From ea9d65470b312f3b925c4a9a7fe173f40d6f00f4 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 28 Oct 2026 14:57:42 -0700 Subject: [PATCH 001/100] feat(planner): add retune.objective to score retune on a DGDR objective metric Replay retune scored every candidate on SLA goodput per GPU. Add `retune.objective`, whose values are the DGDR v1beta2 `spec.objective.metric` enum spelled the same (Throughput, ThroughputPerGPU, ThroughputPerUser, TTFT, E2ELatency, Goodput, GoodputPerGPU), so a later live executor can copy it into a Request. The default, GoodputPerGPU, keeps the current scoring. - score_summary scores the AISimulate target the operator lowers each metric to, in the metric's units. Only Goodput and GoodputPerGPU are rescaled from the replay duration to the window: every candidate emits the same output tokens, so window-normalized throughput would tie every candidate, while the SLA already penalizes a slow goodput candidate. A missing input (no SLA, no GPU-hours, no latency samples) fails the candidate instead of scoring 0 or infinity. - _decide ranks best first in the objective's direction and applies a candidate only when it beats the current score by more than min_improvement_percentage of |current score|; ties still go to the current point, and non-finite scores never rank. - sla_ttft_ms/sla_itl_ms apply only to the Goodput objectives. - With load or throughput scaling, only GoodputPerGPU and ThroughputPerGPU are accepted: other metrics improve with more GPUs, and retune can search min_workers and the GPU budgets. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/config/planner_config.py | 63 +++++++- .../src/dynamo/planner/retune/controller.py | 45 ++++-- components/src/dynamo/planner/retune/sweep.py | 144 ++++++++++++++---- .../integration/test_retune_sweep_workers.py | 48 +++++- .../planner/tests/retune_sweep_helpers.py | 2 + .../planner/tests/unit/test_retune_config.py | 49 +++++- .../tests/unit/test_retune_controller.py | 81 ++++++++++ .../planner/tests/unit/test_retune_sweep.py | 131 ++++++++++++++++ .../planner/aisimulate-planner-replay.mdx | 8 +- .../planner/planner-design.md | 48 ++++-- .../planner/planner-guide.md | 11 +- .../components/planner-configuration.mdx | 32 +++- 12 files changed, 587 insertions(+), 75 deletions(-) diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index 9f794e6fffff..e88692aff1c5 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -828,12 +828,32 @@ def replay_kwargs( return kwargs +# Retune scoring metrics: the DGDR v1beta2 ``spec.objective.metric`` values, +# spelled the same so a Request can carry the metric unchanged. +RetuneObjective = Literal[ + "Throughput", + "ThroughputPerGPU", + "ThroughputPerUser", + "TTFT", + "E2ELatency", + "Goodput", + "GoodputPerGPU", +] +# The objectives that divide by provisioned GPUs; with scaling, the others +# favor the candidates that provision the most. +RETUNE_PER_GPU_OBJECTIVES: tuple[RetuneObjective, ...] = ( + "GoodputPerGPU", + "ThroughputPerGPU", +) + + class RetuneConfig(BaseModel): """Periodic replay-based retune of router and planner runtime parameters. Every ``interval_seconds`` the planner replays the previous window of recorded traffic against a grid of candidate parameters and applies the - best point by SLA goodput per GPU. ``router`` and ``planner`` are search + best point by ``objective`` (SLA goodput per GPU by default; see + ``retune.sweep.score_summary``). ``router`` and ``planner`` are search domains in the AISimulate recommendation syntax of the Dynamo router and planner adapters (``RouterRetuneConfig``, ``PlannerRetuneConfig``), stored as written; ``retune.domains`` expands them into a grid keyed by live @@ -902,17 +922,32 @@ class RetuneConfig(BaseModel): "None means interval_seconds." ), ) + objective: RetuneObjective = Field( + default="GoodputPerGPU", + description=( + "Metric each candidate is scored on, named as DGDR v1beta2 " + "objective.metric. TTFT and E2ELatency (mean, ms) are minimized; " + "the others are maximized. With load or throughput scaling, only " + "GoodputPerGPU and ThroughputPerGPU are accepted." + ), + ) sla_ttft_ms: float | None = Field( default=None, gt=0, allow_inf_nan=False, - description="Scoring TTFT SLA. None uses the planner's ttft_ms.", + description=( + "TTFT SLA of the Goodput and GoodputPerGPU objectives. None uses " + "the planner's ttft_ms." + ), ) sla_itl_ms: float | None = Field( default=None, gt=0, allow_inf_nan=False, - description="Scoring ITL SLA. None uses the planner's itl_ms.", + description=( + "ITL SLA of the Goodput and GoodputPerGPU objectives. None uses " + "the planner's itl_ms." + ), ) min_improvement_percentage: float = Field( default=3.0, @@ -920,8 +955,9 @@ class RetuneConfig(BaseModel): lt=100, allow_inf_nan=False, description=( - "Apply a candidate only if its score exceeds the current point's " - "score on the same window by more than this percentage." + "Apply a candidate only if it beats the current point's score on " + "the same window by more than this percentage of that score: " + "higher for a maximized objective, lower for TTFT and E2ELatency." ), ) max_candidates: int = Field( @@ -1026,7 +1062,10 @@ def effective_timeout_seconds(self) -> float: return self.timeout_seconds def scoring_sla(self, ttft_ms: float, itl_ms: float) -> tuple[float, float]: - """Return the scoring ``(ttft_ms, itl_ms)``; unset values use the planner's.""" + """Return the scoring ``(ttft_ms, itl_ms)``; unset values use the planner's. + + Only the Goodput objectives score against it. + """ return ( ttft_ms if self.sla_ttft_ms is None else self.sla_ttft_ms, @@ -1913,6 +1952,18 @@ def _validate_retune(self, retune: RetuneConfig) -> None: raise ValueError( f"retune requires mode='agg' or mode='disagg', got {self.mode!r}" ) + if self.scaling_enabled() and retune.objective not in RETUNE_PER_GPU_OBJECTIVES: + # Every candidate replay runs an inner planner that sizes its own + # replicas, and min_workers or the GPU budgets may be searched; a + # metric that ignores GPU count picks the largest deployment. + raise ValueError( + f"retune.objective {retune.objective!r} does not divide by " + "GPUs, so with load or throughput scaling it favors the " + "candidates that provision the most GPUs; use " + f"{' or '.join(RETUNE_PER_GPU_OBJECTIVES)}, or retune with " + "fixed replicas (enable_load_scaling and " + "enable_throughput_scaling both false)" + ) if not _is_multiple_of( retune.interval_seconds, self.scheduling.scale_interval_seconds ): diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index da15f33dceb3..e8fa02eb00db 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -66,6 +66,7 @@ from dynamo.planner.retune.sweep import ( SweepRequest, SweepResult, + objective_maximized, run_sweep, with_predictor_fields, ) @@ -148,6 +149,7 @@ def __init__( self._sla_ttft_ms, self._sla_itl_ms = retune.scoring_sla( config.ttft_ms, config.itl_ms ) + self._maximize = objective_maximized(retune.objective) self._startup_router_json: dict[str, Any] = dict( inputs.engine.router_config or {} @@ -201,14 +203,15 @@ def __init__( ) logger.info( "retune enabled: interval=%ss grace=%ss %d grid points, router keys %s, " - "planner keys %s, predictor candidates %s, scoring SLA ttft=%sms itl=%sms, " - "advisory=%s", + "planner keys %s, predictor candidates %s, objective %s, " + "scoring SLA ttft=%sms itl=%sms, advisory=%s", retune.interval_seconds, retune.grace_seconds, len(self._grid), list(self._router_keys), list(self._planner_keys), [label for label, _ in self._predictor_candidates or ()], + retune.objective, self._sla_ttft_ms, self._sla_itl_ms, config.advisory, @@ -527,6 +530,7 @@ async def _launch(self, k: int, now_s: float) -> RetuneDecision | None: predictor_candidates=( self._predictor_candidates if base_planner is not None else None ), + objective=self._retune.objective, sla_ttft_ms=self._sla_ttft_ms, sla_itl_ms=self._sla_itl_ms, max_parallel=self._retune.max_parallel, @@ -632,21 +636,32 @@ def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: "stale", reason="searched parameters changed during the sweep" ) baseline_score = result.scores.get(baseline.key()) - if baseline_score is None: - error = result.metrics.get(baseline.key(), {}).get("error") + if baseline_score is None or not math.isfinite(baseline_score): + error = ( + result.metrics.get(baseline.key(), {}).get("error") + if baseline_score is None + else f"non-finite score {baseline_score!r}" + ) return record("failed", reason=f"baseline replay failed: {error}") order = { candidate.key(): index for index, candidate in enumerate(request.candidates) } + sign = 1.0 if self._maximize else -1.0 ranked = sorted( ( (score, candidate) for candidate in request.candidates if (score := result.scores.get(candidate.key())) is not None + and math.isfinite(score) + ), + # Best score first (lowest for a minimized objective); ties go to + # the baseline, then canonical order. + key=lambda item: ( + -sign * item[0], + item[1] != baseline, + order[item[1].key()], ), - # Score descending; ties go to the baseline, then canonical order. - key=lambda item: (-item[0], item[1] != baseline, order[item[1].key()]), ) top: list[tuple[str, float]] = [] for score, candidate in ranked: @@ -660,10 +675,7 @@ def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: "best_score": best_score, "top": top, } - threshold = baseline_score * ( - 1.0 + self._retune.min_improvement_percentage / 100.0 - ) - if best == baseline or best_score <= threshold: + if best == baseline or not self._improves(best_score, baseline_score): return record("kept", **scores) chosen = with_predictor_fields(best, baseline, presweep_fields) @@ -727,6 +739,19 @@ def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: self._pending_planner.update(planner_changes) return decision + def _improves(self, score: float, baseline_score: float) -> bool: + """Whether ``score`` beats the baseline by the required margin. + + The gain is the increase for a maximized objective and the decrease + for a minimized one; it must exceed ``min_improvement_percentage`` of + ``|baseline_score|``. Any strict gain clears a zero baseline, and a + latency cannot fall below zero, so a zero-latency baseline is kept. + """ + + gain = score - baseline_score if self._maximize else baseline_score - score + margin = abs(baseline_score) * self._retune.min_improvement_percentage / 100.0 + return gain > margin + def _skipped( self, k: int, now_s: float, outcome: RetuneOutcome, reason: str ) -> RetuneDecision: diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 93605305791e..64a619757314 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -50,7 +50,11 @@ from aisimulate.sweeper import OptimizationTarget, objective_value -from dynamo.planner.config.planner_config import PlannerConfig, RetuneEngineSpec +from dynamo.planner.config.planner_config import ( + PlannerConfig, + RetuneEngineSpec, + RetuneObjective, +) from dynamo.planner.core.types import ( RetuneParams, TrafficObservation, @@ -89,6 +93,67 @@ _WORKER_TEMP_DIR = "tmp" +@dataclass(frozen=True) +class _Objective: + """How ``score_summary`` scores one ``retune.objective``. + + ``target`` is the AISimulate target the DGDR operator lowers the metric + to. ``inputs`` are the summary values ``objective_value`` reads, which + must be finite numbers; ``positive`` of them must also be above 0. + ``per_window`` rescales the value from the replay's duration to the + window length. + """ + + target: OptimizationTarget + inputs: tuple[str, ...] + positive: tuple[str, ...] = () + per_window: bool = False + + +_OBJECTIVES: dict[RetuneObjective, _Objective] = { + "Throughput": _Objective( + OptimizationTarget.THROUGHPUT, ("output_throughput_tok_s",) + ), + "ThroughputPerGPU": _Objective( + OptimizationTarget.THROUGHPUT_PER_GPU, + ("output_throughput_tok_s", "gpu_hours", "duration_ms"), + ("gpu_hours", "duration_ms"), + ), + "ThroughputPerUser": _Objective( + OptimizationTarget.THROUGHPUT_PER_USER, + ("mean_output_token_throughput_per_user",), + ), + "TTFT": _Objective( + OptimizationTarget.TTFT, + ("mean_ttft_ms", "num_ttft_samples"), + ("num_ttft_samples",), + ), + "E2ELatency": _Objective( + OptimizationTarget.E2E_LATENCY, + ("mean_e2e_latency_ms", "num_e2e_latency_samples"), + ("num_e2e_latency_samples",), + ), + "Goodput": _Objective( + OptimizationTarget.GOODPUT, + (GOODPUT_KEY, "duration_ms"), + ("duration_ms",), + per_window=True, + ), + "GoodputPerGPU": _Objective( + OptimizationTarget.GOODPUT_PER_GPU, + (GOODPUT_KEY, "gpu_hours", "duration_ms"), + ("gpu_hours", "duration_ms"), + per_window=True, + ), +} + + +def objective_maximized(objective: RetuneObjective) -> bool: + """Whether a higher score is better; TTFT and E2ELatency are minimized.""" + + return _OBJECTIVES[objective].target.maximize + + @dataclass(frozen=True) class SweepRequest: """Everything one sweep needs; picklable and independent of the planner. @@ -105,6 +170,7 @@ class SweepRequest: predictors the forecast-loss pre-sweep chooses from, as ``retune.domains`` expands ``retune.planner.load_predictor``; its winner's predictor fields are fixed into every non-baseline candidate. + ``objective`` is the metric ``score_summary`` scores each replay on. """ window_index: int @@ -120,6 +186,7 @@ class SweepRequest: replicas: dict[str, int] base_planner: dict[str, Any] | None predictor_candidates: tuple[PredictorCandidate, ...] | None + objective: RetuneObjective sla_ttft_ms: float sla_itl_ms: float max_parallel: int @@ -209,37 +276,58 @@ def with_predictor_fields( def score_summary( - summary: Mapping[str, Any], window_s: float + summary: Mapping[str, Any], + window_s: float, + objective: RetuneObjective = "GoodputPerGPU", ) -> tuple[float, dict[str, Any]]: - """Score a replay summary as SLA-good output tokens per window-GPU-second. - - ``objective_value(summary, GOODPUT_PER_GPU) * duration_ms / (window_s * - 1000)``: the denominator is the fixed window length, so candidates whose - drains differ are compared on the same time base. Fails closed: a missing - goodput key or a non-positive ``gpu_hours``/``duration_ms`` raises instead - of scoring 0, which would tie every candidate. + """Score a replay summary on ``objective``, in the objective's own units. + + The score is ``objective_value`` of the AISimulate target the DGDR + operator lowers the objective to, unsigned: ``objective_maximized`` says + which way is better. A replay drains every request of the window, so its + ``duration_ms`` is the window plus a drain that differs by candidate, and + each output-token total is the same for every candidate: + + - ``Goodput`` and ``GoodputPerGPU`` are multiplied by ``duration_ms / + (window_s * 1000)``: SLA-good tokens per window second (per GPU). The + SLA already penalizes a slow candidate, and the fixed window keeps the + drain from counting against it a second time. The default score is + SLA-good output tokens per window-GPU-second. + - ``Throughput`` and ``ThroughputPerGPU`` are not rescaled. Over the + window both would equal the offered load for every candidate (per GPU + too, with fixed replicas); over the replay they rank a shorter drain, + which is how a candidate with more capacity shows when the window + backs up. + - ``ThroughputPerUser`` (mean tok/s/user over token gaps), ``TTFT`` and + ``E2ELatency`` (mean ms) are per-request means that the duration does + not enter. + + Fails closed: a missing or non-finite input, a non-positive + ``gpu_hours``/``duration_ms``, or a latency with no samples raises + instead of scoring, so that candidate is unscored rather than tied at 0 + or ranked at infinity. """ - if summary.get(GOODPUT_KEY) is None: + spec = _OBJECTIVES[objective] + if GOODPUT_KEY in spec.inputs and summary.get(GOODPUT_KEY) is None: raise ValueError(f"replay summary has no {GOODPUT_KEY}; scoring needs an SLA") - goodput, gpu_hours, duration_ms = ( - _finite_number(summary, name) - for name in (GOODPUT_KEY, "gpu_hours", "duration_ms") - ) - if gpu_hours <= 0 or duration_ms <= 0: - raise ValueError( - "replay summary needs positive gpu_hours and duration_ms, got " - f"{gpu_hours!r} and {duration_ms!r}" - ) + inputs = {name: _finite_number(summary, name) for name in spec.inputs} + for name in spec.positive: + if inputs[name] <= 0: + raise ValueError( + f"replay summary needs a positive {name}, got {inputs[name]!r}" + ) if not window_s > 0: raise ValueError(f"window_s must be positive, got {window_s!r}") - objective = objective_value(dict(summary), OptimizationTarget.GOODPUT_PER_GPU) - score = objective * duration_ms / (window_s * 1000.0) + value = objective_value(dict(summary), spec.target) + score = value + if spec.per_window: + score = value * inputs["duration_ms"] / (window_s * 1000.0) + if not math.isfinite(score): + raise ValueError(f"{objective} score must be finite, got {score!r}") return score, { - "objective_goodput_per_gpu": objective, - GOODPUT_KEY: goodput, - "gpu_hours": gpu_hours, - "duration_ms": duration_ms, + f"objective_{spec.target.value}": value, + **inputs, "completed_requests": summary.get("completed_requests"), "goodput_completed_requests": summary.get("goodput_completed_requests"), } @@ -265,8 +353,9 @@ def evaluate_candidate( sla_ttft_ms: float, sla_itl_ms: float, window_s: float, + objective: RetuneObjective = "GoodputPerGPU", ) -> tuple[float, dict[str, Any]]: - """Replay the window once with ``params`` and score its summary. + """Replay the window once with ``params`` and score its summary on ``objective``. Without ``base_planner`` the replay keeps ``replicas`` fixed (Scenario 1). With it, an inner planner runs from ``base_planner`` with @@ -310,7 +399,7 @@ def evaluate_candidate( capture_planner_details=False, **kwargs, ) - return score_summary(report.summary, window_s) + return score_summary(report.summary, window_s, objective) def _presweep_summary( @@ -682,6 +771,7 @@ def _run_sweep( request.sla_ttft_ms, request.sla_itl_ms, request.window_s, + request.objective, ), ) for effective_key, (effective, _) in by_effective.items() diff --git a/components/src/dynamo/planner/tests/integration/test_retune_sweep_workers.py b/components/src/dynamo/planner/tests/integration/test_retune_sweep_workers.py index f843082de98c..33dbe53be010 100644 --- a/components/src/dynamo/planner/tests/integration/test_retune_sweep_workers.py +++ b/components/src/dynamo/planner/tests/integration/test_retune_sweep_workers.py @@ -4,17 +4,20 @@ # Optional-dependency preflight must run before the sweep module imports it. """Retune sweeps on real spawned workers: isolation, deadline, cancellation, -the trace read, the pre-sweep and the workers' temporary files. +the trace read, the pre-sweep and the workers' temporary files, plus every +objective scored from a real replay. -Each test runs ``run_sweep`` end to end, so it spawns worker processes that -import the evaluators from ``dynamo.planner.tests.retune_sweep_helpers``. +Each sweep test runs ``run_sweep`` end to end, so it spawns worker processes +that import the evaluators from ``dynamo.planner.tests.retune_sweep_helpers``. """ import errno +import math import multiprocessing import os import threading import time +from typing import get_args import pytest @@ -23,8 +26,9 @@ reason="AI Simulate is an optional Dynamo simulation dependency", ) +from dynamo.planner.config.planner_config import RetuneObjective from dynamo.planner.core.types import RetuneParams -from dynamo.planner.retune.sweep import run_sweep +from dynamo.planner.retune.sweep import evaluate_candidate, run_sweep from dynamo.planner.tests.retune_sweep_helpers import ( ABORT, HANG, @@ -211,6 +215,42 @@ def test_run_sweep_presweep_fixes_the_winner_into_non_baseline_candidates(tmp_pa assert result.predictor_warmup_error is None +@pytest.mark.timeout(240) +def test_a_real_replay_scores_every_objective(tmp_path): + # In process: the summary keys each objective reads exist in a replay. + trace = write_trace( + tmp_path / "trace.jsonl", trace_rows([ORIGIN_MS + 250 * i for i in range(30)]) + ) + request = sweep_request( + [trace], + [retune_params()], + retune_params(), + engine=disagg_engine(), + replicas={"prefill": 1, "decode": 1}, + ) + + scores = {} + for objective in get_args(RetuneObjective): + scores[objective], metrics = evaluate_candidate( + trace, + request.engine, + request.arrival_speedup_ratio, + request.baseline, + request.replicas, + None, + None, + request.sla_ttft_ms, + request.sla_itl_ms, + request.window_s, + objective, + ) + assert metrics["completed_requests"] == 30 + + assert all(math.isfinite(score) and score > 0 for score in scores.values()) + assert scores["ThroughputPerGPU"] < scores["Throughput"] + assert scores["TTFT"] < scores["E2ELatency"] + + @pytest.mark.timeout(240) def test_worker_temp_files_are_removed_with_the_sweep(tmp_path): trace = write_trace( diff --git a/components/src/dynamo/planner/tests/retune_sweep_helpers.py b/components/src/dynamo/planner/tests/retune_sweep_helpers.py index 059bcc2727ff..db19c1987c7d 100644 --- a/components/src/dynamo/planner/tests/retune_sweep_helpers.py +++ b/components/src/dynamo/planner/tests/retune_sweep_helpers.py @@ -114,6 +114,7 @@ def sweep_request(trace_paths, candidates, baseline, **overrides): "replicas": {"aggregated": 2}, "base_planner": None, "predictor_candidates": None, + "objective": "GoodputPerGPU", "sla_ttft_ms": 500.0, "sla_itl_ms": 30.0, "max_parallel": 2, @@ -134,6 +135,7 @@ def fake_evaluator( sla_ttft_ms, sla_itl_ms, window_s, + objective, ): if params.router.get("router_temperature") == HANG: time.sleep(600) diff --git a/components/src/dynamo/planner/tests/unit/test_retune_config.py b/components/src/dynamo/planner/tests/unit/test_retune_config.py index 29e46c38c05d..13255106e8d6 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_config.py @@ -4,9 +4,11 @@ # Optional-dependency preflight must run before the retune imports. """The planner retune config section: a valid round trip, a rejected -startup-only knob and the grid-size limit, plus the retune-only, decision -latency and history rules that only config validation enforces, and one -error per invalid knob.""" +startup-only knob and the grid-size limit, plus the retune-only, objective, +decision latency and history rules that only config validation enforces, and +one error per invalid knob.""" + +from typing import get_args import pytest @@ -17,7 +19,11 @@ from pydantic import ValidationError -from dynamo.planner.config.planner_config import PlannerConfig, RetuneConfig +from dynamo.planner.config.planner_config import ( + PlannerConfig, + RetuneConfig, + RetuneObjective, +) from dynamo.planner.reconfiguration import candidate_config from dynamo.planner.retune.domains import ( expand_planner_domain, @@ -281,6 +287,41 @@ def test_retune_only_mode_needs_both_scaling_flags_explicitly_false(): _config(optimization_target="sla", **RETUNE_ONLY) +def test_objective_takes_the_dgdr_metric_names_and_defaults_to_goodput_per_gpu(): + assert RetuneConfig(interval_seconds=600).objective == "GoodputPerGPU" + # The DGDR v1beta2 ObjectiveMetric enum, spelled as a Request writes it. + assert get_args(RetuneObjective) == ( + "Throughput", + "ThroughputPerGPU", + "ThroughputPerUser", + "TTFT", + "E2ELatency", + "Goodput", + "GoodputPerGPU", + ) + for objective in get_args(RetuneObjective): + config = _config(mode="agg", retune=_retune(objective=objective), **RETUNE_ONLY) + assert config.retune.objective == objective + # AISimulate target names and other spellings are not the DGDR names. + for objective in ("goodput_per_gpu", "goodputPerGPU", "MinGPUs", "Pareto"): + with pytest.raises(ValidationError, match="objective"): + RetuneConfig(interval_seconds=600, objective=objective) + + +@pytest.mark.parametrize("objective", get_args(RetuneObjective)) +def test_scaling_retune_needs_a_per_gpu_objective(objective): + # Easy mode forces load scaling on, so each candidate replay sizes its + # own replicas; a metric that ignores GPU count would buy the most. + if objective in ("GoodputPerGPU", "ThroughputPerGPU"): + config = _config(retune=_retune(objective=objective)) + assert config.scaling_enabled() + return + with pytest.raises(ValidationError, match=f"{objective}' does not divide by GPUs"): + _config(retune=_retune(objective=objective)) + with pytest.raises(ValidationError, match="does not divide by GPUs"): + _config(retune=_retune(objective=objective), **SLA_THROUGHPUT) + + def test_measured_decision_latency_must_fit_the_interval(): # A measured latency is at most decision_latency_scale x timeout_seconds, # which must fit in the interval. diff --git a/components/src/dynamo/planner/tests/unit/test_retune_controller.py b/components/src/dynamo/planner/tests/unit/test_retune_controller.py index 3536a580a12f..b589a0151ed4 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_controller.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_controller.py @@ -6,6 +6,7 @@ """Unit tests for the retune controller with a scripted ``run_sweep``.""" import json +import math import threading from collections import deque @@ -535,6 +536,86 @@ async def test_min_improvement_percentage_boundary(sweeps, best_score, outcome): assert decision.best_score == best_score +@pytest.mark.parametrize( + ("objective", "baseline_score", "best_score", "outcome"), + [ + # Maximized: the best must exceed the baseline by more than 3% of it. + ("Goodput", 100.0, 100.0, "kept"), + ("Goodput", 0.0, 0.0, "kept"), + ("Goodput", 0.0, 1e-9, "applied"), + ("Goodput", -100.0, -97.0, "kept"), + ("Goodput", -100.0, -96.99, "applied"), + # Minimized: the best must fall below the baseline by more than 3%. + ("TTFT", 100.0, 97.0, "kept"), + ("TTFT", 100.0, 96.99, "applied"), + ("TTFT", 100.0, 100.0, "kept"), + ("TTFT", 0.0, 0.0, "kept"), + ("E2ELatency", 1_000.0, 969.99, "applied"), + ("E2ELatency", -100.0, -103.01, "applied"), + ], +) +async def test_min_improvement_follows_the_objective_direction( + sweeps, objective, baseline_score, best_score, outcome +): + # Every candidate but BEST scores the baseline's value, so a tie with + # BEST goes to the baseline. + sweeps.default = _best_wins(best_score=best_score, other_score=baseline_score) + controller = _controller(_s1(objective=objective)) + + ticks = await _run(controller, 70) + + ((_, decision),) = _decisions(ticks) + assert decision.outcome == outcome + assert (decision.baseline_score, decision.best_score) == ( + baseline_score, + best_score, + ) + assert decision.chosen == (BEST if outcome == "applied" else STARTUP) + + +@pytest.mark.parametrize( + ("objective", "best_score", "top"), + [ + ("TTFT", 50.0, [("best", 50.0), ("baseline", 100.0)]), + ("TTFT", 150.0, [("baseline", 100.0), ("other", 100.0)]), + ("Goodput", 150.0, [("best", 150.0), ("baseline", 100.0)]), + ("Goodput", 50.0, [("baseline", 100.0), ("other", 100.0)]), + ], +) +async def test_candidates_rank_best_first_in_the_objective_direction( + sweeps, objective, best_score, top +): + sweeps.default = _best_wins(best_score=best_score, other_score=100.0) + controller = _controller(_s1(objective=objective)) + + ticks = await _run(controller, 70) + + ((_, decision),) = _decisions(ticks) + (request,) = sweeps.requests + assert request.objective == objective + names = {STARTUP.key(): "baseline", BEST.key(): "best"} + ranked = [(names.get(key, "other"), score) for key, score in decision.top] + assert ranked[:2] == top + assert decision.best_score == top[0][1] + + +@pytest.mark.parametrize("objective", ["Goodput", "TTFT"]) +async def test_non_finite_scores_never_rank(sweeps, objective): + # A non-finite candidate is unscored; a non-finite baseline fails. + sweeps.default = _best_wins(best_score=math.nan, other_score=100.0) + controller = _controller(_s1(objective=objective)) + ticks = await _run(controller, 70) + ((_, decision),) = _decisions(ticks) + assert (decision.outcome, decision.best_score) == ("kept", 100.0) + + sweeps.default = _scored(lambda c, _r: math.inf if c == STARTUP else 1.0) + controller = _controller(_s1(objective=objective)) + ticks = await _run(controller, 70) + ((_, decision),) = _decisions(ticks) + assert decision.outcome == "failed" + assert "baseline replay failed: non-finite score inf" in decision.reason + + async def test_advisory_records_the_best_point_without_actuating(sweeps): sweeps.default = _best_wins() controller = _controller(_s1(advisory=True)) diff --git a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py index 38aa197bd15f..cee34295ea2f 100644 --- a/components/src/dynamo/planner/tests/unit/test_retune_sweep.py +++ b/components/src/dynamo/planner/tests/unit/test_retune_sweep.py @@ -20,6 +20,7 @@ import types from collections import deque from multiprocessing.reduction import ForkingPickler +from typing import get_args import pytest @@ -28,10 +29,13 @@ reason="AI Simulate is an optional Dynamo simulation dependency", ) +from dynamo.planner.config.planner_config import RetuneObjective from dynamo.planner.core.types import RetuneParams, TrafficObservation from dynamo.planner.retune import sweep as sweep_module from dynamo.planner.retune.sweep import ( + _OBJECTIVES, GOODPUT_KEY, + objective_maximized, score_summary, with_predictor_fields, ) @@ -78,6 +82,111 @@ def test_score_uses_the_fixed_window_denominator(): score_summary(bad, window_s=10.0) +# Four average GPUs over a 1,800 s replay of a 1,500 s window. +SUMMARY = { + "output_throughput_tok_s": 300.0, + GOODPUT_KEY: 120.0, + "gpu_hours": 2.0, + "duration_ms": 1_800_000.0, + "mean_output_token_throughput_per_user": 40.0, + "mean_ttft_ms": 250.0, + "num_ttft_samples": 10, + "mean_e2e_latency_ms": 900.0, + "num_e2e_latency_samples": 10, +} + + +@pytest.mark.parametrize( + ("objective", "target", "raw", "score", "maximized"), + [ + ("Throughput", "throughput", 300.0, 300.0, True), + ("ThroughputPerGPU", "throughput_per_gpu", 75.0, 75.0, True), + ("ThroughputPerUser", "throughput_per_user", 40.0, 40.0, True), + ("TTFT", "ttft", 250.0, 250.0, False), + ("E2ELatency", "e2e_latency", 900.0, 900.0, False), + # Goodput is rescaled from the 1,800 s replay to the 1,500 s window. + ("Goodput", "goodput", 120.0, 144.0, True), + ("GoodputPerGPU", "goodput_per_gpu", 30.0, 36.0, True), + ], +) +def test_each_objective_scores_the_target_the_dgdr_operator_lowers_it_to( + objective, target, raw, score, maximized +): + # The targets are those of the DGDR operator's v1beta2 publicMetric. + assert get_args(RetuneObjective) == tuple(_OBJECTIVES) + assert _OBJECTIVES[objective].target.value == target + + value, metrics = score_summary(SUMMARY, 1_500.0, objective) + + assert value == pytest.approx(score) + assert metrics[f"objective_{target}"] == pytest.approx(raw) + assert objective_maximized(objective) is maximized + + +def test_only_goodput_objectives_ignore_the_drain(): + # One window of 600k output tokens, 300k of them SLA-good, on 2 GPUs, + # with a 100 s or a 300 s drain. The mean latencies are the same. + def replay(duration_s): + return { + **SUMMARY, + "output_throughput_tok_s": 600_000 / duration_s, + GOODPUT_KEY: 300_000 / duration_s, + "gpu_hours": 2 * duration_s / 3_600, + "duration_ms": duration_s * 1_000.0, + } + + fast, slow = replay(1_000.0), replay(1_200.0) + scores = { + objective: ( + score_summary(fast, 900.0, objective)[0], + score_summary(slow, 900.0, objective)[0], + ) + for objective in get_args(RetuneObjective) + } + + # Good tokens per window second (per GPU): the drain does not count. + assert scores["Goodput"] == pytest.approx((300_000 / 900,) * 2) + assert scores["GoodputPerGPU"] == pytest.approx((300_000 / 900 / 2,) * 2) + # The token total is fixed, so throughput ranks the shorter drain. + assert scores["Throughput"] == pytest.approx((600.0, 500.0)) + assert scores["ThroughputPerGPU"] == pytest.approx((300.0, 250.0)) + for objective in ("ThroughputPerUser", "TTFT", "E2ELatency"): + assert scores[objective][0] == scores[objective][1] + + +@pytest.mark.parametrize( + ("objective", "update", "match"), + [ + ("GoodputPerGPU", {GOODPUT_KEY: None}, "needs an SLA"), + ("Goodput", {"duration_ms": 0.0}, "positive duration_ms"), + ("ThroughputPerGPU", {"gpu_hours": 0.0}, "positive gpu_hours"), + ("Throughput", {"output_throughput_tok_s": None}, "must be numeric"), + ( + "ThroughputPerUser", + {"mean_output_token_throughput_per_user": math.inf}, + "finite", + ), + # AISimulate scores a latency without samples as infinite; retune + # leaves that candidate unscored instead. + ("TTFT", {"num_ttft_samples": 0}, "positive num_ttft_samples"), + ("E2ELatency", {"num_e2e_latency_samples": 0}, "positive num_e2e"), + ("E2ELatency", {"mean_e2e_latency_ms": math.nan}, "finite"), + ], +) +def test_a_missing_score_input_fails_the_candidate(objective, update, match): + with pytest.raises(ValueError, match=match): + score_summary({**SUMMARY, **update}, 1_500.0, objective) + + +def test_only_goodput_objectives_need_the_sla_and_gpu_hours(): + # A latency replay without goodput (no SLA) or GPU accounting scores. + summary = {"mean_ttft_ms": 250.0, "num_ttft_samples": 10} + + assert score_summary(summary, 1_500.0, "TTFT")[0] == 250.0 + with pytest.raises(ValueError, match="needs an SLA"): + score_summary(summary, 1_500.0, "Goodput") + + def test_a_grid_point_equivalent_to_the_baseline_gets_its_key(): # With planner: default the current point carries every predictor field, # a grid point only the searched ones; a winner equal to the current @@ -385,6 +494,28 @@ def counting_evaluator(*args): assert result.scores[same.key()] == result.scores[baseline.key()] is not None +def test_every_candidate_is_scored_on_the_request_objective(tmp_path, monkeypatch): + monkeypatch.setattr(sweep_module, "_WorkerPool", _InlinePool) + objectives = [] + + def recording_evaluator(*args): + objectives.append(args[-1]) + return fake_evaluator(*args) + + trace = write_trace( + tmp_path / "trace.jsonl", trace_rows([ORIGIN_MS + 100 * i for i in range(5)]) + ) + a, b = retune_params(0.0), retune_params(1.0) + + result = sweep_module.run_sweep( + sweep_request([trace], [a, b], a, objective="TTFT"), + evaluator=recording_evaluator, + ) + + assert result.status == "ok", result.error + assert objectives == ["TTFT", "TTFT"] + + def test_a_predictor_winner_is_prepared_in_the_sweep(tmp_path, monkeypatch): monkeypatch.setattr(sweep_module, "_WorkerPool", _InlinePool) prepared = [] diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx index b2bc1a52485d..c0d92b5b97af 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx @@ -497,7 +497,8 @@ With `aisimulate predict --stack dynamo`: Recommended: set `evaluation.sla` with `ttft_ms` and `itl_ms`. Predict fills `retune.sla_ttft_ms` and `retune.sla_itl_ms` from it when they are unset, so retune scores candidates against the SLA that AISimulate reports goodput for. Without it, scoring uses the Planner's `ttft_ms` and `itl_ms`, -which then keep their `PlannerConfig` defaults. +which then keep their `PlannerConfig` defaults. The SLA applies only to the default +`retune.objective`, `GoodputPerGPU`, and to `Goodput`. With `dynamo.replay.run_trace_replay`: @@ -682,7 +683,7 @@ Each record has these fields. Times are Planner-clock seconds from the start of | `num_requests`, `num_candidates` | Requests in the window and candidates in the sweep | | `previous`, `chosen` | The point before the decision and the point chosen, as `router` and `planner` maps keyed by live names (`router_temperature`, `load_scaling_down_sensitivity`) | | `planner_changes` | Planner fields the decision changes | -| `baseline_score`, `best_score`, `top` | The current point's score, the best score, and up to five `[candidate, score]` pairs | +| `baseline_score`, `best_score`, `top` | The current point's score, the best score, and up to five `[candidate, score]` pairs, best first. Scores are in the units of [`retune.objective`](../../../../reference/components/planner-configuration.mdx#objectives); for `TTFT` and `E2ELatency` they are milliseconds and lower is better | | `predictor_presweep` | Load-predictor pre-sweep: `winner`, `fields`, per-candidate `losses`, `eval_steps`, `warmup_steps`, and `wall_s` | | `predictor_warmup` | Number of traffic observations that warm a newly selected load predictor; `null` unless an `applied` decision replaces the predictor | | `sweep_wall_s`, `decision_latency_s` | The sweep's wall-clock time, and the delay after launch at which the decision could land; `null` for `busy` and decisions that fail before a sweep runs | @@ -730,7 +731,8 @@ aisimulate predict --stack dynamo --config retune-router.yaml \ ### Measure the Gain Run the same config without retune and compare `summary.goodput_output_throughput_tok_s`, -`summary.gpu_hours`, and the latency percentiles: +`summary.gpu_hours`, and the latency percentiles, or the summary metric of a non-default +`retune.objective`, such as `summary.mean_ttft_ms` for `TTFT`: - For Scenario 1, replace the whole `planner` section with `--set 'planner={policy: disabled}'`. Do not only remove `retune`: without it, the two `false` diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md index 9c4d7d029587..62674dfe6ef8 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md @@ -427,23 +427,45 @@ keeps the current predictor. The decision records the result in `predictor_presw ### Scoring and Outcomes -Each candidate's score is SLA-good output tokens per second of the window per GPU: +Each candidate is scored on `retune.objective`, by default SLA-good output tokens per second of the +window per GPU. The score is AISimulate's `objective_value` for the target that the +[objective](../../../../reference/components/planner-configuration.mdx#objectives) maps to, in the +metric's own units, rescaled for the Goodput objectives only: ```text -score = objective_value(summary, GOODPUT_PER_GPU) × duration_ms / (window_s × 1000) +Goodput, GoodputPerGPU: score = objective_value(summary, target) × duration_ms / (window_s × 1000) +other objectives: score = objective_value(summary, target) ``` -`objective_value` is AISimulate's goodput-per-GPU objective over the replay's `duration_ms`, which -includes the drain after the last arrival. Rescaling to the fixed window length `window_s` keeps -candidates whose drains differ on one time base. A replay whose summary lacks -`goodput_output_throughput_tok_s`, or has a non-positive `gpu_hours` or `duration_ms`, fails that -candidate instead of scoring zero. The current point is re-scored on every window, so every -comparison is paired on the same window file and the same random numbers. - -Candidates rank by score, with ties going to the current point and then to the canonical order. The -best candidate is applied only when its score exceeds the current score × (1 + -`min_improvement_percentage` / 100) and it changes a router parameter or a Planner field. Each -decision has one outcome: +A candidate replay runs until every request of the window completes, so every candidate produces +the same output tokens, and its `duration_ms` is the window plus a drain after the last arrival that +differs by candidate. The rescaling follows from that: + +- **Goodput and GoodputPerGPU** count only SLA-good tokens, so the SLA already penalizes a candidate + that falls behind. Dividing by the replay's duration would penalize its longer drain a second + time, and the drain is an artifact of cutting the trace at the window end: live traffic keeps + arriving. Rescaling to the fixed window length `window_s` puts every candidate on one time base: + SLA-good tokens per window second (per average GPU). +- **Throughput and ThroughputPerGPU** are not rescaled. Over the window, both would equal the + offered load for every candidate (per GPU too, with fixed replicas), so every candidate would tie. + Over the replay's duration, they rank the candidate that finishes the window's tokens sooner, + which is how more serving capacity shows when the window backs up. +- **ThroughputPerUser, TTFT, and E2ELatency** are per-request means, which the duration does not + enter. + +The score fails closed. A replay that lacks an input of the objective fails that candidate instead +of scoring zero, which would tie every candidate, or infinity: for example, a Goodput objective +without `goodput_output_throughput_tok_s`, a per-GPU or Goodput objective with a non-positive +`gpu_hours` or `duration_ms`, or `TTFT` or `E2ELatency` without samples. Only the Goodput +objectives use the scoring SLA. The current point is re-scored on every window, so every comparison +is paired on the same window file and the same random numbers. + +Candidates rank best first, highest score for a maximized objective and lowest for `TTFT` and +`E2ELatency`, with ties going to the current point and then to the canonical order. The best +candidate is applied only when it beats the current score by more than `min_improvement_percentage` +/ 100 × |current score| (above it for a maximized objective, below it for a minimized one) and it +changes a router parameter or a Planner field. A zero current score therefore needs any strict +gain, and a zero latency is never beaten. Each decision has one outcome: | Outcome | Meaning | |---|---| diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md index ea2fbc04ad71..b52f84272098 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md @@ -335,9 +335,12 @@ spec: Retune lets the Planner tune its own runtime parameters from the traffic it served. Every `retune.interval_seconds`, it takes the window of the request trace that just ended and replays it once per candidate parameter point, on the deployment's engine configuration and from the replica -counts at the window start. It scores each candidate by SLA goodput per GPU: SLA-good output tokens -per second of the window, divided by the GPUs provisioned. When the best candidate beats the current -point on the same window by more than `min_improvement_percentage`, the Planner switches to it. +counts at the window start. It scores each candidate on `retune.objective`, by default SLA goodput +per GPU: SLA-good output tokens per second of the window, divided by the GPUs provisioned. The +objective takes the DGDR v1beta2 `spec.objective.metric` values, such as `ThroughputPerGPU` or +`TTFT`; see [Objectives](../../../../reference/components/planner-configuration.mdx#objectives). +When the best candidate beats the current point on the same window by more than +`min_improvement_percentage`, the Planner switches to it. Retune searches parameters a running deployment can change without a restart: @@ -355,7 +358,7 @@ Retune supports two scenarios: | Scenario | Planner | Searched | Each candidate replay | |---|---|---|---| | Fixed replicas | Retune-only mode: `retune` set with both `enable_load_scaling: false` and `enable_throughput_scaling: false`. The Planner never scales | Router parameters | Keeps the replica counts fixed | -| Autoscaling | Load-based or throughput-based scaling, or both | Router parameters, Planner fields, or both | Runs the current Planner with the candidate's fields | +| Autoscaling | Load-based or throughput-based scaling, or both. `retune.objective` must be `GoodputPerGPU` or `ThroughputPerGPU`, because the candidates' GPU counts differ | Router parameters, Planner fields, or both | Runs the current Planner with the candidate's fields | With `advisory: true`, the Planner still runs every sweep but only records the decision it would apply, with outcome `advisory`; the router and Planner parameters keep their starting values. diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 3fd7b80a86ca..c0c292879f95 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -629,7 +629,7 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p ## Retune (Experimental, Simulation Only) -`retune` makes the Planner periodically replay the last window of recorded traffic against a grid of router and Planner parameters and apply the best point by SLA goodput per GPU. This section is the field contract. For the concept, see [Periodic Retune](../../developer-guide/knowledge-base/modular-components/planner/planner-guide.md#periodic-retune-experimental); for windows, scoring, and decision timing, see [Planner Design](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#retune-experimental-simulation-only); to run it, see [Simulate Retune](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#simulate-retune-experimental). +`retune` makes the Planner periodically replay the last window of recorded traffic against a grid of router and Planner parameters and apply the best point by `objective`, SLA goodput per GPU by default. This section is the field contract. For the concept, see [Periodic Retune](../../developer-guide/knowledge-base/modular-components/planner/planner-guide.md#periodic-retune-experimental); for windows, scoring, and decision timing, see [Planner Design](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#retune-experimental-simulation-only); to run it, see [Simulate Retune](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#simulate-retune-experimental). **Experimental.** In this release, retune runs only inside planner-in-the-loop offline replay: `dynamo.replay.run_trace_replay` with `trace_format="dynamo"`, or `aisimulate predict --stack dynamo` on a Dynamo request trace. A live Planner rejects a config that sets `retune` at startup. @@ -664,14 +664,19 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p Wall-clock deadline for one sweep; its workers are killed after it and the decision is `timeout`. `null` means `interval_seconds`. Must be greater than 0. Requires `timeout_seconds` ≤ `interval_seconds`. A numeric `decision_latency_seconds` must also be ≤ `timeout_seconds`; with `measured`, `decision_latency_scale` × `timeout_seconds` must also be ≤ `interval_seconds`. This keeps at most one sweep in flight. + + Metric each candidate replay is scored on, written as the DGDR v1beta2 `spec.objective.metric` value. `TTFT` and `E2ELatency` are minimized; the others are maximized. See [Objectives](#objectives) for what each one scores. With load or throughput scaling enabled, only `GoodputPerGPU` and `ThroughputPerGPU` are accepted; retune-only mode accepts every value. + + Allowed values: Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU + - Scoring TTFT SLA. Must be greater than 0. `null` uses the Planner's `ttft_ms`; in `aisimulate predict`, `null` takes `evaluation.sla.ttft_ms` when it is set (see [Requirements](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#requirements)). + TTFT SLA of the `Goodput` and `GoodputPerGPU` objectives; other objectives ignore it. Must be greater than 0. `null` uses the Planner's `ttft_ms`; in `aisimulate predict`, `null` takes `evaluation.sla.ttft_ms` when it is set (see [Requirements](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#requirements)). - Scoring ITL SLA. Must be greater than 0. `null` uses the Planner's `itl_ms`; in `aisimulate predict`, `null` takes `evaluation.sla.itl_ms` when it is set (see [Requirements](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#requirements)). + ITL SLA of the `Goodput` and `GoodputPerGPU` objectives; other objectives ignore it. Must be greater than 0. `null` uses the Planner's `itl_ms`; in `aisimulate predict`, `null` takes `evaluation.sla.itl_ms` when it is set (see [Requirements](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#requirements)). - Apply the best candidate only if its score exceeds the current point's score on the same window by more than this percentage. Must be in [0, 100). Candidate replays share one random seed, so a candidate that changes no routing or scaling decision scores exactly the current point and is kept. With the pinned AISimulate release, an inner Planner with `optimization_target: sla` is not guaranteed to be bitwise reproducible, so such a candidate can miss an exact tie; see [Candidates and the Sweep](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#candidates-and-the-sweep). To choose a value, see [Measure the Gain](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#measure-the-gain). + Apply the best candidate only if it beats the current point's score on the same window by more than this percentage of that score: a higher score for a maximized `objective`, a lower one for `TTFT` and `E2ELatency`. Must be in [0, 100). Candidate replays share one random seed, so a candidate that changes no routing or scaling decision scores exactly the current point and is kept. With the pinned AISimulate release, an inner Planner with `optimization_target: sla` is not guaranteed to be bitwise reproducible, so such a candidate can miss an exact tie; see [Candidates and the Sweep](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#candidates-and-the-sweep). To choose a value, see [Measure the Gain](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#measure-the-gain). Upper bound on the expanded grid size (the product of all domain sizes), on the values of each knob, and separately on the number of load-predictor candidates. Must be at least 1. A grid that exceeds it is rejected with the size of each dimension. For the candidate set a sweep replays, see [Candidates and the Sweep](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#candidates-and-the-sweep). @@ -687,6 +692,25 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p +### Objectives + +`retune.objective` takes the values of the DGDR v1beta2 `spec.objective.metric` enum, spelled the same, and scores each as the AISimulate optimization target the Dynamo operator maps it to. Scores are in the metric's units, and decisions record them as `baseline_score` and `best_score`. + +| Value | AISimulate target | Score | Better | +|---|---|---|---| +| `Throughput` | `throughput` | Output tokens per second of the replay | Higher | +| `ThroughputPerGPU` | `throughput_per_gpu` | `Throughput` divided by the average provisioned GPUs | Higher | +| `ThroughputPerUser` | `throughput_per_user` | Mean output tokens per second per user, over token gaps | Higher | +| `TTFT` | `ttft` | Mean time to first token, in milliseconds | Lower | +| `E2ELatency` | `e2e_latency` | Mean end-to-end latency, in milliseconds | Lower | +| `Goodput` | `goodput` | SLA-good output tokens per second of the window | Higher | +| `GoodputPerGPU` (default) | `goodput_per_gpu` | `Goodput` divided by the average provisioned GPUs | Higher | + +- **SLA.** Only `Goodput` and `GoodputPerGPU` count SLA-good tokens, against `sla_ttft_ms` and `sla_itl_ms`. The other objectives ignore both fields. +- **Time base.** A candidate replay runs until every request of the window completes, so its duration is the window plus a drain that differs by candidate. `Goodput` and `GoodputPerGPU` are rescaled to the window length; the throughput objectives are not, so they rank the candidate that finishes the window's tokens sooner. [Scoring and Outcomes](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#scoring-and-outcomes) gives the formulas and the reasons. +- **Missing inputs.** A replay that lacks an input of the objective fails that candidate instead of scoring it, for example `TTFT` with no first-token samples, or a per-GPU objective without GPU-hours. +- **Scaling.** With load or throughput scaling enabled, each candidate replay runs a Planner that sizes its own replicas, and `retune.planner` can search `min_workers` and the GPU budgets. A metric that is not divided by GPUs improves with more GPUs and would apply the candidate that provisions the most, so only `GoodputPerGPU` and `ThroughputPerGPU` are accepted. `ThroughputPerGPU` ignores the SLA, so with scaling it favors the candidate that serves the window on the fewest GPUs, whatever its latency. + ### Search domains `retune.router` and `retune.planner` use the knob names, value syntax, and default domains of the Dynamo router and Planner adapters of `aisimulate recommend --stack dynamo`: From ae432960c5590786c1387cf86677f53764edc6e4 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Wed, 28 Oct 2026 18:40:50 -0700 Subject: [PATCH 002/100] feat(planner): enable live retune through DGDR v1beta2 A live planner can now run retune (executor: dgdr). Every interval it cuts the window from the frontends' request traces on a shared RWX claim, replays it through DGDR v1beta2 Requests (one per balanced Cartesian shard, plus a CRN probe), scores the Candidates with the replay score_summary, decides with the unchanged _decide, and applies the winner to every frontend router through PATCH /router_policy. One advisory switch covers scaling and retune. - retune/live: frontend config parsing and premises (P4, P6-P10, P14), the streaming window cutter with per-member gzip validation, the DGDR executor (shards, Strict create, adoption by spec hash, lifecycle state machine, read-back, scoring), the supervisor (Lease leadership, a term-guarded state snapshot, outbox, initialization table, two-phase rollback, revalidation), the router actuator and the reconciler. - Controller, adapter and base hooks: launch_gate, harvest_allowed, drop_pending/rewind, epoch-phase windows, replace_retune, a disabled adapter state, SIGTERM handling; replay is unchanged. - Config: retune.executor and retune.dgdr with live premises P1-P3, P5, P13. - Replay: trace runs pass _deterministic=True (common random numbers). - Helm: planner RBAC for DGDR v1beta2 objects and Leases when dgdrV1Beta2.enabled; docs for the experimental live retune. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/planner/__main__.py | 71 +- .../dynamo/planner/config/planner_config.py | 569 +++- components/src/dynamo/planner/core/base.py | 186 +- components/src/dynamo/planner/core/types.py | 17 +- .../plugins/orchestrator/engine_adapter.py | 166 +- .../src/dynamo/planner/retune/controller.py | 451 ++- .../dynamo/planner/retune/live/__init__.py | 11 + .../dynamo/planner/retune/live/actuator.py | 868 +++++ .../dynamo/planner/retune/live/bootstrap.py | 638 ++++ .../dynamo/planner/retune/live/contracts.py | 470 +++ .../src/dynamo/planner/retune/live/dgdr.py | 1909 +++++++++++ .../planner/retune/live/frontend_config.py | 1794 ++++++++++ .../src/dynamo/planner/retune/live/kube.py | 397 +++ .../src/dynamo/planner/retune/live/metrics.py | 262 ++ .../dynamo/planner/retune/live/reconciler.py | 919 ++++++ .../src/dynamo/planner/retune/live/state.py | 889 +++++ .../dynamo/planner/retune/live/supervisor.py | 2019 ++++++++++++ .../dynamo/planner/retune/live/window_cut.py | 1401 ++++++++ components/src/dynamo/planner/retune/sweep.py | 26 +- .../tests/integration/test_live_retune_e2e.py | 1238 +++++++ .../dynamo/planner/tests/live_retune_fakes.py | 1098 +++++++ .../src/dynamo/planner/tests/retune_stubs.py | 41 +- .../tests/unit/test_live_retune_actuator.py | 1356 ++++++++ .../tests/unit/test_live_retune_adapter.py | 922 ++++++ .../tests/unit/test_live_retune_config.py | 619 ++++ .../unit/test_live_retune_controller_hooks.py | 1236 +++++++ .../tests/unit/test_live_retune_dgdr.py | 1884 +++++++++++ .../unit/test_live_retune_frontend_config.py | 1889 +++++++++++ .../tests/unit/test_live_retune_kube.py | 530 +++ .../tests/unit/test_live_retune_metrics.py | 277 ++ .../tests/unit/test_live_retune_reconciler.py | 1031 ++++++ .../tests/unit/test_live_retune_state.py | 562 ++++ .../tests/unit/test_live_retune_supervisor.py | 2915 +++++++++++++++++ .../tests/unit/test_live_retune_window_cut.py | 1253 +++++++ components/src/dynamo/replay/simulation.py | 9 + .../replay/tests/test_simulation_crn.py | 140 + .../operator/templates/planner.yaml | 33 + .../platform/tests/planner_rbac_test.yaml | 185 +- docs/fern/index.yml | 2 + .../planner/aisimulate-planner-replay.mdx | 18 +- .../planner/planner-design.md | 38 +- .../planner/planner-guide.md | 27 +- .../auto-deployment/planner-live-retune.md | 377 +++ .../components/planner-configuration.mdx | 100 +- .../planner/planner-guide.md | 2 +- 45 files changed, 30751 insertions(+), 94 deletions(-) create mode 100644 components/src/dynamo/planner/retune/live/__init__.py create mode 100644 components/src/dynamo/planner/retune/live/actuator.py create mode 100644 components/src/dynamo/planner/retune/live/bootstrap.py create mode 100644 components/src/dynamo/planner/retune/live/contracts.py create mode 100644 components/src/dynamo/planner/retune/live/dgdr.py create mode 100644 components/src/dynamo/planner/retune/live/frontend_config.py create mode 100644 components/src/dynamo/planner/retune/live/kube.py create mode 100644 components/src/dynamo/planner/retune/live/metrics.py create mode 100644 components/src/dynamo/planner/retune/live/reconciler.py create mode 100644 components/src/dynamo/planner/retune/live/state.py create mode 100644 components/src/dynamo/planner/retune/live/supervisor.py create mode 100644 components/src/dynamo/planner/retune/live/window_cut.py create mode 100644 components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py create mode 100644 components/src/dynamo/planner/tests/live_retune_fakes.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_config.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_kube.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_state.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py create mode 100644 components/src/dynamo/replay/tests/test_simulation_crn.py create mode 100644 docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md diff --git a/components/src/dynamo/planner/__main__.py b/components/src/dynamo/planner/__main__.py index e2f495deb106..800685f8220d 100644 --- a/components/src/dynamo/planner/__main__.py +++ b/components/src/dynamo/planner/__main__.py @@ -16,6 +16,9 @@ import argparse import asyncio import logging +import signal +from collections.abc import Awaitable +from typing import Any from pydantic import BaseModel @@ -25,6 +28,9 @@ logger = logging.getLogger(__name__) +# How long a stopped planner waits for its endpoint to stop serving. +_SERVE_STOP_TIMEOUT_S = 10.0 + class RequestType(BaseModel): text: str @@ -43,13 +49,66 @@ async def generate(request: RequestType): # once the handler is registered with the transport. Running it concurrently # with planner.run() ensures the system server (/health, /live) reports the # planner as ready only after _async_init() has completed. - await asyncio.gather( - generate_endpoint.serve_endpoint( - generate, # type: ignore[arg-type] - health_check_payload={"text": "health"}, - ), - planner.run(), + serve = generate_endpoint.serve_endpoint( + generate, # type: ignore[arg-type] + health_check_payload={"text": "health"}, ) + if config.retune is None: + await asyncio.gather(serve, planner.run()) + return + await _run_until_sigterm(runtime, serve, planner.run()) + + +async def _run_until_sigterm( + runtime: Any, serve: Awaitable[Any], run: Awaitable[None] +) -> None: + """Serve and run the planner; SIGTERM stops the planner cleanly. + + Live retune holds a leadership Lease and may have a sweep in flight. + Without a handler SIGTERM kills the process before ``run()``'s + ``finally`` closes the retune controller and releases the Lease, so the + next leader would wait out the Lease. The handler cancels only the run + task; once its shutdown finished, the runtime is shut down to stop + serving the endpoint and the call returns normally. + """ + + loop = asyncio.get_running_loop() + run_task = asyncio.ensure_future(run) + serve_task = asyncio.ensure_future(serve) + terminating = False + + def on_sigterm() -> None: + nonlocal terminating + if terminating: + return # a second SIGTERM must not interrupt the shutdown + terminating = True + logger.info("SIGTERM received; stopping the planner") + run_task.cancel() + + try: + loop.add_signal_handler(signal.SIGTERM, on_sigterm) + installed = True + except (NotImplementedError, RuntimeError, ValueError) as exc: + # No signal handlers on this platform or off the main thread. + logger.warning("Cannot install a planner SIGTERM handler: %s", exc) + installed = False + try: + await asyncio.gather(serve_task, run_task) + except asyncio.CancelledError: + if not (terminating and run_task.done() and run_task.cancelled()): + raise + logger.info("Planner stopped; shutting down the runtime") + runtime.shutdown() + done, _ = await asyncio.wait({serve_task}, timeout=_SERVE_STOP_TIMEOUT_S) + if not done: + serve_task.cancel() + elif not serve_task.cancelled() and serve_task.exception() is not None: + logger.warning( + "Planner endpoint stopped with an error: %s", serve_task.exception() + ) + finally: + if installed: + loop.remove_signal_handler(signal.SIGTERM) async def init_planner(runtime: DistributedRuntime, config: PlannerConfig): diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index e88692aff1c5..c33e7eec70a8 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -23,7 +23,7 @@ from dataclasses import asdict from enum import Enum from functools import lru_cache -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Annotated, Any, Dict, Literal, Optional, Protocol, cast from urllib.parse import parse_qsl, urlsplit @@ -847,6 +847,375 @@ def replay_kwargs( ) +class DgdrRunConfig(BaseModel): + """How each DGDR v1beta2 Run of a live retune sweep evaluates its points.""" + + model_config = ConfigDict(extra="forbid") + + parallelism: int = Field( + default=8, + ge=1, + description=( + "AISimulate optimizer.parallelism of each Run. The Run Job's CPU " + "and memory may lower the effective value." + ), + ) + candidate_timeout_seconds: float = Field( + default=1800.0, + gt=0, + allow_inf_nan=False, + description="AISimulate optimizer.candidate_timeout_seconds of each Run.", + ) + max_shards: int = Field( + default=3, + ge=1, + le=16, + description=( + "Most grid shards (Requests) per window. A shard is a Cartesian " + "sub-grid of at most 100 points; off-grid points and the CRN probe " + "are extra single-point Requests and do not count." + ), + ) + + +class DgdrExecutorConfig(BaseModel): + """Live retune through DGDR v1beta2 (``retune.executor: dgdr``). + + The planner cuts each window from the frontend request traces on a shared + RWX claim, creates DGDR v1beta2 Requests that replay it, scores their + Candidates and applies the winner to the frontends' routers. + """ + + model_config = ConfigDict(extra="forbid") + + phase_seconds: float = Field( + default=0.0, + ge=0, + allow_inf_nan=False, + description=( + "Window phase: windows are [phase + k*interval, phase + (k+1)*interval) " + "in epoch seconds, so they survive planner restarts." + ), + ) + claim_name: str = Field( + min_length=1, + description="RWX PersistentVolumeClaim, in the DGD namespace, holding traces.", + ) + mount_path: str = Field( + default="/trace", + min_length=1, + description="Where the planner container mounts claim_name.", + ) + segments_dir: str = Field( + default="segments", + min_length=1, + description="Claim-relative directory of frontend trace segments: ///.", + ) + windows_dir: str = Field( + default="windows", + min_length=1, + description="Claim-relative directory of window files: //.", + ) + startup_router_config: dict[str, Any] | None = Field( + default=None, + description=( + "Optional startup KvRouterConfig JSON. It must equal what the planner " + "parses from the frontend's args and env; parsing is the source." + ), + ) + model: str = Field( + min_length=1, + description="HF repo id the workers serve (worker --model); AISimulate resolves it.", + ) + sku: str = Field(min_length=1, description="spec.hardware.gpu.sku of each Request.") + engine: dict[str, Any] = Field( + description=( + "AISimulate public engine section with every leaf a scalar " + "(mode, backend, backend_version, context_length, " + "workers..parallelism.preset, scheduler, kv_cache). Replicas " + "come from the DGD spec at each launch and must not be set here." + ), + ) + cut_budget_seconds: float = Field( + default=900.0, + gt=0, + allow_inf_nan=False, + description="Deadline of the subprocess that cuts a window file.", + ) + job_startup_seconds: float = Field( + default=300.0, + ge=0, + allow_inf_nan=False, + description="Run Job start-up time reserved when deciding whether a shard can still finish.", + ) + run: DgdrRunConfig = Field(default_factory=DgdrRunConfig) + keep_requests: int = Field( + default=5, + ge=0, + description="Windows whose terminal Requests are kept for audit.", + ) + max_window_rows: int = Field( + default=5_000_000, + ge=1, + description="A window with more requests fails (window_too_large).", + ) + promote_last_advisory: bool = Field( + default=False, + description=( + "When switching from advisory to active, apply the last advisory " + "recommendation if it is recent and its fingerprint still matches." + ), + ) + lease_duration_seconds: float = Field( + default=60.0, + ge=15, + allow_inf_nan=False, + description="Leadership Lease duration; renewed every third of it.", + ) + actuator_interval_seconds: float = Field( + default=15.0, + gt=0, + allow_inf_nan=False, + description="Period of the router actuator and reconciler rounds.", + ) + trace_incomplete_threshold: float = Field( + default=0.05, + ge=0, + le=1, + description=( + "With a Prometheus endpoint, a window whose request count is below " + "the frontend's by more than this fraction fails (trace_incomplete)." + ), + ) + + +# Live retune (``retune.executor: dgdr``) limits. A sweep must end this long +# before the next window's launch: harvest, the decision write and clean-up +# run in that margin. +DGDR_TIMEOUT_MARGIN_SECONDS = 600.0 +# DGDR v1beta2 ``spec.recommendation.maxCandidates`` maximum: one Request (a +# grid shard) carries at most this many router points. +DGDR_MAX_SHARD_POINTS = 100 +# Router knobs a DGDR v1beta2 Request can search, by public name +# (``contracts.DGDR_ROUTER_KNOBS``); the Request's router section has no +# place for the retune extension knobs. +_DGDR_ROUTER_SECTION_KEYS = frozenset( + { + "overlap_score_credit", + "prefill_load_scale", + "temperature", + "policy", + "prefill_load_model", + } +) +_DGDR_ENGINE_MODES = {"agg": "aggregated", "disagg": "disaggregated"} +_DGDR_ENGINE_ROLES = {"agg": ("aggregated",), "disagg": ("prefill", "decode")} +# ``parallelism.preset`` entry keys: AISimulate requires all of these plus +# ``replicas``, which the executor sets from the DGD spec at each launch. +_DGDR_PRESET_KEYS = ("tensor", "pipeline", "attention_data", "moe_tensor", "moe_expert") +_DGDR_PRESET_OPTIONAL_KEYS = ("prefill_context", "decode_context") +# Scheduler knobs AISimulate searches over a default domain when omitted. +_DGDR_SCHEDULER_KNOBS = ("max_batched_tokens", "max_sequences") + + +def dgdr_need_seconds( + points: int, dgdr: DgdrExecutorConfig, *, needs_cut: bool +) -> float: + """Wall-clock seconds one DGDR shard of ``points`` router points needs. + + ``cut_budget`` (when the window file must still be cut) + Run Job + start-up + one round per ``run.parallelism`` candidates, plus one: the + AISimulate optimizer evaluates its first candidate alone. ``parallelism`` + is the configured value; Job resources may lower it, so this is + optimistic. + """ + + if points < 1: + raise ValueError(f"a DGDR shard has at least one point, got {points}") + rounds = 1 + math.ceil((points - 1) / dgdr.run.parallelism) + cut = dgdr.cut_budget_seconds if needs_cut else 0.0 + return cut + dgdr.job_startup_seconds + rounds * dgdr.run.candidate_timeout_seconds + + +def balanced_shard_parts(sizes: Sequence[int], max_shards: int) -> tuple[int, ...]: + """How many chunks to cut each grid dimension into for DGDR sharding. + + ``sizes`` are the value counts of the router grid dimensions. Cutting + dimension ``i`` into ``parts[i]`` contiguous chunks of at most + ``ceil(sizes[i] / parts[i])`` values gives ``prod(parts)`` Cartesian + shards, the largest with ``largest_shard_points(sizes, parts)`` points. + Returns the parts with ``prod(parts) <= max_shards`` that minimize the + largest shard, then the shard count, then lexicographically, so every + caller derives the same split: the default 3 x 9 x 4 grid with + ``max_shards=3`` gives ``(1, 3, 1)``, three shards of 36 points. + """ + + if max_shards < 1: + raise ValueError(f"max_shards must be at least 1, got {max_shards}") + if any(size < 1 for size in sizes): + raise ValueError(f"every grid dimension needs a value, got sizes {sizes}") + + def splits(index: int, budget: int): + if index == len(sizes): + yield () + return + for part in range(1, min(sizes[index], budget) + 1): + for rest in splits(index + 1, budget // part): + yield (part, *rest) + + return min( + splits(0, max_shards), + key=lambda parts: ( + largest_shard_points(sizes, parts), + math.prod(parts), + parts, + ), + ) + + +def largest_shard_points(sizes: Sequence[int], parts: Sequence[int]) -> int: + """Points in the largest shard of ``balanced_shard_parts``' split.""" + + return math.prod( + math.ceil(size / part) for size, part in zip(sizes, parts, strict=True) + ) + + +def _validate_dgdr_engine( + engine: Mapping[str, Any], planner_mode: str, backend: str +) -> None: + """P5: ``retune.dgdr.engine`` pins one engine with scalars only. + + The section is the Request's ``inputOverrides.engine`` merge patch, so + any AISimulate search domain left in it (a ``choices`` or ``range`` + mapping, a parallelism preset list or a default domain) would make the + Run search engine settings next to the router knobs. A written + ``backend`` must be the deployed one (``PlannerConfig.backend``), which + every Request names and which fills an omitted ``backend``. + """ + + name = "retune.dgdr.engine" + _check_dgdr_engine_scalars(engine, (name,)) + written = engine.get("backend") + if written is not None and written != backend: + raise ValueError( + f"{name}.backend {written!r} differs from the deployed backend " + f"{backend!r} (P5); remove it or set it to {backend!r}" + ) + expected_mode = _DGDR_ENGINE_MODES[planner_mode] + if engine.get("mode") != expected_mode: + raise ValueError( + f"{name}.mode must be {expected_mode!r} for planner mode " + f"{planner_mode!r} (P5), got {engine.get('mode')!r}; an omitted mode " + "searches both aggregated and disaggregated engines" + ) + for key, owner in (("model", "model"), ("hardware", "sku")): + if key in engine: + raise ValueError( + f"{name}.{key} would override the Request's typed field (P5); " + f"set retune.dgdr.{owner} instead" + ) + workers = engine.get("workers") + if not isinstance(workers, Mapping): + raise ValueError( + f"{name}.workers must be a mapping that pins every worker role of " + f"planner mode {planner_mode!r} (P5)" + ) + used = _DGDR_ENGINE_ROLES[planner_mode] + for role, worker in workers.items(): + if role not in used and worker is not None: + raise ValueError( + f"{name}.workers.{role} is not a worker role of planner mode " + f"{planner_mode!r}; write it as null or remove it (P5)" + ) + for role in used: + path = f"{name}.workers.{role}" + worker = workers.get(role) + if not isinstance(worker, Mapping): + raise ValueError( + f"{path} is required for planner mode {planner_mode!r} (P5)" + ) + _validate_dgdr_parallelism(worker.get("parallelism"), f"{path}.parallelism") + scheduler = worker.get("scheduler") + scheduler = scheduler if isinstance(scheduler, Mapping) else {} + for knob in _DGDR_SCHEDULER_KNOBS: + if not _is_positive_int(scheduler.get(knob)): + raise ValueError( + f"{path}.scheduler.{knob} must be a positive integer (P5); " + "omitted, AISimulate searches its default domain" + ) + + +def _validate_dgdr_parallelism(parallelism: Any, path: str) -> None: + preset = parallelism.get("preset") if isinstance(parallelism, Mapping) else None + if not ( + isinstance(preset, list) and len(preset) == 1 and isinstance(preset[0], Mapping) + ): + raise ValueError( + f"{path}.preset must be a list of exactly one mapping with " + f"{', '.join(_DGDR_PRESET_KEYS)} (P5); got {preset!r}" + ) + others = sorted(set(parallelism) - {"preset"}) + if others: + raise ValueError( + f"{path} sets {others} next to preset; pin the parallelism in the " + "single preset entry (P5)" + ) + entry = preset[0] + missing = [key for key in _DGDR_PRESET_KEYS if key not in entry] + unknown = sorted( + set(entry) - set(_DGDR_PRESET_KEYS) - set(_DGDR_PRESET_OPTIONAL_KEYS) + ) + if missing or unknown: + raise ValueError( + f"{path}.preset[0] must set exactly {', '.join(_DGDR_PRESET_KEYS)} " + f"(P5); missing={missing}, unknown={unknown}" + ) + for key, value in entry.items(): + if not _is_positive_int(value): + raise ValueError( + f"{path}.preset[0].{key} must be a positive integer (P5), got {value!r}" + ) + + +def _check_dgdr_engine_scalars(value: Any, path: tuple[str, ...]) -> None: + dotted = ".".join(path) + if isinstance(value, Mapping): + domain = sorted({"choices", "range"} & set(value)) + if domain: + raise ValueError( + f"{dotted} is a search domain ({domain[0]}); every " + "retune.dgdr.engine value must be a scalar (P5)" + ) + for key, item in value.items(): + if key == "replicas": + raise ValueError( + f"{dotted}.replicas must not be set: the DGD spec provides " + "replicas at each launch (P5)" + ) + _check_dgdr_engine_scalars(item, (*path, str(key))) + elif isinstance(value, list): + # Only workers..parallelism.preset is a list; its shape is + # checked per used role. + if not ( + len(path) == 5 + and path[1] == "workers" + and path[3:] == ("parallelism", "preset") + ): + raise ValueError( + f"{dotted} is a list; every retune.dgdr.engine value except " + "workers..parallelism.preset must be a scalar (P5)" + ) + for index, item in enumerate(value): + _check_dgdr_engine_scalars(item, (*path[:-1], f"{path[-1]}[{index}]")) + elif value is not None and not isinstance(value, (str, int, float)): + raise ValueError(f"{dotted} must be a scalar, got {type(value).__name__} (P5)") + + +def _is_positive_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + class RetuneConfig(BaseModel): """Periodic replay-based retune of router and planner runtime parameters. @@ -981,6 +1350,17 @@ class RetuneConfig(BaseModel): "AISimulate default domain; null does not search the router." ), ) + executor: Literal["local", "dgdr"] = Field( + default="local", + description=( + "Where sweeps run: local worker processes (replay) or DGDR v1beta2 " + "Requests (live; requires dgdr)." + ), + ) + dgdr: DgdrExecutorConfig | None = Field( + default=None, + description="Live DGDR executor settings; required iff executor is dgdr.", + ) planner: Literal["default"] | dict[str, Any] | None = Field( default=None, description=( @@ -1019,6 +1399,12 @@ def _validate_domain_syntax( parse_retune_section(section, value) # type: ignore[call-overload] return value + @model_validator(mode="after") + def _validate_executor(self) -> "RetuneConfig": + if (self.executor == "dgdr") != (self.dgdr is not None): + raise ValueError("retune.dgdr is required iff retune.executor is 'dgdr'") + return self + @model_validator(mode="after") def _validate_timing(self) -> "RetuneConfig": timeout = self.effective_timeout_seconds @@ -1456,8 +1842,8 @@ def _validate_ca_bundle_path(cls, v: Optional[str]) -> Optional[str]: ) # Advisory mode: compute, log and report scaling decisions without - # executing them, live and in planner-in-the-loop replay; replay records - # retune decisions (replay-only) the same way. + # executing them, live and in planner-in-the-loop replay; retune records + # a decision that would apply as advisory and applies nothing. advisory: bool = SLAPlannerDefaults.advisory # --- Power-aware budget (read-only caps; DGD-owned) --- @@ -1565,9 +1951,11 @@ def _validate_ca_bundle_path(cls, v: Optional[str]) -> Optional[str]: default=None, description=( "Opt-in periodic replay-based retune of router and planner runtime " - "parameters. Replay-only in this release. Setting both " - "enable_load_scaling and enable_throughput_scaling explicitly to " - "false with retune set runs retune-only with fixed replicas." + "parameters. executor 'local' runs in planner-in-the-loop replay; " + "a live Kubernetes planner requires executor 'dgdr' " + "(experimental). Setting both enable_load_scaling and " + "enable_throughput_scaling explicitly to false with retune set " + "runs retune-only with fixed replicas." ), ) @@ -1952,6 +2340,9 @@ def _validate_retune(self, retune: RetuneConfig) -> None: raise ValueError( f"retune requires mode='agg' or mode='disagg', got {self.mode!r}" ) + if retune.executor == "dgdr": + # Before the domain expansion, whose errors assume replay. + self._validate_live_retune(retune) if self.scaling_enabled() and retune.objective not in RETUNE_PER_GPU_OBJECTIVES: # Every candidate replay runs an inner planner that sizes its own # replicas, and min_workers or the GPU budgets may be searched; a @@ -1999,6 +2390,172 @@ def _validate_retune(self, retune: RetuneConfig) -> None: "omitted AISimulate knob searches its default domain, so pin " "knobs with a scalar to shrink the grid" ) + if retune.executor == "dgdr": + self._validate_live_retune_budget(retune, router_domain) + + def _validate_live_retune(self, retune: RetuneConfig) -> None: + """Configuration premises P1, P2, P3 and P5 of live retune. + + A live planner sweeps through DGDR v1beta2 Requests in the Kubernetes + namespace of its DGD, retunes the frontend router only (Scenario 1), + and decides when a sweep returns; the frontend-side premises are + checked at runtime against the DGD. + """ + + dgdr = retune.dgdr + if dgdr is None: # RetuneConfig requires it with executor dgdr + raise ValueError("retune.dgdr is required iff retune.executor is 'dgdr'") + if self.environment != "kubernetes": + raise ValueError( + "retune.executor 'dgdr' requires environment='kubernetes' (P1), " + f"got {self.environment!r}" + ) + latency = retune.decision_latency_seconds + if latency == "measured": + if retune.decision_latency_scale != 1.0: + raise ValueError( + "retune.executor 'dgdr' decides when the sweep returns, so " + "decision_latency_seconds: measured requires " + "decision_latency_scale 1 (P1), got " + f"{retune.decision_latency_scale}" + ) + elif latency != 0: + raise ValueError( + "retune.executor 'dgdr' decides when the sweep returns, so " + "retune.decision_latency_seconds must be 0 or 'measured' (P1), " + f"got {latency}" + ) + if retune.timeout_seconds is None: + raise ValueError( + "retune.executor 'dgdr' requires an explicit " + "retune.timeout_seconds (P1): the sweep deadline, window cut " + "included" + ) + limit = retune.interval_seconds - DGDR_TIMEOUT_MARGIN_SECONDS + if retune.timeout_seconds > limit: + raise ValueError( + f"retune.timeout_seconds must be <= interval_seconds - " + f"{DGDR_TIMEOUT_MARGIN_SECONDS:g} = {limit:g} with executor " + f"'dgdr' (P1), got {retune.timeout_seconds:g}" + ) + if self.scaling_enabled(): + raise ValueError( + "retune.executor 'dgdr' supports retune-only mode (P2): set " + "enable_load_scaling and enable_throughput_scaling both to false" + ) + if retune.planner is not None: + raise ValueError( + "retune.executor 'dgdr' retunes the router only (P2); remove " + "retune.planner" + ) + self._validate_live_retune_router(retune) + _validate_dgdr_engine(dgdr.engine, self.mode, self.backend) + # Claim paths: the planner, the frontends and the Run Jobs mount the + # same claim, and clean-up deletes under these directories. A DGDR + # trace path rejects empty, '.' and '..' segments. + dirs: dict[str, list[str]] = {} + for name in ("segments_dir", "windows_dir"): + value = getattr(dgdr, name) + dirs[name] = value.split("/") + if any(segment in ("", ".", "..") for segment in dirs[name]): + raise ValueError( + f"retune.dgdr.{name} must be a relative directory inside " + "the claim: '/'-separated names without empty, '.' or " + f"'..' segments, got {value!r}" + ) + if not PurePosixPath(dgdr.mount_path).is_absolute(): + raise ValueError( + f"retune.dgdr.mount_path must be absolute, got {dgdr.mount_path!r}" + ) + segments, windows = dirs["segments_dir"], dirs["windows_dir"] + shorter = min(len(segments), len(windows)) + if segments[:shorter] == windows[:shorter]: + raise ValueError( + "retune.dgdr.segments_dir and retune.dgdr.windows_dir must not " + f"contain one another, got {dgdr.segments_dir!r} and " + f"{dgdr.windows_dir!r}" + ) + + @staticmethod + def _validate_live_retune_router(retune: RetuneConfig) -> None: + """P3: the router domain fits the DGDR v1beta2 router section. + + A Request searches ``overlap_score_credit``, ``prefill_load_scale`` + and ``temperature`` of a ``kv_router`` with ``prefill_load_model: + none``; ``RouterRetuneConfig`` already pins the policy. + """ + + spec = retune.router + if not isinstance(spec, Mapping): + return + extra = sorted(set(spec) - _DGDR_ROUTER_SECTION_KEYS) + if extra: + raise ValueError( + f"retune.router {extra} cannot be searched with executor 'dgdr' " + "(P3): a DGDR Request searches only overlap_score_credit, " + "prefill_load_scale and temperature" + ) + if "prefill_load_model" not in spec: + return + # Imported on use: the adapter schema imports this module. + from aisimulate.config.common import Choices + + from dynamo.planner.retune.domains import parse_retune_section + + model = parse_retune_section("router", spec).prefill_load_model + if "type" not in model.model_fields_set: + return + written = ( + list(model.type.choices) + if isinstance(model.type, Choices) + else [model.type] + ) + if written != ["none"]: + raise ValueError( + "retune.router.prefill_load_model.type must be 'none' with " + f"executor 'dgdr' (P3), got {written}" + ) + + @staticmethod + def _validate_live_retune_budget( + retune: RetuneConfig, router_domain: Mapping[str, Sequence[float]] + ) -> None: + """P13: the largest grid shard can finish within the sweep deadline. + + The baseline can fall in any grid shard, and a window whose baseline + shard cannot finish fails with ``launch_too_late``, so the largest + shard must fit. The domain is the widest one: a zero startup credit + only narrows it at runtime. + """ + + dgdr = retune.dgdr + if dgdr is None or retune.timeout_seconds is None: + # _validate_live_retune rejects both first. + raise ValueError("retune.executor 'dgdr' requires dgdr and timeout_seconds") + sizes = [len(values) for values in router_domain.values()] + parts = balanced_shard_parts(sizes, dgdr.run.max_shards) + points = largest_shard_points(sizes, parts) + if points > DGDR_MAX_SHARD_POINTS: + raise ValueError( + f"the retune router grid of {math.prod(sizes)} points cannot be " + f"split into at most retune.dgdr.run.max_shards=" + f"{dgdr.run.max_shards} Cartesian shards of at most " + f"{DGDR_MAX_SHARD_POINTS} points (P13); the largest would have " + f"{points}. Raise max_shards or pin router knobs with a scalar" + ) + need = dgdr_need_seconds(points, dgdr, needs_cut=True) + if need > retune.timeout_seconds: + rounds = 1 + math.ceil((points - 1) / dgdr.run.parallelism) + raise ValueError( + f"the largest retune grid shard ({points} points) needs " + f"{need:g}s: cut_budget_seconds {dgdr.cut_budget_seconds:g} + " + f"job_startup_seconds {dgdr.job_startup_seconds:g} + {rounds} " + f"rounds x candidate_timeout_seconds " + f"{dgdr.run.candidate_timeout_seconds:g}, more than " + f"retune.timeout_seconds={retune.timeout_seconds:g} (P13), so " + "every window would fail launch_too_late. Raise timeout_seconds " + "or run.parallelism, add run.max_shards, or shrink the grid" + ) def _validate_retune_planner_grid( self, diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index 3203d1948ec9..cc7ddd93ed0a 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -12,11 +12,12 @@ from __future__ import annotations import asyncio +import importlib import logging import time from collections.abc import AsyncIterator, Iterable from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Optional, Protocol import aiohttp.web from prometheus_client import start_http_server @@ -59,12 +60,59 @@ from dynamo.runtime import DistributedRuntime if TYPE_CHECKING: + from dynamo.planner.core.types import RetuneInputs from dynamo.planner.monitoring.worker_info import WorkerInfo from dynamo.planner.plugins.builtins.local_planner import BuiltinLoadPredict + from dynamo.planner.plugins.orchestrator.engine_adapter import RetuneHooks logger = logging.getLogger(__name__) + +class LiveRetuneHandle(Protocol): + """What ``retune.live.bootstrap.build_live_retune`` returns to the base. + + ``retune_inputs`` is ``None`` while the frontend's startup router config + does not satisfy the retune premises (the engine then starts with retune + disabled). ``start`` runs after the engine is bootstrapped and may call + back ``replace_live_retune``; ``on_effects`` sees every tick's effects + (it must be cheap and never block); ``stop`` releases leadership. + """ + + def retune_inputs(self) -> Optional[RetuneInputs]: + ... + + def hooks(self) -> RetuneHooks: + ... + + async def start(self, host: NativePlannerBase) -> None: + ... + + async def stop(self) -> None: + ... + + def on_effects(self, effects: PlannerEffects) -> None: + ... + + +class RetuneCleanupHandle(Protocol): + """What ``retune.live.bootstrap.build_legacy_cleanup`` returns (§4.11). + + A planner without retune config rolls back what a removed live retune + deployed, then deletes its Requests and Leases; ``start`` runs after the + engine is bootstrapped and ``stop`` releases leadership. + """ + + async def start(self, host: NativePlannerBase) -> None: + ... + + async def stop(self) -> None: + ... + + _CONFIG_LOCK_TIMEOUT_SECONDS = 10.0 +# Building the legacy retune clean-up reads two Leases and the DGD; startup +# never waits longer than this for it. +_LEGACY_RETUNE_CLEANUP_TIMEOUT_SECONDS = 30.0 # A tick waits briefly for the common fast submission path, then leaves the # task running. It is deliberately not cancelled: a remote request may have # been accepted even when its response has not arrived, so retrying could @@ -131,8 +179,13 @@ def __init__( config: PlannerConfig, environment: PlannerEnvironment, ) -> None: - if config.retune is not None: - raise ValueError("planner.retune is replay-only in this release") + if config.retune is not None and config.retune.executor == "local": + # Local sweeps replay in this process; a live planner retunes + # through DGDR v1beta2 Requests only. + raise ValueError( + "planner.retune is replay-only with executor 'local'; " + "use executor: dgdr" + ) self.runtime = runtime self.config = config self.environment = environment @@ -174,6 +227,10 @@ def __init__( self._environment_initialized = False self._engine: Optional[EngineProtocol] = None self._last_worker_counts: Optional[WorkerCounts] = None + # Live retune (``retune.executor: dgdr``): built in _async_init. + self._live_retune: Optional[LiveRetuneHandle] = None + # §4.11 legacy clean-up of a removed live retune (no retune config). + self._legacy_retune_cleanup: Optional[RetuneCleanupHandle] = None async def _async_init(self) -> None: # Shutdown is safe for a partially initialized environment and is @@ -183,9 +240,20 @@ async def _async_init(self) -> None: await self.environment.initialize() self._validate_min_endpoint_budgets_at_startup() + if self.config.retune is not None: + # Before the first _ensure_engine (regression bootstrap builds + # the engine): the engine takes the live retune inputs/hooks. + self._live_retune = await self._build_live_retune() + else: + self._legacy_retune_cleanup = await self._build_legacy_retune_cleanup() + await self._bootstrap_regression() await self._bootstrap_engine_plugins_if_needed() + if self._live_retune is not None: + await self._live_retune.start(self) + await self._start_legacy_retune_cleanup() + if self.config.enable_runtime_reconfiguration: if self.runtime is None: raise ValueError("system-port reconfiguration requires a runtime") @@ -221,6 +289,87 @@ async def _async_init(self) -> None: await self._shutdown_runtime() raise + async def _build_live_retune(self) -> LiveRetuneHandle: + # Imported on use: the live retune stack talks to Kubernetes, and + # planners without retune never load it. + bootstrap = importlib.import_module("dynamo.planner.retune.live.bootstrap") + return await bootstrap.build_live_retune(self) + + def _uses_kubernetes_connector(self) -> bool: + controller = getattr(self.environment, "controller", None) + if controller is None: + return False + try: + from dynamo.planner.connectors.kubernetes import KubernetesConnector + except ImportError: + return False + return isinstance(controller, KubernetesConnector) + + async def _build_legacy_retune_cleanup(self) -> Optional[RetuneCleanupHandle]: + """§4.11 clean-up of a live retune whose config was removed. + + Only for a Kubernetes planner. ``None`` when nothing was left over; + it never fails startup: missing RBAC (403), a missing DGD or Lease + (404) and any other error are logged and skipped. + """ + + if not self._uses_kubernetes_connector(): + return None + try: + bootstrap = importlib.import_module("dynamo.planner.retune.live.bootstrap") + return await asyncio.wait_for( + bootstrap.build_legacy_cleanup(self), + timeout=_LEGACY_RETUNE_CLEANUP_TIMEOUT_SECONDS, + ) + except Exception as exc: + logger.info("Live retune legacy clean-up skipped: %r", exc) + logger.debug("Live retune legacy clean-up setup failed", exc_info=True) + return None + + async def _start_legacy_retune_cleanup(self) -> None: + cleanup = self._legacy_retune_cleanup + if cleanup is None: + return + try: + await cleanup.start(self) + except Exception as exc: + logger.info("Live retune legacy clean-up did not start: %r", exc) + logger.debug("Live retune legacy clean-up start failed", exc_info=True) + self._legacy_retune_cleanup = None + try: + await cleanup.stop() + except Exception: + logger.debug("Stopping the legacy clean-up failed", exc_info=True) + + async def replace_live_retune( + self, + inputs: Optional[RetuneInputs], + *, + generation: int, + last_decided_k: Optional[int], + ) -> None: + """Swap the engine's live retune controller (supervisor callback). + + Serialized with ticks by the config lock, so no tick runs a + half-replaced controller. ``inputs=None`` disables retune; a + construction error propagates to the caller with retune disabled. + """ + + from dynamo.planner.plugins.orchestrator.engine_adapter import ( + OrchestratorEngineAdapter, + ) + + async with self._config_lock: + engine = self._engine + if not isinstance(engine, OrchestratorEngineAdapter): + raise RuntimeError("live retune requires the orchestrator engine") + engine.replace_retune( + inputs, + generation=generation, + now_s=time.time(), + last_decided_k=last_decided_k, + ) + async def _shutdown_runtime(self) -> None: """Release initialized planner resources after normal or failed startup.""" @@ -257,6 +406,24 @@ async def _shutdown_runtime(self) -> None: except Exception: logger.exception("Failed to stop planner engine") + # After the engine: its shutdown closed the retune controller (and + # stopped its sweep), so the supervisor can release leadership. + live_retune = self._live_retune + self._live_retune = None + if live_retune is not None: + try: + await live_retune.stop() + except Exception: + logger.exception("Failed to stop planner live retune") + + legacy_retune_cleanup = self._legacy_retune_cleanup + self._legacy_retune_cleanup = None + if legacy_retune_cleanup is not None: + try: + await legacy_retune_cleanup.stop() + except Exception: + logger.exception("Failed to stop the live retune legacy clean-up") + effect_submission_task = self._effect_submission_task self._effect_submission_task = None if effect_submission_task is not None: @@ -467,6 +634,7 @@ def _ensure_engine(self) -> EngineProtocol: OrchestratorEngineAdapter, ) + live_retune = self._live_retune self._engine = OrchestratorEngineAdapter( self.config, self._build_worker_capabilities(), @@ -475,6 +643,10 @@ def _ensure_engine(self) -> EngineProtocol: require_prefill=self.require_prefill, require_decode=self.require_decode, ), + retune_inputs=( + None if live_retune is None else live_retune.retune_inputs() + ), + retune_hooks=None if live_retune is None else live_retune.hooks(), ) return self._engine @@ -1000,6 +1172,14 @@ async def _run_one_tick( async with self._config_lock: config_generation = self._config_generation effects = await engine.tick(tick, tick_input) + if self._live_retune is not None: + # Retune decisions go to the live supervisor's outbox, which + # persists them before actuating routers; _apply_effects never + # sees router parameters. + try: + self._live_retune.on_effects(effects) + except Exception: + logger.exception("Live retune failed to take the tick's effects") await self._submit_effects(effects, config_generation) emit_diagnostics = self._should_emit_tick_diagnostics(tick, effects) if emit_diagnostics: diff --git a/components/src/dynamo/planner/core/types.py b/components/src/dynamo/planner/core/types.py index cec4ca94d9fc..195c625c7b15 100644 --- a/components/src/dynamo/planner/core/types.py +++ b/components/src/dynamo/planner/core/types.py @@ -339,7 +339,9 @@ def __hash__(self) -> int: class RetuneDecision: """Audit record of one retune decision. - ``decision_id`` is ``retune--w`` and is the live reconfiguration + ``decision_id`` is ``retune--w`` in replay and + ``retune-w`` live (windows anchored at an epoch phase, so + the id survives planner restarts); it is the live reconfiguration ``request_id``. Times are planner-clock seconds: epoch seconds live, virtual seconds in replay. ``as_of_s`` is the censoring cutoff (the scheduled launch time). ``chosen`` equals ``previous`` unless the outcome @@ -352,7 +354,10 @@ class RetuneDecision: ``(params.key(), score)`` pairs, best first. ``sweep_wall_s`` is the sweep's wall-clock time and ``decision_latency_s`` the delay the decision waited for after launch (at least one tick; ``decided_at_s`` is the tick - that observed it). + that observed it). ``failure_reason`` is a stable machine-readable reason + next to the free-text ``reason`` (for example ``replicas_changed``, or + the live sweep executor's), and ``details`` carries JSON-shaped + executor facts; both are empty in replay. """ decision_id: str @@ -376,6 +381,8 @@ class RetuneDecision: decision_latency_s: Optional[float] = None top: list[tuple[str, float]] = field(default_factory=list) reason: Optional[str] = None + failure_reason: Optional[str] = None + details: dict[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: _require_non_empty_string("decision_id", self.decision_id) @@ -391,6 +398,10 @@ def __post_init__(self) -> None: raise ValueError("previous and chosen must be RetuneParams") if self.predictor_warmup is not None: object.__setattr__(self, "predictor_warmup", tuple(self.predictor_warmup)) + if self.failure_reason is not None: + _require_non_empty_string("failure_reason", self.failure_reason) + # A private copy: the record must not change with the sweep result. + object.__setattr__(self, "details", dict(self.details)) def to_dict(self) -> dict[str, Any]: """JSON-shaped record; ``predictor_warmup`` becomes its length.""" @@ -423,6 +434,8 @@ def to_dict(self) -> dict[str, Any]: "decision_latency_s": self.decision_latency_s, "top": [[key, score] for key, score in self.top], "reason": self.reason, + "failure_reason": self.failure_reason, + "details": json.loads(json.dumps(self.details)), } diff --git a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py index 5ec1f4d2ef36..be3ab8179639 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py +++ b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py @@ -60,7 +60,8 @@ from __future__ import annotations import logging -from collections.abc import Iterable +from collections.abc import Callable, Iterable +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional, Sequence from dynamo.common.forward_pass_metrics import encode as _encode_fpm_record @@ -68,9 +69,12 @@ from dynamo.planner.monitoring.signal_health import PlannerSignalHealth if TYPE_CHECKING: + from concurrent.futures import Future + import grpc.aio from dynamo.planner.retune.controller import RetuneController + from dynamo.planner.retune.live.contracts import HarvestAllowed, LaunchGate from dynamo.planner.core.budget import ( apply_power_budget, @@ -130,6 +134,22 @@ log = logging.getLogger(__name__) +@dataclass(frozen=True) +class RetuneHooks: + """Live retune hooks a planner host hands every retune controller. + + ``sweep_fn(request, *, cancel, abort)`` runs one sweep (the DGDR + executor); ``launch_gate`` and ``harvest_allowed`` are the live + supervisor's predicates (see ``retune.controller``); ``phase_s`` is the + epoch phase the window schedule is anchored at. Replay passes no hooks. + """ + + sweep_fn: Callable[..., Any] + launch_gate: LaunchGate + harvest_allowed: HarvestAllowed + phase_s: float = 0.0 + + class OrchestratorEngineAdapter: """``EngineProtocol``-compatible wrapper around the builtin plugin chain. @@ -149,6 +169,13 @@ class OrchestratorEngineAdapter: ``RetuneController`` built from ``retune_inputs``: ``initial_tick`` anchors its window schedule, every ``tick`` runs it after the plugin pipeline, and ``shutdown`` closes it. + + A live host also passes ``retune_hooks``. Live, missing + ``retune_inputs`` (the frontend's startup router config does not satisfy + the retune premises) or a controller that fails construction leaves the + adapter in a disabled state, retune configured but no controller, + instead of failing the planner; ``replace_retune`` later installs or + removes a controller without rebuilding the engine. """ def __init__( @@ -159,6 +186,7 @@ def __init__( observe_plugin: Optional[EnvironmentObserver] = None, clock: Optional[Clock] = None, retune_inputs: Optional[RetuneInputs] = None, + retune_hooks: Optional[RetuneHooks] = None, ) -> None: self._config = config self._capabilities = capabilities @@ -325,15 +353,47 @@ def _factory(plugin_id, endpoint, *, in_process_instance=None): self._plugins_bootstrapped = False self._in_process_plugins_loaded = False self._register_builtin_plugins() - self._retune = self._build_retune_controller(retune_inputs) + self._retune_hooks = retune_hooks + self._retune: Optional[RetuneController] = None + # A replaced controller's sweep that may still be running while + # retune is disabled; the next generation's first launch waits for it. + self._retune_predecessor: Optional[Future[Any]] = None + if retune_hooks is None or self._config.retune is None: + self._retune = self._build_retune_controller(retune_inputs) + elif retune_inputs is None: + log.warning( + "planner.retune is configured but its live premises do not " + "hold; retune stays disabled until they do" + ) + else: + try: + self._retune = self._build_retune_controller( + retune_inputs, generation=0 + ) + except ValueError as exc: + # A live planner never exits on a retune premise: the + # supervisor keeps running (to roll routers back) and may + # install a controller later through replace_retune. + log.error( + "planner.retune controller construction failed; retune " + "stays disabled: %s", + exc, + ) def _build_retune_controller( - self, retune_inputs: Optional[RetuneInputs] + self, + retune_inputs: Optional[RetuneInputs], + *, + generation: int = 0, + predecessor: Optional[Future[Any]] = None, ) -> Optional[RetuneController]: """Build the retune controller iff ``config.retune`` is set. The controller validates the retune setup in its constructor, so an - invalid one fails here, at engine construction. + invalid one fails here, at engine construction. With live hooks the + controller gets them, ``generation`` and ``predecessor`` (the sweep + its first launch waits for); without (replay) it is built exactly as + before. """ if self._config.retune is None: return None @@ -348,14 +408,98 @@ def _build_retune_controller( # this module. from dynamo.planner.retune.controller import RetuneController + hooks = self._retune_hooks + if hooks is None: + return RetuneController( + self._config, + retune_inputs, + self._capabilities, + self._clock, + self._scale_interval, + ) return RetuneController( self._config, retune_inputs, self._capabilities, self._clock, self._scale_interval, + sweep_fn=hooks.sweep_fn, + launch_gate=hooks.launch_gate, + harvest_allowed=hooks.harvest_allowed, + generation=generation, + predecessor=predecessor, ) + @property + def retune_controller(self) -> Optional[RetuneController]: + """The installed retune controller; ``None`` without retune or when + a live retune is disabled.""" + + return self._retune + + def replace_retune( + self, + retune_inputs: Optional[RetuneInputs], + *, + generation: int, + now_s: float, + last_decided_k: Optional[int], + ) -> None: + """Swap the live retune controller without rebuilding the engine. + + The old controller's sweep is aborted (``drop_pending``) and the + controller closed. With ``retune_inputs`` a new controller of + ``generation`` is built with the same hooks, anchored at ``now_s`` + and the hooks' phase, and rewound to ``last_decided_k`` when given; + with ``None`` the adapter stays disabled. The new controller's first + launch queues behind the old controller's sweep while that is still + running (``running_future``), also across disabled generations, so + two generations never sweep at once. A construction error is + raised after the old controller is gone, leaving the adapter + disabled. Callers serialize this with ticks (the planner's config + lock). + """ + hooks = self._retune_hooks + if self._config.retune is None or hooks is None: + raise RuntimeError("replace_retune requires a live retune configuration") + old = self._retune + self._retune = None + if old is not None: + try: + old.drop_pending() + except Exception: + log.exception("retune controller drop_pending failed") + try: + old.close() + except Exception: + log.exception("retune controller close failed") + # After drop_pending and close: the sweep that still holds the + # old controller's thread (and, transitively, older ones). + running = old.running_future() + if running is not None: + self._retune_predecessor = running + if self._retune_predecessor is not None and self._retune_predecessor.done(): + self._retune_predecessor = None + if retune_inputs is None: + log.info( + "retune controller generation %d: disabled (premises do not hold)", + generation, + ) + return + controller = self._build_retune_controller( + retune_inputs, + generation=generation, + predecessor=self._retune_predecessor, + ) + assert controller is not None + # The new controller now carries it (and hands it on when replaced). + self._retune_predecessor = None + controller.start(now_s, hooks.phase_s) + if last_decided_k is not None: + controller.rewind(last_decided_k) + self._retune = controller + log.info("retune controller generation %d installed", generation) + @property def plugins_bootstrapped(self) -> bool: return self._plugins_bootstrapped @@ -600,8 +744,12 @@ def initial_tick(self, start_s: float) -> ScheduledTick: if plugin.is_builtin and plugin.last_call_at == float("-inf"): plugin.registered_at = self._last_tick_monotonic if self._retune is not None: - # Anchors the retune window schedule (s0) at the planner start. - self._retune.start(start_s) + if self._retune_hooks is None: + # Anchors the retune window schedule (s0) at the planner start. + self._retune.start(start_s) + else: + # Live: anchored at the configured epoch phase instead. + self._retune.start(start_s, self._retune_hooks.phase_s) return self._compute_next_scheduled_tick() def apply_runtime_configuration( @@ -702,7 +850,11 @@ async def tick( # 4. Project PipelineOutcome onto PlannerEffects. A retune-only # planner (retune with both scaling modes off) keeps the # deployment's replicas: no GPU-budget or min-endpoint repair. - retune_only = self._retune is not None and not self._config.scaling_enabled() + # Decided by config, so a disabled live retune (no controller) + # keeps them too. + retune_only = ( + self._config.retune is not None and not self._config.scaling_enabled() + ) scale_to = ( None if retune_only diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index e8fa02eb00db..0060320f6945 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -23,6 +23,26 @@ known once the sweep is done. In replay the old parameters therefore serve traffic for the modeled sweep time; live, ``measured`` with scale 1 decides on the first tick after the sweep finishes. + +A live host (``retune.live``) passes hooks that replay never sets, so replay +behaviour is unchanged: + +- ``start(start_s, phase_s)`` anchors ``s0`` at an epoch phase instead of + the planner start, so windows and ``decision_id = retune-w`` + survive restarts and leader changes. +- ``launch_gate(k, window)`` is asked before a due window is launched or + recorded ``busy``: ``not_leader``/``not_ready`` leave the window due (it + is retried on a later tick), ``already_decided`` consumes it silently. +- ``harvest_allowed(generation)`` is asked before a finished sweep becomes + a decision; a refused or abandoned (``NON_DECISION_FAILURES``) sweep + records nothing and rewinds the window so it can be relaunched or adopted. +- ``drop_pending``/``rewind`` let the host abandon an in-flight sweep + without consuming its window; ``restore_desired`` re-installs persisted + router values. +- ``predecessor``/``running_future`` carry the single sweep thread across + controller generations: a rebuilt controller's first launch queues behind + the previous generation's still-running sweep, as a relaunch queues + behind a dropped sweep within one controller. """ from __future__ import annotations @@ -33,8 +53,9 @@ import math import threading from collections import deque -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import wait as wait_futures from dataclasses import dataclass, field from typing import Any @@ -63,6 +84,12 @@ grid_points, required_history_seconds, ) +from dynamo.planner.retune.live.contracts import ( + NON_DECISION_FAILURES, + GateResult, + HarvestAllowed, + LaunchGate, +) from dynamo.planner.retune.sweep import ( SweepRequest, SweepResult, @@ -76,6 +103,11 @@ _TOP_SCORES = 5 _EPSILON = 1e-9 +# Gate reasons that leave a due window due, to be retried on a later tick. +_GATE_RETRY_REASONS = frozenset({"not_leader", "not_ready"}) + +# ``sweep_fn(request, *, cancel)`` in replay; live hooks add ``abort``. +SweepFn = Callable[..., SweepResult] @dataclass(frozen=True) @@ -105,6 +137,12 @@ class _PendingSweep: # Both None while a ``measured`` sweep is still running. decision_latency_s: float | None ready_at_s: float | None + # Set by the supervisor to stop this sweep (lost leadership, rebuild). + abort: threading.Event | None = None + # When submitted behind a dropped (or predecessor generation's) sweep + # that was still running: ``(submitted_monotonic, [started_monotonic])``. + # ``launched_at_s`` then moves to the moment the sweep really started. + queued: tuple[float, list[float]] | None = None class RetuneController: @@ -118,6 +156,16 @@ class RetuneController: from it. Under ``config.advisory`` a decision that would apply is recorded as ``advisory`` and leaves the desired state at the deployed point. + + The keyword-only hooks are for a live host and default to replay + behaviour: ``sweep_fn`` replaces ``run_sweep`` (and must accept + ``abort=`` when ``launch_gate`` is set), ``launch_gate`` and + ``harvest_allowed`` are the supervisor's predicates (module docstring), + and ``generation`` identifies this controller among the ones a live host + builds over its lifetime; every sweep request carries it. + ``predecessor`` is the previous generation's ``running_future()``: this + controller's first launch waits for it, so two generations never sweep + at once. """ def __init__( @@ -127,12 +175,29 @@ def __init__( capabilities: WorkerCapabilities, clock: Clock, scale_interval_s: float, + *, + sweep_fn: SweepFn | None = None, + launch_gate: LaunchGate | None = None, + harvest_allowed: HarvestAllowed | None = None, + generation: int = 0, + predecessor: Future[SweepResult] | None = None, ) -> None: retune = config.retune if retune is None: raise ValueError("RetuneController requires config.retune") if not (math.isfinite(scale_interval_s) and scale_interval_s > 0): raise ValueError("scale_interval_s must be positive and finite") + if launch_gate is not None and sweep_fn is None: + # run_sweep takes no abort event; a gated launch passes one. + raise ValueError("RetuneController launch_gate requires a sweep_fn") + if isinstance(generation, bool) or not isinstance(generation, int): + raise ValueError("generation must be an integer") + if predecessor is not None and not isinstance(predecessor, Future): + raise ValueError("predecessor must be a concurrent.futures.Future") + self._sweep_fn = sweep_fn + self._launch_gate = launch_gate + self._harvest_allowed = harvest_allowed + self._generation = generation self._config = config self._retune = retune self._inputs = inputs @@ -156,6 +221,7 @@ def __init__( ) self._router_maps: dict[str, dict[str, dict[str, float]]] = {} router_domain = self._router_domain(retune) + self._router_domain_values = router_domain self._router_keys = tuple(sorted(router_domain)) self._router_searched = bool(router_domain) @@ -193,10 +259,17 @@ def __init__( self._startup_point = self.current_point() self._s0 = 0.0 + # Live: s0 is an epoch phase and decision ids carry the window end. + self._phase_anchored = False self._started = False self._last_k = -1 self._history: deque[tuple[float, dict[str, int]]] = deque() self._pending: _PendingSweep | None = None + # A dropped sweep that may still hold the single sweep thread, or + # the previous generation's sweep that a first launch waits for. + self._dropped: Future[SweepResult] | None = ( + predecessor if predecessor is not None and not predecessor.done() else None + ) self._cancel = threading.Event() self._executor = ThreadPoolExecutor( max_workers=1, thread_name_prefix="planner-retune" @@ -226,10 +299,23 @@ def __init__( # ------------------------------------------------------------------ API - def start(self, start_s: float) -> None: - """Anchor the window schedule at the planner's first tick time.""" + def start(self, start_s: float, phase_s: float | None = None) -> None: + """Anchor the window schedule. - self._s0 = float(start_s) + Replay anchors ``s0`` at the planner's first tick time ``start_s``. + A live host passes ``phase_s``: ``s0`` is then that epoch phase, so + every planner process and controller generation cuts the same + windows, and decision ids become ``retune-w``. + """ + + if phase_s is None: + self._s0 = float(start_s) + self._phase_anchored = False + else: + if not math.isfinite(phase_s): + raise ValueError("phase_s must be finite") + self._s0 = float(phase_s) + self._phase_anchored = True self._started = True self._last_k = -1 @@ -252,6 +338,8 @@ async def on_tick( pending = self._pending if pending is not None and pending.future.done(): + if pending.queued is not None: + self._settle_queue_delay(pending) if pending.ready_at_s is None: pending.decision_latency_s = self._decision_latency_s(pending.future) pending.ready_at_s = pending.launched_at_s + pending.decision_latency_s @@ -259,19 +347,33 @@ async def on_tick( # must land in the tick that launches the next window. if now_s + _EPSILON * self._retune.interval_seconds >= pending.ready_at_s: self._pending = None - decisions.append(self._harvest(pending, now_s)) + harvested = self._harvest_or_rewind(pending, now_s) + if harvested is None: + audit.append(f"retune:dropped w={pending.request.window_index}") + else: + decisions.append(harvested) due = self._due_window(now_s) if due > self._last_k: - self._last_k = due - if self._pending is not None: - decisions.append(self._skipped(due, now_s, "busy", "sweep in flight")) + # The gate runs before both branches: a window this process may + # not work on is neither launched nor recorded busy. + gate = self._gate(due) + if gate is not None and gate.reason in _GATE_RETRY_REASONS: + pass + elif gate is not None and gate.reason == "already_decided": + self._last_k = due else: - launched = await self._launch(due, now_s) - if launched is None: - audit.append(f"retune:launch w={due}") + self._last_k = due + if self._pending is not None: + decisions.append( + self._skipped(due, now_s, "busy", "sweep in flight") + ) else: - decisions.append(launched) + launched = await self._launch(due, now_s, gate) + if launched is None: + audit.append(f"retune:launch w={due}") + else: + decisions.append(launched) for decision in decisions: audit.append(f"retune:{decision.outcome} w={decision.window_index}") @@ -288,6 +390,101 @@ def close(self) -> None: self._cancel.set() self._executor.shutdown(wait=False, cancel_futures=True) + def drop_pending(self) -> None: + """Abandon the in-flight sweep without deciding its window. + + Sets the sweep's abort event, cancels it if it has not started, and + forgets it, so a following ``rewind`` takes effect and the window is + relaunched (or adopted by another leader) instead of recorded busy. + A sweep that is already running keeps the single sweep thread until + it sees the abort; a later launch queues behind it. + """ + + pending = self._pending + if pending is None: + return + self._pending = None + if pending.abort is not None: + pending.abort.set() + if not pending.future.cancel() and not pending.future.done(): + # Running, so any earlier dropped sweep has finished, or a + # predecessor generation's sweep is still running and this one + # waits for it: either way this one is now what a new launch + # would queue behind. + self._dropped = pending.future + logger.info( + "retune dropped the in-flight sweep of window %d", + pending.request.window_index, + ) + + def rewind(self, last_decided_k: int) -> None: + """Make every window after ``last_decided_k`` due again. + + A no-op while a sweep is pending (``drop_pending`` first): its + window is consumed until it is harvested or dropped. + """ + + if self._pending is None: + self._last_k = min(self._last_k, int(last_decided_k)) + + def restore_desired(self, flat: Mapping[str, float]) -> None: + """Re-install persisted desired router values (live re-election). + + ``flat`` maps live router knob names to values and must cover every + searched key; extra keys are ignored. The startup point is not + touched: it stays a candidate of every sweep. + """ + + try: + values = {key: flat[key] for key in self._router_keys} + except KeyError as error: + raise ValueError( + f"restore_desired needs every searched router key " + f"{list(self._router_keys)}; missing {error.args[0]!r}" + ) from None + # Validates finite, non-negative values and a resolvable config, so + # the per-tick desired map can never fail afterwards. + router = RetuneParams(router=values).router + if self._router_searched: + self._router_map(router) + self._desired_router = router + + def running_future(self) -> Future[SweepResult] | None: + """The sweep that holds or will hold the sweep thread, if any. + + The pending sweep while it runs, else a dropped (or predecessor + generation's) sweep that is still running, else a pending sweep + that has not started; ``None`` once all of them are done. A host + replacing this controller calls it after ``drop_pending`` and + ``close`` and passes it as the successor's ``predecessor``. + """ + + pending = self._pending.future if self._pending is not None else None + if pending is not None and pending.running(): + return pending + for future in (self._dropped, pending): + if future is not None and not future.done(): + return future + return None + + @property + def generation(self) -> int: + """The generation the host built this controller with.""" + + return self._generation + + @property + def router_keys(self) -> tuple[str, ...]: + """Searched router knobs, by live (PATCH) name, sorted.""" + + return self._router_keys + + @property + def router_domain(self) -> dict[str, list[float]]: + """The searched router domain: live knob name -> candidate values.""" + + return {key: list(values) for key, values in self._router_domain_values.items()} + def current_point(self) -> RetuneParams: """The point in effect: desired router values and live planner values. @@ -491,13 +688,140 @@ def _ms(self, time_s: float) -> float: return inputs.trace_origin_ms + time_s * 1000.0 * inputs.arrival_speedup_ratio def _decision_id(self, k: int) -> str: + if self._phase_anchored: + _, end_s, _ = self._window_times(k) + return f"retune-w{int(round(self._ms(end_s)))}" return f"retune-{int(self._s0)}-w{k}" - async def _launch(self, k: int, now_s: float) -> RetuneDecision | None: + def _gate(self, k: int) -> GateResult | None: + """Ask the live host whether window ``k`` may launch now. + + ``None`` without a gate (replay). A gate that raises or answers an + unknown reason holds the window like ``not_ready``: the window stays + due and the gate is asked again on the next tick. + """ + + if self._launch_gate is None: + return None + try: + gate = self._launch_gate(k, self._window_times(k)) + except Exception: + logger.exception("retune launch gate failed for window %d", k) + return GateResult(reason="not_ready") + if gate.reason is not None and gate.reason not in ( + *_GATE_RETRY_REASONS, + "already_decided", + ): + logger.warning( + "retune launch gate returned unknown reason %r for window %d; " + "holding the window", + gate.reason, + k, + ) + return GateResult(reason="not_ready") + return gate + + def _harvest_or_rewind( + self, pending: _PendingSweep, now_s: float + ) -> RetuneDecision | None: + """Turn a finished sweep into a decision, or abandon its window. + + The window is abandoned (no decision, ``_last_k`` rewound so it is + due again) when the host refuses the harvest, for instance because + this process lost leadership or rebuilt its controller, or when the + sweep reports a ``NON_DECISION_FAILURES`` reason. + """ + + k = pending.request.window_index + allowed = True + if self._harvest_allowed is not None: + try: + allowed = bool(self._harvest_allowed(pending.request.generation)) + except Exception: + logger.exception("retune harvest_allowed failed for window %d", k) + allowed = False + if not allowed: + logger.info("retune discards the sweep of window %d: harvest refused", k) + self.rewind(k - 1) + return None + result = self._sweep_result(pending) + if result.failure_reason in NON_DECISION_FAILURES: + logger.info( + "retune abandons window %d: sweep stopped (%s)", + k, + result.failure_reason, + ) + self.rewind(k - 1) + return None + return self._decide(pending, result, now_s) + + def _settle_queue_delay(self, pending: _PendingSweep) -> None: + """Move a queued sweep's launch time to when it actually started.""" + + assert pending.queued is not None + submitted, started = pending.queued + pending.queued = None + if not started: + return # never ran (cancelled at close); keep the submit time + delay = max(0.0, started[0] - submitted) + pending.launched_at_s += delay + if pending.ready_at_s is not None: + pending.ready_at_s += delay + + def _replicas_for_launch( + self, start_s: float, end_s: float, spec_replicas: Mapping[str, int] | None + ) -> tuple[dict[str, int] | None, bool, str | None]: + """``(replicas, verified, mismatch)`` for a window launch. + + Without ``spec_replicas`` (replay), the ready counts at window start, + verified. With them (live), every ready count recorded over the + window, including the one in effect at its start, must equal the + spec; a mismatch is reported. The spec replicas are used either way, + unverified when no recorded count covers the window start (the + window began before this controller started). + """ + + if spec_replicas is None: + return self._replicas_at(start_s), True, None + spec = {key: int(value) for key, value in spec_replicas.items()} + covering: dict[str, int] | None = None + observed: list[dict[str, int]] = [] + for at_s, counts in self._history: + if at_s <= start_s + _EPSILON: + covering = counts + elif at_s < end_s - _EPSILON: + observed.append(counts) + if covering is not None: + observed.insert(0, covering) + for counts in observed: + if counts != spec: + return spec, covering is not None, json.dumps(counts, sort_keys=True) + return spec, covering is not None, None + + async def _launch( + self, k: int, now_s: float, gate: GateResult | None = None + ) -> RetuneDecision | None: """Submit the sweep of window ``k``; a decision only if it cannot run.""" start_s, end_s, as_of_s = self._window_times(k) - replicas = self._replicas_at(start_s) + spec_replicas = gate.spec_replicas if gate is not None else None + replicas, verified, mismatch = self._replicas_for_launch( + start_s, end_s, spec_replicas + ) + if mismatch is not None: + return self._skipped( + k, + now_s, + "failed", + f"ready worker counts {mismatch} differ from the DGD spec " + f"replicas {json.dumps(spec_replicas, sort_keys=True)} during " + "the window", + failure_reason="replicas_changed", + details={ + "spec_replicas": dict(spec_replicas or {}), + "observed_replicas": json.loads(mismatch), + }, + ) if replicas is None: return self._skipped( k, now_s, "failed", "no ready worker counts recorded at window start" @@ -535,9 +859,12 @@ async def _launch(self, k: int, now_s: float) -> RetuneDecision | None: sla_itl_ms=self._sla_itl_ms, max_parallel=self._retune.max_parallel, timeout_s=self._retune.effective_timeout_seconds, + # 0 in replay, the field's default. + generation=self._generation, + **self._live_request_fields(as_of_s, spec_replicas, verified), ) - # Looked up at call time so tests can substitute the sweep function. - future = self._executor.submit(run_sweep, request, cancel=self._cancel) + abort = gate.abort if gate is not None else None + future, queued = self._submit(request, abort, gated=gate is not None) if isinstance(self._clock, VirtualClock): # Virtual time is frozen while the sweep runs; wait for it here. await asyncio.wait([asyncio.wrap_future(future)]) @@ -555,17 +882,80 @@ async def _launch(self, k: int, now_s: float) -> RetuneDecision | None: launched_at_s=now_s, decision_latency_s=latency, ready_at_s=None if latency is None else now_s + latency, + abort=abort, + queued=queued, ) logger.info( - "retune launched window %d [%s, %s) with %d candidates, replicas %s", + "retune launched window %d [%s, %s) with %d candidates, replicas %s%s", k, start_s, end_s, len(candidates), replicas, + "" + if verified + else " (unverified: the window began before this controller)", ) return None + def _live_request_fields( + self, + as_of_s: float, + spec_replicas: Mapping[str, int] | None, + verified: bool, + ) -> dict[str, Any]: + """The ``SweepRequest`` fields only a gated (live) launch sets.""" + + if self._launch_gate is None: + return {} + return { + "router_domain": { + key: tuple(float(value) for value in values) + for key, values in self._router_domain_values.items() + }, + "deadline_s": as_of_s + self._retune.effective_timeout_seconds, + "spec_replicas": None if spec_replicas is None else dict(spec_replicas), + "replicas_verified": verified, + } + + def _submit( + self, + request: SweepRequest, + abort: threading.Event | None, + *, + gated: bool, + ) -> tuple[Future[SweepResult], tuple[float, list[float]] | None]: + """Submit the sweep to the single sweep thread. + + ``run_sweep`` is looked up at call time so tests can substitute it. + A gated launch also passes the gate's abort event. Behind a dropped + sweep that still holds the thread, or a predecessor generation's + sweep that still runs on its own controller's thread, the call waits + for it and records when it really starts (``_settle_queue_delay``). + """ + + sweep_fn: SweepFn = self._sweep_fn if self._sweep_fn is not None else run_sweep + kwargs: dict[str, Any] = {"cancel": self._cancel} + if gated: + kwargs["abort"] = abort + if self._dropped is not None and self._dropped.done(): + self._dropped = None + if self._dropped is None: + return self._executor.submit(sweep_fn, request, **kwargs), None + clock = self._clock + blocker = self._dropped + started: list[float] = [] + + def queued_sweep() -> SweepResult: + # A dropped sweep of this controller has already finished (one + # sweep thread); a predecessor generation's may still be running. + wait_futures((blocker,)) + started.append(clock.monotonic()) + return sweep_fn(request, **kwargs) + + future = self._executor.submit(queued_sweep) + return future, (clock.monotonic(), started) + def _decision_latency_s(self, future: Future[SweepResult]) -> float: """Delay from launch to decision; ``measured`` needs a done sweep.""" latency = self._retune.decision_latency_seconds @@ -581,15 +971,16 @@ def _decision_latency_s(self, future: Future[SweepResult]) -> float: ) return max(latency, self._scale_interval_s) - def _harvest(self, pending: _PendingSweep, now_s: float) -> RetuneDecision: + @staticmethod + def _sweep_result(pending: _PendingSweep) -> SweepResult: + """The finished sweep's result; a raised exception is ``failed``.""" + error = pending.future.exception() if error is not None: - result = SweepResult( + return SweepResult( status="failed", error=f"{type(error).__name__}: {error}" ) - else: - result = pending.future.result() - return self._decide(pending, result, now_s) + return pending.future.result() def _decide( self, pending: _PendingSweep, result: SweepResult, now_s: float @@ -621,6 +1012,8 @@ def record(outcome: RetuneOutcome, **overrides: Any) -> RetuneDecision: "predictor_presweep": result.predictor_presweep, "sweep_wall_s": result.wall_s, "decision_latency_s": pending.decision_latency_s, + "failure_reason": result.failure_reason, + "details": result.details, } values.update(overrides) return RetuneDecision(**values) @@ -753,7 +1146,14 @@ def _improves(self, score: float, baseline_score: float) -> bool: return gain > margin def _skipped( - self, k: int, now_s: float, outcome: RetuneOutcome, reason: str + self, + k: int, + now_s: float, + outcome: RetuneOutcome, + reason: str, + *, + failure_reason: str | None = None, + details: Mapping[str, Any] | None = None, ) -> RetuneDecision: start_s, end_s, as_of_s = self._window_times(k) current = self.current_point() @@ -773,6 +1173,8 @@ def _skipped( baseline_score=None, best_score=None, reason=reason, + failure_reason=failure_reason, + details=dict(details or {}), ) # ------------------------------------------------------- router helpers @@ -814,7 +1216,7 @@ def _config_value(self, key: str) -> Any: def _log_decision(decision: RetuneDecision) -> None: logger.info( "retune decision %s: %s (baseline=%s best=%s requests=%d " - "chosen=%s planner_changes=%s reason=%s)", + "chosen=%s planner_changes=%s reason=%s failure_reason=%s)", decision.decision_id, decision.outcome, decision.baseline_score, @@ -827,6 +1229,7 @@ def _log_decision(decision: RetuneDecision) -> None: ), decision.planner_changes, decision.reason, + decision.failure_reason, ) diff --git a/components/src/dynamo/planner/retune/live/__init__.py b/components/src/dynamo/planner/retune/live/__init__.py new file mode 100644 index 000000000000..452ce8294de1 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/__init__.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live retune: run the retune sweep through DGDR v1beta2 and apply the winner. + +The replay retune loop (``RetuneController`` and ``_decide``) is unchanged; +this package supplies its live pieces: the DGDR sweep executor, the +supervisor that owns leadership and persisted state, the router actuator, +the reconciler and the trace window cutter. ``contracts`` holds the types +they share. +""" diff --git a/components/src/dynamo/planner/retune/live/actuator.py b/components/src/dynamo/planner/retune/live/actuator.py new file mode 100644 index 000000000000..3f2282beecfe --- /dev/null +++ b/components/src/dynamo/planner/retune/live/actuator.py @@ -0,0 +1,868 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Apply the retune router parameters to every frontend router (design §4.8). + +``RouterActuator.reconcile`` runs one round: list the DGD's frontend pods, +``GET /router_policy`` on each, keep the routers of the DGD's dynamo +namespace, latch each router's initial configuration, check it against the +parsed startup maps (P14), write what the target needs and report whether +every router has converged. + +Wire format (``lib/llm/src/http/service/router_policy.rs``):: + + GET /router_policy -> {"routers": [{router_id, model, namespace, + component, endpoint, worker_type, + configuration: {revision, parameters} | null}]} + PATCH /router_policy {router_id, expected_revision, parameters: {...}} + {router_id, expected_revision, reset: true} + -> one router object as above + +A PATCH replaces only the keys it names; ``reset: true`` restores every +parameter to the router's startup values. Both bump the router's revision +(``lib/kv-router/src/plugins/worker_selection/runtime.rs``), and GET never +returns the startup values, so the actuator latches them itself: the +configuration of a router seen at revision 0, or returned by a reset, is that +router's initial. + +Modes (``ActuationTarget.mode``): + +- ``apply``: PATCH explicit values for every one of the 8 policy parameters + that differs from ``desired_by_wt[worker_type]``. A router whose initial is + unknown (revision > 0, never latched) is reset once first to learn it; a + router whose initial differs from the startup map (P14) is never written, + so planner-parsed values cannot leak into keys the sweep did not search. +- ``rollback``: reset only routers whose configuration differs from their + latched initial. Never by revision: a reset itself bumps the revision. +- ``observe``: no writes; latch initials and check parity only. + +``ActuationTarget.learn_unknown=False`` (advisory) never resets a router to +learn its initial: such a router is left alone, and rollback does not wait +for it. + +Targeted pods: Ready frontends (plus NotReady ones with ``include_not_ready``) +that have an IP, are not terminating and are not in phase Succeeded/Failed +(an evicted pod can keep its IP but serves nothing). + +Convergence: at least ``max(1, expected_frontends)`` Ready frontends, every +Ready frontend answered GET and has a router with a runtime configuration +for every worker type of the planner mode, and every router seen has its 8 +parameters equal to the target (``apply``: the desired map; +``rollback``/``observe``: the router's latched initial). A NotReady pod +that answers GET is checked like the others; one that does not answer is +ignored (a restarted frontend comes back at revision 0, its initial). No +routers at all is not converged. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Optional + +import aiohttp + +from dynamo.planner.retune.live.contracts import ( + POD_LABEL_COMPONENT_TYPE, + POD_LABEL_DGD_NAME, + POLICY_PARAMETER_KEYS, + ActuationReport, + ActuationTarget, + DgdRef, + PodInfo, + PolicyCallResult, + RetuneKube, + RetuneMetricsSink, + RouterInfo, + RouterPolicyClient, +) + +logger = logging.getLogger(__name__) + +ROUTER_POLICY_PATH = "/router_policy" +# Container port of the frontend when the pod has no port named ``http`` +# (deploy/operator/internal/consts/consts.go DynamoServicePort). +DEFAULT_FRONTEND_PORT = 8000 +# Consecutive rounds of GET 404 on a frontend after which its admin API is +# reported disabled (P12). +ADMIN_API_DISABLED_ROUNDS = 3 +# Largest error text kept from a failed call. +_MAX_ERROR_CHARS = 512 +# Relative/absolute tolerance of parameter comparisons. Values round-trip +# exactly through serde_json and Python floats; the tolerance only absorbs +# formatting noise from hand-written targets. +_REL_TOL = 1e-9 +_ABS_TOL = 1e-12 +# Statuses after which the pod is re-read once and the write retried. +_RETRY_AFTER_REGET = frozenset({404, 409, 503}) +# The frontend bounds each router's control call by CONTROL_TIMEOUT +# (lib/llm/src/http/service/router_policy.rs): GET then answers 200 without +# the routers that timed out, and PATCH answers 503. The client's total +# timeout must exceed it, or the client gives up first and turns those +# answers into status 0 (a whole pod lost for one stuck router). +ROUTER_CONTROL_TIMEOUT_S = 5.0 +DEFAULT_CLIENT_TIMEOUT_S = ROUTER_CONTROL_TIMEOUT_S + 3.0 +# Pod phases after which no container runs again. +_FINISHED_POD_PHASES = frozenset({"Succeeded", "Failed"}) + + +# ----------------------------------------------------------------- client + + +class AiohttpRouterPolicyClient: + """``RouterPolicyClient`` over a dedicated ``aiohttp.ClientSession``. + + The session is created on first use inside the running event loop and + closed by ``close()``. Every call returns a ``PolicyCallResult`` and never + raises for HTTP, network or decoding failures: ``status`` is the HTTP + status, or 0 when no well-formed response arrived (connection error, + timeout, a pod without an IP, or a body that is not the documented JSON). + ``timeout_s`` bounds each call and must stay above the server's + ``ROUTER_CONTROL_TIMEOUT_S``. + """ + + def __init__( + self, + timeout_s: float = DEFAULT_CLIENT_TIMEOUT_S, + scheme: str = "http", + *, + default_port: int = DEFAULT_FRONTEND_PORT, + max_connections: int = 32, + ) -> None: + if not (math.isfinite(timeout_s) and timeout_s > 0): + raise ValueError("timeout_s must be positive and finite") + if scheme not in ("http", "https"): + raise ValueError(f"scheme must be 'http' or 'https', got {scheme!r}") + self._timeout = aiohttp.ClientTimeout(total=float(timeout_s)) + self._scheme = scheme + self._default_port = int(default_port) + self._max_connections = int(max_connections) + self._session: Optional[aiohttp.ClientSession] = None + + def url(self, pod: PodInfo) -> Optional[str]: + """The pod's ``/router_policy`` URL, or None without a pod IP.""" + + if not pod.ip: + return None + host = f"[{pod.ip}]" if ":" in pod.ip else pod.ip + port = pod.http_port if pod.http_port is not None else self._default_port + return f"{self._scheme}://{host}:{port}{ROUTER_POLICY_PATH}" + + async def get(self, pod: PodInfo) -> PolicyCallResult: + return await self._call(pod, "GET", None, _parse_list) + + async def patch( + self, + pod: PodInfo, + router_id: str, + expected_revision: int, + parameters: Mapping[str, float], + ) -> PolicyCallResult: + if not parameters: + # The server rejects an empty patch with 400; do not send one. + return PolicyCallResult(status=0, error="empty parameters") + body = { + "router_id": router_id, + "expected_revision": int(expected_revision), + "parameters": {key: float(value) for key, value in parameters.items()}, + } + return await self._call(pod, "PATCH", body, _parse_single) + + async def reset( + self, pod: PodInfo, router_id: str, expected_revision: int + ) -> PolicyCallResult: + body = { + "router_id": router_id, + "expected_revision": int(expected_revision), + "reset": True, + } + return await self._call(pod, "PATCH", body, _parse_single) + + async def close(self) -> None: + session, self._session = self._session, None + if session is not None and not session.closed: + await session.close() + + async def _call( + self, + pod: PodInfo, + method: str, + body: Optional[dict[str, Any]], + parse: Any, + ) -> PolicyCallResult: + url = self.url(pod) + if url is None: + return PolicyCallResult(status=0, error=f"pod {pod.name} has no IP") + try: + session = self._ensure_session() + async with session.request(method, url, json=body) as response: + raw = await response.read() + status = response.status + except asyncio.CancelledError: + raise + except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as exc: + return PolicyCallResult( + status=0, error=_truncate(f"{method} {url}: {exc!r}") + ) + if status != 200: + return PolicyCallResult(status=status, error=_error_text(raw)) + try: + return parse(json.loads(raw)) + except (ValueError, TypeError, KeyError) as exc: + return PolicyCallResult( + status=0, error=_truncate(f"malformed {method} response: {exc}") + ) + + def _ensure_session(self) -> aiohttp.ClientSession: + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession( + timeout=self._timeout, + connector=aiohttp.TCPConnector(limit=self._max_connections), + trust_env=False, + ) + return self._session + + +def _parse_router(item: Mapping[str, Any]) -> RouterInfo: + configuration = item.get("configuration") + parameters: Optional[dict[str, float]] = None + revision = 0 + if configuration is not None: + revision = int(configuration["revision"]) + parameters = { + str(key): float(value) + for key, value in dict(configuration["parameters"]).items() + if isinstance(value, (int, float)) and not isinstance(value, bool) + } + model = item.get("model") + return RouterInfo( + router_id=str(item["router_id"]), + namespace=str(item["namespace"]), + worker_type=str(item["worker_type"]), + revision=revision, + configuration=parameters, + model=None if model is None else str(model), + ) + + +def _parse_list(payload: Any) -> PolicyCallResult: + routers = payload["routers"] + if not isinstance(routers, list): + raise TypeError("routers is not a list") + return PolicyCallResult( + status=200, routers=tuple(_parse_router(item) for item in routers) + ) + + +def _parse_single(payload: Any) -> PolicyCallResult: + return PolicyCallResult(status=200, router=_parse_router(payload)) + + +def _error_text(raw: bytes) -> str: + text = raw.decode("utf-8", errors="replace") + try: + payload = json.loads(text) + except ValueError: + return _truncate(text) + if isinstance(payload, dict) and isinstance(payload.get("error"), str): + return _truncate(payload["error"]) + return _truncate(text) + + +def _truncate(text: str) -> str: + return text if len(text) <= _MAX_ERROR_CHARS else text[:_MAX_ERROR_CHARS] + "..." + + +# --------------------------------------------------------------- actuator + + +@dataclass +class _Round: + """Mutable bookkeeping of one ``reconcile`` call.""" + + target: ActuationTarget + initials: Mapping[str, Mapping[str, float]] + errors: dict[str, int] = field(default_factory=dict) + + def error(self, kind: str) -> None: + self.errors[kind] = self.errors.get(kind, 0) + 1 + + +@dataclass +class _PodOutcome: + pod: PodInfo + ok: bool # the last GET of this pod succeeded + get_status: Optional[int] = None + routers: tuple[RouterInfo, ...] = () # in-scope routers, final state + + +class RouterActuator: + """Converges the frontends' router policies to an ``ActuationTarget``. + + ``startup_maps`` is the parsed startup ``policy_parameters(wt)`` per + worker type of the planner mode (``aggregated``, or ``prefill`` and + ``decode``); its keys are the worker types the actuator manages. The + actuator keeps a local cache of the initials it latched, so a write in + the same round, or a state write that has not landed yet, never makes a + router look like it has an unknown initial. ``reconcile`` never raises + (except ``asyncio.CancelledError``). + """ + + def __init__( + self, + kube: RetuneKube, + client: RouterPolicyClient, + dgd: DgdRef, + startup_maps: dict[str, dict[str, float]], + metrics: RetuneMetricsSink, + max_concurrency: int = 8, + *, + list_timeout_s: float = 10.0, + admin_disabled_rounds: int = ADMIN_API_DISABLED_ROUNDS, + ) -> None: + if max_concurrency < 1: + raise ValueError("max_concurrency must be >= 1") + if admin_disabled_rounds < 1: + raise ValueError("admin_disabled_rounds must be >= 1") + self._kube = kube + self._client = client + self._dgd = dgd + self._metrics = metrics + self._max_concurrency = int(max_concurrency) + self._list_timeout_s = float(list_timeout_s) + self._admin_disabled_rounds = int(admin_disabled_rounds) + # router_id -> initial configuration latched by this process. + self._latched: dict[str, dict[str, float]] = {} + # router_id -> name of the pod it was last seen on. + self._router_pod: dict[str, str] = {} + # router_id -> signature of a write the router rejected with 422; + # the same write is not retried until the target changes. + self._rejected: dict[str, str] = {} + # pod uid -> consecutive rounds whose GET returned 404. + self._get_404_rounds: dict[str, int] = {} + self._last_pods: Optional[tuple[PodInfo, ...]] = None + self._startup_maps: dict[str, dict[str, float]] = {} + self.set_startup_maps(startup_maps) + + # ------------------------------------------------------------------ API + + @property + def label_selector(self) -> str: + return ( + f"{POD_LABEL_DGD_NAME}={self._dgd.name}," + f"{POD_LABEL_COMPONENT_TYPE}=frontend" + ) + + @property + def worker_types(self) -> tuple[str, ...]: + return tuple(self._startup_maps) + + @property + def last_pods(self) -> Optional[tuple[PodInfo, ...]]: + """Frontend pods of the last round's listing; None if it failed.""" + + return self._last_pods + + def set_startup_maps(self, startup_maps: Mapping[str, Mapping[str, float]]) -> None: + """Replace the parsed startup maps (after a frontend spec change).""" + + if not startup_maps: + raise ValueError("startup_maps must name at least one worker type") + self._startup_maps = { + worker_type: {key: float(value) for key, value in parameters.items()} + for worker_type, parameters in startup_maps.items() + } + self._rejected.clear() + + async def close(self) -> None: + """Close the policy client when it has a ``close`` coroutine (the + ``RouterPolicyClient`` protocol does not require one).""" + + close = getattr(self._client, "close", None) + if close is not None: + await close() + + async def reconcile( + self, + target: ActuationTarget, + *, + initials: Mapping[str, Mapping[str, float]], + expected_frontends: int, + ) -> ActuationReport: + """Run one round; see the module docstring. Never raises.""" + + try: + report = await self._reconcile(target, initials, expected_frontends) + except asyncio.CancelledError: + raise + except Exception: # never raise: the supervisor retries next round + logger.exception("retune router actuator round failed") + report = ActuationReport( + converged=False, + seen_router_ids=frozenset(initials) | frozenset(self._latched), + errors={"internal": 1}, + ) + self._metrics.router_converged(report.converged) + return report + + # ------------------------------------------------------------- the round + + async def _reconcile( + self, + target: ActuationTarget, + initials: Mapping[str, Mapping[str, float]], + expected_frontends: int, + ) -> ActuationReport: + state = _Round(target=target, initials=initials) + target_error = self._target_error(target) + if target_error is not None: + logger.error("retune router actuator: invalid target: %s", target_error) + state.error("invalid_target") + # Latch and check parity without writing; never report converged. + state.target = ActuationTarget(mode="observe") + + try: + pods = await asyncio.to_thread( + self._kube.list_pods, + self.label_selector, + timeout_s=self._list_timeout_s, + ) + except Exception as exc: # KubeError or a client bug: retry next round + logger.warning( + "retune router actuator: listing frontend pods failed: %s", exc + ) + self._last_pods = None + state.error("list_pods") + return ActuationReport( + converged=False, + seen_router_ids=frozenset(initials) | frozenset(self._latched), + errors=dict(state.errors), + ) + self._last_pods = tuple(pods) + + live = [pod for pod in pods if _serving(pod)] + ready = [pod for pod in live if pod.ready and pod.ip] + targeted = [ + pod + for pod in live + if pod.ip and (pod.ready or state.target.include_not_ready) + ] + + semaphore = asyncio.Semaphore(self._max_concurrency) + + async def run(pod: PodInfo) -> _PodOutcome: + async with semaphore: + return await self._process_pod(pod, state) + + outcomes = list(await asyncio.gather(*(run(pod) for pod in targeted))) + + self._track_admin_api(pods, outcomes) + converged = target_error is None and self._converged( + state, ready, outcomes, max(1, int(expected_frontends)) + ) + routers_present = self._routers_present( + ready, outcomes, max(1, int(expected_frontends)) + ) + parity_ok, mismatches = self._parity(state, outcomes) + seen = self._seen_router_ids(state, pods, outcomes) + self._prune(seen) + new_initials = { + router_id: dict(configuration) + for router_id, configuration in self._latched.items() + if router_id in seen + and not _same_map(initials.get(router_id), configuration) + } + admin_api_disabled = any( + count >= self._admin_disabled_rounds + for count in self._get_404_rounds.values() + ) + if mismatches: + logger.warning( + "retune router actuator: P14 parity mismatch: %s", "; ".join(mismatches) + ) + return ActuationReport( + converged=converged, + new_initials=new_initials, + seen_router_ids=frozenset(seen), + parity_ok=parity_ok, + parity_mismatches=tuple(mismatches), + admin_api_disabled=admin_api_disabled, + errors=dict(state.errors), + routers_present=routers_present, + ) + + def _routers_present( + self, + ready: Sequence[PodInfo], + outcomes: Sequence[_PodOutcome], + required_ready: int, + ) -> bool: + """P12: enough Ready frontends, each GET answered with a configured + router for every worker type of the planner mode.""" + + if len(ready) < required_ready: + return False + ready_names = {pod.name for pod in ready} + answered = 0 + for outcome in outcomes: + if outcome.pod.name not in ready_names: + continue + if not outcome.ok: + return False + covered = { + router.worker_type + for router in outcome.routers + if router.configuration is not None + } + if covered != set(self._startup_maps): + return False + answered += 1 + return answered >= required_ready + + def _target_error(self, target: ActuationTarget) -> Optional[str]: + if target.mode not in ("apply", "rollback", "observe"): + return f"unknown mode {target.mode!r}" + if target.mode != "apply": + return None + desired = target.desired_by_wt + if desired is None: + return "apply needs desired_by_wt" + for worker_type in self._startup_maps: + parameters = desired.get(worker_type) + if parameters is None: + return f"desired_by_wt has no {worker_type!r} map" + for key in POLICY_PARAMETER_KEYS: + value = parameters.get(key) + if not _valid_parameter(value): + return f"desired_by_wt[{worker_type!r}][{key!r}] is {value!r}" + return None + + # ------------------------------------------------------------- per pod + + async def _process_pod(self, pod: PodInfo, state: _Round) -> _PodOutcome: + result = await self._client.get(pod) + if result.status != 200: + state.error(f"get:{_code(result.status)}") + logger.debug( + "retune router actuator: GET %s -> %s %s", + pod.name, + result.status, + result.error, + ) + return _PodOutcome(pod=pod, ok=False, get_status=result.status) + routers = self._in_scope(pod, result.routers, state) + retried = False + while True: + final: list[RouterInfo] = [] + retry = False + for router in routers: + current, needs_reget = await self._drive(pod, router, state) + final.append(current) + retry = retry or needs_reget + if not retry or retried: + return _PodOutcome( + pod=pod, ok=True, get_status=200, routers=tuple(final) + ) + # 409, 503 or PATCH 404: re-read the pod once and retry. + retried = True + result = await self._client.get(pod) + if result.status != 200: + state.error(f"get:{_code(result.status)}") + return _PodOutcome(pod=pod, ok=False, get_status=result.status) + routers = self._in_scope(pod, result.routers, state) + + def _in_scope( + self, pod: PodInfo, routers: Sequence[RouterInfo], state: _Round + ) -> list[RouterInfo]: + """Routers of the DGD's dynamo namespace and the planner's worker + types; latch the initials of routers at revision 0.""" + + scoped = [] + for router in routers: + if router.namespace != self._dgd.dynamo_namespace: + continue + if router.worker_type not in self._startup_maps: + continue + self._router_pod[router.router_id] = pod.name + if router.configuration is None: + scoped.append(router) + continue + if router.revision == 0 and self._initial(router.router_id, state) is None: + self._latch(router.router_id, router.configuration) + scoped.append(router) + return scoped + + def _initial(self, router_id: str, state: _Round) -> Optional[dict[str, float]]: + latched = self._latched.get(router_id) + if latched is not None: + return latched + persisted = state.initials.get(router_id) + return None if persisted is None else dict(persisted) + + def _latch(self, router_id: str, configuration: Mapping[str, float]) -> None: + self._latched[router_id] = { + key: float(value) for key, value in configuration.items() + } + + async def _drive( + self, pod: PodInfo, router: RouterInfo, state: _Round + ) -> tuple[RouterInfo, bool]: + """Write what ``router`` needs; return its final state and whether + the pod should be re-read and the router retried.""" + + current = router + # At most a learning reset followed by one PATCH. + for _ in range(2): + action = self._plan(current, state) + if action is None: + return current, False + kind, parameters = action + signature = self._signature(kind, current.worker_type, state) + if self._rejected.get(current.router_id) == signature: + return current, False + if kind == "reset": + result = await self._client.reset( + pod, current.router_id, current.revision + ) + else: + result = await self._client.patch( + pod, current.router_id, current.revision, parameters + ) + if result.status != 200 or result.router is None: + code = _code(result.status) + state.error(f"{kind}:{code}") + self._metrics.patch_error(code) + logger.warning( + "retune router actuator: %s %s router %s rev %d -> %s %s", + kind, + pod.name, + current.router_id, + current.revision, + result.status, + result.error, + ) + if result.status == 422: + self._rejected[current.router_id] = signature + return current, result.status in _RETRY_AFTER_REGET + self._rejected.pop(current.router_id, None) + current = result.router + if kind == "reset" and current.configuration is not None: + # A reset restores every parameter to the router's startup + # values: that response is the router's initial. + self._latch(current.router_id, current.configuration) + return current, False + + def _plan( + self, router: RouterInfo, state: _Round + ) -> Optional[tuple[str, dict[str, float]]]: + """``("reset", {})``, ``("patch", diff)`` or None for one router.""" + + mode = state.target.mode + configuration = router.configuration + if mode == "observe" or configuration is None: + return None + initial = self._initial(router.router_id, state) + if initial is None and not state.target.learn_unknown: + # Never latched, and learning by reset is off (advisory): the + # planner never wrote it, so it is not the planner's to touch. + return None + if mode == "rollback": + if initial is None or not _same_map(configuration, initial): + return ("reset", {}) + return None + # apply + if initial is None: + # Revision > 0 and never latched: learn the initial first. + return ("reset", {}) + if not _same_map(initial, self._startup_maps[router.worker_type]): + # P14 failed for this router; writing explicit values would put + # planner-parsed values into keys the sweep never searched. + return None + desired = state.target.desired_by_wt[router.worker_type] # type: ignore[index] + diff = { + key: float(desired[key]) + for key in POLICY_PARAMETER_KEYS + if not _close(configuration.get(key), desired[key]) + } + return ("patch", diff) if diff else None + + @staticmethod + def _signature(kind: str, worker_type: str, state: _Round) -> str: + """Identity of a write for the 422 memory: mode, kind and target.""" + + desired = (state.target.desired_by_wt or {}).get(worker_type) + return json.dumps( + [state.target.mode, kind, desired], sort_keys=True, default=float + ) + + # ------------------------------------------------------------ verdicts + + def _left_alone(self, router: RouterInfo, state: _Round) -> bool: + """A rollback without learning skips routers with no known initial.""" + + return ( + state.target.mode == "rollback" + and not state.target.learn_unknown + and self._initial(router.router_id, state) is None + ) + + def _target_map( + self, router: RouterInfo, state: _Round + ) -> Optional[Mapping[str, float]]: + if state.target.mode == "apply": + return state.target.desired_by_wt[router.worker_type] # type: ignore[index] + return self._initial(router.router_id, state) + + def _converged( + self, + state: _Round, + ready: Sequence[PodInfo], + outcomes: Sequence[_PodOutcome], + required_ready: int, + ) -> bool: + if len(ready) < required_ready: + return False + configured = 0 + for outcome in outcomes: + if not outcome.ok: + if outcome.pod.ready: + return False + # An unreachable NotReady pod (crash loop, still starting) is + # ignored: it comes back at revision 0, i.e. at its initial. + continue + covered: set[str] = set() + for router in outcome.routers: + if router.configuration is None: + continue + covered.add(router.worker_type) + configured += 1 + if self._left_alone(router, state): + continue + expected = self._target_map(router, state) + if expected is None or not _same_map(router.configuration, expected): + return False + if outcome.pod.ready and covered != set(self._startup_maps): + return False + return configured > 0 + + def _parity( + self, state: _Round, outcomes: Sequence[_PodOutcome] + ) -> tuple[Optional[bool], list[str]]: + mismatches: list[str] = [] + known = 0 + unknown = 0 + for outcome in outcomes: + for router in outcome.routers: + if router.configuration is None: + continue + initial = self._initial(router.router_id, state) + if initial is None: + unknown += 1 + continue + known += 1 + startup = self._startup_maps[router.worker_type] + differing = [ + f"{key} initial={initial.get(key)} startup={startup.get(key)}" + for key in POLICY_PARAMETER_KEYS + if not _close(initial.get(key), startup.get(key)) + ] + if differing: + mismatches.append( + f"{outcome.pod.name}/{router.router_id}/{router.worker_type}: " + + ", ".join(differing) + ) + if mismatches: + return False, mismatches + if known == 0 or unknown: + return None, mismatches + return True, mismatches + + def _seen_router_ids( + self, + state: _Round, + pods: Sequence[PodInfo], + outcomes: Sequence[_PodOutcome], + ) -> set[str]: + """Routers observed this round plus known ones not provably gone. + + A known router is gone when its pod no longer exists (or has + finished: Succeeded/Failed) or answered GET without it; with an + unknown pod, when every live frontend with an IP answered GET this + round. + """ + + observed = { + router.router_id for outcome in outcomes for router in outcome.routers + } + existing = {pod.name for pod in pods if pod.phase not in _FINISHED_POD_PHASES} + answered = {outcome.pod.name for outcome in outcomes if outcome.ok} + complete = all(pod.name in answered for pod in pods if _serving(pod) and pod.ip) + seen = set(observed) + for router_id in set(state.initials) | set(self._latched): + if router_id in observed: + continue + pod_name = self._router_pod.get(router_id) + if pod_name is None: + gone = complete + else: + gone = pod_name not in existing or pod_name in answered + if not gone: + seen.add(router_id) + return seen + + def _prune(self, seen: set[str]) -> None: + for cache in (self._latched, self._router_pod, self._rejected): + for router_id in [key for key in cache if key not in seen]: + del cache[router_id] + + def _track_admin_api( + self, pods: Sequence[PodInfo], outcomes: Sequence[_PodOutcome] + ) -> None: + live_uids = {pod.uid for pod in pods} + for uid in [uid for uid in self._get_404_rounds if uid not in live_uids]: + del self._get_404_rounds[uid] + for outcome in outcomes: + uid = outcome.pod.uid + if outcome.get_status == 404: + self._get_404_rounds[uid] = self._get_404_rounds.get(uid, 0) + 1 + elif outcome.get_status is not None: + self._get_404_rounds.pop(uid, None) + + +# ---------------------------------------------------------------- helpers + + +def _serving(pod: PodInfo) -> bool: + """Not terminating and not finished: its frontend may still answer.""" + + return not pod.terminating and pod.phase not in _FINISHED_POD_PHASES + + +def _valid_parameter(value: Any) -> bool: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + return math.isfinite(value) and value >= 0 + + +def _code(status: int) -> str: + return str(status) if status else "network" + + +def _close(left: Optional[float], right: Optional[float]) -> bool: + if left is None or right is None: + return False + return math.isclose(float(left), float(right), rel_tol=_REL_TOL, abs_tol=_ABS_TOL) + + +def _same_map( + left: Optional[Mapping[str, float]], right: Optional[Mapping[str, float]] +) -> bool: + """Equal on the 8 policy parameters (a missing key is never equal).""" + + if left is None or right is None: + return False + return all(_close(left.get(key), right.get(key)) for key in POLICY_PARAMETER_KEYS) diff --git a/components/src/dynamo/planner/retune/live/bootstrap.py b/components/src/dynamo/planner/retune/live/bootstrap.py new file mode 100644 index 000000000000..6231f759146b --- /dev/null +++ b/components/src/dynamo/planner/retune/live/bootstrap.py @@ -0,0 +1,638 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build the live retune supervisor of a Kubernetes planner (design §4.2, §4.3). + +``build_live_retune(host)`` runs in ``NativePlannerBase._async_init`` after +``environment.initialize()`` and before the engine is built. It reads the +planner's DGD, parses the frontend's startup router configuration with a +pure function of the DGD spec, evaluates the spec premises, and returns a +``LiveRetuneSupervisor`` whose ``retune_inputs()`` the engine's first +controller is built from (``None`` while P4 fails: the adapter then starts +with retune disabled) and whose ``hooks()`` every controller gets. + +Every collaborator (Kubernetes client, router actuator, reconciler, DGDR +executor, window cutter) is imported on use, so planners without live retune +never load them. ``build_supervisor`` takes them injectable for tests. + +``build_legacy_cleanup(host)`` is the §4.11 clean-up for a planner whose +retune was removed: if a previous live retune left its Leases behind, it +returns a supervisor in decommission mode that rolls back what that retune +deployed, then deletes its Requests and Leases. ``NativePlannerBase`` +builds it in ``_async_init`` for a Kubernetes planner without retune config, +starts it after the engine and stops it in ``_shutdown_runtime``; a failure +to build it never fails startup. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import posixpath +import socket +import uuid +from collections.abc import Callable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Optional + +from dynamo.planner.retune.live.contracts import ( + DgdRef, + FrontendStartup, + KubeAlreadyExists, + KubeError, + KubeInvalid, + PremiseReport, + RetuneKube, + RetuneMetricsSink, + WindowCutter, +) +from dynamo.planner.retune.live.state import ( + fingerprint, + leadership_lease_name, + state_lease_name, +) +from dynamo.planner.retune.live.supervisor import ( + ActuatorFactory, + LiveRetuneSupervisor, + SpecEvaluation, + SupervisorSettings, + worker_types_for, +) + +if TYPE_CHECKING: + from dynamo.planner.config.planner_config import PlannerConfig + from dynamo.planner.core.types import RetuneInputs + +logger = logging.getLogger(__name__) + +# Shard id and window of the P11 canary Request (dry run only, never stored). +CANARY_SHARD_ID = "canary" +CANARY_WINDOW_END_MS = 0 + + +# ------------------------------------------------------------- resolver + + +class KvRouterResolver: + """``RouterResolver`` over the parsed startup ``KvRouterConfig`` JSON. + + Resolves exactly as the retune controller does: the startup value of a + searched key is its serde value, or its aggregated resolution when the + startup leaves it unset; a point's per worker type parameters are + ``policy_parameters(wt)`` of ``startup ⊕ point``. + """ + + def __init__( + self, router_config: Mapping[str, Any], worker_types: Sequence[str] + ) -> None: + self._router_config = dict(router_config) + self._worker_types = tuple(worker_types) + + def startup_flat(self, keys: Sequence[str]) -> dict[str, float]: + from dynamo.llm import KvRouterConfig + + startup = KvRouterConfig.from_json(json.dumps(self._router_config)) + serialized = json.loads(startup.to_json()) + resolved = startup.policy_parameters("aggregated") + values: dict[str, float] = {} + for key in keys: + value = serialized.get(key) + values[key] = float(resolved[key] if value is None else value) + return values + + def by_wt(self, flat: Mapping[str, float]) -> dict[str, dict[str, float]]: + from dynamo.llm import KvRouterConfig + + config = KvRouterConfig.from_json(json.dumps({**self._router_config, **flat})) + return { + worker_type: { + key: float(value) + for key, value in dict(config.policy_parameters(worker_type)).items() + } + for worker_type in self._worker_types + } + + +# ------------------------------------------------------------ DGD reads + + +def worker_spec_replicas( + dgd: Mapping[str, Any], planner_mode: str +) -> Optional[dict[str, int]]: + """DGD spec replicas per worker type of the planner mode. + + ``None`` when a worker component cannot be resolved or has no replicas. + An omitted ``replicas`` is the workload default of 1. + """ + + from dynamo.planner.config.defaults import SubComponentType + from dynamo.planner.monitoring.dgd_services import get_component_from_type_or_name + + roles = ( + (("prefill", SubComponentType.PREFILL), ("decode", SubComponentType.DECODE)) + if planner_mode == "disagg" + else (("aggregated", SubComponentType.DECODE),) + ) + replicas: dict[str, int] = {} + for worker_type, role in roles: + try: + service = get_component_from_type_or_name(dict(dgd), role) + except Exception as error: + logger.warning( + "retune: cannot resolve the %s workers: %s", worker_type, error + ) + return None + raw = service.service.get("replicas") + try: + count = 1 if raw is None else int(raw) + except (TypeError, ValueError): + return None + if count < 1: + return None + replicas[worker_type] = count + return replicas + + +def build_retune_inputs( + config: PlannerConfig, + startup: FrontendStartup, + *, + dgd_name: str, + model_name: Optional[str], +) -> RetuneInputs: + """Live ``RetuneInputs``: the parsed startup, traces on the claim. + + The engine pins only the router (the DGDR Run's engine comes from + ``retune.dgdr.engine``); ``prefill/decode_engine_args`` are empty maps in + disagg so the controller's engine-shape check holds. Live trace time is + epoch milliseconds (origin 0, speedup 1). + """ + + from dynamo.planner.config.planner_config import RetuneEngineSpec + from dynamo.planner.core.types import RetuneInputs + + retune = config.retune + if retune is None or retune.dgdr is None: + raise ValueError("live retune requires retune.executor 'dgdr'") + dgdr = retune.dgdr + disagg = config.mode == "disagg" + engine = RetuneEngineSpec( + router_mode=startup.router_mode, # type: ignore[arg-type] + router_config=dict(startup.router_config), + model_name=model_name or None, + prefill_engine_args={} if disagg else None, + decode_engine_args={} if disagg else None, + extra_engine_args=None, + ) + trace = posixpath.join(dgdr.mount_path, dgdr.segments_dir.strip("/"), dgd_name) + return RetuneInputs( + engine=engine, + trace_paths=(trace,), + trace_origin_ms=0.0, + arrival_speedup_ratio=1.0, + ) + + +def evaluate_spec( + dgd: Mapping[str, Any], + *, + config: PlannerConfig, + k8s_namespace: str, + dynamo_namespace: str, + frontend: Any = None, +) -> SpecEvaluation: + """Parse the frontend and evaluate the spec premises of one DGD read. + + ``frontend`` is the ``frontend_config`` module (injectable for tests). + """ + + if frontend is None: + from dynamo.planner.retune.live import frontend_config as frontend + + metadata = dgd.get("metadata") or {} + name = str(metadata.get("name") or "") + uid = metadata.get("uid") + if not name or not uid: + raise ValueError("the DGD has no metadata.name or metadata.uid") + ref = DgdRef( + name=name, + namespace=str(metadata.get("namespace") or k8s_namespace), + uid=str(uid), + dynamo_namespace=dynamo_namespace, + ) + parsed = frontend.parse_frontend(dgd, planner_mode=config.mode) + startup = parsed if isinstance(parsed, FrontendStartup) else None + parse_error = None if startup is not None else str(parsed) + report: PremiseReport = frontend.evaluate_premises( + dgd, + planner_config=config, + startup=startup, + parse_error=parse_error, + planner_replicas=None, + ) + inputs = None + if report.p4_ok and startup is not None: + try: + inputs = build_retune_inputs( + config, + startup, + dgd_name=name, + model_name=_served_model(frontend, dgd), + ) + except ValueError as error: + logger.error("retune: building the live retune inputs failed: %s", error) + report = PremiseReport( + p4_ok=False, + spec_ok=False, + reasons=(*report.reasons, f"P4: {error}"), + warnings=report.warnings, + startup=report.startup, + ) + generation = metadata.get("generation") + return SpecEvaluation( + report=report, + startup=startup, + inputs=inputs, + spec_replicas=worker_spec_replicas(dgd, config.mode), + dgd=ref, + generation=generation if isinstance(generation, int) else None, + ) + + +def _served_model(frontend: Any, dgd: Mapping[str, Any]) -> Optional[str]: + served = getattr(frontend, "served_model_name", None) + if served is None: + return None + try: + return served(dgd) + except Exception: + return None + + +# --------------------------------------------------------- DGDR pieces + + +def dgdr_context_factory( + config: PlannerConfig, cut_window: Optional[WindowCutter] = None +) -> Callable[[LiveRetuneSupervisor], Any]: + """Bind a ``dgdr.DgdrContext`` to the supervisor's current DGD.""" + + def factory(supervisor: LiveRetuneSupervisor) -> Any: + from dynamo.planner.retune.live.dgdr import DgdrContext + + retune = config.retune + assert retune is not None and retune.dgdr is not None + cutter: WindowCutter + if cut_window is not None: + cutter = cut_window + else: + from dynamo.planner.retune.live import window_cut + + cutter = window_cut.cut_window + return DgdrContext( + dgd=supervisor.dgd, + cfg=retune.dgdr, + retune=retune, + backend=str(config.backend), + kube=supervisor.kube, + registry=supervisor.registry, + cut_window=cutter, + is_leader=supervisor.is_leader, + ) + + return factory + + +def dgdr_canary( + config: PlannerConfig, *, timeout_s: float = 10.0 +) -> Callable[[LiveRetuneSupervisor], bool]: + """P11: v1beta2 accepts ``workload.trace`` (a Strict dry-run create). + + The canary is the single-point Request of the startup router point that + a real window would create, so a CRD or webhook that drops or rejects any + field the executor relies on fails it. ``KubeInvalid`` is a failed + premise; other API errors propagate (the supervisor counts them as + failing and retries next round). + """ + + def canary(supervisor: LiveRetuneSupervisor) -> bool: + from dynamo.planner.retune.live.dgdr import Shard, build_request + + retune = config.retune + assert retune is not None and retune.dgdr is not None + replicas = supervisor.evaluation.spec_replicas + point = supervisor.startup_flat() + if not replicas or not point: + return False + sla_ttft_ms, sla_itl_ms = retune.scoring_sla(config.ttft_ms, config.itl_ms) + shard = Shard( + id=CANARY_SHARD_ID, + kind="point", + choices={key: (value,) for key, value in point.items()}, + points=(dict(point),), + ) + body = build_request( + dgd=supervisor.dgd, + cfg=retune.dgdr, + retune=retune, + sla_ttft_ms=sla_ttft_ms, + sla_itl_ms=sla_itl_ms, + window_end_ms=CANARY_WINDOW_END_MS, + shard=shard, + spec_replicas=replicas, + backend=str(config.backend), + attempt=0, + ) + try: + supervisor.kube.create_request(body, dry_run=True, timeout_s=timeout_s) + except KubeAlreadyExists: + return True + except KubeInvalid as error: + logger.error( + "retune P11: the DGDR v1beta2 API rejects the Request: %s", error + ) + return False + return True + + return canary + + +def default_actuator_factory( + kube: RetuneKube, metrics: RetuneMetricsSink +) -> ActuatorFactory: + def factory(dgd: DgdRef, startup_maps: dict[str, dict[str, float]]) -> Any: + from dynamo.planner.retune.live.actuator import ( + AiohttpRouterPolicyClient, + RouterActuator, + ) + + return RouterActuator( + kube, AiohttpRouterPolicyClient(), dgd, startup_maps, metrics + ) + + return factory + + +def planner_identity() -> str: + """``/`` (§4.7.1).""" + + pod = ( + os.environ.get("POD_NAME") + or os.environ.get("DYN_POD_NAME") + or os.environ.get("HOSTNAME") + or socket.gethostname() + ) + return f"{pod}/{uuid.uuid4().hex[:12]}" + + +def searched_router_keys(config: PlannerConfig) -> tuple[str, ...]: + """Live names of the searched router keys (independent of the startup).""" + + retune = config.retune + if retune is None: + return () + try: + from dynamo.planner.retune.domains import expand_router_domain + + return tuple(sorted(expand_router_domain(retune))) + except Exception: + logger.exception("retune: expanding the router domain failed") + return () + + +def _runtime_namespace(host: Any) -> str: + environment = getattr(host, "environment", None) + getter = getattr(environment, "runtime_namespace", None) + if callable(getter): + try: + value = getter() + if value: + return str(value) + except Exception: + logger.debug("retune: runtime namespace unavailable", exc_info=True) + return str(host.config.namespace) + + +# --------------------------------------------------------------- build + + +async def build_supervisor( + config: PlannerConfig, + *, + kube: RetuneKube, + dgd_name: str, + namespace_fn: Callable[[], str], + metrics: Optional[RetuneMetricsSink] = None, + identity: Optional[str] = None, + frontend: Any = None, + actuator_factory: Optional[ActuatorFactory] = None, + reconciler: Any = None, + canary: Optional[Callable[[LiveRetuneSupervisor], bool]] = None, + context_factory: Optional[Callable[[LiveRetuneSupervisor], Any]] = None, + sweep_runner: Optional[Callable[..., Any]] = None, + **supervisor_kwargs: Any, +) -> LiveRetuneSupervisor: + """A supervisor for ``dgd_name`` with the production collaborators by default.""" + + retune = config.retune + if retune is None or retune.executor != "dgdr" or retune.dgdr is None: + raise ValueError( + "live retune requires retune.executor 'dgdr' (retune.executor " + f"is {None if retune is None else retune.executor!r})" + ) + dgdr = retune.dgdr + if metrics is None: + from dynamo.planner.retune.live.metrics import RetuneMetrics + + metrics = RetuneMetrics() + dgd_obj = await asyncio.to_thread(kube.get_dgd, dgd_name) + + def evaluate(obj: dict) -> SpecEvaluation: + return evaluate_spec( + obj, + config=config, + k8s_namespace=kube.namespace, + dynamo_namespace=namespace_fn(), + frontend=frontend, + ) + + evaluation = evaluate(dgd_obj) + if reconciler is None: + from dynamo.planner.retune.live.reconciler import Reconciler + + reconciler = Reconciler( + kube, evaluation.dgd, dgdr, retune, dgdr.mount_path, metrics + ) + worker_types = worker_types_for(config.mode) + settings = SupervisorSettings.from_config( + config, router_keys=searched_router_keys(config) + ) + supervisor = LiveRetuneSupervisor( + settings=settings, + kube=kube, + evaluation=evaluation, + identity=identity or planner_identity(), + premise_evaluator=evaluate, + actuator_factory=actuator_factory or default_actuator_factory(kube, metrics), + fingerprint_fn=lambda startup: fingerprint(config, startup), + resolver_factory=lambda startup: ( + None + if startup is None + else KvRouterResolver(startup.router_config, worker_types) + ), + reconciler=reconciler, + metrics=metrics, + canary=canary or dgdr_canary(config), + sweep_runner=sweep_runner, + context_factory=context_factory or dgdr_context_factory(config), + **supervisor_kwargs, + ) + report = evaluation.report + logger.info( + "live retune for DGD %s/%s (uid %s, dynamo namespace %s) as %s: " + "P4 %s, spec_ok %s, spec replicas %s, advisory %s", + evaluation.dgd.namespace, + evaluation.dgd.name, + evaluation.dgd.uid, + evaluation.dgd.dynamo_namespace, + supervisor.identity, + "ok" if report.p4_ok else "failed", + report.spec_ok, + evaluation.spec_replicas, + settings.advisory, + ) + return supervisor + + +async def build_live_retune(host: Any) -> LiveRetuneSupervisor: + """The supervisor of a ``NativePlannerBase`` with ``retune.executor: dgdr``. + + ``host.environment.controller`` is the planner's ``KubernetesConnector``; + its ``kube_api`` client and ``parent_dgd_name`` name the DGD. + """ + + config = host.config + retune = config.retune + if retune is None or retune.executor != "dgdr": + raise ValueError( + "build_live_retune requires retune.executor 'dgdr'; executor " + f"{None if retune is None else retune.executor!r} is replay-only" + ) + connector = host.environment.controller + from dynamo.planner.retune.live.kube import KubernetesRetuneKube + + kube = KubernetesRetuneKube.from_kube_api(connector.kube_api) + return await build_supervisor( + config, + kube=kube, + dgd_name=connector.parent_dgd_name, + namespace_fn=lambda: _runtime_namespace(host), + ) + + +async def build_legacy_cleanup(host: Any) -> Optional[LiveRetuneSupervisor]: + """§4.11: a decommission supervisor if a removed retune left Leases. + + ``None`` when there is nothing to clean up or the planner cannot read + Leases or the DGD (403/404: logged at info, never fatal). Run it like the + live supervisor (``start``/``stop``); it rolls back what the old retune + deployed, then deletes the DGD's retune Requests and both Leases and + stops itself (``decommissioned``). + """ + + config = host.config + if config.retune is not None: + return None + try: + connector = host.environment.controller + kube_api = connector.kube_api + dgd_name = connector.parent_dgd_name + except AttributeError: + return None + from dynamo.planner.retune.live.kube import KubernetesRetuneKube + + kube = KubernetesRetuneKube.from_kube_api(kube_api) + try: + leftovers = [ + await asyncio.to_thread(kube.get_lease, name) + for name in (leadership_lease_name(dgd_name), state_lease_name(dgd_name)) + ] + if not any(lease is not None for lease in leftovers): + return None + dgd_obj = await asyncio.to_thread(kube.get_dgd, dgd_name) + except KubeError as error: + logger.info("retune legacy clean-up skipped: %s", error) + return None + logger.info( + "retune legacy clean-up: DGD %s has Leases of a removed live retune; " + "rolling back what it deployed, then deleting its Requests and Leases", + dgd_name, + ) + return build_decommission_supervisor( + config, + kube=kube, + dgd=dgd_obj, + dynamo_namespace=_runtime_namespace(host), + ) + + +def build_decommission_supervisor( + config: PlannerConfig, + *, + kube: RetuneKube, + dgd: Mapping[str, Any], + dynamo_namespace: str, + metrics: Optional[RetuneMetricsSink] = None, + actuator_factory: Optional[ActuatorFactory] = None, + frontend: Any = None, + identity: Optional[str] = None, + **supervisor_kwargs: Any, +) -> LiveRetuneSupervisor: + """A supervisor that only rolls back and deletes (no retune config).""" + + if frontend is None: + from dynamo.planner.retune.live import frontend_config as frontend + if metrics is None: + from dynamo.planner.retune.live.metrics import RetuneMetrics + + metrics = RetuneMetrics() + metadata = dgd.get("metadata") or {} + ref = DgdRef( + name=str(metadata.get("name")), + namespace=str(metadata.get("namespace") or kube.namespace), + uid=str(metadata.get("uid")), + dynamo_namespace=dynamo_namespace, + ) + parsed = frontend.parse_frontend(dgd, planner_mode=config.mode) + startup = parsed if isinstance(parsed, FrontendStartup) else None + + def evaluate(obj: dict) -> SpecEvaluation: + return SpecEvaluation( + report=PremiseReport(p4_ok=False, spec_ok=True), + startup=startup, + inputs=None, + spec_replicas=None, + dgd=ref, + ) + + evaluation = evaluate(dict(dgd)) + settings = SupervisorSettings( + interval_s=3600.0, + grace_s=0.0, + timeout_s=3000.0, + advisory=True, + worker_types=worker_types_for(config.mode), + decommission=True, + ) + return LiveRetuneSupervisor( + settings=settings, + kube=kube, + evaluation=evaluation, + identity=identity or planner_identity(), + premise_evaluator=evaluate, + actuator_factory=actuator_factory or default_actuator_factory(kube, metrics), + fingerprint_fn=lambda _: "decommission", + metrics=metrics, + **supervisor_kwargs, + ) diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py new file mode 100644 index 000000000000..cfa462a592b1 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -0,0 +1,470 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Types shared by the live retune modules. + +Every module in ``dynamo.planner.retune.live`` depends on this one and on +nothing else in the package except through the protocols defined here, so +each can be built and tested with fakes. +""" + +from __future__ import annotations + +import threading +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal, Optional, Protocol + +# The eight builtin default-policy parameters a frontend router exposes on +# GET/PATCH /router_policy, by their live (PATCH) names. +POLICY_PARAMETER_KEYS: tuple[str, ...] = ( + "decode_active_request_weight", + "disk_cache_hit_weight", + "host_cache_hit_weight", + "overlap_score_credit", + "overlap_score_credit_decay", + "prefill_load_scale", + "router_temperature", + "shared_cache_multiplier", +) + +# Router knobs a DGDR v1beta2 Request can search, AISimulate public name -> +# live name. ``dynamo.router.simulation.config.ROUTER_PUBLIC_TO_LIVE`` is the +# source of truth; this mirrors it for modules that must not import the +# router simulation stack. +DGDR_ROUTER_KNOBS: dict[str, str] = { + "overlap_score_credit": "overlap_score_credit", + "prefill_load_scale": "prefill_load_scale", + "temperature": "router_temperature", +} + +LABEL_PREFIX = "nvidia.com/planner-retune-" +LABEL_DGD = LABEL_PREFIX + "dgd" +LABEL_DGD_UID = LABEL_PREFIX + "dgd-uid" +LABEL_WINDOW = LABEL_PREFIX + "window" +LABEL_SHARD = LABEL_PREFIX + "shard" +LABEL_SPEC = LABEL_PREFIX + "spec" +LABEL_ATTEMPT = LABEL_PREFIX + "attempt" + +# Pod labels the operator sets (deploy/operator/internal/consts/consts.go). +POD_LABEL_DGD_NAME = "nvidia.com/dynamo-graph-deployment-name" +POD_LABEL_COMPONENT_TYPE = "nvidia.com/dynamo-component-type" + +DGDR_GROUP = "nvidia.com" +DGDR_VERSION = "v1beta2" +DGDR_REQUESTS = "dynamographdeploymentrequests" +DGDR_RUNS = "dynamographdeploymentruns" +DGDR_CANDIDATES = "dynamographdeploymentcandidates" +RUN_UID_LABEL = "nvidia.com/dgdr-run-uid" + +# SweepResult.failure_reason values that mean "this process stopped working +# on the window", not "the window failed": no decision is recorded and the +# controller rewinds so the window can be adopted (§4.7.5). +NON_DECISION_FAILURES = frozenset({"shutdown", "lost_leadership"}) + +GateReason = Literal["not_leader", "not_ready", "already_decided"] + + +# --------------------------------------------------------------- controller + + +@dataclass(frozen=True) +class GateResult: + """What ``launch_gate`` tells the controller about a due window. + + ``reason`` is ``None`` to let the window launch (or record ``busy``). + ``not_leader``/``not_ready`` keep ``_last_k`` so the window is retried on + a later tick; ``already_decided`` advances it without a decision. + ``spec_replicas`` (worker type -> DGD spec replicas) and ``abort`` (set + by the supervisor when this process stops leading) are only meaningful + when ``reason`` is ``None``. + """ + + reason: Optional[GateReason] = None + spec_replicas: Optional[dict[str, int]] = None + abort: Optional[threading.Event] = None + + +# (k, (window_start_s, window_end_s, as_of_s)) -> GateResult +LaunchGate = Callable[[int, tuple[float, float, float]], GateResult] +# controller generation of the sweep's controller -> may it be harvested? +HarvestAllowed = Callable[[int], bool] + + +# ------------------------------------------------------------ kubernetes + + +class KubeError(Exception): + """A Kubernetes call failed (network, timeout, 5xx, unexpected status).""" + + +class KubeNotFound(KubeError): + pass + + +class KubeConflict(KubeError): + """409 on update (stale resourceVersion).""" + + +class KubeAlreadyExists(KubeError): + """409 on create (the name is taken).""" + + +class KubeInvalid(KubeError): + """422 / 400: the API server rejected the object (e.g. Strict pruning).""" + + +@dataclass(frozen=True) +class DgdRef: + """The DynamoGraphDeployment this planner serves.""" + + name: str + namespace: str # Kubernetes namespace + uid: str + dynamo_namespace: str # the routers' ``namespace`` on GET /router_policy + + +@dataclass(frozen=True) +class PodInfo: + name: str + uid: str + ip: Optional[str] + ready: bool + terminating: bool + labels: Mapping[str, str] = field(default_factory=dict) + # Container port named ``http`` of the frontend container, if any. + http_port: Optional[int] = None + # ``status.phase`` (Pending, Running, Succeeded, Failed, Unknown). + phase: Optional[str] = None + + +class RetuneKube(Protocol): + """Synchronous Kubernetes access for live retune. + + Every method takes an optional ``timeout_s`` (connect+read, applied as + the kubernetes client's ``_request_timeout`` tuple of ints) and raises + the ``Kube*`` errors above instead of client exceptions. Callers on the + event loop wrap calls in ``asyncio.to_thread``. + """ + + namespace: str + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + ... + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + ... + + def create_request( + self, + body: dict, + *, + dry_run: bool = False, + timeout_s: float = 10.0, + ) -> dict: + """Create with ``field_validation='Strict'`` (and ``dry_run='All'``).""" + ... + + def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def list_requests( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + ... + + def delete_request( + self, + name: str, + *, + propagation: Literal["Background", "Foreground"] = "Background", + timeout_s: float = 10.0, + ) -> None: + """Missing objects are not an error.""" + ... + + def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def list_candidates( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + ... + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + ... + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + """Update with ``body['metadata']['resourceVersion']``; 409 -> KubeConflict.""" + ... + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + ... + + +# ------------------------------------------------------- frontend config + + +@dataclass(frozen=True) +class FrontendStartup: + """The frontend router configuration parsed from the DGD spec. + + ``router_config`` is the startup ``KvRouterConfig`` JSON (only keys the + frontend sets); ``startup_maps`` is ``policy_parameters(wt)`` for each + worker type of the planner mode. ``fingerprint_source`` is a canonical + JSON string of everything parsed that the state fingerprint covers. + """ + + router_mode: str # "kv_router" or "round_robin" + router_config: dict[str, Any] + startup_maps: dict[str, dict[str, float]] + frontend_component: str # DGD component name of the frontend + replicas: int + replica_sync: bool + fingerprint_source: str + + +@dataclass(frozen=True) +class PremiseReport: + """Spec-class premise evaluation (P4, P7-P10; P14 is the actuator's). + + ``p4_ok`` alone decides whether a controller can be built; ``spec_ok`` + is the conjunction of every spec premise. ``reasons`` explain each + failed premise as ``"P: ..."``; ``warnings`` carry warn-only ones. + """ + + p4_ok: bool + spec_ok: bool + reasons: tuple[str, ...] = () + warnings: tuple[str, ...] = () + startup: Optional[FrontendStartup] = None + + +# ---------------------------------------------------------- router policy + + +@dataclass(frozen=True) +class RouterInfo: + """One router on GET /router_policy.""" + + router_id: str + namespace: str + worker_type: str + revision: int + # The 8 PolicyParameters; None for a custom policy without runtime config. + configuration: Optional[dict[str, float]] + model: Optional[str] = None + + +@dataclass(frozen=True) +class PolicyCallResult: + """Outcome of one GET or PATCH. ``status`` is the HTTP status (0 on a + network error or timeout).""" + + status: int + routers: tuple[RouterInfo, ...] = () # GET + router: Optional[RouterInfo] = None # PATCH/reset response, when 200 + error: Optional[str] = None + + +class RouterPolicyClient(Protocol): + async def get(self, pod: PodInfo) -> PolicyCallResult: + ... + + async def patch( + self, + pod: PodInfo, + router_id: str, + expected_revision: int, + parameters: Mapping[str, float], + ) -> PolicyCallResult: + ... + + async def reset( + self, pod: PodInfo, router_id: str, expected_revision: int + ) -> PolicyCallResult: + """PATCH ``{router_id, expected_revision, reset: true}``.""" + ... + + +ActuationMode = Literal["apply", "rollback", "observe"] + + +@dataclass(frozen=True) +class ActuationTarget: + """What one actuator round converges to. + + ``apply``: PATCH explicit values so every router's 8 parameters equal + ``desired_by_wt[worker_type]``. ``rollback``: reset every router whose + configuration differs from its latched initial (never by revision). + ``observe``: no writes; latch initials and check parity only. + ``include_not_ready`` also targets NotReady, non-terminating pods. + ``learn_unknown`` lets the actuator reset a router whose initial was + never latched (revision > 0) once to learn it; that is active mode only + (§4.8). With False (advisory) such a router is never written: rollback + leaves it alone and does not wait for it to converge. + """ + + mode: ActuationMode + desired_by_wt: Optional[dict[str, dict[str, float]]] = None + include_not_ready: bool = False + learn_unknown: bool = True + + +@dataclass(frozen=True) +class ActuationReport: + converged: bool + # router_id -> configuration latched this round (revision 0 or reset). + new_initials: dict[str, dict[str, float]] = field(default_factory=dict) + # router_ids seen this round (to prune stale initials). + seen_router_ids: frozenset[str] = frozenset() + # P14: every known initial equals startup_maps[worker_type]. + parity_ok: Optional[bool] = None + parity_mismatches: tuple[str, ...] = () + admin_api_disabled: bool = False + errors: dict[str, int] = field(default_factory=dict) # by status code / kind + # P12: enough Ready frontends, each with a configured router for every + # worker type, whatever their values. Gates launches; convergence does not. + routers_present: Optional[bool] = None + + +# ----------------------------------------------------------- reconciler + + +class SweepRegistry(Protocol): + """Thread-safe link between a running sweep and the reconciler. + + Protection is reference-counted per window: every ``protect`` call + takes one reference and needs one matching ``release``. A window, with + the union of every Request name given to it, stays protected while any + reference is held, so two sweeps of one window (a rebuilt controller + relaunching it while the old sweep winds down) cannot drop each + other's protection. + """ + + def protect(self, window_end_ms: int, request_names: Iterable[str]) -> None: + """Mark a window file and Requests as in use by this process (one + reference).""" + ... + + def release(self, window_end_ms: int) -> None: + """Drop one reference taken by ``protect``.""" + ... + + def enqueue_delete(self, request_names: Iterable[str]) -> None: + ... + + +# ---------------------------------------------------------------- window + + +@dataclass(frozen=True) +class WindowCutResult: + path: Optional[str] # None for an empty window (no file written) + num_requests: int + trace_block_size: Optional[int] + row_counts: dict[str, int] + dropped_members: int = 0 + dropped_bytes: int = 0 + reused: bool = False # the final file already existed (EEXIST) + + +class WindowCutter(Protocol): + def __call__( + self, + segments_root: str, + start_ms: int, + end_ms: int, + as_of_ms: int, + out_path: str, + *, + max_rows: int, + deadline_s: float, + cancel: Optional[threading.Event] = None, + ) -> WindowCutResult: + ... + + +# --------------------------------------------------------------- metrics + + +class RetuneMetricsSink(Protocol): + """``dynamo_planner_retune_*`` metrics; every method is cheap and safe.""" + + def decision(self, outcome: str) -> None: + ... + + def failure(self, reason: str) -> None: + ... + + def skipped(self, reason: str) -> None: + ... + + def supervisor_state( + self, + *, + leader: bool, + ready: bool, + initialized: bool, + rollback_pending: bool, + generation: int, + ) -> None: + ... + + def scores( + self, + *, + baseline: Optional[float], + best: Optional[float], + sweep_seconds: Optional[float], + missing_candidates: int, + ) -> None: + ... + + def router_converged(self, converged: bool) -> None: + ... + + def patch_error(self, code: str) -> None: + ... + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + ... + + def dropped_members(self, count: int) -> None: + ... + + +class NullMetrics: + """A ``RetuneMetricsSink`` that records nothing (tests, disabled metrics).""" + + def decision(self, outcome: str) -> None: + pass + + def failure(self, reason: str) -> None: + pass + + def skipped(self, reason: str) -> None: + pass + + def supervisor_state(self, **_: Any) -> None: + pass + + def scores(self, **_: Any) -> None: + pass + + def router_converged(self, converged: bool) -> None: + pass + + def patch_error(self, code: str) -> None: + pass + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + pass + + def dropped_members(self, count: int) -> None: + pass diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py new file mode 100644 index 000000000000..f44eed905346 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -0,0 +1,1909 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live retune sweep executor: one window through DGDR v1beta2 Requests. + +``run_dgdr_sweep`` is the live ``sweep_fn`` of ``RetuneController`` (design +§4.5). It runs on the controller's single sweep thread and never raises: + +1. Shard the router grid (``plan_shards``): balanced Cartesian sub-grids of + at most 100 points (a Request's ``maxCandidates`` bound), one + single-point shard for an off-grid baseline or startup point, and a CRN + probe that evaluates one fixed grid point a second time. +2. Adopt the Requests a previous leader created for this window, matched by + ``(dgd-uid, window, shard, spec hash)``; others of the window are queued + for deletion. A window whose every shard is adopted as Succeeded is + harvested without cutting it again. An adopted attempt whose Run failed + or ran another input is replaced by attempt+1 while the shard still + fits before the deadline. +3. Cut the window file (a subprocess, ``context.cut_window``), then create + one Request per remaining shard with Strict field validation and check + that the API server kept every submitted field. +4. Wait for each Request's Run, check that ``Run.status.aisimulate.input`` + is what the Request asked for, and read the Run's Candidates. +5. Map each Candidate's router adapter parameters back to live names, + require identical non-router parameters across the window, score each + with ``score_summary`` on ``retune.objective``, and check the CRN probe. + +The executor never deletes anything itself: Requests to remove go to the +``SweepRegistry`` deletion queue, which the reconciler drains while this +process leads. Shutdown (``cancel``) and loss of leadership (``abort``) +return at once with ``failure_reason`` ``shutdown``/``lost_leadership`` and +leave every Request in place for the next leader to adopt. Every mutating +call first checks ``abort`` and ``context.is_leader()``. + +A shard that holds the baseline, or the probe, is critical: when it fails +the window fails. Any other failed shard only leaves its points unscored. +""" + +from __future__ import annotations + +import copy +import functools +import hashlib +import itertools +import json +import logging +import math +import os +import re +import threading +import time +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Literal, NoReturn, Optional + +from dynamo.planner.config.planner_config import DgdrExecutorConfig, RetuneConfig +from dynamo.planner.core.types import RetuneParams +from dynamo.planner.retune.live.contracts import ( + DGDR_GROUP, + DGDR_ROUTER_KNOBS, + DGDR_VERSION, + LABEL_ATTEMPT, + LABEL_DGD, + LABEL_DGD_UID, + LABEL_PREFIX, + LABEL_SHARD, + LABEL_SPEC, + LABEL_WINDOW, + RUN_UID_LABEL, + DgdRef, + KubeAlreadyExists, + KubeError, + KubeInvalid, + KubeNotFound, + RetuneKube, + SweepRegistry, + WindowCutResult, + WindowCutter, +) +from dynamo.planner.retune.sweep import ( + SweepRequest, + SweepResult, + SweepStatus, + score_summary, +) + +logger = logging.getLogger(__name__) + +ShardKind = Literal["grid", "point", "probe"] + +# A Request publishes at most recommendation.maxCandidates (<= 100) points. +MAX_SHARD_POINTS = 100 +# Seconds a new Request may go without an Accepted condition before the +# window fails as operator_not_reconciling. +ACCEPT_TIMEOUT_S = 60.0 +# Relative score difference the CRN probe tolerates. +CRN_REL_TOLERANCE = 1e-9 +# Oldest Dynamo runtime whose trace replay runs with common random numbers. +# None: the version is not checked and the CRN probe alone decides. +CRN_MIN_VERSION: Optional[str] = None +# Annotation holding the cut window's row counts, so a window adopted as +# Succeeded reports them without cutting the window again. +ROWS_ANNOTATION = LABEL_PREFIX + "rows" +# Operator constants (deploy/operator at #17483). +REQUEST_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" +TRACE_MOUNT_PATH = "/var/run/aisimulate/trace" +ROUTER_ADAPTER = "dynamo.router" +_ROLES = ("aggregated", "prefill", "decode") +_TEMPERATURE = DGDR_ROUTER_KNOBS["temperature"] +# spec.objective.metric -> AISimulate optimization.target (operator lowering). +_METRIC_TARGETS = { + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", +} +# Statuses a retry cannot fix (bad request, auth, missing API). +_PERMANENT_STATUSES = frozenset({400, 401, 403, 404, 405, 422}) +# How the API server words a rejection by an admission webhook. +_ADMISSION_DENIED = re.compile(r'admission webhook "[^"]*" denied the request') +_K8S_TIMEOUT_S = 10.0 +_SNAP_TOLERANCE = 1e-9 +_MIN_K8S_TIMEOUT_S = 1.0 +_ABSENT = object() + + +# ------------------------------------------------------------------ shards + + +@dataclass(frozen=True) +class Shard: + """One Request's slice of the window's candidates. + + ``choices`` maps each live router knob to the values the shard spans; + ``points`` is their Cartesian product (live names). A ``point`` or + ``probe`` shard has one value per knob. + """ + + id: str + kind: ShardKind + choices: dict[str, tuple[float, ...]] + points: tuple[dict[str, float], ...] + + @property + def size(self) -> int: + return len(self.points) + + def contains(self, point: Mapping[str, float]) -> bool: + return all(point.get(key) in values for key, values in self.choices.items()) + + +def plan_shards( + router_domain: Mapping[str, Sequence[float]], + *, + baseline: Mapping[str, float], + startup: Mapping[str, float], + max_shards: int, + probe: bool = True, + parallelism: int | None = None, +) -> list[Shard]: + """Split ``router_domain`` (live knob -> values) into Request shards. + + The grid becomes at most ``max_shards`` Cartesian sub-grids of at most + ``MAX_SHARD_POINTS`` points: each knob's sorted values are cut into + contiguous, near-equal chunks. The split minimizes the largest shard's + wall time, then the shard count: with the Run ``parallelism`` the time + is its evaluation rounds (``1 + ceil((n - 1) / parallelism)``, as in + ``need_seconds``), so a shard that would not shorten the window is not + added; without it, the largest shard's size. ``baseline`` and ``startup`` + (live knob -> value) each get a single-point shard when off the grid. + With ``probe``, a ``probe`` shard repeats one fixed grid point (the + middle value of each knob, a positive temperature when the grid has + one) to check common random numbers across Runs. Raises ``ValueError`` + when no split fits or a point does not cover the domain's knobs. + """ + + if not router_domain: + raise ValueError("router domain is empty; there is nothing to shard") + if max_shards < 1: + raise ValueError(f"max_shards must be at least 1, got {max_shards}") + keys = tuple(sorted(router_domain)) + axes: dict[str, tuple[float, ...]] = {} + for key in keys: + values = tuple(sorted({float(value) for value in router_domain[key]})) + if not values: + raise ValueError(f"router domain {key!r} has no values") + if not all(math.isfinite(value) for value in values): + raise ValueError(f"router domain {key!r} has a non-finite value") + axes[key] = values + for name, point in (("baseline", baseline), ("startup", startup)): + if set(point) != set(keys): + raise ValueError( + f"{name} router point keys {sorted(point)} do not match the " + f"router domain {list(keys)}" + ) + + if parallelism is not None and parallelism < 1: + raise ValueError(f"parallelism must be at least 1, got {parallelism}") + sizes = [len(axes[key]) for key in keys] + best: tuple[float, int, int, tuple[int, ...]] | None = None + for splits in _splits(sizes, max_shards): + largest = math.prod( + math.ceil(size / split) for size, split in zip(sizes, splits) + ) + # An oversized shard can never be chosen over one that fits. + cost = ( + math.inf + if largest > MAX_SHARD_POINTS + else largest + if parallelism is None + else _rounds(largest, parallelism) + ) + candidate = (cost, math.prod(splits), largest, splits) + if best is None or candidate < best: + best = candidate + assert best is not None # all-ones always fits max_shards >= 1 + _, _, largest, splits = best + if largest > MAX_SHARD_POINTS: + raise ValueError( + f"the router grid of {math.prod(sizes)} points cannot be split into " + f"at most {max_shards} Cartesian shards of at most " + f"{MAX_SHARD_POINTS} points (the smallest largest shard has " + f"{largest}); raise retune.dgdr.run.max_shards or shrink the domain" + ) + + shards: list[Shard] = [] + chunked = [_chunks(axes[key], split) for key, split in zip(keys, splits)] + for index, combo in enumerate(itertools.product(*chunked)): + choices = dict(zip(keys, combo)) + shards.append( + Shard( + id=f"grid-{index}", + kind="grid", + choices=choices, + points=_product(choices), + ) + ) + + def in_grid(point: Mapping[str, float]) -> bool: + return all(float(point[key]) in axes[key] for key in keys) + + def single(shard_id: str, kind: ShardKind, point: Mapping[str, float]) -> Shard: + values = {key: float(point[key]) for key in keys} + return Shard( + id=shard_id, + kind=kind, + choices={key: (value,) for key, value in values.items()}, + points=(values,), + ) + + baseline_point = {key: float(baseline[key]) for key in keys} + startup_point = {key: float(startup[key]) for key in keys} + if not in_grid(baseline_point): + shards.append(single("point-baseline", "point", baseline_point)) + if not in_grid(startup_point) and startup_point != baseline_point: + shards.append(single("point-startup", "point", startup_point)) + if probe: + shards.append(single("probe", "probe", _probe_point(axes))) + return shards + + +def _rounds(points: int, parallelism: int) -> int: + """Evaluation rounds of a Run: the first candidate alone, then batches.""" + + return 1 + math.ceil((points - 1) / parallelism) + + +def _splits(sizes: Sequence[int], max_shards: int) -> Iterable[tuple[int, ...]]: + """Every per-knob split count whose product is at most ``max_shards``.""" + + def walk(index: int, budget: int) -> Iterable[tuple[int, ...]]: + if index == len(sizes): + yield () + return + for split in range(1, min(sizes[index], budget) + 1): + for rest in walk(index + 1, budget // split): + yield (split, *rest) + + return walk(0, max_shards) + + +def _chunks(values: tuple[float, ...], count: int) -> list[tuple[float, ...]]: + """``values`` cut into ``count`` contiguous chunks whose sizes differ by <= 1.""" + + base, extra = divmod(len(values), count) + chunks: list[tuple[float, ...]] = [] + start = 0 + for index in range(count): + end = start + base + (1 if index < extra else 0) + chunks.append(values[start:end]) + start = end + return chunks + + +def _product(choices: Mapping[str, tuple[float, ...]]) -> tuple[dict[str, float], ...]: + keys = sorted(choices) + return tuple( + dict(zip(keys, values)) + for values in itertools.product(*(choices[key] for key in keys)) + ) + + +def _probe_point(axes: Mapping[str, tuple[float, ...]]) -> dict[str, float]: + """The fixed CRN probe point: each knob's middle value; a positive + temperature when the grid has one, so the probe exercises sampling.""" + + point: dict[str, float] = {} + for key, values in axes.items(): + pool = values + if key == _TEMPERATURE: + positive = tuple(value for value in values if value > 0) + pool = positive or values + point[key] = pool[len(pool) // 2] + return point + + +def need_seconds(shard: Shard, cfg: DgdrExecutorConfig, *, needs_cut: bool) -> float: + """Wall time a shard needs from now to finish (design §4.1). + + ``(cut_budget if needs_cut) + job_startup + (1 + ceil((n - 1) / + parallelism)) * candidate_timeout``: AISimulate evaluates the first + candidate alone, then ``parallelism`` at a time. + """ + + rounds = _rounds(shard.size, cfg.run.parallelism) + return ( + (cfg.cut_budget_seconds if needs_cut else 0.0) + + cfg.job_startup_seconds + + rounds * cfg.run.candidate_timeout_seconds + ) + + +# ----------------------------------------------------------------- Request + + +def spec_hash(spec: Mapping[str, Any]) -> str: + """First 16 hex digits of the SHA-1 of the canonical JSON of ``spec``.""" + + canonical = json.dumps(spec, sort_keys=True, separators=(",", ":"), allow_nan=False) + return hashlib.sha1(canonical.encode("utf-8")).hexdigest()[:16] + + +def window_claim_path(cfg: DgdrExecutorConfig, dgd: DgdRef, window_end_ms: int) -> str: + """Claim-relative path of a window file: ``//.jsonl.gz``.""" + + return f"{cfg.windows_dir.strip('/')}/{dgd.name}/{int(window_end_ms)}.jsonl.gz" + + +def request_name( + dgd: DgdRef, window_end_ms: int, shard_id: str, digest: str, attempt: int +) -> str: + """``-rt-``.""" + + key = f"{dgd.uid}|{int(window_end_ms)}|{shard_id}|{digest}|{attempt}" + suffix = hashlib.sha1(key.encode("utf-8")).hexdigest()[:10] + prefix = dgd.name[:40].rstrip("-.") or "dgd" + return f"{prefix}-rt-{suffix}" + + +def _label_value(value: str) -> str: + """``value`` when it is a valid label value, else a hashed 63-char form.""" + + if len(value) <= 63 and re.fullmatch( + r"([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?)?", value + ): + return value + digest = hashlib.sha1(value.encode("utf-8")).hexdigest()[:10] + head = re.sub(r"[^-A-Za-z0-9_.]", "-", value)[:52].strip("-_.") + return f"{head}-{digest}" if head else digest + + +def build_request( + *, + dgd: DgdRef, + cfg: DgdrExecutorConfig, + retune: RetuneConfig, + sla_ttft_ms: float, + sla_itl_ms: float, + window_end_ms: int, + shard: Shard, + spec_replicas: Mapping[str, int], + backend: str, + attempt: int, +) -> dict: + """The DGDR v1beta2 Request that replays one shard of a window (§4.5.2). + + The engine is ``cfg.engine`` with every leaf a scalar and each used + role's ``parallelism.preset[0].replicas`` set from ``spec_replicas`` + (unused roles are deleted with ``null``); the GPU budget is the sum of + replicas times GPUs per worker (tensor x pipeline x attention_data). + Router knobs use their AISimulate public names: ``{choices: [...]}`` + for a grid shard, a scalar for a point or probe shard. Raises + ``ValueError`` for an engine or shard the Request cannot express. + """ + + engine = _pinned_engine(cfg.engine, spec_replicas, backend) + size = shard.size + if not 1 <= size <= MAX_SHARD_POINTS: + raise ValueError(f"shard {shard.id} has {size} points; 1..100 fit a Request") + spec: dict[str, Any] = { + "model": {"name": cfg.model}, + "backends": [backend], + "hardware": { + "gpu": {"sku": cfg.sku, "budget": _gpu_budget(engine, spec_replicas)} + }, + "workload": { + "trace": { + "format": "dynamo", + "source": { + "pvc": { + "claimName": cfg.claim_name, + "path": window_claim_path(cfg, dgd, window_end_ms), + } + }, + } + }, + "objective": { + "mode": "Optimize", + "metric": retune.objective, + "sla": {"ttftMs": float(sla_ttft_ms), "itlMs": float(sla_itl_ms)}, + }, + "recommendation": {"maxCandidates": size}, + "aisimulate": { + "inputOverrides": { + "engine": engine, + "router": _router_section(shard), + "optimizer": { + "algorithm": "random", + "max_trials": 2 * size, + "parallelism": cfg.run.parallelism, + "candidate_timeout_seconds": cfg.run.candidate_timeout_seconds, + }, + "optimization": {"strict_sla": False}, + "dgdr_run": {"generator": "direct"}, + } + }, + } + digest = spec_hash(spec) + return { + "apiVersion": f"{DGDR_GROUP}/{DGDR_VERSION}", + "kind": "DynamoGraphDeploymentRequest", + "metadata": { + "name": request_name(dgd, window_end_ms, shard.id, digest, attempt), + "namespace": dgd.namespace, + "labels": { + LABEL_DGD: _label_value(dgd.name), + LABEL_DGD_UID: dgd.uid, + LABEL_WINDOW: str(int(window_end_ms)), + LABEL_SHARD: shard.id, + LABEL_SPEC: digest, + LABEL_ATTEMPT: str(attempt), + }, + "ownerReferences": [ + { + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeployment", + "name": dgd.name, + "uid": dgd.uid, + "blockOwnerDeletion": False, + } + ], + }, + "spec": spec, + } + + +def _pinned_engine( + engine: Mapping[str, Any], spec_replicas: Mapping[str, int], backend: str +) -> dict[str, Any]: + pinned = copy.deepcopy(dict(engine)) + roles = set(spec_replicas) + if roles not in ({"aggregated"}, {"prefill", "decode"}): + raise ValueError( + "spec_replicas must name 'aggregated', or 'prefill' and 'decode'; " + f"got {sorted(roles)}" + ) + written = pinned.get("backend") + if written is None: + pinned["backend"] = backend + elif written != backend: + raise ValueError( + f"retune.dgdr.engine.backend {written!r} differs from the deployed " + f"backend {backend!r}" + ) + workers = pinned.get("workers") + if not isinstance(workers, dict): + raise ValueError("retune.dgdr.engine.workers must be a mapping") + for role in _ROLES: + if role not in roles: + workers[role] = None + continue + worker = workers.get(role) + preset = ( + worker.get("parallelism", {}).get("preset") + if isinstance(worker, dict) and isinstance(worker.get("parallelism"), dict) + else None + ) + if ( + not isinstance(preset, list) + or len(preset) != 1 + or not isinstance(preset[0], dict) + ): + raise ValueError( + f"retune.dgdr.engine.workers.{role}.parallelism.preset must be a " + "list of exactly one entry" + ) + replicas = int(spec_replicas[role]) + if replicas < 1: + raise ValueError( + f"{role} has {replicas} spec replicas; at least 1 is needed" + ) + preset[0]["replicas"] = replicas + return pinned + + +def _gpu_budget(engine: Mapping[str, Any], spec_replicas: Mapping[str, int]) -> int: + budget = 0 + for role, replicas in spec_replicas.items(): + entry = engine["workers"][role]["parallelism"]["preset"][0] + per_worker = 1 + for knob in ("tensor", "pipeline", "attention_data"): + value = entry.get(knob, 1) + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError( + f"retune.dgdr.engine.workers.{role}.parallelism.preset[0]." + f"{knob} must be a positive integer, got {value!r}" + ) + per_worker *= value + budget += int(replicas) * per_worker + return budget + + +def _router_section(shard: Shard) -> dict[str, Any]: + live_knobs = set(DGDR_ROUTER_KNOBS.values()) + if set(shard.choices) != live_knobs: + raise ValueError( + f"shard {shard.id} router knobs {sorted(shard.choices)} must be exactly " + f"{sorted(live_knobs)}, the knobs a DGDR Request can express" + ) + section: dict[str, Any] = { + "policy": "kv_router", + "prefill_load_model": {"type": "none"}, + } + for public, live in DGDR_ROUTER_KNOBS.items(): + values = [float(value) for value in shard.choices[live]] + section[public] = {"choices": values} if shard.kind == "grid" else values[0] + return section + + +# --------------------------------------------------------- input read-back + + +def merge_patch(target: Any, patch: Any) -> Any: + """RFC 7386 JSON merge patch (the operator's lowering); never mutates.""" + + if not isinstance(patch, Mapping): + return copy.deepcopy(patch) + result = dict(target) if isinstance(target, Mapping) else {} + for key, value in patch.items(): + if value is None: + result.pop(key, None) + else: + result[key] = merge_patch(result.get(key), value) + return result + + +def _get(obj: Any, path: Sequence[str], default: Any = None) -> Any: + for key in path: + if not isinstance(obj, Mapping) or key not in obj: + return default + obj = obj[key] + return obj + + +def _same_json(left: Any, right: Any) -> bool: + """JSON equality: numbers by value (2 == 2.0), booleans never equal numbers.""" + + if isinstance(left, bool) or isinstance(right, bool): + return isinstance(left, bool) and isinstance(right, bool) and left == right + if isinstance(left, (int, float)) and isinstance(right, (int, float)): + return float(left) == float(right) + if isinstance(left, Mapping) and isinstance(right, Mapping): + return set(left) == set(right) and all( + _same_json(left[key], right[key]) for key in left + ) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all( + _same_json(a, b) for a, b in zip(left, right) + ) + return type(left) is type(right) and left == right + + +def missing_fields(stored: Any, submitted: Any, path: str = "") -> list[str]: + """Submitted leaves the stored object lacks or changed (pruned or rejected). + + ``stored`` may hold more (admission defaults); a submitted ``null`` only + requires the stored value to be absent or ``null``. + """ + + if isinstance(submitted, Mapping): + if not isinstance(stored, Mapping): + return [path or "."] + problems: list[str] = [] + for key, value in submitted.items(): + child = f"{path}.{key}" if path else key + if value is None: + if stored.get(key) is not None: + problems.append(child) + continue + if key not in stored: + problems.append(child) + continue + problems.extend(missing_fields(stored[key], value, child)) + return problems + return [] if _same_json(stored, submitted) else [path or "."] + + +def input_mismatches( + spec: Mapping[str, Any], effective: Mapping[str, Any] +) -> list[str]: + """How ``Run.status.aisimulate.input`` differs from what ``spec`` asks for. + + The expected input follows the operator's lowering at #17483: the + public role underlay, ``spec.model``/``backends``/``hardware`` into + ``engine``, then ``inputOverrides`` as a merge patch. ``engine``, + ``router`` and ``optimizer`` must match exactly; the objective, SLA, + budget, trace source and the direct generator are checked by value. + """ + + overrides = _get(spec, ("aisimulate", "inputOverrides"), {}) or {} + typed_engine: dict[str, Any] = { + "workers": {role: {} for role in _ROLES}, + "model": _get(spec, ("model", "name")), + "backend": {"choices": list(spec.get("backends") or [])}, + "hardware": _get(spec, ("hardware", "gpu", "sku")), + } + expected_trace = _get(spec, ("workload", "trace", "source", "pvc", "path"), "") + mounted = TRACE_MOUNT_PATH + ( + "/trace.jsonl.gz" if str(expected_trace).endswith(".gz") else "/trace.jsonl" + ) + sla = _get(spec, ("objective", "sla"), {}) or {} + checks: list[tuple[tuple[str, ...], Any]] = [ + (("engine",), merge_patch(typed_engine, overrides.get("engine") or {})), + (("router",), merge_patch({}, overrides.get("router") or {})), + (("optimizer",), merge_patch({}, overrides.get("optimizer") or {})), + ( + ("optimization", "target"), + _METRIC_TARGETS.get(_get(spec, ("objective", "metric"), ""), _ABSENT), + ), + ( + ("optimization", "strict_sla"), + _get(overrides, ("optimization", "strict_sla")), + ), + ( + ("optimization", "constraints", "max_candidate_gpus"), + _get(spec, ("hardware", "gpu", "budget")), + ), + (("evaluation", "sla", "ttft_ms"), sla.get("ttftMs", _ABSENT)), + (("evaluation", "sla", "itl_ms"), sla.get("itlMs", _ABSENT)), + (("evaluation", "sla", "e2e_ms"), sla.get("e2eMs", _ABSENT)), + ( + ("traffic", "source"), + {"type": "trace", "format": "dynamo", "paths": [mounted]}, + ), + (("traffic", "load"), {"type": "trace_timestamps"}), + (("dgdr_run", "generator"), _get(overrides, ("dgdr_run", "generator"))), + ( + ("dgdr_run", "max_candidates"), + _get(spec, ("recommendation", "maxCandidates")), + ), + ] + # Normalize like the API server (JSON numbers, no tuples). + effective = json.loads(json.dumps(effective)) + problems: list[str] = [] + for path, expected in checks: + expected = expected if expected is _ABSENT else json.loads(json.dumps(expected)) + actual = _get(effective, path, _ABSENT) + if expected is _ABSENT: + if actual is not _ABSENT and actual is not None: + problems.append(f"{'.'.join(path)}: expected unset, got {actual!r}") + elif actual is _ABSENT or not _same_json(actual, expected): + shown = "unset" if actual is _ABSENT else repr(actual) + problems.append(f"{'.'.join(path)}: expected {expected!r}, got {shown}") + return problems + + +def _version_tuple(text: str | None) -> tuple[int, ...] | None: + if not text: + return None + found = re.match(r"^v?(\d+)\.(\d+)(?:\.(\d+))?", text.strip()) + if found is None: + return None + return tuple(int(part) for part in found.groups(default="0")) + + +def crn_unsupported( + runtime_version: str | None, runtime_image: str | None, minimum: str | None +) -> bool: + """Whether a parseable runtime version is below ``minimum`` (None: unknown).""" + + floor = _version_tuple(minimum) + if floor is None: + return False + version = _version_tuple(runtime_version) + if version is None and runtime_image: + # registry/repo:tag@digest -> tag + last = runtime_image.split("@", 1)[0].rsplit("/", 1)[-1] + if ":" in last: + version = _version_tuple(last.split(":", 1)[1]) + return version is not None and version < floor + + +# ---------------------------------------------------------------- executor + + +@dataclass(frozen=True) +class DgdrContext: + """Everything ``run_dgdr_sweep`` needs besides the ``SweepRequest``. + + ``backend`` is the deployed backend (``vllm``, ``sglang``, ``trtllm``). + ``is_leader`` is checked before every mutating call. ``now``/``sleep`` + are wall-clock (``deadline_s`` is epoch seconds). ``crn_min_version`` is + the oldest runtime that replays traces with common random numbers. + """ + + dgd: DgdRef + cfg: DgdrExecutorConfig + retune: RetuneConfig + backend: str + kube: RetuneKube + registry: SweepRegistry + cut_window: WindowCutter + is_leader: Callable[[], bool] + now: Callable[[], float] = time.time + sleep: Callable[[float], None] = time.sleep + poll_interval_s: float = 5.0 + crn_min_version: Optional[str] = CRN_MIN_VERSION + + +def run_dgdr_sweep( + request: SweepRequest, + *, + cancel: threading.Event, + abort: Optional[threading.Event], + context: DgdrContext, +) -> SweepResult: + """Sweep one window through DGDR v1beta2 Requests; never raises. + + Scores are keyed by ``candidate.key()`` for every candidate of + ``request`` (live router names, no planner part); a candidate no Run + published scores ``None``. See the module docstring for the protocol. + """ + + started = context.now() + try: + return _DgdrSweep(request, cancel, abort, context, started).run() + except Exception as error: # defensive: the executor must never raise + logger.exception("retune DGDR sweep of window %s failed", request.window_index) + return SweepResult( + status="failed", + wall_s=max(0.0, context.now() - started), + error=f"{type(error).__name__}: {error}", + failure_reason="internal_error", + details={}, + ) + + +class _Interrupted(Exception): + """Shutdown or loss of leadership: return without a decision.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +class _WindowFailed(Exception): + """The window ends without scores (``failed``, ``timeout`` or ``empty``).""" + + def __init__( + self, status: SweepStatus, reason: Optional[str], message: str + ) -> None: + super().__init__(message) + self.status = status + self.reason = reason + self.message = message + + +class _EitherEvent(threading.Event): + """An event that reads as set once any of its sources is set. + + Hands the window cutter one ``cancel`` for both shutdown and abort. + """ + + def __init__(self, *sources: Optional[threading.Event]) -> None: + super().__init__() + self._sources = [source for source in sources if source is not None] + + def is_set(self) -> bool: + return super().is_set() or any(source.is_set() for source in self._sources) + + isSet = is_set + + def wait(self, timeout: Optional[float] = None) -> bool: + end = None if timeout is None else time.monotonic() + timeout + while not self.is_set(): + remaining = None if end is None else end - time.monotonic() + if remaining is not None and remaining <= 0: + return False + super().wait(0.05 if remaining is None else min(0.05, remaining)) + return True + + +ShardState = Literal["create", "unknown", "wait", "done", "missing"] + + +@dataclass +class _ShardRun: + """Progress of one shard's Request within this sweep.""" + + shard: Shard + critical: bool + spec: dict + digest: str + attempt: int = 1 + state: ShardState = "create" + name: Optional[str] = None + adopted: bool = False + # This sweep sent a create for the current attempt that may have landed; + # False for an attempt a previous leader created. + created: bool = False + reason: Optional[str] = None + message: Optional[str] = None + run_name: Optional[str] = None + run_uid: Optional[str] = None + checked_run_uid: Optional[str] = None + unaccepted_since: Optional[float] = None + pending_since: Optional[float] = None + run_pending: bool = False + rows: Optional[dict] = None + published: int = 0 + + def summary(self) -> dict[str, Any]: + return { + "kind": self.shard.kind, + "points": self.shard.size, + "critical": self.critical, + "request": self.name, + "attempt": self.attempt, + "adopted": self.adopted, + "run": self.run_name, + "state": self.state, + "reason": self.reason, + "message": self.message, + "published": self.published, + "run_pending": self.run_pending, + } + + +@dataclass +class _Scored: + score: Optional[float] + entry: dict[str, Any] = field(default_factory=dict) + + +class _DgdrSweep: + def __init__( + self, + request: SweepRequest, + cancel: threading.Event, + abort: Optional[threading.Event], + context: DgdrContext, + started: float, + ) -> None: + self.request = request + self.cancel = cancel + self.abort = abort + self.ctx = context + self.cfg = context.cfg + self.started = started + deadline = getattr(request, "deadline_s", None) + self.deadline = ( + float(deadline) if deadline is not None else started + request.timeout_s + ) + self.window_end_ms = int(round(request.window_end_ms)) + # Registry references this sweep took; each is released at the end. + self._protections = 0 + self.claim_path = window_claim_path(self.cfg, context.dgd, self.window_end_ms) + self.local_path = os.path.join(self.cfg.mount_path, self.claim_path) + self.shards: dict[str, _ShardRun] = {} + self.cut_done = False + self.num_requests = 0 + self.row_counts: dict[str, int] = {} + self.details: dict[str, Any] = { + "window": {"path": self.claim_path, "cut": False}, + "deadline_s": self.deadline, + "generation": getattr(request, "generation", 0), + "replicas_verified": getattr(request, "replicas_verified", True), + } + self.k8s_errors = 0 + self.runtime: dict[str, Any] = {} + + # ------------------------------------------------------------ driver + + def run(self) -> SweepResult: + try: + self._prepare() + self._check_interrupt() + self._adopt() + self._create_all() + self._poll() + return self._harvest() + except _Interrupted as stop: + logger.info( + "retune window %s sweep stopped (%s); its Requests stay for the " + "next leader", + self.request.window_index, + stop.reason, + ) + return self._result( + "failed", + failure_reason=stop.reason, + error=f"sweep stopped: {stop.reason}", + ) + except _WindowFailed as failed: + self._abandon_in_flight() + log = logger.info if failed.status == "empty" else logger.warning + log( + "retune window %s %s (%s): %s", + self.request.window_index, + failed.status, + failed.reason, + failed.message, + ) + return self._result( + failed.status, + failure_reason=failed.reason, + error=failed.message if failed.status != "empty" else None, + ) + finally: + # One release per protect: the registry counts references per + # window, and another generation's sweep may hold its own. + try: + for _ in range(self._protections): + self.ctx.registry.release(self.window_end_ms) + except Exception: # pragma: no cover - registry is in-process + logger.exception("retune registry release failed") + self._protections = 0 + + def _protect(self, names: list[str]) -> None: + self.ctx.registry.protect(self.window_end_ms, names) + self._protections += 1 + + def _result( + self, + status: SweepStatus, + *, + failure_reason: Optional[str] = None, + error: Optional[str] = None, + scores: Optional[dict[str, Optional[float]]] = None, + metrics: Optional[dict[str, dict[str, Any]]] = None, + ) -> SweepResult: + details = dict(self.details) + details["shards"] = {key: run.summary() for key, run in self.shards.items()} + details["k8s_errors"] = self.k8s_errors + details.update(self.runtime) + return SweepResult( + status=status, + num_requests=self.num_requests, + row_counts=dict(self.row_counts), + scores=scores or {}, + metrics=metrics or {}, + wall_s=max(0.0, self._now() - self.started), + error=error, + failure_reason=failure_reason, + details=details, + ) + + # ----------------------------------------------------------- helpers + + def _now(self) -> float: + return self.ctx.now() + + def _timeout(self) -> float: + return max(_MIN_K8S_TIMEOUT_S, min(_K8S_TIMEOUT_S, self.deadline - self._now())) + + def _check_interrupt(self) -> None: + if self.cancel.is_set(): + raise _Interrupted("shutdown") + if self.abort is not None and self.abort.is_set(): + raise _Interrupted("lost_leadership") + + def _check_mutation(self) -> None: + self._check_interrupt() + if not self.ctx.is_leader(): + raise _Interrupted("lost_leadership") + + def _fail( + self, reason: Optional[str], message: str, status: SweepStatus = "failed" + ) -> NoReturn: + raise _WindowFailed(status, reason, message) + + def _retrying(self, what: str, call: Callable[[], Any]) -> Any: + """``call`` until it succeeds; a permanent error or the deadline fails.""" + + while True: + self._check_interrupt() + try: + return call() + except KubeError as error: + self.k8s_errors += 1 + if _permanent(error): + self._fail("k8s_error", f"{what}: {error}") + if self._now() + self.ctx.poll_interval_s >= self.deadline: + self._fail("k8s_error", f"{what}: {error}", "timeout") + logger.warning("retune %s failed, retrying: %s", what, error) + self.ctx.sleep(self.ctx.poll_interval_s) + + def _read_failed(self, what: str, error: KubeError) -> None: + """A GET failed: a permanent error fails the window; else poll again.""" + + self.k8s_errors += 1 + if _permanent(error): + self._fail("k8s_error", f"{what}: {error}") + logger.warning("retune %s failed: %s", what, error) + + def _enqueue_delete(self, names: Iterable[Optional[str]]) -> None: + present = [name for name in names if name] + if present: + self.ctx.registry.enqueue_delete(present) + + def _abandon_in_flight(self) -> None: + """Queue this window's unfinished Requests for deletion.""" + + self._enqueue_delete( + run.name + for run in self.shards.values() + if run.state in ("wait", "unknown") and run.name + ) + + # ------------------------------------------------------------- setup + + def _prepare(self) -> None: + request = self.request + domain = getattr(request, "router_domain", None) + if not domain: + self._fail("invalid_request", "the sweep request has no router domain") + if any(candidate.planner for candidate in request.candidates): + self._fail( + "invalid_request", + "a DGDR sweep searches router parameters only (Scenario 1)", + ) + keys = set(domain) + startup = self._startup_point(domain) + try: + shards = plan_shards( + domain, + baseline=request.baseline.router, + startup=startup, + max_shards=self.cfg.run.max_shards, + parallelism=self.cfg.run.parallelism, + ) + except (KeyError, ValueError) as error: + self._fail("invalid_domain", str(error)) + replicas = getattr(request, "spec_replicas", None) or request.replicas + self.details["replicas"] = dict(replicas) + probe = next((shard for shard in shards if shard.kind == "probe"), None) + for shard in shards: + try: + body = build_request( + dgd=self.ctx.dgd, + cfg=self.cfg, + retune=self.ctx.retune, + sla_ttft_ms=request.sla_ttft_ms, + sla_itl_ms=request.sla_itl_ms, + window_end_ms=self.window_end_ms, + shard=shard, + spec_replicas=replicas, + backend=self.ctx.backend, + attempt=1, + ) + except (KeyError, TypeError, ValueError) as error: + self._fail("invalid_engine", f"shard {shard.id}: {error}") + # The baseline must be scored, and the probe's grid twin is what + # the probe is compared with. + critical = ( + shard.kind == "probe" + or shard.contains(request.baseline.router) + or (probe is not None and shard.contains(probe.points[0])) + ) + self.shards[shard.id] = _ShardRun( + shard=shard, + critical=critical, + spec=body["spec"], + digest=body["metadata"]["labels"][LABEL_SPEC], + ) + uncovered = [ + candidate.router + for candidate in request.candidates + if set(candidate.router) != keys + or not any( + run.shard.kind != "probe" and run.shard.contains(candidate.router) + for run in self.shards.values() + ) + ] + if uncovered: + self.details["uncovered_candidates"] = uncovered[:10] + + def _startup_point(self, domain: Mapping[str, Sequence[float]]) -> dict[str, float]: + """The off-grid candidate other than the baseline, else the baseline. + + The controller's candidates are the grid plus the startup point and + the baseline, so an off-grid candidate is one of those two. + """ + + baseline = self.request.baseline + for candidate in self.request.candidates: + router = candidate.router + if candidate == baseline or set(router) != set(domain): + continue + if not all(float(router[key]) in domain[key] for key in domain): + return dict(router) + return dict(baseline.router) + + def _build(self, run: _ShardRun) -> dict: + body = build_request( + dgd=self.ctx.dgd, + cfg=self.cfg, + retune=self.ctx.retune, + sla_ttft_ms=self.request.sla_ttft_ms, + sla_itl_ms=self.request.sla_itl_ms, + window_end_ms=self.window_end_ms, + shard=run.shard, + spec_replicas=self.details["replicas"], + backend=self.ctx.backend, + attempt=run.attempt, + ) + rows = {"num_requests": self.num_requests, "row_counts": self.row_counts} + body["metadata"]["annotations"] = { + ROWS_ANNOTATION: json.dumps(rows, sort_keys=True, separators=(",", ":")) + } + return body + + # ------------------------------------------------------------- adopt + + def _adopt(self) -> None: + """Adopt the latest attempt of each shard; queue the rest for deletion. + + Each adopted attempt is observed once here: a Succeeded one needs no + cut, and a failed one is replaced while there is time (``_run_failed``). + """ + + selector = ( + f"{LABEL_DGD_UID}={self.ctx.dgd.uid},{LABEL_WINDOW}={self.window_end_ms}" + ) + existing = self._retrying( + "list window Requests", + lambda: self.ctx.kube.list_requests(selector, timeout_s=self._timeout()), + ) + stale: list[Optional[str]] = [] + latest: dict[str, dict] = {} + for obj in existing: + metadata = obj.get("metadata") or {} + labels = metadata.get("labels") or {} + name = metadata.get("name") + run = self.shards.get(labels.get(LABEL_SHARD, "")) + if run is None or labels.get(LABEL_SPEC) != run.digest: + stale.append(name) + continue + attempt = _int(labels.get(LABEL_ATTEMPT), 0) + if metadata.get("deletionTimestamp"): + # Being deleted: never adopt it, and never reuse its attempt. + run.attempt = max(run.attempt, attempt + 1) + continue + current = latest.get(run.shard.id) + if current is None or attempt > _int( + current["metadata"]["labels"].get(LABEL_ATTEMPT), 0 + ): + if current is not None: + stale.append(current["metadata"]["name"]) + latest[run.shard.id] = obj + else: + stale.append(name) + self._enqueue_delete(stale) + for shard_id, obj in latest.items(): + run = self.shards[shard_id] + metadata = obj["metadata"] + run.name = metadata["name"] + run.attempt = max( + run.attempt, _int(metadata["labels"].get(LABEL_ATTEMPT), 1) + ) + run.adopted = True + run.state = "wait" + run.rows = _rows(metadata.get("annotations")) + logger.info( + "retune window %s adopts Request %s for shard %s", + self.request.window_index, + run.name, + shard_id, + ) + if latest: + self._protect([obj["metadata"]["name"] for obj in latest.values()]) + # Classify the adopted Requests: a window whose shards all + # succeeded is harvested without cutting it again. + for run in self.shards.values(): + if run.state == "wait": + self._observe(run) + + # ------------------------------------------------------------ create + + def _create_all(self) -> None: + pending = [run for run in self._ordered() if run.state == "create"] + if not pending: + rows = next( + (run.rows for run in self.shards.values() if run.rows is not None), + None, + ) + if rows is not None: + self.num_requests = _int(rows.get("num_requests"), 0) + self.row_counts = dict(rows.get("row_counts") or {}) + return + # Gate every shard before the cut, so a critical shard that cannot + # finish fails the window without spending the cut budget. + for run in pending: + self._too_late(run) + for run in pending: + if run.state == "create": + self._create(run) + + def _ordered(self) -> list[_ShardRun]: + """Critical shards first, so they get the time that is left.""" + + return sorted(self.shards.values(), key=lambda run: not run.critical) + + def _too_late(self, run: _ShardRun) -> bool: + """Apply the launch gate; True when the shard was given up.""" + + need = need_seconds(run.shard, self.cfg, needs_cut=not self.cut_done) + left = self.deadline - self._now() + if left >= need: + return False + message = ( + f"shard {run.shard.id} needs {need:.0f}s but {max(left, 0.0):.0f}s are " + "left before the deadline" + ) + if run.critical: + self._fail("launch_too_late", message, "timeout") + run.state = "missing" + run.reason = "launch_too_late" + run.message = message + logger.warning("retune window %s: %s", self.request.window_index, message) + return True + + def _create(self, run: _ShardRun) -> None: + self._check_mutation() + if self._too_late(run): + return + if not self.cut_done: + self._cut() + if self._too_late(run): + return + self._check_mutation() + body = self._build(run) + name = body["metadata"]["name"] + run.name = name + run.rows = None + self._protect([name]) + try: + created = self.ctx.kube.create_request(body, timeout_s=self._timeout()) + except KubeAlreadyExists: + self._adopt_existing(run) + return + except KubeError as error: + self.k8s_errors += 1 + if _admission_denied(error): + self._fail("request_rejected", f"Request {name} rejected: {error}") + if isinstance(error, KubeInvalid): + self._fail("schema_unsupported", f"Request {name} rejected: {error}") + if _permanent(error): + self._fail("k8s_error", f"create Request {name}: {error}") + # The create may have landed; look it up on the next poll. + run.created = True + run.state = "unknown" + run.message = f"create: {error}" + logger.warning("retune create of Request %s failed: %s", name, error) + return + run.created = True + dropped = missing_fields(created.get("spec"), body["spec"], "spec") + labels = (created.get("metadata") or {}).get("labels") or {} + dropped += [ + f"metadata.labels.{key}" + for key, value in body["metadata"]["labels"].items() + if labels.get(key) != value + ] + if dropped: + self._enqueue_delete([name]) + run.state = "missing" + self._fail( + "schema_unsupported", + f"the API server dropped or changed {sorted(dropped)} of Request {name}", + ) + run.state = "wait" + run.unaccepted_since = self._now() + logger.info( + "retune window %s created Request %s (shard %s, %d points, attempt %d)", + self.request.window_index, + name, + run.shard.id, + run.shard.size, + run.attempt, + ) + + def _adopt_existing(self, run: _ShardRun) -> None: + """The name exists (an earlier create landed): adopt it if it is ours.""" + + existing = self._retrying( + f"get Request {run.name}", + lambda: self.ctx.kube.get_request(run.name, timeout_s=self._timeout()), + ) + if existing is None: + # Deleted in between; create it again on the next poll. + run.state = "create" + return + labels = (existing.get("metadata") or {}).get("labels") or {} + if ( + labels.get(LABEL_SPEC) == run.digest + and labels.get(LABEL_DGD_UID) == self.ctx.dgd.uid + and labels.get(LABEL_SHARD) == run.shard.id + ): + run.state = "wait" + run.adopted = True + run.unaccepted_since = self._now() + return + self._shard_failed( + run, "request_conflict", f"Request {run.name} exists with other labels" + ) + + # ------------------------------------------------------------ window + + def _cut(self) -> None: + """Cut the window file; an empty window ends the sweep as ``empty``.""" + + self._check_interrupt() + segments_root = ( + self.request.trace_paths[0] + if self.request.trace_paths + else os.path.join( + self.cfg.mount_path, self.cfg.segments_dir, self.ctx.dgd.name + ) + ) + budget = min(self.cfg.cut_budget_seconds, self.deadline - self._now()) + if budget <= 0: + self._fail("launch_too_late", "no time left to cut the window", "timeout") + # WindowCutter takes the deadline as absolute wall-clock seconds. + cut_deadline_s = self._now() + budget + cancel = _EitherEvent(self.cancel, self.abort) + try: + os.makedirs(os.path.dirname(self.local_path), exist_ok=True) + result: WindowCutResult = self.ctx.cut_window( + segments_root, + int(round(self.request.window_start_ms)), + self.window_end_ms, + int(round(self.request.as_of_ms)), + self.local_path, + max_rows=self.cfg.max_window_rows, + deadline_s=cut_deadline_s, + cancel=cancel, + ) + except Exception as error: + # A cut cancelled by shutdown or abort reports which one it was. + self._check_interrupt() + reason = getattr(error, "reason", None) + if reason not in ( + "window_cut_timeout", + "window_too_large", + "trace_corrupt", + "window_cut_failed", + ): + reason = "window_cut_failed" + self._fail(reason, f"window cut: {type(error).__name__}: {error}") + self.cut_done = True + self.num_requests = int(result.num_requests) + self.row_counts = dict(result.row_counts) + self.details["window"].update( + cut=True, + num_requests=result.num_requests, + reused=result.reused, + dropped_members=result.dropped_members, + dropped_bytes=result.dropped_bytes, + trace_block_size=result.trace_block_size, + ) + if result.path is None or result.num_requests == 0: + self._fail(None, "empty window", "empty") + + # -------------------------------------------------------------- wait + + def _poll(self) -> None: + while True: + self._check_interrupt() + active = [ + run + for run in self._ordered() + if run.state in ("create", "unknown", "wait") + ] + if not active: + return + if self._now() >= self.deadline: + pending = [run.shard.id for run in active if run.run_pending] + self._fail( + "run_pending" if pending else "timeout", + "the deadline passed with shards " + f"{sorted(run.shard.id for run in active)} unfinished" + + (f" (Runs pending: {sorted(pending)})" if pending else ""), + "timeout", + ) + for run in active: + self._check_interrupt() + if run.state == "create": + self._create(run) + elif run.state == "unknown": + self._resolve_unknown(run) + else: + self._observe(run) + if any( + run.state in ("create", "unknown", "wait") + for run in self.shards.values() + ): + left = self.deadline - self._now() + self.ctx.sleep(max(0.0, min(self.ctx.poll_interval_s, left))) + + def _resolve_unknown(self, run: _ShardRun) -> None: + """After a create whose outcome is unknown: adopt it or create again.""" + + try: + existing = self.ctx.kube.get_request(run.name, timeout_s=self._timeout()) + except KubeError as error: + self._read_failed(f"get Request {run.name}", error) + return + if existing is None: + run.state = "create" + return + self._adopt_existing(run) + + def _shard_failed(self, run: _ShardRun, reason: str, message: str) -> None: + """A shard ended without scores; its Request (if any) is kept for audit. + + A critical shard fails the window; any other leaves its points unscored. + """ + + run.state = "missing" + run.reason = reason + run.message = message + if run.critical: + self._fail(reason, f"shard {run.shard.id}: {message}") + logger.warning( + "retune window %s shard %s %s: %s; its points are unscored", + self.request.window_index, + run.shard.id, + reason, + message, + ) + + def _observe(self, run: _ShardRun) -> None: + """One look at a shard's Request and Run (the §4.5.3 state machine).""" + + kube = self.ctx.kube + try: + obj = kube.get_request(run.name, timeout_s=self._timeout()) + except KubeError as error: + self._read_failed(f"get Request {run.name}", error) + return + if obj is None or (obj.get("metadata") or {}).get("deletionTimestamp"): + self._shard_failed(run, "request_deleted", f"Request {run.name} is gone") + return + if run.rows is None: + run.rows = _rows((obj.get("metadata") or {}).get("annotations")) + generation = _int((obj.get("metadata") or {}).get("generation"), 1) + accepted = _condition(obj, "Accepted") + if accepted is None or _int(accepted.get("observedGeneration"), 0) < generation: + if run.unaccepted_since is None: + run.unaccepted_since = self._now() + elif self._now() - run.unaccepted_since >= ACCEPT_TIMEOUT_S: + self._fail( + "operator_not_reconciling", + f"Request {run.name} has no Accepted condition after " + f"{ACCEPT_TIMEOUT_S:.0f}s", + ) + return + run.unaccepted_since = None + if accepted.get("status") != "True": + if accepted.get("reason") == "InvalidSpec": + self._fail( + "invalid_spec", + f"Request {run.name}: {accepted.get('message', '')}", + ) + # RunCreating, RunNameConflict, CancelingSupersededRun: wait. + return + try: + found = self._find_run(obj) + except KubeError as error: + self._read_failed(f"get the Run of Request {run.name}", error) + return + if found is None: + return # created, not yet visible + run.run_name = found["metadata"]["name"] + run_uid = found["metadata"].get("uid") + status = found.get("status") or {} + completed = _condition(found, "Completed") or {} + terminal = completed.get("status") == "True" + succeeded = terminal and completed.get("reason") == "SearchSucceeded" + outcome = ( + f"Run {run.run_name} {completed.get('reason')}: " + f"{completed.get('message', '')}" + ) + if not terminal: + self._track_pending(run, status) + if run.checked_run_uid != run_uid: + effective = _get(status, ("aisimulate", "input")) + if not isinstance(effective, Mapping): + if not terminal: + return # the input is recorded together with jobRef + if not succeeded: + self._run_failed(run, outcome) + return + self._input_mismatch(run, ["status.aisimulate.input is missing"]) + return + if not self._check_input(run, effective): + return + run.checked_run_uid = run_uid + if succeeded: + run.state = "done" + run.run_uid = run_uid + elif terminal: + self._run_failed(run, outcome) + + def _track_pending(self, run: _ShardRun, status: Mapping[str, Any]) -> None: + """Flag a live Run that has had no Job for a long time (``run_pending``). + + The operator writes ``status.aisimulate.input`` and ``jobRef`` in one + status patch once the Job exists; until then the Run has no status + at all, so this runs before the input is read back. + """ + + if status.get("jobRef") or status.get("startTime"): + run.pending_since = None + run.run_pending = False + return + now = self._now() + if run.pending_since is None: + run.pending_since = now + elif now - run.pending_since >= max( + self.cfg.job_startup_seconds, self.ctx.poll_interval_s + ): + if not run.run_pending: + logger.warning( + "retune Run %s has no Job after %.0fs (run_pending)", + run.run_name, + now - run.pending_since, + ) + run.run_pending = True + + def _run_failed(self, run: _ShardRun, outcome: str) -> None: + """A Run ended without success. + + An attempt a previous leader created is replaced while there is time + (design §4.5.3 adopts only a live or Succeeded attempt); otherwise + the shard fails. + """ + + if run.adopted and not run.created: + name = run.name + if self._replace_attempt(run, outcome): + logger.warning( + "retune window %s replaces adopted Request %s of shard %s: %s", + self.request.window_index, + name, + run.shard.id, + outcome, + ) + return + self._shard_failed(run, "run_failed", outcome) + + def _find_run(self, obj: Mapping[str, Any]) -> Optional[dict]: + """The Run of the Request's current generation, by the operator's refs.""" + + metadata = obj.get("metadata") or {} + status = obj.get("status") or {} + generation = _int(metadata.get("generation"), 1) + names: list[str] = [] + active = (status.get("activeRunRef") or {}).get("name") + if active: + names.append(active) + names += [ + ref.get("name") + for ref in status.get("recentRunRefs") or () + if ref.get("name") + ] + names.append(f"{metadata.get('name')}-{generation}") + uid = metadata.get("uid") + for name in dict.fromkeys(names): + found = self.ctx.kube.get_run(name, timeout_s=self._timeout()) + if found is None: + continue + run_meta = found.get("metadata") or {} + annotations = run_meta.get("annotations") or {} + owners = run_meta.get("ownerReferences") or () + if annotations.get(REQUEST_GENERATION_ANNOTATION) == str( + generation + ) and any(owner.get("uid") == uid for owner in owners): + return found + return None + + def _check_input(self, run: _ShardRun, effective: Mapping[str, Any]) -> bool: + """Read back the Run's lowered input; False when the shard moved on.""" + + dgdr_run = effective.get("dgdr_run") or {} + image = dgdr_run.get("runtime_image") + version = dgdr_run.get("runtime_version_override") + if image or version: + self.runtime = {"runtime_image": image, "runtime_version": version} + problems = input_mismatches(run.spec, effective) + if problems: + self._input_mismatch(run, problems) + return False + if crn_unsupported(version, image, self.ctx.crn_min_version): + self._fail( + "crn_unsupported", + f"Run {run.run_name} runtime {version or image} predates CRN " + f"trace replay ({self.ctx.crn_min_version})", + ) + return True + + def _input_mismatch(self, run: _ShardRun, problems: list[str]) -> None: + """An adopted attempt is replaced while there is time; else it fails.""" + + message = f"Run {run.run_name} input differs: {'; '.join(problems[:5])}" + logger.warning("retune window %s: %s", self.request.window_index, message) + if run.adopted and self._replace_attempt(run, message): + return + self._enqueue_delete([run.name]) + run.state = "missing" + if run.critical: + self._fail("input_mismatch", message) + run.reason = "input_mismatch" + run.message = message + + def _replace_attempt(self, run: _ShardRun, message: str) -> bool: + """Queue the attempt for deletion and create attempt+1, if it fits. + + False (nothing changed) when ``D_k - now < need(shard)``. + """ + + need = need_seconds(run.shard, self.cfg, needs_cut=not self.cut_done) + if self.deadline - self._now() < need: + return False + self._enqueue_delete([run.name]) + run.attempt += 1 + run.state = "create" + run.adopted = False + run.created = False + run.name = None + run.run_name = None + run.checked_run_uid = None + run.unaccepted_since = None + run.pending_since = None + run.run_pending = False + run.message = message + return True + + # ----------------------------------------------------------- harvest + + def _harvest(self) -> SweepResult: + request = self.request + domain_keys = set(getattr(request, "router_domain", None) or {}) + results: dict[str, _Scored] = {} + probe_results: dict[str, _Scored] = {} + engine_signature: Optional[str] = None + signature_source: Optional[str] = None + unmapped: list[str] = [] + for run in self._ordered(): + if run.state != "done": + continue + selector = f"{RUN_UID_LABEL}={run.run_uid}" + candidates = self._retrying( + f"list Candidates of Run {run.run_name}", + functools.partial(self._list_candidates, selector), + ) + run.published = len(candidates) + for candidate in sorted( + candidates, key=lambda obj: (obj.get("metadata") or {}).get("name", "") + ): + name = (candidate.get("metadata") or {}).get("name", "") + parameters = _get(candidate, ("spec", "parameters")) + try: + point, signature = _map_parameters(parameters, run.shard) + except _Unmapped as error: + if error.fatal: + self._fail("engine_not_pinned", f"Candidate {name}: {error}") + unmapped.append(f"{name}: {error}") + continue + if engine_signature is None: + engine_signature, signature_source = signature, name + elif signature != engine_signature: + self._fail( + "engine_not_pinned", + f"Candidates {signature_source} and {name} differ in " + "non-router parameters", + ) + key = RetuneParams(router=point).key() + target = probe_results if run.shard.kind == "probe" else results + if key in target: + continue + metrics = _get(candidate, ("status", "metrics"), {}) or {} + entry: dict[str, Any] = { + "params": RetuneParams(router=point).to_dict(), + "shard": run.shard.id, + "candidate": name, + } + try: + score, info = score_summary( + metrics, request.window_s, request.objective + ) + except ValueError as error: + entry["error"] = str(error) + target[key] = _Scored(None, entry) + continue + entry.update(info) + target[key] = _Scored(score, entry) + if unmapped: + self.details["unmapped_candidates"] = unmapped[:10] + self._check_probe(results, probe_results) + + scores: dict[str, Optional[float]] = {} + metrics_out: dict[str, dict[str, Any]] = {} + missing = 0 + for candidate in request.candidates: + key = candidate.key() + found = results.get(key) + if found is not None and set(candidate.router) == domain_keys: + scores[key] = found.score + metrics_out[key] = found.entry + else: + scores[key] = None + metrics_out[key] = { + "params": candidate.to_dict(), + "error": self._missing_reason(candidate.router), + } + if scores[key] is None: + missing += 1 + self.details["missing_candidates"] = missing + logger.info( + "retune window %s harvested %d of %d candidates from %d Requests", + request.window_index, + len(scores) - missing, + len(scores), + sum(1 for run in self.shards.values() if run.state == "done"), + ) + return self._result("ok", scores=scores, metrics=metrics_out) + + def _list_candidates(self, selector: str) -> list[dict]: + return self.ctx.kube.list_candidates(selector, timeout_s=self._timeout()) + + def _missing_reason(self, point: Mapping[str, float]) -> str: + for run in self.shards.values(): + if run.shard.kind != "probe" and run.shard.contains(point): + if run.state == "missing": + return f"shard {run.shard.id} {run.reason}: {run.message}" + return f"not published by Run {run.run_name} (infeasible or failed)" + return "not covered by any shard" + + def _check_probe( + self, results: Mapping[str, _Scored], probe_results: Mapping[str, _Scored] + ) -> None: + probe = self.shards.get("probe") + if probe is None: + return + point = probe.shard.points[0] + key = RetuneParams(router=point).key() + grid = results.get(key) + repeat = probe_results.get(key) + grid_score = None if grid is None else grid.score + probe_score = None if repeat is None else repeat.score + report: dict[str, Any] = { + "point": point, + "grid_score": grid_score, + "probe_score": probe_score, + } + self.details["crn_probe"] = report + if grid_score is None or probe_score is None: + self._fail( + "crn_mismatch", + f"CRN probe point {point} is unscored (grid {grid_score}, " + f"probe {probe_score})", + ) + scale = max(abs(grid_score), abs(probe_score)) + error = 0.0 if scale == 0 else abs(grid_score - probe_score) / scale + report["relative_error"] = error + if not error <= CRN_REL_TOLERANCE: + self._fail( + "crn_mismatch", + f"CRN probe point {point} scored {probe_score!r}, its grid " + f"evaluation {grid_score!r} (relative error {error:.3g})", + ) + + +class _Unmapped(Exception): + """A Candidate that maps to no point; ``fatal`` when the engine drifted.""" + + def __init__(self, message: str, *, fatal: bool = False) -> None: + super().__init__(message) + self.fatal = fatal + + +def _map_parameters(parameters: Any, shard: Shard) -> tuple[dict[str, float], str]: + """A Candidate's live router point and its non-router signature. + + The signature is the canonical JSON of the parameters without the three + searched knobs; every Candidate of a window must share it. Raises + ``_Unmapped`` for parameters that name no point of ``shard``, fatally + for a router other than the pinned ``kv_router`` without a load model. + """ + + if not isinstance(parameters, Mapping): + raise _Unmapped("spec.parameters is not an object") + adapters = parameters.get("adapters") + if not isinstance(adapters, Mapping): + raise _Unmapped("spec.parameters.adapters is missing") + router = adapters.get(ROUTER_ADAPTER) + if not isinstance(router, Mapping): + raise _Unmapped(f"spec.parameters.adapters[{ROUTER_ADAPTER!r}] is missing") + load_model = router.get("prefill_load_model") + load_type = ( + load_model.get("type") if isinstance(load_model, Mapping) else load_model + ) + if router.get("policy") != "kv_router" or load_type != "none": + raise _Unmapped( + f"router policy {router.get('policy')!r} with prefill load model " + f"{load_type!r}; the Request pinned kv_router and none", + fatal=True, + ) + point: dict[str, float] = {} + for public, live in DGDR_ROUTER_KNOBS.items(): + value = router.get(public) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise _Unmapped(f"router knob {public} is {value!r}") + snapped = _snap(float(value), shard.choices.get(live, ())) + if snapped is None: + raise _Unmapped( + f"router knob {public}={value!r} is not one of the shard's values" + ) + point[live] = snapped + rest = dict(parameters) + rest["adapters"] = { + **adapters, + ROUTER_ADAPTER: { + key: value for key, value in router.items() if key not in DGDR_ROUTER_KNOBS + }, + } + signature = json.dumps(rest, sort_keys=True, separators=(",", ":"), default=str) + return point, signature + + +def _snap(value: float, values: Sequence[float]) -> Optional[float]: + """The domain value ``value`` denotes, allowing for float round-trips.""" + + for candidate in values: + if abs(value - candidate) <= _SNAP_TOLERANCE * max(1.0, abs(candidate)): + return candidate + return None + + +def _condition(obj: Mapping[str, Any], kind: str) -> Optional[dict]: + for condition in _get(obj, ("status", "conditions"), ()) or (): + if isinstance(condition, Mapping) and condition.get("type") == kind: + return dict(condition) + return None + + +def _permanent(error: KubeError) -> bool: + """A Kubernetes error a retry cannot fix (bad request, auth, missing API).""" + + return getattr(error, "status", None) in _PERMANENT_STATUSES or isinstance( + error, (KubeInvalid, KubeNotFound) + ) + + +def _admission_denied(error: KubeError) -> bool: + """A permanent rejection whose message comes from an admission webhook.""" + + return _permanent(error) and _ADMISSION_DENIED.search(str(error)) is not None + + +def _int(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _rows(annotations: Any) -> Optional[dict]: + if not isinstance(annotations, Mapping) or ROWS_ANNOTATION not in annotations: + return None + try: + rows = json.loads(annotations[ROWS_ANNOTATION]) + except (TypeError, ValueError): + return None + return rows if isinstance(rows, dict) else None diff --git a/components/src/dynamo/planner/retune/live/frontend_config.py b/components/src/dynamo/planner/retune/live/frontend_config.py new file mode 100644 index 000000000000..76485dbb7f10 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/frontend_config.py @@ -0,0 +1,1794 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Frontend router startup parsed from the DGD spec, and the spec premises. + +Live retune replays each window in DGDR with AISimulate's router defaults +plus the searched knobs, while the live frontends start from whatever their +argv and env say. The two agree only when the frontend leaves every +unsearched routing field at its default (premise P4), so the planner reads +the frontend's startup from the DynamoGraphDeployment spec, never from a +running pod (design §4.3): + +- ``find_frontend`` picks the one standalone frontend component. +- ``parse_frontend`` rebuilds the frontend's effective argv (the operator's + ``command``/``args`` defaulting) and env (operator-injected, graph ``env`` + and the main container's, with Kubernetes ``$(VAR)`` expansion), then + parses them with the frontend's own argument groups. Those groups read + ``os.environ`` while registering their arguments; here they run as copies + of their functions whose ``os`` sees only the frontend's env, so the + planner process's environment is neither read nor modified. A + shell-wrapped command, a ``--router-policy-config`` file, router env set + through ``valueFrom`` and an argument whose unreadable ``$(VAR)`` could + set a router flag make the frontend unparseable. ``envFrom`` sources are + assumed to set no router env; ``evaluate_premises`` warns about them. +- ``evaluate_premises`` checks P4 and P7-P10 and warns on P6 and P7; P14 + is the router actuator's. P4 also parses every worker component's router + flags and env the same way: a worker that resolves a router mode + advertises its own router configuration, which replaces the frontend's + for its worker set. ``p4_ok`` alone decides whether a retune controller + can be built. + +Every function is pure (a dict in, a value out, no Kubernetes calls) and +never reads the planner's own environment. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import posixpath +import re +import shlex +import types +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from functools import lru_cache +from typing import TYPE_CHECKING, Any, Optional + +from dynamo.common.configuration.groups.kv_router_args import ( + _KV_ROUTER_FIELDS, + KvRouterConfigBase, +) +from dynamo.common.configuration.groups.router_args import RouterConfigBase +from dynamo.llm import KvRouterConfig +from dynamo.planner.retune.live.contracts import FrontendStartup, PremiseReport + +if TYPE_CHECKING: + from dynamo.planner.config.planner_config import PlannerConfig, RetuneConfig + +logger = logging.getLogger(__name__) + +FRONTEND_TYPE = "frontend" +PLANNER_TYPE = "planner" +EPP_TYPE = "epp" +# Component types that run inference workers; v1beta1 ``type`` is optional. +WORKER_TYPES = frozenset({"worker", "prefill", "decode", ""}) + +MAIN_CONTAINER = "main" +# The operator's frontend invocation (component_frontend.go GetBaseContainer). +DEFAULT_FRONTEND_COMMAND = ("python3",) +DEFAULT_FRONTEND_ARGS = ("-m", "dynamo.frontend") +FRONTEND_MODULE = "dynamo.frontend" + +# Env the operator injects into every component (component_common.go +# getCommonContainer); user env of the same name overrides it. +ENV_PARENT_DGD_NAME = "DYN_PARENT_DGD_K8S_NAME" +ENV_PARENT_DGD_NAMESPACE = "DYN_PARENT_DGD_K8S_NAMESPACE" +_OPERATOR_FIELD_REF_ENV = ("POD_NAME", "POD_NAMESPACE", "POD_UID") + +# The trace recipe of design §4.4.1 (P10). +ENV_POD_NAME = "DYN_POD_NAME" +ENV_TRACE = "DYN_REQUEST_TRACE" +ENV_TRACE_SINKS = "DYN_REQUEST_TRACE_SINKS" +ENV_TRACE_FORMAT = "DYN_REQUEST_TRACE_FILE_FORMAT" +ENV_TRACE_RECORDS = "DYN_REQUEST_TRACE_RECORDS" +ENV_TRACE_PATH = "DYN_REQUEST_TRACE_FILE_PATH" +# lib/truthy is_truthy; lib/llm/src/request_trace/config.rs parse_file_format. +_TRUTHY = frozenset({"1", "true", "on", "yes"}) +_GZ_FORMATS = frozenset({"jsonl_gz", "jsonl.gz"}) + +ENV_WORKER_SELECTION_POLICY = "DYN_ROUTER_WORKER_SELECTION_POLICY" +ENV_LOAD_AWARE = "DYN_ROUTER_LOAD_AWARE" +# A worker that resolves a router mode (router_args.add_worker_router_arguments) +# advertises a RouterConfig in its model card. +ENV_ROUTER_MODE = "DYN_ROUTER_MODE" +_ROUTER_MODE_DEST = "router_mode" +_ROUTER_ENV_PREFIX = "DYN_ROUTER_" +# Router env the Rust router reads directly (no frontend flag) that tunes +# transport or startup only, never which worker is picked. +_TRANSPORT_ROUTER_ENV = frozenset( + { + "DYN_ROUTER_ACTIVE_SEQUENCE_DIRECT_ZMQ", + "DYN_ROUTER_ACTIVE_SEQUENCE_ZMQ_RCVHWM", + "DYN_ROUTER_MODEL_CARD_WAIT_SECS", + "DYN_ROUTER_ZMQ_ENDPOINTS_PER_SUB", + } +) +# The deprecated overlap_score_weight: KvRouterArgGroup reads the first name +# when it is set and the second otherwise. +_OVERLAP_WEIGHT_ENV = ("DYN_ROUTER_KV_OVERLAP_SCORE_WEIGHT", "DYN_OVERLAP_SCORE_WEIGHT") + +# RouterConfig (not KvRouterConfig) fields that change routing or admission: +# session affinity pins requests and the busy thresholds reject them. The +# DGDR replay models neither. +_ROUTER_ADMISSION_FIELDS = ( + "session_affinity_ttl_secs", + "active_decode_blocks_threshold", + "active_prefill_tokens_threshold", + "active_prefill_tokens_threshold_frac", +) +# KvRouterConfig fields serde skips (to_json drops them and from_json +# rejects them), so they are checked on the parsed flags. +_SERDE_SKIPPED_DEFAULTS: dict[str, Any] = { + "router_approximate_cache_policy": "ttl", + "router_prefill_policy": None, + "router_decode_policy": None, +} +# KvRouterConfig JSON keys a frontend may set to a non-default value: they +# do not change which worker the router picks, or another premise covers +# them. +TOLERATED_ROUTER_FIELDS = frozenset( + { + "router_replica_sync", # P8 + "router_event_threads", # indexer thread count + # Hashing of router-derived active-sequence identities. + "router_tracking_hash", + "router_tracking_key_file", + "router_tracking_key_id", + "serve_indexer", # serves this router's indexer to other routers + "enable_session_prefix_index", # tracked, not consumed by routing + "skip_initial_worker_wait", # startup only + } +) +# Fields with their own P4 message. +_LOAD_MODEL = "router_prefill_load_model" +_CONDITIONAL_DISAGG = "conditional_disagg_enabled" + +_TLS_DESTS = ("tls_cert_path", "tls_key_path") +# Parsed destinations whose value the startup depends on. +_RELEVANT_DESTS = frozenset( + { + *_KV_ROUTER_FIELDS, + *_ROUTER_ADMISSION_FIELDS, + *_TLS_DESTS, + "router_mode", + "session_affinity_mode", + "load_aware", + "conditional_disagg_config", + } +) +# runtime_status.go ResolveComponentRuntimeStatus, in its priority order. +_MODEL_FLAGS = ("--served-model-name", "--model-name", "--model", "--model-path") +_SHELLS = frozenset({"sh", "bash", "dash", "zsh", "ash", "ksh"}) +# Python options that may precede ``-m``; the second set takes a value. +_PYTHON_FLAGS = frozenset({"-u", "-B", "-E", "-I", "-O", "-OO", "-P", "-q", "-s", "-S"}) +_PYTHON_FLAGS_WITH_VALUE = frozenset({"-X", "-W"}) +# Marks an argument part that references env the planner cannot read. +_OPAQUE = "\x00opaque:" +_OPAQUE_REPR = repr(_OPAQUE)[1:-1] + +# Receives the frontend argument groups' own log lines (deprecation and +# removed-flag notices), which would otherwise repeat on every parse. +_SILENT_LOGGER = logging.getLogger(f"{__name__}.frontend_args") +_SILENT_LOGGER.disabled = True + + +class FrontendParseError(ValueError): + """The frontend's router configuration cannot be derived from the spec.""" + + +# ------------------------------------------------------------------ API + + +def worker_types(planner_mode: str) -> tuple[str, ...]: + """Router worker types of a planner mode, as the retune controller uses them.""" + + return ("prefill", "decode") if planner_mode == "disagg" else ("aggregated",) + + +def canonical_json(value: Any) -> str: + """Deterministic JSON: sorted keys, no whitespace.""" + + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def find_frontend(dgd: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: + """Return the name and spec of the DGD's one standalone frontend component. + + v1beta1 lists components in ``spec.components``; a v1alpha1 + ``spec.services`` map is converted to the same shape (``type``, + ``replicas``, ``podTemplate.spec`` with a ``main`` container), so callers + read one layout. Raises ``FrontendParseError`` ("P7: ...") unless there + is exactly one standalone frontend, naming an EPP or ``frontendSidecar`` + layout when that is why. + """ + + components = _components(dgd) + frontends = [c for c in components if _type(c) == FRONTEND_TYPE] + if len(frontends) == 1: + return str(frontends[0].get("name", "")), dict(frontends[0]) + if frontends: + names = ", ".join(sorted(str(c.get("name")) for c in frontends)) + raise FrontendParseError( + f"P7: the DGD has {len(frontends)} frontend components ({names}); " + "live retune needs exactly one" + ) + sidecars = _sidecar_components(components) + if sidecars: + raise FrontendParseError( + f"P7: the frontend runs as a frontendSidecar of {', '.join(sidecars)}; " + "live retune needs a standalone frontend component" + ) + if any(_type(c) == EPP_TYPE for c in components): + raise FrontendParseError( + "P7: the DGD routes through an EPP component and has no standalone " + "frontend component" + ) + raise FrontendParseError("P7: the DGD has no frontend component") + + +def served_model_names( + dgd: Mapping[str, Any], +) -> tuple[dict[str, str], tuple[str, ...]]: + """Served model name of each worker component, and the undetermined ones. + + Uses the operator's ``status.components[].servedModelName`` while + the status is current, else the first of ``--served-model-name``, + ``--model-name``, ``--model`` and ``--model-path`` in the main + container's command and args (the operator's order), after Kubernetes + ``$(VAR)`` expansion against the component's env, skipping values that + reference env the planner cannot read. + """ + + projected = _projected_status(dgd) + models: dict[str, str] = {} + unknown: list[str] = [] + for component in _components(dgd): + if _type(component) not in WORKER_TYPES: + continue + name = str(component.get("name", "")) + model = (projected.get(name) or {}).get("servedModelName") or _flag_model( + _model_flags(component, _component_env(dgd, component)) + ) + if model: + models[name] = str(model) + else: + unknown.append(name) + return models, tuple(sorted(unknown)) + + +def served_model_name(dgd: Mapping[str, Any]) -> Optional[str]: + """The DGD's served model when exactly one name is known, else ``None``.""" + + names = set(served_model_names(dgd)[0].values()) + return next(iter(names)) if len(names) == 1 else None + + +def parse_frontend( + dgd: Mapping[str, Any], *, planner_mode: str +) -> FrontendStartup | str: + """Parse the frontend's startup router configuration from the DGD spec. + + Returns the startup, or a ``"P4: ..."`` / ``"P7: ..."`` reason when it + cannot be derived. ``router_config`` holds only the keys whose serde + value differs from ``KvRouterConfig()``; ``startup_maps`` resolves + ``policy_parameters`` for each worker type of ``planner_mode``. A + frontend that parses but violates other P4 conditions (a non-KV router + mode, a non-default field) still returns its startup; see + ``evaluate_premises``. + """ + + analysis = _analyze(dgd, planner_mode) + if analysis.startup is None: + return analysis.error or "P4: the frontend configuration cannot be parsed" + return analysis.startup + + +def evaluate_premises( + dgd: Mapping[str, Any], + *, + planner_config: PlannerConfig, + startup: FrontendStartup | None, + parse_error: str | None, + planner_replicas: int | None, +) -> PremiseReport: + """Evaluate the spec premises P4 and P7-P10, and the P6 and P7 warnings. + + ``startup``/``parse_error`` are ``parse_frontend``'s result for ``dgd``; + what a ``FrontendStartup`` cannot carry (serde-skipped flags, session + affinity, unread router env, TLS, the trace recipe) is re-derived from + ``dgd`` (cached, so cheap). P4 also fails for a worker component that + advertises its own router configuration. ``planner_replicas``, when + given, replaces the planner component replicas read from the DGD (P9). + A malformed DGD never raises; every problem becomes a reason. Raises + ``ValueError`` only when ``planner_config.retune.dgdr`` is unset. + """ + + retune = planner_config.retune + if retune is None or retune.dgdr is None: + raise ValueError("evaluate_premises requires planner_config.retune.dgdr") + dgdr = retune.dgdr + mode = str(planner_config.mode) + analysis = _analyze(dgd, mode) + components = _components(dgd) + p4: list[str] = [] + others: list[str] = [] + warnings: list[str] = [] + + # P4: a parsed KV router whose unsearched fields are the defaults and + # whose startup passes the retune controller's constructor checks. + if startup is None: + p4.append(parse_error or analysis.error or "P4: the frontend was not parsed") + else: + p4.extend(analysis.p4_violations) + p4.extend(_startup_errors(startup, retune, mode)) + if dgdr.startup_router_config is not None: + mismatch = _startup_config_mismatch(dgdr.startup_router_config, startup) + if mismatch is not None: + p4.append(mismatch) + # A worker's advertised router configuration replaces the frontend's for + # its worker set (watcher.rs effective_router_config), whatever the + # frontend's flags say. + p4.extend(_worker_router_errors(dgd, components, warnings)) + + # P7: one served model, one standalone frontend, no EPP, no TLS. + models, unknown = served_model_names(dgd) + distinct = sorted(set(models.values())) + if len(distinct) > 1: + others.append( + f"P7: the DGD serves {len(distinct)} models ({', '.join(distinct)}); " + "live retune needs one" + ) + if unknown: + warnings.append( + f"P7: cannot determine the served model of {', '.join(unknown)}; " + "live retune needs the DGD to serve one model" + ) + # Every DGDR Run simulates retune.dgdr.model; an alias given with + # --served-model-name can hide the HF id, so a miss only warns. + names = _worker_model_names(dgd, components) + if names and dgdr.model not in names: + warnings.append( + f"P7: retune.dgdr.model {dgdr.model!r} is not among the workers' " + f"--model/--model-path/--served-model-name values " + f"({', '.join(sorted(names))}); every DGDR Run simulates " + "retune.dgdr.model, so make sure it is the model the DGD serves" + ) + if analysis.component is None: + if analysis.error is not None and analysis.error.startswith("P7"): + others.append(analysis.error) + else: + if any(_type(c) == EPP_TYPE for c in components): + others.append("P7: the DGD routes through an EPP component (GAIE)") + sidecars = _sidecar_components(components) + if sidecars: + others.append(f"P7: {', '.join(sidecars)} run a frontendSidecar") + if analysis.tls: + others.append( + "P7: the frontend serves TLS (--tls-cert-path / DYN_TLS_CERT_PATH)" + ) + + if analysis.env is not None: + for source in analysis.env.env_from: + if _env_from_may_set_router_env(source): + warnings.append( + f"P4: the frontend container uses envFrom {_describe_env_from(source)}; " + "the planner cannot read it and assumes it sets no router env " + "(P14 still checks the routers' policy parameters)" + ) + + # P8: one frontend replica, or synchronized router replicas. + if startup is not None and startup.replicas > 1 and not startup.replica_sync: + others.append( + f"P8: the frontend has {startup.replicas} replicas without " + "router_replica_sync; AISimulate models a single router" + ) + + # P9: worker replicas only change through the DGD spec, and one planner. + for component in components: + if _type(component) in WORKER_TYPES and _scaling_adapter(component): + others.append( + f"P9: worker component {component.get('name')} has a " + "scalingAdapter; its replicas could change during a window" + ) + others.extend(_planner_replica_errors(components, planner_replicas, warnings)) + + # P10: the frontend records request_end traces onto the claim. + if analysis.component is not None and analysis.env is not None: + others.extend( + _trace_recipe_errors( + analysis, + dgd_name=_dgd_name(dgd), + claim_name=dgdr.claim_name, + segments_dir=dgdr.segments_dir, + warnings=warnings, + ) + ) + + # P6 (warn only): AISimulate resolves the model by its HF id. + warning = _model_warning(dgdr.model) + if warning is not None: + warnings.append(warning) + + reasons = tuple(_unique([*p4, *others])) + return PremiseReport( + p4_ok=not p4, + spec_ok=not reasons, + reasons=reasons, + warnings=tuple(_unique(warnings)), + startup=startup, + ) + + +# --------------------------------------------------------- frontend facts + + +@dataclass(frozen=True) +class _Env: + """A component main container's env as far as the planner can know it. + + ``entries`` is the merged raw env keyed by name (operator-injected, then + graph ``env``, then the container's; later wins). ``values`` holds each + name whose value the planner can compute, after ``$(VAR)`` expansion; + ``opaque`` the names it cannot (``valueFrom``, or a reference to one). + """ + + entries: Mapping[str, Mapping[str, Any]] + values: Mapping[str, str] + opaque: frozenset[str] + env_from: tuple[Mapping[str, Any], ...] + + +@dataclass(frozen=True) +class _Analysis: + component: Optional[str] = None + spec: Optional[Mapping[str, Any]] = None + env: Optional[_Env] = None + startup: Optional[FrontendStartup] = None + error: Optional[str] = None + # P4 violations a FrontendStartup cannot show. + p4_violations: tuple[str, ...] = () + tls: bool = False + + +@dataclass(frozen=True) +class _Parsed: + """The cacheable outcome of parsing one frontend argv and env.""" + + error: Optional[str] = None + router_mode: str = "" + router_config: Optional[Mapping[str, Any]] = None + startup_maps: Optional[Mapping[str, Mapping[str, float]]] = None + replica_sync: bool = False + p4_violations: tuple[str, ...] = () + tls: bool = False + + +def _analyze(dgd: Mapping[str, Any], planner_mode: str) -> _Analysis: + try: + name, spec = find_frontend(dgd) + except FrontendParseError as error: + return _Analysis(error=str(error)) + env = _component_env(dgd, spec) + try: + argv = _effective_argv(spec) + except FrontendParseError as error: + return _Analysis(component=name, spec=spec, env=env, error=str(error)) + parsed = _parse_cached( + canonical_json( + { + "argv": list(argv), + "values": dict(env.values), + "opaque": sorted(env.opaque), + "env_from": [dict(source) for source in env.env_from], + } + ), + worker_types(planner_mode), + ) + if parsed.error is not None: + return _Analysis(component=name, spec=spec, env=env, error=parsed.error) + assert parsed.router_config is not None and parsed.startup_maps is not None + router_config = dict(parsed.router_config) + startup_maps = {wt: dict(params) for wt, params in parsed.startup_maps.items()} + startup = FrontendStartup( + router_mode=parsed.router_mode, + router_config=router_config, + startup_maps=startup_maps, + frontend_component=name, + replicas=_replicas(spec), + replica_sync=parsed.replica_sync, + # Replicas stay out: scaling the frontend is P8's concern and must not + # reset the persisted state. + fingerprint_source=canonical_json( + { + "router_mode": parsed.router_mode, + "router_config": router_config, + "startup_maps": startup_maps, + } + ), + ) + return _Analysis( + component=name, + spec=spec, + env=env, + startup=startup, + p4_violations=parsed.p4_violations, + tls=parsed.tls, + ) + + +@lru_cache(maxsize=32) +def _parse_cached(key: str, wts: tuple[str, ...]) -> _Parsed: + """Parse once per distinct frontend (revalidation re-parses every round).""" + + inputs = json.loads(key) + try: + return _parse( + inputs["argv"], + inputs["values"], + frozenset(inputs["opaque"]), + inputs["env_from"], + wts, + ) + except FrontendParseError as error: + return _Parsed(error=str(error)) + except Exception as error: # noqa: BLE001 - fail closed, never crash the planner + logger.exception("live retune could not parse the frontend configuration") + return _Parsed( + error=( + "P4: the planner failed to parse the frontend configuration " + f"({type(error).__name__}: {error})" + ) + ) + + +def _parse( + argv: Sequence[str], + values: Mapping[str, str], + opaque: frozenset[str], + env_from: Sequence[Mapping[str, Any]], + wts: tuple[str, ...], +) -> _Parsed: + args = _frontend_args(argv) + parser, registered = _frontend_parser(values) + + def relevant(name: str) -> bool: + if name in registered: + return registered[name] in _RELEVANT_DESTS + return name in _OVERLAP_WEIGHT_ENV or ( + name.startswith(_ROUTER_ENV_PREFIX) and name not in _TRANSPORT_ROUTER_ENV + ) + + # Router env the planner cannot read makes the startup unknowable. + hidden = sorted(name for name in opaque if relevant(name)) + if hidden: + raise FrontendParseError( + f"P4: frontend router env {', '.join(hidden)} is set through valueFrom " + "(or references such a value); the planner cannot read it, so set " + "it with a literal value" + ) + # envFrom sources (typically an HF token secret) are assumed not to set + # router env; evaluate_premises warns, and P14 compares the routers' + # policy parameters with the parsed startup. + del env_from + + # An unreadable $(VAR) becomes a marker the parser carries as a value: a + # non-router flag keeps it unconverted, a router flag fails the parse, + # and one left over where a flag could stand fails below. + expanded = [_expand_arg(arg, values, opaque) for arg in args] + namespace, unknown = parser.parse_known_args(expanded) + hidden_flag = _opaque_flag_position(unknown) + if hidden_flag is not None: + original = dict(zip(expanded, args)).get(hidden_flag, hidden_flag) + raise FrontendParseError( + f"P4: frontend arg {original!r} references env the planner cannot " + "read and is not the value of a frontend flag, so it could set a " + "router flag" + ) + unresolved = sorted( + dest + for dest, value in vars(namespace).items() + if dest in _RELEVANT_DESTS and _OPAQUE in str(value) + ) + if unresolved: + raise FrontendParseError( + f"P4: frontend args set {', '.join(unresolved)} from env the planner " + "cannot read" + ) + + view = _RouterView.from_cli_args(namespace) + # FrontendConfig.validate: the load-aware preset implies KV routing. + if view.load_aware: + view.router_mode = "kv" + if view.router_policy_config is not None: + raise FrontendParseError( + "P4: the frontend sets --router-policy-config " + f"({view.router_policy_config}); the planner cannot read that file, " + "and live retune needs the builtin default policy" + ) + try: + view.apply_load_aware_preset() + view.apply_conditional_disagg_config() + kwargs = {name: getattr(view, name) for name in _KV_ROUTER_FIELDS} + config = KvRouterConfig(**kwargs) + except (ValueError, TypeError) as error: + raise FrontendParseError( + f"P4: the frontend's router configuration is invalid: {error}" + ) from None + + full = json.loads(config.to_json()) + defaults = json.loads(KvRouterConfig().to_json()) + router_config = { + key: value + for key, value in full.items() + if key not in defaults or defaults[key] != value + } + try: + maps = _policy_maps(canonical_json(router_config), wts) + except ValueError as error: + raise FrontendParseError( + f"P4: cannot resolve the frontend's startup policy parameters: {error}" + ) from None + + violations: list[str] = [] + if kwargs["overlap_score_weight"] is not None: + violations.append( + "P4: the frontend sets the deprecated overlap_score_weight, which " + "overrides prefill_load_scale" + ) + for name, default in _SERDE_SKIPPED_DEFAULTS.items(): + if kwargs[name] != default: + violations.append( + f"P4: the frontend sets {name}={kwargs[name]!r} (default " + f"{default!r}), which live retune cannot model" + ) + if view.load_aware: + violations.append( + f"P4: the frontend enables the load-aware preset (--load-aware / " + f"{ENV_LOAD_AWARE})" + ) + for name in _ROUTER_ADMISSION_FIELDS: + value = getattr(view, name) + if value is not None: + violations.append( + f"P4: the frontend sets {name}={value!r}; live retune cannot " + "model session affinity or busy-worker rejection" + ) + for name in sorted(values): + if ( + not name.startswith(_ROUTER_ENV_PREFIX) + or name in registered + or name in _TRANSPORT_ROUTER_ENV + ): + continue + if name == ENV_WORKER_SELECTION_POLICY: + violations.append( + f"P4: the frontend sets {name}, which selects a worker-selection " + "policy instance instead of the builtin default policy" + ) + else: + violations.append( + f"P4: the frontend sets {name}, a router setting live retune " + "cannot model" + ) + + return _Parsed( + router_mode=_router_mode(str(view.router_mode)), + router_config=router_config, + startup_maps=maps, + replica_sync=bool(full.get("router_replica_sync", False)), + p4_violations=tuple(violations), + tls=any(getattr(view, dest, None) for dest in _TLS_DESTS), + ) + + +class _RouterView(RouterConfigBase, KvRouterConfigBase): + """The router fields of ``FrontendConfig``, without its other checks.""" + + tls_cert_path: Optional[str] = None + tls_key_path: Optional[str] = None + + +@lru_cache(maxsize=256) +def _policy_maps( + router_config_json: str, wts: tuple[str, ...] +) -> dict[str, dict[str, float]]: + config = KvRouterConfig.from_json(router_config_json) + return { + wt: {key: float(value) for key, value in config.policy_parameters(wt).items()} + for wt in wts + } + + +def _router_mode(cli_mode: str) -> str: + """``--router-mode`` in the ``RetuneEngineSpec.router_mode`` spelling.""" + + return "kv_router" if cli_mode == "kv" else cli_mode.replace("-", "_") + + +# ----------------------------------------------------------- argv and env + + +def _effective_argv(spec: Mapping[str, Any]) -> tuple[str, ...]: + """The frontend container's argv after the operator's defaulting. + + graph.go: an authored frontend ``command`` drops the default args; with + none, the command stays ``python3`` and the args default to + ``-m dynamo.frontend`` unless authored. + """ + + if spec.get("roles"): + raise FrontendParseError( + "P4: the frontend component uses roles; live retune needs a " + "single-pod frontend" + ) + main = _main_container(spec) + command = [str(part) for part in main.get("command") or []] + args = [str(part) for part in main.get("args") or []] + if command: + return tuple(command + args) + return (*DEFAULT_FRONTEND_COMMAND, *(args or DEFAULT_FRONTEND_ARGS)) + + +def _frontend_args(argv: Sequence[str]) -> list[str]: + """The arguments after ``python -m dynamo.frontend``; anything else fails.""" + + if not argv: + raise FrontendParseError("P4: the frontend container has an empty command") + program = posixpath.basename(argv[0]) + if program in _SHELLS or (len(argv) > 1 and argv[1] == "-c"): + raise FrontendParseError( + f"P4: the frontend command is shell-wrapped ({' '.join(argv[:2])} ...); " + "write it as command: [python3] and args: [-m, dynamo.frontend, ...]" + ) + if not program.startswith("python"): + raise FrontendParseError( + f"P4: the frontend command starts with {argv[0]!r}, not a python " + f"interpreter running -m {FRONTEND_MODULE}" + ) + index = 1 + while index < len(argv): + token = argv[index] + if token in _PYTHON_FLAGS: + index += 1 + continue + if token in _PYTHON_FLAGS_WITH_VALUE: + index += 2 + continue + if token == "-m" and index + 1 < len(argv): + module, rest = argv[index + 1], index + 2 + elif token.startswith("-m") and len(token) > 2: + module, rest = token[2:], index + 1 + else: + break + if module == FRONTEND_MODULE: + return list(argv[rest:]) + break + raise FrontendParseError( + f"P4: the frontend command {' '.join(argv[:4])!r} does not run " + f"python -m {FRONTEND_MODULE}" + ) + + +def _component_env(dgd: Mapping[str, Any], spec: Mapping[str, Any]) -> _Env: + """Merge a component's env the way the operator renders it. + + The operator injects ``DYN_PARENT_DGD_K8S_*`` and ``POD_*``, MergeEnvs + overlays graph ``env`` and the main container's ``env`` and sorts the + result by name; Kubernetes expands ``$(VAR)`` against earlier names. + """ + + metadata = dgd.get("metadata") or {} + entries: dict[str, Mapping[str, Any]] = { + ENV_PARENT_DGD_NAME: { + "name": ENV_PARENT_DGD_NAME, + "value": str(metadata.get("name", "")), + }, + ENV_PARENT_DGD_NAMESPACE: { + "name": ENV_PARENT_DGD_NAMESPACE, + "value": str(metadata.get("namespace", "")), + }, + } + for name in _OPERATOR_FIELD_REF_ENV: + entries[name] = {"name": name, "valueFrom": {"fieldRef": {}}} + main = _main_container(spec) + for entry in [*_graph_env(dgd), *(main.get("env") or [])]: + if isinstance(entry, Mapping) and entry.get("name"): + entries[str(entry["name"])] = entry + + values: dict[str, str] = {} + opaque: set[str] = set() + for name in sorted(entries): + entry = entries[name] + expanded = ( + None + if entry.get("valueFrom") is not None + else _expand(str(entry.get("value") or ""), values, opaque) + ) + if expanded is None: + opaque.add(name) + else: + values[name] = expanded + env_from = tuple( + source for source in main.get("envFrom") or [] if isinstance(source, Mapping) + ) + return _Env( + entries=entries, + values=values, + opaque=frozenset(opaque), + env_from=env_from, + ) + + +def _expand( + value: str, + values: Mapping[str, str], + opaque: Iterable[str], + *, + mark_opaque: bool = False, +) -> Optional[str]: + """Kubernetes ``$(VAR)`` expansion. + + ``$$`` escapes a ``$`` and a reference to an undefined name stays as + written. A reference to an opaque name makes the result ``None``, or, + with ``mark_opaque``, is replaced by a marker. + """ + + hidden = set(opaque) + out: list[str] = [] + index = 0 + while index < len(value): + if value.startswith("$$", index): + out.append("$") + index += 2 + continue + if value.startswith("$(", index): + close = value.find(")", index + 2) + if close != -1: + name = value[index + 2 : close] + if name in hidden: + if not mark_opaque: + return None + out.append(f"{_OPAQUE}{name}\x00") + else: + out.append(values.get(name, value[index : close + 1])) + index = close + 1 + continue + out.append(value[index]) + index += 1 + return "".join(out) + + +def _expand_arg(arg: str, values: Mapping[str, str], opaque: Iterable[str]) -> str: + """Expand one argument, marking opaque references for the parser.""" + + expanded = _expand(arg, values, opaque, mark_opaque=True) + assert expanded is not None + return expanded + + +def _opaque_flag_position(unknown: Sequence[str]) -> Optional[str]: + """A left-over argument whose unreadable env could stand for a flag. + + The frontend parser consumes the value of every flag it knows, so a + marked argument it leaves over sits where a flag could, unless it + follows a left-over option (a flag the frontend hands to its chat + processor, and this its value). A left-over option whose name is + marked always could. + """ + + previous: Optional[str] = None + for token in unknown: + if token.startswith("-"): + if _OPAQUE in token.split("=", 1)[0]: + return token + elif token.startswith(_OPAQUE) and not ( + previous is not None + and previous.startswith("-") + and "=" not in previous + and _OPAQUE not in previous + ): + return token + previous = token + return None + + +def _opaque_reference(text: str) -> Optional[str]: + """``$(NAME)`` for the first opaque marker in ``text``, or ``None``. + + argparse quotes a rejected value with ``repr``, so the marker may appear + escaped. + """ + + for marker, end_marker in ((_OPAQUE, "\x00"), (_OPAQUE_REPR, "\\x00")): + index = text.find(marker) + if index != -1: + start = index + len(marker) + end = text.find(end_marker, start) + return f"$({text[start:end] if end != -1 else text[start:]})" + return None + + +def _env_from_may_set_router_env(source: Mapping[str, Any]) -> bool: + """Whether an envFrom source could define a ``DYN_*`` name.""" + + prefix = str(source.get("prefix") or "") + return "DYN_".startswith(prefix) or prefix.startswith("DYN_") + + +def _describe_env_from(source: Mapping[str, Any]) -> str: + for kind in ("configMapRef", "secretRef"): + reference = source.get(kind) + if isinstance(reference, Mapping): + return f"{kind} {reference.get('name')!r}" + return canonical_json(dict(source)) + + +# ---------------------------------------------------- frontend arg parser + + +def _parse_error_message(message: str) -> str: + reference = _opaque_reference(message) + if reference is None: + return f"P4: cannot parse the frontend args: {message.strip()}" + return ( + f"P4: a frontend router arg references {reference}, which the planner " + "cannot read" + ) + + +def _worker_parse_error_message(message: str) -> str: + reference = _opaque_reference(message) + if reference is None: + return f"has router flags the planner cannot parse ({message.strip()})" + return ( + f"has a router flag that references {reference}, which the planner " + "cannot read" + ) + + +class _ArgsParser(argparse.ArgumentParser): + """An ``ArgumentParser`` that raises instead of printing and exiting. + + A value carrying the opaque marker skips type conversion and the + choices check unless its destination is in ``relevant``, so an + unreadable value of a flag that does not matter here parses, while one + of a flag that does fails with a message naming the reference. + """ + + def __init__( + self, + *, + relevant: frozenset[str] = _RELEVANT_DESTS, + describe: Callable[[str], str] = _parse_error_message, + ) -> None: + super().__init__(add_help=False) + self._relevant = relevant + self._describe = describe + + def error(self, message: str): # type: ignore[override] + raise FrontendParseError(self._describe(message)) + + def exit(self, status: int = 0, message: Optional[str] = None): # type: ignore[override] + raise FrontendParseError( + self._describe(message or f"argument parsing exited with {status}") + ) + + def _opaque_irrelevant(self, action: argparse.Action, value: Any) -> bool: + return ( + isinstance(value, str) + and _OPAQUE in value + and action.dest not in self._relevant + ) + + def _get_value(self, action, arg_string): # type: ignore[no-untyped-def] + if self._opaque_irrelevant(action, arg_string): + return arg_string + return super()._get_value(action, arg_string) + + def _check_value(self, action, value): # type: ignore[no-untyped-def] + if self._opaque_irrelevant(action, value): + return None + return super()._check_value(action, value) + + +class _OsWithEnv: + """``os`` with ``environ`` replaced by the frontend's env.""" + + def __init__(self, environ: Mapping[str, str]) -> None: + self.environ = dict(environ) + + def __getattr__(self, name: str) -> Any: + return getattr(os, name) + + +class _QuietIgnore(argparse.Action): + """Accept and drop a removed flag, without the frontend's warning.""" + + def __call__(self, parser, namespace, values, option_string=None) -> None: # type: ignore[no-untyped-def] + return None + + +class _NoArguments: + """Replaces an argument group none of whose flags matter here.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + def add_arguments(self, parser: Any) -> None: + return None + + +def _rebind( + function: Callable[..., Any], overrides: Mapping[str, Any] +) -> Callable[..., Any]: + """A copy of ``function`` whose module globals are overlaid with ``overrides``. + + The module is untouched, so other threads keep running the original. + """ + + rebound = types.FunctionType( + function.__code__, + {**function.__globals__, **overrides}, + function.__name__, + function.__defaults__, + function.__closure__, + ) + rebound.__kwdefaults__ = function.__kwdefaults__ + return rebound + + +def _frontend_parser( + env: Mapping[str, str], +) -> tuple[argparse.ArgumentParser, dict[str, str]]: + """The frontend's argument parser with every env default read from ``env``. + + Runs ``FrontendArgGroup.add_arguments`` and the router groups it adds as + copies whose ``os``, ``env_or_default``, ``add_argument`` and + ``add_negatable_bool_argument`` see only ``env``. Deprecation warnings, + the AISimulate perf-model flags (whose type reads files) and the + ``--model-path`` directory check are dropped. Returns the parser and the + env name -> dest map of everything it registered. + """ + + from dynamo.common.configuration.groups import kv_router_args, router_args + from dynamo.frontend import frontend_args + + overrides, bound, registered = _env_bound(env) + add_frontend_arguments = _rebind( + frontend_args.FrontendArgGroup.add_arguments, + { + **overrides, + "RouterArgGroup": bound(router_args.RouterArgGroup), + "KvRouterArgGroup": bound(kv_router_args.KvRouterArgGroup), + "AisPerfArgGroup": _NoArguments, + "validate_model_path": str, + }, + ) + parser = _ArgsParser() + try: + add_frontend_arguments(frontend_args.FrontendArgGroup(), parser) + except (ValueError, TypeError, argparse.ArgumentTypeError) as error: + # The frontend itself fails to start on such env. + raise FrontendParseError(f"P4: invalid frontend env: {error}") from None + + # The removed flags only log when used; keep their arity, drop the noise. + for action in parser._actions: + if isinstance(action, router_args._DeprecatedEnforceDisaggAction): + action.__class__ = argparse.BooleanOptionalAction # type: ignore[assignment] + elif isinstance(action, router_args._IgnoredAdmissionControlAction): + action.__class__ = _QuietIgnore # type: ignore[assignment] + return parser, registered + + +def _worker_router_parser( + env: Mapping[str, str], +) -> tuple[argparse.ArgumentParser, dict[str, str]]: + """A worker's router-advertisement parser with env defaults from ``env``. + + Runs ``router_args.add_worker_router_arguments``, which every backend + (vLLM, SGLang, TensorRT-LLM, mocker) uses, the way ``_frontend_parser`` + runs the frontend's groups. Only ``router_mode`` is relevant: it decides + whether the worker advertises a router configuration at all. + """ + + from dynamo.common.configuration.groups import kv_router_args, router_args + + overrides, bound, registered = _env_bound(env) + add_worker_arguments = _rebind( + router_args.add_worker_router_arguments, + { + "RouterArgGroup": bound(router_args.RouterArgGroup), + "KvRouterArgGroup": bound(kv_router_args.KvRouterArgGroup), + }, + ) + parser = _ArgsParser( + relevant=frozenset({_ROUTER_MODE_DEST}), + describe=_worker_parse_error_message, + ) + try: + add_worker_arguments(parser) + except (ValueError, TypeError, argparse.ArgumentTypeError) as error: + # The worker itself fails to start on such env. + raise FrontendParseError(f"has invalid router env ({error})") from None + return parser, registered + + +def _env_bound( + env: Mapping[str, str], +) -> tuple[dict[str, Any], Callable[[type], type], dict[str, str]]: + """Globals that make an argument group read only ``env``. + + Returns the overrides for ``_rebind``, a function that subclasses an + ``ArgGroup`` with its ``add_arguments`` rebound to them, and the env + name -> dest map the bound groups fill while registering. + """ + + from dynamo.common.configuration import utils + + fake_os = _OsWithEnv(env) + registered: dict[str, str] = {} + env_or_default = _rebind(utils.env_or_default, {"os": fake_os}) + base_add = _rebind( + utils.add_argument, {"os": fake_os, "env_or_default": env_or_default} + ) + + def add_argument(parser: Any, **kwargs: Any) -> None: + kwargs.pop("deprecated", None) + flag = str(kwargs["flag_name"]) + registered[str(kwargs["env_var"])] = str( + kwargs.get("dest") or flag.lstrip("-").replace("-", "_") + ) + base_add(parser, **kwargs) + + overrides = { + "os": fake_os, + "env_or_default": env_or_default, + "add_argument": add_argument, + "add_negatable_bool_argument": _rebind( + utils.add_negatable_bool_argument, {"add_argument": add_argument} + ), + "logger": _SILENT_LOGGER, + } + + def bound(group: Any) -> type: + return type( + group.__name__, + (group,), + {"add_arguments": _rebind(group.add_arguments, overrides)}, + ) + + return overrides, bound, registered + + +# --------------------------------------------------------- premise checks + + +def _startup_errors( + startup: FrontendStartup, retune: RetuneConfig, planner_mode: str +) -> list[str]: + """P4 on a parsed startup: KV mode, defaults, the controller's checks.""" + + if startup.router_mode != "kv_router": + return [ + f"P4: the frontend routes with {startup.router_mode!r}; live retune " + "needs --router-mode kv" + ] + try: + searched = set(_router_domain_keys(retune)) + except ValueError as error: + return [f"P4: cannot expand retune.router: {error}"] + config = startup.router_config + errors: list[str] = [] + load_model = config.get(_LOAD_MODEL, "none") + if load_model != "none": + errors.append( + f"P4: the frontend's {_LOAD_MODEL} is {load_model!r}; live retune " + "needs 'none'" + ) + if config.get(_CONDITIONAL_DISAGG): + errors.append( + "P4: the frontend enables conditional disaggregation, which replay " + "and live routers resolve differently" + ) + for key in sorted(config): + if ( + key in searched + or key in TOLERATED_ROUTER_FIELDS + or key in (_LOAD_MODEL, _CONDITIONAL_DISAGG) + ): + continue + errors.append( + f"P4: the frontend sets {key}={config[key]!r}; only the searched " + f"router knobs ({', '.join(sorted(searched)) or 'none'}) may differ " + "from their defaults" + ) + if errors: + return errors + return _controller_errors(retune, config, worker_types(planner_mode)) + + +def _worker_router_errors( + dgd: Mapping[str, Any], + components: Sequence[Mapping[str, Any]], + warnings: list[str], +) -> list[str]: + """P4 on the workers: none advertises its own router configuration. + + A worker that resolves a router mode (``--router-mode`` or + ``DYN_ROUTER_MODE``, from its container or the graph env) puts a + ``RouterConfig`` in its model card, and the frontend routes that worker + set with it instead of its own flags; replay would model the frontend's. + """ + + errors: list[str] = [] + env_from: list[str] = [] + for component in components: + if _type(component) not in WORKER_TYPES: + continue + name = str(component.get("name", "")) + env = _component_env(dgd, component) + argv, shell = _worker_argv(component, env) + detail = _worker_router_cached( + canonical_json( + { + "argv": argv, + "shell": shell, + "values": dict(env.values), + "opaque": sorted(env.opaque), + } + ) + ) + if detail is not None: + errors.append( + f"P4: worker component {name} {detail}; a worker's router " + "configuration replaces the frontend's for its workers, so set " + f"--router-mode and {ENV_ROUTER_MODE} on the frontend only (a " + "graph-level env entry reaches every worker)" + ) + if any(_env_from_may_set_router_env(source) for source in env.env_from): + env_from.append(name) + if env_from: + warnings.append( + f"P4: worker components {', '.join(sorted(env_from))} use envFrom; " + "the planner cannot read it and assumes it sets no router env (a " + f"{ENV_ROUTER_MODE} there would make the worker advertise its own " + "router configuration)" + ) + return errors + + +@lru_cache(maxsize=64) +def _worker_router_cached(key: str) -> Optional[str]: + """Why one worker advertises a router configuration, or ``None``.""" + + inputs = json.loads(key) + try: + return _worker_router_mode( + inputs["argv"], + bool(inputs["shell"]), + inputs["values"], + frozenset(inputs["opaque"]), + ) + except FrontendParseError as error: + return str(error) + except Exception as error: # noqa: BLE001 - fail closed, never crash the planner + logger.exception("live retune could not parse a worker's router flags") + return ( + "could not be checked for a router configuration " + f"({type(error).__name__}: {error})" + ) + + +def _worker_router_mode( + argv: Sequence[str], + shell: bool, + values: Mapping[str, str], + opaque: frozenset[str], +) -> Optional[str]: + parser, registered = _worker_router_parser(values) + mode_env = sorted( + name for name, dest in registered.items() if dest == _ROUTER_MODE_DEST + ) + hidden = [name for name in mode_env if name in opaque] + if hidden: + return ( + f"sets {', '.join(hidden)} through valueFrom (or a reference to such " + "a value), so the planner cannot tell whether it advertises a router " + "configuration" + ) + # A shell script can also export the env inline. + inline = sorted( + {token.split("=", 1)[0] for token in argv if shell and "=" in token} + & set(mode_env) + ) + if inline: + return f"sets {', '.join(inline)} in its shell command" + namespace, _ = parser.parse_known_args(list(argv)) + mode = getattr(namespace, _ROUTER_MODE_DEST, None) + if mode is None: + return None + return ( + f"advertises its own router configuration (router mode {mode!r} from " + f"--router-mode or {ENV_ROUTER_MODE})" + ) + + +def _worker_argv(component: Mapping[str, Any], env: _Env) -> tuple[list[str], bool]: + """A worker's argv after ``$(VAR)`` expansion, and whether a shell runs it. + + Unreadable references stay marked. A shell-wrapped command is split + into the script's words, so the flags of the command it runs are seen. + """ + + main = _main_container(component) + parts: list[str] = [] + for part in [*(main.get("command") or []), *(main.get("args") or [])]: + expanded = _expand(str(part), env.values, env.opaque, mark_opaque=True) + assert expanded is not None + parts.append(expanded) + shell = bool(parts) and ( + posixpath.basename(parts[0]) in _SHELLS or parts[1:2] == ["-c"] + ) + if not shell: + return parts, False + words: list[str] = [] + for part in parts: + try: + words.extend(shlex.split(part)) + except ValueError: + words.extend(part.split()) + return words, True + + +def _router_domain_keys(retune: RetuneConfig) -> list[str]: + if retune.router is None: + return [] + from dynamo.planner.retune.domains import expand_router_domain + + return list(expand_router_domain(retune)) + + +def _controller_errors( + retune: RetuneConfig, router_config: Mapping[str, Any], wts: tuple[str, ...] +) -> list[str]: + """The startup-dependent checks of ``RetuneController._router_domain``. + + A startup failing them would make the controller constructor raise, so + P4 rejects it before a controller is built. + """ + + if retune.router is None: + return [] + from dynamo.planner.retune.domains import ( + OVERLAP_SCORE_CREDIT, + expand_router_domain, + grid_points, + ) + + startup_json = canonical_json(dict(router_config)) + try: + startup_maps = _policy_maps(startup_json, wts) + load_model = json.loads(KvRouterConfig.from_json(startup_json).to_json())[ + _LOAD_MODEL + ] + except ValueError as error: + return [ + f"P4: the frontend startup is not a valid router configuration: {error}" + ] + # A worker type that starts with no overlap credit can never get it. + zero_credit = sorted( + wt + for wt, params in startup_maps.items() + if wt != "decode" and params[OVERLAP_SCORE_CREDIT] == 0 + ) + try: + domain = expand_router_domain( + retune, + startup_load_model=load_model, + zero_startup_credit=bool(zero_credit), + ) + except ValueError as error: + return [f"P4: retune.router does not fit the frontend startup: {error}"] + for point in grid_points(domain, {}): + try: + maps = _policy_maps(canonical_json({**router_config, **point.router}), wts) + except ValueError as error: + return [ + f"P4: retune.router candidate {point.router} is not a valid " + f"router configuration for this frontend: {error}" + ] + for wt in zero_credit: + if maps[wt][OVERLAP_SCORE_CREDIT] != 0: + return [ + f"P4: retune.router candidate {point.router} gives the {wt} " + "router overlap_score_credit > 0, but the frontend starts it " + "at 0 (a live router rejects that update)" + ] + return [] + + +def _startup_config_mismatch( + declared: Mapping[str, Any], startup: FrontendStartup +) -> Optional[str]: + """``retune.dgdr.startup_router_config`` must mean the parsed startup.""" + + try: + declared_full = json.loads( + KvRouterConfig.from_json(json.dumps(dict(declared))).to_json() + ) + except ValueError as error: + return ( + "P4: retune.dgdr.startup_router_config is not a KvRouterConfig " + f"JSON: {error}" + ) + parsed_full = json.loads( + KvRouterConfig.from_json(json.dumps(dict(startup.router_config))).to_json() + ) + keys = sorted( + key + for key in set(declared_full) | set(parsed_full) + if declared_full.get(key) != parsed_full.get(key) + ) + if not keys: + return None + diffs = "; ".join( + f"{key}: declared {declared_full.get(key)!r}, frontend {parsed_full.get(key)!r}" + for key in keys + ) + return f"P4: retune.dgdr.startup_router_config differs from the frontend ({diffs})" + + +def _planner_replica_errors( + components: Sequence[Mapping[str, Any]], + planner_replicas: Optional[int], + warnings: list[str], +) -> list[str]: + if planner_replicas is not None: + if planner_replicas != 1: + return [ + f"P9: the planner has {planner_replicas} replicas; live retune needs 1" + ] + return [] + planners = [c for c in components if _type(c) == PLANNER_TYPE] + if not planners: + warnings.append( + "P9: the DGD has no planner component; its replicas cannot be checked" + ) + return [ + f"P9: planner component {c.get('name')} has {_replicas(c)} replicas; " + "live retune needs 1" + for c in planners + if _replicas(c) != 1 + ] + + +def _trace_recipe_errors( + analysis: _Analysis, + *, + dgd_name: str, + claim_name: str, + segments_dir: str, + warnings: list[str], +) -> list[str]: + """P10: the frontend writes request_end segments onto the claim (§4.4.1).""" + + assert analysis.spec is not None and analysis.env is not None + entries = analysis.env.entries + errors: list[str] = [] + + def literal(name: str) -> Optional[str]: + entry = entries.get(name) + if entry is None: + return None + if entry.get("valueFrom") is not None: + errors.append(f"P10: frontend env {name} must be a literal value") + return None + return str(entry.get("value") or "") + + trace = literal(ENV_TRACE) + if trace is None or trace.strip().lower() not in _TRUTHY: + errors.append(f"P10: the frontend must set {ENV_TRACE}=1") + + sinks = _csv(literal(ENV_TRACE_SINKS)) + if "file" not in sinks or {"jsonl", "jsonl_gz"} & set(sinks): + errors.append(f"P10: the frontend must set {ENV_TRACE_SINKS}=file") + elif set(sinks) != {"file"}: + extra = ", ".join(sorted(set(sinks) - {"file"})) + warnings.append(f"P10: {ENV_TRACE_SINKS} also enables {extra}") + + file_format = literal(ENV_TRACE_FORMAT) + if file_format is not None and file_format.strip().lower() not in _GZ_FORMATS: + errors.append(f"P10: {ENV_TRACE_FORMAT} must be jsonl_gz or unset") + + if set(_csv(literal(ENV_TRACE_RECORDS))) != {"request_end"}: + errors.append(f"P10: the frontend must set {ENV_TRACE_RECORDS}=request_end") + + pod_name = entries.get(ENV_POD_NAME) or {} + field_ref = (pod_name.get("valueFrom") or {}).get("fieldRef") or {} + if field_ref.get("fieldPath") != "metadata.name": + errors.append( + f"P10: the frontend must set {ENV_POD_NAME} from fieldRef metadata.name" + ) + parent = entries.get(ENV_PARENT_DGD_NAME) or {} + if parent.get("valueFrom") is not None or parent.get("value") != dgd_name: + errors.append( + f"P10: the frontend overrides {ENV_PARENT_DGD_NAME}; it must stay the " + f"DGD name {dgd_name!r}" + ) + + mounts, mount_errors = _claim_mounts(analysis.spec, claim_name) + errors.extend(mount_errors) + path = literal(ENV_TRACE_PATH) + if path is None: + errors.append(f"P10: the frontend must set {ENV_TRACE_PATH}") + elif mounts: + segments = segments_dir.strip("/") + prefixes = [ + _join_dir(mount, segments, dgd_dir, f"$({ENV_POD_NAME})") + for mount in mounts + for dgd_dir in (f"$({ENV_PARENT_DGD_NAME})", dgd_name) + ] + # Directly in the pod's directory: the reconciler prunes only the + # segments at its top level. + if not any( + path.startswith(prefix) + and len(path) > len(prefix) + and "/" not in path[len(prefix) :] + for prefix in prefixes + ): + errors.append( + f"P10: {ENV_TRACE_PATH}={path!r} must be a file prefix directly " + f"under {prefixes[0]!r} (no further '/')" + ) + return errors + + +def _claim_mounts( + spec: Mapping[str, Any], claim_name: str +) -> tuple[list[str], list[str]]: + """Writable main-container mount paths of the claim's root, and errors.""" + + pod_spec = _pod_spec(spec) + volumes = { + str(volume.get("name")) + for volume in pod_spec.get("volumes") or [] + if isinstance(volume, Mapping) + and (volume.get("persistentVolumeClaim") or {}).get("claimName") == claim_name + } + if not volumes: + return [], [f"P10: the frontend pod has no volume for claim {claim_name!r}"] + candidates = [ + mount + for mount in _main_container(spec).get("volumeMounts") or [] + if isinstance(mount, Mapping) and mount.get("name") in volumes + ] + if not candidates: + return [], [ + f"P10: the frontend main container does not mount claim {claim_name!r}" + ] + mounts = [ + str(mount["mountPath"]) + for mount in candidates + if mount.get("mountPath") + and not mount.get("readOnly") + and not mount.get("subPath") + and not mount.get("subPathExpr") + ] + if not mounts: + return [], [ + f"P10: the frontend mounts claim {claim_name!r} read-only or through a " + "subPath; mount the claim root writable" + ] + return mounts, [] + + +def _join_dir(mount: str, *parts: str) -> str: + return "/".join([mount.rstrip("/"), *(part for part in parts if part)]) + "/" + + +def _model_warning(model: str) -> Optional[str]: + try: + from aisimulate_core.sdk.common import DefaultHFModels + except Exception: # noqa: BLE001 - optional dependency of a warn-only premise + return None + if model in DefaultHFModels: + return None + return ( + f"P6: retune.dgdr.model {model!r} is not an AISimulate default HF model; " + "each DGDR Run must be able to download it" + ) + + +# ------------------------------------------------------------ DGD helpers + + +def _components(dgd: Mapping[str, Any]) -> list[Mapping[str, Any]]: + spec = dgd.get("spec") or {} + components = spec.get("components") + if isinstance(components, list): + return [c for c in components if isinstance(c, Mapping)] + services = spec.get("services") + if isinstance(services, Mapping): + return [ + _from_v1alpha1(str(name), service) + for name, service in services.items() + if isinstance(service, Mapping) + ] + return [] + + +def _from_v1alpha1(name: str, service: Mapping[str, Any]) -> dict[str, Any]: + """A v1alpha1 service in the v1beta1 component shape this module reads.""" + + extra = service.get("extraPodSpec") or {} + main = dict(extra.get("mainContainer") or {}) + main["name"] = MAIN_CONTAINER + env: dict[str, Any] = {} + for entry in [*(service.get("envs") or []), *(main.get("env") or [])]: + if isinstance(entry, Mapping) and entry.get("name"): + env[str(entry["name"])] = entry + main["env"] = list(env.values()) + env_from = list(main.get("envFrom") or []) + if service.get("envFromSecret"): + env_from.append({"secretRef": {"name": service["envFromSecret"]}}) + main["envFrom"] = env_from + # v1alpha1 volumeMounts name a PVC and mount it at mountPoint. + mounts = list(main.get("volumeMounts") or []) + volumes = list(extra.get("volumes") or []) + for mount in service.get("volumeMounts") or []: + if isinstance(mount, Mapping) and mount.get("name"): + claim = str(mount["name"]) + mounts.append({"name": claim, "mountPath": mount.get("mountPoint")}) + volumes.append( + {"name": claim, "persistentVolumeClaim": {"claimName": claim}} + ) + main["volumeMounts"] = mounts + component: dict[str, Any] = { + "name": name, + "type": service.get("componentType") or "", + "podTemplate": {"spec": {"containers": [main], "volumes": volumes}}, + } + for key in ("replicas", "roles", "modelRef"): + if service.get(key) is not None: + component[key] = service[key] + if service.get("frontendSidecar"): + component["frontendSidecar"] = "sidecar-frontend" + # v1alpha1 needs enabled: true; v1beta1 opts in with any object. + if (service.get("scalingAdapter") or {}).get("enabled"): + component["scalingAdapter"] = {} + return component + + +def _graph_env(dgd: Mapping[str, Any]) -> list[Any]: + spec = dgd.get("spec") or {} + return list(spec.get("env") or spec.get("envs") or []) + + +def _pod_spec(spec: Mapping[str, Any]) -> Mapping[str, Any]: + template = spec.get("podTemplate") + pod_spec = template.get("spec") if isinstance(template, Mapping) else None + return pod_spec if isinstance(pod_spec, Mapping) else {} + + +def _main_container(spec: Mapping[str, Any]) -> Mapping[str, Any]: + for container in _pod_spec(spec).get("containers") or []: + if isinstance(container, Mapping) and container.get("name") == MAIN_CONTAINER: + return container + return {} + + +def _sidecar_components(components: Sequence[Mapping[str, Any]]) -> list[str]: + return sorted(str(c.get("name")) for c in components if c.get("frontendSidecar")) + + +def _type(component: Mapping[str, Any]) -> str: + return str(component.get("type") or "") + + +def _replicas(component: Mapping[str, Any]) -> int: + """Spec replicas; an omitted value is the workload default of 1.""" + + replicas = component.get("replicas") + try: + return 1 if replicas is None else int(replicas) + except (TypeError, ValueError): + return 1 + + +def _scaling_adapter(component: Mapping[str, Any]) -> bool: + return component.get("scalingAdapter") is not None + + +def _dgd_name(dgd: Mapping[str, Any]) -> str: + return str((dgd.get("metadata") or {}).get("name", "")) + + +def _projected_status(dgd: Mapping[str, Any]) -> Mapping[str, Any]: + """``status.components`` while the status is current, else empty.""" + + status = dgd.get("status") or {} + generation = (dgd.get("metadata") or {}).get("generation") + current = generation is not None and generation == status.get("observedGeneration") + return (status.get("components") or {}) if current else {} + + +def _model_flags( + component: Mapping[str, Any], env: _Env +) -> list[tuple[str, Optional[str]]]: + """Each model flag in the main container's command and args, in order. + + Values are ``$(VAR)``-expanded against the component's env; a value + that still references env (or looks like a flag) is ``None``. + """ + + main = _main_container(component) + tokens: list[str] = [] + for part in [*(main.get("command") or []), *(main.get("args") or [])]: + text = _expand(str(part), env.values, env.opaque, mark_opaque=True) + assert text is not None + try: + tokens.extend(shlex.split(text)) + except ValueError: + tokens.append(text) + found: list[tuple[str, Optional[str]]] = [] + for index, token in enumerate(tokens): + for flag in _MODEL_FLAGS: + if token == flag and index + 1 < len(tokens): + value = tokens[index + 1] + elif token.startswith(flag + "="): + value = token[len(flag) + 1 :] + else: + continue + resolved = ( + value + and "$" not in value + and _OPAQUE not in value + and not value.startswith("-") + ) + found.append((flag, value if resolved else None)) + return found + + +def _flag_model(flags: Sequence[tuple[str, Optional[str]]]) -> Optional[str]: + """The operator's served model: the first flag of ``_MODEL_FLAGS`` given.""" + + for flag in _MODEL_FLAGS: + for name, value in flags: + if name == flag: + return value + return None + + +def _worker_model_names( + dgd: Mapping[str, Any], components: Sequence[Mapping[str, Any]] +) -> set[str]: + """Every model name or path the workers give, and their served names.""" + + projected = _projected_status(dgd) + names: set[str] = set() + for component in components: + if _type(component) not in WORKER_TYPES: + continue + served = (projected.get(str(component.get("name", ""))) or {}).get( + "servedModelName" + ) + if served: + names.add(str(served)) + for _, value in _model_flags(component, _component_env(dgd, component)): + if value: + names.update(part for part in re.split(r"[\s,]+", value) if part) + return names + + +def _csv(value: Optional[str]) -> list[str]: + if not value: + return [] + return [part.strip().lower() for part in value.split(",") if part.strip()] + + +def _unique(items: Iterable[str]) -> list[str]: + return list(dict.fromkeys(items)) diff --git a/components/src/dynamo/planner/retune/live/kube.py b/components/src/dynamo/planner/retune/live/kube.py new file mode 100644 index 000000000000..359d3aa92f98 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/kube.py @@ -0,0 +1,397 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``contracts.RetuneKube`` over the official ``kubernetes`` client. + +Every call is synchronous (callers on the event loop use +``asyncio.to_thread``), carries ``_request_timeout=(connect, read)`` and +raises the ``contracts.Kube*`` errors instead of client exceptions. The +error carries the HTTP status as ``error.status`` (``None`` for a network +error or timeout), so a caller can tell 403 from the rest. + +Custom resources (the DGD at v1beta1, DGDR v1beta2 Requests, Runs and +Candidates) come back as the API server's JSON. Leases are read without the +client's model deserialization (``_preload_content=False``) so a Lease dict +round-trips unchanged: deserializing turns ``renewTime`` into a ``datetime`` +whose ``isoformat`` drops a zero fraction, which a later replace would send +back in a form the ``MicroTime`` parser rejects. +""" + +from __future__ import annotations + +import json +import logging +import math +from collections.abc import Callable, Mapping +from typing import Any, Literal, Optional, TypeVar + +from kubernetes import client +from kubernetes.client import ApiException + +from dynamo.planner.retune.live.contracts import ( + DGDR_CANDIDATES, + DGDR_GROUP, + DGDR_REQUESTS, + DGDR_RUNS, + DGDR_VERSION, + KubeAlreadyExists, + KubeConflict, + KubeError, + KubeInvalid, + KubeNotFound, + PodInfo, +) + +logger = logging.getLogger(__name__) + +DGD_VERSION = "v1beta1" +DGD_PLURAL = "dynamographdeployments" +# The frontend's container and HTTP port names (deploy/operator/internal/consts). +MAIN_CONTAINER_NAME = "main" +HTTP_PORT_NAME = "http" +FIELD_MANAGER = "dynamo-planner-retune" + +_T = TypeVar("_T") +_Operation = Literal["read", "create", "update", "delete"] + + +def request_timeout(timeout_s: float) -> tuple[int, int]: + """The client's ``(connect, read)`` timeout for ``timeout_s``, at least 1 s each.""" + + if not math.isfinite(timeout_s) or timeout_s <= 0: + seconds = 1 + else: + seconds = max(1, int(math.ceil(timeout_s))) + return (seconds, seconds) + + +def _kube_error(error: Exception, operation: _Operation, what: str) -> KubeError: + """Map a client exception to the contracts error for ``operation``.""" + + if isinstance(error, ApiException): + status = error.status + detail = _api_message(error) + message = f"{operation} {what}: HTTP {status}: {detail}" + mapped: KubeError + if status == 404: + mapped = KubeNotFound(message) + elif status == 409 and operation == "create": + mapped = KubeAlreadyExists(message) + elif status == 409: + mapped = KubeConflict(message) + elif status in (400, 422): + mapped = KubeInvalid(message) + else: + mapped = KubeError(message) + mapped.status = status # type: ignore[attr-defined] + return mapped + mapped = KubeError(f"{operation} {what}: {type(error).__name__}: {error}") + mapped.status = None # type: ignore[attr-defined] + return mapped + + +def _api_message(error: ApiException) -> str: + """The ``Status.message`` of an API error body, else its reason.""" + + body = error.body + if isinstance(body, bytes): + body = body.decode("utf-8", errors="replace") + if isinstance(body, str) and body: + try: + parsed = json.loads(body) + except ValueError: + return body[:500] + if isinstance(parsed, dict) and parsed.get("message"): + return str(parsed["message"])[:500] + return str(error.reason or "") + + +def _pod_info(pod: Any) -> PodInfo: + """``PodInfo`` of a ``V1Pod``.""" + + metadata = pod.metadata + status = pod.status + ready = any( + condition.type == "Ready" and condition.status == "True" + for condition in (status.conditions if status is not None else None) or () + ) + return PodInfo( + name=metadata.name, + uid=metadata.uid, + ip=status.pod_ip if status is not None else None, + ready=ready, + terminating=metadata.deletion_timestamp is not None, + labels=dict(metadata.labels or {}), + http_port=_http_port(pod.spec), + phase=status.phase if status is not None else None, + ) + + +def _http_port(spec: Any) -> Optional[int]: + """The ``http`` container port of the main container, else of any container.""" + + containers = list((spec.containers if spec is not None else None) or ()) + containers.sort(key=lambda container: container.name != MAIN_CONTAINER_NAME) + for container in containers: + for port in container.ports or (): + if port.name == HTTP_PORT_NAME: + return int(port.container_port) + return None + + +def _decode(response: Any) -> dict: + """The JSON body of a ``_preload_content=False`` response.""" + + data = response.data + if isinstance(data, bytes): + data = data.decode("utf-8") + return json.loads(data) if data else {} + + +class KubernetesRetuneKube: + """``contracts.RetuneKube`` for one namespace. + + ``api_client`` defaults to a new ``ApiClient`` on the default + configuration, which ``KubernetesAPI`` loads (in-cluster or kubeconfig); + ``from_kube_api`` reuses that connector's client instead. + """ + + def __init__(self, namespace: str, api_client: Optional[client.ApiClient] = None): + if not namespace: + raise ValueError("namespace must be set") + self.namespace = namespace + self.api_client = api_client if api_client is not None else client.ApiClient() + self._custom = client.CustomObjectsApi(self.api_client) + self._core = client.CoreV1Api(self.api_client) + self._coordination = client.CoordinationV1Api(self.api_client) + + @classmethod + def from_kube_api(cls, kube_api: Any) -> KubernetesRetuneKube: + """Share a ``KubernetesAPI``'s client and namespace.""" + + return cls( + kube_api.current_namespace, + api_client=kube_api.custom_api.api_client, + ) + + # ------------------------------------------------------------ helpers + + def _call(self, operation: _Operation, what: str, function: Callable[[], _T]) -> _T: + try: + return function() + except KubeError: + raise + except Exception as error: # ApiException, urllib3 and socket errors + raise _kube_error(error, operation, what) from error + + def _get_custom( + self, version: str, plural: str, name: str, timeout_s: float + ) -> Optional[dict]: + try: + return self._call( + "read", + f"{plural}/{name}", + lambda: self._custom.get_namespaced_custom_object( + group=DGDR_GROUP, + version=version, + namespace=self.namespace, + plural=plural, + name=name, + _request_timeout=request_timeout(timeout_s), + ), + ) + except KubeNotFound: + return None + + def _list_custom( + self, version: str, plural: str, label_selector: str, timeout_s: float + ) -> list[dict]: + result = self._call( + "read", + f"{plural}?labelSelector={label_selector}", + lambda: self._custom.list_namespaced_custom_object( + group=DGDR_GROUP, + version=version, + namespace=self.namespace, + plural=plural, + label_selector=label_selector, + _request_timeout=request_timeout(timeout_s), + ), + ) + return list(result.get("items") or []) + + # ----------------------------------------------------------------- DGD + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + found = self._get_custom(DGD_VERSION, DGD_PLURAL, name, timeout_s) + if found is None: + error = KubeNotFound( + f"DynamoGraphDeployment {self.namespace}/{name} not found" + ) + error.status = 404 # type: ignore[attr-defined] + raise error + return found + + # ---------------------------------------------------------------- pods + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + result = self._call( + "read", + f"pods?labelSelector={label_selector}", + lambda: self._core.list_namespaced_pod( + namespace=self.namespace, + label_selector=label_selector, + _request_timeout=request_timeout(timeout_s), + ), + ) + return [_pod_info(pod) for pod in result.items or ()] + + # ---------------------------------------------------- DGDR v1beta2 API + + def create_request( + self, + body: dict, + *, + dry_run: bool = False, + timeout_s: float = 10.0, + ) -> dict: + kwargs: dict[str, Any] = { + "field_validation": "Strict", + "field_manager": FIELD_MANAGER, + "_request_timeout": request_timeout(timeout_s), + } + if dry_run: + kwargs["dry_run"] = "All" + name = (body.get("metadata") or {}).get("name", "") + return self._call( + "create", + f"{DGDR_REQUESTS}/{name}", + lambda: self._custom.create_namespaced_custom_object( + group=DGDR_GROUP, + version=DGDR_VERSION, + namespace=self.namespace, + plural=DGDR_REQUESTS, + body=body, + **kwargs, + ), + ) + + def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + return self._get_custom(DGDR_VERSION, DGDR_REQUESTS, name, timeout_s) + + def list_requests( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + return self._list_custom(DGDR_VERSION, DGDR_REQUESTS, label_selector, timeout_s) + + def delete_request( + self, + name: str, + *, + propagation: Literal["Background", "Foreground"] = "Background", + timeout_s: float = 10.0, + ) -> None: + try: + self._call( + "delete", + f"{DGDR_REQUESTS}/{name}", + lambda: self._custom.delete_namespaced_custom_object( + group=DGDR_GROUP, + version=DGDR_VERSION, + namespace=self.namespace, + plural=DGDR_REQUESTS, + name=name, + propagation_policy=propagation, + _request_timeout=request_timeout(timeout_s), + ), + ) + except KubeNotFound: + pass + + def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + return self._get_custom(DGDR_VERSION, DGDR_RUNS, name, timeout_s) + + def list_candidates( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + return self._list_custom( + DGDR_VERSION, DGDR_CANDIDATES, label_selector, timeout_s + ) + + # -------------------------------------------------------------- Leases + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + try: + return self._call( + "read", + f"leases/{name}", + lambda: _decode( + self._coordination.read_namespaced_lease( + name=name, + namespace=self.namespace, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + except KubeNotFound: + return None + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + name = (body.get("metadata") or {}).get("name", "") + return self._call( + "create", + f"leases/{name}", + lambda: _decode( + self._coordination.create_namespaced_lease( + namespace=self.namespace, + body=_lease_body(body), + field_manager=FIELD_MANAGER, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + return self._call( + "update", + f"leases/{name}", + lambda: _decode( + self._coordination.replace_namespaced_lease( + name=name, + namespace=self.namespace, + body=_lease_body(body), + field_manager=FIELD_MANAGER, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + """Missing Leases are not an error.""" + + try: + self._call( + "delete", + f"leases/{name}", + lambda: self._coordination.delete_namespaced_lease( + name=name, + namespace=self.namespace, + _request_timeout=request_timeout(timeout_s), + ), + ) + except KubeNotFound: + pass + + +def _lease_body(body: Mapping[str, Any]) -> dict: + """A Lease body with its type fields set (the API server requires them).""" + + lease = dict(body) + lease.setdefault("apiVersion", "coordination.k8s.io/v1") + lease.setdefault("kind", "Lease") + return lease diff --git a/components/src/dynamo/planner/retune/live/metrics.py b/components/src/dynamo/planner/retune/live/metrics.py new file mode 100644 index 000000000000..cd28e1c21248 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/metrics.py @@ -0,0 +1,262 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``dynamo_planner_retune_*`` Prometheus metrics of live retune. + +``RetuneMetrics`` implements ``contracts.RetuneMetricsSink``: decision, +failure and skip counters, the supervisor's state, the scores of the latest +decision, router convergence, PATCH errors and trace health (design §4.10). +Every method is cheap and never raises, so callers on the event loop, in the +sweep thread or in the actuator need no guard. + +The planner serves ``prometheus_client.REGISTRY`` on +``metric_reporting_prometheus_port``, so the default registry is the one to +use. The collectors are created once per registry and shared: building a +second ``RetuneMetrics`` on the same registry (a controller rebuild, a test) +does not fail with a duplicate registration. +""" + +from __future__ import annotations + +import logging +import math +import threading +import weakref +from typing import Optional + +from prometheus_client import REGISTRY, CollectorRegistry, Counter, Gauge + +from dynamo.planner.core.types import RETUNE_OUTCOMES + +logger = logging.getLogger(__name__) + +PREFIX = "dynamo_planner_retune" +# Gate reasons (``contracts.GateReason``) and decision outcomes are known up +# front; their series start at 0 so ``increase()`` sees the first event. +_SKIP_REASONS = ("not_leader", "not_ready", "already_decided") +# Live outcomes recorded in the state history besides the controller's. +_LIVE_OUTCOMES = ("promoted", "rolled_back") + + +class _Collectors: + """The metric objects of one registry.""" + + def __init__(self, registry: CollectorRegistry) -> None: + kw = {"registry": registry} + self.decisions_total = Counter( + f"{PREFIX}_decisions_total", + "Retune decisions, counted once the state write persisted them.", + labelnames=["outcome"], + **kw, + ) + self.failures_total = Counter( + f"{PREFIX}_failures_total", + "Failed retune windows and supervisor failures, by reason.", + labelnames=["reason"], + **kw, + ) + self.skipped_total = Counter( + f"{PREFIX}_skipped_total", + "Due windows the launch gate held back, once per window and reason.", + labelnames=["reason"], + **kw, + ) + self.leader = Gauge( + f"{PREFIX}_leader", + "1 while this planner holds the retune leadership Lease.", + **kw, + ) + self.ready = Gauge( + f"{PREFIX}_ready", + "1 while every spec and health premise holds, so windows launch.", + **kw, + ) + self.initialized = Gauge( + f"{PREFIX}_initialized", + "1 once the leader finished initializing from the persisted state.", + **kw, + ) + self.rollback_pending = Gauge( + f"{PREFIX}_rollback_pending", + "1 while routers are being reset to their startup parameters.", + **kw, + ) + self.controller_generation = Gauge( + f"{PREFIX}_controller_generation", + "Generation of the installed retune controller; it grows on each rebuild.", + **kw, + ) + self.baseline_score = Gauge( + f"{PREFIX}_baseline_score", + "Objective score of the current parameters on the latest decided " + "window (NaN when not scored).", + **kw, + ) + self.best_score = Gauge( + f"{PREFIX}_best_score", + "Best candidate score on the latest decided window (NaN when none).", + **kw, + ) + self.sweep_seconds = Gauge( + f"{PREFIX}_sweep_seconds", + "Wall-clock duration of the latest sweep (NaN when it did not run).", + **kw, + ) + self.missing_candidates = Gauge( + f"{PREFIX}_missing_candidates", + "Candidates of the latest sweep without a score.", + **kw, + ) + self.router_converged = Gauge( + f"{PREFIX}_router_converged", + "1 when every frontend router holds the target policy parameters.", + **kw, + ) + self.router_patch_errors_total = Counter( + f"{PREFIX}_router_patch_errors_total", + "Failed /router_policy PATCH or reset calls, by HTTP status or kind.", + labelnames=["code"], + **kw, + ) + self.trace_sink_missing = Gauge( + f"{PREFIX}_trace_sink_missing", + "1 for a Ready frontend pod without its own trace segment directory.", + labelnames=["pod"], + **kw, + ) + self.trace_dropped_members_total = Counter( + f"{PREFIX}_trace_dropped_members_total", + "Trace gzip members dropped by window cuts for a bad CRC, size or " + "truncation.", + **kw, + ) + for outcome in (*RETUNE_OUTCOMES, *_LIVE_OUTCOMES): + self.decisions_total.labels(outcome=outcome) + for reason in _SKIP_REASONS: + self.skipped_total.labels(reason=reason) + for gauge in (self.baseline_score, self.best_score, self.sweep_seconds): + gauge.set(math.nan) + + +_LOCK = threading.Lock() +_BY_REGISTRY: weakref.WeakKeyDictionary[ + CollectorRegistry, _Collectors +] = weakref.WeakKeyDictionary() + + +def _collectors(registry: CollectorRegistry) -> _Collectors: + with _LOCK: + collectors = _BY_REGISTRY.get(registry) + if collectors is None: + collectors = _Collectors(registry) + _BY_REGISTRY[registry] = collectors + return collectors + + +def _label(value: object) -> str: + text = str(value) + return text if text else "unknown" + + +def _number(value: Optional[float]) -> float: + return math.nan if value is None else float(value) + + +class RetuneMetrics: + """``RetuneMetricsSink`` over Prometheus collectors on ``registry``. + + ``registry`` defaults to ``prometheus_client.REGISTRY``, which the + planner's metrics server exposes. Instances on the same registry share + their collectors. + """ + + def __init__(self, registry: CollectorRegistry | None = None) -> None: + self._m = _collectors(REGISTRY if registry is None else registry) + + def decision(self, outcome: str) -> None: + try: + self._m.decisions_total.labels(outcome=_label(outcome)).inc() + except Exception: + logger.debug("retune decision metric failed", exc_info=True) + + def failure(self, reason: str) -> None: + try: + self._m.failures_total.labels(reason=_label(reason)).inc() + except Exception: + logger.debug("retune failure metric failed", exc_info=True) + + def skipped(self, reason: str) -> None: + try: + self._m.skipped_total.labels(reason=_label(reason)).inc() + except Exception: + logger.debug("retune skipped metric failed", exc_info=True) + + def supervisor_state( + self, + *, + leader: bool, + ready: bool, + initialized: bool, + rollback_pending: bool, + generation: int, + ) -> None: + try: + self._m.leader.set(1 if leader else 0) + self._m.ready.set(1 if ready else 0) + self._m.initialized.set(1 if initialized else 0) + self._m.rollback_pending.set(1 if rollback_pending else 0) + self._m.controller_generation.set(generation) + except Exception: + logger.debug("retune supervisor metrics failed", exc_info=True) + + def scores( + self, + *, + baseline: Optional[float], + best: Optional[float], + sweep_seconds: Optional[float], + missing_candidates: int, + ) -> None: + try: + self._m.baseline_score.set(_number(baseline)) + self._m.best_score.set(_number(best)) + self._m.sweep_seconds.set(_number(sweep_seconds)) + self._m.missing_candidates.set(missing_candidates) + except Exception: + logger.debug("retune score metrics failed", exc_info=True) + + def router_converged(self, converged: bool) -> None: + try: + self._m.router_converged.set(1 if converged else 0) + except Exception: + logger.debug("retune router_converged metric failed", exc_info=True) + + def patch_error(self, code: str) -> None: + try: + self._m.router_patch_errors_total.labels(code=_label(code)).inc() + except Exception: + logger.debug("retune patch error metric failed", exc_info=True) + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + """Set the pod's series to 1, or drop it once the sink is back. + + Dropping instead of writing 0 keeps no series for pods that are gone. + """ + + try: + if missing: + self._m.trace_sink_missing.labels(pod=_label(pod)).set(1) + else: + try: + self._m.trace_sink_missing.remove(_label(pod)) + except KeyError: + pass + except Exception: + logger.debug("retune trace_sink_missing metric failed", exc_info=True) + + def dropped_members(self, count: int) -> None: + try: + if count > 0: + self._m.trace_dropped_members_total.inc(count) + except Exception: + logger.debug("retune dropped members metric failed", exc_info=True) diff --git a/components/src/dynamo/planner/retune/live/reconciler.py b/components/src/dynamo/planner/retune/live/reconciler.py new file mode 100644 index 000000000000..7184900bdd78 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/reconciler.py @@ -0,0 +1,919 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Garbage collection of live retune files and DGDR Requests (design §4.4.5). + +The supervisor runs ``Reconciler.run_once`` only while this process leads and +is initialized. It deletes, with ``fs_root`` the planner's mount of the trace +claim, ``T``/``G`` the retune interval/grace and ``timeout`` the sweep +timeout:: + + ////.NNNNNN.jsonl.gz frontend segments + ///.jsonl.gz window files + ///.tmp-- cut temp files + +- Segments, only when the frontend pod set is known and nonempty: never the + highest-numbered segment of a live pod; other segments of a live pod, and + the whole directory of a pod that no longer exists, once every file is + older than ``T + G + timeout + visibility margin``. A live pod's segments + are the files directly in its directory: P10 requires the sink's file + prefix to sit directly under ``$(DYN_POD_NAME)/``. A pod directory that + cannot be read is skipped for the round; the other pods are still + collected. +- Window files, never while protected: ``window_end_ms`` in + ``[last_window_end_ms, due_window_end_ms]`` with ``now < D_k``, or in use + by this process's sweep (``protect``). Otherwise deleted when at least one + Request references the file and every referencing Request's current Run + has ``Completed=True`` and no Job pod left, or when no Request references + it and it is older than ``T + timeout``. +- Temp files older than ``cut_budget_seconds`` plus a margin, unless their + window is in use by this process's sweep. +- Requests of this DGD name, never one this process's sweep protects: a + terminal Request outside the newest ``keep_requests`` windows, a + non-terminal Request whose window deadline has passed, a Request in the + delete queue, and a Request of another DGD uid (the DGD was recreated). + +``run_once(should_continue=...)`` asks the caller before every delete or +unlink and ends the round as soon as it says no (leadership or the term was +lost while the round ran on its worker thread). ``set_dgd`` follows the DGD +identity when the DGD is recreated under the same name. + +``Reconciler`` is also the ``SweepRegistry`` the DGDR sweep uses to protect +its window and Requests and to queue deletions; those methods are thread-safe +(the sweep runs on the controller's executor thread). Protection is +reference-counted per window: each ``protect`` call takes one reference, +each ``release`` drops one, and the window with the union of every name +given to it stays protected while any reference is held. Two sweeps of the +same window (a rebuilt controller relaunching it while the old one winds +down) therefore cannot drop each other's protection. ``run_once`` blocks on +the filesystem and the Kubernetes API: call it with ``asyncio.to_thread``. +""" + +from __future__ import annotations + +import logging +import os +import re +import shutil +import threading +import time +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Optional + +from dynamo.planner.config.planner_config import DgdrExecutorConfig, RetuneConfig +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD, + LABEL_DGD_UID, + LABEL_WINDOW, + DgdRef, + KubeError, + PodInfo, + RetuneKube, + RetuneMetricsSink, +) + +logger = logging.getLogger(__name__) + +# Allowance for RWX attribute caching and the frontend's flush interval on +# top of T + G + timeout before a segment counts as unused. +DEFAULT_VISIBILITY_MARGIN_S = 600.0 +# Allowance on top of cut_budget_seconds before a temp file counts as +# abandoned (the cut subprocess is killed at its deadline). +DEFAULT_TEMP_MARGIN_S = 300.0 + +WINDOW_SUFFIX = ".jsonl.gz" +TEMP_PREFIX = ".tmp-" +# Label the Job controller puts on a Job's pods (batchv1.JobNameLabel). +JOB_NAME_LABEL = "batch.kubernetes.io/job-name" +_FINISHED_POD_PHASES = frozenset({"Succeeded", "Failed"}) +# Annotation the DGDR v1beta2 Request controller puts on each Run: the +# Request generation it was created for. +RUN_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" +RUN_COMPLETED_CONDITION = "Completed" +REQUEST_ACCEPTED_CONDITION = "Accepted" +REQUEST_INVALID_SPEC_REASON = "InvalidSpec" + +_WINDOW_RE = re.compile(r"^(\d+)" + re.escape(WINDOW_SUFFIX) + r"$") +_TEMP_RE = re.compile(r"^" + re.escape(TEMP_PREFIX) + r"(\d+)-") +# A Rust gzip JSONL sink segment: ..jsonl.gz +_SEGMENT_RE = re.compile(r"^(.*)\.(\d{6,})\.jsonl\.gz$") + + +def window_file_name(window_end_ms: int) -> str: + return f"{int(window_end_ms)}{WINDOW_SUFFIX}" + + +def window_relpath(cfg: DgdrExecutorConfig, dgd_name: str, window_end_ms: int) -> str: + """Claim-relative path of a window file (the Request's trace path).""" + + return "/".join( + (cfg.windows_dir.strip("/"), dgd_name, window_file_name(window_end_ms)) + ) + + +@dataclass(frozen=True) +class ReconcileReport: + """What one ``run_once`` round did.""" + + deleted_segments: int = 0 + deleted_pod_dirs: tuple[str, ...] = () + deleted_windows: tuple[int, ...] = () + deleted_temp_files: int = 0 + deleted_requests: tuple[str, ...] = () + # Ready frontend pods without a segment directory (trace_sink_missing). + sink_missing: tuple[str, ...] = () + errors: dict[str, int] = field(default_factory=dict) + # The round ended early: ``should_continue`` said no, or the DGD + # identity changed (``set_dgd``) while it ran. + stopped: bool = False + + +@dataclass +class _RequestView: + name: str + uid: str + dgd_uid: Optional[str] + window_end_ms: Optional[int] + trace_file: Optional[str] # window file name if it reads one of ours + obj: dict[str, Any] + + +@dataclass +class _Protection: + """One window's protection: references held and the names protected.""" + + count: int = 0 + names: set[str] = field(default_factory=set) + + +@dataclass +class _Round: + """Mutable bookkeeping of one ``run_once`` call. Counts accumulate as + deletions happen, so a phase that fails half way still reports them.""" + + now: float + dgd: DgdRef + segments_root: str + windows_root: str + should_continue: Optional[Callable[[], bool]] + errors: dict[str, int] = field(default_factory=dict) + stopped: bool = False + deleted_segments: int = 0 + deleted_dirs: list[str] = field(default_factory=list) + deleted_windows: list[int] = field(default_factory=list) + deleted_temp: int = 0 + deleted_requests: list[str] = field(default_factory=list) + sink_missing: list[str] = field(default_factory=list) + + def error(self, kind: str) -> None: + self.errors[kind] = self.errors.get(kind, 0) + 1 + + +class Reconciler: + """Deletes what live retune no longer needs; see the module docstring.""" + + def __init__( + self, + kube: RetuneKube, + dgd: DgdRef, + cfg: DgdrExecutorConfig, + retune: RetuneConfig, + fs_root: str, + metrics: RetuneMetricsSink, + now: Callable[[], float] = time.time, + *, + visibility_margin_s: float = DEFAULT_VISIBILITY_MARGIN_S, + temp_margin_s: float = DEFAULT_TEMP_MARGIN_S, + kube_timeout_s: float = 10.0, + ) -> None: + self._kube = kube + self._cfg = cfg + self._fs_root = fs_root + self._metrics = metrics + self._now = now + self._timeout_s = float(kube_timeout_s) + interval_s = float(retune.interval_seconds) + sweep_timeout_s = float(retune.effective_timeout_seconds) + self._segment_max_age_s = ( + interval_s + + float(retune.grace_seconds) + + sweep_timeout_s + + float(visibility_margin_s) + ) + self._window_max_age_s = interval_s + sweep_timeout_s + self._temp_max_age_s = float(cfg.cut_budget_seconds) + float(temp_margin_s) + + self._lock = threading.Lock() + # Guarded by _lock (set_dgd may run while a round is on its thread). + self._dgd = dgd + self._segments_root = self._root(cfg.segments_dir, dgd) + self._windows_root = self._root(cfg.windows_dir, dgd) + # window_end_ms -> references and Request names of running sweeps. + self._protected: dict[int, _Protection] = {} + self._delete_queue: set[str] = set() + # (request uid, generation) of Requests known terminal / finished; + # neither can revert for one generation. + self._terminal: set[tuple[str, int]] = set() + self._finished: set[tuple[str, int]] = set() + self._sink_reported: set[str] = set() + + def _root(self, directory: str, dgd: DgdRef) -> str: + return os.path.join(self._fs_root, directory.strip("/"), dgd.name) + + # ------------------------------------------------------------- identity + + def set_dgd(self, dgd: DgdRef) -> None: + """Follow the DGD identity (it was recreated under the same name). + + Requests labelled with the new uid become this DGD's; those of the + old uid are then deleted as ``other_dgd_uid``. The Request-state + caches are cleared. A round running on another thread ends at its + next delete, since it classified Requests by the old identity. + """ + + with self._lock: + if dgd != self._dgd: + logger.info( + "retune reconciler: DGD %s uid %s -> %s %s", + self._dgd.name, + self._dgd.uid, + dgd.name, + dgd.uid, + ) + self._dgd = dgd + self._segments_root = self._root(self._cfg.segments_dir, dgd) + self._windows_root = self._root(self._cfg.windows_dir, dgd) + self._terminal.clear() + self._finished.clear() + + @property + def dgd(self) -> DgdRef: + with self._lock: + return self._dgd + + # --------------------------------------------------------- SweepRegistry + + def protect(self, window_end_ms: int, request_names: Iterable[str]) -> None: + """Take one reference on the window and add ``request_names`` to + its protected names; each call needs a matching ``release``.""" + + names = set(request_names) + with self._lock: + entry = self._protected.setdefault(int(window_end_ms), _Protection()) + entry.count += 1 + entry.names.update(names) + + def release(self, window_end_ms: int) -> None: + """Drop one reference; the window and its names are unprotected + once no reference is left. Releasing an unknown window is a no-op.""" + + with self._lock: + window = int(window_end_ms) + entry = self._protected.get(window) + if entry is None: + return + entry.count -= 1 + if entry.count <= 0: + del self._protected[window] + + def enqueue_delete(self, request_names: Iterable[str]) -> None: + names = set(request_names) + with self._lock: + self._delete_queue.update(names) + + # ---------------------------------------------------------- inspection + + @property + def segments_root(self) -> str: + with self._lock: + return self._segments_root + + @property + def windows_root(self) -> str: + with self._lock: + return self._windows_root + + def protected_windows(self) -> frozenset[int]: + with self._lock: + return frozenset(self._protected) + + def protected_requests(self) -> frozenset[str]: + with self._lock: + return frozenset( + name for entry in self._protected.values() for name in entry.names + ) + + def pending_deletes(self) -> frozenset[str]: + with self._lock: + return frozenset(self._delete_queue) + + # ---------------------------------------------------------------- round + + def run_once( + self, + *, + last_window_end_ms: Optional[int], + due_window_end_ms: Optional[int], + deadline_for: Callable[[int], float], + frontend_pods: Optional[Sequence[PodInfo]], + should_continue: Optional[Callable[[], bool]] = None, + ) -> ReconcileReport: + """One GC round. ``deadline_for(window_end_ms)`` is that window's + sweep deadline ``D_k`` in epoch seconds; ``frontend_pods`` is the + actuator's listing (None when it failed). ``should_continue`` is + called before every delete or unlink; once it returns False (or + raises) the round deletes nothing more and returns.""" + + now = float(self._now()) + with self._lock: + state = _Round( + now=now, + dgd=self._dgd, + segments_root=self._segments_root, + windows_root=self._windows_root, + should_continue=should_continue, + ) + + def in_window_range(window_end_ms: int) -> bool: + """The design's protected range: undecided and before D_k.""" + + if last_window_end_ms is not None and window_end_ms < last_window_end_ms: + return False + if due_window_end_ms is not None and window_end_ms > due_window_end_ms: + return False + return now < deadline_for(window_end_ms) + + try: + self._collect_segments(state, frontend_pods) + except Exception: # keep the sink check, Request and window GC running + logger.exception("retune reconciler: segment GC failed") + state.error("segments") + if not state.stopped: + try: + state.sink_missing = self._report_sinks(state, frontend_pods) + except Exception: + logger.exception("retune reconciler: trace sink check failed") + state.error("sinks") + + requests: Optional[list[_RequestView]] = None + if not state.stopped: + try: + requests = self._list_requests(state.dgd) + except KubeError as exc: + logger.warning("retune reconciler: listing Requests failed: %s", exc) + state.error("list_requests") + except Exception: + logger.exception("retune reconciler: listing Requests failed") + state.error("list_requests") + try: + self._collect_requests(state, requests, in_window_range, deadline_for) + except Exception: + logger.exception("retune reconciler: Request GC failed") + state.error("requests") + + if not state.stopped: + try: + remaining = ( + None + if requests is None + else [ + view + for view in requests + if view.name not in state.deleted_requests + ] + ) + self._collect_windows(state, remaining, in_window_range) + except Exception: + logger.exception("retune reconciler: window GC failed") + state.error("windows") + + report = ReconcileReport( + deleted_segments=state.deleted_segments, + deleted_pod_dirs=tuple(sorted(state.deleted_dirs)), + deleted_windows=tuple(sorted(state.deleted_windows)), + deleted_temp_files=state.deleted_temp, + deleted_requests=tuple(sorted(state.deleted_requests)), + sink_missing=tuple(sorted(state.sink_missing)), + errors=state.errors, + stopped=state.stopped, + ) + if ( + report.deleted_segments + or report.deleted_pod_dirs + or report.deleted_windows + or report.deleted_temp_files + or report.deleted_requests + ): + logger.info( + "retune reconciler: deleted %d segments, pod dirs %s, windows %s, " + "%d temp files, Requests %s", + report.deleted_segments, + list(report.deleted_pod_dirs), + list(report.deleted_windows), + report.deleted_temp_files, + list(report.deleted_requests), + ) + return report + + def _proceed(self, state: _Round) -> bool: + """Whether the round may delete one more thing; asked right before + every delete or unlink. A no is final for the round.""" + + if state.stopped: + return False + with self._lock: + current = self._dgd + if current != state.dgd: + logger.info( + "retune reconciler: the DGD identity changed during the round; " + "stopping it" + ) + state.stopped = True + return False + if state.should_continue is None: + return True + try: + proceed = bool(state.should_continue()) + except Exception: + logger.exception("retune reconciler: should_continue failed") + proceed = False + if not proceed: + logger.info("retune reconciler: told to stop; ending the round early") + state.stopped = True + return proceed + + # ------------------------------------------------------------- segments + + def _collect_segments( + self, state: _Round, frontend_pods: Optional[Sequence[PodInfo]] + ) -> None: + if not frontend_pods: + # Unknown or empty pod set: every directory would look orphaned. + return + live = {pod.name for pod in frontend_pods} + cutoff = state.now - self._segment_max_age_s + for entry in _scandir(state.segments_root): + if state.stopped: + return + # One pod directory that cannot be read (another uid, umask 077) + # must not starve the pods after it in scandir order. + try: + if not entry.is_dir(follow_symlinks=False): + continue + if entry.name in live: + self._collect_pod_segments(state, entry.path, cutoff) + continue + if _newest_mtime(entry.path) >= cutoff: + continue + if not self._proceed(state): + return + shutil.rmtree(entry.path) + state.deleted_dirs.append(entry.name) + except FileNotFoundError: + continue + except OSError as exc: + logger.warning( + "retune reconciler: segment GC of %s failed: %s", entry.path, exc + ) + state.error("segments") + + def _collect_pod_segments( + self, state: _Round, directory: str, cutoff: float + ) -> None: + # Top level only: P10 puts the sink's file prefix directly under the + # pod directory, so there is no subdirectory to walk. + segments: dict[str, list[tuple[int, os.DirEntry[str]]]] = {} + for entry in _scandir(directory): + match = _SEGMENT_RE.match(entry.name) + if match is None or not entry.is_file(follow_symlinks=False): + continue + segments.setdefault(match.group(1), []).append((int(match.group(2)), entry)) + for entries in segments.values(): + entries.sort(key=lambda item: item[0]) + # The highest-numbered segment is the one the frontend appends to. + for _, entry in entries[:-1]: + try: + if entry.stat(follow_symlinks=False).st_mtime >= cutoff: + continue + if not self._proceed(state): + return + os.unlink(entry.path) + state.deleted_segments += 1 + except FileNotFoundError: + continue + except OSError as exc: + logger.warning( + "retune reconciler: removing %s failed: %s", entry.path, exc + ) + state.error("segments") + + def _report_sinks( + self, state: _Round, frontend_pods: Optional[Sequence[PodInfo]] + ) -> list[str]: + if frontend_pods is None: + return [] + missing: list[str] = [] + current: set[str] = set() + for pod in frontend_pods: + if not pod.ready or pod.terminating: + continue + current.add(pod.name) + absent = not os.path.isdir(os.path.join(state.segments_root, pod.name)) + if absent: + missing.append(pod.name) + self._metrics.trace_sink_missing(pod.name, absent) + for name in self._sink_reported - current: + self._metrics.trace_sink_missing(name, False) + self._sink_reported = current + if missing: + logger.warning( + "retune reconciler: Ready frontends without a trace directory " + "under %s: %s", + state.segments_root, + missing, + ) + return missing + + # -------------------------------------------------------------- windows + + def _collect_windows( + self, + state: _Round, + requests: Optional[list[_RequestView]], + in_window_range: Callable[[int], bool], + ) -> None: + protected = self.protected_windows() + references: dict[str, list[_RequestView]] = {} + for view in requests or (): + if view.trace_file is not None: + references.setdefault(view.trace_file, []).append(view) + + for entry in _scandir(state.windows_root): + if state.stopped: + return + try: + self._collect_window_entry( + state, entry, requests, references, protected, in_window_range + ) + except FileNotFoundError: + continue + except OSError as exc: + logger.warning( + "retune reconciler: window GC of %s failed: %s", entry.path, exc + ) + state.error("windows") + + def _collect_window_entry( + self, + state: _Round, + entry: os.DirEntry[str], + requests: Optional[list[_RequestView]], + references: Mapping[str, list[_RequestView]], + protected: frozenset[int], + in_window_range: Callable[[int], bool], + ) -> None: + if not entry.is_file(follow_symlinks=False): + return + if entry.name.startswith(TEMP_PREFIX): + temp = _TEMP_RE.match(entry.name) + window = int(temp.group(1)) if temp is not None else None + # A temp file of a window this process's sweep is cutting + # belongs to the running cut subprocess. + if window is not None and window in self.protected_windows(): + return + if self._older_than(entry, state.now, self._temp_max_age_s): + if self._unlink(state, entry.path): + state.deleted_temp += 1 + return + match = _WINDOW_RE.match(entry.name) + if match is None or requests is None: + # Unknown file, or references unknown: keep it. + return + window = int(match.group(1)) + if window in protected or in_window_range(window): + return + referrers = references.get(entry.name, []) + if referrers: + if not all(self._finished_with(state, view) for view in referrers): + return + elif not self._older_than(entry, state.now, self._window_max_age_s): + return + # The sweep may have protected it since the snapshot above. + if window in self.protected_windows(): + return + if self._unlink(state, entry.path): + state.deleted_windows.append(window) + + # ------------------------------------------------------------- requests + + def _list_requests(self, dgd: DgdRef) -> list[_RequestView]: + items = self._kube.list_requests( + f"{LABEL_DGD}={dgd.name}", timeout_s=self._timeout_s + ) + views = [] + prefix = self._cfg.windows_dir.strip("/") + "/" + dgd.name + "/" + for obj in items: + metadata = obj.get("metadata") or {} + labels = metadata.get("labels") or {} + pvc = _dig(obj, "spec", "workload", "trace", "source", "pvc") or {} + path = pvc.get("path") + trace_file = None + if ( + pvc.get("claimName") == self._cfg.claim_name + and isinstance(path, str) + and path.startswith(prefix) + and "/" not in path[len(prefix) :] + ): + trace_file = path[len(prefix) :] + views.append( + _RequestView( + name=str(metadata.get("name", "")), + uid=str(metadata.get("uid", "")), + dgd_uid=labels.get(LABEL_DGD_UID), + window_end_ms=_int_or_none(labels.get(LABEL_WINDOW)), + trace_file=trace_file, + obj=obj, + ) + ) + return [view for view in views if view.name] + + def _collect_requests( + self, + state: _Round, + requests: Optional[list[_RequestView]], + in_window_range: Callable[[int], bool], + deadline_for: Callable[[int], float], + ) -> None: + now = state.now + dgd_uid = state.dgd.uid + reasons: dict[str, str] = {name: "queued" for name in self.pending_deletes()} + if requests is not None: + listed = {view.uid for view in requests} + with self._lock: + for cache in (self._terminal, self._finished): + cache.difference_update( + [key for key in cache if key[0] not in listed] + ) + ours = [view for view in requests if view.dgd_uid == dgd_uid] + for view in requests: + if view.dgd_uid is None: + logger.warning( + "retune reconciler: Request %s has no %s label", + view.name, + LABEL_DGD_UID, + ) + elif view.dgd_uid != dgd_uid: + # The DGD was deleted and recreated under the same name. + reasons.setdefault(view.name, "other_dgd_uid") + states = {view.name: self._is_terminal(state, view) for view in ours} + terminal = {name for name, value in states.items() if value is True} + kept_windows = sorted( + { + view.window_end_ms + for view in ours + if view.name in terminal and view.window_end_ms is not None + }, + reverse=True, + )[: self._cfg.keep_requests] + for view in ours: + window = view.window_end_ms + if window is None: + logger.warning( + "retune reconciler: Request %s has no %s label", + view.name, + LABEL_WINDOW, + ) + continue + value = states[view.name] + if value is None: + continue # Run state unknown this round + if value: + if window not in kept_windows and not in_window_range(window): + reasons.setdefault(view.name, "terminal") + elif now >= deadline_for(window): + # Not terminal and past D_k: no sweep can adopt it anymore. + reasons.setdefault(view.name, "deadline_passed") + + for name in sorted(reasons): + if name in self.protected_requests(): + continue + if not self._proceed(state): + return + try: + self._kube.delete_request(name, timeout_s=self._timeout_s) + except KubeError as exc: + logger.warning( + "retune reconciler: deleting Request %s (%s) failed: %s", + name, + reasons[name], + exc, + ) + state.error("delete_request") + continue + logger.info( + "retune reconciler: deleted Request %s (%s)", name, reasons[name] + ) + state.deleted_requests.append(name) + with self._lock: + self._delete_queue.discard(name) + + # ------------------------------------------------- Request / Run state + + def _cached(self, cache: set[tuple[str, int]], key: tuple[str, int]) -> bool: + with self._lock: + return key in cache + + def _remember( + self, state: _Round, cache: set[tuple[str, int]], key: tuple[str, int] + ) -> None: + """Cache a verdict unless ``set_dgd`` cleared the caches meanwhile + (the caches are cleared in place, never replaced).""" + + with self._lock: + if self._dgd == state.dgd: + cache.add(key) + + def _is_terminal(self, state: _Round, view: _RequestView) -> Optional[bool]: + """Whether the Request's current-generation Run completed, or the + operator rejected the current generation (InvalidSpec); None when + the Run could not be read.""" + + generation = _int_or_none(_dig(view.obj, "metadata", "generation")) or 0 + key = (view.uid, generation) + if self._cached(self._terminal, key): + return True + try: + run = self._current_run(view, generation) + except KubeError as exc: + logger.warning( + "retune reconciler: reading the Run of Request %s failed: %s", + view.name, + exc, + ) + state.error("get_run") + return None + terminal = ( + _condition_true(run, RUN_COMPLETED_CONDITION) + if run is not None + else _invalid_spec(view.obj, generation) + ) + if terminal: + self._remember(state, self._terminal, key) + return terminal + + def _finished_with(self, state: _Round, view: _RequestView) -> bool: + """The Request no longer reads its window file: terminal, no active + Run, and no pod of the current Run's Job left.""" + + generation = _int_or_none(_dig(view.obj, "metadata", "generation")) or 0 + key = (view.uid, generation) + if self._cached(self._finished, key): + return True + if _dig(view.obj, "status", "activeRunRef", "name"): + return False + try: + run = self._current_run(view, generation) + if run is None: + finished = _invalid_spec(view.obj, generation) + elif not _condition_true(run, RUN_COMPLETED_CONDITION): + finished = False + else: + job = _dig(run, "status", "jobRef", "name") + # A Succeeded or Failed pod no longer reads the trace; one + # with no phase reported is treated as still running. + finished = job is None or not any( + pod.phase not in _FINISHED_POD_PHASES + for pod in self._kube.list_pods( + f"{JOB_NAME_LABEL}={job}", timeout_s=self._timeout_s + ) + ) + except KubeError as exc: + logger.warning( + "retune reconciler: reading Run state of Request %s failed: %s", + view.name, + exc, + ) + return False + if finished: + self._remember(state, self._finished, key) + return finished + + def _current_run( + self, view: _RequestView, generation: int + ) -> Optional[dict[str, Any]]: + """The Run created for the Request's current generation, if any. + + The Run's generation annotation identifies it, whatever its name; + the Request status names the active and the retained terminal Runs. + """ + + status = view.obj.get("status") or {} + names = [] + active = (status.get("activeRunRef") or {}).get("name") + if active: + names.append(active) + names.extend( + ref.get("name") + for ref in status.get("recentRunRefs") or () + if ref.get("name") + ) + for name in names: + run = self._kube.get_run(name, timeout_s=self._timeout_s) + if run is None: + continue + annotations = _dig(run, "metadata", "annotations") or {} + if _int_or_none(annotations.get(RUN_GENERATION_ANNOTATION)) == generation: + return run + return None + + # ---------------------------------------------------------- filesystem + + @staticmethod + def _older_than(entry: os.DirEntry[str], now: float, age_s: float) -> bool: + try: + return entry.stat(follow_symlinks=False).st_mtime < now - age_s + except FileNotFoundError: + return False + + def _unlink(self, state: _Round, path: str) -> bool: + if not self._proceed(state): + return False + try: + os.unlink(path) + return True + except FileNotFoundError: + return False + except OSError as exc: + logger.warning("retune reconciler: removing %s failed: %s", path, exc) + state.error("windows") + return False + + +# ---------------------------------------------------------------- helpers + + +def _scandir(path: str) -> list[os.DirEntry[str]]: + """Entries of ``path``; none when it is missing. Other ``OSError``s + (e.g. ``PermissionError``) propagate to the caller's per-entry handler.""" + + try: + with os.scandir(path) as entries: + return list(entries) + except (FileNotFoundError, NotADirectoryError): + return [] + + +def _newest_mtime(path: str) -> float: + """Newest mtime of ``path`` and everything under it.""" + + newest = os.lstat(path).st_mtime + for root, dirs, files in os.walk(path): + for name in dirs + files: + try: + newest = max(newest, os.lstat(os.path.join(root, name)).st_mtime) + except FileNotFoundError: + continue + return newest + + +def _dig(obj: Any, *keys: str) -> Any: + for key in keys: + if not isinstance(obj, Mapping): + return None + obj = obj.get(key) + return obj + + +def _int_or_none(value: Any) -> Optional[int]: + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _conditions(obj: Optional[Mapping[str, Any]]) -> list[Mapping[str, Any]]: + conditions = _dig(obj, "status", "conditions") + return [c for c in conditions or () if isinstance(c, Mapping)] + + +def _condition_true(obj: Optional[Mapping[str, Any]], kind: str) -> bool: + return any( + c.get("type") == kind and c.get("status") == "True" for c in _conditions(obj) + ) + + +def _invalid_spec(request: Mapping[str, Any], generation: int) -> bool: + """The operator rejected this generation; only a new one changes that.""" + + for condition in _conditions(request): + if ( + condition.get("type") == REQUEST_ACCEPTED_CONDITION + and condition.get("status") == "False" + and condition.get("reason") == REQUEST_INVALID_SPEC_REASON + and _int_or_none(condition.get("observedGeneration")) == generation + ): + return True + return False diff --git a/components/src/dynamo/planner/retune/live/state.py b/components/src/dynamo/planner/retune/live/state.py new file mode 100644 index 000000000000..598fc2f9996d --- /dev/null +++ b/components/src/dynamo/planner/retune/live/state.py @@ -0,0 +1,889 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Persisted live retune state and the leadership Lease (design §4.7.1-§4.7.2). + +Two ``coordination.k8s.io/v1`` Leases per DGD, both owned by the DGD so the +garbage collector removes them with it: + +- ``-planner-retune``: leadership. ``LeaseLeadership`` renews it with + get -> check holder -> update(resourceVersion), takes over an empty holder + at once and a foreign holder only after observing it unchanged (holder and + resourceVersion) for a whole lease duration on the local monotonic clock. +- ``-planner-retune-state``: the state snapshot (``RetuneState``) as + JSON in two annotations. Every write replaces the snapshot and is + conditional on the resourceVersion read; the supervisor additionally + requires ``state.leader_term`` to equal its own term for every write but + the claim. + +The write-rule functions below return a new ``RetuneState`` and never touch +Kubernetes; ``StateStore`` reads and writes the snapshot. Everything here is +synchronous: callers on the event loop run it in a thread. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import logging +import math +import threading +import time +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Literal, Optional + +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD, + LABEL_DGD_UID, + DgdRef, + FrontendStartup, + KubeAlreadyExists, + KubeConflict, + KubeError, + RetuneKube, +) + +if TYPE_CHECKING: + from dynamo.planner.config.planner_config import PlannerConfig + +logger = logging.getLogger(__name__) + +STATE_ANNOTATION = "nvidia.com/planner-retune-state" +HISTORY_ANNOTATION = "nvidia.com/planner-retune-history" +HISTORY_LIMIT = 50 +STATE_FORMAT_VERSION = 1 + +DGD_API_VERSION = "nvidia.com/v1beta1" +DGD_KIND = "DynamoGraphDeployment" +LEASE_API_VERSION = "coordination.k8s.io/v1" + +RetuneMode = Literal["active", "advisory"] +_MODES = ("active", "advisory") +_REL_TOL = 1e-9 +_ABS_TOL = 1e-12 + + +def leadership_lease_name(dgd_name: str) -> str: + return f"{dgd_name}-planner-retune" + + +def state_lease_name(dgd_name: str) -> str: + return f"{dgd_name}-planner-retune-state" + + +def owner_reference(dgd: DgdRef) -> dict[str, Any]: + """ownerReference to the DGD; the GC deletes the Leases with it.""" + + return { + "apiVersion": DGD_API_VERSION, + "kind": DGD_KIND, + "name": dgd.name, + "uid": dgd.uid, + "blockOwnerDeletion": False, + } + + +def leader_term(lease_transitions: int, identity: str, term_seq: int) -> str: + """``//``. + + ``term_seq`` is bumped on every entry into initialization, so a write of + an earlier term (another process, or this one before re-initializing) + always fails the ``leader_term`` precondition. + """ + + return f"{int(lease_transitions)}/{identity}/{int(term_seq)}" + + +def micro_time(epoch_s: float) -> str: + """A Kubernetes ``MicroTime`` (RFC 3339 with microseconds, UTC).""" + + stamp = datetime.fromtimestamp(epoch_s, tz=timezone.utc) + return stamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def canonical_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def maps_equal( + left: Optional[Mapping[str, Any]], right: Optional[Mapping[str, Any]] +) -> bool: + """Deep equality of parameter maps, floats within a tiny tolerance.""" + + if left is None or right is None: + return left is None and right is None + if set(left) != set(right): + return False + for key, value in left.items(): + other = right[key] + if isinstance(value, Mapping) or isinstance(other, Mapping): + if not ( + isinstance(value, Mapping) + and isinstance(other, Mapping) + and maps_equal(value, other) + ): + return False + elif isinstance(value, (int, float)) and isinstance(other, (int, float)): + if not math.isclose( + float(value), float(other), rel_tol=_REL_TOL, abs_tol=_ABS_TOL + ): + return False + elif value != other: + return False + return True + + +def fingerprint(config: PlannerConfig, startup: Optional[FrontendStartup]) -> str: + """What the deployed point was chosen under; a change resets it. + + Covers the router search domain as configured (its expansion is a pure + function of it and the startup), the parsed frontend startup, the pinned + DGDR engine section with its model, SKU and backend, the planner mode, + the objective and the scoring SLA. ``advisory`` is not part of it: the + state's ``mode`` records it. + """ + + retune = config.retune + if retune is None: + raise ValueError("fingerprint requires config.retune") + dgdr = retune.dgdr + sla_ttft_ms, sla_itl_ms = retune.scoring_sla(config.ttft_ms, config.itl_ms) + payload = { + "version": STATE_FORMAT_VERSION, + "router": retune.router, + "planner": retune.planner, + "objective": retune.objective, + "sla": [float(sla_ttft_ms), float(sla_itl_ms)], + "mode": config.mode, + "backend": config.backend, + "engine": None if dgdr is None else dgdr.engine, + "model": None if dgdr is None else dgdr.model, + "sku": None if dgdr is None else dgdr.sku, + "startup": None if startup is None else startup.fingerprint_source, + } + digest = hashlib.sha1(canonical_json(payload).encode("utf-8")).hexdigest() + return digest[:16] + + +# ------------------------------------------------------------------ state + + +class StateFormatError(ValueError): + """The state annotations do not hold a readable snapshot.""" + + +def _float_map(value: Any, what: str) -> dict[str, float]: + if not isinstance(value, Mapping): + raise StateFormatError(f"{what} must be an object") + out: dict[str, float] = {} + for key, item in value.items(): + if isinstance(item, bool) or not isinstance(item, (int, float)): + raise StateFormatError(f"{what}.{key} must be a number") + out[str(key)] = float(item) + return out + + +def _nested_map(value: Any, what: str) -> dict[str, dict[str, float]]: + if not isinstance(value, Mapping): + raise StateFormatError(f"{what} must be an object") + return {str(key): _float_map(item, f"{what}.{key}") for key, item in value.items()} + + +@dataclass(frozen=True) +class RetuneState: + """The state snapshot of design §4.7.2. + + ``mode is None`` is "no state": only the claim (``dgd_uid`` and + ``leader_term``) has been written. ``by_wt`` is the deployed per worker + type 8-parameter map, ``flat`` the controller's router point (live + names), ``initials`` the latched initial configuration of each router. + ``history`` is newest last, at most ``HISTORY_LIMIT`` entries. + """ + + dgd_uid: str + leader_term: str = "" + fingerprint: Optional[str] = None + mode: Optional[RetuneMode] = None + rollback_pending: bool = False + deploy_seq: int = 0 + flat: dict[str, float] = field(default_factory=dict) + by_wt: dict[str, dict[str, float]] = field(default_factory=dict) + initials: dict[str, dict[str, float]] = field(default_factory=dict) + last_window_end_ms: Optional[int] = None + updated_ms: int = 0 + history: tuple[dict[str, Any], ...] = () + + @property + def has_deployment(self) -> bool: + """Whether an initialization has recorded what is deployed.""" + + return self.mode is not None + + def to_annotations(self) -> dict[str, str]: + body = { + "version": STATE_FORMAT_VERSION, + "dgd_uid": self.dgd_uid, + "leader_term": self.leader_term, + "fingerprint": self.fingerprint, + "mode": self.mode, + "rollback_pending": self.rollback_pending, + "deploy_seq": self.deploy_seq, + "flat": self.flat, + "by_wt": self.by_wt, + "initials": self.initials, + "last_window_end_ms": self.last_window_end_ms, + "updated_ms": self.updated_ms, + } + return { + STATE_ANNOTATION: canonical_json(body), + HISTORY_ANNOTATION: canonical_json(list(self.history)), + } + + @classmethod + def from_annotations( + cls, annotations: Optional[Mapping[str, str]] + ) -> Optional[RetuneState]: + """Parse the snapshot; ``None`` when the lease holds none. + + Raises ``StateFormatError`` for a snapshot that cannot be read. + """ + + raw = (annotations or {}).get(STATE_ANNOTATION) + if raw is None: + return None + try: + body = json.loads(raw) + except ValueError as error: + raise StateFormatError(f"state annotation is not JSON: {error}") from None + if not isinstance(body, Mapping): + raise StateFormatError("state annotation must be a JSON object") + dgd_uid = body.get("dgd_uid") + if not isinstance(dgd_uid, str) or not dgd_uid: + raise StateFormatError("state.dgd_uid must be a non-empty string") + mode = body.get("mode") + if mode is not None and mode not in _MODES: + raise StateFormatError(f"state.mode must be one of {_MODES} or null") + deploy_seq = body.get("deploy_seq", 0) + if isinstance(deploy_seq, bool) or not isinstance(deploy_seq, int): + raise StateFormatError("state.deploy_seq must be an integer") + last = body.get("last_window_end_ms") + if last is not None and ( + isinstance(last, bool) or not isinstance(last, (int, float)) + ): + raise StateFormatError("state.last_window_end_ms must be a number") + history_raw = (annotations or {}).get(HISTORY_ANNOTATION) + history: list[dict[str, Any]] = [] + if history_raw: + try: + parsed = json.loads(history_raw) + except ValueError: + # History is an audit trail; a damaged one never blocks. + logger.warning("retune state: unreadable history annotation dropped") + parsed = [] + if isinstance(parsed, list): + history = [dict(item) for item in parsed if isinstance(item, Mapping)] + return cls( + dgd_uid=dgd_uid, + leader_term=str(body.get("leader_term") or ""), + fingerprint=body.get("fingerprint"), + mode=mode, + rollback_pending=bool(body.get("rollback_pending", False)), + deploy_seq=int(deploy_seq), + flat=_float_map(body.get("flat") or {}, "state.flat"), + by_wt=_nested_map(body.get("by_wt") or {}, "state.by_wt"), + initials=_nested_map(body.get("initials") or {}, "state.initials"), + last_window_end_ms=None if last is None else int(last), + updated_ms=int(body.get("updated_ms") or 0), + history=tuple(history[-HISTORY_LIMIT:]), + ) + + +def deployed_off( + state: Optional[RetuneState], + startup_by_wt: Optional[Mapping[str, Mapping[str, float]]], +) -> bool: + """Whether the routers may be off their startup values (§4.7.3 step 4). + + ``rollback_pending ∨ by_wt ≠ startup``, false without a state, with one + refinement: only an ``active`` record can have deployed anything. An + ``advisory`` record without ``rollback_pending`` was written by a planner + that only observed (or finished rolling back), so its ``by_wt`` is the + startup it saw, not a deployment; after a frontend startup change it + must not make an advisory planner reset routers it never wrote. With an + unknown startup (the frontend could not be parsed) an active record + counts as deployed: only the routers' latched initials can tell. + """ + + if state is None or not state.has_deployment: + return False + if state.rollback_pending: + return True + if state.mode != "active": + return False + if startup_by_wt is None: + return True + return not maps_equal(state.by_wt, startup_by_wt) + + +# -------------------------------------------------------- write rules (§4.7.2) + + +def claim(state: Optional[RetuneState], *, dgd_uid: str, term: str) -> RetuneState: + """Claim write: only ``leader_term`` changes (a new record without state).""" + + if state is None: + return RetuneState(dgd_uid=dgd_uid, leader_term=term) + return dataclasses.replace(state, leader_term=term) + + +def _replace_deployment( + state: RetuneState, + *, + fingerprint: str, + mode: RetuneMode, + flat: Mapping[str, float], + by_wt: Mapping[str, Mapping[str, float]], + rollback_pending: bool, +) -> RetuneState: + return dataclasses.replace( + state, + fingerprint=fingerprint, + mode=mode, + flat={key: float(value) for key, value in flat.items()}, + by_wt={wt: dict(params) for wt, params in by_wt.items()}, + rollback_pending=bool(rollback_pending), + deploy_seq=state.deploy_seq + 1, + ) + + +def deploy( + state: RetuneState, + *, + fingerprint: str, + mode: RetuneMode, + flat: Mapping[str, float], + by_wt: Mapping[str, Mapping[str, float]], +) -> RetuneState: + """applied / promote: replace the deployment, clear rollback_pending.""" + + return _replace_deployment( + state, + fingerprint=fingerprint, + mode=mode, + flat=flat, + by_wt=by_wt, + rollback_pending=False, + ) + + +def rollback_start( + state: RetuneState, + *, + fingerprint: str, + mode: RetuneMode, + flat: Mapping[str, float], + by_wt: Mapping[str, Mapping[str, float]], +) -> RetuneState: + """Rollback start (advisory, or spec_ok false): startup, pending set.""" + + return _replace_deployment( + state, + fingerprint=fingerprint, + mode=mode, + flat=flat, + by_wt=by_wt, + rollback_pending=True, + ) + + +def reset( + state: RetuneState, + *, + fingerprint: str, + mode: RetuneMode, + flat: Mapping[str, float], + by_wt: Mapping[str, Mapping[str, float]], + rollback_pending: bool, +) -> RetuneState: + """Reset (fingerprint change, no state, mode switch): startup, + ``rollback_pending = deployed_off``.""" + + return _replace_deployment( + state, + fingerprint=fingerprint, + mode=mode, + flat=flat, + by_wt=by_wt, + rollback_pending=rollback_pending, + ) + + +def rollback_complete( + state: RetuneState, + *, + expected_deploy_seq: int, + entry: Optional[Mapping[str, Any]] = None, +) -> Optional[RetuneState]: + """Flip only ``rollback_pending`` to false. + + ``None`` (discard, never retry) when the deployment changed since the + rollback started (``deploy_seq``) or the rollback is already complete. + ``entry`` (a ``rolled_back`` record) is appended to the history. + """ + + if not state.rollback_pending or state.deploy_seq != expected_deploy_seq: + return None + new = dataclasses.replace(state, rollback_pending=False) + if entry is not None: + new = append_history(new, entry) + return new + + +def merge_initials( + state: RetuneState, + latched: Mapping[str, Mapping[str, float]], + *, + keep: Optional[Iterable[str]] = None, +) -> RetuneState: + """Merge newly latched initials; drop router ids not in ``keep``.""" + + initials = {rid: dict(cfg) for rid, cfg in state.initials.items()} + for router_id, configuration in latched.items(): + initials[str(router_id)] = {k: float(v) for k, v in configuration.items()} + if keep is not None: + kept = set(keep) + initials = {rid: cfg for rid, cfg in initials.items() if rid in kept} + return dataclasses.replace(state, initials=initials) + + +def append_history(state: RetuneState, entry: Mapping[str, Any]) -> RetuneState: + """Append one entry, deduplicated by ``decision_id``; keep the newest 50.""" + + decision_id = entry.get("decision_id") + history = [ + item + for item in state.history + if decision_id is None or item.get("decision_id") != decision_id + ] + history.append(dict(entry)) + return dataclasses.replace(state, history=tuple(history[-HISTORY_LIMIT:])) + + +def has_decision(state: RetuneState, decision_id: str) -> bool: + return any(item.get("decision_id") == decision_id for item in state.history) + + +def record_decision( + state: RetuneState, entry: Mapping[str, Any], window_end_ms: Optional[int] +) -> RetuneState: + """Every decision: history entry, ``last_window_end_ms = max(old, end)``.""" + + new = append_history(state, entry) + if window_end_ms is None: + return new + last = state.last_window_end_ms + return dataclasses.replace( + new, + last_window_end_ms=int(window_end_ms) + if last is None + else max(last, int(window_end_ms)), + ) + + +# ----------------------------------------------------------------- store + + +@dataclass(frozen=True) +class StateSnapshot: + """One read of the state Lease. ``lease is None``: it does not exist.""" + + state: Optional[RetuneState] + resource_version: Optional[str] + lease: Optional[dict[str, Any]] + # The annotations held a snapshot that could not be parsed. + corrupt: bool = False + + +class StateStore: + """Reads and conditionally writes ``-planner-retune-state``.""" + + def __init__( + self, + kube: RetuneKube, + dgd: DgdRef, + *, + timeout_s: float = 5.0, + clock: Callable[[], float] = time.time, + ) -> None: + self._kube = kube + self._dgd = dgd + self._timeout_s = float(timeout_s) + self._clock = clock + self.name = state_lease_name(dgd.name) + + @property + def dgd(self) -> DgdRef: + return self._dgd + + def read(self) -> StateSnapshot: + lease = self._kube.get_lease(self.name, timeout_s=self._timeout_s) + if lease is None: + return StateSnapshot(state=None, resource_version=None, lease=None) + metadata = lease.get("metadata") or {} + rv = metadata.get("resourceVersion") + try: + state = RetuneState.from_annotations(metadata.get("annotations")) + except StateFormatError as error: + logger.error("retune state lease %s is unreadable: %s", self.name, error) + return StateSnapshot( + state=None, resource_version=rv, lease=lease, corrupt=True + ) + return StateSnapshot(state=state, resource_version=rv, lease=lease) + + def write(self, state: RetuneState, snapshot: StateSnapshot) -> StateSnapshot: + """Replace the snapshot read as ``snapshot`` (create it if absent). + + A changed resourceVersion, or a lease created by someone else in the + meantime, raises ``KubeConflict``; other failures raise ``KubeError`` + (the write may still have landed: re-read before deciding). + """ + + state = dataclasses.replace(state, updated_ms=int(round(self._clock() * 1000))) + annotations = state.to_annotations() + if snapshot.lease is None: + body = self._new_body(annotations) + try: + created = self._kube.create_lease(body, timeout_s=self._timeout_s) + except KubeAlreadyExists as error: + raise KubeConflict(str(error)) from error + return StateSnapshot( + state=state, + resource_version=(created.get("metadata") or {}).get("resourceVersion"), + lease=created, + ) + body = json.loads(json.dumps(snapshot.lease)) + metadata = body.setdefault("metadata", {}) + metadata["resourceVersion"] = snapshot.resource_version + merged = dict(metadata.get("annotations") or {}) + merged.update(annotations) + metadata["annotations"] = merged + metadata["ownerReferences"] = [owner_reference(self._dgd)] + labels = dict(metadata.get("labels") or {}) + labels.update({LABEL_DGD: self._dgd.name, LABEL_DGD_UID: self._dgd.uid}) + metadata["labels"] = labels + replaced = self._kube.replace_lease(self.name, body, timeout_s=self._timeout_s) + return StateSnapshot( + state=state, + resource_version=(replaced.get("metadata") or {}).get("resourceVersion"), + lease=replaced, + ) + + def delete(self) -> None: + self._kube.delete_lease(self.name, timeout_s=self._timeout_s) + + def _new_body(self, annotations: Mapping[str, str]) -> dict[str, Any]: + return { + "apiVersion": LEASE_API_VERSION, + "kind": "Lease", + "metadata": { + "name": self.name, + "namespace": self._kube.namespace, + "labels": {LABEL_DGD: self._dgd.name, LABEL_DGD_UID: self._dgd.uid}, + "annotations": dict(annotations), + "ownerReferences": [owner_reference(self._dgd)], + }, + "spec": {}, + } + + +# ------------------------------------------------------------ leadership + + +LeaseStep = Literal["acquired", "renewed", "standby", "lost", "error"] + + +class LeaseLeadership: + """Leadership over ``-planner-retune`` (design §4.7.1). + + ``step`` runs one renewal/acquisition attempt and returns what happened: + + - ``acquired``: this process now leads (a new lease, an empty holder, an + expired foreign holder, or its own lease after it had lost track). + - ``renewed``: it led and still does. + - ``lost``: it led and another holder now owns the lease. + - ``standby``: another holder leads. + - ``error``: the API call failed; leadership, if any, lasts until + ``leader_until``. + + ``is_leader`` holds from a successful write until ``leader_until`` = + send time + lease duration - ``margin_s``, all on the local monotonic + clock: a contender takes over only after watching the holder unchanged + for a whole lease duration, which starts after the send. + """ + + def __init__( + self, + kube: RetuneKube, + dgd: DgdRef, + *, + identity: str, + lease_duration_s: float = 60.0, + margin_s: Optional[float] = None, + timeout_s: float = 5.0, + monotonic: Callable[[], float] = time.monotonic, + clock: Callable[[], float] = time.time, + ) -> None: + if not identity: + raise ValueError("identity must be non-empty") + if not (math.isfinite(lease_duration_s) and lease_duration_s > 0): + raise ValueError("lease_duration_s must be positive and finite") + self._kube = kube + self._dgd = dgd + self.identity = identity + self.name = leadership_lease_name(dgd.name) + self.lease_duration_s = float(lease_duration_s) + self._duration_int = max(1, int(math.ceil(lease_duration_s))) + if margin_s is None: + margin_s = 3.0 * timeout_s + # Never leave less than a third of the duration to lead in. + self.margin_s = min(float(margin_s), self.lease_duration_s * 2.0 / 3.0) + self._timeout_s = float(timeout_s) + self._monotonic = monotonic + self._clock = clock + self._lock = threading.Lock() + self._held = False + self._leader_until = -math.inf + self.transitions = 0 + # (holderIdentity, resourceVersion) of a foreign lease, and when this + # process first saw it unchanged. + self._observed: Optional[tuple[str, Optional[str]]] = None + self._observed_since = 0.0 + + # ------------------------------------------------------------- queries + + def is_leader(self) -> bool: + with self._lock: + return self._held and self._monotonic() < self._leader_until + + @property + def held(self) -> bool: + with self._lock: + return self._held + + @property + def leader_until(self) -> float: + with self._lock: + return self._leader_until + + def set_owner(self, dgd: DgdRef) -> None: + """The DGD the Lease's ownerReference names from now on.""" + + self._dgd = dgd + + def drop(self) -> None: + """Forget leadership locally (no API call).""" + + with self._lock: + self._held = False + self._leader_until = -math.inf + + # ---------------------------------------------------------------- step + + def step(self) -> LeaseStep: + t_send = self._monotonic() + try: + lease = self._kube.get_lease(self.name, timeout_s=self._timeout_s) + except KubeError as error: + logger.warning( + "retune leadership: reading lease %s failed: %s", self.name, error + ) + return "error" + if lease is None: + return self._create(t_send) + + spec = lease.get("spec") or {} + metadata = lease.get("metadata") or {} + holder = spec.get("holderIdentity") or "" + rv = metadata.get("resourceVersion") + transitions = _int(spec.get("leaseTransitions")) + + if holder == self.identity: + # Expired locally counts as lost: renewing it starts a new term. + was_held = self.is_leader() + body = self._body(lease, transitions=transitions, acquire=not was_held) + try: + updated = self._kube.replace_lease( + self.name, body, timeout_s=self._timeout_s + ) + except KubeConflict: + # Changed since the read: re-read on the next step. + logger.info("retune leadership: renewal of %s conflicted", self.name) + return "error" + except KubeError as error: + logger.warning( + "retune leadership: renewing %s failed: %s", self.name, error + ) + return "error" + new_holder = (updated.get("spec") or {}).get("holderIdentity") or "" + if new_holder != self.identity: + return self._lose() + self._lead(t_send, transitions) + return "renewed" if was_held else "acquired" + + lost = self.held + if lost: + self.drop() + logger.warning( + "retune leadership: lease %s is now held by %r", + self.name, + holder or None, + ) + if not holder: + # Released by its holder: take it over at once. + return self._take_over(lease, t_send, transitions, lost) + observed = (holder, rv) + if self._observed != observed: + self._observed = observed + self._observed_since = t_send + return "lost" if lost else "standby" + duration = _positive(spec.get("leaseDurationSeconds")) or self.lease_duration_s + if t_send - self._observed_since < duration: + return "lost" if lost else "standby" + logger.info( + "retune leadership: holder %r of %s unchanged for %.0fs; taking over", + holder, + self.name, + t_send - self._observed_since, + ) + return self._take_over(lease, t_send, transitions, lost) + + def release(self) -> bool: + """Clear the holder if it is still this process (conditional update).""" + + held = self.held + self.drop() + if not held: + return False + try: + lease = self._kube.get_lease(self.name, timeout_s=self._timeout_s) + if lease is None: + return False + spec = lease.get("spec") or {} + if (spec.get("holderIdentity") or "") != self.identity: + return False + body = json.loads(json.dumps(lease)) + body_spec = body.setdefault("spec", {}) + body_spec["holderIdentity"] = None + body_spec["renewTime"] = micro_time(self._clock()) + self._kube.replace_lease(self.name, body, timeout_s=self._timeout_s) + logger.info("retune leadership: released lease %s", self.name) + return True + except KubeError as error: + logger.warning( + "retune leadership: releasing %s failed: %s", self.name, error + ) + return False + + # ------------------------------------------------------------ helpers + + def _lead(self, t_send: float, transitions: int) -> None: + with self._lock: + self._held = True + self._leader_until = t_send + self.lease_duration_s - self.margin_s + self.transitions = transitions + self._observed = None + + def _lose(self) -> LeaseStep: + lost = self.held + self.drop() + return "lost" if lost else "standby" + + def _create(self, t_send: float) -> LeaseStep: + now = micro_time(self._clock()) + body = { + "apiVersion": LEASE_API_VERSION, + "kind": "Lease", + "metadata": { + "name": self.name, + "namespace": self._kube.namespace, + "labels": {LABEL_DGD: self._dgd.name, LABEL_DGD_UID: self._dgd.uid}, + "ownerReferences": [owner_reference(self._dgd)], + }, + "spec": { + "holderIdentity": self.identity, + "leaseDurationSeconds": self._duration_int, + "acquireTime": now, + "renewTime": now, + "leaseTransitions": 0, + }, + } + try: + self._kube.create_lease(body, timeout_s=self._timeout_s) + except (KubeAlreadyExists, KubeConflict): + return self._lose() + except KubeError as error: + logger.warning( + "retune leadership: creating %s failed: %s", self.name, error + ) + return "error" + self._lead(t_send, 0) + logger.info( + "retune leadership: acquired new lease %s as %s", self.name, self.identity + ) + return "acquired" + + def _take_over( + self, lease: dict[str, Any], t_send: float, transitions: int, lost: bool + ) -> LeaseStep: + body = self._body(lease, transitions=transitions + 1, acquire=True) + try: + self._kube.replace_lease(self.name, body, timeout_s=self._timeout_s) + except KubeConflict: + return "lost" if lost else "standby" + except KubeError as error: + logger.warning( + "retune leadership: taking over %s failed: %s", self.name, error + ) + return "error" + self._lead(t_send, transitions + 1) + logger.info( + "retune leadership: acquired lease %s as %s (transitions=%d)", + self.name, + self.identity, + transitions + 1, + ) + return "acquired" + + def _body( + self, lease: dict[str, Any], *, transitions: int, acquire: bool + ) -> dict[str, Any]: + body = json.loads(json.dumps(lease)) + metadata = body.setdefault("metadata", {}) + metadata["ownerReferences"] = [owner_reference(self._dgd)] + spec = body.setdefault("spec", {}) + now = micro_time(self._clock()) + spec["holderIdentity"] = self.identity + spec["leaseDurationSeconds"] = self._duration_int + spec["renewTime"] = now + spec["leaseTransitions"] = int(transitions) + if acquire or not spec.get("acquireTime"): + spec["acquireTime"] = now + return body + + +def _int(value: Any) -> int: + if isinstance(value, bool): + return 0 + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +def _positive(value: Any) -> Optional[float]: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return float(value) if value > 0 else None diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py new file mode 100644 index 000000000000..518b422cd51b --- /dev/null +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -0,0 +1,2019 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``LiveRetuneSupervisor``: leadership, persisted state and actuation (design §4.7). + +The retune controller decides exactly as in replay; the supervisor decides +when it may launch and harvest, persists every decision before anything is +applied, and owns the routers' parameters. + +Hooks the controller calls on the planner's event loop: + +- ``launch_gate(k, window)`` before both the launch and the busy branch: + ``not_leader``/``not_ready`` keep the window due, ``already_decided`` skips + a window the state already records, otherwise it hands out the DGD spec + replicas and a fresh abort event. +- ``harvest_allowed(generation)``: leader, initialized and the sweep's + controller generation is current. +- ``sweep_fn(request, *, cancel, abort)`` on the sweep thread: binds a + ``DgdrContext`` and runs ``dgdr.run_dgdr_sweep``; never raises. + +The host calls ``on_effects`` after every engine tick; it enqueues the tick's +decisions in the outbox bound to (term_seq, generation, fingerprint). + +Background tasks (all catch every exception; every Kubernetes call runs in a +thread with a timeout): + +- lease: renew or acquire ``-planner-retune`` every third of the lease + duration. Becoming leader, re-acquiring after losing track, or a re-init + request enters initialization (§4.7.3) with ``term_seq + 1``; losing + leadership runs the local loss (§4.7.5). +- control: re-validate the spec premises (§4.7.7, rebuilding the controller + through ``host.replace_live_retune`` when the startup, ``spec_ok`` or P4 + change), then, as an initialized leader, the P11 canary and one actuator + round (§4.8), and it starts one reconciler round (§4.4.5). +- reconciler: one round at a time in its own task, never awaited by the + control loop, within ``reconcile_budget_s``. A round still running (a hung + filesystem or slow API calls) skips the next ones; the reconciler stops + before its next delete once this process is no longer the initialized + leader of the round's epoch or the budget has passed. +- outbox: write each decision with the term and resourceVersion + preconditions (§4.7.4); N consecutive failures re-initialize. + +Local leadership loss, a failed outbox, and a controller rebuild share one +path: clear ``initialized``, stop the outbox writer and the in-flight +actuator/reconciler round, clear the outbox, abort the sweep and drop it +(``drop_pending``), and rewind the controller to the last persisted window, +so a due window can be relaunched or adopted before its deadline. + +The actuation target follows the state: ``rollback`` while +``rollback_pending`` (an advisory planner resets only routers whose initial +it latched, §4.8); ``observe`` (never write) in advisory; ``rollback`` when a +spec premise fails; otherwise ``apply`` the state's ``by_wt``. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import functools +import inspect +import logging +import math +import threading +import time +from collections import deque +from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal, Optional, Protocol, Union + +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD_UID, + ActuationReport, + ActuationTarget, + DgdRef, + FrontendStartup, + GateReason, + GateResult, + KubeConflict, + KubeError, + KubeNotFound, + NullMetrics, + PremiseReport, + RetuneKube, + RetuneMetricsSink, + SweepRegistry, +) +from dynamo.planner.retune.live.state import ( + LeaseLeadership, + RetuneMode, + RetuneState, + StateSnapshot, + StateStore, + append_history, + claim, + deploy, + deployed_off, + has_decision, + maps_equal, + merge_initials, + record_decision, + reset, + rollback_complete, + rollback_start, +) + +if TYPE_CHECKING: + from dynamo.planner.config.planner_config import PlannerConfig + from dynamo.planner.core.types import PlannerEffects, RetuneDecision, RetuneInputs + from dynamo.planner.retune.sweep import SweepRequest, SweepResult + +logger = logging.getLogger(__name__) + +_EPSILON = 1e-9 +# Claim-write attempts on resourceVersion conflicts before giving up the +# initialization (the next lease round retries with term_seq + 1). +_CLAIM_ATTEMPTS = 3 +# Conflict retries of one state write. +_WRITE_ATTEMPTS = 3 +# Remembered (k, reason) pairs of skipped launches (the metric counts each +# pair once). +_SKIP_MEMORY = 64 + +BlockingRunner = Callable[[Callable[[], Any]], Awaitable[Any]] + + +async def _to_thread(call: Callable[[], Any]) -> Any: + return await asyncio.to_thread(call) + + +# ---------------------------------------------------------------- types + + +@dataclass(frozen=True) +class SpecEvaluation: + """One evaluation of the DGD spec (frontend parse and spec premises). + + ``inputs`` is the live ``RetuneInputs``, ``None`` iff P4 fails. + ``spec_replicas`` maps worker type to the DGD spec replicas (``None`` + when the workers cannot be resolved). ``dgd`` carries the DGD's current + uid and the routers' dynamo namespace. + """ + + report: PremiseReport + startup: Optional[FrontendStartup] + inputs: Optional[RetuneInputs] + spec_replicas: Optional[dict[str, int]] + dgd: DgdRef + generation: Optional[int] = None + + +# DGD object -> evaluation; may raise (the round is then skipped). +PremiseEvaluator = Callable[[dict], SpecEvaluation] + + +class RouterResolver(Protocol): + """Resolves router points against the parsed startup ``KvRouterConfig``.""" + + def startup_flat(self, keys: Sequence[str]) -> dict[str, float]: + """The startup value of each searched key (the controller's start).""" + ... + + def by_wt(self, flat: Mapping[str, float]) -> dict[str, dict[str, float]]: + """``policy_parameters(wt)`` of ``startup ⊕ flat`` per worker type.""" + ... + + +ResolverFactory = Callable[[Optional[FrontendStartup]], Optional[RouterResolver]] + + +class ActuatorLike(Protocol): + async def reconcile( + self, + target: ActuationTarget, + *, + initials: Mapping[str, Mapping[str, float]], + expected_frontends: int, + ) -> ActuationReport: + ... + + def set_startup_maps(self, startup_maps: Mapping[str, Mapping[str, float]]) -> None: + ... + + +# (DgdRef, startup maps per worker type) -> actuator +ActuatorFactory = Callable[[DgdRef, dict[str, dict[str, float]]], ActuatorLike] +SweepRunner = Callable[..., "SweepResult"] + + +class LiveRetuneHost(Protocol): + """What the supervisor needs from ``NativePlannerBase``. + + ``replace_live_retune`` takes the planner's config lock and calls the + engine adapter's ``replace_retune``. The supervisor reaches the current + controller through ``host._engine.retune_controller`` and serializes its + calls on it with ``host._config_lock``. + """ + + async def replace_live_retune( + self, + inputs: Optional[RetuneInputs], + *, + generation: int, + last_decided_k: Optional[int], + ) -> None: + ... + + +@dataclass(frozen=True) +class SupervisorSettings: + """Timing and policy of one supervisor (from ``PlannerConfig``).""" + + interval_s: float + grace_s: float + timeout_s: float + phase_s: float = 0.0 + advisory: bool = False + promote_last_advisory: bool = False + worker_types: tuple[str, ...] = ("aggregated",) + # Searched router keys (live names); the controller's router_keys. + router_keys: tuple[str, ...] = () + lease_duration_s: float = 60.0 + actuator_interval_s: float = 15.0 + k8s_timeout_s: float = 5.0 + write_failure_limit: int = 3 + canary_interval_s: float = 600.0 + stop_timeout_s: float = 10.0 + # How long one reconciler round may run before the reconciler is told to + # stop (a blocked filesystem call cannot be interrupted; later rounds + # wait for it instead of piling up threads). + reconcile_budget_s: float = 120.0 + # Legacy clean-up (§4.11): roll back what an earlier retune deployed, + # then delete the Requests and both Leases. + decommission: bool = False + + def __post_init__(self) -> None: + for name in ( + "interval_s", + "lease_duration_s", + "actuator_interval_s", + "reconcile_budget_s", + ): + value = getattr(self, name) + if not (math.isfinite(value) and value > 0): + raise ValueError(f"{name} must be positive and finite") + if self.write_failure_limit < 1: + raise ValueError("write_failure_limit must be >= 1") + + @property + def renew_interval_s(self) -> float: + return self.lease_duration_s / 3.0 + + @property + def io_timeout_s(self) -> float: + """Backstop on one blocking call; the client applies its own.""" + + return 2.0 * self.k8s_timeout_s + 1.0 + + @classmethod + def from_config( + cls, config: PlannerConfig, *, router_keys: Sequence[str] = () + ) -> SupervisorSettings: + retune = config.retune + if retune is None or retune.dgdr is None: + raise ValueError("live retune requires retune.executor 'dgdr'") + dgdr = retune.dgdr + return cls( + interval_s=float(retune.interval_seconds), + grace_s=float(retune.grace_seconds), + timeout_s=float(retune.effective_timeout_seconds), + phase_s=float(dgdr.phase_seconds), + advisory=bool(config.advisory), + promote_last_advisory=bool(dgdr.promote_last_advisory), + worker_types=worker_types_for(config.mode), + router_keys=tuple(sorted(router_keys)), + lease_duration_s=float(dgdr.lease_duration_seconds), + actuator_interval_s=float(dgdr.actuator_interval_seconds), + ) + + +def worker_types_for(planner_mode: str) -> tuple[str, ...]: + return ("prefill", "decode") if planner_mode == "disagg" else ("aggregated",) + + +def due_window_end_ms( + now_s: float, *, phase_s: float, interval_s: float, grace_s: float +) -> int: + """End of the latest window whose launch time is at or before ``now_s``. + + Same arithmetic as ``RetuneController._due_window`` with ``s0 = phase``. + """ + + k = math.floor((now_s - phase_s - grace_s) / interval_s + _EPSILON) - 1 + return int(round((phase_s + (k + 1) * interval_s) * 1000.0)) + + +def current_window_end_ms(now_s: float, *, phase_s: float, interval_s: float) -> int: + """End of the window in progress at ``now_s`` (it contains ``now_s``). + + It ends after the due window (``due_window_end_ms``) and after every + window that ended but is not due yet (within the grace). + """ + + k = math.floor((now_s - phase_s) / interval_s + _EPSILON) + return int(round((phase_s + (k + 1) * interval_s) * 1000.0)) + + +def window_index(window_end_ms: int, *, phase_s: float, interval_s: float) -> int: + """The controller's ``k`` of the window ending at ``window_end_ms``.""" + + return int(round((window_end_ms / 1000.0 - phase_s) / interval_s)) - 1 + + +# ----------------------------------------------------- initialization (§4.7.3) + +InitRow = Literal[ + "spec_not_ok", + "advisory_rollback", + "advisory_observe", + "active_restore", + "active_reset", + "promote", +] + + +@dataclass(frozen=True) +class InitPlan: + """What initialization writes and restores. + + ``state`` is the snapshot to persist (``write`` says whether it differs + from the claimed one); ``restore_flat`` is passed to the controller's + ``restore_desired`` (``None``: leave the controller alone). + """ + + row: InitRow + state: RetuneState + write: bool + restore_flat: Optional[dict[str, float]] + + +def plan_initialization( + state: RetuneState, + *, + spec_ok: bool, + advisory: bool, + fingerprint: str, + startup_flat: Optional[Mapping[str, float]], + startup_by_wt: Optional[Mapping[str, Mapping[str, float]]], + router_keys: Sequence[str], + promote: bool, + now_ms: int, + interval_ms: int, + first_window_end_ms: int, +) -> InitPlan: + """The initialization table of design §4.7.3, first matching row wins. + + ``state`` is the snapshot after the claim write. A state without a + deployment record is "no state". The first initialization of a DGD also + persists ``last_window_end_ms = first_window_end_ms`` (the end of the + window in progress now): neither the due window nor the window whose + trace began mid-way (tracing enabled with the planner) is ever swept, + even after a restart. + """ + + mode: RetuneMode = "advisory" if advisory else "active" + off = deployed_off(state, startup_by_wt) + flat0 = {key: float(value) for key, value in (startup_flat or {}).items()} + by0 = {wt: dict(params) for wt, params in (startup_by_wt or {}).items()} + stale = ( + not state.has_deployment + or state.mode != mode + or state.fingerprint != fingerprint + ) + restore: Optional[dict[str, float]] = None + + if not spec_ok: + row: InitRow = "spec_not_ok" + if off: + new = rollback_start( + state, fingerprint=fingerprint, mode=mode, flat=flat0, by_wt=by0 + ) + elif stale: + new = reset( + state, + fingerprint=fingerprint, + mode=mode, + flat=flat0, + by_wt=by0, + rollback_pending=False, + ) + else: + new = state + elif advisory: + if off: + row = "advisory_rollback" + new = rollback_start( + state, fingerprint=fingerprint, mode=mode, flat=flat0, by_wt=by0 + ) + else: + row = "advisory_observe" + new = ( + reset( + state, + fingerprint=fingerprint, + mode=mode, + flat=flat0, + by_wt=by0, + rollback_pending=False, + ) + if stale + else state + ) + elif ( + not stale + and not state.rollback_pending + and set(state.flat) == set(router_keys) + and state.by_wt + ): + row = "active_restore" + new = state + restore = dict(state.flat) + else: + entry = ( + _promotable(state, fingerprint, router_keys, now_ms, interval_ms) + if promote + else None + ) + if entry is not None: + row = "promote" + flat = {key: float(value) for key, value in entry["flat"].items()} + by_wt = {wt: dict(params) for wt, params in entry["by_wt"].items()} + new = deploy( + state, fingerprint=fingerprint, mode="active", flat=flat, by_wt=by_wt + ) + new = append_history( + new, + { + "decision_id": f"promote-{entry.get('decision_id')}", + "outcome": "promoted", + "window_end_ms": entry.get("window_end_ms"), + "fingerprint": fingerprint, + "flat": flat, + "by_wt": by_wt, + "deploy_seq": new.deploy_seq, + "at_ms": now_ms, + }, + ) + restore = flat + else: + row = "active_reset" + new = reset( + state, + fingerprint=fingerprint, + mode="active", + flat=flat0, + by_wt=by0, + rollback_pending=off, + ) + restore = flat0 + if new.last_window_end_ms is None: + new = dataclasses.replace(new, last_window_end_ms=int(first_window_end_ms)) + return InitPlan(row=row, state=new, write=new != state, restore_flat=restore) + + +def _promotable( + state: RetuneState, + fingerprint: str, + router_keys: Sequence[str], + now_ms: int, + interval_ms: int, +) -> Optional[Mapping[str, Any]]: + """The last advisory recommendation, if §4.7.6 lets it be promoted.""" + + if not state.history: + return None + entry = state.history[-1] + if entry.get("outcome") != "advisory" or entry.get("fingerprint") != fingerprint: + return None + window_end_ms = entry.get("window_end_ms") + if ( + not isinstance(window_end_ms, (int, float)) + or now_ms - window_end_ms > interval_ms + ): + return None + flat = entry.get("flat") + by_wt = entry.get("by_wt") + if not isinstance(flat, Mapping) or set(flat) != set(router_keys): + return None + if not isinstance(by_wt, Mapping) or not by_wt: + return None + return entry + + +# ------------------------------------------------------------------ outbox + + +@dataclass(frozen=True) +class OutboxEntry: + """One decision waiting to be persisted, bound to the term that made it.""" + + decision: RetuneDecision + router_parameters: Optional[dict[str, dict[str, float]]] + term_seq: int + generation: int + fingerprint: str + epoch: int + + +WriteOutcome = Literal["ok", "already", "discarded", "lost_term", "error"] +_ALREADY = object() +Mutation = Callable[[RetuneState], Union[RetuneState, None, object]] + + +class _NullRegistry: + def protect(self, window_end_ms: int, request_names: Any) -> None: + pass + + def release(self, window_end_ms: int) -> None: + pass + + def enqueue_delete(self, request_names: Any) -> None: + pass + + +# -------------------------------------------------------------- supervisor + + +class LiveRetuneSupervisor: + """Leadership, state, outbox, actuation and clean-up of live retune. + + Collaborators are injected so tests never need a cluster: ``kube`` (a + ``RetuneKube``), ``premise_evaluator`` (DGD -> ``SpecEvaluation``), + ``actuator_factory`` (a ``RouterActuator``), ``reconciler`` (a + ``Reconciler``, also the sweep registry), ``canary`` (P11), + ``sweep_runner``/``context_factory`` (``dgdr.run_dgdr_sweep`` and its + ``DgdrContext``), the clocks, ``sleep`` and ``blocking`` (how a blocking + call runs off the event loop). + """ + + def __init__( + self, + *, + settings: SupervisorSettings, + kube: RetuneKube, + evaluation: SpecEvaluation, + identity: str, + premise_evaluator: PremiseEvaluator, + actuator_factory: ActuatorFactory, + fingerprint_fn: Callable[[Optional[FrontendStartup]], str], + resolver_factory: Optional[ResolverFactory] = None, + reconciler: Optional[Any] = None, + metrics: Optional[RetuneMetricsSink] = None, + canary: Optional[Callable[[LiveRetuneSupervisor], bool]] = None, + sweep_runner: Optional[SweepRunner] = None, + context_factory: Optional[Callable[[LiveRetuneSupervisor], Any]] = None, + clock: Callable[[], float] = time.time, + monotonic: Callable[[], float] = time.monotonic, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, + blocking: Optional[BlockingRunner] = None, + leadership: Optional[LeaseLeadership] = None, + store: Optional[StateStore] = None, + ) -> None: + self._settings = settings + self._kube = kube + self._identity = identity + self._premise_evaluator = premise_evaluator + self._actuator_factory = actuator_factory + self._fingerprint_fn = fingerprint_fn + self._resolver_factory = resolver_factory + self._reconciler = reconciler + self._metrics: RetuneMetricsSink = ( + metrics if metrics is not None else NullMetrics() + ) + self._canary = canary + self._sweep_runner = sweep_runner + self._context_factory = context_factory + self._clock = clock + self._monotonic = monotonic + self._sleep = sleep + self._blocking: BlockingRunner = ( + blocking if blocking is not None else _to_thread + ) + + self._dgd = evaluation.dgd + self._leadership = leadership or LeaseLeadership( + kube, + evaluation.dgd, + identity=identity, + lease_duration_s=settings.lease_duration_s, + margin_s=3.0 * settings.k8s_timeout_s, + timeout_s=settings.k8s_timeout_s, + monotonic=monotonic, + clock=clock, + ) + self._store = store or StateStore( + kube, evaluation.dgd, timeout_s=settings.k8s_timeout_s, clock=clock + ) + + # Spec evaluation and what is derived from it. + self._evaluation = evaluation + self._p14_ok: Optional[bool] = None + self._spec_ok = bool(evaluation.report.spec_ok) + self._fingerprint = fingerprint_fn(evaluation.startup) + self._resolver = self._make_resolver(evaluation.startup) + self._actuator = actuator_factory( + evaluation.dgd, self._actuator_maps(evaluation.startup) + ) + self._log_report(evaluation.report) + + # Leadership, term and the cached state snapshot. + self._term_seq = 0 + self._term: Optional[str] = None + self._snapshot: Optional[StateSnapshot] = None + self._initialized = False + self._reinit_requested = False + # Bumped by every local loss: results of work started before it are + # discarded. + self._epoch = 0 + + # Controller. + self._host: Optional[Any] = None + self._generation = 0 + self._controller_ok = evaluation.inputs is not None + + # Outbox. + self._outbox: deque[OutboxEntry] = deque() + self._write_failures = 0 + + # Health (P11, P12) and actuation. + self._canary_ok: Optional[bool] = None + self._canary_at: Optional[float] = None + self._last_report: Optional[ActuationReport] = None + # (deploy_seq, monotonic since) of a converged rollback. + self._rollback_since: Optional[tuple[int, float]] = None + + # Sweeps. + self._aborts: list[threading.Event] = [] + self._aborts_lock = threading.Lock() + self._sweeps_in_flight = 0 + self._skips_seen: deque[tuple[int, str]] = deque(maxlen=_SKIP_MEMORY) + + # Tasks and locks. + self._init_lock = asyncio.Lock() + self._write_lock = asyncio.Lock() + self._tasks: list[asyncio.Task[None]] = [] + self._outbox_task: Optional[asyncio.Task[None]] = None + self._round_task: Optional[asyncio.Task[None]] = None + # The reconciler round's task (it waits up to the budget) and its + # work (the reconciler call itself, done only once it has returned). + self._reconcile_task: Optional[asyncio.Task[None]] = None + self._reconcile_work: Optional[asyncio.Future[Any]] = None + self._lease_wake = asyncio.Event() + self._outbox_wake = asyncio.Event() + self._background = False + self._stopping = False + self._stopped = False + self.decommissioned = False + + # ------------------------------------------------------------ queries + + @property + def identity(self) -> str: + return self._identity + + @property + def generation(self) -> int: + return self._generation + + @property + def term(self) -> Optional[str]: + return self._term + + @property + def term_seq(self) -> int: + return self._term_seq + + @property + def initialized(self) -> bool: + return self._initialized + + @property + def state(self) -> Optional[RetuneState]: + return None if self._snapshot is None else self._snapshot.state + + @property + def spec_ok(self) -> bool: + return self._spec_ok + + @property + def fingerprint(self) -> str: + return self._fingerprint + + @property + def evaluation(self) -> SpecEvaluation: + return self._evaluation + + @property + def dgd(self) -> DgdRef: + return self._dgd + + @property + def kube(self) -> RetuneKube: + return self._kube + + @property + def registry(self) -> SweepRegistry: + reconciler = self._reconciler + if reconciler is not None and hasattr(reconciler, "protect"): + return reconciler + return _NullRegistry() + + @property + def actuator(self) -> ActuatorLike: + return self._actuator + + @property + def outbox_size(self) -> int: + return len(self._outbox) + + @property + def last_report(self) -> Optional[ActuationReport]: + return self._last_report + + @property + def reconcile_task(self) -> Optional[asyncio.Task[None]]: + """The latest reconciler round's task (done once the round returned + or its budget passed); tests await it after ``control_round``.""" + + return self._reconcile_task + + def is_leader(self) -> bool: + """Thread-safe; ``False`` once ``leader_until`` has passed.""" + + return self._leadership.is_leader() + + def ready(self) -> bool: + """Whether a due window may launch (spec, controller and health).""" + + return ( + self._spec_ok + and self._controller_ok + and self._evaluation.spec_replicas is not None + and self._canary_ok is True + and self._p12_ok() + and not self._settings.decommission + ) + + def retune_inputs(self) -> Optional[RetuneInputs]: + """Inputs for the engine's first controller; ``None`` if P4 fails.""" + + return self._evaluation.inputs + + def hooks(self) -> Any: + """``RetuneHooks`` for ``OrchestratorEngineAdapter``.""" + + from dynamo.planner.plugins.orchestrator.engine_adapter import RetuneHooks + + return RetuneHooks( + sweep_fn=self.sweep_fn, + launch_gate=self.launch_gate, + harvest_allowed=self.harvest_allowed, + phase_s=self._settings.phase_s, + ) + + def target(self) -> Optional[ActuationTarget]: + """What the actuator converges to now (``None``: nothing).""" + + state = self.state + if not self._initialized or state is None: + return None + if state.rollback_pending: + # §4.8: only an active planner resets a router it never latched + # (revision > 0) to learn its initial; an advisory planner leaves + # it alone. + return ActuationTarget( + mode="rollback", + include_not_ready=True, + learn_unknown=not ( + self._settings.advisory or self._settings.decommission + ), + ) + if self._settings.advisory or self._settings.decommission: + return ActuationTarget(mode="observe") + if not self._spec_ok: + return ActuationTarget(mode="rollback", include_not_ready=True) + if not state.by_wt: + return ActuationTarget(mode="observe") + return ActuationTarget( + mode="apply", + desired_by_wt={wt: dict(params) for wt, params in state.by_wt.items()}, + ) + + # ----------------------------------------------------- controller hooks + + def launch_gate(self, k: int, window: tuple[float, float, float]) -> GateResult: + """Whether window ``k`` may launch now; see the module docstring.""" + + reason: Optional[GateReason] + _, end_s, _ = window + state = self.state + if not self.is_leader(): + reason = "not_leader" + elif not self._initialized or state is None or not self.ready(): + reason = "not_ready" + elif ( + state.last_window_end_ms is not None + and int(round(end_s * 1000.0)) <= state.last_window_end_ms + ): + reason = "already_decided" + else: + reason = None + if reason is not None: + if (k, reason) not in self._skips_seen: + self._skips_seen.append((k, reason)) + self._metrics.skipped(reason) + logger.info("retune gate: window %d %s", k, reason) + return GateResult(reason=reason) + abort = threading.Event() + with self._aborts_lock: + self._aborts = [event for event in self._aborts if not event.is_set()][-8:] + self._aborts.append(abort) + return GateResult( + reason=None, + spec_replicas=dict(self._evaluation.spec_replicas or {}), + abort=abort, + ) + + def harvest_allowed(self, generation: int) -> bool: + """Read-only: leader, initialized, and the sweep's generation is current.""" + + return ( + self.is_leader() + and self._initialized + and int(generation) == self._generation + ) + + def sweep_fn( + self, + request: SweepRequest, + *, + cancel: threading.Event, + abort: Optional[threading.Event] = None, + ) -> SweepResult: + """Run one DGDR sweep on the controller's sweep thread; never raises.""" + + with self._aborts_lock: + self._sweeps_in_flight += 1 + try: + runner = self._sweep_runner + if runner is None: + from dynamo.planner.retune.live.dgdr import run_dgdr_sweep + + runner = run_dgdr_sweep + if self._context_factory is None: + raise RuntimeError("no DGDR sweep context factory is configured") + context = self._context_factory(self) + return runner(request, cancel=cancel, abort=abort, context=context) + except Exception as error: # the sweep contract: never raise + logger.exception("retune sweep of window %s failed", request.window_index) + from dynamo.planner.retune.sweep import SweepResult + + return SweepResult( + status="failed", + error=f"{type(error).__name__}: {error}", + failure_reason="sweep_error", + ) + finally: + with self._aborts_lock: + self._sweeps_in_flight -= 1 + + def on_effects(self, effects: PlannerEffects) -> None: + """Enqueue the tick's decisions; never raises into the tick.""" + + decisions = list(getattr(effects, "retune_decisions", None) or ()) + if not decisions: + return + if not (self.is_leader() and self._initialized): + logger.warning( + "retune: dropping %d decision(s) %s: not an initialized leader", + len(decisions), + [decision.decision_id for decision in decisions], + ) + self._rewind_now() + return + router_parameters = getattr(effects, "router_parameters", None) + for decision in decisions: + self._outbox.append( + OutboxEntry( + decision=decision, + router_parameters=( + None + if router_parameters is None + else {wt: dict(p) for wt, p in router_parameters.items()} + ), + term_seq=self._term_seq, + generation=self._generation, + fingerprint=self._fingerprint, + epoch=self._epoch, + ) + ) + self._outbox_wake.set() + + # ------------------------------------------------------------ lifecycle + + async def start(self, host: Any, *, background: bool = True) -> None: + """Bind the planner host and start the background tasks. + + With ``background=False`` (tests) nothing runs on its own: drive + ``lease_round``, ``control_round`` and ``outbox_round`` directly. + ``control_round`` still starts the reconciler round as a task (it is + never waited for); await ``reconcile_task`` to finish it. + """ + + self._host = host + controller = self._controller() + if controller is not None: + self._generation = int(getattr(controller, "generation", self._generation)) + self._controller_ok = controller is not None + self._background = background + self._publish() + if not background: + return + self._tasks = [ + asyncio.create_task(self._lease_loop(), name="planner-retune-lease"), + asyncio.create_task(self._control_loop(), name="planner-retune-control"), + ] + + async def stop(self) -> None: + """Abort sweeps, wait for them briefly and release the Lease (§4.7.8).""" + + if self._stopped: + return + self._stopping = True + tasks = list(self._tasks) + self._tasks = [] + for task in tasks: + task.cancel() + if tasks: + await asyncio.wait(tasks) + try: + async with self._init_lock: + await self._local_loss_locked("shutdown") + except Exception: + logger.exception("retune supervisor: shutdown clean-up failed") + self._abort_all() + deadline = self._monotonic() + self._settings.stop_timeout_s + while self._sweeps_in_flight > 0 and self._monotonic() < deadline: + await self._sleep(0.1) + if self._sweeps_in_flight > 0: + logger.warning( + "retune supervisor: %d sweep(s) still running at shutdown", + self._sweeps_in_flight, + ) + try: + await self._io(self._leadership.release) + except Exception: + logger.exception("retune supervisor: releasing the lease failed") + await self._close_actuator(self._actuator) + self._stopped = True + self._publish() + + # ------------------------------------------------------------- rounds + + async def lease_round(self) -> None: + """One renewal/acquisition; initialize or run the local loss.""" + + if self._stopping: + return + try: + step = await self._io(self._leadership.step) + except Exception as error: + logger.warning("retune lease round failed: %s", error) + step = "error" + try: + if self.is_leader(): + if ( + step == "acquired" + or self._reinit_requested + or not self._initialized + ): + async with self._init_lock: + if self.is_leader() and not self._stopping: + await self._initialize_locked() + elif self._initialized or self._term is not None: + async with self._init_lock: + if self._term is not None: + await self._local_loss_locked(f"not leader ({step})") + self._term = None + except Exception: + logger.exception("retune lease round: initialization failed") + self._publish() + + async def control_round(self) -> None: + """Re-validate the spec; as an initialized leader, actuate and start + a reconciler round (never waited for).""" + + if self._stopping: + return + try: + await self.revalidate() + except Exception: + logger.exception("retune revalidation failed") + if self._initialized and self.is_leader(): + task = asyncio.create_task(self._leader_round()) + self._round_task = task + await asyncio.wait([task]) + if self._round_task is task: + self._round_task = None + if not task.cancelled() and task.exception() is not None: + logger.error("retune actuation round failed", exc_info=task.exception()) + self._publish() + + async def outbox_round(self) -> None: + """Write the queued decisions in order; stop at the first failure.""" + + while self._outbox and not self._stopping: + entry = self._outbox[0] + if not self._entry_current(entry): + self._outbox.popleft() + logger.info( + "retune outbox: dropped %s of an earlier term/generation", + entry.decision.decision_id, + ) + continue + outcome = await self._write_decision(entry) + if outcome in ("ok", "already", "discarded"): + if self._outbox and self._outbox[0] is entry: + self._outbox.popleft() + self._write_failures = 0 + continue + if outcome == "lost_term": + self._request_reinit("the state's leader_term is not ours") + return + # "error": counted as failures_total{reason="state_write"} by + # _write_state. + self._write_failures += 1 + logger.warning( + "retune outbox: writing %s failed (%d/%d)", + entry.decision.decision_id, + self._write_failures, + self._settings.write_failure_limit, + ) + if self._write_failures >= self._settings.write_failure_limit: + self._request_reinit("consecutive state write failures") + return + + async def revalidate(self) -> None: + """§4.7.7: re-read the DGD and rebuild on a spec-class change.""" + + try: + dgd_obj = await self._io(self._kube.get_dgd, self._dgd.name) + except KubeNotFound: + logger.warning("retune: DGD %s not found", self._dgd.name) + return + except (KubeError, asyncio.TimeoutError) as error: + logger.warning("retune: reading DGD %s failed: %s", self._dgd.name, error) + return + evaluation = self._premise_evaluator(dgd_obj) + await self.apply_evaluation(evaluation) + + async def apply_evaluation(self, evaluation: SpecEvaluation) -> None: + """Adopt an evaluation; rebuild when the startup, ``spec_ok``, P4 or + the DGD identity changed.""" + + old = self._evaluation + startup_changed = _startup_key(old.startup) != _startup_key(evaluation.startup) + p14 = None if startup_changed else self._p14_ok + spec_ok = bool(evaluation.report.spec_ok) and p14 is not False + reasons = [] + if startup_changed: + reasons.append("startup changed") + if spec_ok != self._spec_ok: + reasons.append(f"spec_ok {self._spec_ok} -> {spec_ok}") + if bool(old.report.p4_ok) != bool(evaluation.report.p4_ok): + reasons.append(f"P4 {old.report.p4_ok} -> {evaluation.report.p4_ok}") + if (old.dgd.name, old.dgd.namespace, old.dgd.uid) != ( + evaluation.dgd.name, + evaluation.dgd.namespace, + evaluation.dgd.uid, + ): + reasons.append(f"DGD uid {old.dgd.uid} -> {evaluation.dgd.uid}") + if reasons: + await self.rebuild(evaluation, "; ".join(reasons)) + return + # Replicas, warnings or the routers' dynamo namespace (a worker + # rollout) change without a rebuild. + self._evaluation = evaluation + if old.dgd != evaluation.dgd: + logger.info( + "retune: routers' dynamo namespace %s -> %s", + old.dgd.dynamo_namespace, + evaluation.dgd.dynamo_namespace, + ) + await self._swap_actuator(evaluation) + + async def rebuild(self, evaluation: SpecEvaluation, reason: str) -> None: + """Local loss, generation + 1, ``replace_live_retune``, re-initialize.""" + + async with self._init_lock: + logger.info("retune: rebuilding the controller: %s", reason) + await self._local_loss_locked(f"rebuild: {reason}") + await self._install_evaluation(evaluation) + self._generation += 1 + host = self._host + if host is not None: + try: + await host.replace_live_retune( + evaluation.inputs, + generation=self._generation, + last_decided_k=self._last_decided_k(), + ) + self._controller_ok = evaluation.inputs is not None + except Exception: + logger.exception( + "retune: installing controller generation %d failed; not ready", + self._generation, + ) + self._controller_ok = False + else: + self._controller_ok = evaluation.inputs is not None + if self.is_leader() and not self._stopping: + await self._initialize_locked() + self._publish() + + # ------------------------------------------------- init and local loss + + async def _initialize_locked(self) -> bool: + """§4.7.3 under ``_init_lock``; ``True`` once initialized.""" + + await self._local_loss_locked("initialize") + # Confirm on the API server right before the claim: a holder change + # this process has not seen yet (it believes it leads until + # leader_until) aborts the initialization instead of claiming. + try: + await self._io(self._leadership.step) + except Exception as error: + logger.warning("retune init: confirming leadership failed: %s", error) + if not self.is_leader(): + return False + self._term_seq += 1 + self._reinit_requested = False + term = f"{self._leadership.transitions}/{self._identity}/{self._term_seq}" + self._term = term + epoch = self._epoch + settings = self._settings + async with self._write_lock: + snapshot = await self._read_own_snapshot() + if snapshot is None: + return False + # Claim: only leader_term, conditional on the resourceVersion. + for attempt in range(_CLAIM_ATTEMPTS): + if not self.is_leader() or epoch != self._epoch: + return False + claimed = claim(snapshot.state, dgd_uid=self._dgd.uid, term=term) + try: + snapshot = await self._io(self._store.write, claimed, snapshot) + break + except KubeConflict: + snapshot = await self._read_own_snapshot() + if snapshot is None: + return False + except (KubeError, asyncio.TimeoutError) as error: + logger.warning("retune init: claim write failed: %s", error) + self._metrics.failure("state_write") + return False + else: + logger.warning("retune init: claim write kept conflicting") + self._metrics.failure("state_write") + return False + assert snapshot.state is not None + now_s = self._clock() + plan = plan_initialization( + snapshot.state, + spec_ok=self._spec_ok, + advisory=settings.advisory or settings.decommission, + fingerprint=self._fingerprint, + startup_flat=self._startup_flat(), + startup_by_wt=self._startup_by_wt(), + router_keys=self._router_keys(), + promote=settings.promote_last_advisory, + now_ms=int(round(now_s * 1000.0)), + interval_ms=int(round(settings.interval_s * 1000.0)), + # §4.7.2 first enable: skip the due window and the window in + # progress, whose trace may have begun mid-way. + first_window_end_ms=current_window_end_ms( + now_s, phase_s=settings.phase_s, interval_s=settings.interval_s + ), + ) + if plan.write: + if not self.is_leader() or epoch != self._epoch: + return False + try: + snapshot = await self._io(self._store.write, plan.state, snapshot) + except (KubeError, asyncio.TimeoutError) as error: + logger.warning("retune init: %s write failed: %s", plan.row, error) + self._metrics.failure("state_write") + return False + self._snapshot = snapshot + logger.info( + "retune init: term %s row %s (mode=%s rollback_pending=%s deploy_seq=%s " + "last_window_end_ms=%s spec_ok=%s)", + term, + plan.row, + plan.state.mode, + plan.state.rollback_pending, + plan.state.deploy_seq, + plan.state.last_window_end_ms, + self._spec_ok, + ) + controller = self._controller() + async with self._host_lock(): + if plan.restore_flat is not None and controller is not None: + try: + controller.restore_desired(plan.restore_flat) + except Exception: + logger.exception("retune init: restore_desired failed") + return False + if controller is not None: + self._rewind(controller) + if epoch != self._epoch or not self.is_leader(): + return False + self._initialized = True + self._rollback_since = None + if self._background: + self._outbox_task = asyncio.create_task( + self._outbox_loop(), name="planner-retune-outbox" + ) + self._outbox_wake.set() + return True + + async def _read_own_snapshot(self) -> Optional[StateSnapshot]: + """Read the state; drop a snapshot of another DGD uid (§4.7.3 step 2).""" + + try: + snapshot = await self._io(self._store.read) + if snapshot.state is not None and snapshot.state.dgd_uid != self._dgd.uid: + logger.warning( + "retune init: state belongs to DGD uid %s, not %s; recreating it", + snapshot.state.dgd_uid, + self._dgd.uid, + ) + await self._io(self._store.delete) + await self._delete_foreign_requests(snapshot.state.dgd_uid) + snapshot = StateSnapshot(state=None, resource_version=None, lease=None) + elif snapshot.corrupt: + # Replaced by the claim (conditional on its resourceVersion). + snapshot = dataclasses.replace(snapshot, state=None) + return snapshot + except (KubeError, asyncio.TimeoutError) as error: + logger.warning("retune init: reading the state failed: %s", error) + return None + + async def _delete_foreign_requests(self, dgd_uid: str) -> None: + try: + requests = await self._io( + self._kube.list_requests, f"{LABEL_DGD_UID}={dgd_uid}" + ) + except (KubeError, asyncio.TimeoutError) as error: + logger.warning( + "retune: listing Requests of DGD uid %s failed: %s", dgd_uid, error + ) + return + names = [(request.get("metadata") or {}).get("name") for request in requests] + self.registry.enqueue_delete([name for name in names if name]) + + async def _local_loss_locked(self, reason: str) -> None: + """§4.7.5 under ``_init_lock``.""" + + was = self._initialized or bool(self._outbox) or self._term is not None + self._initialized = False + self._epoch += 1 + self._rollback_since = None + # Health is re-established by the next term's first actuator round. + self._last_report = None + await self._cancel(self._outbox_task) + self._outbox_task = None + await self._cancel(self._round_task) + self._round_task = None + # The reconciler stops before its next delete (its should_continue + # checks the epoch); _reconcile_work stays pending until it returns. + await self._cancel(self._reconcile_task) + self._reconcile_task = None + dropped = len(self._outbox) + self._outbox.clear() + self._write_failures = 0 + self._abort_all() + controller = self._controller() + if controller is not None: + async with self._host_lock(): + try: + controller.drop_pending() + except Exception: + logger.exception("retune: drop_pending failed") + self._rewind(controller) + if was: + logger.info( + "retune: local leadership loss (%s); dropped %d queued decision(s)", + reason, + dropped, + ) + + def _request_reinit(self, reason: str) -> None: + logger.warning("retune: re-initializing: %s", reason) + self._initialized = False + self._reinit_requested = True + self._lease_wake.set() + + # ------------------------------------------------------ state writes + + async def _write_state(self, mutate: Mutation, what: str) -> WriteOutcome: + """Conditional write (term and resourceVersion) of ``mutate(state)``. + + ``mutate`` returns the new state, ``None`` to discard, or + ``_ALREADY`` when the server state already has it. A timeout is + resolved by re-reading: a write that landed is a success. Every + failed write (``error``) counts as ``failures_total{state_write}``. + """ + + outcome = await self._conditional_write(mutate, what) + if outcome == "error": + self._metrics.failure("state_write") + return outcome + + async def _conditional_write(self, mutate: Mutation, what: str) -> WriteOutcome: + epoch = self._epoch + async with self._write_lock: + snapshot = self._snapshot + for attempt in range(_WRITE_ATTEMPTS): + if epoch != self._epoch or not self.is_leader(): + return "discarded" + if snapshot is None or snapshot.state is None: + snapshot = await self._reread() + if snapshot is None or snapshot.state is None: + return "error" + state = snapshot.state + if state.leader_term != self._term: + return "lost_term" + new = mutate(state) + if new is _ALREADY: + self._snapshot = snapshot + return "already" + if new is None: + self._snapshot = snapshot + return "discarded" + assert isinstance(new, RetuneState) + try: + written = await self._io(self._store.write, new, snapshot) + except KubeConflict: + snapshot = await self._reread() + continue + except (KubeError, asyncio.TimeoutError) as error: + logger.warning("retune state write (%s) failed: %s", what, error) + snapshot = await self._reread() + if ( + snapshot is not None + and snapshot.state is not None + and snapshot.state.leader_term == self._term + and mutate(snapshot.state) is _ALREADY + ): + self._snapshot = snapshot + return "ok" + return "error" + if epoch != self._epoch: + return "discarded" + self._snapshot = written + return "ok" + return "error" + + async def _reread(self) -> Optional[StateSnapshot]: + try: + snapshot = await self._io(self._store.read) + except (KubeError, asyncio.TimeoutError) as error: + logger.warning("retune: re-reading the state failed: %s", error) + return None + if snapshot.state is not None and snapshot.state.dgd_uid != self._dgd.uid: + return StateSnapshot(state=None, resource_version=None, lease=None) + self._snapshot = snapshot + return snapshot + + async def _write_decision(self, entry: OutboxEntry) -> WriteOutcome: + decision = entry.decision + history = self._history_entry(entry) + window_end_ms = history["window_end_ms"] + applied = decision.outcome == "applied" + by_wt = entry.router_parameters + + def mutate(state: RetuneState) -> Union[RetuneState, None, object]: + if not self._entry_current(entry): + return None + if has_decision(state, decision.decision_id): + return _ALREADY + new = record_decision(state, history, window_end_ms) + if applied: + if not by_wt: + logger.error( + "retune: applied decision %s has no router parameters", + decision.decision_id, + ) + return new + new = deploy( + new, + fingerprint=entry.fingerprint, + mode="active", + flat=dict(decision.chosen.router), + by_wt=by_wt, + ) + return new + + outcome = await self._write_state(mutate, f"decision {decision.decision_id}") + if outcome in ("ok", "already"): + self._count_decision(decision) + logger.info( + "retune: persisted decision %s (%s)%s", + decision.decision_id, + decision.outcome, + " - deploying" if applied else "", + ) + return outcome + + def _count_decision(self, decision: RetuneDecision) -> None: + self._metrics.decision(decision.outcome) + failure_reason = getattr(decision, "failure_reason", None) + if decision.outcome in ("failed", "timeout"): + self._metrics.failure(failure_reason or decision.outcome) + details = getattr(decision, "details", None) or {} + # The DGDR executor reports the window cut under details["window"]. + window = details.get("window") + dropped = ( + window.get("dropped_members") + if isinstance(window, Mapping) + else details.get("dropped_members") + ) + if isinstance(dropped, int) and dropped > 0: + self._metrics.dropped_members(dropped) + if decision.launched_at_s is not None: + missing = details.get("missing_candidates", 0) + self._metrics.scores( + baseline=_finite(decision.baseline_score), + best=_finite(decision.best_score), + sweep_seconds=_finite(decision.sweep_wall_s), + missing_candidates=int(missing) if isinstance(missing, int) else 0, + ) + + def _history_entry(self, entry: OutboxEntry) -> dict[str, Any]: + decision = entry.decision + flat = {key: float(value) for key, value in decision.chosen.router.items()} + item: dict[str, Any] = { + "decision_id": decision.decision_id, + "window_end_ms": int(round(decision.window_end_s * 1000.0)), + "outcome": decision.outcome, + "failure_reason": getattr(decision, "failure_reason", None), + "reason": decision.reason, + "fingerprint": entry.fingerprint, + "baseline": _finite(decision.baseline_score), + "best": _finite(decision.best_score), + "flat": flat, + "decided_ms": int(round(decision.decided_at_s * 1000.0)), + } + if decision.outcome == "applied" and entry.router_parameters is not None: + item["by_wt"] = entry.router_parameters + elif decision.outcome == "advisory": + item["by_wt"] = self._resolve_by_wt(flat) + details = getattr(decision, "details", None) or {} + image = { + key: details[key] + for key in ("runtime_image", "runtime_version", "runtime_version_override") + if isinstance(details.get(key), str) + } + if image: + item["image"] = image + return item + + def _entry_current(self, entry: OutboxEntry) -> bool: + return ( + self._initialized + and self.is_leader() + and entry.epoch == self._epoch + and entry.term_seq == self._term_seq + and entry.generation == self._generation + and entry.fingerprint == self._fingerprint + ) + + # ----------------------------------------------------------- actuation + + async def _leader_round(self) -> None: + epoch = self._epoch + if self._settings.decommission: + await self._actuate() + await self._finish_decommission() + return + await self._check_canary() + await self._actuate() + # Off the control loop's critical path: revalidation and actuation + # never wait for the reconciler. + self._start_reconcile(epoch) + + async def _actuate(self) -> None: + target = self.target() + state = self.state + if target is None or state is None: + return + epoch = self._epoch + startup = self._evaluation.startup + report = await self._actuator.reconcile( + target, + initials=state.initials, + expected_frontends=max(1, startup.replicas if startup is not None else 1), + ) + if epoch != self._epoch: + return + self._last_report = report + if startup is not None and report.parity_ok is not None: + if report.parity_ok != self._p14_ok: + logger.info( + "retune: P14 parity %s -> %s", self._p14_ok, report.parity_ok + ) + self._p14_ok = report.parity_ok + await self._persist_initials(report) + await self._track_rollback(target, report) + + async def _persist_initials(self, report: ActuationReport) -> None: + state = self.state + if state is None: + return + merged = merge_initials(state, report.new_initials, keep=report.seen_router_ids) + if maps_equal(merged.initials, state.initials): + return + latched = dict(report.new_initials) + keep = frozenset(report.seen_router_ids) + + def mutate(current: RetuneState) -> Union[RetuneState, None, object]: + new = merge_initials(current, latched, keep=keep) + return _ALREADY if maps_equal(new.initials, current.initials) else new + + outcome = await self._write_state(mutate, "initials") + if outcome == "lost_term": + self._request_reinit("the state's leader_term is not ours") + + async def _track_rollback( + self, target: ActuationTarget, report: ActuationReport + ) -> None: + """Write "rollback complete" after a lease duration of convergence.""" + + state = self.state + if state is None or target.mode != "rollback" or not state.rollback_pending: + self._rollback_since = None + return + if not report.converged: + self._rollback_since = None + return + now = self._monotonic() + since = self._rollback_since + if since is None or since[0] != state.deploy_seq: + self._rollback_since = (state.deploy_seq, now) + return + if now - since[1] < self._settings.lease_duration_s: + return + expected = since[0] + entry = { + "decision_id": f"rolled_back-{expected}", + "outcome": "rolled_back", + "deploy_seq": expected, + "fingerprint": self._fingerprint, + "at_ms": int(round(self._clock() * 1000.0)), + } + + def mutate(current: RetuneState) -> Union[RetuneState, None, object]: + return rollback_complete(current, expected_deploy_seq=expected, entry=entry) + + outcome = await self._write_state(mutate, "rollback complete") + self._rollback_since = None + if outcome == "ok": + logger.info("retune: rollback complete (deploy_seq %d)", expected) + elif outcome == "lost_term": + self._request_reinit("the state's leader_term is not ours") + + async def _check_canary(self) -> None: + """P11: a dry-run Strict Request with ``workload.trace``.""" + + canary = self._canary + if canary is None: + self._canary_ok = True + return + now = self._monotonic() + if ( + self._canary_ok + and self._canary_at is not None + and now - self._canary_at < self._settings.canary_interval_s + ): + return + try: + ok = bool(await self._io(canary, self)) + except Exception as error: + logger.warning("retune canary (P11) failed: %s", error) + ok = False + if ok != self._canary_ok: + logger.info("retune canary (P11): %s", "ok" if ok else "failing") + self._canary_ok = ok + self._canary_at = now + + def _reconcile_allowed(self, epoch: int) -> bool: + """§4.4.5: only the initialized leader of ``epoch`` cleans up. + + Called from the reconciler's thread too (``is_leader`` is + thread-safe, the rest are single reads). + """ + + return ( + not self._stopping + and self._initialized + and epoch == self._epoch + and self.is_leader() + ) + + def reconcile_running(self) -> bool: + """Whether a reconciler round (its task or its call) is still running.""" + + task, work = self._reconcile_task, self._reconcile_work + return (task is not None and not task.done()) or ( + work is not None and not work.done() + ) + + def _start_reconcile(self, epoch: int) -> None: + """Start one reconciler round in its own task, unless one still runs.""" + + if self._reconciler is None or not self._reconcile_allowed(epoch): + return + if self.reconcile_running(): + logger.debug("retune reconciler: the previous round is still running") + return + self._reconcile_task = asyncio.create_task( + self._reconcile(epoch), name="planner-retune-reconcile" + ) + + async def _reconcile(self, epoch: int) -> None: + """One reconciler round, waited for at most ``reconcile_budget_s``. + + The reconciler calls ``should_continue`` before every delete or + unlink and stops once it is ``False``: this process is no longer the + initialized leader of ``epoch``, or the budget has passed. A blocking + call cannot be cancelled, so ``_reconcile_work`` stays pending until + it returns and no other round starts meanwhile. + """ + + reconciler = self._reconciler + if reconciler is None or not self._reconcile_allowed(epoch): + return + state = self.state + settings = self._settings + expired = threading.Event() + + def should_continue() -> bool: + return not expired.is_set() and self._reconcile_allowed(epoch) + + call = functools.partial( + reconciler.run_once, + last_window_end_ms=None if state is None else state.last_window_end_ms, + due_window_end_ms=due_window_end_ms( + self._clock(), + phase_s=settings.phase_s, + interval_s=settings.interval_s, + grace_s=settings.grace_s, + ), + deadline_for=self._deadline_for, + frontend_pods=getattr(self._actuator, "last_pods", None), + should_continue=should_continue, + ) + cancellable = inspect.iscoroutinefunction(reconciler.run_once) + work: asyncio.Future[Any] = asyncio.ensure_future( + call() if cancellable else self._blocking(call) + ) + work.add_done_callback(_log_reconcile_failure) + self._reconcile_work = work + try: + await asyncio.wait_for( + asyncio.shield(work), timeout=settings.reconcile_budget_s + ) + except asyncio.TimeoutError: + logger.warning( + "retune reconciler: the round exceeded its %.0fs budget; it " + "stops before its next delete, later rounds wait for it", + settings.reconcile_budget_s, + ) + except asyncio.CancelledError: + raise + except Exception: + pass # logged by _log_reconcile_failure + finally: + if not work.done(): + expired.set() + if cancellable: + work.cancel() + + def _deadline_for(self, window_end_ms: int) -> float: + """``D_k = L_k + timeout`` in epoch seconds.""" + + settings = self._settings + return window_end_ms / 1000.0 + settings.grace_s + settings.timeout_s + + async def _finish_decommission(self) -> None: + """§4.11: once nothing is left to roll back, delete Requests and Leases.""" + + state = self.state + if self.decommissioned or state is None or state.rollback_pending: + return + dgd = self._dgd + try: + requests = await self._io( + self._kube.list_requests, f"{LABEL_DGD_UID}={dgd.uid}" + ) + for request in requests: + name = (request.get("metadata") or {}).get("name") + if name: + await self._io(self._kube.delete_request, name) + await self._io(self._store.delete) + except (KubeError, asyncio.TimeoutError) as error: + logger.warning("retune clean-up: deleting Requests/state failed: %s", error) + return + try: + await self._io(self._kube.delete_lease, self._leadership.name) + except (KubeError, asyncio.TimeoutError) as error: + logger.warning("retune clean-up: deleting the lease failed: %s", error) + self._leadership.drop() + self._initialized = False + self.decommissioned = True + self._stopping = True + logger.info("retune clean-up of DGD %s finished", dgd.name) + + # ------------------------------------------------------------- helpers + + async def _install_evaluation(self, evaluation: SpecEvaluation) -> None: + old = self._evaluation + startup_changed = _startup_key(old.startup) != _startup_key(evaluation.startup) + if startup_changed: + self._p14_ok = None + self._evaluation = evaluation + self._spec_ok = bool(evaluation.report.spec_ok) and self._p14_ok is not False + self._fingerprint = self._fingerprint_fn(evaluation.startup) + if startup_changed: + self._resolver = self._make_resolver(evaluation.startup) + if old.dgd != evaluation.dgd: + if old.dgd.uid != evaluation.dgd.uid: + # Before the re-initialization deletes the old uid's + # Requests: the reconciler must treat the new uid's Requests + # as its own and the old ones as another DGD's. + self._set_reconciler_dgd(evaluation.dgd) + self._store = StateStore( + self._kube, + evaluation.dgd, + timeout_s=self._settings.k8s_timeout_s, + clock=self._clock, + ) + self._leadership.set_owner(evaluation.dgd) + await self._swap_actuator(evaluation) + elif startup_changed: + self._actuator.set_startup_maps(self._actuator_maps(evaluation.startup)) + self._last_report = None + self._log_report(evaluation.report) + + def _set_reconciler_dgd(self, dgd: DgdRef) -> None: + setter = getattr(self._reconciler, "set_dgd", None) + if setter is None: + return + try: + setter(dgd) + except Exception: + logger.exception( + "retune: moving the reconciler to DGD uid %s failed", dgd.uid + ) + + async def _swap_actuator(self, evaluation: SpecEvaluation) -> None: + """A new actuator for a new DGD identity or dynamo namespace.""" + + self._dgd = evaluation.dgd + previous = self._actuator + self._actuator = self._actuator_factory( + evaluation.dgd, self._actuator_maps(evaluation.startup) + ) + self._last_report = None + await self._close_actuator(previous) + + def _actuator_maps( + self, startup: Optional[FrontendStartup] + ) -> dict[str, dict[str, float]]: + if startup is not None and startup.startup_maps: + return {wt: dict(params) for wt, params in startup.startup_maps.items()} + # Unknown startup: rollback needs only the latched initials. + return {wt: {} for wt in self._settings.worker_types} + + def _make_resolver( + self, startup: Optional[FrontendStartup] + ) -> Optional[RouterResolver]: + if startup is None or self._resolver_factory is None: + return None + try: + return self._resolver_factory(startup) + except Exception: + logger.exception("retune: building the router resolver failed") + return None + + def _router_keys(self) -> tuple[str, ...]: + controller = self._controller() + keys = ( + getattr(controller, "router_keys", None) if controller is not None else None + ) + return tuple(keys) if keys else self._settings.router_keys + + def _startup_by_wt(self) -> Optional[dict[str, dict[str, float]]]: + startup = self._evaluation.startup + if startup is None: + return None + return {wt: dict(params) for wt, params in startup.startup_maps.items()} + + def startup_flat(self) -> Optional[dict[str, float]]: + """The controller's startup router point (live names), if known.""" + + return self._startup_flat() + + def _startup_flat(self) -> Optional[dict[str, float]]: + resolver = self._resolver + keys = self._router_keys() + if resolver is None or not keys: + return None + try: + return resolver.startup_flat(keys) + except Exception: + logger.exception("retune: resolving the startup router point failed") + return None + + def _resolve_by_wt( + self, flat: Mapping[str, float] + ) -> Optional[dict[str, dict[str, float]]]: + resolver = self._resolver + if resolver is None: + return None + try: + return resolver.by_wt(flat) + except Exception: + logger.exception("retune: resolving router point %s failed", dict(flat)) + return None + + def _p12_ok(self) -> bool: + """P12 from the last actuator round (routers present, admin API on).""" + + report = self._last_report + if report is None or report.admin_api_disabled: + return False + present = report.routers_present + return bool(report.converged if present is None else present) + + def _last_decided_k(self) -> Optional[int]: + state = self.state + if state is None or state.last_window_end_ms is None: + return None + return window_index( + state.last_window_end_ms, + phase_s=self._settings.phase_s, + interval_s=self._settings.interval_s, + ) + + def _rewind(self, controller: Any) -> None: + last = self._last_decided_k() + try: + controller.rewind(-1 if last is None else last) + except Exception: + logger.exception("retune: rewind failed") + + def _rewind_now(self) -> None: + controller = self._controller() + if controller is not None: + self._rewind(controller) + + def _abort_all(self) -> None: + with self._aborts_lock: + aborts, self._aborts = self._aborts, [] + for event in aborts: + event.set() + + def _controller(self) -> Any: + host = self._host + if host is None: + return None + engine = getattr(host, "_engine", None) + return getattr(engine, "retune_controller", None) + + @contextlib.asynccontextmanager + async def _host_lock(self) -> AsyncIterator[None]: + lock = getattr(self._host, "_config_lock", None) + if lock is None: + yield + return + async with lock: + yield + + async def _io(self, function: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: + call = functools.partial(function, *args, **kwargs) + return await asyncio.wait_for( + self._blocking(call), timeout=self._settings.io_timeout_s + ) + + @staticmethod + async def _cancel(task: Optional[asyncio.Task[Any]]) -> None: + if task is None or task.done() or task is asyncio.current_task(): + return + task.cancel() + await asyncio.wait([task]) + + @staticmethod + async def _close_actuator(actuator: Any) -> None: + client = getattr(actuator, "_client", None) + close = getattr(client, "close", None) + if close is None: + return + try: + result = close() + if inspect.isawaitable(result): + await result + except Exception: + logger.debug( + "retune: closing the router policy client failed", exc_info=True + ) + + def _log_report(self, report: PremiseReport) -> None: + for reason in report.reasons: + logger.warning("retune premise failed: %s", reason) + for warning in report.warnings: + logger.warning("retune premise warning: %s", warning) + + def _publish(self) -> None: + state = self.state + try: + self._metrics.supervisor_state( + leader=self.is_leader(), + ready=self.ready(), + initialized=self._initialized, + rollback_pending=bool(state is not None and state.rollback_pending), + generation=self._generation, + ) + except Exception: + logger.debug("retune: publishing supervisor metrics failed", exc_info=True) + + # ---------------------------------------------------------------- loops + + async def _lease_loop(self) -> None: + while not self._stopping: + try: + await self.lease_round() + except asyncio.CancelledError: + raise + except Exception: + logger.exception("retune lease loop iteration failed") + await self._wait(self._lease_wake, self._settings.renew_interval_s) + + async def _control_loop(self) -> None: + while not self._stopping: + try: + await self.control_round() + except asyncio.CancelledError: + raise + except Exception: + logger.exception("retune control loop iteration failed") + await self._wait(None, self._settings.actuator_interval_s) + + async def _outbox_loop(self) -> None: + while not self._stopping: + await self._outbox_wake.wait() + self._outbox_wake.clear() + try: + await self.outbox_round() + except asyncio.CancelledError: + raise + except Exception: + logger.exception("retune outbox round failed") + if self._reinit_requested: + return + if self._outbox: + # A failed write: retry after a short pause. + await self._sleep(min(5.0, self._settings.k8s_timeout_s)) + self._outbox_wake.set() + + async def _wait(self, event: Optional[asyncio.Event], timeout_s: float) -> None: + """Sleep ``timeout_s`` (the injectable sleep) or until ``event`` is set.""" + + sleeper = asyncio.ensure_future(self._sleep(timeout_s)) + waiters: set[asyncio.Future[Any]] = {sleeper} + if event is not None: + waiters.add(asyncio.ensure_future(event.wait())) + try: + await asyncio.wait(waiters, return_when=asyncio.FIRST_COMPLETED) + finally: + for waiter in waiters: + waiter.cancel() + await asyncio.wait(waiters) + if event is not None: + event.clear() + + +def _log_reconcile_failure(work: asyncio.Future[Any]) -> None: + """Done callback of a reconciler call (it may outlive its round's task).""" + + if work.cancelled(): + return + error = work.exception() + if error is not None: + logger.error("retune reconciler round failed", exc_info=error) + + +def _startup_key(startup: Optional[FrontendStartup]) -> Optional[str]: + return None if startup is None else startup.fingerprint_source + + +def _finite(value: Optional[float]) -> Optional[float]: + if value is None: + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py new file mode 100644 index 000000000000..026261454064 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -0,0 +1,1401 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Cut a retune window file from the frontends' live request-trace segments. + +The frontends append ``dynamo.request.trace.v1`` rows to rotating +``//req.NNNNNN.jsonl.gz`` segments on a shared RWX claim, +one gzip member per flush (``lib/llm/src/telemetry/jsonl_gz.rs``). +``cut_window`` turns the ``request_end`` rows that arrived in +``[start_ms, end_ms)`` and completed by ``as_of_ms`` into the bare-row window +file a DGDR Run replays. Selection, censoring, de-duplication, the single +block-size rule and ``row_counts`` are those of +``retune.window.read_trace_window``, and the rows are the same bytes; only +the output is gzip-compressed and the memory is bounded. + +* **Members.** Each gzip member is decoded on its own and checked against its + CRC32 and ISIZE before any of its lines is read. A member that fails (cut + short by a segment still being written, a zero-filled hole a lagging RWX + client shows, a flipped bit) is dropped whole and the reader resynchronises + on the next gzip header, so one run of bad bytes may hold many members. + Runs of bad bytes above ``MAX_DROPPED_MEMBER_RATIO`` of the members, or + bad bytes above that ratio of the gzip bytes, fail the cut + (``trace_corrupt``); a run that reaches the end of a writer's newest + segment is its in-flight flush (or a crashed writer's last one) and is + counted but does not count toward either ratio. +* **Pass 1** parses every row as the replay reader does (a row it would + reject, in a member whose CRC is good, fails the cut as ``trace_corrupt``) + and keeps only columns for each selected row (~45 B: arrival, completion, + a 128-bit digest of ``request_id``, where its line is, block size, flags). + Duplicates are removed by digest, keeping the first in file order, + whenever the columns outgrow ``max_rows`` by a slack, so ``max_rows`` + bounds memory and is checked on de-duplicated rows. +* **Pass 2** re-reads only the members that hold kept rows, in output order + (arrival, completion, then ``request_id`` within equal pairs), re-parses + each line with its hashes and streams ``_window_record`` rows to gzip, + compressed and written by a thread. Decoded lines wait in a bounded cache + that evicts the member needed furthest in the future. A member or row + that no longer matches pass 1 fails the cut (the segment changed). +* **Atomic output.** The file is written to ``.tmp--`` + next to ``out_path``, fsynced, made 0644 and hard-linked into place; an + existing ``out_path`` is reused (another process cut the same window). +* **Child process.** ``cut_window`` runs the cut in a ``spawn`` child, like + the replay sweep's workers, and kills it at the deadline or on cancel, so + a hung RWX read or a runaway cut never holds the sweep thread. +""" + +from __future__ import annotations + +import errno +import gzip +import hashlib +import json +import logging +import math +import multiprocessing +import os +import queue +import re +import secrets +import stat +import struct +import sys +import threading +import time +import zlib +from array import array +from collections import OrderedDict +from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from multiprocessing.connection import Connection, wait +from operator import attrgetter +from pathlib import Path +from typing import Any, BinaryIO, Literal, Optional, Protocol, Union, get_args + +import numpy as np + +from dynamo.planner.offline.trace_data import _reject_duplicate_copies +from dynamo.planner.retune.live.contracts import WindowCutResult +from dynamo.planner.retune.window import ( + _dumps, + _new_counts, + _parse_request_end, + _request_end_event, + _RequestRow, + _window_record, +) + +logger = logging.getLogger(__name__) + +StrPath = Union[str, "os.PathLike[str]"] + + +class _ByteSink(Protocol): + def write(self, data: bytes, /) -> Any: + ... + + +WindowCutFailure = Literal[ + "window_cut_timeout", # the deadline passed; the child was killed + "window_too_large", # more than max_rows de-duplicated rows + "trace_corrupt", # too many bad gzip members, or a malformed row + "window_cut_failed", # I/O error, mixed block sizes, child died, ... + "cancelled", # the caller's cancel event was set; the child was killed +] +WINDOW_CUT_FAILURES: tuple[str, ...] = get_args(WindowCutFailure) + +# Runs of bad bytes (other than in-flight tails) over all members, or their +# bytes over all gzip bytes, above which the cut fails as trace_corrupt. A run +# counts once however many members it wiped out (a zero-filled hole); its +# bytes measure how much of the trace it took. +MAX_DROPPED_MEMBER_RATIO = 0.01 +# Window files are written fast rather than small: on hash-chain rows level +# 1 is ~5x faster than level 6 for ~8% more bytes, and the cut budget pays +# for compression. +OUTPUT_COMPRESSLEVEL = 1 +# Temporary window files: ``.tmp--`` next to out_path. +TEMP_PREFIX = ".tmp-" + +_TEMP_NAME = re.compile(r"^\.tmp-(\d+)-([0-9A-Za-z]+)$") +_SEGMENT_INDEX = re.compile(r"\.(\d+)\.jsonl(?:\.gz)?$") +_GZIP_MAGIC = b"\x1f\x8b\x08" # ID1, ID2, CM=deflate +_GZIP_WBITS = 16 + zlib.MAX_WBITS # one gzip member, header and trailer +_READ_CHUNK = 1 << 20 +_DECODE_STEP = 1 << 16 +_COMPACT_AT = 1 << 20 # consumed reader bytes before the buffer is compacted +_PLAIN_BLOCK = 1 << 20 # bytes of whole lines per plain-JSONL pseudo-member +# A trace file the reconciler deleted under the walk (ESTALE: on NFS). +_VANISHED = frozenset({errno.ENOENT, errno.ESTALE}) +# Pass-1 columns may hold this fraction of max_rows (at least the minimum) +# beyond max_rows before duplicates are removed. +_DEDUP_SLACK_RATIO = 0.25 +_DEDUP_SLACK_MIN = 1024 +_PASS2_CACHE_BYTES = 256 << 20 +_PASS2_BLOCK_ROWS = 1 << 16 +_LINE_OVERHEAD = 64 # bytes object and dict slot per cached line +_OPEN_FILES = 32 +_WRITE_BUFFER = 1 << 20 +_WRITE_QUEUE = 4 # batches waiting for the writer thread +_FLAG_AGENT_CONTEXT = 1 +_FLAG_NO_TOTAL_TIME = 2 +_POLL_INTERVAL_S = 0.25 +_EXIT_GRACE_S = 1.0 +_JOIN_TIMEOUT_S = 5.0 +_CHILD_NICENESS = 10 +_CHILD_SWITCH_INTERVAL_S = 0.0005 +_MAX_DROP_EXAMPLES = 5 + + +class WindowCutError(Exception): + """A window cut failed; ``reason`` is one of ``WINDOW_CUT_FAILURES``.""" + + def __init__(self, reason: WindowCutFailure, message: str) -> None: + if reason not in WINDOW_CUT_FAILURES: + raise ValueError(f"unknown window cut failure reason {reason!r}") + super().__init__(message) + self.reason: WindowCutFailure = reason + self.message = message + + def __reduce__(self) -> tuple[Any, ...]: + return (type(self), (self.reason, self.message)) + + +def temp_file_name(window_end_ms: float, nonce: Optional[str] = None) -> str: + """The name of a window's temporary file (``.tmp--``).""" + + if nonce is None: + nonce = secrets.token_hex(8) + elif not re.fullmatch(r"[0-9A-Za-z]+", nonce): + raise ValueError(f"nonce must be alphanumeric, got {nonce!r}") + return f"{TEMP_PREFIX}{int(round(window_end_ms))}-{nonce}" + + +def temp_file_window_end(name: str) -> Optional[int]: + """The window end of a ``temp_file_name`` name; ``None`` for any other name.""" + + match = _TEMP_NAME.match(name) + return int(match.group(1)) if match else None + + +def segment_index(name: str) -> Optional[int]: + """The ``NNNNNN`` of a ``.NNNNNN.jsonl[.gz]`` segment name, else ``None``.""" + + match = _SEGMENT_INDEX.search(name) + return int(match.group(1)) if match else None + + +# ------------------------------------------------------------- entry points + + +def cut_window( + segments_root: StrPath, + start_ms: float, + end_ms: float, + as_of_ms: float, + out_path: StrPath, + *, + max_rows: int, + deadline_s: float, + cancel: Optional[threading.Event] = None, +) -> WindowCutResult: + """Cut a window file in a spawned child process (``contracts.WindowCutter``). + + ``deadline_s`` is absolute wall-clock time (``time.time()`` seconds), e.g. + ``min(now + cut_budget_seconds, D_k)``. The child is killed at the + deadline (``window_cut_timeout``) or as soon as ``cancel`` is set + (``cancelled``; any object with ``is_set()`` works, so a caller can pass + one that watches several events). Every failure raises ``WindowCutError``; + invalid arguments raise ``ValueError`` before a child is started. + """ + + _validate(start_ms, end_ms, as_of_ms, out_path, max_rows) + if not math.isfinite(deadline_s): + raise ValueError("deadline_s must be finite") + deadline = time.monotonic() + (deadline_s - time.time()) + out_path = os.fspath(out_path) + temp_path = os.path.join( + os.path.dirname(os.path.abspath(out_path)), temp_file_name(end_ms) + ) + started = time.monotonic() + result = _run_in_child( + cut_window_inprocess, + (os.fspath(segments_root), start_ms, end_ms, as_of_ms, out_path), + {"max_rows": max_rows, "temp_path": temp_path}, + deadline=deadline, + cancel=cancel, + cleanup=(temp_path,), + name=f"retune-window-cut-{int(round(end_ms))}", + ) + logger.info( + "retune window [%s, %s) as of %s: %d requests in %s (%s) in %.1fs; " + "dropped %d gzip members (%d bytes)", + start_ms, + end_ms, + as_of_ms, + result.num_requests, + result.path, + "reused" if result.reused else "written", + time.monotonic() - started, + result.dropped_members, + result.dropped_bytes, + ) + return result + + +def cut_window_inprocess( + segments_root: StrPath, + start_ms: float, + end_ms: float, + as_of_ms: float, + out_path: StrPath, + *, + max_rows: int, + temp_path: Optional[StrPath] = None, +) -> WindowCutResult: + """Cut a window file in this process; the body of ``cut_window``'s child. + + An empty window writes no file (``path=None``). ``temp_path`` names the + temporary file (it must be in ``out_path``'s directory); by default a + fresh ``temp_file_name`` there. Failures raise ``WindowCutError``. + """ + + _validate(start_ms, end_ms, as_of_ms, out_path, max_rows) + out_path = os.fspath(out_path) + directory = os.path.dirname(os.path.abspath(out_path)) + if temp_path is None: + temp_path = os.path.join(directory, temp_file_name(end_ms)) + temp_path = os.fspath(temp_path) + if os.path.dirname(os.path.abspath(temp_path)) != directory: + # os.link needs one filesystem; the reconciler looks for it there. + raise ValueError(f"temp_path {temp_path!r} is not next to {out_path!r}") + try: + files = _segment_files(os.fspath(segments_root)) + selection = _Selection(start_ms, end_ms, as_of_ms, max_rows) + items = _trace_items(files) + try: + selection.scan(items) + finally: + items.close() + selection.finish() + + def result(path: Optional[str], reused: bool = False) -> WindowCutResult: + return WindowCutResult( + path=path, + num_requests=selection.num_rows, + trace_block_size=selection.block_size, + row_counts=dict(selection.counts), + dropped_members=selection.dropped_members, + dropped_bytes=selection.dropped_bytes, + reused=reused, + ) + + if selection.num_rows == 0: + return result(None) + if os.path.lexists(out_path): + # Another process (an earlier leader, an adopted Request's cut) + # already published this window; DGDR Requests name it by path. + return result(out_path, reused=True) + paths = [path for path, _ in files] + created = _publish( + out_path, temp_path, lambda sink: _write_rows(selection, paths, sink) + ) + return result(out_path, reused=not created) + except OSError as error: + raise WindowCutError( + "window_cut_failed", f"{type(error).__name__}: {error}" + ) from error + + +def _validate( + start_ms: float, + end_ms: float, + as_of_ms: float, + out_path: StrPath, + max_rows: int, +) -> None: + for name, value in ( + ("start_ms", start_ms), + ("end_ms", end_ms), + ("as_of_ms", as_of_ms), + ): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be a number, got {value!r}") + if not math.isfinite(value): + raise ValueError(f"{name} must be finite") + if not start_ms < end_ms: + raise ValueError("start_ms must be before end_ms") + if as_of_ms < end_ms: + raise ValueError("as_of_ms must not be before end_ms") + if isinstance(max_rows, bool) or not isinstance(max_rows, int) or max_rows < 1: + raise ValueError(f"max_rows must be a positive integer, got {max_rows!r}") + if not os.fspath(out_path).endswith(".gz"): + raise ValueError(f"window file must be gzip (.gz): {os.fspath(out_path)!r}") + + +# ------------------------------------------------------------ child process + + +def _run_in_child( + function: Callable[..., Any], + args: Sequence[Any], + kwargs: Mapping[str, Any], + *, + deadline: float, + cancel: Optional[threading.Event], + cleanup: Sequence[str] = (), + name: str = "retune-window-cut", +) -> Any: + """``function(*args, **kwargs)`` in a ``spawn`` child; its return value. + + ``deadline`` is ``time.monotonic()`` time. The child is killed at the + deadline, on cancel, or once it has answered; ``cleanup`` paths are + removed unless it answered with a result (a killed child leaves its + temporary file behind). A ``WindowCutError`` raised by ``function`` is + re-raised with its reason; any other failure, the child's death included, + is ``window_cut_failed``. + """ + + _check_interrupted(deadline, cancel) + context = multiprocessing.get_context("spawn") + receiver, sender = context.Pipe(duplex=False) + process = context.Process( + target=_child_main, + args=(sender, function, tuple(args), dict(kwargs)), + name=name, + daemon=True, + ) + outcome: Optional[tuple[Any, ...]] = None + try: + try: + process.start() + except Exception as error: + raise WindowCutError( + "window_cut_failed", f"cannot start the window cut process: {error}" + ) from error + finally: + sender.close() + while outcome is None: + _check_interrupted(deadline, cancel) + timeout = max(0.0, min(deadline - time.monotonic(), _POLL_INTERVAL_S)) + if not wait([receiver, process.sentinel], timeout): + continue + try: + outcome = receiver.recv() + except (EOFError, OSError): + process.join(_JOIN_TIMEOUT_S) + raise WindowCutError( + "window_cut_failed", + f"window cut process exited with code {process.exitcode} " + "before returning a result", + ) from None + finally: + _stop(process, grace_s=_EXIT_GRACE_S if outcome is not None else 0.0) + receiver.close() + if outcome is None or outcome[0] != "ok": + for path in cleanup: + _unlink_quietly(path) + assert outcome is not None # the loop only ends with one + if outcome[0] == "ok": + return outcome[1] + _, reason, message = outcome + raise WindowCutError(reason, message) + + +def _check_interrupted(deadline: float, cancel: Optional[threading.Event]) -> None: + if cancel is not None and cancel.is_set(): + raise WindowCutError("cancelled", "window cut cancelled") + if time.monotonic() >= deadline: + raise WindowCutError("window_cut_timeout", "window cut exceeded its deadline") + + +def _stop(process: multiprocessing.process.BaseProcess, *, grace_s: float) -> None: + """Let ``process`` exit for ``grace_s``, then kill it and reap it. + + A child blocked in uninterruptible I/O (a hard-mounted RWX volume) does + not die on SIGKILL until the I/O returns; it is left behind rather than + holding the sweep thread. SIGKILL takes effect before it returns to user + space, so it cannot publish a window afterwards. + """ + + if process.pid is None: + return + if grace_s > 0: + process.join(grace_s) + if process.is_alive(): + process.kill() + process.join(_JOIN_TIMEOUT_S) + if process.is_alive(): + logger.error( + "retune window cut process %s (pid %s) did not exit after SIGKILL; " + "leaving it behind", + process.name, + process.pid, + ) + + +def _child_main( + conn: Connection, + function: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> None: + """Child entry point: answer ``("ok", value)`` or ``("error", reason, message)``.""" + + try: + os.nice(_CHILD_NICENESS) + except OSError: + pass + # The writer thread takes the GIL back after every zlib block; at the + # default 5 ms interval it would wait behind parsing (pass 2 ~30% slower). + sys.setswitchinterval(_CHILD_SWITCH_INTERVAL_S) + try: + outcome: tuple[Any, ...] = ("ok", function(*args, **kwargs)) + except WindowCutError as error: + outcome = ("error", error.reason, error.message) + except BaseException as error: # noqa: BLE001 - every failure is reported + outcome = ("error", "window_cut_failed", f"{type(error).__name__}: {error}") + try: + conn.send(outcome) + except Exception as error: + conn.send(("error", "window_cut_failed", f"unsendable result: {error}")) + finally: + conn.close() + + +# ------------------------------------------------------------ gzip members + + +@dataclass(frozen=True) +class _Member: + """A gzip member that decoded and matched its CRC32 and ISIZE.""" + + offset: int + end: int + payload: bytes + + +@dataclass(frozen=True) +class _Garbage: + """A maximal run of bytes ``[offset, end)`` that holds no valid member.""" + + offset: int + end: int + at_eof: bool + + +class _GzipMemberReader: + """Iterate a gzip file as validated members and runs of bad bytes. + + Every byte from ``offset`` to the end of the file is in exactly one item, + in file order. A member is yielded only after zlib reached its end and + its trailer's CRC32 and ISIZE match the decompressed bytes (zlib checks + them too; the explicit check does not depend on how zlib was built). + Anything else - a member cut short by the end of the file, a corrupt + one, zeros - starts a ``_Garbage`` run that ends at the next gzip header + that decodes, or at the end of the file. Memory holds one member's + compressed and decompressed bytes plus a read chunk. + """ + + def __init__(self, source: BinaryIO, offset: int = 0) -> None: + if offset: + source.seek(offset) + self._source = source + self._buf = bytearray() + self._pos = 0 # current position in _buf + self._base = offset # file offset of _buf[0] + self._eof = False + self._pending: Optional[_Member] = None + + def __iter__(self) -> _GzipMemberReader: + return self + + def __next__(self) -> Union[_Member, _Garbage]: + if self._pending is not None: + pending, self._pending = self._pending, None + return pending + if self._pos >= len(self._buf) and not self._fill(): + raise StopIteration + start = self._offset() + member = self._decode() + if member is not None: + return member + while self._skip_to_magic(): + member = self._decode() + if member is not None: + self._pending = member + return _Garbage(start, member.offset, at_eof=False) + return _Garbage(start, self._offset(), at_eof=True) + + def _offset(self) -> int: + return self._base + self._pos + + def _fill(self) -> bool: + if self._eof: + return False + chunk = self._source.read(_READ_CHUNK) + if not chunk: + self._eof = True + return False + self._buf += chunk + return True + + def _consume(self, count: int) -> None: + self._pos += count + if self._pos >= _COMPACT_AT and 2 * self._pos >= len(self._buf): + del self._buf[: self._pos] + self._base += self._pos + self._pos = 0 + + def _decode(self) -> Optional[_Member]: + """Decode and consume the member at the current position. + + Returns ``None``, consuming nothing, when no valid member starts here. + """ + + decoder = zlib.decompressobj(wbits=_GZIP_WBITS) + parts: list[bytes] = [] + index = self._pos + while True: + if index >= len(self._buf) and not self._fill(): + return None # cut short by the end of the file + piece = self._buf[index : index + _DECODE_STEP] + try: + parts.append(decoder.decompress(piece)) + except zlib.error: + return None + if decoder.eof: + end = index + len(piece) - len(decoder.unused_data) + payload = b"".join(parts) + if not _trailer_matches(self._buf[end - 8 : end], payload): + return None + member = _Member(self._offset(), self._base + end, payload) + self._consume(end - self._pos) + return member + index += len(piece) + + def _skip_to_magic(self) -> bool: + """Consume at least one byte, then up to the next gzip magic. + + ``False`` (everything consumed) when no magic is left in the file. + """ + + self._consume(1) + keep = len(_GZIP_MAGIC) - 1 # a magic may straddle two reads + while True: + found = self._buf.find(_GZIP_MAGIC, self._pos) + if found >= 0: + self._consume(found - self._pos) + return True + self._consume(max(0, len(self._buf) - keep - self._pos)) + if not self._fill(): + self._consume(len(self._buf) - self._pos) + return False + + +def _trailer_matches(trailer: Union[bytes, bytearray], payload: bytes) -> bool: + crc, size = struct.unpack(" Optional[bytes]: + """The payload of ``data`` when it is exactly one valid gzip member.""" + + decoder = zlib.decompressobj(wbits=_GZIP_WBITS) + try: + payload = decoder.decompress(data) + except zlib.error: + return None + if not decoder.eof or decoder.unused_data or len(data) < 8: + return None + return payload if _trailer_matches(data[-8:], payload) else None + + +# ---------------------------------------------------------------- pass 1 + + +def _segment_files(root: str) -> list[tuple[str, bool]]: + """Trace files under ``root`` in replay-reader order, and which are newest. + + The files and their order are those of ``resolve_trace_files([root])``: + sorted ``*.jsonl`` and ``*.jsonl.gz`` regular files, and a file present + both compressed and not is an error. A directory or file under ``root`` + that vanishes during the walk (ENOENT, or ESTALE on NFS: the reconciler + deletes a dead pod's segments) is skipped; any other error listing or + stat-ing one (EACCES, EIO, ETIMEDOUT on a soft mount, ...) fails the cut + as ``window_cut_failed`` rather than silently losing a pod's rows, as + does any error on ``root`` itself. Each file comes with whether it is the + newest segment (highest ``NNNNNN``) of its directory, the one its writer + may still be appending to. + """ + + try: + root_mode = os.stat(root).st_mode + except OSError as error: + if error.errno in _VANISHED: + raise WindowCutError( + "window_cut_failed", f"trace segments root {root} does not exist" + ) from error + raise _unreadable(root, error) from error + if stat.S_ISDIR(root_mode): + + def walk_error(error: OSError) -> None: + if error.filename != root and error.errno in _VANISHED: + logger.info( + "retune window cut: trace directory %s vanished (%s); " + "skipping it", + error.filename, + error, + ) + return + raise _unreadable(error.filename, error) from error + + found: list[Path] = [] + for directory, _, names in os.walk(root, onerror=walk_error): + for name in names: + if name.endswith(".jsonl") or name.endswith(".jsonl.gz"): + candidate = os.path.join(directory, name) + if _is_regular_file(candidate): + found.append(Path(candidate)) + try: + paths = _reject_duplicate_copies(sorted(found)) + except ValueError as error: + raise WindowCutError("window_cut_failed", str(error)) from error + else: + paths = [Path(root)] + newest: dict[Path, tuple[int, str]] = {} + for path in paths: + index = segment_index(path.name) + key = (-1 if index is None else index, path.name) + if path.parent not in newest or key > newest[path.parent]: + newest[path.parent] = key + return [(str(path), newest[path.parent][1] == path.name) for path in paths] + + +def _is_regular_file(path: str) -> bool: + """Whether ``path`` is a regular file (following symlinks, like ``is_file``). + + ``False`` when it vanished; any other stat error fails the cut. + """ + + try: + return stat.S_ISREG(os.stat(path).st_mode) + except OSError as error: + if error.errno in _VANISHED: + return False + raise _unreadable(path, error) from error + + +def _unreadable(path: Any, error: OSError) -> WindowCutError: + return WindowCutError( + "window_cut_failed", + f"cannot read trace segments at {path}: {type(error).__name__}: {error}", + ) + + +@dataclass(frozen=True) +class _PlainBlock: + """Whole lines ``[offset, end)`` of a plain JSONL file. + + ``dropped`` counts the bytes of an unterminated last line that does not + parse (still being written); they are not in ``payload``. + """ + + offset: int + end: int + payload: bytes + dropped: int = 0 + + +# (file index, path, newest, item); an OSError item means the file vanished. +_TraceItem = tuple[int, str, bool, Union[_Member, _Garbage, _PlainBlock, OSError]] + + +def _trace_items( + files: Sequence[tuple[str, bool]], +) -> Generator[_TraceItem, None, None]: + """Every member, bad byte run and plain block of ``files``, in file order. + + A file that vanishes (ENOENT, or ESTALE on NFS) before or while it is + read yields its error and the walk goes on; other I/O errors raise. + """ + + for file_index, (path, newest) in enumerate(files): + try: + with open(path, "rb") as source: + items: Iterator[Union[_Member, _Garbage, _PlainBlock]] = ( + _GzipMemberReader(source) + if path.endswith(".gz") + else _plain_blocks(source) + ) + for item in items: + yield file_index, path, newest, item + except OSError as error: + if error.errno not in _VANISHED: + raise + yield file_index, path, newest, error + + +def _plain_blocks(source: BinaryIO) -> Iterator[_PlainBlock]: + """A plain JSONL file as blocks of about ``_PLAIN_BLOCK`` bytes of lines. + + Reading stops at the first block that does not end in a newline: its + last line was the end of the file when read. A writer may finish that + line before the next read, which would then start inside it. + """ + + offset = 0 + while True: + lines = source.readlines(_PLAIN_BLOCK) + if not lines: + return + block = b"".join(lines) + if block.endswith(b"\n"): + yield _PlainBlock(offset, offset + len(block), block) + offset += len(block) + continue + # Only the file's last line can be unterminated. One that does not + # parse is still being written; one that does is a row, as the + # replay reader reads it. + payload, dropped = block, 0 + cut = block.rfind(b"\n") + 1 + tail = block[cut:] + if tail.strip() and not _is_json(tail): + payload, dropped = block[:cut], len(tail) + yield _PlainBlock(offset, offset + len(payload), payload, dropped) + return + + +class _Selection: + """Pass 1: the selected rows of every trace file, as columns. + + Row ``i`` is ``arrival[i]``, ``completion[i]``, ``rid[16*i:16*i+16]`` + (BLAKE2b of ``request_id``), line ``line[i]`` of member + ``members[member[i]]``, ``block[i]`` and ``flags[i]``. Rows are in file + order, the order the replay reader sees them, so de-duplication keeps the + same (first) row it does. + """ + + def __init__( + self, start_ms: float, end_ms: float, as_of_ms: float, max_rows: int + ) -> None: + self.start_ms = start_ms + self.end_ms = end_ms + self.as_of_ms = as_of_ms + self.max_rows = max_rows + self._dedup_at = max_rows + max( + _DEDUP_SLACK_MIN, int(max_rows * _DEDUP_SLACK_RATIO) + ) + self.counts = _new_counts() + # (file index, offset, end, is gzip) of each member with a selected + # row; a plain JSONL file is cut into pseudo-members of whole lines. + self.members: list[tuple[int, int, int, bool]] = [] + self.arrival = array("q") + self.completion = array("q") + self.rid = bytearray() + self.member = array("I") + self.line = array("I") + self.block = array("q") + self.flags = array("B") + self.duplicates = 0 + self.valid_members = 0 + # Runs of bad bytes (``_Garbage``), each counted once however many + # members it held, and their bytes (plus plain-JSONL partial tails). + self.dropped_members = 0 + self.dropped_bytes = 0 + # The runs, and their bytes, that count toward MAX_DROPPED_MEMBER_RATIO. + self.corrupt_members = 0 + self.corrupt_bytes = 0 + # Bytes of every gzip member and run of bad bytes. + self.gzip_bytes = 0 + self.block_size: Optional[int] = None + self._drop_examples: list[str] = [] + + @property + def num_rows(self) -> int: + return len(self.arrival) + + def digest(self, row: int) -> bytes: + return bytes(self.rid[16 * row : 16 * row + 16]) + + def scan(self, items: Iterable[_TraceItem]) -> None: + """Consume ``_trace_items`` (in file order).""" + + for file_index, path, newest, item in items: + if isinstance(item, _Member): + self.valid_members += 1 + self.gzip_bytes += item.end - item.offset + self._scan_payload( + item.payload, file_index, path, item.offset, item.end, True + ) + elif isinstance(item, _PlainBlock): + self.dropped_bytes += item.dropped + if item.payload: + self._scan_payload( + item.payload, file_index, path, item.offset, item.end, False + ) + elif isinstance(item, _Garbage): + self._dropped(path, item, newest=newest) + else: + # The reconciler deletes only segments far older than any + # window being cut; a row taken from this one fails pass 2. + logger.warning( + "retune window cut: trace file %s vanished (%s); skipping it", + path, + item, + ) + + def finish(self) -> None: + """De-duplicate, then apply the corruption, size and block-size rules.""" + + self._dedup() + self.counts["duplicate"] = self.duplicates + total = self.valid_members + self.dropped_members + # A run counts as one member: its bytes catch a hole that wiped out + # many members. + if ( + self.corrupt_members > MAX_DROPPED_MEMBER_RATIO * total + or self.corrupt_bytes > MAX_DROPPED_MEMBER_RATIO * self.gzip_bytes + ): + raise WindowCutError( + "trace_corrupt", + f"{self.corrupt_members} of {total} gzip members of the trace " + f"failed validation ({self.corrupt_bytes} of {self.gzip_bytes} " + f"bytes; limit {MAX_DROPPED_MEMBER_RATIO:.0%} of either); " + f"dropped byte runs: {', '.join(self._drop_examples)}", + ) + if self.num_rows > self.max_rows: + raise _too_large(self.max_rows) + if not self.num_rows: + return + blocks = np.unique(np.frombuffer(self.block, dtype=self.block.typecode)) + if len(blocks) > 1: + raise WindowCutError( + "window_cut_failed", + "trace window mixes trace_block_size values " + f"{blocks.tolist()}; replay needs one block size", + ) + self.block_size = int(blocks[0]) + flags: np.ndarray = np.frombuffer(self.flags, dtype=self.flags.typecode) + self.counts["agent_context_stripped"] = int( + np.count_nonzero(flags & _FLAG_AGENT_CONTEXT) + ) + self.counts["censor_unavailable"] = int( + np.count_nonzero(flags & _FLAG_NO_TOTAL_TIME) + ) + + def _dropped(self, path: str, item: _Garbage, *, newest: bool) -> None: + size = item.end - item.offset + self.dropped_members += 1 + self.dropped_bytes += size + self.gzip_bytes += size + if item.at_eof and newest: + # The flush its writer is appending, or a crashed writer's last + # one: at most one batch, written after the cut began. + logger.debug( + "retune window cut: %d trailing bytes of %s hold no complete " + "gzip member (in-flight write)", + size, + path, + ) + return + self.corrupt_members += 1 + self.corrupt_bytes += size + if len(self._drop_examples) < _MAX_DROP_EXAMPLES: + self._drop_examples.append(f"{path}@{item.offset}+{size}") + logger.warning( + "retune window cut: dropped %d bytes at %s offset %d that hold no " + "valid gzip member", + size, + path, + item.offset, + ) + + def _scan_payload( + self, + payload: bytes, + file_index: int, + path: str, + offset: int, + end: int, + is_gzip: bool, + ) -> None: + lines = payload.split(b"\n") + if not lines[-1]: + lines.pop() + member_id = -1 + start_ms, end_ms, as_of_ms = self.start_ms, self.end_ms, self.as_of_ms + counts = self.counts + for index, raw in enumerate(lines): + if not raw.strip(): + continue + location = f"{path}@{offset}:{index + 1}" + event = _event(raw, location, "trace_corrupt") + if event is None: + continue + row = _request_row(event, location, "trace_corrupt") + if row is None: + counts["missing_output_tokens"] += 1 + continue + if not start_ms <= row.arrival_ms < end_ms: + continue + if row.completion_ms > as_of_ms: + counts["censored"] += 1 + continue + if member_id < 0: + member_id = len(self.members) + self.members.append((file_index, offset, end, is_gzip)) + self._append(row, member_id, index, location) + + def _append( + self, row: _RequestRow, member_id: int, index: int, location: str + ) -> None: + try: + self.arrival.append(row.arrival_ms) + self.completion.append(row.completion_ms) + self.block.append(row.trace_block_size) + except OverflowError as error: + raise WindowCutError( + "trace_corrupt", f"trace row value out of range at {location}" + ) from error + self.rid += _digest(row.request_id) + self.member.append(member_id) + self.line.append(index) + self.flags.append( + (_FLAG_AGENT_CONTEXT if row.had_agent_context else 0) + | (_FLAG_NO_TOTAL_TIME if row.total_time_ms is None else 0) + ) + if len(self.arrival) > self._dedup_at: + # Duplicates only shrink the window, and rows already kept stay + # kept: past max_rows after de-duplication, the window is too large. + self._dedup() + if len(self.arrival) > self.max_rows: + raise _too_large(self.max_rows) + + def _dedup(self) -> None: + count = len(self.arrival) + if count < 2: + return + keep = _first_occurrences(self.rid, count) + kept = int(np.count_nonzero(keep)) + if kept == count: + return + self.duplicates += count - kept + self.arrival = _compress(self.arrival, keep) + self.completion = _compress(self.completion, keep) + self.member = _compress(self.member, keep) + self.line = _compress(self.line, keep) + self.block = _compress(self.block, keep) + self.flags = _compress(self.flags, keep) + self.rid = bytearray( + np.frombuffer(self.rid, dtype=np.uint8).reshape(count, 16)[keep].tobytes() + ) + + +def _first_occurrences(rid: bytearray, count: int) -> np.ndarray: + """Mask of the rows whose 128-bit digest has not occurred before them.""" + + words: np.ndarray = np.frombuffer(rid, dtype=np.uint64).reshape(count, 2) + high, low = words[:, 0], words[:, 1] + order = np.lexsort((np.arange(count), low, high)) + high, low = high[order], low[order] + repeat: np.ndarray = np.zeros(count, dtype=bool) + repeat[1:] = (high[1:] == high[:-1]) & (low[1:] == low[:-1]) + keep: np.ndarray = np.ones(count, dtype=bool) + keep[order[repeat]] = False + return keep + + +def _compress(column: array, keep: np.ndarray) -> array: + kept = array(column.typecode) + kept.frombytes(np.frombuffer(column, dtype=column.typecode)[keep].tobytes()) + return kept + + +def _too_large(max_rows: int) -> WindowCutError: + return WindowCutError( + "window_too_large", + f"the window has more than {max_rows} requests (max_window_rows)", + ) + + +def _digest(request_id: str) -> bytes: + # UTF-8 with surrogatepass is injective on str, lone surrogates included. + return hashlib.blake2b( + request_id.encode("utf-8", "surrogatepass"), digest_size=16 + ).digest() + + +def _is_json(raw: bytes) -> bool: + try: + json.loads(raw) + except ValueError: + return False + return True + + +def _event( + raw: bytes, location: str, failure: WindowCutFailure +) -> Optional[dict[str, Any]]: + """The ``request_end`` event of a trace line, as the replay reader reads it.""" + + try: + record = json.loads(raw) + except ValueError as error: # JSONDecodeError, UnicodeDecodeError + raise WindowCutError(failure, f"invalid JSON at {location}: {error}") from error + if not isinstance(record, dict): + raise WindowCutError(failure, f"trace row at {location} must be a JSON object") + try: + return _request_end_event(record, location) + except ValueError as error: + raise WindowCutError(failure, str(error)) from error + + +def _request_row( + event: dict[str, Any], location: str, failure: WindowCutFailure +) -> Optional[_RequestRow]: + try: + return _parse_request_end(event, location, need_hashes=True) + except ValueError as error: + raise WindowCutError(failure, str(error)) from error + + +# ---------------------------------------------------------------- pass 2 + + +def _publish(out_path: str, temp_path: str, write: Callable[[_ByteSink], None]) -> bool: + """Write ``out_path`` atomically through ``temp_path``. + + Returns ``False`` when ``out_path`` already existed: it is kept and the + new file is discarded. ``temp_path`` is removed in every case. + """ + + directory = os.path.dirname(os.path.abspath(out_path)) + os.makedirs(directory, exist_ok=True) + try: + descriptor = os.open( + temp_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), + 0o600, + ) + with open(descriptor, "wb") as raw: + # No name and no mtime in the header: equal cuts are equal bytes. + with gzip.GzipFile( + filename="", + mode="wb", + fileobj=raw, + compresslevel=OUTPUT_COMPRESSLEVEL, + mtime=0, + ) as sink: + write(sink) + raw.flush() + os.fsync(raw.fileno()) + os.chmod(temp_path, 0o644) + try: + os.link(temp_path, out_path) + created = True + except FileExistsError: + created = False + finally: + _unlink_quietly(temp_path) + _fsync_directory(directory) + return created + + +def _fsync_directory(directory: str) -> None: + try: + descriptor = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + except OSError as error: + logger.debug("retune window cut: cannot open %s to fsync: %s", directory, error) + return + try: + os.fsync(descriptor) + except OSError as error: + # Some network filesystems reject fsync on a directory; their LINK + # is synchronous at the server anyway. + logger.debug("retune window cut: fsync of %s failed: %s", directory, error) + finally: + os.close(descriptor) + + +def _unlink_quietly(path: str) -> None: + try: + os.unlink(path) + except FileNotFoundError: + pass + except OSError as error: + logger.warning("retune window cut: cannot remove %s: %s", path, error) + + +def _write_rows(selection: _Selection, paths: Sequence[str], sink: _ByteSink) -> None: + """Pass 2: write the kept rows to ``sink`` in window-file order. + + The order is ``read_trace_window``'s: (arrival, completion, request_id). + The columns order rows by the first two; rows with an equal pair are + collected and ordered by their ``request_id`` before they are written. + """ + + count = selection.num_rows + arrival: np.ndarray = np.frombuffer( + selection.arrival, dtype=selection.arrival.typecode + ) + completion: np.ndarray = np.frombuffer( + selection.completion, dtype=selection.completion.typecode + ) + members: np.ndarray = np.frombuffer( + selection.member, dtype=selection.member.typecode + ) + lines: np.ndarray = np.frombuffer(selection.line, dtype=selection.line.typecode) + order = np.lexsort((completion, arrival)) + position: np.ndarray = np.empty(count, dtype=np.int64) + position[order] = np.arange(count, dtype=np.int64) + # Each member's rows, in the order they are written. + by_member = np.lexsort((position, members)) + bounds = np.asarray( + np.searchsorted(members[by_member], np.arange(len(selection.members) + 1)) + ) + cache = _MemberLines( + paths, + selection.members, + bounds, + position[by_member], + lines[by_member], + capacity=_PASS2_CACHE_BYTES, + ) + del position, by_member + group: list[_RequestRow] = [] + group_key: Optional[tuple[int, int]] = None + writer = _LineWriter(sink) + try: + for first in range(0, count, _PASS2_BLOCK_ROWS): + block = order[first : first + _PASS2_BLOCK_ROWS] + for row, arrival_ms, completion_ms, member_id, line_index in zip( + block.tolist(), + arrival[block].tolist(), + completion[block].tolist(), + members[block].tolist(), + lines[block].tolist(), + ): + if (arrival_ms, completion_ms) != group_key: + _write_group(group, writer) + group = [] + group_key = (arrival_ms, completion_ms) + raw = cache.take(member_id, line_index) + location = cache.location(member_id, line_index) + event = _event(raw, location, "window_cut_failed") + parsed = ( + None + if event is None + else _request_row(event, location, "window_cut_failed") + ) + if ( + parsed is None + or parsed.arrival_ms != arrival_ms + or parsed.completion_ms != completion_ms + or _digest(parsed.request_id) != selection.digest(row) + ): + raise _changed(f"the row at {location} differs") + group.append(parsed) + _write_group(group, writer) + writer.close() + except BaseException: + writer.abort() + raise + finally: + cache.close() + + +def _write_group(group: list[_RequestRow], writer: _LineWriter) -> None: + if len(group) > 1: + group.sort(key=attrgetter("request_id")) + for row in group: + writer.write(_dumps(_window_record(row))) + + +def _changed(detail: str) -> WindowCutError: + return WindowCutError( + "window_cut_failed", f"trace segment changed between passes: {detail}" + ) + + +class _LineWriter: + """Batches window lines and writes them to the gzip sink in a thread. + + zlib and file writes release the GIL, so compressing and writing one + batch overlaps parsing the next; at most ``_WRITE_QUEUE`` batches wait. + ``close`` writes what is left and re-raises the thread's error; + ``abort`` stops it without writing more. + """ + + def __init__(self, sink: _ByteSink) -> None: + self._sink = sink + self._parts: list[str] = [] + self._size = 0 + self._queue: queue.Queue[Optional[bytes]] = queue.Queue(maxsize=_WRITE_QUEUE) + self._error: Optional[BaseException] = None + self._aborted = False + self._stopped = False + self._thread = threading.Thread( + target=self._run, name="retune-window-write", daemon=True + ) + self._thread.start() + + def write(self, line: str) -> None: + self._parts.append(line) + self._parts.append("\n") + self._size += len(line) + 1 + if self._size >= _WRITE_BUFFER: + self._send() + + def close(self) -> None: + self._send() + self._stop() + if self._error is not None: + raise self._error + + def abort(self) -> None: + self._aborted = True + self._stop() + + def _send(self) -> None: + if self._error is not None: + raise self._error + if self._parts: + self._queue.put("".join(self._parts).encode("utf-8")) + self._parts = [] + self._size = 0 + + def _stop(self) -> None: + if not self._stopped: + self._stopped = True + self._queue.put(None) + self._thread.join() + + def _run(self) -> None: + while True: + data = self._queue.get() + if data is None: + return + if self._error is not None or self._aborted: + continue + try: + self._sink.write(data) + except BaseException as error: # noqa: BLE001 - re-raised by close() + self._error = error + + +class _MemberLines: + """The lines pass 2 still has to write, decoded member by member. + + Rows are taken in output order. A member is read and decoded when one of + its rows is first needed and keeps only its lines still to be written; it + is dropped when its last one is taken. Above ``capacity`` bytes, the + members whose next row is written last are evicted (Belady) and decoded + again when needed, so memory stays bounded whatever the request durations. + + ``bounds[m]:bounds[m+1]`` indexes member ``m``'s rows in ``positions`` + (their output positions, ascending) and ``lines`` (their line indices). + """ + + def __init__( + self, + paths: Sequence[str], + members: Sequence[tuple[int, int, int, bool]], + bounds: np.ndarray, + positions: np.ndarray, + lines: np.ndarray, + *, + capacity: int, + ) -> None: + self._paths = paths + self._members = members + self._cursor: np.ndarray = bounds[:-1].astype(np.int64) + self._ends = bounds[1:] + self._positions = positions + self._lines = lines + self._capacity = capacity + self._loaded: dict[int, dict[int, bytes]] = {} + self._sizes: dict[int, int] = {} + self._bytes = 0 + self._files: OrderedDict[int, BinaryIO] = OrderedDict() + self.loads = 0 + + def location(self, member_id: int, line_index: int) -> str: + file_index, offset, _, _ = self._members[member_id] + return f"{self._paths[file_index]}@{offset}:{line_index + 1}" + + def take(self, member_id: int, line_index: int) -> bytes: + lines = self._loaded.get(member_id) + if lines is None: + lines = self._load(member_id) + raw = lines.pop(line_index, None) + if raw is None: + raise WindowCutError( + "window_cut_failed", + f"window cut lost track of the row at " + f"{self.location(member_id, line_index)}", + ) + self._cursor[member_id] += 1 + size = len(raw) + _LINE_OVERHEAD + self._sizes[member_id] -= size + self._bytes -= size + if not lines: + del self._loaded[member_id] + del self._sizes[member_id] + elif self._bytes > self._capacity: + self._evict() + return raw + + def close(self) -> None: + for source in self._files.values(): + source.close() + self._files.clear() + + def _load(self, member_id: int) -> dict[int, bytes]: + file_index, offset, end, is_gzip = self._members[member_id] + data = self._read(file_index, offset, end) + payload = _decode_exact_member(data) if is_gzip else data + if payload is None: + raise _changed( + f"the gzip member at {self._paths[file_index]}@{offset} " + "no longer decodes" + ) + pieces = payload.split(b"\n") + wanted = self._lines[self._cursor[member_id] : self._ends[member_id]].tolist() + if max(wanted) >= len(pieces): + raise _changed( + f"the member at {self._paths[file_index]}@{offset} has fewer lines" + ) + lines = {index: pieces[index] for index in wanted} + size = sum(map(len, lines.values())) + _LINE_OVERHEAD * len(lines) + self._loaded[member_id] = lines + self._sizes[member_id] = size + self._bytes += size + self.loads += 1 + return lines + + def _read(self, file_index: int, offset: int, end: int) -> bytes: + source = self._files.pop(file_index, None) + if source is None: + if len(self._files) >= _OPEN_FILES: + _, oldest = self._files.popitem(last=False) + oldest.close() + source = open(self._paths[file_index], "rb") + self._files[file_index] = source + source.seek(offset) + data = source.read(end - offset) + if len(data) != end - offset: + raise _changed(f"{self._paths[file_index]} is shorter than when scanned") + return data + + def _evict(self) -> None: + target = self._capacity * 3 // 4 + furthest_first = sorted( + self._loaded, + key=lambda member_id: int(self._positions[self._cursor[member_id]]), + reverse=True, + ) + for member_id in furthest_first: + if self._bytes <= target: + return + del self._loaded[member_id] + self._bytes -= self._sizes.pop(member_id) diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 64a619757314..1823d6bfc4ad 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -171,6 +171,17 @@ class SweepRequest: ``retune.domains`` expands ``retune.planner.load_predictor``; its winner's predictor fields are fixed into every non-baseline candidate. ``objective`` is the metric ``score_summary`` scores each replay on. + + The remaining fields are set only by a live controller (``launch_gate`` + set) and keep their defaults in replay, where ``run_sweep`` ignores them: + ``router_domain`` is the controller's router domain (live knob name -> + values), ``deadline_s`` the absolute epoch deadline ``D_k = L_k + + timeout``, ``spec_replicas`` the DGD spec replicas by worker type the + gate reported, ``replicas_verified`` whether the recorded ready counts + cover the whole window (``False`` when the window began before the + controller started), and ``generation`` the launching controller's + generation. Abort events are passed to the sweep function, never stored + here, so the request stays picklable. """ window_index: int @@ -192,6 +203,11 @@ class SweepRequest: max_parallel: int timeout_s: float history_paths: tuple[str, ...] = () + router_domain: dict[str, tuple[float, ...]] | None = None + deadline_s: float | None = None + spec_replicas: dict[str, int] | None = None + replicas_verified: bool = True + generation: int = 0 def __post_init__(self) -> None: keys = [candidate.key() for candidate in self.candidates] @@ -207,6 +223,8 @@ def __post_init__(self) -> None: raise ValueError("max_parallel must be at least 1") if not self.timeout_s > 0: raise ValueError("timeout_s must be positive") + if self.deadline_s is not None and not math.isfinite(self.deadline_s): + raise ValueError("deadline_s must be finite") if self.predictor_candidates: labels = [label for label, _ in self.predictor_candidates] if len(set(labels)) != len(labels): @@ -238,7 +256,11 @@ class SweepResult: prepares it on ``warmup_observations`` as a host does before a swap; ``predictor_warmup_error`` is that preparation's error, if any. ``row_counts`` holds the window reader's counts and, when the history - was read, ``history_requests``. + was read, ``history_requests``. ``failure_reason`` is a stable, + machine-readable reason a live executor sets next to the free-text + ``error`` (``retune.live.contracts.NON_DECISION_FAILURES`` mean the + window was abandoned, not failed); ``details`` carries JSON-shaped + executor facts for the decision record. ``run_sweep`` sets neither. """ status: SweepStatus @@ -251,6 +273,8 @@ class SweepResult: predictor_warmup_error: str | None = None wall_s: float = 0.0 error: str | None = None + failure_reason: str | None = None + details: dict[str, Any] = field(default_factory=dict) def with_predictor_fields( diff --git a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py new file mode 100644 index 000000000000..8d91f87a4945 --- /dev/null +++ b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py @@ -0,0 +1,1238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the retune imports. + +"""Live retune end to end: the real modules, faked only at the edges. + +Real: ``bootstrap.build_supervisor`` (frontend parse, spec premises, the P11 +canary), ``LiveRetuneSupervisor`` with its Lease leadership and state store, +``OrchestratorEngineAdapter`` with its ``RetuneController``, +``dgdr.run_dgdr_sweep``, the window cutter, ``RouterActuator`` and +``Reconciler``. + +Fake: the Kubernetes API (``FakeRetuneKube``, with ``FakeDgdrOperator`` +driving each DGDR v1beta2 Request to a Run and its Candidates), the +frontends' ``/router_policy`` (revision-checked PATCH and reset with the +semantics of ``router_policy.rs`` and ``worker_selection/runtime.rs``), the +trace claim (a temporary directory of real ``jsonl.gz`` segments) and the +clocks. The test advances the clocks in 15 s steps; each step runs every +live planner the way ``NativePlannerBase`` and the supervisor's background +tasks do: lease round, the due planner tick (its decisions go to the +outbox), outbox round, control round (revalidation, canary, actuator, +reconciler). A launched sweep runs on the controller's sweep thread and the +test waits for it before the next step, so every run is deterministic. The +DGDR operator advances one phase per poll of the sweep. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import gzip +import json +import math +import os +import time +import uuid +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Optional + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from dynamo.llm import KvRouterConfig +from dynamo.planner.config.planner_config import PlannerConfig +from dynamo.planner.core.types import RetuneDecision, TickInput, WorkerCounts +from dynamo.planner.plugins.clock import WallClock +from dynamo.planner.plugins.orchestrator.engine_adapter import OrchestratorEngineAdapter +from dynamo.planner.retune.live import bootstrap +from dynamo.planner.retune.live.actuator import RouterActuator +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD_UID, + LABEL_SHARD, + LABEL_WINDOW, + DgdRef, + NullMetrics, + PodInfo, + PolicyCallResult, + RouterInfo, +) +from dynamo.planner.retune.live.reconciler import Reconciler +from dynamo.planner.retune.live.state import ( + RetuneState, + leadership_lease_name, + state_lease_name, +) +from dynamo.planner.retune.live.supervisor import LiveRetuneSupervisor +from dynamo.planner.retune.live.window_cut import cut_window, cut_window_inprocess +from dynamo.planner.tests.live_retune_fakes import ( + FakeDgdrOperator, + FakeRetuneKube, + default_metrics, +) +from dynamo.planner.tests.retune_stubs import caps + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.integration, + pytest.mark.planner, +] + +INTERVAL = 3600.0 +GRACE = 120.0 +TIMEOUT = 3000.0 +SCALE_INTERVAL = 60.0 +# Lease renewal (a third of the lease) and the actuator cadence both fit it. +STEP = 15.0 +LEASE = 60.0 +# Window n is [T0 + (n - 1) * INTERVAL, T0 + n * INTERVAL), launched GRACE +# after it ends. T0 is far ahead of the real clock: files the cutter writes +# (real mtimes) look old to the reconciler, and a real deadline derived from +# the simulated one is always in the future. +T0 = 3_600_000_000.0 +DGD_NAME = "graph" +K8S_NAMESPACE = "retune-e2e" +DGD_UID = "dgd-uid-1" +DYNAMO_NAMESPACE = "retune-e2e-graph" +CLAIM = "request-traces" +MODEL = "Qwen/Qwen3-0.6B" +FRONTENDS = ("graph-frontend-a", "graph-frontend-b") +WORKER_REPLICAS = 2 +REQUESTS_PER_POD = 12 +ROWS_PER_WINDOW = REQUESTS_PER_POD * len(FRONTENDS) +TRACE_SCHEMA = "dynamo.request.trace.v1" +TRACE_FILE_PATH = "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" +CREDIT = "overlap_score_credit" +LOAD = "prefill_load_scale" +TEMP = "router_temperature" +STARTUP = {CREDIT: 1.0, LOAD: 1.0, TEMP: 0.0} +# The frontend runs ``--router-mode kv --router-replica-sync``. +FRONTEND_ROUTER_CONFIG = {"router_replica_sync": True} + + +def end_s(n: int) -> float: + return T0 + n * INTERVAL + + +def end_ms(n: int) -> int: + return int(round(end_s(n) * 1000.0)) + + +def launch_s(n: int) -> float: + return end_s(n) + GRACE + + +def decision_id(n: int) -> str: + return f"retune-w{end_ms(n)}" + + +def by_wt(point: Mapping[str, float]) -> dict[str, dict[str, float]]: + """``policy_parameters`` of the frontend's startup config with ``point``.""" + + config = KvRouterConfig.from_json(json.dumps({**FRONTEND_ROUTER_CONFIG, **point})) + return { + "aggregated": { + key: float(value) + for key, value in dict(config.policy_parameters("aggregated")).items() + } + } + + +def startup_map() -> dict[str, float]: + return by_wt({})["aggregated"] + + +# ------------------------------------------------------------- DGD, config + + +def dgd_object() -> dict[str, Any]: + """A DGD that satisfies every spec premise (P4, P6-P10).""" + + env = [ + { + "name": "DYN_POD_NAME", + "valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}}, + }, + {"name": "DYN_REQUEST_TRACE", "value": "1"}, + {"name": "DYN_REQUEST_TRACE_RECORDS", "value": "request_end"}, + {"name": "DYN_REQUEST_TRACE_SINKS", "value": "file"}, + {"name": "DYN_REQUEST_TRACE_FILE_FORMAT", "value": "jsonl_gz"}, + {"name": "DYN_REQUEST_TRACE_FILE_PATH", "value": TRACE_FILE_PATH}, + ] + frontend = { + "name": "Frontend", + "type": "frontend", + "replicas": len(FRONTENDS), + "podTemplate": { + "spec": { + "containers": [ + { + "name": "main", + "image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.8.0", + "command": ["python3"], + "args": [ + "-m", + "dynamo.frontend", + "--router-mode", + "kv", + "--router-replica-sync", + ], + "env": env, + "volumeMounts": [{"name": "traces", "mountPath": "/trace"}], + } + ], + "volumes": [ + {"name": "traces", "persistentVolumeClaim": {"claimName": CLAIM}} + ], + } + }, + } + worker = { + "name": "worker", + "type": "worker", + "replicas": WORKER_REPLICAS, + "podTemplate": { + "spec": { + "containers": [ + { + "name": "main", + "command": ["python3", "-m", "dynamo.vllm"], + "args": [f"--model={MODEL}"], + } + ] + } + }, + } + planner = { + "name": "Planner", + "type": "planner", + "replicas": 1, + "podTemplate": {"spec": {"containers": [{"name": "main"}]}}, + } + return { + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeployment", + "metadata": {"name": DGD_NAME, "namespace": K8S_NAMESPACE, "uid": DGD_UID}, + "spec": {"components": [frontend, worker, planner]}, + } + + +def planner_config(mount_path: Path, *, advisory: bool) -> PlannerConfig: + return PlannerConfig( + mode="agg", + environment="kubernetes", + advisory=advisory, + enable_load_scaling=False, + enable_throughput_scaling=False, + load_adjustment_interval_seconds=int(SCALE_INTERVAL), + scheduling={"scale_interval_seconds": SCALE_INTERVAL}, + retune={ + "interval_seconds": INTERVAL, + "grace_seconds": GRACE, + "timeout_seconds": TIMEOUT, + "router": { + "overlap_score_credit": {"choices": [0.5, 1.0]}, + "prefill_load_scale": 1.0, + "temperature": {"choices": [0.0, 0.2]}, + }, + "executor": "dgdr", + "dgdr": { + "claim_name": CLAIM, + "mount_path": str(mount_path), + "model": MODEL, + "sku": "h200_sxm", + "engine": { + "mode": "aggregated", + "backend": "vllm", + "workers": { + "aggregated": { + "parallelism": { + "preset": [ + { + "tensor": 1, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, + } + ] + }, + "scheduler": { + "max_batched_tokens": 8192, + "max_sequences": 256, + }, + }, + "prefill": None, + "decode": None, + }, + }, + "run": { + "parallelism": 8, + "candidate_timeout_seconds": 600, + "max_shards": 3, + }, + "cut_budget_seconds": 300, + "job_startup_seconds": 60, + "keep_requests": 1, + "lease_duration_seconds": LEASE, + "actuator_interval_seconds": STEP, + }, + }, + ) + + +# ------------------------------------------------------------------ fakes + + +class SimClock: + """Wall and monotonic time, advanced only by the test.""" + + def __init__(self, wall: float) -> None: + self.wall = wall + self.mono = 1_000.0 + + def time(self) -> float: + return self.wall + + def monotonic(self) -> float: + return self.mono + + def advance(self, seconds: float) -> None: + self.wall += seconds + self.mono += seconds + + +async def inline(call): + return call() + + +async def no_sleep(_seconds: float) -> None: + return None + + +class RecordingMetrics(NullMetrics): + def __init__(self) -> None: + self.events: list[tuple[str, Any]] = [] + + def decision(self, outcome: str) -> None: + self.events.append(("decision", outcome)) + + def failure(self, reason: str) -> None: + self.events.append(("failure", reason)) + + def skipped(self, reason: str) -> None: + self.events.append(("skipped", reason)) + + def patch_error(self, code: str) -> None: + self.events.append(("patch_error", code)) + + def of(self, kind: str) -> list[Any]: + return [value for event, value in self.events if event == kind] + + +@dataclasses.dataclass +class FakeRouter: + """One frontend-local router: startup values, current values, revision.""" + + router_id: str + initial: dict[str, float] + parameters: dict[str, float] + revision: int = 0 + worker_type: str = "aggregated" + + def info(self) -> RouterInfo: + return RouterInfo( + router_id=self.router_id, + namespace=DYNAMO_NAMESPACE, + worker_type=self.worker_type, + revision=self.revision, + configuration=dict(self.parameters), + model=MODEL, + ) + + +@dataclasses.dataclass(frozen=True) +class PolicyWrite: + """One PATCH or reset that reached a frontend, and the persisted retune + state at that moment (to check persist-before-actuate).""" + + writer: str + kind: str + pod: str + router_id: str + expected_revision: int + parameters: Optional[dict[str, Any]] + status: int + state: Optional[RetuneState] + + +class FakeFrontends: + """The DGD's frontend pods and their routers.""" + + def __init__(self, kube: FakeRetuneKube) -> None: + self.kube = kube + self.routers: dict[str, list[FakeRouter]] = {} + self.writes: list[PolicyWrite] = [] + + def start(self, pod: str) -> None: + self.kube.add_pod(pod, dgd=DGD_NAME, component_type="frontend", ready=True) + self.restart(pod) + + def restart(self, pod: str) -> None: + """A new frontend process: a new router id at revision 0 with the + startup values (``RouterPolicyRegistry::register``).""" + + startup = startup_map() + self.routers[pod] = [ + FakeRouter( + router_id=str(uuid.uuid4()), + initial=dict(startup), + parameters=dict(startup), + ) + ] + + def router(self, pod: str) -> FakeRouter: + (router,) = self.routers[pod] + return router + + def router_ids(self) -> set[str]: + return {self.router(pod).router_id for pod in FRONTENDS} + + def persisted_state(self) -> Optional[RetuneState]: + lease = self.kube.leases.get(state_lease_name(DGD_NAME)) + if lease is None: + return None + return RetuneState.from_annotations(lease["metadata"].get("annotations")) + + +class FakeRouterPolicyClient: + """``RouterPolicyClient`` with the ``/router_policy`` semantics of + ``lib/llm/src/http/service/router_policy.rs`` and + ``lib/kv-router/src/plugins/worker_selection/runtime.rs``: an empty patch + is 400, an unknown router 404, a stale ``expected_revision`` 409, an + unknown parameter or an invalid value 422; a PATCH replaces only the + keys it names (``null`` restores that key's startup value), a reset + restores every startup value, and both bump the revision.""" + + def __init__(self, frontends: FakeFrontends, writer: str) -> None: + self.frontends = frontends + self.writer = writer + self.gets = 0 + + async def get(self, pod: PodInfo) -> PolicyCallResult: + self.gets += 1 + routers = self.frontends.routers.get(pod.name) + if routers is None: + return PolicyCallResult(status=0, error="connection refused") + return PolicyCallResult( + status=200, routers=tuple(router.info() for router in routers) + ) + + async def patch( + self, + pod: PodInfo, + router_id: str, + expected_revision: int, + parameters: Mapping[str, float], + ) -> PolicyCallResult: + return self._update( + pod, "patch", router_id, expected_revision, dict(parameters) + ) + + async def reset( + self, pod: PodInfo, router_id: str, expected_revision: int + ) -> PolicyCallResult: + return self._update(pod, "reset", router_id, expected_revision, None) + + def _update( + self, + pod: PodInfo, + kind: str, + router_id: str, + expected_revision: int, + parameters: Optional[dict[str, Any]], + ) -> PolicyCallResult: + status, router, error = self._apply( + pod.name, router_id, expected_revision, parameters + ) + self.frontends.writes.append( + PolicyWrite( + writer=self.writer, + kind=kind, + pod=pod.name, + router_id=router_id, + expected_revision=expected_revision, + parameters=None if parameters is None else dict(parameters), + status=status, + state=self.frontends.persisted_state(), + ) + ) + if status != 200 or router is None: + return PolicyCallResult(status=status, error=error) + return PolicyCallResult(status=200, router=router.info()) + + def _apply( + self, + pod: str, + router_id: str, + expected_revision: int, + parameters: Optional[dict[str, Any]], + ) -> tuple[int, Optional[FakeRouter], Optional[str]]: + if parameters is not None and not parameters: + return 400, None, "provide either nonempty parameters or reset: true" + router = next( + ( + r + for r in self.frontends.routers.get(pod, ()) + if r.router_id == router_id + ), + None, + ) + if router is None: + return 404, None, "router instance not found" + if expected_revision != router.revision: + return 409, None, "policy revision conflict" + if parameters is None: + updated = dict(router.initial) + else: + updated = dict(router.parameters) + for key, value in parameters.items(): + if key not in router.initial: + return 422, None, f"unknown parameter {key!r}" + if value is None: + updated[key] = router.initial[key] + continue + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + ): + return 422, None, f"invalid value {value!r} for {key!r}" + updated[key] = float(value) + router.parameters = updated + router.revision += 1 + return 200, router, None + + +class TraceClaim: + """The RWX trace claim: ``/segments///req.NNNNNN.jsonl.gz``.""" + + def __init__(self, mount: Path) -> None: + self.mount = mount + self.segments = mount / "segments" / DGD_NAME + self.windows = mount / "windows" / DGD_NAME + self._index = 0 + for pod in FRONTENDS: + (self.segments / pod).mkdir(parents=True) + + def write_window(self, n: int, now_s: float) -> None: + """A new segment per frontend with window ``n``'s request_end records, + one gzip member per record as the frontend's gzip sink flushes.""" + + self._index += 1 + start = end_ms(n - 1) + span = end_ms(n) - start + for p, pod in enumerate(FRONTENDS): + path = self.segments / pod / f"req.{self._index:06d}.jsonl.gz" + with open(path, "wb") as sink: + for i in range(REQUESTS_PER_POD): + received = start + (i + 1) * span // (REQUESTS_PER_POD + 1) + p + total = 400 + 25 * i + record = { + "schema": TRACE_SCHEMA, + "event_type": "request_end", + "event_time_unix_ms": received + total, + "request": { + "request_id": f"w{n}-{pod}-{i}", + "model": MODEL, + "request_received_ms": received, + "total_time_ms": float(total), + "output_tokens": 16 + i, + "replay": { + "trace_block_size": 64, + "input_length": 256 + 8 * i, + "input_sequence_hashes": [n, p, i, 11 * i + 3], + }, + }, + } + envelope = {"timestamp": received + total, "event": record} + sink.write(gzip.compress(json.dumps(envelope).encode() + b"\n")) + os.utime(path, (now_s, now_s)) + + def segment_files(self, pod: str) -> list[str]: + return sorted(os.listdir(self.segments / pod)) + + def window_files(self) -> set[int]: + if not self.windows.is_dir(): + return set() + return { + int(name.split(".", 1)[0]) + for name in os.listdir(self.windows) + if name.endswith(".jsonl.gz") + } + + +# ---------------------------------------------------------------- planners + + +class PlannerHost: + """The slice of ``NativePlannerBase`` the supervisor calls back into.""" + + def __init__(self, engine: OrchestratorEngineAdapter, clock: SimClock) -> None: + self._engine = engine + self._config_lock = asyncio.Lock() + self._clock = clock + + async def replace_live_retune(self, inputs, *, generation, last_decided_k): + async with self._config_lock: + self._engine.replace_retune( + inputs, + generation=generation, + now_s=self._clock.time(), + last_decided_k=last_decided_k, + ) + + +class Planner: + """One planner process: its supervisor, engine, actuator and reconciler.""" + + def __init__(self, world: World, name: str, *, advisory: bool) -> None: + self.world = world + self.name = name + self.identity = f"{name}/{uuid.uuid4().hex[:12]}" + self.config = planner_config(world.claim.mount, advisory=advisory) + self.metrics = RecordingMetrics() + self.client = FakeRouterPolicyClient(world.frontends, name) + self.decisions: list[RetuneDecision] = [] + self.router_parameters: Optional[dict[str, dict[str, float]]] = None + self.supervisor: LiveRetuneSupervisor + self.engine: OrchestratorEngineAdapter + self.host: PlannerHost + + async def boot(self) -> None: + """``NativePlannerBase._async_init`` then ``run``'s first tick.""" + + world = self.world + clock = world.clock + retune = self.config.retune + assert retune is not None and retune.dgdr is not None + reconciler = Reconciler( + world.kube, + world.dgd_ref, + retune.dgdr, + retune, + retune.dgdr.mount_path, + self.metrics, + now=clock.time, + ) + contexts = bootstrap.dgdr_context_factory(self.config, cut_window=world.cutter) + + def context_factory(supervisor: LiveRetuneSupervisor) -> Any: + # The executor's clock is the simulated one; each poll of the + # sweep lets the DGDR operator advance one phase. + return dataclasses.replace( + contexts(supervisor), + now=clock.time, + sleep=world.operator_sleep, + poll_interval_s=5.0, + ) + + self.supervisor = await bootstrap.build_supervisor( + self.config, + kube=world.kube, + dgd_name=DGD_NAME, + namespace_fn=lambda: DYNAMO_NAMESPACE, + metrics=self.metrics, + identity=self.identity, + actuator_factory=lambda dgd, maps: RouterActuator( + world.kube, self.client, dgd, maps, self.metrics + ), + reconciler=reconciler, + context_factory=context_factory, + clock=clock.time, + monotonic=clock.monotonic, + sleep=no_sleep, + blocking=inline, + ) + self.engine = OrchestratorEngineAdapter( + self.config, + caps(), + clock=WallClock(), + retune_inputs=self.supervisor.retune_inputs(), + retune_hooks=self.supervisor.hooks(), + ) + self.host = PlannerHost(self.engine, clock) + await self.supervisor.start(self.host, background=False) + self.next_tick = self.engine.initial_tick(clock.time()) + + async def step(self) -> None: + supervisor = self.supervisor + await supervisor.lease_round() + if self.world.clock.time() >= self.next_tick.at_s: + await self.tick() + await supervisor.outbox_round() + await supervisor.control_round() + # The reconciler runs as its own task; finish it to keep each step + # deterministic. + if supervisor.reconcile_task is not None: + await supervisor.reconcile_task + + async def tick(self) -> None: + """``NativePlannerBase._run_one_tick`` without the environment.""" + + counts = WorkerCounts( + ready_num_decode=WORKER_REPLICAS, expected_num_decode=WORKER_REPLICAS + ) + async with self.host._config_lock: + effects = await self.engine.tick( + self.next_tick, + TickInput(now_s=self.world.clock.time(), worker_counts=counts), + ) + self.supervisor.on_effects(effects) + assert effects.next_tick is not None + self.next_tick = effects.next_tick + self.decisions.extend(effects.retune_decisions) + self.router_parameters = effects.router_parameters + # Live, the sweep runs on the controller's thread while ticks go on; + # finishing it here keeps the interleaving deterministic. + controller = self.engine.retune_controller + pending = getattr(controller, "_pending", None) + if pending is not None and not pending.future.done(): + await asyncio.wait_for(asyncio.wrap_future(pending.future), timeout=60) + + async def stop(self) -> None: + """SIGTERM: ``_shutdown_runtime`` stops the engine, then the supervisor.""" + + self.world.planners.remove(self) + await self.engine.shutdown() + await self.supervisor.stop() + + def crash(self) -> None: + """SIGKILL: nothing of this process runs again; its Lease stays.""" + + self.world.planners.remove(self) + self.world.crashed.append(self) + + def decision(self, n: int) -> RetuneDecision: + (found,) = [d for d in self.decisions if d.decision_id == decision_id(n)] + return found + + +class World: + """The cluster, the frontends, the trace claim and the clocks.""" + + def __init__(self, tmp_path: Path, *, spawn_cutter: bool = False) -> None: + # Boot inside window 0: first enable skips it (its trace starts + # mid-way), so window 1 is the first one recorded in full. + self.clock = SimClock(T0 - 600.0) + self.kube = FakeRetuneKube(K8S_NAMESPACE, now=self.clock.time) + self.kube.put_dgd(dgd_object()) + self.dgd_ref = DgdRef(DGD_NAME, K8S_NAMESPACE, DGD_UID, DYNAMO_NAMESPACE) + self.frontends = FakeFrontends(self.kube) + for pod in FRONTENDS: + self.frontends.start(pod) + self.claim = TraceClaim(tmp_path / "claim") + # window end ms -> the router point (live names) the replay favours. + self.preferences: dict[int, Mapping[str, float]] = {} + self.operator = FakeDgdrOperator(self.kube, evaluate=self._evaluate) + self.spawn_cutter = spawn_cutter + self.cuts: list[tuple[int, int, bool]] = [] + self.polls = 0 + self.planners: list[Planner] = [] + self.crashed: list[Planner] = [] + self._next_trace_window = 1 + + # ----------------------------------------------------------- edges + + def prefer(self, n: int, point: Mapping[str, float]) -> None: + self.preferences[end_ms(n)] = dict(point) + + def _evaluate(self, request: dict, point: Mapping[str, float]) -> dict: + """Replay summary of a router point (AISimulate public names): the + window's preferred point scores best; deterministic, so the CRN + probe always matches its grid twin.""" + + window = int(request["metadata"]["labels"][LABEL_WINDOW]) + preferred = self.preferences.get(window, STARTUP) + distance = abs(point["overlap_score_credit"] - preferred[CREDIT]) + abs( + point["temperature"] - preferred[TEMP] + ) + metrics = default_metrics(point) + metrics["goodput_output_throughput_tok_s"] = 1000.0 - 200.0 * distance + return metrics + + def operator_sleep(self, _seconds: float) -> None: + self.polls += 1 + if self.polls > 10_000: + raise AssertionError("the DGDR sweep never finished polling") + self.operator.step() + + def cutter( + self, + segments_root, + start_ms, + end_ms_, + as_of_ms, + out_path, + *, + max_rows, + deadline_s, + cancel=None, + ): + if self.spawn_cutter: + # The child runs on the real clock; keep the remaining budget. + real_deadline = time.time() + (deadline_s - self.clock.time()) + result = cut_window( + segments_root, + start_ms, + end_ms_, + as_of_ms, + out_path, + max_rows=max_rows, + deadline_s=real_deadline, + cancel=cancel, + ) + else: + result = cut_window_inprocess( + segments_root, start_ms, end_ms_, as_of_ms, out_path, max_rows=max_rows + ) + self.cuts.append((int(end_ms_), result.num_requests, result.reused)) + return result + + # ------------------------------------------------------------ driving + + async def boot(self, name: str, *, advisory: bool = False) -> Planner: + planner = Planner(self, name, advisory=advisory) + await planner.boot() + self.planners.append(planner) + return planner + + async def run_until(self, until_s: float) -> None: + while self.clock.wall < until_s: + self.clock.advance(min(STEP, until_s - self.clock.wall)) + while self.clock.wall >= end_s(self._next_trace_window): + self.claim.write_window(self._next_trace_window, self.clock.wall) + self._next_trace_window += 1 + for planner in list(self.planners): + await planner.step() + + async def close(self) -> None: + for planner in list(self.planners): + await planner.stop() + for planner in self.crashed: + await planner.engine.shutdown() + + # ---------------------------------------------------------- inspection + + def state(self) -> RetuneState: + state = self.frontends.persisted_state() + assert state is not None + return state + + def history(self, decision: str) -> dict[str, Any]: + (entry,) = [e for e in self.state().history if e["decision_id"] == decision] + return entry + + def lease_spec(self) -> dict[str, Any]: + return self.kube.leases[leadership_lease_name(DGD_NAME)]["spec"] + + def requests_of(self, n: int) -> dict[str, dict]: + return { + name: obj + for name, obj in self.kube.requests.items() + if obj["metadata"]["labels"][LABEL_WINDOW] == str(end_ms(n)) + } + + def request_windows(self) -> set[int]: + return { + int(obj["metadata"]["labels"][LABEL_WINDOW]) + for obj in self.kube.requests.values() + } + + def created(self, n: Optional[int] = None) -> list[str]: + """Names of the stored (non dry-run) Request creates, in order.""" + + return [ + call.target["metadata"]["name"] + for call in self.kube.calls_of("create_request") + if not call.kwargs["dry_run"] + and ( + n is None + or call.target["metadata"]["labels"][LABEL_WINDOW] == str(end_ms(n)) + ) + ] + + def canaries(self) -> int: + return sum( + 1 for call in self.kube.calls_of("create_request") if call.kwargs["dry_run"] + ) + + def deleted(self) -> set[str]: + return {name for name, _ in self.kube.deleted_requests} + + def configurations(self) -> dict[str, tuple[int, dict[str, float]]]: + return { + pod: ( + self.frontends.router(pod).revision, + self.frontends.router(pod).parameters, + ) + for pod in FRONTENDS + } + + +def converged_to(world: World, point: Mapping[str, float], revision: int) -> None: + expected = by_wt(point)["aggregated"] + assert world.configurations() == {pod: (revision, expected) for pod in FRONTENDS} + + +@pytest.fixture +async def make_world(tmp_path): + worlds: list[World] = [] + + def make(**kwargs: Any) -> World: + world = World(tmp_path, **kwargs) + worlds.append(world) + return world + + yield make + for world in worlds: + await world.close() + + +# ------------------------------------------------------------------ tests + + +@pytest.mark.timeout(180) +async def test_advisory_records_recommendations_and_never_writes(make_world): + world = make_world(spawn_cutter=True) + recommended_1 = {CREDIT: 0.5, LOAD: 1.0, TEMP: 0.0} + recommended_2 = {CREDIT: 0.5, LOAD: 1.0, TEMP: 0.2} + world.prefer(1, recommended_1) + world.prefer(2, recommended_2) + planner = await world.boot("planner-a", advisory=True) + + await world.run_until(launch_s(1) - STEP) + supervisor = planner.supervisor + assert supervisor.is_leader() and supervisor.initialized and supervisor.ready() + state = world.state() + # First enable: an advisory record of the startup point; the window in + # progress at start-up (its trace began mid-way) counts as decided. + assert (state.mode, state.rollback_pending, state.deploy_seq) == ( + "advisory", + False, + 1, + ) + assert state.flat == STARTUP and state.by_wt == by_wt(STARTUP) + assert state.last_window_end_ms == end_ms(0) + assert state.initials == { + router_id: startup_map() for router_id in world.frontends.router_ids() + } + assert planner.decisions == [] and world.created() == [] + assert world.canaries() >= 1 and world.kube.requests == {} + + # Window 1: swept through DGDR, recorded as a recommendation only. + await world.run_until(launch_s(1) + SCALE_INTERVAL + 2 * STEP) + decision = planner.decision(1) + assert decision.outcome == "advisory", decision + assert decision.previous.router == STARTUP + assert decision.chosen.router == recommended_1 + assert decision.num_requests == ROWS_PER_WINDOW + assert decision.failure_reason is None + assert decision.best_score > decision.baseline_score + details = decision.details + assert details["window"]["cut"] is True + assert details["window"]["num_requests"] == ROWS_PER_WINDOW + assert details["window"]["dropped_members"] == 0 + assert details["missing_candidates"] == 0 + assert details["crn_probe"]["relative_error"] == 0.0 + assert set(details["shards"]) == {"grid-0", "probe"} + assert world.cuts == [(end_ms(1), ROWS_PER_WINDOW, False)] + + state = world.state() + assert state.mode == "advisory" and state.flat == STARTUP + assert state.by_wt == by_wt(STARTUP) and state.deploy_seq == 1 + assert state.last_window_end_ms == end_ms(1) + entry = world.history(decision_id(1)) + assert entry["outcome"] == "advisory" + assert entry["flat"] == recommended_1 + assert entry["by_wt"] == by_wt(recommended_1) + assert supervisor.target().mode == "observe" + assert supervisor.last_report.converged + assert planner.router_parameters == by_wt(STARTUP) + + shards = { + obj["metadata"]["labels"][LABEL_SHARD] for obj in world.requests_of(1).values() + } + assert shards == {"grid-0", "probe"} + assert all( + obj["metadata"]["labels"][LABEL_DGD_UID] == DGD_UID + for obj in world.kube.requests.values() + ) + + # Window 2: another recommendation; window 1's Requests are cleaned up + # once window 2's are terminal (keep_requests: 1). + window_1_requests = set(world.requests_of(1)) + await world.run_until(launch_s(2) + SCALE_INTERVAL + 2 * STEP) + decision = planner.decision(2) + assert decision.outcome == "advisory" + assert decision.previous.router == STARTUP + assert decision.chosen.router == recommended_2 + assert world.history(decision_id(2))["by_wt"] == by_wt(recommended_2) + assert world.request_windows() == {end_ms(2)} + assert window_1_requests <= world.deleted() + assert world.claim.window_files() == {end_ms(2)} + + # Advisory never writes a router. + assert world.frontends.writes == [] + converged_to(world, STARTUP, revision=0) + assert planner.metrics.of("decision") == ["advisory", "advisory"] + assert planner.metrics.of("patch_error") == [] + + +@pytest.mark.timeout(180) +async def test_apply_restart_handover_and_advisory_rollback(make_world): + world = make_world() + applied_1 = {CREDIT: 0.5, LOAD: 1.0, TEMP: 0.0} + applied_3 = {CREDIT: 0.5, LOAD: 1.0, TEMP: 0.2} + world.prefer(1, applied_1) + world.prefer(2, applied_1) + world.prefer(3, applied_3) + world.prefer(4, applied_1) + pods = sorted(FRONTENDS) + + # ---------------------------------------- active: apply a better point + p1 = await world.boot("planner-a") + await world.run_until(launch_s(1) - STEP) + state = world.state() + assert (state.mode, state.rollback_pending, state.deploy_seq) == ( + "active", + False, + 1, + ) + assert state.flat == STARTUP and state.last_window_end_ms == end_ms(0) + assert state.initials == { + router_id: startup_map() for router_id in world.frontends.router_ids() + } + assert world.frontends.writes == [] and p1.decisions == [] + + await world.run_until(launch_s(1) + SCALE_INTERVAL + 2 * STEP) + decision = p1.decision(1) + assert decision.outcome == "applied", decision + assert decision.previous.router == STARTUP + assert decision.chosen.router == applied_1 + assert decision.num_requests == ROWS_PER_WINDOW + assert decision.best_score > decision.baseline_score + assert decision.details["replicas"] == {"aggregated": WORKER_REPLICAS} + # The planner started before window 1, so its replicas were observed. + assert decision.details["replicas_verified"] is True + assert decision.details["crn_probe"]["relative_error"] == 0.0 + + state = world.state() + assert (state.mode, state.rollback_pending, state.deploy_seq) == ( + "active", + False, + 2, + ) + assert state.flat == applied_1 and state.by_wt == by_wt(applied_1) + assert state.last_window_end_ms == end_ms(1) + assert world.history(decision_id(1))["by_wt"] == by_wt(applied_1) + + # The routers converge with explicit values, only for the changed key, + # and only after the decision is persisted. + writes = world.frontends.writes + assert sorted( + (w.pod, w.kind, w.expected_revision, w.parameters, w.status) for w in writes + ) == [(pod, "patch", 0, {CREDIT: 0.5}, 200) for pod in pods] + assert all(w.writer == "planner-a" for w in writes) + assert all(w.state.flat == applied_1 and w.state.deploy_seq == 2 for w in writes) + converged_to(world, applied_1, revision=1) + assert p1.supervisor.last_report.converged + assert p1.router_parameters == by_wt(applied_1) + window_1_requests = set(world.requests_of(1)) + assert { + obj["metadata"]["labels"][LABEL_SHARD] for obj in world.requests_of(1).values() + } == {"grid-0", "probe"} + assert sorted(world.created()) == sorted(window_1_requests) + + # A frontend restarts: its new router (revision 0, startup values) is + # latched and re-applied with explicit values; the persisted initials + # follow the new router id (the final rollback depends on them). + restarted = FRONTENDS[1] + gone = world.frontends.router(restarted).router_id + world.frontends.restart(restarted) + writes_before = len(world.frontends.writes) + await world.run_until(launch_s(1) + SCALE_INTERVAL + 4 * STEP) + reapplied = world.frontends.writes[writes_before:] + assert [ + (w.pod, w.kind, w.expected_revision, w.parameters, w.status) for w in reapplied + ] == [(restarted, "patch", 0, {CREDIT: 0.5}, 200)] + converged_to(world, applied_1, revision=1) + state = world.state() + assert gone not in state.initials + assert state.initials == { + router_id: startup_map() for router_id in world.frontends.router_ids() + } + assert state.deploy_seq == 2 + + # -------------------------------- restart: restore, never roll back + restart_at = launch_s(1) + 600.0 + await world.run_until(restart_at) + await p1.stop() + assert not world.lease_spec().get("holderIdentity") # released + writes_before = len(world.frontends.writes) + p2 = await world.boot("planner-b") + await world.run_until(restart_at + 5 * STEP) + assert p2.supervisor.is_leader() and p2.supervisor.initialized + assert world.lease_spec()["holderIdentity"] == p2.identity + assert world.lease_spec()["leaseTransitions"] == 1 + state = world.state() + assert state.leader_term.startswith(f"1/{p2.identity}/") + assert (state.mode, state.rollback_pending, state.deploy_seq) == ( + "active", + False, + 2, + ) + assert state.flat == applied_1 and state.by_wt == by_wt(applied_1) + controller = p2.engine.retune_controller + assert controller is not None and controller.current_point().router == applied_1 + assert p2.router_parameters == by_wt(applied_1) + assert len(world.frontends.writes) == writes_before + converged_to(world, applied_1, revision=1) + + # The decided window is not swept again. + await world.run_until(launch_s(2) - STEP) + assert p2.decisions == [] + assert sorted(world.created()) == sorted(window_1_requests) + assert [cut[0] for cut in world.cuts] == [end_ms(1)] + + # Window 2: the restarted planner keeps the applied point. + await world.run_until(launch_s(2) + SCALE_INTERVAL + 2 * STEP) + kept = p2.decision(2) + assert kept.outcome == "kept", kept + assert kept.previous.router == applied_1 and kept.chosen.router == applied_1 + assert kept.best_score == kept.baseline_score + state = world.state() + assert state.deploy_seq == 2 and state.last_window_end_ms == end_ms(2) + assert len(world.frontends.writes) == writes_before + # Window 1's Requests and window file are gone (keep_requests: 1). + assert world.request_windows() == {end_ms(2)} + assert window_1_requests <= world.deleted() + assert world.claim.window_files() == {end_ms(2)} + window_2_requests = set(world.requests_of(2)) + + # ---------------------------- leadership handover (holder crashes) + p3 = await world.boot("planner-c") + await world.run_until(launch_s(3)) + # The leader launched window 3 (its Requests all succeeded); the + # standby never launched anything. + assert not p3.supervisor.is_leader() and not p3.supervisor.initialized + assert p3.decisions == [] and "not_leader" in p3.metrics.of("skipped") + window_3_requests = set(world.requests_of(3)) + assert window_3_requests and set(world.created(3)) == window_3_requests + assert [d.decision_id for d in p2.decisions] == [decision_id(2)] + transitions = world.lease_spec()["leaseTransitions"] + p2.crash() + await world.run_until(launch_s(3) + LEASE + 2 * SCALE_INTERVAL + 2 * STEP) + assert p3.supervisor.is_leader() and p3.supervisor.initialized + assert world.lease_spec()["holderIdentity"] == p3.identity + assert world.lease_spec()["leaseTransitions"] == transitions + 1 + + decision = p3.decision(3) + assert decision.outcome == "applied", decision + assert decision.previous.router == applied_1 + assert decision.chosen.router == applied_3 + assert decision.num_requests == ROWS_PER_WINDOW + # The new leader adopted the crashed leader's Requests: no new create, + # no second cut. + assert all(shard["adopted"] for shard in decision.details["shards"].values()) + assert decision.details["window"]["cut"] is False + assert set(world.created(3)) == window_3_requests + assert [cut[0] for cut in world.cuts] == [end_ms(1), end_ms(2), end_ms(3)] + + state = world.state() + assert state.leader_term.startswith(f"{transitions + 1}/{p3.identity}/") + assert (state.mode, state.deploy_seq) == ("active", 3) + assert state.flat == applied_3 and state.by_wt == by_wt(applied_3) + new_writes = world.frontends.writes[writes_before:] + assert sorted( + (w.pod, w.kind, w.expected_revision, w.parameters, w.status) for w in new_writes + ) == [(pod, "patch", 1, {TEMP: 0.2}, 200) for pod in pods] + assert all( + w.writer == "planner-c" and w.state.flat == applied_3 for w in new_writes + ) + converged_to(world, applied_3, revision=2) + + # ----------------------- switch to advisory: two-phase rollback + switch_at = launch_s(3) + 600.0 + await world.run_until(switch_at) + await p3.stop() + writes_before = len(world.frontends.writes) + p4 = await world.boot("planner-d", advisory=True) + await world.run_until(switch_at + 2 * STEP) + state = world.state() + # Phase 1: rollback_pending persisted, then the routers are reset. + assert (state.mode, state.rollback_pending, state.deploy_seq) == ( + "advisory", + True, + 4, + ) + assert state.flat == STARTUP and state.by_wt == by_wt(STARTUP) + resets = world.frontends.writes[writes_before:] + assert sorted( + (w.pod, w.kind, w.expected_revision, w.parameters, w.status) for w in resets + ) == [(pod, "reset", 2, None, 200) for pod in pods] + assert all( + w.writer == "planner-d" + and w.state.rollback_pending + and w.state.mode == "advisory" + and w.state.deploy_seq == 4 + for w in resets + ) + converged_to(world, STARTUP, revision=3) + assert p4.supervisor.target().mode == "rollback" + + # Phase 2: converged for a lease duration -> rollback complete. + await world.run_until(switch_at + 2 * STEP + LEASE + 3 * STEP) + state = world.state() + assert (state.mode, state.rollback_pending, state.deploy_seq) == ( + "advisory", + False, + 4, + ) + last = state.history[-1] + assert (last["outcome"], last["deploy_seq"]) == ("rolled_back", 4) + assert p4.supervisor.target().mode == "observe" + assert len(world.frontends.writes) == writes_before + len(pods) + + # Window 4 under advisory: a recommendation, no write. + await world.run_until(launch_s(4) + SCALE_INTERVAL + 2 * STEP) + decision = p4.decision(4) + assert decision.outcome == "advisory", decision + assert decision.previous.router == STARTUP + assert decision.chosen.router == applied_1 + state = world.state() + assert state.flat == STARTUP and state.last_window_end_ms == end_ms(4) + assert len(world.frontends.writes) == writes_before + len(pods) + converged_to(world, STARTUP, revision=3) + + # Clean-up across the four windows: only the newest window's Requests + # and window file remain, and old segments are gone (the newest of each + # pod is never deleted). + assert world.request_windows() == {end_ms(4)} + assert window_1_requests | window_2_requests | window_3_requests <= world.deleted() + assert world.claim.window_files() == {end_ms(4)} + for pod in FRONTENDS: + assert world.claim.segment_files(pod) == [ + "req.000003.jsonl.gz", + "req.000004.jsonl.gz", + ] + for planner in (p1, p2, p3, p4): + assert planner.metrics.of("patch_error") == [] diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py new file mode 100644 index 000000000000..0429ec30f493 --- /dev/null +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -0,0 +1,1098 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""In-memory fakes of the Kubernetes side of live retune. + +``FakeRetuneKube`` implements ``contracts.RetuneKube`` over plain dicts with +the API-server behaviour the live modules depend on: ``resourceVersion`` +compare-and-swap on Lease replace, ``AlreadyExists`` on create, Strict field +validation and silent pruning on Request create, label selectors, cascading +Request deletion, and injectable faults (including writes that time out but +land). ``FakeDgdrOperator`` drives the Requests in a ``FakeRetuneKube`` +through the DGDR v1beta2 lifecycle (Accepted -> Run -> Completed with +Candidates) the way the operator at #17483 does, lowering each Request into +``Run.status.aisimulate.input`` with the same merge-patch rules. + +This is a plain module, not a conftest: tests import what they need. +""" + +from __future__ import annotations + +import copy +import functools +import itertools +import json +import re +import threading +import time +import uuid +from collections import deque +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field, replace +from typing import Any, Literal, Optional + +from dynamo.planner.retune.live.contracts import ( + DGDR_ROUTER_KNOBS, + POD_LABEL_COMPONENT_TYPE, + POD_LABEL_DGD_NAME, + RUN_UID_LABEL, + KubeAlreadyExists, + KubeConflict, + KubeError, + KubeInvalid, + KubeNotFound, + PodInfo, +) + +# Operator constants (deploy/operator/internal/controller at #17483). +REQUEST_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" +CANDIDATE_ID_LABEL = "nvidia.com/dgdr-candidate-id" +TRACE_MOUNT_PATH = "/var/run/aisimulate/trace" +DEFAULT_RUNTIME_IMAGE = "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.8.0" +DEFAULT_DYNAMO_VERSION = "1.8.0" +ROUTER_ADAPTER = "dynamo.router" + +_METRIC_TARGETS = { + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", +} + +KubeMethod = Literal[ + "get_dgd", + "list_pods", + "create_request", + "get_request", + "list_requests", + "delete_request", + "get_run", + "list_candidates", + "get_lease", + "create_lease", + "replace_lease", + "delete_lease", +] + + +# ------------------------------------------------------------------ helpers + + +def merge_patch(target: Any, patch: Any) -> Any: + """RFC 7386 JSON merge patch; returns a new value, never mutates.""" + + if not isinstance(patch, Mapping): + return copy.deepcopy(patch) + result = dict(target) if isinstance(target, Mapping) else {} + for key, value in patch.items(): + if value is None: + result.pop(key, None) + else: + result[key] = merge_patch(result.get(key), value) + return result + + +def nested_get(obj: Any, *path: str, default: Any = None) -> Any: + for key in path: + if not isinstance(obj, Mapping) or key not in obj: + return default + obj = obj[key] + return obj + + +def _nested_set(obj: dict, path: tuple[str, ...], value: Any) -> None: + for key in path[:-1]: + child = obj.get(key) + if not isinstance(child, dict): + child = {} + obj[key] = child + obj = child + obj[path[-1]] = value + + +def _nested_pop(obj: Any, path: tuple[str, ...]) -> bool: + for key in path[:-1]: + if not isinstance(obj, dict) or key not in obj: + return False + obj = obj[key] + if isinstance(obj, dict) and path[-1] in obj: + del obj[path[-1]] + return True + return False + + +def _split(path: str) -> tuple[str, ...]: + return tuple(path.split(".")) + + +def rfc3339(epoch_s: float) -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch_s)) + + +_SELECTOR_TERM = re.compile( + r"^\s*(!?)\s*([A-Za-z0-9._/-]+)\s*(?:(==|=|!=)\s*(\S*))?\s*$" +) + + +def match_labels(labels: Mapping[str, str] | None, selector: str) -> bool: + """Equality-based label selector: ``k=v``, ``k==v``, ``k!=v``, ``k``, ``!k``.""" + + labels = labels or {} + if not selector.strip(): + return True + for term in selector.split(","): + found = _SELECTOR_TERM.match(term) + if found is None: + raise ValueError(f"unsupported label selector term {term!r}") + negate, key, op, value = found.groups() + if op is None: + present = key in labels + if present == bool(negate): + return False + elif negate: + raise ValueError(f"unsupported label selector term {term!r}") + elif op in ("=", "=="): + if labels.get(key) != value: + return False + elif labels.get(key) == value: # != + return False + return True + + +def lower_request_input( + request: Mapping[str, Any], + *, + run_name: str, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + gpus_per_node: int = 8, +) -> dict[str, Any]: + """``Run.status.aisimulate.input`` for a Request, as the operator lowers it. + + Mirrors ``aisimulate.LowerWithUnderlay`` and ``dgdrRunInput`` at #17483: + the public role underlay, the ``dgdr_run`` static underlay, the typed + underlay, the derived defaults, ``inputOverrides`` as the final merge + patch, then the operator-owned ``dgdr_run`` fields and the trace source. + """ + + spec = request.get("spec") or {} + max_candidates = int(nested_get(spec, "recommendation", "maxCandidates", default=1)) + underlay: dict[str, Any] = { + "engine": {"workers": {"aggregated": {}, "prefill": {}, "decode": {}}} + } + underlay = merge_patch( + underlay, {"dgdr_run": {"name": run_name, "max_candidates": max_candidates}} + ) + typed: dict[str, Any] = {} + model = nested_get(spec, "model", "name") + if model is not None: + _nested_set(typed, ("engine", "model"), model) + if spec.get("backends"): + _nested_set(typed, ("engine", "backend", "choices"), list(spec["backends"])) + sku = nested_get(spec, "hardware", "gpu", "sku") + if sku: + _nested_set(typed, ("engine", "hardware"), sku) + budget = nested_get(spec, "hardware", "gpu", "budget") + if budget is not None: + _nested_set( + typed, ("optimization", "constraints", "max_candidate_gpus"), int(budget) + ) + trace = nested_get(spec, "workload", "trace") + trace_source = None + if trace is not None: + path = nested_get(trace, "source", "pvc", "path", default="") + mounted = TRACE_MOUNT_PATH + ( + "/trace.jsonl.gz" if path.endswith(".gz") else "/trace.jsonl" + ) + trace_source = { + "type": "trace", + "format": trace.get("format"), + "paths": [mounted], + } + _nested_set(typed, ("traffic", "source"), dict(trace_source)) + _nested_set(typed, ("traffic", "load", "type"), "trace_timestamps") + speedup = nested_get(trace, "replay", "arrivalSpeedupRatio") + if speedup is not None: + _nested_set(typed, ("traffic", "load", "speedup"), speedup) + objective = spec.get("objective") or {} + if objective.get("metric") is not None: + _nested_set( + typed, ("optimization", "target"), _METRIC_TARGETS[objective["metric"]] + ) + sla = objective.get("sla") or {} + for api_key, input_key in ( + ("ttftMs", "ttft_ms"), + ("itlMs", "itl_ms"), + ("e2eMs", "e2e_ms"), + ): + if sla.get(api_key) is not None: + _nested_set(typed, ("evaluation", "sla", input_key), sla[api_key]) + underlay = merge_patch(underlay, typed) + if "traffic" in underlay: + source_type = nested_get(underlay, "traffic", "source", "type") + if source_type != "trace" and nested_get(underlay, "traffic", "stop") is None: + _nested_set(underlay, ("traffic", "stop", "requests"), 1000.0) + if nested_get(underlay, "optimizer", "max_trials") is None: + _nested_set(underlay, ("optimizer", "max_trials"), 320.0) + if nested_get(underlay, "evaluation", "sla") and ( + nested_get(underlay, "optimization", "strict_sla") is None + ): + _nested_set(underlay, ("optimization", "strict_sla"), True) + overrides = nested_get(spec, "aisimulate", "inputOverrides", default={}) or {} + effective = merge_patch(underlay, overrides) + reserved = { + "name": run_name, + "max_candidates": max_candidates, + "snapshot_dir": "/var/run/aisimulate/output", + "namespace": nested_get(request, "metadata", "namespace"), + "runtime_image": runtime_image, + "runtime_version_override": dynamo_version, + "derive_runtime_images": True, + "num_gpus_per_node": gpus_per_node, + } + for key, value in reserved.items(): + _nested_set(effective, ("dgdr_run", key), value) + if trace_source is not None: + traffic = effective.setdefault("traffic", {}) + traffic.setdefault("source", {}).update(trace_source) + # Round-trip through JSON like the API server does. + return json.loads(json.dumps(effective)) + + +def router_points(effective_input: Mapping[str, Any]) -> list[dict[str, float]]: + """The router points (public knob names) an input's router section spans.""" + + router = effective_input.get("router") or {} + axes: list[tuple[str, list[float]]] = [] + for public in DGDR_ROUTER_KNOBS: + value = router.get(public) + if isinstance(value, Mapping): + axes.append((public, [float(v) for v in value["choices"]])) + elif value is not None: + axes.append((public, [float(value)])) + return [ + dict(zip([name for name, _ in axes], values)) + for values in itertools.product(*(values for _, values in axes)) + ] + + +def engine_parameters(effective_input: Mapping[str, Any]) -> dict[str, Any]: + """The non-router parameters every Candidate of a pinned engine shares.""" + + engine = effective_input.get("engine") or {} + workers = engine.get("workers") or {} + params: dict[str, Any] = { + "backend": engine.get("backend"), + "deployment_mode": "disagg" if engine.get("mode") == "disaggregated" else "agg", + "model": engine.get("model"), + "hardware": engine.get("hardware"), + } + for role, worker in sorted(workers.items()): + if isinstance(worker, Mapping): + preset = nested_get(worker, "parallelism", "preset", default=[{}]) + params[f"{role}_parallelism"] = dict(preset[0]) if preset else {} + return json.loads(json.dumps(params)) + + +def candidate_parameters( + effective_input: Mapping[str, Any], point_public: Mapping[str, float] +) -> dict[str, Any]: + """``Candidate.spec.parameters`` of one evaluated router point.""" + + router = effective_input.get("router") or {} + return { + **engine_parameters(effective_input), + "adapters": { + ROUTER_ADAPTER: { + "policy": router.get("policy", "kv_router"), + "prefill_load_model": dict( + router.get("prefill_load_model") or {"type": "none"} + ), + **{name: float(value) for name, value in point_public.items()}, + } + }, + } + + +def default_metrics(point_public: Mapping[str, float]) -> dict[str, float]: + """Deterministic replay summary metrics of a router point (for tests).""" + + credit = point_public.get("overlap_score_credit", 1.0) + scale = point_public.get("prefill_load_scale", 1.0) + temperature = point_public.get("temperature", 0.0) + goodput = 1000.0 + 100.0 * credit - 3.0 * abs(scale - 2.0) - 10.0 * temperature + return { + "goodput_output_throughput_tok_s": goodput, + "output_throughput_tok_s": goodput * 1.1, + "gpu_hours": 2.0, + "duration_ms": 3_600_000.0, + "mean_ttft_ms": 100.0 + scale, + "num_ttft_samples": 1000.0, + "mean_e2e_latency_ms": 1000.0 + scale, + "num_e2e_latency_samples": 1000.0, + "mean_output_token_throughput_per_user": 50.0 + credit, + } + + +# ------------------------------------------------------------------- faults + + +@dataclass +class Fault: + """One injected failure of a ``FakeRetuneKube`` method. + + ``landed=True`` performs the call and then raises (a write that timed + out but reached the API server). ``match`` limits the fault to calls + whose first positional argument (a name, selector or body) satisfies it. + """ + + error: Exception + times: int = 1 + landed: bool = False + match: Optional[Callable[[Any], bool]] = None + + +@dataclass +class KubeCall: + method: str + target: Any + kwargs: dict[str, Any] = field(default_factory=dict) + + +# --------------------------------------------------------------------- kube + + +class FakeRetuneKube: + """In-memory ``contracts.RetuneKube`` (thread-safe). + + Stores DGDs, pods (``PodInfo``), Requests, Runs, Candidates and Leases + as plain dicts. Objects come back as deep copies, so callers can never + alias the store. ``now`` stamps ``creationTimestamp``. + + Request create honours ``field_validation='Strict'``: a path listed in + ``unknown_fields`` (dotted, e.g. ``spec.workload.trace``) is rejected + with ``KubeInvalid``; a path in ``pruned_fields`` is dropped silently + (an API server that ignores the validation directive). ``reject_create`` + rejects every create with ``KubeInvalid``. ``on_request_created`` hooks + run after each stored (non dry-run) create. + """ + + def __init__( + self, + namespace: str = "test-ns", + *, + now: Callable[[], float] = time.time, + ) -> None: + self.namespace = namespace + self.now = now + self.dgds: dict[str, dict] = {} + self.pods: dict[str, PodInfo] = {} + self.requests: dict[str, dict] = {} + self.runs: dict[str, dict] = {} + # Each Run's lowered input, recorded in its status by ``create_job``. + self.run_inputs: dict[str, dict] = {} + self.candidates: dict[str, dict] = {} + self.leases: dict[str, dict] = {} + self.calls: list[KubeCall] = [] + self.deleted_requests: list[tuple[str, str]] = [] + self.unknown_fields: set[str] = set() + self.pruned_fields: set[str] = set() + self.reject_create: Optional[str] = None + self.on_request_created: list[Callable[[FakeRetuneKube, dict], None]] = [] + self.cascade_deletes = True + self._faults: dict[str, deque[Fault]] = {} + self._rv = itertools.count(1) + self._lock = threading.RLock() + + # ------------------------------------------------------------ faults + + def fail( + self, + method: KubeMethod, + error: Exception | None = None, + *, + times: int = 1, + landed: bool = False, + match: Optional[Callable[[Any], bool]] = None, + ) -> None: + """Make the next ``times`` matching calls of ``method`` raise ``error``.""" + + with self._lock: + self._faults.setdefault(method, deque()).append( + Fault( + error=error or KubeError(f"injected {method} failure"), + times=times, + landed=landed, + match=match, + ) + ) + + def clear_faults(self) -> None: + with self._lock: + self._faults.clear() + + def calls_of(self, method: str) -> list[KubeCall]: + with self._lock: + return [call for call in self.calls if call.method == method] + + def _call( + self, + method: str, + target: Any, + kwargs: dict[str, Any], + operation: Callable[[], Any], + ) -> Any: + with self._lock: + self.calls.append(KubeCall(method, copy.deepcopy(target), dict(kwargs))) + fault = self._take_fault(method, target) + if fault is not None and not fault.landed: + raise fault.error + result = operation() + if fault is not None: + raise fault.error + return result + + def _take_fault(self, method: str, target: Any) -> Optional[Fault]: + queue = self._faults.get(method) + if not queue: + return None + for fault in list(queue): + if fault.match is None or fault.match(target): + fault.times -= 1 + if fault.times <= 0: + queue.remove(fault) + return fault + return None + + def _stamp(self, obj: dict, *, kind: str, api_version: str) -> dict: + metadata = obj.setdefault("metadata", {}) + obj.setdefault("apiVersion", api_version) + obj.setdefault("kind", kind) + metadata.setdefault("namespace", self.namespace) + metadata.setdefault("uid", str(uuid.uuid4())) + metadata.setdefault("creationTimestamp", rfc3339(self.now())) + metadata.setdefault("generation", 1) + metadata["resourceVersion"] = str(next(self._rv)) + return obj + + def bump(self, obj: dict) -> None: + """Give a stored object a new ``resourceVersion``.""" + + obj["metadata"]["resourceVersion"] = str(next(self._rv)) + + # --------------------------------------------------------------- DGD + + def put_dgd(self, dgd: dict) -> dict: + with self._lock: + stored = copy.deepcopy(dgd) + self._stamp( + stored, kind="DynamoGraphDeployment", api_version="nvidia.com/v1beta1" + ) + self.dgds[stored["metadata"]["name"]] = stored + return copy.deepcopy(stored) + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + if name not in self.dgds: + raise KubeNotFound(f"DynamoGraphDeployment {name!r} not found") + return copy.deepcopy(self.dgds[name]) + + return self._call("get_dgd", name, {"timeout_s": timeout_s}, operation) + + # -------------------------------------------------------------- pods + + def add_pod( + self, + name: str, + *, + dgd: str | None = None, + component_type: str = "frontend", + ready: bool = True, + terminating: bool = False, + ip: str | None = None, + http_port: int | None = 8000, + labels: Mapping[str, str] | None = None, + ) -> PodInfo: + all_labels = dict(labels or {}) + if dgd is not None: + all_labels.setdefault(POD_LABEL_DGD_NAME, dgd) + if component_type: + all_labels.setdefault(POD_LABEL_COMPONENT_TYPE, component_type) + pod = PodInfo( + name=name, + uid=str(uuid.uuid4()), + ip=ip if ip is not None else f"10.0.0.{len(self.pods) + 1}", + ready=ready, + terminating=terminating, + labels=all_labels, + http_port=http_port, + ) + with self._lock: + self.pods[name] = pod + return pod + + def update_pod(self, name: str, **changes: Any) -> PodInfo: + with self._lock: + pod = replace(self.pods[name], **changes) + self.pods[name] = pod + return pod + + def remove_pod(self, name: str) -> None: + with self._lock: + self.pods.pop(name, None) + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + def operation() -> list[PodInfo]: + return [ + pod + for pod in self.pods.values() + if match_labels(pod.labels, label_selector) + ] + + return self._call( + "list_pods", label_selector, {"timeout_s": timeout_s}, operation + ) + + # ----------------------------------------------------------- Requests + + def create_request( + self, + body: dict, + *, + dry_run: bool = False, + timeout_s: float = 10.0, + ) -> dict: + def operation() -> dict: + if self.reject_create is not None: + raise KubeInvalid(self.reject_create) + stored = copy.deepcopy(body) + for path in sorted(self.unknown_fields): + if nested_get(stored, *_split(path)) is not None: + raise KubeInvalid(f'strict decoding error: unknown field "{path}"') + for path in self.pruned_fields: + _nested_pop(stored, _split(path)) + name = nested_get(stored, "metadata", "name") + if not name: + raise KubeInvalid("metadata.name is required") + if name in self.requests: + raise KubeAlreadyExists( + f"dynamographdeploymentrequests {name!r} already exists" + ) + self._stamp( + stored, + kind="DynamoGraphDeploymentRequest", + api_version="nvidia.com/v1beta2", + ) + if dry_run: + return stored + self.requests[name] = stored + for hook in list(self.on_request_created): + hook(self, copy.deepcopy(stored)) + return copy.deepcopy(stored) + + return self._call( + "create_request", + body, + {"dry_run": dry_run, "timeout_s": timeout_s}, + operation, + ) + + def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.requests.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_request", name, {"timeout_s": timeout_s}, operation) + + def list_requests( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + def operation() -> list[dict]: + return [ + copy.deepcopy(obj) + for obj in self.requests.values() + if match_labels(obj["metadata"].get("labels"), label_selector) + ] + + return self._call( + "list_requests", label_selector, {"timeout_s": timeout_s}, operation + ) + + def delete_request( + self, + name: str, + *, + propagation: Literal["Background", "Foreground"] = "Background", + timeout_s: float = 10.0, + ) -> None: + def operation() -> None: + request = self.requests.pop(name, None) + if request is None: + return + self.deleted_requests.append((name, propagation)) + if not self.cascade_deletes: + return + request_uid = request["metadata"]["uid"] + for run_name, run in list(self.runs.items()): + owners = run["metadata"].get("ownerReferences") or [] + if any(owner.get("uid") == request_uid for owner in owners): + self._delete_run(run_name) + + return self._call( + "delete_request", + name, + {"propagation": propagation, "timeout_s": timeout_s}, + operation, + ) + + # --------------------------------------------------- Runs, Candidates + + def _delete_run(self, run_name: str) -> None: + run = self.runs.pop(run_name, None) + self.run_inputs.pop(run_name, None) + if run is None: + return + run_uid = run["metadata"]["uid"] + for candidate_name, candidate in list(self.candidates.items()): + labels = candidate["metadata"].get("labels") or {} + if labels.get(RUN_UID_LABEL) == run_uid: + del self.candidates[candidate_name] + + def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.runs.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_run", name, {"timeout_s": timeout_s}, operation) + + def list_candidates( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + def operation() -> list[dict]: + return [ + copy.deepcopy(obj) + for obj in self.candidates.values() + if match_labels(obj["metadata"].get("labels"), label_selector) + ] + + return self._call( + "list_candidates", label_selector, {"timeout_s": timeout_s}, operation + ) + + # ------------------------------------------------------------- Leases + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.leases.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_lease", name, {"timeout_s": timeout_s}, operation) + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + stored = copy.deepcopy(body) + name = nested_get(stored, "metadata", "name") + if not name: + raise KubeInvalid("metadata.name is required") + if name in self.leases: + raise KubeAlreadyExists(f"leases {name!r} already exists") + stored["metadata"].pop("resourceVersion", None) + self._stamp(stored, kind="Lease", api_version="coordination.k8s.io/v1") + self.leases[name] = stored + return copy.deepcopy(stored) + + return self._call("create_lease", body, {"timeout_s": timeout_s}, operation) + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + current = self.leases.get(name) + if current is None: + raise KubeNotFound(f"leases {name!r} not found") + sent = nested_get(body, "metadata", "resourceVersion") + if sent != current["metadata"]["resourceVersion"]: + raise KubeConflict( + f"leases {name!r}: resourceVersion {sent!r} is stale " + f"(current {current['metadata']['resourceVersion']!r})" + ) + stored = copy.deepcopy(body) + metadata = stored.setdefault("metadata", {}) + for key in ("uid", "creationTimestamp", "namespace", "generation"): + metadata[key] = current["metadata"].get(key) + metadata["name"] = name + stored.setdefault("apiVersion", current.get("apiVersion")) + stored.setdefault("kind", current.get("kind")) + self.bump(stored) + self.leases[name] = stored + return copy.deepcopy(stored) + + return self._call( + "replace_lease", (name, body), {"timeout_s": timeout_s}, operation + ) + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + def operation() -> None: + self.leases.pop(name, None) + + return self._call("delete_lease", name, {"timeout_s": timeout_s}, operation) + + # ------------------------------------------------- operator simulation + + def set_request_condition( + self, + name: str, + *, + status: str, + reason: str, + message: str = "", + observed_generation: int | None = None, + ) -> None: + """Set the Request's ``Accepted`` condition.""" + + with self._lock: + request = self.requests[name] + generation = request["metadata"].get("generation", 1) + conditions = [ + condition + for condition in request.setdefault("status", {}).get("conditions", []) + if condition.get("type") != "Accepted" + ] + conditions.append( + { + "type": "Accepted", + "status": status, + "reason": reason, + "message": message, + "observedGeneration": ( + generation + if observed_generation is None + else observed_generation + ), + "lastTransitionTime": rfc3339(self.now()), + } + ) + request["status"]["conditions"] = conditions + self.bump(request) + + def accept_request( + self, + name: str, + *, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + mutate_input: Optional[Callable[[dict], None]] = None, + ) -> dict: + """Create the Request's Run (``-``) and mark it Accepted. + + As with the operator, the new Run has no status yet: ``create_job`` + later records its lowered input (kept in ``run_inputs`` until then) + together with ``jobRef``. + """ + + with self._lock: + request = self.requests[name] + generation = int(request["metadata"].get("generation", 1)) + run_name = f"{name}-{generation}" + effective = lower_request_input( + request, + run_name=run_name, + runtime_image=runtime_image, + dynamo_version=dynamo_version, + ) + if mutate_input is not None: + mutate_input(effective) + run: dict[str, Any] = { + "metadata": { + "name": run_name, + "annotations": {REQUEST_GENERATION_ANNOTATION: str(generation)}, + "ownerReferences": [ + { + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRequest", + "name": name, + "uid": request["metadata"]["uid"], + "controller": True, + "blockOwnerDeletion": True, + } + ], + }, + "spec": copy.deepcopy( + {k: v for k, v in request.get("spec", {}).items() if k != "v1beta1"} + ), + "status": {}, + } + self._stamp( + run, kind="DynamoGraphDeploymentRun", api_version="nvidia.com/v1beta2" + ) + self.runs[run_name] = run + self.run_inputs[run_name] = effective + status = request.setdefault("status", {}) + status["observedGeneration"] = generation + status["activeRunRef"] = {"name": run_name} + self.set_request_condition( + name, + status="True", + reason="RunCreated", + message=f"Run {run_name!r} exists for generation {generation}", + ) + return copy.deepcopy(run) + + def create_job(self, run_name: str, *, record_input: bool = True) -> None: + """Create the Run's Job, as the operator's ``reconcileRunStatus`` does. + + One status write records ``status.aisimulate.input`` and ``jobRef`` + with ``Completed=False`` (``JobPending``). ``record_input=False`` + leaves the input out (an operator that does not record it). + """ + + with self._lock: + run = self.runs[run_name] + status = run.setdefault("status", {}) + if record_input: + status["aisimulate"] = { + "input": copy.deepcopy(self.run_inputs[run_name]) + } + status["jobRef"] = {"name": f"{run_name}-job", "uid": str(uuid.uuid4())} + status["phase"] = "Pending" + self._set_completed( + run, "False", "JobPending", "AISimulate Job is waiting to start" + ) + + def start_run(self, run_name: str, *, record_input: bool = True) -> None: + """Start the Run's Job (creating it first, see ``create_job``).""" + + with self._lock: + run = self.runs[run_name] + if not run["status"].get("jobRef"): + self.create_job(run_name, record_input=record_input) + status = run["status"] + status["startTime"] = rfc3339(self.now()) + status["phase"] = "Running" + self._set_completed(run, "False", "JobRunning", "AISimulate Job is running") + + def add_candidate( + self, + run_name: str, + parameters: Mapping[str, Any], + metrics: Mapping[str, Any], + ) -> dict: + with self._lock: + run = self.runs[run_name] + canonical = json.dumps(parameters, sort_keys=True, separators=(",", ":")) + candidate_id = ( + "evaluated-point-" + uuid.uuid5(uuid.NAMESPACE_OID, canonical).hex[:12] + ) + candidate = { + "metadata": { + "name": f"{run_name}-{candidate_id}", + "labels": { + RUN_UID_LABEL: run["metadata"]["uid"], + CANDIDATE_ID_LABEL: candidate_id, + }, + "ownerReferences": [ + { + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRun", + "name": run_name, + "uid": run["metadata"]["uid"], + "controller": True, + } + ], + }, + "spec": {"parameters": json.loads(canonical)}, + "status": {"metrics": json.loads(json.dumps(dict(metrics)))}, + } + self._stamp( + candidate, + kind="DynamoGraphDeploymentCandidate", + api_version="nvidia.com/v1beta2", + ) + self.candidates[candidate["metadata"]["name"]] = candidate + return copy.deepcopy(candidate) + + def complete_run( + self, + run_name: str, + *, + reason: str = "SearchSucceeded", + message: str = "", + candidates: Iterable[tuple[Mapping[str, Any], Mapping[str, Any]]] = (), + ) -> None: + """Publish ``candidates`` (parameters, metrics) and complete the Run.""" + + with self._lock: + run = self.runs[run_name] + refs = [ + { + "name": self.add_candidate(run_name, params, metrics)["metadata"][ + "name" + ] + } + for params, metrics in candidates + ] + status = run["status"] + status["candidateRefs"] = refs + status["phase"] = "Succeeded" if reason == "SearchSucceeded" else "Failed" + status["completionTime"] = rfc3339(self.now()) + self._set_completed(run, "True", reason, message) + owner = (run["metadata"].get("ownerReferences") or [{}])[0] + request = self.requests.get(owner.get("name", "")) + if request is not None: + request_status = request.setdefault("status", {}) + active = request_status.get("activeRunRef") or {} + if active.get("name") == run_name: + request_status.pop("activeRunRef", None) + recent = [ + ref + for ref in request_status.get("recentRunRefs", []) + if ref.get("name") != run_name + ] + request_status["recentRunRefs"] = [{"name": run_name}, *recent] + self.bump(request) + + def _set_completed(self, run: dict, status: str, reason: str, message: str) -> None: + conditions = [ + condition + for condition in run["status"].get("conditions", []) + if condition.get("type") != "Completed" + ] + conditions.append( + { + "type": "Completed", + "status": status, + "reason": reason, + "message": message, + "lastTransitionTime": rfc3339(self.now()), + } + ) + run["status"]["conditions"] = conditions + self.bump(run) + + def run_of(self, request_name: str) -> Optional[dict]: + """The stored Run a Request owns, if any (the newest).""" + + with self._lock: + request = self.requests.get(request_name) + if request is None: + return None + owned = [ + run + for run in self.runs.values() + if any( + owner.get("uid") == request["metadata"]["uid"] + for owner in run["metadata"].get("ownerReferences") or [] + ) + ] + return copy.deepcopy(owned[-1]) if owned else None + + +Evaluator = Callable[[dict, Mapping[str, float]], Optional[Mapping[str, Any]]] + + +class FakeDgdrOperator: + """Drives every Request of a ``FakeRetuneKube`` one phase per ``step()``. + + Phases: accept (a Run with no status, as before its Job exists), start + (the lowered input, jobRef and startTime; the operator records the + input together with jobRef), complete (one Candidate per router point + the input spans, metrics from ``evaluate(request, point_public)``; + ``None`` leaves the point unpublished, as AISimulate drops an + infeasible point). ``hold`` keeps + named Requests in their current phase; ``fail_runs`` maps a Request + name to the ``Completed`` reason its Run ends with. ``mutate_input`` + edits a Run's lowered input and ``mutate_parameters`` a Candidate's + parameters (both get the Request name), to simulate version skew. + ``max_phase`` stops every Request at ``accepted`` or ``started``. + """ + + def __init__( + self, + kube: FakeRetuneKube, + *, + evaluate: Optional[Evaluator] = None, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + mutate_input: Optional[Callable[[str, dict], None]] = None, + mutate_parameters: Optional[Callable[[str, dict], None]] = None, + ) -> None: + self.kube = kube + self.evaluate: Evaluator = evaluate or ( + lambda _request, point: default_metrics(point) + ) + self.runtime_image = runtime_image + self.dynamo_version = dynamo_version + self.mutate_input = mutate_input + self.mutate_parameters = mutate_parameters + self.max_phase: Optional[Literal["accepted", "started"]] = None + self.hold: set[str] = set() + self.fail_runs: dict[str, str] = {} + self.steps = 0 + + def step(self) -> int: + """Advance every Request one phase; returns how many moved.""" + + moved = 0 + with self.kube._lock: + self.steps += 1 + for name in list(self.kube.requests): + if name in self.hold: + continue + run = self.kube.run_of(name) + if run is None: + hook = self.mutate_input + self.kube.accept_request( + name, + runtime_image=self.runtime_image, + dynamo_version=self.dynamo_version, + mutate_input=( + None if hook is None else functools.partial(hook, name) + ), + ) + moved += 1 + continue + run_name = run["metadata"]["name"] + completed = next( + ( + condition + for condition in run["status"].get("conditions", []) + if condition.get("type") == "Completed" + ), + None, + ) + if completed is not None and completed.get("status") == "True": + continue + if self.max_phase == "accepted": + continue + if not run["status"].get("startTime"): + self.kube.start_run(run_name) + moved += 1 + continue + if self.max_phase == "started": + continue + reason = self.fail_runs.get(name) + if reason is not None: + self.kube.complete_run(run_name, reason=reason, message="injected") + moved += 1 + continue + effective = run["status"]["aisimulate"]["input"] + request = self.kube.requests[name] + published = [] + for point in router_points(effective): + metrics = self.evaluate(copy.deepcopy(request), point) + if metrics is not None: + parameters = candidate_parameters(effective, point) + if self.mutate_parameters is not None: + self.mutate_parameters(name, parameters) + published.append((parameters, metrics)) + self.kube.complete_run(run_name, candidates=published) + moved += 1 + return moved + + def run_to_completion(self, max_steps: int = 10) -> None: + for _ in range(max_steps): + if not self.step(): + return diff --git a/components/src/dynamo/planner/tests/retune_stubs.py b/components/src/dynamo/planner/tests/retune_stubs.py index aedfa4518e78..c81dc5afdd7e 100644 --- a/components/src/dynamo/planner/tests/retune_stubs.py +++ b/components/src/dynamo/planner/tests/retune_stubs.py @@ -50,21 +50,54 @@ class StubController: """Records the host's calls; returns one queued tick result per tick.""" instances: list[StubController] = [] - - def __init__(self, config, inputs, capabilities, clock, scale_interval_s): + # Raised by the next construction, then cleared (live rebuild tests). + construct_error: Exception | None = None + + def __init__(self, config, inputs, capabilities, clock, scale_interval_s, **hooks): + error = StubController.construct_error + if error is not None: + StubController.construct_error = None + raise error self.args = (config, inputs, capabilities, clock, scale_interval_s) + # Live hooks (sweep_fn, launch_gate, harvest_allowed, generation); + # empty when the host built the controller the replay way. + self.hooks = hooks self.config = config self.started: list[float] = [] + # phase_s of each start(); None when the host passed none. + self.phases: list[float | None] = [] self.ticks: list[tuple[float, Any]] = [] # Searched planner values as the controller saw them on each tick. self.seen_planner_values: list[dict[str, Any]] = [] self.results: list[StubTickResult] = [] self.closed = 0 self.close_error: Exception | None = None + self.dropped = 0 + self.rewinds: list[int] = [] + # Ordered lifecycle calls: "start", "rewind", "drop_pending", "close". + self.calls: list[str] = [] StubController.instances.append(self) - def start(self, start_s: float) -> None: + @property + def generation(self) -> int: + return self.hooks.get("generation", 0) + + def start(self, start_s: float, phase_s: float | None = None) -> None: self.started.append(start_s) + self.phases.append(phase_s) + self.calls.append("start") + + def drop_pending(self) -> None: + self.dropped += 1 + self.calls.append("drop_pending") + + def rewind(self, last_decided_k: int) -> None: + self.rewinds.append(last_decided_k) + self.calls.append("rewind") + + def running_future(self): + """No sweep thread: nothing for a successor to queue behind.""" + return None async def on_tick(self, now_s, worker_counts): self.ticks.append((now_s, worker_counts)) @@ -78,6 +111,7 @@ async def on_tick(self, now_s, worker_counts): def close(self) -> None: self.closed += 1 + self.calls.append("close") if self.close_error is not None: raise self.close_error @@ -89,6 +123,7 @@ def install_stub_controller(monkeypatch) -> type[StubController]: module.RetuneController = StubController # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, CONTROLLER_MODULE, module) StubController.instances = [] + StubController.construct_error = None return StubController diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py b/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py new file mode 100644 index 000000000000..5f9311ae9c15 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py @@ -0,0 +1,1356 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the live retune router actuator and its HTTP client. + +``FakePolicyClient`` implements the ``/router_policy`` semantics of +``lib/kv-router/src/plugins/worker_selection/runtime.rs`` and +``lib/llm/src/http/service/router_policy.rs``: CAS on ``expected_revision``, +per-key PATCH merge, ``reset`` restores the startup values, and every +successful write bumps the revision. +""" + +from __future__ import annotations + +import asyncio +import json +import re +import socket +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +import pytest +from aiohttp import test_utils, web + +from dynamo.planner.retune.live.actuator import ( + DEFAULT_CLIENT_TIMEOUT_S, + ROUTER_CONTROL_TIMEOUT_S, + AiohttpRouterPolicyClient, + RouterActuator, +) +from dynamo.planner.retune.live.contracts import ( + POD_LABEL_COMPONENT_TYPE, + POD_LABEL_DGD_NAME, + POLICY_PARAMETER_KEYS, + ActuationTarget, + DgdRef, + KubeError, + PodInfo, + PolicyCallResult, + RouterInfo, +) + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +DGD = DgdRef(name="dgd", namespace="ns", uid="dgd-uid", dynamo_namespace="dyn-ns") + +# KvRouterConfig defaults resolved by policy_parameters("aggregated"). +STARTUP = { + "decode_active_request_weight": 0.0, + "disk_cache_hit_weight": 0.25, + "host_cache_hit_weight": 0.75, + "overlap_score_credit": 1.0, + "overlap_score_credit_decay": 0.0, + "prefill_load_scale": 1.0, + "router_temperature": 0.0, + "shared_cache_multiplier": 0.0, +} +# overlap_score_credit = 0 also zeroes the inherited cache weights. +ZERO_CREDIT = { + **STARTUP, + "overlap_score_credit": 0.0, + "disk_cache_hit_weight": 0.0, + "host_cache_hit_weight": 0.0, +} +CHOSEN = { + **STARTUP, + "overlap_score_credit": 0.5, + "prefill_load_scale": 2.0, + "router_temperature": 0.2, +} +# policy_parameters("decode") of the defaults: plain decode has no cache input. +DECODE_STARTUP = { + **STARTUP, + "overlap_score_credit": 0.0, + "disk_cache_hit_weight": 0.0, + "host_cache_hit_weight": 0.0, +} +_CACHE_KEYS = ( + "overlap_score_credit", + "host_cache_hit_weight", + "disk_cache_hit_weight", + "shared_cache_multiplier", +) + + +# ------------------------------------------------------------------ fakes + + +@dataclass +class FakeRouter: + router_id: str + initial: dict[str, float] + worker_type: str = "aggregated" + namespace: str = DGD.dynamo_namespace + revision: int = 0 + current: Optional[dict[str, float]] = None + runtime_config: bool = True # False: a custom policy, configuration null + plain_decode: bool = False # rejects nonzero cache inputs with 422 + + def __post_init__(self) -> None: + if self.current is None: + self.current = dict(self.initial) + + def info(self) -> RouterInfo: + return RouterInfo( + router_id=self.router_id, + namespace=self.namespace, + worker_type=self.worker_type, + revision=self.revision if self.runtime_config else 0, + configuration=dict(self.current) if self.runtime_config else None, + model="model", + ) + + +@dataclass +class FakeFrontend: + routers: dict[str, FakeRouter] = field(default_factory=dict) + # Statuses returned by the next GETs before the real answer. + get_failures: list[int] = field(default_factory=list) + + def add(self, router: FakeRouter) -> FakeRouter: + self.routers[router.router_id] = router + return router + + +class FakePolicyClient: + """In-memory frontends keyed by pod name.""" + + def __init__(self) -> None: + self.frontends: dict[str, FakeFrontend] = {} + self.calls: list[tuple[Any, ...]] = [] + # Statuses returned by the next writes (any router) before applying. + self.write_failures: list[int] = [] + # Called before a write is applied: (op, pod name, router_id). + self.before_write: Optional[Callable[[str, str, str], None]] = None + self.raise_on_get: Optional[BaseException] = None + self.active = 0 + self.max_active = 0 + self.get_delay_s = 0.0 + + def frontend(self, pod: str) -> FakeFrontend: + return self.frontends.setdefault(pod, FakeFrontend()) + + def writes(self) -> list[tuple[Any, ...]]: + return [call for call in self.calls if call[0] in ("patch", "reset")] + + async def get(self, pod: PodInfo) -> PolicyCallResult: + self.calls.append(("get", pod.name)) + if self.raise_on_get is not None: + raise self.raise_on_get + self.active += 1 + self.max_active = max(self.max_active, self.active) + try: + if self.get_delay_s: + await asyncio.sleep(self.get_delay_s) + frontend = self.frontends.get(pod.name) + if frontend is None: + return PolicyCallResult(status=0, error="connection refused") + if frontend.get_failures: + return PolicyCallResult(status=frontend.get_failures.pop(0)) + return PolicyCallResult( + status=200, + routers=tuple(router.info() for router in frontend.routers.values()), + ) + finally: + self.active -= 1 + + async def patch( + self, + pod: PodInfo, + router_id: str, + expected_revision: int, + parameters: Mapping[str, float], + ) -> PolicyCallResult: + self.calls.append( + ("patch", pod.name, router_id, expected_revision, dict(parameters)) + ) + return self._write("patch", pod, router_id, expected_revision, parameters) + + async def reset( + self, pod: PodInfo, router_id: str, expected_revision: int + ) -> PolicyCallResult: + self.calls.append(("reset", pod.name, router_id, expected_revision)) + return self._write("reset", pod, router_id, expected_revision, None) + + def _write( + self, + op: str, + pod: PodInfo, + router_id: str, + expected_revision: int, + parameters: Optional[Mapping[str, float]], + ) -> PolicyCallResult: + if self.before_write is not None: + self.before_write(op, pod.name, router_id) + if self.write_failures: + return PolicyCallResult(status=self.write_failures.pop(0), error="injected") + router = self.frontend(pod.name).routers.get(router_id) + if router is None: + return PolicyCallResult(status=404, error="router instance not found") + if not router.runtime_config: + return PolicyCallResult(status=422, error="unsupported") + if expected_revision != router.revision: + return PolicyCallResult(status=409, error="policy revision conflict") + if parameters is None: + updated = dict(router.initial) + else: + if not parameters: + return PolicyCallResult(status=400, error="empty") + updated = dict(router.current or {}) + for key, value in parameters.items(): + if key not in router.initial: + return PolicyCallResult(status=422, error=f"unknown {key}") + updated[key] = router.initial[key] if value is None else float(value) + if router.plain_decode and any(updated[key] != 0.0 for key in _CACHE_KEYS): + return PolicyCallResult(status=422, error="decode has no cache inputs") + router.current = updated + router.revision += 1 + return PolicyCallResult(status=200, router=router.info()) + + +class FakeKube: + namespace = DGD.namespace + + def __init__(self, pods: Optional[list[PodInfo]] = None) -> None: + self.pods: list[PodInfo] = list(pods or []) + self.selectors: list[str] = [] + self.fail: Optional[Exception] = None + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + self.selectors.append(label_selector) + if self.fail is not None: + raise self.fail + return list(self.pods) + + +class RecordingMetrics: + def __init__(self) -> None: + self.converged: list[bool] = [] + self.patch_errors: list[str] = [] + + def router_converged(self, converged: bool) -> None: + self.converged.append(converged) + + def patch_error(self, code: str) -> None: + self.patch_errors.append(code) + + def __getattr__(self, name: str) -> Callable[..., None]: + return lambda *args, **kwargs: None + + +def pod( + name: str, + *, + ready: bool = True, + terminating: bool = False, + ip: Optional[str] = "10.0.0.1", + phase: Optional[str] = None, +) -> PodInfo: + return PodInfo( + name=name, + uid=f"{name}-uid", + ip=ip, + ready=ready, + terminating=terminating, + labels={POD_LABEL_DGD_NAME: DGD.name, POD_LABEL_COMPONENT_TYPE: "frontend"}, + http_port=8000, + phase=phase, + ) + + +def make_actuator( + pods: list[PodInfo], + *, + startup_maps: Optional[dict[str, dict[str, float]]] = None, + max_concurrency: int = 8, +) -> tuple[RouterActuator, FakeKube, FakePolicyClient, RecordingMetrics]: + kube = FakeKube(pods) + client = FakePolicyClient() + metrics = RecordingMetrics() + actuator = RouterActuator( + kube, + client, + DGD, + startup_maps or {"aggregated": dict(STARTUP)}, + metrics, + max_concurrency=max_concurrency, + ) + return actuator, kube, client, metrics + + +def apply(desired: Mapping[str, float], wt: str = "aggregated") -> ActuationTarget: + return ActuationTarget(mode="apply", desired_by_wt={wt: dict(desired)}) + + +OBSERVE = ActuationTarget(mode="observe") +ROLLBACK = ActuationTarget(mode="rollback", include_not_ready=True) + + +def diff_keys(left: Mapping[str, float], right: Mapping[str, float]) -> set[str]: + return {key for key in POLICY_PARAMETER_KEYS if left[key] != right[key]} + + +# ---------------------------------------------------------------- latching + + +async def test_observe_latches_initials_and_never_writes() -> None: + actuator, kube, client, metrics = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + + assert kube.selectors == [ + f"{POD_LABEL_DGD_NAME}=dgd,{POD_LABEL_COMPONENT_TYPE}=frontend" + ] + assert client.writes() == [] + assert report.converged + assert report.new_initials == {"r1": STARTUP} + assert report.seen_router_ids == frozenset({"r1"}) + assert report.parity_ok is True + assert report.parity_mismatches == () + assert metrics.converged == [True] + + # Persisted initials are no longer reported as new. + report = await actuator.reconcile( + OBSERVE, initials={"r1": STARTUP}, expected_frontends=1 + ) + assert report.new_initials == {} + assert report.converged + + +async def test_new_initials_reported_until_persisted() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + + await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + # The state write has not landed: the router is at revision 1 now but + # its initial stays known locally (no learning reset) and is reported. + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + assert [call[0] for call in client.writes()] == ["patch"] + assert report.new_initials == {"r1": STARTUP} + assert report.converged + + +async def test_observe_with_unknown_initial_writes_nothing() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add( + FakeRouter("r1", STARTUP, revision=4, current=dict(CHOSEN)) + ) + + for _ in range(3): + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + assert not report.converged + assert report.parity_ok is None + assert report.new_initials == {} + assert client.writes() == [] + + +# ------------------------------------------------------------------- apply + + +async def test_apply_patches_only_differing_keys_with_explicit_values() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + + assert client.writes() == [ + ( + "patch", + "fe-0", + "r1", + 0, + { + "overlap_score_credit": 0.5, + "prefill_load_scale": 2.0, + "router_temperature": 0.2, + }, + ) + ] + assert router.current == CHOSEN + assert report.converged + assert report.parity_ok is True + + # Converged: the next rounds write nothing. + for _ in range(2): + report = await actuator.reconcile( + apply(CHOSEN), initials={"r1": STARTUP}, expected_frontends=1 + ) + assert report.converged + assert len(client.writes()) == 1 + + +async def test_zero_credit_round_trip_restores_cache_weights() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + initials: dict[str, dict[str, float]] = {} + + report = await actuator.reconcile( + apply(ZERO_CREDIT), initials=initials, expected_frontends=1 + ) + initials.update(report.new_initials) + assert set(client.writes()[-1][4]) == diff_keys(STARTUP, ZERO_CREDIT) + assert router.current == ZERO_CREDIT + + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) + # host/disk come back with the credit, as explicit values. + assert client.writes()[-1][4] == { + key: CHOSEN[key] for key in diff_keys(ZERO_CREDIT, CHOSEN) + } + assert router.current == CHOSEN + assert report.converged + + await actuator.reconcile( + apply(ZERO_CREDIT), initials=initials, expected_frontends=1 + ) + report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) + assert client.writes()[-1] == ("reset", "fe-0", "r1", 3) + assert router.current == STARTUP + assert router.current["host_cache_hit_weight"] == 0.75 + assert router.current["disk_cache_hit_weight"] == 0.25 + assert report.converged + + +async def test_apply_learns_unknown_initial_then_patches_in_one_round() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + tampered = {**STARTUP, "router_temperature": 0.9} + router = client.frontend("fe-0").add( + FakeRouter("r1", STARTUP, revision=2, current=tampered) + ) + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + + assert [call[0] for call in client.writes()] == ["reset", "patch"] + assert client.writes()[0] == ("reset", "fe-0", "r1", 2) + assert client.writes()[1][3] == 3 + assert router.current == CHOSEN + assert report.new_initials == {"r1": STARTUP} + assert report.parity_ok is True + assert report.converged + + await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + assert len(client.writes()) == 2 + + +async def test_parity_mismatch_blocks_apply_and_never_loops() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0"), pod("fe-1", ip="10.0.0.2")]) + odd = {**STARTUP, "overlap_score_credit_decay": 0.5} + client.frontend("fe-0").add(FakeRouter("good", STARTUP)) + client.frontend("fe-1").add(FakeRouter("odd", odd)) + + for _ in range(3): + report = await actuator.reconcile( + apply(CHOSEN), initials={}, expected_frontends=2 + ) + assert report.parity_ok is False + assert len(report.parity_mismatches) == 1 + assert report.parity_mismatches[0].startswith("fe-1/odd/aggregated:") + assert "overlap_score_credit_decay initial=0.5 startup=0.0" in ( + report.parity_mismatches[0] + ) + assert not report.converged + # Only the router whose initial matches startup was written, once. + assert [call[2] for call in client.writes()] == ["good"] + assert client.frontend("fe-1").routers["odd"].current == odd + + +async def test_rollback_with_parity_mismatch_resets_to_each_routers_initial() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + odd = {**STARTUP, "overlap_score_credit_decay": 0.5} + router = client.frontend("fe-0").add(FakeRouter("odd", odd)) + await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + router.current = {**odd, "router_temperature": 1.0} + router.revision = 1 + + report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) + + assert client.writes() == [("reset", "fe-0", "odd", 1)] + assert router.current == odd + assert report.converged # rollback converges to the latched initial + assert report.parity_ok is False + + +# ---------------------------------------------------------------- rollback + + +async def test_rollback_resets_only_deviating_routers_and_never_by_revision() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + frontend = client.frontend("fe-0") + # At its initial although its revision is not 0 (patched, then reset). + at_initial = frontend.add(FakeRouter("a", STARTUP, revision=3)) + deviating = frontend.add(FakeRouter("b", STARTUP, revision=1, current=dict(CHOSEN))) + initials = {"a": dict(STARTUP), "b": dict(STARTUP)} + + report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) + + assert client.writes() == [("reset", "fe-0", "b", 1)] + assert deviating.current == STARTUP + assert deviating.revision == 2 + assert at_initial.revision == 3 + assert report.converged + + # The reset bumped b's revision; comparing configurations, not + # revisions, means no further resets. + for _ in range(3): + report = await actuator.reconcile( + ROLLBACK, initials=initials, expected_frontends=1 + ) + assert report.converged + assert len(client.writes()) == 1 + + +async def test_rollback_learns_unknown_initial_with_a_single_reset() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + router = client.frontend("fe-0").add( + FakeRouter("r1", STARTUP, revision=5, current=dict(CHOSEN)) + ) + + report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) + assert client.writes() == [("reset", "fe-0", "r1", 5)] + assert report.new_initials == {"r1": STARTUP} + assert router.current == STARTUP + assert report.converged + + for _ in range(2): + report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) + assert report.converged + assert len(client.writes()) == 1 + + +async def test_rollback_targets_not_ready_pods_but_never_terminating_ones() -> None: + pods = [ + pod("ready", ip="10.0.0.1"), + pod("not-ready", ready=False, ip="10.0.0.2"), + pod("pending", ready=False, ip=None), + pod("leaving", terminating=True, ip="10.0.0.4"), + ] + actuator, _, client, _ = make_actuator(pods) + initials = {} + for name in ("ready", "not-ready", "leaving"): + client.frontend(name).add( + FakeRouter(f"r-{name}", STARTUP, revision=1, current=dict(CHOSEN)) + ) + initials[f"r-{name}"] = dict(STARTUP) + + report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) + + assert sorted(call[1] for call in client.writes()) == ["not-ready", "ready"] + assert ("get", "leaving") not in client.calls + assert ("get", "pending") not in client.calls + assert report.converged + + # apply never targets NotReady pods. + client.calls.clear() + await actuator.reconcile(apply(CHOSEN), initials=initials, expected_frontends=1) + assert {call[1] for call in client.calls} == {"ready"} + + +async def test_rollback_converges_without_an_unreachable_not_ready_pod() -> None: + actuator, _, client, _ = make_actuator( + [pod("ready"), pod("crashing", ready=False, ip="10.0.0.2")] + ) + router = client.frontend("ready").add( + FakeRouter("r1", STARTUP, revision=1, current=dict(CHOSEN)) + ) + # CrashLoopBackOff: the GET is refused on every round. + client.frontend("crashing").get_failures = [0, 0, 0] + + for _ in range(3): + report = await actuator.reconcile( + ROLLBACK, initials={"r1": STARTUP}, expected_frontends=1 + ) + assert report.converged + assert report.errors == {"get:network": 1} + assert router.current == STARTUP + assert client.writes() == [("reset", "ready", "r1", 1)] + + +async def test_an_unreachable_ready_pod_blocks_convergence() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0"), pod("fe-1", ip="10.0.0.2")]) + client.frontend("fe-0").add(FakeRouter("r0", STARTUP)) + client.frontend("fe-1").add(FakeRouter("r1", STARTUP)) + client.frontend("fe-1").get_failures = [0] + + report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) + assert not report.converged + + report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) + assert report.converged + + +async def test_an_answering_not_ready_pod_is_still_checked() -> None: + actuator, _, client, _ = make_actuator( + [pod("ready"), pod("starting", ready=False, ip="10.0.0.2")] + ) + client.frontend("ready").add(FakeRouter("r0", STARTUP)) + stuck = client.frontend("starting").add( + FakeRouter("r1", STARTUP, revision=2, current=dict(CHOSEN)) + ) + initials = {"r0": dict(STARTUP), "r1": dict(STARTUP)} + client.write_failures = [500] + + report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) + assert not report.converged # it answered, and it is not at its initial + assert stuck.current == CHOSEN + + report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) + assert report.converged + assert stuck.current == STARTUP + + +@pytest.mark.parametrize("phase", ["Failed", "Succeeded"]) +async def test_finished_pods_are_never_targeted_and_their_routers_are_gone( + phase: str, +) -> None: + actuator, kube, client, _ = make_actuator( + [pod("fe-0"), pod("evicted", ip="10.0.0.2")] + ) + client.frontend("fe-0").add(FakeRouter("r0", STARTUP)) + client.frontend("evicted").add(FakeRouter("r-ev", STARTUP)) + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + initials = dict(report.new_initials) + assert set(initials) == {"r0", "r-ev"} + + # Evicted: the pod object stays, with its IP, in a finished phase. + kube.pods = [pod("fe-0"), pod("evicted", ready=False, ip="10.0.0.2", phase=phase)] + client.frontend("evicted").get_failures = [0] + client.calls.clear() + + report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) + + assert ("get", "evicted") not in client.calls + assert report.converged + assert report.errors == {} + assert report.seen_router_ids == frozenset({"r0"}) + + +async def test_rollback_without_learning_leaves_unknown_routers_alone() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + frontend = client.frontend("fe-0") + # Changed by someone else before any planner latched it. + foreign = frontend.add( + FakeRouter("foreign", STARTUP, revision=4, current=dict(CHOSEN)) + ) + ours = frontend.add(FakeRouter("ours", STARTUP, revision=1, current=dict(CHOSEN))) + target = ActuationTarget( + mode="rollback", include_not_ready=True, learn_unknown=False + ) + + for _ in range(2): + report = await actuator.reconcile( + target, initials={"ours": STARTUP}, expected_frontends=1 + ) + assert report.converged + assert client.writes() == [("reset", "fe-0", "ours", 1)] + assert ours.current == STARTUP + assert foreign.current == CHOSEN and foreign.revision == 4 + assert report.new_initials == {} + assert report.parity_ok is None # the foreign router's initial is unknown + + # The default (active) target learns it with one reset. + report = await actuator.reconcile( + ROLLBACK, initials={"ours": STARTUP}, expected_frontends=1 + ) + assert client.writes()[-1] == ("reset", "fe-0", "foreign", 4) + assert foreign.current == STARTUP + assert report.converged + + +async def test_apply_without_learning_never_resets_an_unknown_router() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add( + FakeRouter("r1", STARTUP, revision=2, current=dict(CHOSEN)) + ) + target = ActuationTarget( + mode="apply", + desired_by_wt={"aggregated": dict(ZERO_CREDIT)}, + learn_unknown=False, + ) + + report = await actuator.reconcile(target, initials={}, expected_frontends=1) + + assert client.writes() == [] + assert not report.converged + + +# ----------------------------------------------------------- status codes + + +async def test_conflict_regets_and_retries_once() -> None: + actuator, _, client, metrics = make_actuator([pod("fe-0")]) + router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + interfered = [] + + def someone_else_writes(op: str, pod_name: str, router_id: str) -> None: + if not interfered: + interfered.append(True) + router.current = {**STARTUP, "router_temperature": 0.7} + router.revision += 1 + + client.before_write = someone_else_writes + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + + patches = [call for call in client.writes() if call[0] == "patch"] + assert [call[3] for call in patches] == [0, 1] + # The retry diff comes from the fresh GET (temperature 0.7 -> 0.2). + assert patches[1][4]["router_temperature"] == 0.2 + assert router.current == CHOSEN + assert report.errors == {"patch:409": 1} + assert metrics.patch_errors == ["409"] + assert report.converged + + +async def test_conflict_twice_gives_up_until_next_round() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + client.write_failures = [409, 409] + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + assert len(client.writes()) == 2 + assert not report.converged + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + assert router.current == CHOSEN + assert report.converged + + +async def test_patch_404_regets_and_patches_the_rebuilt_router() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + frontend = client.frontend("fe-0") + frontend.add(FakeRouter("old", STARTUP)) + + def rebuild(op: str, pod_name: str, router_id: str) -> None: + if router_id == "old": + del frontend.routers["old"] + frontend.add(FakeRouter("new", STARTUP)) + + client.before_write = rebuild + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + + assert [(call[0], call[2]) for call in client.writes()] == [ + ("patch", "old"), + ("patch", "new"), + ] + assert frontend.routers["new"].current == CHOSEN + assert report.errors == {"patch:404": 1} + assert report.new_initials == {"new": STARTUP} + assert report.seen_router_ids == frozenset({"new"}) + assert report.converged + + +async def test_503_regets_then_retries() -> None: + actuator, _, client, metrics = make_actuator([pod("fe-0")]) + router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + client.write_failures = [503] + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + + assert client.calls[:3] == [ + ("get", "fe-0"), + ("patch", "fe-0", "r1", 0, client.writes()[0][4]), + ("get", "fe-0"), + ] + assert router.current == CHOSEN + assert metrics.patch_errors == ["503"] + assert report.converged + + +async def test_422_is_not_retried_until_the_target_changes() -> None: + actuator, _, client, metrics = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + client.write_failures = [422] + + for _ in range(3): + report = await actuator.reconcile( + apply(CHOSEN), initials={}, expected_frontends=1 + ) + assert not report.converged + assert len(client.writes()) == 1 + assert metrics.patch_errors == ["422"] + + other = {**CHOSEN, "router_temperature": 0.4} + report = await actuator.reconcile(apply(other), initials={}, expected_frontends=1) + assert len(client.writes()) == 2 + assert report.converged + + +async def test_get_404_for_three_rounds_reports_admin_api_disabled() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + frontend = client.frontend("fe-0") + frontend.add(FakeRouter("r1", STARTUP)) + frontend.get_failures = [404, 404, 404] + + flags = [] + for _ in range(3): + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + flags.append(report.admin_api_disabled) + assert report.errors == {"get:404": 1} + assert flags == [False, False, True] + + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + assert not report.admin_api_disabled + assert report.converged + + +async def test_get_404_counter_needs_consecutive_rounds() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + frontend = client.frontend("fe-0") + frontend.add(FakeRouter("r1", STARTUP)) + frontend.get_failures = [404, 404] + await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) # 200 + frontend.get_failures = [404] + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + assert not report.admin_api_disabled + + +# ------------------------------------------------------------- convergence + + +async def test_not_converged_below_the_expected_ready_frontends() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=2) + assert not report.converged + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=0) + assert report.converged # max(1, 0) + + +async def test_not_converged_without_any_router() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0") + + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + assert not report.converged + assert report.parity_ok is None + + +async def test_routers_present_does_not_need_the_target_values() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + + # Present (P12) even though the routers do not match the apply target. + report = await actuator.reconcile( + ActuationTarget(mode="observe"), initials={}, expected_frontends=1 + ) + assert report.routers_present is True + report = await actuator.reconcile( + ActuationTarget(mode="observe"), initials={}, expected_frontends=2 + ) + assert report.routers_present is False + + +async def test_routers_present_needs_a_configured_router() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("custom", STARTUP, runtime_config=False)) + + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + assert report.routers_present is False + + +async def test_a_router_without_runtime_configuration_does_not_count() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("custom", STARTUP, runtime_config=False)) + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + assert not report.converged + assert client.writes() == [] + + +async def test_disagg_needs_both_worker_types_and_respects_plain_decode() -> None: + maps = {"prefill": dict(STARTUP), "decode": dict(DECODE_STARTUP)} + actuator, _, client, _ = make_actuator([pod("fe-0")], startup_maps=maps) + frontend = client.frontend("fe-0") + prefill = frontend.add(FakeRouter("p", STARTUP, worker_type="prefill")) + + target = ActuationTarget( + mode="apply", + desired_by_wt={ + "prefill": dict(CHOSEN), + "decode": {**DECODE_STARTUP, "router_temperature": 0.2}, + }, + ) + report = await actuator.reconcile(target, initials={}, expected_frontends=1) + assert not report.converged # no decode router yet + assert prefill.current == CHOSEN + + decode = frontend.add( + FakeRouter("d", DECODE_STARTUP, worker_type="decode", plain_decode=True) + ) + report = await actuator.reconcile(target, initials={}, expected_frontends=1) + assert decode.current["router_temperature"] == 0.2 + assert decode.current["overlap_score_credit"] == 0.0 + assert report.converged + assert report.parity_ok is True + + +async def test_only_the_dgd_namespace_and_planner_worker_types_are_managed() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + frontend = client.frontend("fe-0") + ours = frontend.add(FakeRouter("ours", STARTUP)) + foreign = frontend.add(FakeRouter("foreign", STARTUP, namespace="other-ns")) + encode = frontend.add(FakeRouter("encode", STARTUP, worker_type="encode")) + + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + + assert {call[2] for call in client.writes()} == {"ours"} + assert ours.current == CHOSEN + assert foreign.current == STARTUP and encode.current == STARTUP + assert report.seen_router_ids == frozenset({"ours"}) + assert report.converged + + +async def test_frontend_restart_fills_in_without_reset() -> None: + actuator, kube, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + initials: dict[str, dict[str, float]] = {} + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) + initials.update(report.new_initials) + + # The frontend restarts: Ready is 0 for a while. + kube.pods = [pod("fe-0", ready=False)] + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) + assert not report.converged + + # A new pod with a fresh router at its startup values: PATCH, no reset. + kube.pods = [pod("fe-0b", ip="10.0.0.9")] + client.frontend("fe-0b").add(FakeRouter("r2", STARTUP)) + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) + assert [call[0] for call in client.writes()] == ["patch", "patch"] + assert client.frontend("fe-0b").routers["r2"].current == CHOSEN + assert report.converged + assert report.seen_router_ids == frozenset({"r2"}) + + +# -------------------------------------------------------- seen router ids + + +async def test_seen_router_ids_drop_only_routers_that_are_provably_gone() -> None: + pods = [pod("fe-0"), pod("fe-1", ip="10.0.0.2")] + actuator, kube, client, _ = make_actuator(pods) + frontend0 = client.frontend("fe-0") + frontend0.add(FakeRouter("a", STARTUP)) + client.frontend("fe-1").add(FakeRouter("b", STARTUP)) + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=2) + initials = dict(report.new_initials) + assert report.seen_router_ids == frozenset({"a", "b"}) + + # fe-1 does not answer: b is kept; a left fe-0, which answered: dropped. + del frontend0.routers["a"] + frontend0.add(FakeRouter("a2", STARTUP)) + client.frontend("fe-1").get_failures = [0] + report = await actuator.reconcile(OBSERVE, initials=initials, expected_frontends=2) + assert report.seen_router_ids == frozenset({"a2", "b"}) + + # fe-1 is gone from the listing: b goes too. + kube.pods = [pod("fe-0")] + report = await actuator.reconcile(OBSERVE, initials=initials, expected_frontends=1) + assert report.seen_router_ids == frozenset({"a2"}) + + +async def test_persisted_initials_of_unknown_pods_survive_an_incomplete_view() -> None: + pods = [pod("fe-0"), pod("fe-1", ready=False, ip="10.0.0.2")] + actuator, _, client, _ = make_actuator(pods) + client.frontend("fe-0").add(FakeRouter("a", STARTUP)) + persisted = {"a": dict(STARTUP), "unknown": dict(STARTUP)} + + # apply does not query the NotReady fe-1, so "unknown" may live there. + report = await actuator.reconcile(OBSERVE, initials=persisted, expected_frontends=1) + assert report.seen_router_ids == frozenset({"a", "unknown"}) + + # rollback queries every live pod; "unknown" is nowhere: dropped. + client.frontend("fe-1") + report = await actuator.reconcile( + ROLLBACK, initials=persisted, expected_frontends=1 + ) + assert report.seen_router_ids == frozenset({"a"}) + + +# --------------------------------------------------------- failure paths + + +async def test_list_pods_failure_reports_without_raising() -> None: + actuator, kube, _, metrics = make_actuator([pod("fe-0")]) + kube.fail = KubeError("apiserver down") + + report = await actuator.reconcile( + OBSERVE, initials={"r1": STARTUP}, expected_frontends=1 + ) + + assert not report.converged + assert report.errors == {"list_pods": 1} + assert report.seen_router_ids == frozenset({"r1"}) + assert actuator.last_pods is None + assert metrics.converged == [False] + + +async def test_unexpected_client_exception_never_escapes() -> None: + actuator, _, client, metrics = make_actuator([pod("fe-0")]) + client.raise_on_get = RuntimeError("bug") + + report = await actuator.reconcile( + apply(CHOSEN), initials={"r1": STARTUP}, expected_frontends=1 + ) + + assert not report.converged + assert report.errors == {"internal": 1} + assert report.seen_router_ids == frozenset({"r1"}) + assert metrics.converged == [False] + + +async def test_cancellation_propagates() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.raise_on_get = asyncio.CancelledError() + + with pytest.raises(asyncio.CancelledError): + await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + + +@pytest.mark.parametrize( + "target", + [ + ActuationTarget(mode="apply"), + ActuationTarget(mode="apply", desired_by_wt={"prefill": dict(CHOSEN)}), + ActuationTarget( + mode="apply", + desired_by_wt={"aggregated": {**CHOSEN, "router_temperature": -1.0}}, + ), + ActuationTarget( + mode="apply", + desired_by_wt={ + "aggregated": { + k: v for k, v in CHOSEN.items() if k != "prefill_load_scale" + } + }, + ), + ], +) +async def test_invalid_apply_target_writes_nothing(target: ActuationTarget) -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + + report = await actuator.reconcile(target, initials={}, expected_frontends=1) + + assert client.writes() == [] + assert not report.converged + assert report.errors == {"invalid_target": 1} + assert report.new_initials == {"r1": STARTUP} # still latched + + +async def test_max_concurrency_bounds_parallel_gets() -> None: + pods = [pod(f"fe-{i}", ip=f"10.0.0.{i + 1}") for i in range(6)] + actuator, _, client, _ = make_actuator(pods, max_concurrency=2) + for i in range(6): + client.frontend(f"fe-{i}").add(FakeRouter(f"r{i}", STARTUP)) + client.get_delay_s = 0.01 + + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=6) + + assert client.max_active == 2 + assert report.converged + assert actuator.last_pods is not None and len(actuator.last_pods) == 6 + + +def test_constructor_validation() -> None: + kube, client, metrics = FakeKube(), FakePolicyClient(), RecordingMetrics() + with pytest.raises(ValueError): + RouterActuator(kube, client, DGD, {}, metrics) + with pytest.raises(ValueError): + RouterActuator(kube, client, DGD, {"aggregated": STARTUP}, metrics, 0) + actuator = RouterActuator( + kube, client, DGD, {"prefill": STARTUP, "decode": DECODE_STARTUP}, metrics + ) + assert actuator.worker_types == ("prefill", "decode") + actuator.set_startup_maps({"aggregated": STARTUP}) + assert actuator.worker_types == ("aggregated",) + + +async def test_close_closes_the_client_when_it_can() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + await actuator.close() # FakePolicyClient has no close: no-op + + http = AiohttpRouterPolicyClient() + actuator = RouterActuator( + FakeKube(), http, DGD, {"aggregated": STARTUP}, RecordingMetrics() + ) + session = http._ensure_session() # what the first call does + assert not session.closed + await actuator.close() + assert session.closed and http._session is None + + +# ------------------------------------------------------------- HTTP client + + +class _FrontendApp: + """A tiny aiohttp server speaking the /router_policy wire format.""" + + def __init__(self) -> None: + self.requests: list[tuple[str, Any]] = [] + self.patch_status = 200 + self.list_body: Any = { + "routers": [ + { + "router_id": "6f1c", + "model": "Qwen/Qwen3-0.6B", + "namespace": "dyn-ns", + "component": "backend", + "endpoint": "generate", + "worker_type": "aggregated", + "configuration": {"revision": 3, "parameters": dict(STARTUP)}, + }, + { + "router_id": "custom", + "model": "Qwen/Qwen3-0.6B", + "namespace": "dyn-ns", + "component": "backend", + "endpoint": "generate", + "worker_type": "decode", + "configuration": None, + }, + ] + } + + def app(self) -> web.Application: + app = web.Application() + app.router.add_get("/router_policy", self.list) + app.router.add_patch("/router_policy", self.update) + return app + + async def list(self, request: web.Request) -> web.Response: + self.requests.append(("GET", dict(request.query))) + if isinstance(self.list_body, str): + return web.Response(text=self.list_body, content_type="application/json") + return web.json_response(self.list_body) + + async def update(self, request: web.Request) -> web.Response: + body = await request.json() + self.requests.append(("PATCH", body)) + if self.patch_status != 200: + return web.json_response( + {"error": "policy revision conflict: expected 1, current 2"}, + status=self.patch_status, + ) + parameters = dict(STARTUP) + parameters.update(body.get("parameters") or {}) + return web.json_response( + { + "router_id": body["router_id"], + "model": "Qwen/Qwen3-0.6B", + "namespace": "dyn-ns", + "component": "backend", + "endpoint": "generate", + "worker_type": "aggregated", + "configuration": { + "revision": body["expected_revision"] + 1, + "parameters": parameters, + }, + } + ) + + +@pytest.fixture +async def frontend_server(): + frontend = _FrontendApp() + server = test_utils.TestServer(frontend.app(), host="127.0.0.1") + await server.start_server() + client = AiohttpRouterPolicyClient(timeout_s=2.0) + target = PodInfo( + name="fe-0", + uid="u", + ip="127.0.0.1", + ready=True, + terminating=False, + http_port=server.port, + ) + try: + yield frontend, client, target + finally: + await client.close() + await server.close() + + +async def test_client_get_parses_the_router_list(frontend_server) -> None: + frontend, client, target = frontend_server + + result = await client.get(target) + + assert result.status == 200 and result.error is None + assert frontend.requests == [("GET", {})] # no ?model= + first, custom = result.routers + assert first == RouterInfo( + router_id="6f1c", + namespace="dyn-ns", + worker_type="aggregated", + revision=3, + configuration=STARTUP, + model="Qwen/Qwen3-0.6B", + ) + assert custom.configuration is None and custom.revision == 0 + + +async def test_client_patch_sends_explicit_parameters(frontend_server) -> None: + frontend, client, target = frontend_server + + result = await client.patch(target, "6f1c", 3, {"overlap_score_credit": 0}) + + assert frontend.requests == [ + ( + "PATCH", + { + "router_id": "6f1c", + "expected_revision": 3, + "parameters": {"overlap_score_credit": 0.0}, + }, + ) + ] + assert result.status == 200 + assert result.router is not None + assert result.router.revision == 4 + assert result.router.configuration["overlap_score_credit"] == 0.0 + + +async def test_client_reset_sends_reset_true_without_parameters( + frontend_server, +) -> None: + frontend, client, target = frontend_server + + result = await client.reset(target, "6f1c", 7) + + assert frontend.requests == [ + ("PATCH", {"router_id": "6f1c", "expected_revision": 7, "reset": True}) + ] + assert result.status == 200 and result.router.revision == 8 + + +async def test_client_reports_the_error_status_and_message(frontend_server) -> None: + frontend, client, target = frontend_server + frontend.patch_status = 409 + + result = await client.patch(target, "6f1c", 1, {"router_temperature": 0.1}) + + assert result.status == 409 + assert result.router is None + assert result.error == "policy revision conflict: expected 1, current 2" + + +@pytest.mark.parametrize( + "body", ["not json", json.dumps({"routers": "x"}), json.dumps({"other": []})] +) +async def test_client_malformed_body_is_status_zero(frontend_server, body) -> None: + frontend, client, target = frontend_server + frontend.list_body = body + + result = await client.get(target) + + assert result.status == 0 + assert result.error and result.error.startswith("malformed GET response") + + +async def test_client_network_error_is_status_zero() -> None: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + client = AiohttpRouterPolicyClient(timeout_s=1.0) + target = PodInfo( + name="fe", + uid="u", + ip="127.0.0.1", + ready=True, + terminating=False, + http_port=port, + ) + try: + result = await client.get(target) + finally: + await client.close() + assert result.status == 0 + assert result.error + + +async def test_client_without_pod_ip_or_parameters_sends_nothing() -> None: + client = AiohttpRouterPolicyClient() + no_ip = PodInfo(name="fe", uid="u", ip=None, ready=False, terminating=False) + result = await client.get(no_ip) + assert result.status == 0 and "no IP" in (result.error or "") + with_ip = PodInfo(name="fe", uid="u", ip="10.0.0.1", ready=True, terminating=False) + result = await client.patch(with_ip, "r", 0, {}) + assert result.status == 0 and result.error == "empty parameters" + await client.close() + + +def test_client_url() -> None: + client = AiohttpRouterPolicyClient(scheme="http") + base = dict(name="fe", uid="u", ready=True, terminating=False) + assert ( + client.url(PodInfo(ip="10.1.2.3", http_port=8123, **base)) + == "http://10.1.2.3:8123/router_policy" + ) + assert ( + client.url(PodInfo(ip="10.1.2.3", **base)) + == "http://10.1.2.3:8000/router_policy" + ) + assert client.url(PodInfo(ip="fd00::5", http_port=80, **base)) == ( + "http://[fd00::5]:80/router_policy" + ) + with pytest.raises(ValueError): + AiohttpRouterPolicyClient(scheme="ftp") + with pytest.raises(ValueError): + AiohttpRouterPolicyClient(timeout_s=0) + + +def test_client_timeout_outlasts_the_server_control_timeout() -> None: + # A stuck router makes the server wait CONTROL_TIMEOUT before answering + # GET 200 without it (or PATCH 503); the client must still be listening. + assert DEFAULT_CLIENT_TIMEOUT_S == 8.0 + assert DEFAULT_CLIENT_TIMEOUT_S > ROUTER_CONTROL_TIMEOUT_S + assert AiohttpRouterPolicyClient()._timeout.total == DEFAULT_CLIENT_TIMEOUT_S + + source = next( + ( + parent / "lib/llm/src/http/service/router_policy.rs" + for parent in Path(__file__).resolve().parents + if (parent / "lib/llm/src/http/service/router_policy.rs").is_file() + ), + None, + ) + if source is None: + pytest.skip("router_policy.rs is not in this checkout") + match = re.search( + r"CONTROL_TIMEOUT: Duration = Duration::from_secs\((\d+)\)", + source.read_text(), + ) + assert match is not None + assert float(match.group(1)) == ROUTER_CONTROL_TIMEOUT_S diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py new file mode 100644 index 000000000000..9f992c9842f2 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py @@ -0,0 +1,922 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live retune wiring of the engine adapter, the planner base and ``__main__``. + +The retune controller is the recording stub of ``retune_stubs`` (one test +builds the real one), and the live supervisor is a fake ``LiveRetuneHandle``, +so these tests cover only the host contract: the disabled state, +``replace_retune``, the base's supervisor lifecycle and the SIGTERM path. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import logging +import os +import signal +import threading +from concurrent.futures import Future +from unittest.mock import patch + +import pytest + +from dynamo.planner import __main__ as planner_main +from dynamo.planner.config.planner_config import PlannerConfig, RetuneConfig +from dynamo.planner.core.base import NativePlannerBase +from dynamo.planner.core.types import ( + PlannerEffects, + RetuneDecision, + RetuneInputs, + RetuneParams, + ScheduledTick, + TickInput, + WorkerCounts, +) +from dynamo.planner.environment.state import DeploymentState +from dynamo.planner.plugins.clock import VirtualClock, WallClock +from dynamo.planner.plugins.orchestrator.engine_adapter import ( + OrchestratorEngineAdapter, + RetuneHooks, +) +from dynamo.planner.retune.live.contracts import GateResult +from dynamo.planner.tests.retune_stubs import ( + ROUTER_MAP, + StubTickResult, + caps, + install_stub_controller, + retune_inputs, + retune_only_config, +) + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +PHASE = 120.0 + + +def _sweep(request, *, cancel, abort): + raise AssertionError("the stub controller never sweeps") + + +def _hooks(**overrides) -> RetuneHooks: + values = { + "sweep_fn": _sweep, + "launch_gate": lambda _k, _window: GateResult(), + "harvest_allowed": lambda _generation: True, + "phase_s": PHASE, + } + values.update(overrides) + return RetuneHooks(**values) + + +def _stub_running_future(self): + """``running_future`` for the stub: ``self.running``, recorded in calls.""" + + self.calls.append("running_future") + return getattr(self, "running", None) + + +@pytest.fixture +def stub_controller(monkeypatch): + stub = install_stub_controller(monkeypatch) + monkeypatch.setattr(stub, "running_future", _stub_running_future, raising=False) + return stub + + +def _adapter(*, inputs=None, hooks=None, clock=None, **overrides): + return OrchestratorEngineAdapter( + retune_only_config(**overrides), + caps(), + clock=clock or WallClock(), + retune_inputs=inputs, + retune_hooks=hooks if hooks is not None else _hooks(), + ) + + +def _decision() -> RetuneDecision: + point = RetuneParams(router={"router_temperature": 0.5}) + return RetuneDecision( + decision_id="retune-w60000", + outcome="kept", + window_index=0, + window_start_s=0.0, + window_end_s=60.0, + as_of_s=120.0, + launched_at_s=120.0, + decided_at_s=125.0, + num_requests=3, + num_candidates=2, + previous=point, + chosen=point, + baseline_score=1.0, + best_score=1.0, + ) + + +# --------------------------------------------------------------------------- +# Engine adapter: construction and the disabled state +# --------------------------------------------------------------------------- + + +def test_live_controller_gets_the_hooks_and_the_phase(stub_controller): + hooks = _hooks() + engine = _adapter(inputs=retune_inputs(), hooks=hooks) + + (controller,) = stub_controller.instances + assert engine.retune_controller is controller + assert controller.hooks == { + "sweep_fn": hooks.sweep_fn, + "launch_gate": hooks.launch_gate, + "harvest_allowed": hooks.harvest_allowed, + "generation": 0, + "predecessor": None, + } + engine.initial_tick(1_000.0) + assert (controller.started, controller.phases) == ([1_000.0], [PHASE]) + + +def test_replay_controller_is_built_and_started_as_before(stub_controller): + engine = OrchestratorEngineAdapter( + retune_only_config(), + caps(), + clock=VirtualClock(), + retune_inputs=retune_inputs(), + ) + + (controller,) = stub_controller.instances + assert controller.hooks == {} + engine.initial_tick(5.0) + assert (controller.started, controller.phases) == ([5.0], [None]) + + +def test_replay_without_inputs_still_fails_at_construction(stub_controller): + with pytest.raises(ValueError, match="requires retune inputs"): + OrchestratorEngineAdapter(retune_only_config(), caps(), clock=VirtualClock()) + + +def test_hooks_without_retune_config_build_nothing(stub_controller): + engine = OrchestratorEngineAdapter( + PlannerConfig(mode="agg"), + caps(), + clock=WallClock(), + retune_inputs=retune_inputs(), + retune_hooks=_hooks(), + ) + + assert engine.retune_controller is None + assert stub_controller.instances == [] + + +def test_live_without_inputs_starts_disabled(stub_controller, caplog): + with caplog.at_level(logging.WARNING): + engine = _adapter(inputs=None) + + assert engine.retune_controller is None + assert stub_controller.instances == [] + assert "retune stays disabled" in caplog.text + engine.initial_tick(0.0) # no controller to start + + +def test_live_controller_construction_failure_starts_disabled(stub_controller, caplog): + stub_controller.construct_error = ValueError("retune.router requires kv_router") + + with caplog.at_level(logging.ERROR): + engine = _adapter(inputs=retune_inputs()) + + assert engine.retune_controller is None + assert "requires kv_router" in caplog.text + + +@pytest.mark.parametrize( + "ready, overrides", + [(16, {}), (1, {"min_endpoint": 2})], + ids=["above-default-max-gpu-budget", "below-min-endpoint"], +) +async def test_disabled_retune_only_planner_never_reconciles_replicas( + stub_controller, ready, overrides +): + # retune_only follows the config, not the controller: a disabled live + # retune keeps the deployment's replicas and stays quiet. + engine = _adapter(inputs=None, optimization_target="sla", **overrides) + counts = WorkerCounts(ready_num_decode=ready, expected_num_decode=ready) + + scheduled = engine.initial_tick(0.0) + for _ in range(10): + assert not scheduled.run_load_scaling + assert not scheduled.need_worker_fpm + effects = await engine.tick( + scheduled, TickInput(now_s=scheduled.at_s, worker_counts=counts) + ) + assert effects.scale_to is None + assert effects.router_parameters is None + assert effects.retune_decisions == [] + assert effects.diagnostics.audit_events == [] + assert effects.next_tick is not None + scheduled = effects.next_tick + + +# --------------------------------------------------------------------------- +# Engine adapter: replace_retune +# --------------------------------------------------------------------------- + + +async def test_replace_retune_swaps_the_controller(stub_controller): + engine = _adapter(inputs=retune_inputs()) + engine.initial_tick(0.0) + (old,) = stub_controller.instances + inputs = RetuneInputs(engine=retune_inputs().engine, trace_paths=("/trace/b",)) + + engine.replace_retune(inputs, generation=4, now_s=500.0, last_decided_k=41) + + old_, new = stub_controller.instances + assert old_ is old + # Abort the old sweep first, close the old controller, then read the + # sweep still holding its thread (none here) for the new one to wait on. + assert old.calls == ["start", "drop_pending", "close", "running_future"] + assert new.args[1] is inputs + assert new.generation == 4 + assert new.hooks["predecessor"] is None + assert new.hooks["launch_gate"] is old.hooks["launch_gate"] + assert new.calls == ["start", "rewind"] + assert (new.started, new.phases, new.rewinds) == ([500.0], [PHASE], [41]) + assert engine.retune_controller is new + + # The next tick runs the new controller only. + new.results.append( + StubTickResult(decisions=[_decision()], router_parameters=ROUTER_MAP) + ) + scheduled = ScheduledTick(at_s=505.0, at_monotonic_s=505.0) + effects = await engine.tick( + scheduled, + TickInput(now_s=505.0, worker_counts=WorkerCounts(ready_num_decode=1)), + ) + assert old.ticks == [] and len(new.ticks) == 1 + assert effects.retune_decisions == [_decision()] + assert effects.router_parameters == ROUTER_MAP + + await engine.shutdown() + assert (old.closed, new.closed) == (1, 1) + + +def test_replace_retune_without_last_decided_k_does_not_rewind(stub_controller): + engine = _adapter(inputs=None) + + engine.replace_retune( + retune_inputs(), generation=1, now_s=10.0, last_decided_k=None + ) + + (controller,) = stub_controller.instances + assert controller.calls == ["start"] + assert engine.retune_controller is controller + + +def test_replace_retune_with_no_inputs_disables_retune(stub_controller): + engine = _adapter(inputs=retune_inputs()) + (old,) = stub_controller.instances + + engine.replace_retune(None, generation=2, now_s=10.0, last_decided_k=7) + + assert engine.retune_controller is None + assert old.calls == ["drop_pending", "close", "running_future"] + assert len(stub_controller.instances) == 1 + + +def test_replace_retune_queues_the_new_controller_behind_the_old_sweep( + stub_controller, +): + engine = _adapter(inputs=retune_inputs()) + (old,) = stub_controller.instances + old.running = Future() + + engine.replace_retune( + retune_inputs(), generation=1, now_s=10.0, last_decided_k=None + ) + + new = stub_controller.instances[1] + assert new.hooks["predecessor"] is old.running + # The new controller carries it on: replacing it asks it, not the adapter. + engine.replace_retune( + retune_inputs(), generation=2, now_s=20.0, last_decided_k=None + ) + assert stub_controller.instances[2].hooks["predecessor"] is None + + +def test_replace_retune_keeps_the_old_sweep_across_a_disabled_generation( + stub_controller, +): + engine = _adapter(inputs=retune_inputs()) + (old,) = stub_controller.instances + old.running = Future() + + engine.replace_retune(None, generation=1, now_s=10.0, last_decided_k=None) + stub_controller.construct_error = ValueError("zero startup credit") + with pytest.raises(ValueError, match="zero startup credit"): + engine.replace_retune( + retune_inputs(), generation=2, now_s=20.0, last_decided_k=None + ) + engine.replace_retune( + retune_inputs(), generation=3, now_s=30.0, last_decided_k=None + ) + + assert stub_controller.instances[1].hooks["predecessor"] is old.running + + +def test_replace_retune_forgets_an_old_sweep_that_has_finished(stub_controller): + engine = _adapter(inputs=retune_inputs()) + (old,) = stub_controller.instances + old.running = Future() + + engine.replace_retune(None, generation=1, now_s=10.0, last_decided_k=None) + old.running.set_result(None) + engine.replace_retune( + retune_inputs(), generation=2, now_s=20.0, last_decided_k=None + ) + + assert stub_controller.instances[1].hooks["predecessor"] is None + + +def test_replace_retune_construction_failure_leaves_retune_disabled( + stub_controller, +): + engine = _adapter(inputs=retune_inputs()) + (old,) = stub_controller.instances + stub_controller.construct_error = ValueError("zero startup credit") + + with pytest.raises(ValueError, match="zero startup credit"): + engine.replace_retune( + retune_inputs(), generation=3, now_s=10.0, last_decided_k=None + ) + + assert old.closed == 1 + assert engine.retune_controller is None + + +def test_replace_retune_survives_an_old_controller_that_fails_to_close( + stub_controller, +): + engine = _adapter(inputs=retune_inputs()) + (old,) = stub_controller.instances + old.close_error = RuntimeError("worker stuck") + + engine.replace_retune( + retune_inputs(), generation=1, now_s=10.0, last_decided_k=None + ) + + assert engine.retune_controller is stub_controller.instances[1] + + +def test_replace_retune_requires_live_hooks(stub_controller): + engine = OrchestratorEngineAdapter( + retune_only_config(), + caps(), + clock=VirtualClock(), + retune_inputs=retune_inputs(), + ) + + with pytest.raises(RuntimeError, match="live retune"): + engine.replace_retune(None, generation=1, now_s=0.0, last_decided_k=None) + + +async def test_real_controller_with_live_hooks_anchors_at_the_phase(): + pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", + ) + from dynamo.planner.retune.sweep import SweepResult + + def sweep(request, *, cancel, abort): + return SweepResult( + status="ok", + num_requests=1, + scores={candidate.key(): 1.0 for candidate in request.candidates}, + ) + + gate_calls = [] + + def gate(k, window): + gate_calls.append((k, window)) + return GateResult(spec_replicas={"aggregated": 2}) + + engine = OrchestratorEngineAdapter( + retune_only_config(), + caps(), + clock=VirtualClock(), + retune_inputs=retune_inputs(), + retune_hooks=_hooks(sweep_fn=sweep, launch_gate=gate, phase_s=0.0), + ) + controller = engine.retune_controller + assert controller is not None and controller.generation == 0 + counts = WorkerCounts(ready_num_decode=2, expected_num_decode=2) + try: + # Started mid-window W_1 = [60, 120): W_1 launches at L_1 = 120 + grace. + scheduled = engine.initial_tick(90.0) + decisions = [] + while scheduled.at_s <= 240.0: + effects = await engine.tick( + scheduled, TickInput(now_s=scheduled.at_s, worker_counts=counts) + ) + decisions.extend(effects.retune_decisions) + assert effects.next_tick is not None + scheduled = effects.next_tick + finally: + await engine.shutdown() + + assert gate_calls[0][0] == 0 + assert [d.decision_id for d in decisions][:2] == ["retune-w60000", "retune-w120000"] + + +async def test_rebuilt_real_controllers_never_sweep_at_once(): + """Two generations of one window never run their sweeps concurrently. + + Generation 0's sweep of window k is dropped but, like a DGDR sweep that + polls, only returns a while later; generation 1 relaunches window k in + the meantime. Its sweep must start only after generation 0's returned, + and its decision's launch time moves to that start. + """ + + pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", + ) + from dynamo.planner.retune.sweep import SweepResult + + lock = threading.Lock() + running: list[int] = [] + overlaps: list[list[int]] = [] + order: list[tuple[int, int]] = [] + gen0_started = threading.Event() + gen1_started = threading.Event() + release = threading.Event() + + def sweep(request, *, cancel, abort): + with lock: + running.append(request.generation) + order.append((request.generation, request.window_index)) + if len(running) > 1: + overlaps.append(list(running)) + try: + if request.generation == 0: + gen0_started.set() + # Sees the abort late, as a DGDR sweep at its next poll. + release.wait(30) + return SweepResult(status="failed", failure_reason="lost_leadership") + gen1_started.set() + return SweepResult( + status="ok", + num_requests=1, + scores={candidate.key(): 1.0 for candidate in request.candidates}, + ) + finally: + with lock: + running.remove(request.generation) + + engine = _adapter( + inputs=retune_inputs(), + hooks=_hooks( + sweep_fn=sweep, + launch_gate=lambda _k, _window: GateResult( + spec_replicas={"aggregated": 2}, abort=threading.Event() + ), + phase_s=0.0, + ), + ) + counts = WorkerCounts(ready_num_decode=2, expected_num_decode=2) + # On a window boundary: W_k = [now - 120, now - 60) launches at now and + # stays the only due window until now + 60. + now = 1_800_000_000.0 + try: + scheduled = engine.initial_tick(now) + await engine.tick(scheduled, TickInput(now_s=now, worker_counts=counts)) + assert gen0_started.wait(30) + + engine.replace_retune( + retune_inputs(), generation=1, now_s=now, last_decided_k=None + ) + await engine.tick(scheduled, TickInput(now_s=now + 1.0, worker_counts=counts)) + controller = engine.retune_controller + assert controller is not None and controller.generation == 1 + relaunch = controller._pending.future + # Queued behind generation 0's sweep, which still runs. + assert not gen1_started.wait(0.2) + + release.set() + relaunch.result(timeout=30) + effects = await engine.tick( + scheduled, TickInput(now_s=now + 30.0, worker_counts=counts) + ) + finally: + release.set() + await engine.shutdown() + + assert overlaps == [] + ((_, k0), (_, k1)) = order + assert [g for g, _ in order] == [0, 1] and k0 == k1 + (decision,) = effects.retune_decisions + assert decision.window_index == k0 + # Launched when generation 0's sweep returned, not at the tick. + assert decision.launched_at_s > now + 1.0 + + +# --------------------------------------------------------------------------- +# Planner base wiring +# --------------------------------------------------------------------------- + + +class _Environment: + def __init__(self) -> None: + self.state = DeploymentState() + self.state.decode.num_gpus = 1 + self.calls: list[str] = [] + + def deployment_state(self) -> DeploymentState: + return self.state + + async def initialize(self) -> None: + self.calls.append("environment.initialize") + + async def refresh(self) -> None: + pass + + async def shutdown(self) -> None: + self.calls.append("environment.shutdown") + + +class FakeLiveRetune: + """A ``LiveRetuneHandle`` that records the base's calls.""" + + def __init__(self, calls, *, inputs=None, hooks=None): + self.calls = calls + self.inputs = inputs + self._hooks = hooks or _hooks() + self.effects: list[PlannerEffects] = [] + self.host = None + self.on_effects_error: Exception | None = None + self.stop_error: Exception | None = None + + def retune_inputs(self): + self.calls.append("live.retune_inputs") + return self.inputs + + def hooks(self): + return self._hooks + + async def start(self, host): + self.calls.append("live.start") + self.host = host + + async def stop(self): + self.calls.append("live.stop") + if self.stop_error is not None: + raise self.stop_error + + def on_effects(self, effects): + self.effects.append(effects) + if self.on_effects_error is not None: + raise self.on_effects_error + + +def _live_config(executor: str = "dgdr") -> PlannerConfig: + config = PlannerConfig( + namespace="test-ns", + mode="agg", + enable_load_scaling=False, + enable_throughput_scaling=False, + metric_reporting_prometheus_port=0, + live_dashboard_port=0, + control_api_port=0, + max_gpu_budget=8, + ) + retune = { + "interval_seconds": 21600, + "grace_seconds": 120, + "timeout_seconds": 20400, + "router": {"overlap_score_credit": {"choices": [0.0, 1.0]}}, + "executor": executor, + } + if executor == "dgdr": + retune["dgdr"] = { + "claim_name": "request-traces", + "model": "Qwen/Qwen3-0.6B", + "sku": "h200_sxm", + "engine": {"mode": "aggregated", "backend": "vllm"}, + } + # Assigned after construction: the base wiring is under test here, not + # the live retune config validation of PlannerConfig. + config.retune = RetuneConfig.model_validate(retune) + return config + + +def _base(config: PlannerConfig, environment: _Environment) -> NativePlannerBase: + with patch("dynamo.planner.core.base.PlannerPrometheusMetrics"): + return NativePlannerBase(None, config, environment) + + +def _planner(executor="dgdr", **live_kwargs): + environment = _Environment() + planner = _base(_live_config(executor), environment) + live = FakeLiveRetune(environment.calls, **live_kwargs) + + async def build(): + environment.calls.append("live.build") + return live + + planner._build_live_retune = build # type: ignore[method-assign] + return planner, environment, live + + +def test_live_planner_rejects_the_local_executor(): + with pytest.raises( + ValueError, match="replay-only with executor 'local'; use executor: dgdr" + ): + _base(_live_config("local"), _Environment()) + + +def test_live_planner_accepts_the_dgdr_executor(): + planner, _, _ = _planner() + + assert planner._live_retune is None + + +async def test_async_init_builds_the_engine_with_live_retune_and_starts_it( + stub_controller, +): + planner, environment, live = _planner(inputs=retune_inputs()) + + async def bootstrap_regression(): + environment.calls.append("bootstrap_regression") + planner._ensure_engine() + + planner._bootstrap_regression = bootstrap_regression # type: ignore[method-assign] + await planner._async_init() + try: + assert environment.calls[:5] == [ + "environment.initialize", + "live.build", + "bootstrap_regression", + "live.retune_inputs", + "live.start", + ] + assert live.host is planner + engine = planner._engine + assert isinstance(engine, OrchestratorEngineAdapter) + (controller,) = stub_controller.instances + assert engine.retune_controller is controller + assert controller.hooks["launch_gate"] is live.hooks().launch_gate + finally: + await planner._shutdown_runtime() + + # Engine (and controller) first, then the supervisor, then the environment. + assert controller.closed == 1 + assert environment.calls[-2:] == ["live.stop", "environment.shutdown"] + assert planner._live_retune is None + + +async def test_async_init_with_failing_premises_starts_disabled(stub_controller): + planner, _, live = _planner(inputs=None) + + await planner._async_init() + try: + assert planner._engine.retune_controller is None # type: ignore[union-attr] + assert live.host is planner + finally: + await planner._shutdown_runtime() + + +async def test_failed_init_still_stops_live_retune(stub_controller): + planner, environment, live = _planner(inputs=retune_inputs()) + + async def failing_start(_host): + environment.calls.append("live.start") + raise RuntimeError("lease API unavailable") + + live.start = failing_start # type: ignore[method-assign] + with pytest.raises(RuntimeError, match="lease API unavailable"): + await planner._async_init() + + assert environment.calls[-2:] == ["live.stop", "environment.shutdown"] + + +async def test_shutdown_logs_a_failing_live_retune_stop(stub_controller, caplog): + planner, environment, live = _planner(inputs=None) + await planner._async_init() + live.stop_error = RuntimeError("lease release timed out") + + with caplog.at_level(logging.ERROR): + await planner._shutdown_runtime() + + assert "Failed to stop planner live retune" in caplog.text + assert environment.calls[-1] == "environment.shutdown" + + +class _FakeEngine: + def __init__(self, effects): + self.effects = effects + + async def observe(self, tick, now_s): + return TickInput(now_s=now_s) + + async def tick(self, tick, tick_input): + return self.effects + + +async def test_each_tick_hands_its_effects_to_live_retune(caplog): + planner, _, live = _planner() + planner._live_retune = live + effects = PlannerEffects( + next_tick=ScheduledTick(at_s=10.0), + router_parameters=ROUTER_MAP, + retune_decisions=[_decision()], + ) + applied = [] + + async def apply_effects(applied_effects): + applied.append(applied_effects) + + planner._apply_effects = apply_effects # type: ignore[method-assign] + engine = _FakeEngine(effects) + + next_tick = await planner._run_one_tick(engine, ScheduledTick(at_s=5.0)) + assert next_tick.at_s == 10.0 + assert live.effects == [effects] + assert applied == [effects] + + # A failing outbox never fails the tick or blocks the effects. + live.on_effects_error = RuntimeError("outbox full") + with caplog.at_level(logging.ERROR): + await planner._run_one_tick(engine, ScheduledTick(at_s=5.0)) + assert "Live retune failed to take the tick's effects" in caplog.text + assert len(applied) == 2 + + +async def test_replace_live_retune_holds_the_config_lock(stub_controller): + planner, _, _ = _planner(inputs=None) + await planner._async_init() + try: + await planner._config_lock.acquire() + task = asyncio.create_task( + planner.replace_live_retune(retune_inputs(), generation=2, last_decided_k=9) + ) + await asyncio.sleep(0.05) + assert not task.done() and stub_controller.instances == [] + planner._config_lock.release() + await asyncio.wait_for(task, timeout=5) + + (controller,) = stub_controller.instances + assert planner._engine.retune_controller is controller # type: ignore[union-attr] + assert controller.generation == 2 + assert controller.rewinds == [9] + assert controller.phases == [PHASE] + finally: + await planner._shutdown_runtime() + + +async def test_replace_live_retune_needs_the_orchestrator_engine(): + planner, _, _ = _planner() + + with pytest.raises(RuntimeError, match="orchestrator engine"): + await planner.replace_live_retune(None, generation=1, last_decided_k=None) + + +# --------------------------------------------------------------------------- +# SIGTERM +# --------------------------------------------------------------------------- + + +class _Runtime: + def __init__(self) -> None: + self.stopped = asyncio.Event() + self.shutdowns = 0 + + def shutdown(self) -> None: + self.shutdowns += 1 + self.stopped.set() + + +async def test_sigterm_cancels_the_planner_run_and_stops_serving(): + runtime = _Runtime() + events: list[str] = [] + running = asyncio.Event() + + async def serve(): + await runtime.stopped.wait() + events.append("serve stopped") + + async def run(): + running.set() + try: + await asyncio.sleep(3600) + finally: + # run()'s finally: the planner shutdown that releases the Lease. + await asyncio.sleep(0) + events.append("run finally") + + main = asyncio.create_task(planner_main._run_until_sigterm(runtime, serve(), run())) + await asyncio.wait_for(running.wait(), timeout=5) + loop = asyncio.get_running_loop() + assert signal.SIGTERM in loop._signal_handlers # type: ignore[attr-defined] + os.kill(os.getpid(), signal.SIGTERM) + os.kill(os.getpid(), signal.SIGTERM) # a second one changes nothing + await asyncio.wait_for(main, timeout=5) + + assert events == ["run finally", "serve stopped"] + assert runtime.shutdowns == 1 + assert signal.SIGTERM not in loop._signal_handlers # type: ignore[attr-defined] + + +async def test_without_sigterm_a_run_failure_propagates(): + runtime = _Runtime() + + async def serve(): + await runtime.stopped.wait() + + async def run(): + raise RuntimeError("tick failed") + + with pytest.raises(RuntimeError, match="tick failed"): + await planner_main._run_until_sigterm(runtime, serve(), run()) + assert runtime.shutdowns == 0 + loop = asyncio.get_running_loop() + assert signal.SIGTERM not in loop._signal_handlers # type: ignore[attr-defined] + runtime.shutdown() # let the serve task finish + + +async def test_outer_cancellation_is_not_mistaken_for_sigterm(): + runtime = _Runtime() + started = asyncio.Event() + + async def serve(): + await runtime.stopped.wait() + + async def run(): + started.set() + await asyncio.sleep(3600) + + main = asyncio.create_task(planner_main._run_until_sigterm(runtime, serve(), run())) + await asyncio.wait_for(started.wait(), timeout=5) + main.cancel() + with pytest.raises(asyncio.CancelledError): + await main + assert runtime.shutdowns == 0 + + +async def test_platforms_without_signal_handlers_still_run(monkeypatch, caplog): + runtime = _Runtime() + loop = asyncio.get_running_loop() + + def unsupported(*_args): + raise NotImplementedError + + monkeypatch.setattr(loop, "add_signal_handler", unsupported) + + async def serve(): + return None + + async def run(): + return None + + with caplog.at_level(logging.WARNING): + await planner_main._run_until_sigterm(runtime, serve(), run()) + assert "Cannot install a planner SIGTERM handler" in caplog.text + + +async def test_start_planner_installs_sigterm_only_with_retune(monkeypatch): + seen: list[tuple[bool, str]] = [] + + class _Planner: + async def _async_init(self): + pass + + async def run(self): + return None + + class _Endpoint: + async def serve_endpoint(self, *_args, **_kwargs): + return None + + class _StartRuntime: + def endpoint(self, _name): + return _Endpoint() + + async def fake_run_until_sigterm(runtime, serve, run): + seen.append((True, type(runtime).__name__)) + await asyncio.gather(serve, run) + + monkeypatch.setattr(planner_main, "construct_planner", lambda **_: _Planner()) + monkeypatch.setattr(planner_main, "_run_until_sigterm", fake_run_until_sigterm) + + await planner_main.start_planner(_StartRuntime(), PlannerConfig(mode="agg")) + assert seen == [] + await planner_main.start_planner(_StartRuntime(), _live_config()) + assert seen == [(True, "_StartRuntime")] + + +def test_hooks_are_frozen(): + hooks = _hooks() + with pytest.raises(dataclasses.FrozenInstanceError): + hooks.phase_s = 1.0 # type: ignore[misc] diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py new file mode 100644 index 000000000000..7ce2be19ba81 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py @@ -0,0 +1,619 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the retune imports. + +"""Configuration premises of live retune (``retune.executor: dgdr``). + +P1 (Kubernetes, decision latency, explicit timeout within the interval), P2 +(retune-only, router only), P3 (the DGDR router knobs), P5 (a pinned engine +section) and P13 (the largest grid shard fits the sweep deadline), plus the +shard-split and need-seconds helpers the executor shares with validation. +""" + +from __future__ import annotations + +import copy +import math + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from pydantic import ValidationError + +from dynamo.planner.config.planner_config import ( + DGDR_MAX_SHARD_POINTS, + DgdrExecutorConfig, + PlannerConfig, + balanced_shard_parts, + dgdr_need_seconds, + largest_shard_points, +) +from dynamo.planner.retune.domains import expand_router_domain + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +PRESET = { + "tensor": 2, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, +} +SCHEDULER = {"max_batched_tokens": 8192, "max_sequences": 256} + + +def _worker(**overrides): + return { + "parallelism": {"preset": [dict(PRESET)]}, + "scheduler": dict(SCHEDULER), + "kv_cache": {"block_size": 64, "capacity": {"memory_fraction": 0.9}}, + **overrides, + } + + +ENGINE_AGG = { + "mode": "aggregated", + "backend": "vllm", + "backend_version": "0.11.0", + "context_length": 32768, + "workers": {"aggregated": _worker(), "prefill": None, "decode": None}, +} +ENGINE_DISAGG = { + "mode": "disaggregated", + "backend": "vllm", + "backend_version": {"vllm": "0.11.0"}, + "context_length": "max", + "workers": {"prefill": _worker(), "decode": _worker()}, +} +# Design §4.1 defaults: T = 6 h, timeout = T - 20 min. +INTERVAL = 21600.0 +TIMEOUT = 20400.0 +# The default router domain is 3 x 9 x 4 = 108 points; with max_shards 3 the +# largest shard has 36: 900 + 300 + (1 + ceil(35 / 8)) x 1800 = 12000 s. +DEFAULT_NEED = 12000.0 + + +def _dgdr(**overrides): + return { + "claim_name": "request-traces", + "model": "Qwen/Qwen3-32B", + "sku": "h200_sxm", + "engine": copy.deepcopy(ENGINE_AGG), + **overrides, + } + + +def _retune(**overrides): + return { + "interval_seconds": INTERVAL, + "timeout_seconds": TIMEOUT, + "router": "default", + "executor": "dgdr", + "dgdr": _dgdr(), + **overrides, + } + + +def _config(retune=None, **overrides) -> PlannerConfig: + values = { + "namespace": "test-ns", + "environment": "kubernetes", + "mode": "agg", + "enable_load_scaling": False, + "enable_throughput_scaling": False, + "retune": _retune() if retune is None else retune, + **overrides, + } + return PlannerConfig(**values) + + +def _rejects(match: str, retune=None, **overrides) -> None: + with pytest.raises(ValidationError, match=match): + _config(retune, **overrides) + + +def _engine(mode: str = "agg") -> dict: + return copy.deepcopy(ENGINE_AGG if mode == "agg" else ENGINE_DISAGG) + + +# ------------------------------------------------------------------ valid + + +@pytest.mark.parametrize("mode", ["agg", "disagg"]) +def test_live_retune_config_accepts_the_design_example(mode): + retune = _retune(dgdr=_dgdr(engine=_engine(mode))) + config = _config(retune, mode=mode) + + assert config.retune is not None + assert config.retune.executor == "dgdr" + assert config.retune.dgdr is not None + assert config.retune.dgdr.engine == _engine(mode) + # Validation never rewrites the sections it checks. + assert config.retune.router == "default" + + +def test_live_retune_accepts_a_measured_decision_latency_at_scale_one(): + config = _config(_retune(decision_latency_seconds="measured")) + assert config.retune is not None + assert config.retune.decision_latency_seconds == "measured" + + +def test_live_retune_accepts_the_largest_timeout_the_interval_allows(): + config = _config(_retune(timeout_seconds=INTERVAL - 600)) + assert config.retune is not None + + +def test_live_retune_accepts_the_three_dgdr_router_knobs_and_none_load_model(): + router = { + "overlap_score_credit": {"choices": [0.0, 1.0]}, + "prefill_load_scale": {"range": {"min": 0.5, "max": 2.0, "step": 0.5}}, + "temperature": 0.0, + "policy": "kv_router", + "prefill_load_model": {"type": "none"}, + } + assert _config(_retune(router=router)).retune is not None + router["prefill_load_model"] = {"type": {"choices": ["none"]}} + assert _config(_retune(router=router)).retune is not None + + +def test_replay_retune_keeps_its_rules(): + # executor local: none of the live premises apply. + config = _config( + {"interval_seconds": 600, "router": "default"}, + environment="virtual", + ) + assert config.retune is not None + assert config.retune.executor == "local" + + +# --------------------------------------------------------------------- P1 + + +def test_p1_requires_kubernetes(): + _rejects(r"environment='kubernetes' \(P1\)", environment="virtual") + + +@pytest.mark.parametrize("latency", [30.0, 0.5]) +def test_p1_rejects_a_fixed_nonzero_decision_latency(latency): + _rejects( + r"decision_latency_seconds must be 0 or 'measured' \(P1\)", + _retune(decision_latency_seconds=latency), + ) + + +def test_p1_rejects_a_scaled_measured_decision_latency(): + # Valid for RetuneConfig (2 x 10000 <= 21600), rejected by P1 before the + # time budget is checked. + _rejects( + r"decision_latency_scale 1 \(P1\)", + _retune( + decision_latency_seconds="measured", + decision_latency_scale=2.0, + timeout_seconds=10000.0, + ), + ) + + +def test_p1_requires_an_explicit_timeout(): + retune = _retune() + del retune["timeout_seconds"] + _rejects(r"explicit retune.timeout_seconds \(P1\)", retune) + + +def test_p1_rejects_a_timeout_within_ten_minutes_of_the_interval(): + _rejects( + r"timeout_seconds must be <= interval_seconds - 600 = 21000 .*\(P1\)", + _retune(timeout_seconds=INTERVAL - 599), + ) + + +# --------------------------------------------------------------------- P2 + + +@pytest.mark.parametrize( + "scaling", + [ + {"enable_load_scaling": True, "enable_throughput_scaling": False}, + { + "enable_load_scaling": False, + "enable_throughput_scaling": True, + "optimization_target": "sla", + }, + ], +) +def test_p2_requires_retune_only(scaling): + _rejects(r"retune-only mode \(P2\)", **scaling) + + +def test_p2_requires_both_scaling_flags_explicitly(): + # Without both flags written, easy mode keeps load scaling on. + with pytest.raises(ValidationError, match=r"retune-only mode \(P2\)"): + PlannerConfig( + namespace="test-ns", + environment="kubernetes", + mode="agg", + enable_throughput_scaling=False, + retune=_retune(), + ) + + +def test_p2_rejects_a_planner_domain(): + _rejects( + r"retunes the router only \(P2\)", + _retune(planner={"load_sensitivity": {"preset": ["default"]}}), + ) + + +# --------------------------------------------------------------------- P3 + + +@pytest.mark.parametrize( + "knob", ["overlap_score_credit_decay", "decode_active_request_weight"] +) +def test_p3_rejects_retune_extension_knobs(knob): + _rejects( + rf"retune.router \['{knob}'\] cannot be searched with executor 'dgdr' \(P3\)", + _retune(router={knob: {"choices": [0.0, 1.0]}}), + ) + + +@pytest.mark.parametrize( + "load_model", + [{"type": "ais"}, {"type": {"choices": ["none", "ais"]}}], +) +def test_p3_requires_the_none_prefill_load_model(load_model): + _rejects( + r"prefill_load_model.type must be 'none' with executor 'dgdr' \(P3\)", + _retune(router={"temperature": 0.0, "prefill_load_model": load_model}), + ) + + +def test_p3_keeps_the_schema_errors_for_other_policies(): + # RouterRetuneConfig already pins the policy; its error stays first. + _rejects( + r"retune.router.policy is fixed at startup", + _retune(router={"policy": "round_robin"}), + ) + + +# --------------------------------------------------------------------- P5 + + +def _rejects_engine(match: str, engine: dict, mode: str = "agg") -> None: + _rejects(match, _retune(dgdr=_dgdr(engine=engine)), mode=mode) + + +@pytest.mark.parametrize( + ("path", "value", "match"), + [ + ( + ("workers", "aggregated", "scheduler", "max_sequences"), + {"choices": [128, 256]}, + r"engine.workers.aggregated.scheduler.max_sequences is a search " + r"domain \(choices\); .* \(P5\)", + ), + ( + ("workers", "aggregated", "kv_cache", "capacity", "memory_fraction"), + {"range": {"min": 0.8, "max": 0.9}}, + r"memory_fraction is a search domain \(range\)", + ), + ( + ("backend",), + {"choices": ["vllm", "sglang"]}, + r"engine.backend is a search domain", + ), + ( + ("workers", "aggregated", "timing", "systems_paths"), + ["/opt/systems"], + r"engine.workers.aggregated.timing.systems_paths is a list; .* \(P5\)", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "replicas"), + 2, + r"engine.workers.aggregated.parallelism.preset\[0\].replicas must " + r"not be set: the DGD spec provides replicas at each launch \(P5\)", + ), + ( + ("workers", "aggregated", "parallelism", "replicas"), + 2, + r"parallelism.replicas must not be set: the DGD spec", + ), + ( + ("workers", "aggregated", "parallelism", "preset"), + [dict(PRESET), dict(PRESET, tensor=4)], + r"parallelism.preset must be a list of exactly one mapping .* \(P5\)", + ), + ( + ("workers", "aggregated", "parallelism", "preset"), + "default", + r"parallelism.preset must be a list of exactly one mapping", + ), + ( + ("workers", "aggregated", "parallelism", "tensor"), + 2, + r"parallelism sets \['tensor'\] next to preset", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "tensor"), + True, + r"preset\[0\].tensor must be a positive integer \(P5\)", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "expert"), + 1, + r"preset\[0\] must set exactly .*unknown=\['expert'\]", + ), + ( + ("workers", "aggregated", "scheduler", "max_batched_tokens"), + None, + r"scheduler.max_batched_tokens must be a positive integer \(P5\); " + r"omitted, AISimulate searches its default domain", + ), + ( + ("mode",), + "disaggregated", + r"engine.mode must be 'aggregated' for planner mode 'agg' \(P5\)", + ), + ( + ("model",), + "Qwen/Qwen3-32B", + r"engine.model would override the Request's typed field \(P5\); " + r"set retune.dgdr.model", + ), + ( + ("hardware",), + "h200_sxm", + r"set retune.dgdr.sku instead", + ), + ( + ("workers", "decode"), + _worker(), + r"engine.workers.decode is not a worker role of planner mode 'agg'", + ), + ( + ("workers", "encoder"), + {"tensor": 1}, + r"engine.workers.encoder is not a worker role", + ), + ], +) +def test_p5_rejects_an_unpinned_engine(path, value, match): + engine = _engine() + target = engine + for key in path[:-1]: + target = target.setdefault(key, {}) if isinstance(key, str) else target[key] + target[path[-1]] = value + _rejects_engine(match, engine) + + +def test_p5_rejects_a_missing_preset_knob(): + engine = _engine() + del engine["workers"]["aggregated"]["parallelism"]["preset"][0]["moe_expert"] + _rejects_engine(r"missing=\['moe_expert'\]", engine) + + +def test_p5_requires_mode(): + engine = _engine() + del engine["mode"] + _rejects_engine(r"an omitted mode searches both", engine) + + +def test_p5_requires_every_role_of_the_planner_mode(): + engine = _engine("disagg") + del engine["workers"]["decode"] + _rejects_engine( + r"engine.workers.decode is required for planner mode 'disagg' \(P5\)", + engine, + mode="disagg", + ) + + +def test_p5_requires_the_disaggregated_engine_mode_in_disagg(): + engine = _engine("disagg") + engine["mode"] = "aggregated" + _rejects_engine( + r"engine.mode must be 'disaggregated' for planner mode 'disagg'", + engine, + mode="disagg", + ) + + +def test_p5_engine_backend_must_be_the_deployed_backend(): + engine = _engine() + engine["backend"] = "sglang" + # PlannerConfig.backend defaults to vllm; every Run would fail its build. + _rejects_engine( + r"retune.dgdr.engine.backend 'sglang' differs from the deployed " + r"backend 'vllm' \(P5\)", + engine, + ) + config = _config(_retune(dgdr=_dgdr(engine=engine)), backend="sglang") + assert config.retune is not None and config.retune.dgdr is not None + assert config.retune.dgdr.engine["backend"] == "sglang" + _rejects( + r"retune.dgdr.engine.backend 'vllm' differs from the deployed backend " + r"'trtllm' \(P5\)", + _retune(dgdr=_dgdr(engine=_engine())), + backend="trtllm", + ) + + +def test_p5_accepts_an_omitted_engine_backend(): + # The Request fills it with the deployed backend. + engine = _engine() + del engine["backend"] + for backend in ("vllm", "sglang"): + config = _config( + _retune(dgdr=_dgdr(engine=copy.deepcopy(engine))), backend=backend + ) + assert config.retune is not None + + +def test_live_retune_accepts_nested_claim_directories(): + dgdr = _dgdr(segments_dir="retune/segments", windows_dir="retune/windows") + assert _config(_retune(dgdr=dgdr)).retune is not None + + +def test_p5_accepts_unused_roles_absent_or_null(): + engine = _engine() + del engine["workers"]["prefill"] + engine["workers"]["encoder"] = None + assert _config(_retune(dgdr=_dgdr(engine=engine))).retune is not None + + +def test_p5_accepts_the_context_parallel_preset_knobs(): + engine = _engine() + engine["workers"]["aggregated"]["parallelism"]["preset"][0]["decode_context"] = 1 + assert _config(_retune(dgdr=_dgdr(engine=engine))).retune is not None + + +# -------------------------------------------------------------- claim paths + + +@pytest.mark.parametrize( + ("overrides", "match"), + [ + ({"segments_dir": "/trace/segments"}, r"segments_dir must be a relative"), + ({"windows_dir": "../windows"}, r"windows_dir must be a relative"), + ({"windows_dir": "."}, r"windows_dir must be a relative"), + ({"windows_dir": "a/./windows"}, r"windows_dir must be a relative"), + ({"windows_dir": "windows/"}, r"windows_dir must be a relative"), + ({"segments_dir": "a//segments"}, r"segments_dir must be a relative"), + ({"mount_path": "trace"}, r"mount_path must be absolute"), + ({"windows_dir": "segments"}, r"must not contain one another"), + ({"windows_dir": "segments/windows"}, r"must not contain one another"), + ], +) +def test_live_retune_rejects_unsafe_claim_paths(overrides, match): + _rejects(match, _retune(dgdr=_dgdr(**overrides))) + + +# -------------------------------------------------------------------- P13 + + +def test_p13_accepts_a_shard_that_exactly_fits(): + assert _config(_retune(timeout_seconds=DEFAULT_NEED)).retune is not None + + +def test_p13_rejects_a_largest_shard_that_cannot_finish(): + _rejects( + r"largest retune grid shard \(36 points\) needs 12000s: " + r"cut_budget_seconds 900 \+ job_startup_seconds 300 \+ 6 rounds x " + r"candidate_timeout_seconds 1800, more than " + r"retune.timeout_seconds=11999 \(P13\)", + _retune(timeout_seconds=DEFAULT_NEED - 1), + ) + + +def test_p13_counts_the_largest_shard_of_fewer_max_shards(): + # max_shards 2: 3 x 9 x 2 = 54 points -> 1 + ceil(53 / 8) = 8 rounds. + dgdr = _dgdr(run={"max_shards": 2}) + need = 900 + 300 + 8 * 1800 + assert _config(_retune(dgdr=dgdr, timeout_seconds=need)).retune is not None + _rejects(r"\(54 points\)", _retune(dgdr=dgdr, timeout_seconds=need - 1)) + + +def test_p13_rejects_a_grid_that_needs_more_shards(): + _rejects( + r"router grid of 108 points cannot be split into at most " + r"retune.dgdr.run.max_shards=1 Cartesian shards of at most 100 points " + r"\(P13\); the largest would have 108", + _retune(dgdr=_dgdr(run={"max_shards": 1})), + ) + + +def test_p13_uses_the_configured_run_parallelism_and_timeouts(): + dgdr = _dgdr( + cut_budget_seconds=60, + job_startup_seconds=0, + run={"parallelism": 36, "candidate_timeout_seconds": 100}, + ) + # 60 + 0 + (1 + ceil(35 / 36)) x 100 = 260. + assert _config(_retune(dgdr=dgdr, timeout_seconds=260)).retune is not None + _rejects(r"needs 260s", _retune(dgdr=dgdr, timeout_seconds=259)) + + +# --------------------------------------------------------------- helpers + + +@pytest.mark.parametrize( + ("sizes", "max_shards", "parts", "largest"), + [ + ((3, 9, 4), 3, (1, 3, 1), 36), + ((3, 9, 4), 2, (1, 1, 2), 54), + ((3, 9, 4), 1, (1, 1, 1), 108), + # 15 shards of 1 x 2 x 4 beat 12 of 1 x 9 x 1. + ((3, 9, 4), 16, (3, 5, 1), 8), + ((10,), 3, (3,), 4), + ((2, 2), 16, (2, 2), 1), + ((), 3, (), 1), + ], +) +def test_balanced_shard_parts(sizes, max_shards, parts, largest): + assert balanced_shard_parts(sizes, max_shards) == parts + assert largest_shard_points(sizes, parts) == largest + assert math.prod(parts) <= max_shards + + +def test_balanced_shard_parts_minimizes_the_largest_shard_exhaustively(): + for sizes in ((3, 9, 4), (5, 7), (2, 3, 5), (12,), (1, 1, 30)): + for max_shards in range(1, 17): + parts = balanced_shard_parts(sizes, max_shards) + best = min( + largest_shard_points(sizes, (a, b, c)[: len(sizes)]) + for a in range(1, max_shards + 1) + for b in range(1, max_shards + 1) + for c in range(1, max_shards + 1) + if math.prod((a, b, c)[: len(sizes)]) <= max_shards + and all(p <= n for p, n in zip((a, b, c)[: len(sizes)], sizes)) + ) + assert largest_shard_points(sizes, parts) == best + + +@pytest.mark.parametrize( + ("sizes", "max_shards"), + [((3, 0), 3), ((3,), 0)], +) +def test_balanced_shard_parts_rejects_bad_input(sizes, max_shards): + with pytest.raises(ValueError): + balanced_shard_parts(sizes, max_shards) + + +def test_default_router_grid_splits_into_three_shards_of_36(): + retune = _config().retune + assert retune is not None + sizes = [len(values) for values in expand_router_domain(retune).values()] + assert sizes == [3, 9, 4] + parts = balanced_shard_parts(sizes, 3) + assert largest_shard_points(sizes, parts) == 36 <= DGDR_MAX_SHARD_POINTS + + +@pytest.mark.parametrize( + ("points", "needs_cut", "expected"), + [ + (1, True, 900 + 300 + 1 * 1800), + (1, False, 300 + 1 * 1800), + (9, True, 900 + 300 + 2 * 1800), + (10, True, 900 + 300 + 3 * 1800), + (36, False, 300 + 6 * 1800), + ], +) +def test_dgdr_need_seconds(points, needs_cut, expected): + dgdr = DgdrExecutorConfig.model_validate(_dgdr()) + assert dgdr_need_seconds(points, dgdr, needs_cut=needs_cut) == expected + + +def test_dgdr_need_seconds_rejects_an_empty_shard(): + dgdr = DgdrExecutorConfig.model_validate(_dgdr()) + with pytest.raises(ValueError): + dgdr_need_seconds(0, dgdr, needs_cut=True) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py new file mode 100644 index 000000000000..b02e5297c3d3 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py @@ -0,0 +1,1236 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the controller imports the sweep. + +"""Live retune hooks of ``RetuneController`` with a scripted sweep function. + +Covers what a live host adds on top of the replay loop: epoch-phase +anchoring, the launch gate, harvest gating and abandoned sweeps, +``drop_pending``/``rewind``, the predecessor generation's sweep, the DGD +spec replicas check, ``restore_desired`` and the structured failure reason +of a decision. +""" + +from __future__ import annotations + +import json +import threading +from collections import deque +from concurrent.futures import Future + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +import dynamo.planner.retune.controller as controller_module +from dynamo.llm import KvRouterConfig +from dynamo.planner.config.planner_config import PlannerConfig, RetuneEngineSpec +from dynamo.planner.core.types import ( + RetuneDecision, + RetuneInputs, + RetuneParams, + WorkerCapabilities, + WorkerCounts, +) +from dynamo.planner.plugins.clock import VirtualClock, WallClock +from dynamo.planner.retune.controller import RetuneController +from dynamo.planner.retune.live.contracts import NON_DECISION_FAILURES, GateResult +from dynamo.planner.retune.sweep import SweepResult + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +INTERVAL = 60.0 +GRACE = 5.0 +STEP = 5.0 +# An epoch-like planner time on a window boundary (phase 0): W_K = [T0-60, T0). +T0 = 1_800_000_000.0 +K = int(T0 // INTERVAL) - 1 +L_K = T0 + GRACE +ROUTER = { + "overlap_score_credit": {"choices": [0.0, 1.0]}, + "prefill_load_scale": 1.0, + "temperature": {"choices": [0.0, 0.5]}, +} +ROUTER_DOMAIN = { + "overlap_score_credit": (0.0, 1.0), + "prefill_load_scale": (1.0,), + "router_temperature": (0.0, 0.5), +} +SPEC = {"aggregated": 2} + + +def _point(credit, temperature): + return RetuneParams( + router={ + "overlap_score_credit": credit, + "prefill_load_scale": 1.0, + "router_temperature": temperature, + } + ) + + +STARTUP = _point(1.0, 0.0) +BEST = _point(0.0, 0.5) + + +class FakeSweeps: + """Scripted sweep function: records each call, returns queued results.""" + + def __init__(self): + self.requests = [] + self.cancels = [] + self.aborts = [] + self.results = deque() + self.default = None + + def __call__(self, request, *, cancel=None, abort=None): + self.requests.append(request) + self.cancels.append(cancel) + self.aborts.append(abort) + result = self.results.popleft() if self.results else self.default + return result(request) if callable(result) else result + + +class Gate: + """Scripted ``launch_gate``: a queue of results, then ``default``.""" + + def __init__(self, *results, default=None): + self.results = deque(results) + self.default = default if default is not None else GateResult() + self.calls = [] + + def __call__(self, k, window): + self.calls.append((k, window)) + result = self.results.popleft() if self.results else self.default + if isinstance(result, Exception): + raise result + return result + + +class Harvest: + """Scripted ``harvest_allowed``.""" + + def __init__(self, *answers, default=True): + self.answers = deque(answers) + self.default = default + self.calls = [] + + def __call__(self, generation): + self.calls.append(generation) + return self.answers.popleft() if self.answers else self.default + + +class ManualClock(WallClock): + """A wall clock (no inline await) whose monotonic time tests set.""" + + def __init__(self): + self.mono = 0.0 + + def monotonic(self) -> float: + return self.mono + + +def _scored(score_of, *, wall_s=0.5, **extra): + def build(request): + return SweepResult( + status="ok", + num_requests=10, + scores={c.key(): score_of(c) for c in request.candidates}, + wall_s=wall_s, + **extra, + ) + + return build + + +def _flat(score=1.0, **extra): + return _scored(lambda _c: score, **extra) + + +def _config(**retune): + return PlannerConfig( + namespace="test-ns", + mode="agg", + enable_load_scaling=False, + enable_throughput_scaling=False, + retune={ + "interval_seconds": INTERVAL, + "grace_seconds": GRACE, + "router": ROUTER, + **retune, + }, + ) + + +def _inputs(origin_ms=0.0): + return RetuneInputs( + engine=RetuneEngineSpec( + router_mode="kv_router", + router_config=json.loads(KvRouterConfig().to_json()), + ), + trace_paths=("/trace/segments/dgd",), + trace_origin_ms=origin_ms, + ) + + +def _live( + sweeps, + *, + gate=None, + harvest=None, + clock=None, + generation=3, + start=T0, + phase=0.0, + config=None, + predecessor=None, +): + controller = RetuneController( + config or _config(), + _inputs(), + WorkerCapabilities(), + clock or VirtualClock(), + STEP, + sweep_fn=sweeps, + launch_gate=( + gate if gate is not None else Gate(default=GateResult(spec_replicas=SPEC)) + ), + harvest_allowed=harvest, + generation=generation, + predecessor=predecessor, + ) + controller.start(start, phase) + return controller + + +def _counts(decode=2): + return WorkerCounts(ready_num_decode=decode) + + +async def _tick(controller, now, decode=2): + return await controller.on_tick(now, _counts(decode)) + + +def _window_ms(k): + return k * INTERVAL * 1000.0, (k + 1) * INTERVAL * 1000.0 + + +# --------------------------------------------------------------------------- +# Construction and properties +# --------------------------------------------------------------------------- + + +def test_launch_gate_requires_a_sweep_fn(): + with pytest.raises(ValueError, match="launch_gate requires a sweep_fn"): + RetuneController( + _config(), + _inputs(), + WorkerCapabilities(), + VirtualClock(), + STEP, + launch_gate=Gate(), + ) + + +def test_generation_router_keys_and_domain_are_exposed(): + controller = _live(FakeSweeps(), generation=7) + try: + assert controller.generation == 7 + assert controller.router_keys == ( + "overlap_score_credit", + "prefill_load_scale", + "router_temperature", + ) + assert controller.router_domain == { + key: list(values) for key, values in ROUTER_DOMAIN.items() + } + # A copy: callers cannot change the controller's domain. + controller.router_domain["router_temperature"].append(9.0) + assert controller.router_domain["router_temperature"] == [0.0, 0.5] + finally: + controller.close() + + +# --------------------------------------------------------------------------- +# Phase anchoring +# --------------------------------------------------------------------------- + + +async def test_phase_anchoring_gives_restart_stable_windows_and_ids(): + sweeps = FakeSweeps() + sweeps.default = _flat() + # Two processes started at different times share the window schedule. + first = _live(sweeps, start=T0 - 7.0) + second = _live(sweeps, start=T0 + 3.0) + try: + for controller in (first, second): + await _tick(controller, L_K) + result = await _tick(controller, L_K + STEP) + (decision,) = result.decisions + start_ms, end_ms = _window_ms(K) + assert decision.window_index == K + assert decision.decision_id == f"retune-w{int(end_ms)}" + assert (decision.window_start_s, decision.window_end_s) == ( + T0 - INTERVAL, + T0, + ) + first_request, second_request = sweeps.requests + assert (first_request.window_start_ms, first_request.window_end_ms) == ( + start_ms, + end_ms, + ) + assert first_request.as_of_ms == (T0 + GRACE) * 1000.0 + assert (second_request.window_start_ms, second_request.window_end_ms) == ( + start_ms, + end_ms, + ) + finally: + first.close() + second.close() + + +async def test_replay_start_keeps_its_decision_ids(monkeypatch): + sweeps = FakeSweeps() + sweeps.default = _flat() + monkeypatch.setattr(controller_module, "run_sweep", sweeps) + controller = RetuneController( + _config(), _inputs(), WorkerCapabilities(), VirtualClock(), STEP + ) + controller.start(10.0) + try: + await _tick(controller, 75.0) + (decision,) = (await _tick(controller, 80.0)).decisions + finally: + controller.close() + + assert decision.decision_id == "retune-10-w0" + (request,) = sweeps.requests + # Replay requests keep the live-only fields at their defaults. + assert request.router_domain is None + assert request.deadline_s is None + assert request.spec_replicas is None + assert request.replicas_verified is True + assert request.generation == 0 + assert sweeps.aborts == [None] + + +def test_phase_must_be_finite(): + controller = _live(FakeSweeps()) + try: + with pytest.raises(ValueError, match="phase_s must be finite"): + controller.start(T0, float("nan")) + finally: + controller.close() + + +# --------------------------------------------------------------------------- +# Launch gate +# --------------------------------------------------------------------------- + + +async def test_gated_launch_passes_abort_and_live_request_fields(): + sweeps = FakeSweeps() + sweeps.default = _flat() + abort = threading.Event() + gate = Gate(GateResult(spec_replicas=SPEC, abort=abort)) + controller = _live(sweeps, gate=gate, generation=4) + try: + result = await _tick(controller, L_K) + finally: + controller.close() + + assert result.audit_events == [f"retune:launch w={K}"] + assert gate.calls == [(K, (T0 - INTERVAL, T0, T0 + GRACE))] + (request,) = sweeps.requests + assert sweeps.aborts == [abort] + assert sweeps.cancels == [controller._cancel] + assert request.router_domain == ROUTER_DOMAIN + # D_k = L_k + timeout (timeout defaults to the interval). + assert request.deadline_s == L_K + INTERVAL + assert request.spec_replicas == SPEC + assert request.replicas == SPEC + # Started at the window end: no recorded count covers the window start. + assert request.replicas_verified is False + assert request.generation == 4 + assert request.trace_paths == ("/trace/segments/dgd",) + + +@pytest.mark.parametrize("reason", ["not_leader", "not_ready"]) +async def test_retry_reasons_hold_the_window_until_it_may_launch(reason): + sweeps = FakeSweeps() + sweeps.default = _flat() + gate = Gate( + GateResult(reason=reason), + GateResult(reason=reason), + default=GateResult(spec_replicas=SPEC), + ) + controller = _live(sweeps, gate=gate) + try: + held = [await _tick(controller, L_K + i * STEP) for i in range(2)] + launched = await _tick(controller, L_K + 2 * STEP) + decided = await _tick(controller, L_K + 3 * STEP) + finally: + controller.close() + + for result in held: + assert result.decisions == [] and result.audit_events == [] + assert [k for k, _ in gate.calls] == [K, K, K] + assert launched.audit_events == [f"retune:launch w={K}"] + assert [d.window_index for d in decided.decisions] == [K] + assert decided.decisions[0].launched_at_s == L_K + 2 * STEP + assert len(sweeps.requests) == 1 + + +async def test_already_decided_consumes_the_window_without_a_decision(): + sweeps = FakeSweeps() + sweeps.default = _flat() + gate = Gate( + GateResult(reason="already_decided"), default=GateResult(spec_replicas=SPEC) + ) + controller = _live(sweeps, gate=gate) + try: + consumed = await _tick(controller, L_K) + quiet = await _tick(controller, L_K + STEP) + next_window = await _tick(controller, L_K + INTERVAL) + finally: + controller.close() + + assert consumed.decisions == [] and consumed.audit_events == [] + assert quiet.decisions == [] and quiet.audit_events == [] + # Asked once for K (consumed), then for K + 1 at its launch time. + assert [k for k, _ in gate.calls] == [K, K + 1] + assert next_window.audit_events == [f"retune:launch w={K + 1}"] + assert [r.window_index for r in sweeps.requests] == [K + 1] + + +async def test_gate_runs_before_the_busy_branch(): + release = threading.Event() + sweeps = FakeSweeps() + + def blocking(request): + release.wait(30) + return _flat()(request) + + sweeps.default = blocking + gate = Gate( + GateResult(spec_replicas=SPEC), + GateResult(reason="not_leader"), + default=GateResult(spec_replicas=SPEC), + ) + controller = _live(sweeps, gate=gate, clock=WallClock()) + try: + await _tick(controller, L_K) + held = await _tick(controller, L_K + INTERVAL) + busy = await _tick(controller, L_K + INTERVAL + STEP) + finally: + release.set() + controller.close() + + # A window this process may not work on is not recorded busy. + assert held.decisions == [] + assert [(d.outcome, d.window_index) for d in busy.decisions] == [("busy", K + 1)] + assert busy.decisions[0].failure_reason is None + assert len(sweeps.requests) == 1 + + +@pytest.mark.parametrize( + "bad", + [RuntimeError("supervisor bug"), GateResult(reason="bogus")], # type: ignore[arg-type] + ids=["raises", "unknown-reason"], +) +async def test_a_failing_gate_holds_the_window(bad): + sweeps = FakeSweeps() + sweeps.default = _flat() + gate = Gate(bad, default=GateResult(spec_replicas=SPEC)) + controller = _live(sweeps, gate=gate) + try: + held = await _tick(controller, L_K) + launched = await _tick(controller, L_K + STEP) + finally: + controller.close() + + assert held.decisions == [] and held.audit_events == [] + assert launched.audit_events == [f"retune:launch w={K}"] + + +# --------------------------------------------------------------------------- +# Harvest gating and abandoned sweeps +# --------------------------------------------------------------------------- + + +async def test_refused_harvest_drops_the_result_and_relaunches_the_window(): + sweeps = FakeSweeps() + sweeps.default = _flat() + harvest = Harvest(False) + controller = _live(sweeps, harvest=harvest, generation=5) + try: + await _tick(controller, L_K) + refused = await _tick(controller, L_K + STEP) + decided = await _tick(controller, L_K + 2 * STEP) + finally: + controller.close() + + assert harvest.calls == [5, 5] + # No decision; the window is due again and relaunches in the same tick. + assert refused.decisions == [] + assert refused.audit_events == [f"retune:dropped w={K}", f"retune:launch w={K}"] + assert [r.window_index for r in sweeps.requests] == [K, K] + assert [(d.outcome, d.window_index) for d in decided.decisions] == [("kept", K)] + + +async def test_harvest_allowed_that_raises_counts_as_refused(): + sweeps = FakeSweeps() + sweeps.default = _flat() + + def harvest(_generation): + raise RuntimeError("lease read failed") + + controller = _live(sweeps, harvest=harvest) + try: + await _tick(controller, L_K) + refused = await _tick(controller, L_K + STEP) + finally: + controller.close() + + assert refused.decisions == [] + assert f"retune:dropped w={K}" in refused.audit_events + + +@pytest.mark.parametrize("reason", sorted(NON_DECISION_FAILURES)) +async def test_non_decision_failures_record_nothing_and_rewind(reason): + sweeps = FakeSweeps() + sweeps.results.append( + SweepResult(status="failed", error="stopped", failure_reason=reason) + ) + sweeps.default = _flat() + controller = _live(sweeps) + try: + await _tick(controller, L_K) + abandoned = await _tick(controller, L_K + STEP) + decided = await _tick(controller, L_K + 2 * STEP) + finally: + controller.close() + + assert abandoned.decisions == [] + assert abandoned.audit_events == [f"retune:dropped w={K}", f"retune:launch w={K}"] + assert [(d.outcome, d.window_index) for d in decided.decisions] == [("kept", K)] + + +async def test_abandoned_window_is_not_relaunched_once_its_successor_is_due(): + sweeps = FakeSweeps() + sweeps.results.append(SweepResult(status="failed", failure_reason="shutdown")) + sweeps.default = _flat() + controller = _live(sweeps, clock=WallClock()) + try: + await _tick(controller, L_K) + controller._pending.future.result(timeout=30) + # Harvested only after the next window is due: K + 1 launches. + result = await _tick(controller, L_K + INTERVAL) + finally: + controller.close() + + assert result.audit_events == [f"retune:dropped w={K}", f"retune:launch w={K + 1}"] + + +@pytest.mark.parametrize( + "result, outcome", + [ + ( + SweepResult( + status="failed", + error="probe scores differ", + failure_reason="crn_mismatch", + details={"probe": {"expected": 1.0, "got": 1.5}}, + ), + "failed", + ), + ( + SweepResult( + status="timeout", + error="window deadline", + failure_reason="timeout", + details={"requests": ["a", "b"]}, + ), + "timeout", + ), + ( + _flat(failure_reason=None, details={"runtime_image": "ais:0.13"}), + "kept", + ), + ], + ids=["failed", "timeout", "kept"], +) +async def test_failure_reason_and_details_reach_the_decision(result, outcome): + sweeps = FakeSweeps() + sweeps.default = result + controller = _live(sweeps) + try: + await _tick(controller, L_K) + (decision,) = (await _tick(controller, L_K + STEP)).decisions + finally: + controller.close() + + expected = result if isinstance(result, SweepResult) else result(sweeps.requests[0]) + assert decision.outcome == outcome + assert decision.failure_reason == expected.failure_reason + assert decision.details == expected.details + record = decision.to_dict() + assert record["failure_reason"] == expected.failure_reason + assert record["details"] == expected.details + + +async def test_a_raising_sweep_fn_is_a_failed_decision_without_reason(): + def raises(_request, *, cancel, abort): + raise RuntimeError("kube unreachable") + + controller = _live(raises) + try: + await _tick(controller, L_K) + (decision,) = (await _tick(controller, L_K + STEP)).decisions + finally: + controller.close() + + assert decision.outcome == "failed" + assert decision.reason == "RuntimeError: kube unreachable" + assert decision.failure_reason is None + + +# --------------------------------------------------------------------------- +# drop_pending and rewind +# --------------------------------------------------------------------------- + + +async def test_drop_pending_and_rewind_relaunch_instead_of_busy(): + release = threading.Event() + sweeps = FakeSweeps() + + def blocking(request): + release.wait(30) + return _flat()(request) + + sweeps.results.append(blocking) + sweeps.default = _flat() + abort = threading.Event() + gate = Gate( + GateResult(spec_replicas=SPEC, abort=abort), + default=GateResult(spec_replicas=SPEC), + ) + controller = _live(sweeps, gate=gate, clock=WallClock()) + try: + await _tick(controller, L_K) + dropped = controller._pending.future + # A pending sweep keeps its window: rewind alone does nothing. + controller.rewind(K - 1) + assert controller._last_k == K + + controller.drop_pending() + assert abort.is_set() + assert controller._pending is None + controller.rewind(K - 1) + relaunch = await _tick(controller, L_K + STEP) + release.set() + dropped.result(timeout=30) + controller._pending.future.result(timeout=30) + # The relaunch queued behind the dropped sweep, so it launched (and + # becomes ready) a little after L_K + STEP. + decided = await _tick(controller, L_K + 3 * STEP) + finally: + release.set() + controller.close() + + assert relaunch.decisions == [] + assert relaunch.audit_events == [f"retune:launch w={K}"] + assert [r.window_index for r in sweeps.requests] == [K, K] + assert [(d.outcome, d.window_index) for d in decided.decisions] == [("kept", K)] + + +async def test_drop_pending_cancels_a_sweep_that_has_not_started(): + release = threading.Event() + sweeps = FakeSweeps() + + def blocking(request): + release.wait(30) + return _flat()(request) + + sweeps.default = blocking + controller = _live(sweeps, clock=WallClock()) + try: + # Occupy the sweep thread so the next submission stays queued. + blocker = controller._executor.submit(release.wait, 30) + await _tick(controller, L_K) + queued = controller._pending.future + controller.drop_pending() + assert queued.cancelled() + # Nothing holds the thread on its behalf: no queue delay to track. + assert controller._dropped is None + release.set() + blocker.result(timeout=30) + finally: + release.set() + controller.close() + + assert sweeps.requests == [] + + +def test_drop_pending_and_rewind_without_a_sweep(): + controller = _live(FakeSweeps()) + try: + controller.drop_pending() + controller._last_k = K + controller.rewind(K + 3) + assert controller._last_k == K + controller.rewind(K - 2) + assert controller._last_k == K - 2 + finally: + controller.close() + + +async def test_a_sweep_queued_behind_a_dropped_one_launches_when_it_starts(): + release = threading.Event() + sweeps = FakeSweeps() + + def blocking(request): + release.wait(30) + return _flat()(request) + + sweeps.results.append(blocking) + sweeps.default = _flat() + clock = ManualClock() + controller = _live(sweeps, clock=clock) + try: + clock.mono = 100.0 + await _tick(controller, L_K) + dropped = controller._pending.future + controller.drop_pending() + controller.rewind(K - 1) + await _tick(controller, L_K + STEP) # submitted at monotonic 100 + clock.mono = 107.0 # the dropped sweep returns 7 s later + release.set() + dropped.result(timeout=30) + controller._pending.future.result(timeout=30) + (decision,) = (await _tick(controller, L_K + 4 * STEP)).decisions + finally: + release.set() + controller.close() + + assert decision.launched_at_s == L_K + STEP + 7.0 + + +async def test_repeated_drops_keep_tracking_the_sweep_holding_the_thread(): + release = threading.Event() + sweeps = FakeSweeps() + + def blocking(request): + release.wait(30) + return _flat()(request) + + sweeps.results.append(blocking) + sweeps.default = _flat() + clock = ManualClock() + controller = _live(sweeps, clock=clock) + try: + clock.mono = 100.0 + await _tick(controller, L_K) + running = controller._pending.future + controller.drop_pending() + controller.rewind(K - 1) + await _tick(controller, L_K + STEP) # queued behind the running one + queued = controller._pending.future + controller.drop_pending() + assert queued.cancelled() + controller.rewind(K - 1) + await _tick(controller, L_K + 2 * STEP) # still behind the first one + clock.mono = 104.0 + release.set() + running.result(timeout=30) + controller._pending.future.result(timeout=30) + (decision,) = (await _tick(controller, L_K + 5 * STEP)).decisions + finally: + release.set() + controller.close() + + # The cancelled sweep never ran; the last one started when the first ended. + assert len(sweeps.requests) == 2 + assert decision.launched_at_s == L_K + 2 * STEP + 4.0 + + +# --------------------------------------------------------------------------- +# Predecessor generation and running_future +# --------------------------------------------------------------------------- + + +def _blocking_sweeps(release, started=None): + """``FakeSweeps`` whose first sweep runs until ``release`` is set.""" + + sweeps = FakeSweeps() + + def blocking(request): + if started is not None: + started.set() + release.wait(30) + return _flat()(request) + + sweeps.results.append(blocking) + sweeps.default = _flat() + return sweeps + + +async def test_a_first_launch_queues_behind_the_predecessor_generation(): + # The previous generation's sweep, still running on its own thread. + predecessor: Future = Future() + assert predecessor.set_running_or_notify_cancel() + ran = threading.Event() + sweeps = FakeSweeps() + + def record(request): + ran.set() + return _flat()(request) + + sweeps.default = record + clock = ManualClock() + controller = _live(sweeps, clock=clock, predecessor=predecessor) + try: + assert controller.running_future() is predecessor + clock.mono = 100.0 + launch = await _tick(controller, L_K) + assert launch.audit_events == [f"retune:launch w={K}"] + queued = controller._pending.future + # Waits for the predecessor; it holds no other thread, so it runs. + assert controller.running_future() is queued + assert not ran.wait(0.2) + assert sweeps.requests == [] + + clock.mono = 106.0 # the predecessor returns 6 s later + predecessor.set_result(SweepResult(status="failed")) + queued.result(timeout=30) + assert controller.running_future() is None + (decision,) = (await _tick(controller, L_K + 4 * STEP)).decisions + finally: + if not predecessor.done(): + predecessor.cancel() + controller.close() + + assert [r.window_index for r in sweeps.requests] == [K] + assert decision.launched_at_s == L_K + 6.0 + + +async def test_a_done_predecessor_does_not_delay_the_first_launch(): + predecessor: Future = Future() + predecessor.set_result(SweepResult(status="failed")) + sweeps = FakeSweeps() + sweeps.default = _flat() + controller = _live(sweeps, clock=WallClock(), predecessor=predecessor) + try: + assert controller.running_future() is None + await _tick(controller, L_K) + assert controller._pending.queued is None + controller._pending.future.result(timeout=30) + (decision,) = (await _tick(controller, L_K + 2 * STEP)).decisions + finally: + controller.close() + + assert decision.launched_at_s == L_K + + +async def test_a_dropped_launch_hands_the_predecessor_wait_on(): + # A launch dropped while it waits for the predecessor still holds this + # controller's thread: the relaunch, and a successor, queue behind it. + predecessor: Future = Future() + assert predecessor.set_running_or_notify_cancel() + sweeps = FakeSweeps() + sweeps.default = _flat() + controller = _live(sweeps, clock=WallClock(), predecessor=predecessor) + try: + await _tick(controller, L_K) + waiting = controller._pending.future + controller.drop_pending() + assert controller._dropped is waiting + assert controller.running_future() is waiting + controller.rewind(K - 1) + await _tick(controller, L_K + STEP) + relaunch = controller._pending.future + assert relaunch.running() is False + # The relaunch has not started: the waiting launch holds the thread. + assert controller.running_future() is waiting + + predecessor.set_result(SweepResult(status="failed")) + waiting.result(timeout=30) + relaunch.result(timeout=30) + assert controller.running_future() is None + finally: + if not predecessor.done(): + predecessor.cancel() + controller.close() + + assert [r.window_index for r in sweeps.requests] == [K, K] + + +async def test_running_future_tracks_the_sweep_holding_the_thread(): + release = threading.Event() + started = threading.Event() + sweeps = _blocking_sweeps(release, started) + controller = _live(sweeps, clock=WallClock()) + try: + assert controller.running_future() is None + await _tick(controller, L_K) + running = controller._pending.future + assert started.wait(30) + assert controller.running_future() is running + + controller.drop_pending() + assert controller.running_future() is running + controller.rewind(K - 1) + await _tick(controller, L_K + STEP) # queued behind the dropped one + queued = controller._pending.future + assert controller.running_future() is running + + release.set() + running.result(timeout=30) + queued.result(timeout=30) + assert controller.running_future() is None + finally: + release.set() + controller.close() + + +async def test_running_future_after_close_skips_a_cancelled_queued_sweep(): + # The host reads running_future after close: a queued sweep that close + # cancelled never runs, so only the running one is waited on. + release = threading.Event() + started = threading.Event() + sweeps = _blocking_sweeps(release, started) + controller = _live(sweeps, clock=WallClock()) + try: + await _tick(controller, L_K) + running = controller._pending.future + assert started.wait(30) + controller.drop_pending() + controller.rewind(K - 1) + await _tick(controller, L_K + STEP) + queued = controller._pending.future + controller.close() + assert queued.cancelled() + assert controller.running_future() is running + finally: + release.set() + controller.close() + running.result(timeout=30) + assert controller.running_future() is None + + +def test_predecessor_must_be_a_future(): + with pytest.raises(ValueError, match="predecessor must be"): + _live(FakeSweeps(), predecessor=object()) + + +async def test_generations_chained_by_predecessor_never_sweep_at_once(): + # Reviewer scenario: generation 0 is slow to see its abort while + # generation 1 relaunches the same window. Replacing generation 1 before + # it launched hands generation 0's sweep on to generation 2 as well. + lock = threading.Lock() + running: list[int] = [] + overlaps: list[list[int]] = [] + order: list[tuple[int, int]] = [] + release = threading.Event() + started = threading.Event() + + def sweep(request, *, cancel, abort): + with lock: + running.append(request.generation) + order.append((request.generation, request.window_index)) + if len(running) > 1: + overlaps.append(list(running)) + try: + if request.generation == 0: + started.set() + release.wait(30) + return SweepResult(status="failed", failure_reason="lost_leadership") + return _flat()(request) + finally: + with lock: + running.remove(request.generation) + + old = _live(sweep, clock=WallClock(), generation=0) + middle = new = None + try: + await _tick(old, L_K) + assert started.wait(30) + old.drop_pending() + old.close() + middle = _live( + sweep, clock=WallClock(), generation=1, predecessor=old.running_future() + ) + middle.close() + new = _live( + sweep, clock=WallClock(), generation=2, predecessor=middle.running_future() + ) + launched = await _tick(new, L_K + STEP) + assert launched.audit_events == [f"retune:launch w={K}"] + assert new._pending.queued is not None + relaunch = new._pending.future + assert not relaunch.done() + release.set() + relaunch.result(timeout=30) + finally: + release.set() + for controller in (old, middle, new): + if controller is not None: + controller.close() + + assert overlaps == [] + assert order == [(0, K), (2, K)] + + +# --------------------------------------------------------------------------- +# DGD spec replicas +# --------------------------------------------------------------------------- + + +async def _run_window(controller, counts_at, *, start, until): + """Tick from ``start`` to ``until``; return window K's decisions. + + Earlier windows are due from the first tick too; only K is asserted. + """ + + decisions = [] + now = start + while now <= until + 1e-9: + result = await controller.on_tick(now, _counts(counts_at(now))) + decisions.extend((now, d) for d in result.decisions if d.window_index == K) + now += STEP + return decisions + + +def _requests_for_k(sweeps): + return [request for request in sweeps.requests if request.window_index == K] + + +async def test_replicas_verified_when_the_counts_cover_the_window(): + sweeps = FakeSweeps() + sweeps.default = _flat() + start = T0 - INTERVAL - 2 * STEP + controller = _live(sweeps, start=start) + try: + decisions = await _run_window( + controller, lambda _now: 2, start=start, until=L_K + ) + finally: + controller.close() + + assert decisions == [] + (request,) = _requests_for_k(sweeps) + assert request.replicas_verified is True + assert request.replicas == request.spec_replicas == SPEC + + +async def test_a_ready_count_change_in_the_window_fails_it(): + sweeps = FakeSweeps() + sweeps.default = _flat() + start = T0 - INTERVAL - 2 * STEP + + def counts(now): + # One worker restarts for a tick in the middle of the window. + return 1 if now == T0 - INTERVAL / 2 else 2 + + controller = _live(sweeps, start=start) + try: + ((now, decision),) = await _run_window( + controller, counts, start=start, until=L_K + ) + finally: + controller.close() + + assert now == L_K + assert (decision.outcome, decision.failure_reason) == ("failed", "replicas_changed") + assert decision.details == { + "spec_replicas": SPEC, + "observed_replicas": {"aggregated": 1}, + } + assert decision.decision_id == f"retune-w{int(T0 * 1000)}" + assert decision.launched_at_s is None + assert _requests_for_k(sweeps) == [] + + +@pytest.mark.parametrize( + "counts, failed", + [ + # 3 ready before the window, 2 from the tick at its start: clean. + (lambda now: 3 if now < T0 - INTERVAL else 2, False), + # The tick at the window start still reports 3: that count was in + # effect when the window began, so it fails. + (lambda now: 3 if now <= T0 - INTERVAL else 2, True), + ], + ids=["changed-before-start", "changed-after-start"], +) +async def test_the_count_in_effect_at_window_start_must_match_too(counts, failed): + sweeps = FakeSweeps() + sweeps.default = _flat() + start = T0 - INTERVAL - 2 * STEP + controller = _live(sweeps, start=start) + try: + decisions = await _run_window(controller, counts, start=start, until=L_K) + finally: + controller.close() + + if failed: + assert [(d.outcome, d.failure_reason) for _, d in decisions] == [ + ("failed", "replicas_changed") + ] + assert _requests_for_k(sweeps) == [] + else: + assert decisions == [] + (request,) = _requests_for_k(sweeps) + assert request.replicas_verified is True + + +async def test_window_started_before_the_controller_launches_unverified(): + sweeps = FakeSweeps() + sweeps.default = _flat() + start = T0 - INTERVAL / 2 + controller = _live(sweeps, start=start) + try: + decisions = await _run_window( + controller, lambda _now: 2, start=start, until=L_K + ) + finally: + controller.close() + + assert decisions == [] + (request,) = _requests_for_k(sweeps) + assert request.replicas_verified is False + assert request.replicas == SPEC + + +async def test_unverified_window_still_fails_on_an_observed_change(): + sweeps = FakeSweeps() + sweeps.default = _flat() + start = T0 - INTERVAL / 2 + controller = _live(sweeps, start=start) + try: + decisions = await _run_window( + controller, lambda _now: 3, start=start, until=L_K + ) + finally: + controller.close() + + assert [(d.outcome, d.failure_reason) for _, d in decisions] == [ + ("failed", "replicas_changed") + ] + assert _requests_for_k(sweeps) == [] + + +# --------------------------------------------------------------------------- +# restore_desired +# --------------------------------------------------------------------------- + + +async def test_restore_desired_sets_the_deployed_point_but_not_startup(): + sweeps = FakeSweeps() + sweeps.default = _flat() + controller = _live(sweeps) + try: + controller.restore_desired( + {**BEST.router, "decode_active_request_weight": 4.0} # extra: ignored + ) + assert controller.current_point() == BEST + result = await _tick(controller, L_K) + finally: + controller.close() + + expected = KvRouterConfig.from_json( + json.dumps({**json.loads(KvRouterConfig().to_json()), **BEST.router}) + ).policy_parameters("aggregated") + assert result.router_parameters == {"aggregated": dict(expected)} + (request,) = sweeps.requests + # The restored point is the baseline; startup stays a candidate. + assert request.baseline == BEST + assert STARTUP in request.candidates + + +@pytest.mark.parametrize( + "flat, match", + [ + ( + {"overlap_score_credit": 0.0, "prefill_load_scale": 1.0}, + "router_temperature", + ), + ({**BEST.router, "router_temperature": -1.0}, "non-negative"), + ({**BEST.router, "router_temperature": float("inf")}, "finite"), + ], + ids=["missing-key", "negative", "infinite"], +) +def test_restore_desired_rejects_bad_values_and_keeps_the_desired_point(flat, match): + controller = _live(FakeSweeps()) + try: + with pytest.raises(ValueError, match=match): + controller.restore_desired(flat) + assert controller.current_point() == STARTUP + finally: + controller.close() + + +# --------------------------------------------------------------------------- +# Decision record +# --------------------------------------------------------------------------- + + +def test_decision_record_carries_failure_reason_and_a_private_details_copy(): + details = {"shards": [{"id": "g0", "state": "Completed"}]} + decision = RetuneDecision( + decision_id="retune-w1800000000000", + outcome="failed", + window_index=K, + window_start_s=T0 - INTERVAL, + window_end_s=T0, + as_of_s=L_K, + launched_at_s=L_K, + decided_at_s=L_K + STEP, + num_requests=0, + num_candidates=2, + previous=STARTUP, + chosen=STARTUP, + baseline_score=None, + best_score=None, + reason="probe scores differ", + failure_reason="crn_mismatch", + details=details, + ) + details["shards"] = [] + + record = decision.to_dict() + assert record["failure_reason"] == "crn_mismatch" + assert record["details"] == {"shards": [{"id": "g0", "state": "Completed"}]} + json.dumps(record) + with pytest.raises(ValueError, match="failure_reason"): + RetuneDecision(**{**decision.__dict__, "failure_reason": ""}) + + +def test_replay_decision_record_has_empty_failure_fields(): + decision = RetuneDecision( + decision_id="retune-0-w0", + outcome="kept", + window_index=0, + window_start_s=0.0, + window_end_s=60.0, + as_of_s=65.0, + launched_at_s=65.0, + decided_at_s=70.0, + num_requests=1, + num_candidates=1, + previous=STARTUP, + chosen=STARTUP, + baseline_score=1.0, + best_score=1.0, + ) + + record = decision.to_dict() + assert (record["failure_reason"], record["details"]) == (None, {}) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py new file mode 100644 index 000000000000..3fb2c4df72bb --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -0,0 +1,1884 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the executor imports the sweep. + +"""Unit tests for the live retune DGDR executor (``retune.live.dgdr``). + +Each sweep runs against ``FakeRetuneKube`` with ``FakeDgdrOperator`` +advancing every Request one phase per ``sleep`` of a fake clock, so the +Request -> Run -> Candidate lifecycle, adoption, failures and timeouts are +all deterministic. +""" + +from __future__ import annotations + +import itertools +import math +import threading +from collections.abc import Callable, Mapping +from typing import Any, Optional + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from dynamo.planner.config.planner_config import ( + DgdrExecutorConfig, + RetuneConfig, + RetuneEngineSpec, +) +from dynamo.planner.core.types import RetuneParams +from dynamo.planner.retune.live import dgdr +from dynamo.planner.retune.live.contracts import ( + LABEL_ATTEMPT, + LABEL_DGD_UID, + LABEL_SHARD, + LABEL_SPEC, + LABEL_WINDOW, + NON_DECISION_FAILURES, + DgdRef, + KubeAlreadyExists, + KubeError, + KubeInvalid, + KubeNotFound, + WindowCutResult, +) +from dynamo.planner.retune.live.dgdr import ( + DgdrContext, + Shard, + build_request, + crn_unsupported, + input_mismatches, + missing_fields, + need_seconds, + plan_shards, + request_name, + run_dgdr_sweep, + spec_hash, +) +from dynamo.planner.retune.sweep import SweepRequest, score_summary +from dynamo.planner.tests.live_retune_fakes import ( + FakeDgdrOperator, + FakeRetuneKube, + default_metrics, + lower_request_input, +) + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +CREDIT = "overlap_score_credit" +SCALE = "prefill_load_scale" +TEMP = "router_temperature" +LIVE_TO_PUBLIC = {CREDIT: CREDIT, SCALE: SCALE, TEMP: "temperature"} +DEFAULT_DOMAIN = { + CREDIT: (0.0, 0.5, 1.0), + SCALE: (0.0, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0), + TEMP: (0.0, 0.2, 0.5, 1.0), +} +SMALL_DOMAIN = {CREDIT: (0.5, 1.0), SCALE: (1.0, 2.0), TEMP: (0.0, 0.5)} +WINDOW_END_MS = 1_793_000_000_000 +INTERVAL_S = 3600.0 +GRACE_S = 120.0 +TIMEOUT_S = 3000.0 +LAUNCH_S = WINDOW_END_MS / 1000.0 + GRACE_S +POLL_S = 5.0 +DGD = DgdRef(name="llm", namespace="test-ns", uid="dgd-uid-1", dynamo_namespace="dyn") +AGG_ENGINE = { + "mode": "aggregated", + "backend": "vllm", + "context_length": 32768, + "workers": { + "aggregated": { + "parallelism": { + "preset": [ + { + "tensor": 2, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, + } + ] + } + }, + "prefill": None, + "decode": None, + }, +} +DISAGG_ENGINE = { + "mode": "disaggregated", + "backend": "vllm", + "workers": { + "aggregated": None, + "prefill": { + "parallelism": { + "preset": [ + { + "tensor": 1, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, + } + ] + } + }, + "decode": { + "parallelism": { + "preset": [ + { + "tensor": 2, + "pipeline": 1, + "attention_data": 2, + "moe_tensor": 1, + "moe_expert": 1, + } + ] + } + }, + }, +} + + +# ------------------------------------------------------------------ helpers + + +def make_cfg(**overrides: Any) -> DgdrExecutorConfig: + values: dict[str, Any] = { + "claim_name": "request-traces", + "model": "Qwen/Qwen3-32B", + "sku": "h200_sxm", + "engine": AGG_ENGINE, + "cut_budget_seconds": 100.0, + "job_startup_seconds": 60.0, + "run": {"parallelism": 8, "candidate_timeout_seconds": 300.0, "max_shards": 3}, + } + values.update(overrides) + return DgdrExecutorConfig.model_validate(values) + + +def make_retune(cfg: DgdrExecutorConfig, **overrides: Any) -> RetuneConfig: + values: dict[str, Any] = { + "interval_seconds": INTERVAL_S, + "grace_seconds": GRACE_S, + "timeout_seconds": TIMEOUT_S, + "router": "default", + "executor": "dgdr", + "dgdr": cfg, + "objective": "GoodputPerGPU", + } + values.update(overrides) + return RetuneConfig.model_validate(values) + + +def grid(domain: Mapping[str, tuple[float, ...]]) -> list[dict[str, float]]: + keys = sorted(domain) + return [ + dict(zip(keys, values)) + for values in itertools.product(*(domain[key] for key in keys)) + ] + + +def make_request( + domain: Mapping[str, tuple[float, ...]] = SMALL_DOMAIN, + *, + baseline: Optional[Mapping[str, float]] = None, + startup: Optional[Mapping[str, float]] = None, + deadline_s: Optional[float] = LAUNCH_S + TIMEOUT_S, + spec_replicas: Optional[dict[str, int]] = None, + objective: str = "GoodputPerGPU", + router_domain: Any = "same", + planner: Optional[dict[str, Any]] = None, + trace_paths: tuple[str, ...] = ("/trace/segments/llm",), +) -> SweepRequest: + points = grid(domain) + baseline = dict(baseline or points[-1]) + startup = dict(startup or baseline) + candidates = {RetuneParams(router=point) for point in points} + candidates.add(RetuneParams(router=startup)) + base = RetuneParams(router=baseline, planner=planner or {}) + candidates.add(base) + replicas = spec_replicas or {"aggregated": 3} + disagg = "prefill" in replicas + return SweepRequest( + window_index=7, + window_start_ms=WINDOW_END_MS - INTERVAL_S * 1000.0, + window_end_ms=float(WINDOW_END_MS), + history_start_ms=WINDOW_END_MS - INTERVAL_S * 1000.0, + as_of_ms=WINDOW_END_MS + GRACE_S * 1000.0, + trace_paths=trace_paths, + arrival_speedup_ratio=1.0, + engine=RetuneEngineSpec( + router_mode="kv_router", + router_config={}, + prefill_engine_args={} if disagg else None, + decode_engine_args={} if disagg else None, + ), + candidates=tuple(sorted(candidates, key=RetuneParams.key)), + baseline=base, + replicas=dict(replicas), + base_planner=None, + predictor_candidates=None, + objective=objective, # type: ignore[arg-type] + sla_ttft_ms=500.0, + sla_itl_ms=30.0, + max_parallel=1, + timeout_s=TIMEOUT_S, + router_domain=( + {key: tuple(values) for key, values in domain.items()} + if router_domain == "same" + else router_domain + ), + deadline_s=deadline_s, + spec_replicas=dict(replicas), + generation=3, + ) + + +def public(point: Mapping[str, float]) -> dict[str, float]: + return {LIVE_TO_PUBLIC[key]: value for key, value in point.items()} + + +def expected_score(point: Mapping[str, float], request: SweepRequest) -> float: + score, _ = score_summary( + default_metrics(public(point)), request.window_s, request.objective + ) + return score + + +class FakeClock: + """Wall clock for the executor; ``sleep`` advances it and runs hooks.""" + + def __init__(self, start: float = LAUNCH_S) -> None: + self.now = start + self.sleeps: list[float] = [] + self.hooks: list[Callable[[], None]] = [] + + def time(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += max(seconds, 0.001) + for hook in list(self.hooks): + hook() + + +class FakeRegistry: + def __init__(self) -> None: + self.protected: dict[int, set[str]] = {} + self.released: list[int] = [] + self.references: dict[int, int] = {} + self.deleted: list[str] = [] + self.lock = threading.Lock() + + def protect(self, window_end_ms: int, request_names) -> None: + with self.lock: + self.protected.setdefault(window_end_ms, set()).update(request_names) + self.references[window_end_ms] = self.references.get(window_end_ms, 0) + 1 + + def release(self, window_end_ms: int) -> None: + with self.lock: + self.released.append(window_end_ms) + self.references[window_end_ms] = self.references.get(window_end_ms, 0) - 1 + + def balanced(self, window_end_ms: int) -> bool: + """Every reference the sweep took on the window was released.""" + return ( + window_end_ms in self.released + and self.references.get(window_end_ms, 0) == 0 + ) + + def enqueue_delete(self, request_names) -> None: + with self.lock: + self.deleted.extend(request_names) + + +class FakeCutter: + """Records cuts; returns ``num_requests`` rows or raises ``error``.""" + + def __init__( + self, + *, + num_requests: int = 1234, + error: Optional[Exception] = None, + on_cut: Optional[Callable[[], None]] = None, + ) -> None: + self.num_requests = num_requests + self.error = error + self.on_cut = on_cut + self.calls: list[dict[str, Any]] = [] + + def __call__( + self, + segments_root, + start_ms, + end_ms, + as_of_ms, + out_path, + *, + max_rows, + deadline_s, + cancel=None, + ) -> WindowCutResult: + self.calls.append( + { + "segments_root": segments_root, + "start_ms": start_ms, + "end_ms": end_ms, + "as_of_ms": as_of_ms, + "out_path": out_path, + "max_rows": max_rows, + "deadline_s": deadline_s, + "cancel": cancel, + } + ) + if self.on_cut is not None: + self.on_cut() + if self.error is not None: + raise self.error + if self.num_requests == 0: + return WindowCutResult( + path=None, num_requests=0, trace_block_size=None, row_counts={} + ) + return WindowCutResult( + path=out_path, + num_requests=self.num_requests, + trace_block_size=64, + row_counts={"selected": self.num_requests}, + dropped_members=1, + dropped_bytes=10, + ) + + +class CutError(Exception): + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +class Harness: + """A fake cluster, operator, clock, registry and cutter for one window.""" + + def __init__( + self, + tmp_path, + *, + cfg: Optional[DgdrExecutorConfig] = None, + operator: bool = True, + **retune_overrides: Any, + ) -> None: + self.clock = FakeClock() + self.kube = FakeRetuneKube("test-ns", now=self.clock.time) + self.cfg = cfg or make_cfg(mount_path=str(tmp_path / "trace")) + self.retune = make_retune(self.cfg, **retune_overrides) + self.registry = FakeRegistry() + self.cutter = FakeCutter() + self.leader = True + self.cancel = threading.Event() + self.abort = threading.Event() + self.operator = FakeDgdrOperator(self.kube) + if operator: + self.clock.hooks.append(self.operator.step) + + def context(self, **overrides: Any) -> DgdrContext: + values: dict[str, Any] = { + "dgd": DGD, + "cfg": self.cfg, + "retune": self.retune, + "backend": "vllm", + "kube": self.kube, + "registry": self.registry, + "cut_window": self.cutter, + "is_leader": lambda: self.leader, + "now": self.clock.time, + "sleep": self.clock.sleep, + "poll_interval_s": POLL_S, + } + values.update(overrides) + return DgdrContext(**values) + + def sweep(self, request: SweepRequest, **overrides: Any): + return run_dgdr_sweep( + request, + cancel=self.cancel, + abort=self.abort, + context=self.context(**overrides), + ) + + def created(self) -> list[dict]: + return [ + call.target + for call in self.kube.calls_of("create_request") + if not call.kwargs.get("dry_run") + ] + + def request_of_shard(self, shard_id: str) -> dict: + found = [ + obj + for obj in self.kube.requests.values() + if obj["metadata"]["labels"][LABEL_SHARD] == shard_id + ] + assert len(found) == 1, (shard_id, [o["metadata"]["name"] for o in found]) + return found[0] + + +def assert_scores(result, request: SweepRequest, missing=()) -> None: + missing_keys = {RetuneParams(router=point).key() for point in missing} + assert set(result.scores) == {candidate.key() for candidate in request.candidates} + for candidate in request.candidates: + key = candidate.key() + if key in missing_keys: + assert result.scores[key] is None, key + assert "error" in result.metrics[key] + else: + assert result.scores[key] == pytest.approx( + expected_score(candidate.router, request), rel=1e-12 + ), key + assert result.metrics[key]["params"] == candidate.to_dict() + + +def kube_error(cls: type[KubeError], status: Optional[int], message: str) -> KubeError: + """A ``Kube*`` error as ``kube.py`` raises it, with its HTTP ``status``.""" + + error = cls(f"HTTP {status}: {message}") + error.status = status # type: ignore[attr-defined] + return error + + +def lose_leadership_after_start(h: Harness, request: SweepRequest) -> None: + """A first leader creates the window's Requests, the operator starts their + Runs, and the leader loses leadership; the next sweep is another leader.""" + + h.operator.max_phase = "started" + h.clock.hooks.append(lambda: h.abort.set() if len(h.clock.sleeps) >= 2 else None) + assert h.sweep(request).failure_reason == "lost_leadership" + h.abort = threading.Event() + h.operator.max_phase = None + h.clock.hooks = [h.operator.step] + + +# ------------------------------------------------------------------ shards + + +def test_plan_shards_default_grid_is_three_balanced_shards_and_a_probe(): + points = grid(DEFAULT_DOMAIN) + baseline = points[0] + shards = plan_shards( + DEFAULT_DOMAIN, baseline=baseline, startup=baseline, max_shards=3 + ) + grid_shards = [shard for shard in shards if shard.kind == "grid"] + assert [shard.size for shard in grid_shards] == [36, 36, 36] + assert [shard.id for shard in shards] == ["grid-0", "grid-1", "grid-2", "probe"] + covered = [point for shard in grid_shards for point in shard.points] + assert sorted(map(str, covered)) == sorted(map(str, points)) + for shard in grid_shards: + # Each shard is the Cartesian product of its choices. + assert shard.size == math.prod(len(v) for v in shard.choices.values()) + probe = shards[-1] + # The middle value of each knob, a positive temperature. + assert probe.points == ({CREDIT: 0.5, SCALE: 2.0, TEMP: 0.5},) + assert probe.choices == {CREDIT: (0.5,), SCALE: (2.0,), TEMP: (0.5,)} + + +def test_plan_shards_adds_off_grid_baseline_and_startup_points(): + off_baseline = {CREDIT: 0.75, SCALE: 1.0, TEMP: 0.0} + off_startup = {CREDIT: 1.0, SCALE: 3.0, TEMP: 0.0} + shards = plan_shards( + SMALL_DOMAIN, + baseline=off_baseline, + startup=off_startup, + max_shards=3, + parallelism=8, + ) + by_id = {shard.id: shard for shard in shards} + assert set(by_id) == {"grid-0", "point-baseline", "point-startup", "probe"} + assert by_id["point-baseline"].points == (off_baseline,) + assert by_id["point-baseline"].kind == "point" + assert by_id["point-startup"].points == (off_startup,) + # The same off-grid point once. + shards = plan_shards( + SMALL_DOMAIN, + baseline=off_baseline, + startup=off_baseline, + max_shards=3, + parallelism=8, + ) + assert [shard.id for shard in shards] == ["grid-0", "point-baseline", "probe"] + # An on-grid startup needs no shard; no probe when disabled. + shards = plan_shards( + SMALL_DOMAIN, + baseline=grid(SMALL_DOMAIN)[0], + startup=grid(SMALL_DOMAIN)[1], + max_shards=3, + probe=False, + parallelism=8, + ) + assert [shard.id for shard in shards] == ["grid-0"] + + +def test_plan_shards_splits_uneven_dimension_and_rejects_impossible_grids(): + domain = {CREDIT: tuple(float(v) for v in range(250))} + zero = {CREDIT: 0.0} + shards = plan_shards(domain, baseline=zero, startup=zero, max_shards=3, probe=False) + assert [shard.size for shard in shards] == [84, 83, 83] + assert shards[0].choices[CREDIT][0] == 0.0 + assert shards[-1].choices[CREDIT][-1] == 249.0 + with pytest.raises(ValueError, match="cannot be split"): + plan_shards(domain, baseline=zero, startup=zero, max_shards=2) + with pytest.raises(ValueError, match="cannot be split"): + plan_shards( + DEFAULT_DOMAIN, + baseline=grid(DEFAULT_DOMAIN)[0], + startup=grid(DEFAULT_DOMAIN)[0], + max_shards=1, + ) + point = grid(SMALL_DOMAIN)[0] + + def sizes(**kwargs): + shards = plan_shards( + SMALL_DOMAIN, baseline=point, startup=point, probe=False, **kwargs + ) + return [shard.size for shard in shards] + + # Without the Run parallelism the largest shard is minimized. + assert sizes(max_shards=16) == [1] * 8 + assert sizes(max_shards=3) == [4, 4] + # With it, a shard is added only when it saves an evaluation round + # (AISimulate runs the first candidate alone, so single points save one). + assert sizes(max_shards=3, parallelism=8) == [8] + assert sizes(max_shards=16, parallelism=8) == [1] * 8 + assert sizes(max_shards=3, parallelism=1) == [4, 4] + # The default grid is three shards of 36 either way. + default = plan_shards( + DEFAULT_DOMAIN, + baseline=grid(DEFAULT_DOMAIN)[0], + startup=grid(DEFAULT_DOMAIN)[0], + max_shards=3, + parallelism=8, + ) + assert [shard.size for shard in default] == [36, 36, 36, 1] + + +def test_plan_shards_validates_inputs_and_probe_falls_back_to_zero_temperature(): + point = {CREDIT: 0.5, SCALE: 1.0} + with pytest.raises(ValueError, match="do not match"): + plan_shards(SMALL_DOMAIN, baseline=point, startup=point, max_shards=3) + with pytest.raises(ValueError, match="empty"): + plan_shards({}, baseline={}, startup={}, max_shards=3) + domain = {CREDIT: (0.0, 1.0), SCALE: (1.0,), TEMP: (0.0,)} + shards = plan_shards( + domain, baseline=grid(domain)[0], startup=grid(domain)[0], max_shards=3 + ) + assert shards[-1].points == ({CREDIT: 1.0, SCALE: 1.0, TEMP: 0.0},) + + +def test_need_seconds_follows_the_design_formula(): + cfg = make_cfg( + cut_budget_seconds=900.0, + job_startup_seconds=300.0, + run={"parallelism": 8, "candidate_timeout_seconds": 1800.0}, + ) + shard = plan_shards( + DEFAULT_DOMAIN, + baseline=grid(DEFAULT_DOMAIN)[0], + startup=grid(DEFAULT_DOMAIN)[0], + max_shards=3, + )[0] + assert shard.size == 36 + # 1 + ceil(35 / 8) = 6 rounds. + assert need_seconds(shard, cfg, needs_cut=True) == 900 + 300 + 6 * 1800 + assert need_seconds(shard, cfg, needs_cut=False) == 300 + 6 * 1800 + single = Shard( + id="probe", kind="probe", choices={CREDIT: (1.0,)}, points=({CREDIT: 1.0},) + ) + assert need_seconds(single, cfg, needs_cut=False) == 300 + 1800 + + +# ---------------------------------------------------------------- Request + + +def test_spec_hash_is_canonical(): + assert spec_hash({"a": 1, "b": [1, 2]}) == spec_hash({"b": [1, 2], "a": 1}) + assert spec_hash({"a": 1}) != spec_hash({"a": 2}) + assert len(spec_hash({})) == 16 + int(spec_hash({}), 16) + + +def _shards(domain=SMALL_DOMAIN, **kwargs): + point = grid(domain)[0] + return plan_shards( + domain, baseline=point, startup=point, max_shards=3, parallelism=8, **kwargs + ) + + +def test_build_request_aggregated_grid_shard(): + cfg = make_cfg() + retune = make_retune(cfg) + shard = _shards()[0] + body = build_request( + dgd=DGD, + cfg=cfg, + retune=retune, + sla_ttft_ms=500, + sla_itl_ms=30, + window_end_ms=WINDOW_END_MS, + shard=shard, + spec_replicas={"aggregated": 3}, + backend="vllm", + attempt=1, + ) + spec = body["spec"] + assert body["apiVersion"] == "nvidia.com/v1beta2" + assert body["kind"] == "DynamoGraphDeploymentRequest" + assert spec["model"] == {"name": "Qwen/Qwen3-32B"} + assert spec["backends"] == ["vllm"] + # 3 replicas x (tensor 2 x pipeline 1 x attention_data 1). + assert spec["hardware"] == {"gpu": {"sku": "h200_sxm", "budget": 6}} + assert spec["workload"] == { + "trace": { + "format": "dynamo", + "source": { + "pvc": { + "claimName": "request-traces", + "path": f"windows/llm/{WINDOW_END_MS}.jsonl.gz", + } + }, + } + } + assert spec["objective"] == { + "mode": "Optimize", + "metric": "GoodputPerGPU", + "sla": {"ttftMs": 500.0, "itlMs": 30.0}, + } + assert spec["recommendation"] == {"maxCandidates": 8} + overrides = spec["aisimulate"]["inputOverrides"] + workers = overrides["engine"]["workers"] + assert workers["aggregated"]["parallelism"]["preset"][0]["replicas"] == 3 + assert workers["prefill"] is None and workers["decode"] is None + assert overrides["engine"]["backend"] == "vllm" + assert overrides["router"] == { + "policy": "kv_router", + "prefill_load_model": {"type": "none"}, + "overlap_score_credit": {"choices": [0.5, 1.0]}, + "prefill_load_scale": {"choices": [1.0, 2.0]}, + "temperature": {"choices": [0.0, 0.5]}, + } + assert overrides["optimizer"] == { + "algorithm": "random", + "max_trials": 16, + "parallelism": 8, + "candidate_timeout_seconds": 300.0, + } + assert overrides["optimization"] == {"strict_sla": False} + assert overrides["dgdr_run"] == {"generator": "direct"} + # The config's engine is not mutated. + assert ( + "replicas" + not in AGG_ENGINE["workers"]["aggregated"]["parallelism"]["preset"][0] + ) + + metadata = body["metadata"] + digest = spec_hash(spec) + assert metadata["labels"] == { + "nvidia.com/planner-retune-dgd": "llm", + LABEL_DGD_UID: "dgd-uid-1", + LABEL_WINDOW: str(WINDOW_END_MS), + LABEL_SHARD: "grid-0", + LABEL_SPEC: digest, + LABEL_ATTEMPT: "1", + } + assert metadata["ownerReferences"] == [ + { + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeployment", + "name": "llm", + "uid": "dgd-uid-1", + "blockOwnerDeletion": False, + } + ] + assert metadata["name"] == request_name(DGD, WINDOW_END_MS, "grid-0", digest, 1) + assert metadata["name"].startswith("llm-rt-") and len(metadata["name"]) == 17 + assert request_name(DGD, WINDOW_END_MS, "grid-0", digest, 2) != metadata["name"] + + +def test_build_request_point_shard_uses_scalars_and_disagg_budget(): + cfg = make_cfg(engine=DISAGG_ENGINE) + retune = make_retune(cfg, objective="TTFT") + probe = _shards()[-1] + body = build_request( + dgd=DGD, + cfg=cfg, + retune=retune, + sla_ttft_ms=200, + sla_itl_ms=20, + window_end_ms=WINDOW_END_MS, + shard=probe, + spec_replicas={"prefill": 2, "decode": 4}, + backend="vllm", + attempt=3, + ) + spec = body["spec"] + # 2 x 1 + 4 x (2 x 1 x 2). + assert spec["hardware"]["gpu"]["budget"] == 18 + assert spec["objective"]["metric"] == "TTFT" + overrides = spec["aisimulate"]["inputOverrides"] + workers = overrides["engine"]["workers"] + assert workers["aggregated"] is None + assert workers["prefill"]["parallelism"]["preset"][0]["replicas"] == 2 + assert workers["decode"]["parallelism"]["preset"][0]["replicas"] == 4 + assert overrides["router"]["temperature"] == 0.5 + assert overrides["router"]["overlap_score_credit"] == 1.0 + assert overrides["router"]["prefill_load_scale"] == 2.0 + assert overrides["optimizer"]["max_trials"] == 2 + assert spec["recommendation"]["maxCandidates"] == 1 + assert body["metadata"]["labels"][LABEL_ATTEMPT] == "3" + + +@pytest.mark.parametrize( + "engine, replicas, backend, match", + [ + ({**AGG_ENGINE, "backend": "sglang"}, {"aggregated": 1}, "vllm", "differs"), + ( + {**AGG_ENGINE, "workers": {"aggregated": {}}}, + {"aggregated": 1}, + "vllm", + "preset", + ), + (AGG_ENGINE, {"prefill": 1}, "vllm", "spec_replicas"), + (AGG_ENGINE, {"aggregated": 0}, "vllm", "at least 1"), + ], +) +def test_build_request_rejects_engines_it_cannot_pin(engine, replicas, backend, match): + cfg = make_cfg(engine=engine) + with pytest.raises(ValueError, match=match): + build_request( + dgd=DGD, + cfg=cfg, + retune=make_retune(cfg), + sla_ttft_ms=1, + sla_itl_ms=1, + window_end_ms=WINDOW_END_MS, + shard=_shards()[0], + spec_replicas=replicas, + backend=backend, + attempt=1, + ) + + +def test_build_request_requires_the_three_dgdr_knobs(): + cfg = make_cfg() + shard = Shard( + id="grid-0", kind="grid", choices={CREDIT: (1.0,)}, points=({CREDIT: 1.0},) + ) + with pytest.raises(ValueError, match="exactly"): + build_request( + dgd=DGD, + cfg=cfg, + retune=make_retune(cfg), + sla_ttft_ms=1, + sla_itl_ms=1, + window_end_ms=WINDOW_END_MS, + shard=shard, + spec_replicas={"aggregated": 1}, + backend="vllm", + attempt=1, + ) + + +def _body(shard_index=0, **kwargs): + cfg = make_cfg(**kwargs) + return build_request( + dgd=DGD, + cfg=cfg, + retune=make_retune(cfg), + sla_ttft_ms=500, + sla_itl_ms=30, + window_end_ms=WINDOW_END_MS, + shard=_shards()[shard_index], + spec_replicas={"aggregated": 3}, + backend="vllm", + attempt=1, + ) + + +def test_input_read_back_accepts_the_operator_lowering_and_finds_drift(): + body = _body() + body["metadata"]["namespace"] = "test-ns" + effective = lower_request_input(body, run_name="r-1") + assert input_mismatches(body["spec"], effective) == [] + + drifted = lower_request_input(body, run_name="r-1") + drifted["engine"]["workers"]["aggregated"]["parallelism"]["preset"][0][ + "replicas" + ] = 1 + drifted["engine"]["workers"]["prefill"] = {} + drifted["router"]["temperature"] = {"choices": [0.0]} + drifted["evaluation"]["sla"]["e2e_ms"] = 10.0 + drifted["traffic"]["load"]["speedup"] = 2.0 + drifted["optimization"]["strict_sla"] = True + problems = input_mismatches(body["spec"], drifted) + joined = "\n".join(problems) + for path in ( + "engine:", + "router:", + "evaluation.sla.e2e_ms", + "traffic.load", + "optimization.strict_sla", + ): + assert path in joined, (path, problems) + + stripped = lower_request_input(body, run_name="r-1") + del stripped["dgdr_run"]["generator"] + del stripped["optimization"]["constraints"] + problems = input_mismatches(body["spec"], stripped) + assert any(p.startswith("dgdr_run.generator") for p in problems) + assert any("max_candidate_gpus" in p for p in problems) + + +def test_missing_fields_reports_pruned_leaves_and_allows_defaults(): + submitted = {"a": {"b": 1, "c": [1, 2], "gone": None}, "d": True} + stored = {"a": {"b": 1.0, "c": [1, 2], "extra": 5}, "d": True, "default": 1} + assert missing_fields(stored, submitted) == [] + assert missing_fields({"a": {"b": 1}}, submitted, "spec") == ["spec.a.c", "spec.d"] + assert missing_fields( + {"a": {"b": 1, "c": [1, 2], "gone": 3}, "d": 1}, submitted + ) == [ + "a.gone", + "d", + ] + + +def test_crn_unsupported_compares_parseable_versions_only(): + assert not crn_unsupported("1.7.0", None, None) + assert crn_unsupported("1.7.0", None, "1.8.0") + assert not crn_unsupported("1.8.0", None, "1.8.0") + assert not crn_unsupported("1.9.1rc0", None, "1.8.0") + assert crn_unsupported(None, "nvcr.io/nvidia/planner:1.7.2@sha256:ab", "1.8") + assert not crn_unsupported("main", "image-without-tag", "1.8.0") + + +# ------------------------------------------------------------ the sweep + + +def test_fresh_window_creates_cuts_harvests_and_scores(tmp_path): + h = Harness(tmp_path) + request = make_request() + result = h.sweep(request) + + assert result.status == "ok", (result.error, result.failure_reason) + assert result.failure_reason is None + assert result.num_requests == 1234 + assert result.row_counts == {"selected": 1234} + assert_scores(result, request) + # One grid shard (8 points) and the probe. + created = h.created() + assert [obj["metadata"]["labels"][LABEL_SHARD] for obj in created] == [ + "grid-0", + "probe", + ] + assert all( + call.kwargs["timeout_s"] <= 10.0 for call in h.kube.calls_of("create_request") + ) + # The window was cut once, into the claim path the Requests name. + (cut,) = h.cutter.calls + assert cut["out_path"] == str( + tmp_path / "trace" / f"windows/llm/{WINDOW_END_MS}.jsonl.gz" + ) + assert cut["segments_root"] == "/trace/segments/llm" + assert cut["start_ms"] == WINDOW_END_MS - 3_600_000 + assert cut["end_ms"] == WINDOW_END_MS + assert cut["as_of_ms"] == WINDOW_END_MS + 120_000 + # Absolute wall-clock seconds: now plus the cut budget. + assert cut["deadline_s"] == LAUNCH_S + 100.0 + assert cut["max_rows"] == h.cfg.max_window_rows + assert (tmp_path / "trace" / "windows" / "llm").is_dir() + # Requests were protected, nothing deleted, the window released. + assert h.registry.protected[WINDOW_END_MS] == { + o["metadata"]["name"] for o in created + } + assert h.registry.deleted == [] + assert h.registry.balanced(WINDOW_END_MS) + details = result.details + assert details["crn_probe"]["relative_error"] == 0.0 + assert details["runtime_image"].endswith(":1.8.0") + assert details["runtime_version"] == "1.8.0" + assert details["missing_candidates"] == 0 + assert details["shards"]["grid-0"]["critical"] is True + assert details["window"]["dropped_members"] == 1 + assert details["generation"] == 3 + assert result.wall_s > 0 + + +def test_default_grid_with_off_grid_startup_runs_five_requests(tmp_path): + h = Harness(tmp_path) + startup = {CREDIT: 1.0, SCALE: 1.5, TEMP: 0.0} + request = make_request(DEFAULT_DOMAIN, baseline=startup, startup=startup) + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + shard_ids = sorted(obj["metadata"]["labels"][LABEL_SHARD] for obj in h.created()) + assert shard_ids == ["grid-0", "grid-1", "grid-2", "point-baseline", "probe"] + # The probe point's grid twin is critical; the baseline shard is too. + critical = {k for k, v in result.details["shards"].items() if v["critical"]} + assert critical == {"grid-1", "point-baseline", "probe"} + + +def test_disaggregated_window(tmp_path): + h = Harness(tmp_path, cfg=make_cfg(engine=DISAGG_ENGINE, mount_path=str(tmp_path))) + request = make_request(spec_replicas={"prefill": 1, "decode": 2}) + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + spec = h.created()[0]["spec"] + assert spec["hardware"]["gpu"]["budget"] == 1 + 2 * 4 + + +def test_window_adopted_as_succeeded_is_not_cut_again(tmp_path): + h = Harness(tmp_path) + request = make_request() + first = h.sweep(request) + assert first.status == "ok" + creates = len(h.kube.calls_of("create_request")) + + # A new leader (fresh cutter) sweeps the same window. + h.cutter = FakeCutter(num_requests=1) + second = h.sweep(request) + assert second.status == "ok", second.error + assert h.cutter.calls == [] + assert len(h.kube.calls_of("create_request")) == creates + assert second.scores == first.scores + # Row counts come from the Request annotation of the cut. + assert second.num_requests == 1234 + assert second.row_counts == {"selected": 1234} + assert all(shard["adopted"] for shard in second.details["shards"].values()) + + +def test_lost_leadership_leaves_requests_for_the_next_leader(tmp_path): + h = Harness(tmp_path) + h.operator.max_phase = "started" + h.clock.hooks.append(lambda: h.abort.set() if len(h.clock.sleeps) >= 2 else None) + request = make_request() + result = h.sweep(request) + assert result.status == "failed" + assert result.failure_reason == "lost_leadership" + assert result.failure_reason in NON_DECISION_FAILURES + assert h.registry.deleted == [] + assert h.registry.balanced(WINDOW_END_MS) + names = set(h.kube.requests) + assert len(names) == 2 + + # The next leader adopts the in-flight Requests without cutting. + h.abort = threading.Event() + h.operator.max_phase = None + h.clock.hooks = [h.operator.step] + h.cutter = FakeCutter() + second = h.sweep(request) + assert second.status == "ok", second.error + assert set(h.kube.requests) == names + assert h.cutter.calls == [] + assert_scores(second, request) + + +def test_shutdown_returns_at_once_without_deleting(tmp_path): + h = Harness(tmp_path) + h.operator.max_phase = "accepted" + h.clock.hooks.append(h.cancel.set) + result = h.sweep(make_request()) + assert result.failure_reason == "shutdown" + assert result.status == "failed" + assert h.registry.deleted == [] + assert len(h.kube.requests) == 2 + + +def test_not_leader_creates_nothing(tmp_path): + h = Harness(tmp_path) + h.leader = False + result = h.sweep(make_request()) + assert result.failure_reason == "lost_leadership" + assert h.created() == [] + assert h.cutter.calls == [] + + +def test_abort_during_cut_reports_lost_leadership(tmp_path): + h = Harness(tmp_path) + h.cutter = FakeCutter(on_cut=h.abort.set, error=CutError("cancelled")) + result = h.sweep(make_request()) + assert result.failure_reason == "lost_leadership" + assert h.created() == [] + # The cutter's cancel event reflects abort. + assert h.cutter.calls[0]["cancel"].is_set() + + +def test_stale_spec_and_older_attempts_are_queued_for_deletion(tmp_path): + h = Harness(tmp_path) + request = make_request() + h.sweep(request) + grid_request = h.request_of_shard("grid-0") + name = grid_request["metadata"]["name"] + + # A stale shard spec and an older attempt of the same spec. + stale = { + "metadata": { + "name": "llm-rt-stale", + "labels": { + **grid_request["metadata"]["labels"], + LABEL_SPEC: "0" * 16, + }, + }, + "spec": {}, + } + older = { + "metadata": { + "name": "llm-rt-older", + "labels": {**grid_request["metadata"]["labels"], LABEL_ATTEMPT: "0"}, + }, + "spec": {}, + } + unknown_shard = { + "metadata": { + "name": "llm-rt-unknown", + "labels": {**grid_request["metadata"]["labels"], LABEL_SHARD: "grid-9"}, + }, + "spec": {}, + } + for obj in (stale, older, unknown_shard): + h.kube.create_request(obj) + other_window = { + "metadata": { + "name": "llm-rt-other", + "labels": {**grid_request["metadata"]["labels"], LABEL_WINDOW: "1"}, + }, + "spec": {}, + } + h.kube.create_request(other_window) + h.registry.deleted.clear() + result = h.sweep(request) + assert result.status == "ok", result.error + assert sorted(h.registry.deleted) == [ + "llm-rt-older", + "llm-rt-stale", + "llm-rt-unknown", + ] + assert result.details["shards"]["grid-0"]["request"] == name + + +def test_create_that_lands_but_times_out_is_adopted(tmp_path): + h = Harness(tmp_path) + h.kube.fail("create_request", KubeError("timeout"), landed=True) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert len(h.kube.requests) == 2 + assert result.details["k8s_errors"] == 1 + assert_scores(result, request) + + +def test_create_already_exists_with_same_spec_is_adopted(tmp_path): + h = Harness(tmp_path) + h.kube.fail("create_request", KubeAlreadyExists("exists"), landed=True) + result = h.sweep(make_request()) + assert result.status == "ok", result.error + assert len(h.kube.requests) == 2 + + +def test_create_that_fails_is_retried(tmp_path): + h = Harness(tmp_path) + h.kube.fail("create_request", KubeError("503"), times=2) + result = h.sweep(make_request()) + assert result.status == "ok", result.error + assert len(h.kube.requests) == 2 + assert result.details["k8s_errors"] == 2 + + +@pytest.mark.parametrize( + "error", + [ + kube_error(KubeError, 403, "requests is forbidden: cannot create"), + kube_error(KubeNotFound, 404, "the server could not find the resource"), + kube_error(KubeError, 405, "method not allowed"), + KubeNotFound("no such API"), + ], + ids=["403", "404", "405", "not-found"], +) +def test_permanent_create_error_fails_fast(tmp_path, error): + h = Harness(tmp_path) + h.kube.fail("create_request", error, times=1000) + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + assert result.status == "failed" + assert result.failure_reason == "k8s_error" + assert str(error) in result.error + # No create -> unknown -> GET -> create loop until the launch gate trips. + assert len(h.created()) == 1 + assert h.clock.sleeps == [] + assert result.details["k8s_errors"] == 1 + assert h.kube.requests == {} + + +@pytest.mark.parametrize( + "cls, status", [(KubeInvalid, 422), (KubeInvalid, 400), (KubeError, 403)] +) +def test_admission_webhook_rejection_is_request_rejected(tmp_path, cls, status): + h = Harness(tmp_path) + message = ( + 'admission webhook "vdynamographdeploymentrequest-v1beta2.kb.io" denied ' + 'the request: DynamoGraphDeploymentRequest.nvidia.com "x" is invalid: ' + 'spec.hardware.gpu.sku: Invalid value: "h200_sxm": no node offers it' + ) + h.kube.fail("create_request", kube_error(cls, status, message)) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "request_rejected" + assert "no node offers it" in result.error + assert len(h.created()) == 1 + + +def test_webhook_unreachable_is_retried(tmp_path): + h = Harness(tmp_path) + error = kube_error( + KubeError, + 500, + 'Internal error occurred: failed calling webhook "x.kb.io": connection refused', + ) + h.kube.fail("create_request", error, times=2) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert result.details["k8s_errors"] == 2 + assert_scores(result, request) + + +@pytest.mark.parametrize( + "error", + [kube_error(KubeError, 403, "requests is forbidden: cannot get"), KubeNotFound()], + ids=["403", "not-found"], +) +def test_permanent_get_error_fails_fast(tmp_path, error): + h = Harness(tmp_path) + # The create's outcome is unknown, so the next poll looks the Request up. + h.kube.fail("create_request", KubeError("timeout")) + h.kube.fail("get_request", error, times=1000) + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + assert result.status == "failed" + assert result.failure_reason == "k8s_error" + assert "get Request" in result.error + assert h.clock.now <= LAUNCH_S + POLL_S + + +def test_transient_get_errors_are_retried(tmp_path): + h = Harness(tmp_path) + h.kube.fail("get_request", KubeError("timeout"), times=3) + h.kube.fail("get_run", kube_error(KubeError, 503, "unavailable"), times=2) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert result.details["k8s_errors"] == 5 + assert_scores(result, request) + + +def test_name_taken_by_a_foreign_request_fails_a_critical_shard(tmp_path): + h = Harness(tmp_path) + request = make_request() + # Pre-create a foreign object under the grid shard's name. + body = build_request( + dgd=DGD, + cfg=h.cfg, + retune=h.retune, + sla_ttft_ms=500, + sla_itl_ms=30, + window_end_ms=WINDOW_END_MS, + shard=_shards()[0], + spec_replicas={"aggregated": 3}, + backend="vllm", + attempt=1, + ) + h.kube.requests[body["metadata"]["name"]] = { + "metadata": {"name": body["metadata"]["name"], "labels": {"foreign": "yes"}}, + "spec": {}, + } + result = h.sweep(request) + assert result.status == "failed" + assert result.failure_reason == "request_conflict" + assert body["metadata"]["name"] not in h.registry.deleted + + +def test_strict_rejection_is_schema_unsupported(tmp_path): + h = Harness(tmp_path) + h.kube.unknown_fields = {"spec.workload.trace"} + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "schema_unsupported" + assert h.kube.requests == {} + + +def test_silently_pruned_field_is_schema_unsupported(tmp_path): + h = Harness(tmp_path) + h.kube.pruned_fields = {"spec.workload.trace.source"} + result = h.sweep(make_request()) + assert result.failure_reason == "schema_unsupported" + assert "spec.workload.trace.source" in result.error + # The pruned Request is queued for deletion. + assert h.registry.deleted == list(h.kube.requests) + + +def test_invalid_spec_fails_the_window_and_abandons_in_flight(tmp_path): + h = Harness(tmp_path, operator=False) + + def reject(kube, created): + if created["metadata"]["labels"][LABEL_SHARD] == "probe": + kube.set_request_condition( + created["metadata"]["name"], + status="False", + reason="InvalidSpec", + message="bad engine", + ) + + h.kube.on_request_created.append(reject) + result = h.sweep(make_request()) + assert result.failure_reason == "invalid_spec" + assert "bad engine" in result.error + assert sorted(h.registry.deleted) == sorted(h.kube.requests) + + +def test_operator_not_reconciling_after_sixty_seconds(tmp_path): + h = Harness(tmp_path, operator=False) + result = h.sweep(make_request()) + assert result.failure_reason == "operator_not_reconciling" + assert h.clock.now - LAUNCH_S >= dgdr.ACCEPT_TIMEOUT_S + assert h.clock.now - LAUNCH_S < dgdr.ACCEPT_TIMEOUT_S + 2 * POLL_S + + +def test_run_name_conflict_keeps_waiting(tmp_path): + h = Harness(tmp_path, operator=False) + phases = {"steps": 0} + + def operator(): + phases["steps"] += 1 + for name in list(h.kube.requests): + if phases["steps"] < 30: + h.kube.set_request_condition( + name, status="False", reason="RunNameConflict" + ) + if phases["steps"] >= 30: + h.operator.step() + + h.clock.hooks.append(operator) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert h.clock.now - LAUNCH_S > dgdr.ACCEPT_TIMEOUT_S + assert_scores(result, request) + + +def test_run_pending_until_the_deadline_times_out(tmp_path): + h = Harness(tmp_path) + # The operator never creates the Jobs. Like the real operator, it then + # writes no Run status at all: the input arrives together with jobRef. + h.operator.max_phase = "accepted" + request = make_request(deadline_s=LAUNCH_S + 900.0) + result = h.sweep(request) + assert [h.kube.run_of(name)["status"] for name in h.kube.requests] == [{}, {}] + assert result.status == "timeout" + assert result.failure_reason == "run_pending" + assert "Runs pending" in result.error + assert h.clock.now >= LAUNCH_S + 900.0 + assert sorted(h.registry.deleted) == sorted(h.kube.requests) + assert all(shard["run_pending"] for shard in result.details["shards"].values()) + + +def test_run_pending_clears_once_the_job_exists(tmp_path, caplog): + h = Harness(tmp_path) + h.operator.max_phase = "accepted" + + def create_jobs_late(): + if h.clock.now - LAUNCH_S >= 3 * h.cfg.job_startup_seconds: + h.operator.max_phase = None + + h.clock.hooks.insert(0, create_jobs_late) + request = make_request() + with caplog.at_level("WARNING", logger=dgdr.__name__): + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + assert "(run_pending)" in caplog.text + assert not any(shard["run_pending"] for shard in result.details["shards"].values()) + + +def test_job_created_but_not_started_is_not_run_pending(tmp_path): + h = Harness(tmp_path, operator=False) + + def create_jobs(): + for name in list(h.kube.requests): + if h.kube.run_of(name) is None: + h.kube.create_job(h.kube.accept_request(name)["metadata"]["name"]) + + h.clock.hooks.append(create_jobs) + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + for name in h.kube.requests: + status = h.kube.run_of(name)["status"] + assert status["aisimulate"]["input"] and status["jobRef"] + assert "startTime" not in status + assert result.status == "timeout" + assert result.failure_reason == "timeout" + assert not any(shard["run_pending"] for shard in result.details["shards"].values()) + + +def test_running_job_past_the_deadline_times_out(tmp_path): + h = Harness(tmp_path) + h.operator.max_phase = "started" + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + assert result.status == "timeout" + assert result.failure_reason == "timeout" + assert sorted(h.registry.deleted) == sorted(h.kube.requests) + + +def test_launch_too_late_for_a_critical_shard(tmp_path): + h = Harness(tmp_path) + # The 8-point grid shard needs 100 + 60 + 2 * 300 = 760 s. + result = h.sweep(make_request(deadline_s=LAUNCH_S + 700.0)) + assert result.status == "timeout" + assert result.failure_reason == "launch_too_late" + assert h.cutter.calls == [] + assert h.created() == [] + + +def test_launch_too_late_for_an_ordinary_shard_leaves_its_points_missing(tmp_path): + cfg = make_cfg( + mount_path=str(tmp_path), + run={"parallelism": 1, "candidate_timeout_seconds": 300.0, "max_shards": 3}, + ) + h = Harness(tmp_path, cfg=cfg) + domain = { + CREDIT: tuple(float(v) for v in range(10)), + SCALE: (1.0,), + TEMP: (0.0, 0.5), + } + baseline = {CREDIT: 2.5, SCALE: 1.0, TEMP: 0.0} # off the grid + shards = plan_shards( + domain, baseline=baseline, startup=baseline, max_shards=3, parallelism=1 + ) + sizes = {shard.id: shard.size for shard in shards} + # Ten credits in chunks of 4, 3 and 3; the probe (credit 5) is in grid-1. + assert sizes == { + "grid-0": 8, + "grid-1": 6, + "grid-2": 6, + "point-baseline": 1, + "probe": 1, + } + assert shards[-1].points[0][CREDIT] == 5.0 + need6 = need_seconds(shards[1], cfg, needs_cut=True) + need8 = need_seconds(shards[0], cfg, needs_cut=True) + assert need6 < need8 + request = make_request(domain, baseline=baseline, deadline_s=LAUNCH_S + need6 + 1) + result = h.sweep(request) + assert result.status == "ok", result.error + assert result.details["shards"]["grid-0"]["reason"] == "launch_too_late" + assert result.details["shards"]["grid-0"]["request"] is None + assert result.details["shards"]["grid-2"]["state"] == "done" + assert_scores(result, request, missing=shards[0].points) + assert ( + "launch_too_late" + in result.metrics[RetuneParams(router=shards[0].points[0]).key()]["error"] + ) + + +def test_failed_run_of_an_ordinary_shard_leaves_its_points_missing(tmp_path): + h = Harness(tmp_path) + startup = {CREDIT: 1.0, SCALE: 1.5, TEMP: 0.0} + request = make_request(baseline=grid(SMALL_DOMAIN)[0], startup=startup) + + def fail_startup(kube, created): + if created["metadata"]["labels"][LABEL_SHARD] == "point-startup": + h.operator.fail_runs[created["metadata"]["name"]] = "SearchFailed" + + h.kube.on_request_created.append(fail_startup) + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request, missing=[startup]) + shard = result.details["shards"]["point-startup"] + assert shard["reason"] == "run_failed" + assert "SearchFailed" in shard["message"] + # A terminal Request is kept for audit. + assert h.registry.deleted == [] + + +def test_failed_run_of_the_baseline_shard_fails_the_window(tmp_path): + h = Harness(tmp_path) + + def fail_grid(kube, created): + if created["metadata"]["labels"][LABEL_SHARD] == "grid-0": + h.operator.fail_runs[created["metadata"]["name"]] = "JobFailed" + + h.kube.on_request_created.append(fail_grid) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "run_failed" + assert "JobFailed" in result.error + probe = h.request_of_shard("probe")["metadata"]["name"] + grid0 = h.request_of_shard("grid-0")["metadata"]["name"] + # The probe may already be done (kept) or in flight (deleted); the failed + # baseline shard is kept for audit. + assert grid0 not in h.registry.deleted + assert set(h.registry.deleted) <= {probe} + + +def test_input_mismatch_of_a_created_critical_shard_fails(tmp_path): + h = Harness(tmp_path) + + def skew(name, effective): + effective["engine"]["context_length"] = 1024 + + h.operator.mutate_input = skew + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "input_mismatch" + assert "engine" in result.error + assert set(h.registry.deleted) == set(h.kube.requests) + + +def test_adopted_attempt_with_input_mismatch_is_replaced(tmp_path): + h = Harness(tmp_path) + request = make_request() + skewed: set[str] = set() + + def skew(name, effective): + if not skewed or name in skewed: + skewed.add(name) + effective["router"]["prefill_load_model"] = {"type": "ais"} + + h.operator.mutate_input = skew + # The first leader creates the Requests and loses leadership before the + # operator accepts them. + h.clock.hooks = [lambda: h.abort.set() if len(h.clock.sleeps) >= 2 else None] + assert h.sweep(request).failure_reason == "lost_leadership" + + h.abort = threading.Event() + h.clock.hooks = [h.operator.step] + result = h.sweep(request) + (bad,) = skewed + assert result.status == "ok", result.error + assert_scores(result, request) + assert bad in h.registry.deleted + replaced = [s for s in result.details["shards"].values() if s["attempt"] == 2] + assert len(replaced) == 1 + # The replacement needed the window file, so the window was cut. + assert len(h.cutter.calls) == 2 + + +def test_adopted_attempt_whose_run_failed_is_replaced(tmp_path): + h = Harness(tmp_path) + request = make_request() + lose_leadership_after_start(h, request) + old = h.request_of_shard("grid-0")["metadata"]["name"] + probe = h.request_of_shard("probe")["metadata"]["name"] + # While no leader runs, the baseline shard's Job fails. + h.kube.complete_run( + h.kube.run_of(old)["metadata"]["name"], reason="JobFailed", message="evicted" + ) + creates = len(h.created()) + + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + shard = result.details["shards"]["grid-0"] + assert shard["attempt"] == 2 + assert not shard["adopted"] + assert shard["request"] not in (old, None) + assert "JobFailed" in shard["message"] + assert result.details["shards"]["probe"]["request"] == probe + assert h.registry.deleted == [old] + assert len(h.created()) == creates + 1 + # The new attempt needed the window file, so the window was cut again. + assert len(h.cutter.calls) == 2 + + +def test_adopted_failed_run_without_time_for_a_new_attempt_fails(tmp_path): + h = Harness(tmp_path) + # The grid shard needs 100 + 60 + 2 * 300 = 760 s with a cut. + request = make_request(deadline_s=LAUNCH_S + 800.0) + lose_leadership_after_start(h, request) + old = h.request_of_shard("grid-0")["metadata"]["name"] + h.kube.complete_run( + h.kube.run_of(old)["metadata"]["name"], reason="JobFailed", message="evicted" + ) + h.clock.now = LAUNCH_S + 100.0 # 700 s are left + creates = len(h.created()) + + result = h.sweep(request) + assert result.status == "failed" + assert result.failure_reason == "run_failed" + assert "JobFailed" in result.error + assert len(h.created()) == creates + assert result.details["shards"]["grid-0"]["attempt"] == 1 + # The failed attempt is kept for audit. + assert old not in h.registry.deleted + + +def test_adopted_failed_run_of_an_ordinary_shard_is_replaced(tmp_path): + h = Harness(tmp_path) + startup = {CREDIT: 1.0, SCALE: 1.5, TEMP: 0.0} + request = make_request(baseline=grid(SMALL_DOMAIN)[0], startup=startup) + lose_leadership_after_start(h, request) + old = h.request_of_shard("point-startup")["metadata"]["name"] + h.kube.complete_run( + h.kube.run_of(old)["metadata"]["name"], + reason="TraceSourceUnavailable", + message="claim is gone", + ) + + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + assert result.details["shards"]["point-startup"]["attempt"] == 2 + assert h.registry.deleted == [old] + + +def test_failed_run_of_a_request_this_sweep_created_is_not_replaced(tmp_path): + h = Harness(tmp_path) + + def grid0(body) -> bool: + return body["metadata"]["labels"][LABEL_SHARD] == "grid-0" + + # The create lands but reports an error, so the sweep adopts its own + # Request by name; its failed Run is not retried. + h.kube.fail("create_request", KubeError("timeout"), landed=True, match=grid0) + + def fail_grid(kube, created): + if grid0(created): + h.operator.fail_runs[created["metadata"]["name"]] = "JobFailed" + + h.kube.on_request_created.append(fail_grid) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "run_failed" + shard = result.details["shards"]["grid-0"] + assert shard["adopted"] and shard["attempt"] == 1 + assert len([b for b in h.created() if grid0(b)]) == 1 + + +def test_input_mismatch_of_an_ordinary_shard_leaves_points_missing(tmp_path): + h = Harness(tmp_path) + startup = {CREDIT: 1.0, SCALE: 1.5, TEMP: 0.0} + request = make_request(baseline=grid(SMALL_DOMAIN)[0], startup=startup) + + def skew(name, effective): + if h.kube.requests[name]["metadata"]["labels"][LABEL_SHARD] == "point-startup": + effective["optimizer"]["parallelism"] = 1 + + h.operator.mutate_input = skew + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request, missing=[startup]) + assert result.details["shards"]["point-startup"]["reason"] == "input_mismatch" + + +def test_missing_run_input_on_a_succeeded_run_is_a_mismatch(tmp_path): + h = Harness(tmp_path, operator=False) + + def operator(): + for name in list(h.kube.requests): + if h.kube.run_of(name) is None: + h.kube.accept_request(name) + run = h.kube.run_of(name)["metadata"]["name"] + h.kube.start_run(run, record_input=False) + h.kube.complete_run(run) + + h.clock.hooks.append(operator) + result = h.sweep(make_request()) + assert result.failure_reason == "input_mismatch" + assert "status.aisimulate.input is missing" in result.error + + +def test_crn_unsupported_runtime_fails_the_window(tmp_path): + h = Harness(tmp_path) + h.operator.dynamo_version = "1.7.0" + result = h.sweep(make_request(), crn_min_version="1.8.0") + assert result.failure_reason == "crn_unsupported" + assert result.details["runtime_version"] == "1.7.0" + + +def test_crn_probe_mismatch_fails_the_window(tmp_path): + def evaluate(request, point): + metrics = default_metrics(point) + if request["metadata"]["labels"][LABEL_SHARD] == "probe": + metrics["goodput_output_throughput_tok_s"] *= 1.0 + 1e-6 + return metrics + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(make_request()) + assert result.failure_reason == "crn_mismatch" + assert result.details["crn_probe"]["relative_error"] == pytest.approx( + 1e-6, rel=1e-3 + ) + + +def test_crn_probe_within_tolerance_passes(tmp_path): + def evaluate(request, point): + metrics = default_metrics(point) + if request["metadata"]["labels"][LABEL_SHARD] == "probe": + metrics["goodput_output_throughput_tok_s"] *= 1.0 + 1e-13 + return metrics + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(make_request()) + assert result.status == "ok", result.error + + +def test_unscored_probe_is_a_crn_mismatch(tmp_path): + probe_point = {CREDIT: 1.0, SCALE: 2.0, TEMP: 0.5} + + def evaluate(request, point): + if request["metadata"]["labels"][LABEL_SHARD] == "probe": + return None + return default_metrics(point) + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(make_request()) + assert result.failure_reason == "crn_mismatch" + assert result.details["crn_probe"]["point"] == probe_point + + +def test_unpublished_and_unscorable_points_are_none(tmp_path): + request = make_request() + dropped = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.0} + broken = {CREDIT: 0.5, SCALE: 2.0, TEMP: 0.0} + + def evaluate(request_obj, point): + if point == public(dropped): + return None + metrics = default_metrics(point) + if point == public(broken): + del metrics["gpu_hours"] + return metrics + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request, missing=[dropped, broken]) + assert "gpu_hours" in result.metrics[RetuneParams(router=broken).key()]["error"] + assert ( + "not published" in result.metrics[RetuneParams(router=dropped).key()]["error"] + ) + assert result.details["missing_candidates"] == 2 + + +def test_scores_follow_the_retune_objective(tmp_path): + h = Harness(tmp_path, objective="TTFT") + request = make_request(objective="TTFT") + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + assert h.created()[0]["spec"]["objective"]["metric"] == "TTFT" + + +def test_engine_drift_between_candidates_is_engine_not_pinned(tmp_path): + h = Harness(tmp_path) + + def drift(name, parameters): + if h.kube.requests[name]["metadata"]["labels"][LABEL_SHARD] == "probe": + parameters["aggregated_parallelism"]["tensor"] = 4 + + h.operator.mutate_parameters = drift + result = h.sweep(make_request()) + assert result.failure_reason == "engine_not_pinned" + + +def test_router_policy_drift_is_engine_not_pinned(tmp_path): + h = Harness(tmp_path) + + def drift(name, parameters): + parameters["adapters"]["dynamo.router"]["prefill_load_model"] = {"type": "ais"} + + h.operator.mutate_parameters = drift + result = h.sweep(make_request()) + assert result.failure_reason == "engine_not_pinned" + + +def test_candidate_with_an_unknown_point_is_ignored(tmp_path): + h = Harness(tmp_path) + request = make_request() + odd = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.0} + + def drift(name, parameters): + router = parameters["adapters"]["dynamo.router"] + if ( + router["overlap_score_credit"], + router["prefill_load_scale"], + router["temperature"], + ) == (0.5, 1.0, 0.0): + router["prefill_load_scale"] = 7.0 + + h.operator.mutate_parameters = drift + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request, missing=[odd]) + assert "not one of the shard's values" in result.details["unmapped_candidates"][0] + + +def test_unscored_baseline_is_reported_for_the_controller(tmp_path): + request = make_request(baseline=grid(SMALL_DOMAIN)[0]) + baseline = request.baseline.router + + def evaluate(request_obj, point): + return None if point == public(baseline) else default_metrics(point) + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(request) + # Status ok: _decide records the window failed for its missing baseline. + assert result.status == "ok" + assert result.scores[request.baseline.key()] is None + + +def test_empty_window_creates_nothing(tmp_path): + h = Harness(tmp_path) + h.cutter = FakeCutter(num_requests=0) + result = h.sweep(make_request()) + assert result.status == "empty" + assert result.failure_reason is None + assert result.error is None + assert h.created() == [] + + +@pytest.mark.parametrize( + "error, reason", + [ + (CutError("window_too_large"), "window_too_large"), + (CutError("trace_corrupt"), "trace_corrupt"), + (CutError("window_cut_timeout"), "window_cut_timeout"), + (CutError("something_else"), "window_cut_failed"), + (OSError("ENOENT"), "window_cut_failed"), + ], +) +def test_cut_failures_fail_the_window(tmp_path, error, reason): + h = Harness(tmp_path) + h.cutter = FakeCutter(error=error) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == reason + assert h.created() == [] + + +def test_transient_list_failures_are_retried(tmp_path): + h = Harness(tmp_path) + h.kube.fail("list_requests", KubeError("timeout"), times=2) + h.kube.fail("list_candidates", KubeError("timeout")) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert result.details["k8s_errors"] == 3 + assert_scores(result, request) + + +def test_permanent_list_failure_fails_the_window(tmp_path): + h = Harness(tmp_path) + error = KubeError("forbidden") + error.status = 403 # type: ignore[attr-defined] + h.kube.fail("list_requests", error) + result = h.sweep(make_request()) + assert result.failure_reason == "k8s_error" + + +def test_unexpected_exception_never_escapes(tmp_path): + h = Harness(tmp_path) + h.kube.fail("list_requests", RuntimeError("boom")) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "internal_error" + assert "boom" in result.error + + +@pytest.mark.parametrize( + "kwargs, reason", + [ + ({"router_domain": None}, "invalid_request"), + ({"planner": {"load_scaling_down_sensitivity": 50}}, "invalid_request"), + ( + {"router_domain": {CREDIT: tuple(float(v) for v in range(500))}}, + "invalid_domain", + ), + ], +) +def test_invalid_requests_fail_cleanly(tmp_path, kwargs, reason): + h = Harness(tmp_path) + if "router_domain" in kwargs and kwargs["router_domain"] is not None: + # A domain whose keys do not match the candidates' router keys. + request = make_request(router_domain=kwargs["router_domain"]) + else: + request = make_request(**kwargs) + result = h.sweep(request) + assert result.status == "failed" + assert result.failure_reason == reason + assert h.created() == [] + + +def test_invalid_engine_fails_cleanly(tmp_path): + h = Harness( + tmp_path, + cfg=make_cfg( + engine={**AGG_ENGINE, "backend": "sglang"}, mount_path=str(tmp_path) + ), + ) + result = h.sweep(make_request()) + assert result.failure_reason == "invalid_engine" + + +def test_deadline_defaults_to_the_request_timeout(tmp_path): + h = Harness(tmp_path) + h.operator.max_phase = "started" + result = h.sweep(make_request(deadline_s=None)) + assert result.status == "timeout" + assert result.details["deadline_s"] == LAUNCH_S + TIMEOUT_S + assert h.clock.now >= LAUNCH_S + TIMEOUT_S + + +def test_sweep_with_the_real_window_cutter(tmp_path): + """The executor and ``window_cut.cut_window`` agree on paths and deadline.""" + + window_cut = pytest.importorskip("dynamo.planner.retune.live.window_cut") + import gzip + import json + import time + + from dynamo.planner.tests.retune_sweep_helpers import trace_rows + + pod_dir = tmp_path / "trace" / "segments" / "llm" / "frontend-0" + pod_dir.mkdir(parents=True) + arrivals = [WINDOW_END_MS - 1000 * i for i in range(1, 6)] + rows = trace_rows(arrivals) + trace_rows([WINDOW_END_MS + 5], prefix="late") + with gzip.open(pod_dir / "req.000001.jsonl.gz", "wt", encoding="utf-8") as out: + out.writelines(json.dumps(row) + "\n" for row in rows) + + h = Harness(tmp_path) + clock = h.clock + + def real_cut(*args, deadline_s, **kwargs): + # The fake clock runs in the past; keep the same budget in real time. + return window_cut.cut_window( + *args, deadline_s=time.time() + (deadline_s - clock.now), **kwargs + ) + + request = make_request(trace_paths=(str(tmp_path / "trace" / "segments" / "llm"),)) + result = h.sweep(request, cut_window=real_cut) + assert result.status == "ok", result.error + assert result.num_requests == 5 + window_file = tmp_path / "trace" / "windows" / "llm" / f"{WINDOW_END_MS}.jsonl.gz" + with gzip.open(window_file, "rt", encoding="utf-8") as lines: + assert len(lines.readlines()) == 5 + claim_path = h.created()[0]["spec"]["workload"]["trace"]["source"]["pvc"]["path"] + assert tmp_path / "trace" / claim_path == window_file + assert_scores(result, request) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py new file mode 100644 index 000000000000..ce64e08551eb --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py @@ -0,0 +1,1889 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the retune imports. + +"""Frontend startup parsing and the spec premises of live retune. + +Covers ``find_frontend`` (v1beta1 and v1alpha1 layouts), ``parse_frontend`` +(the operator's argv defaulting, env merging and ``$(VAR)`` expansion, the +frontend's own argument groups run against an explicit env, a golden +comparison with the real frontend parser, disaggregated maps) and +``evaluate_premises`` (P4, P6-P10) on table-driven DGD fixtures, including +P4 on workers that advertise their own router configuration (compared with +the real worker parser) and the P7 check of ``retune.dgdr.model``. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +import warnings +from typing import Any + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from dynamo.common.configuration.groups.kv_router_args import _KV_ROUTER_FIELDS +from dynamo.frontend.frontend_args import FrontendArgGroup, FrontendConfig +from dynamo.llm import KvRouterConfig +from dynamo.planner.config.planner_config import ( + DgdrExecutorConfig, + PlannerConfig, + RetuneConfig, +) +from dynamo.planner.retune.live import frontend_config as fc +from dynamo.planner.retune.live.contracts import FrontendStartup, PremiseReport + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +DGD = "llm" +NAMESPACE = "team-a" +CLAIM = "request-traces" +MODEL = "Qwen/Qwen3-32B" +TRACE_PATH = "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" +KV_ARGS = ("-m", "dynamo.frontend", "--router-mode", "kv") +DEFAULT_AGG_MAP = { + "decode_active_request_weight": 0.0, + "disk_cache_hit_weight": 0.25, + "host_cache_hit_weight": 0.75, + "overlap_score_credit": 1.0, + "overlap_score_credit_decay": 0.0, + "prefill_load_scale": 1.0, + "router_temperature": 0.0, + "shared_cache_multiplier": 0.0, +} + +PRESET = { + "tensor": 2, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, +} +ENGINE_AGG = { + "mode": "aggregated", + "backend": "vllm", + "backend_version": "0.11.0", + "context_length": 32768, + "workers": { + "aggregated": { + "parallelism": {"preset": [dict(PRESET)]}, + "scheduler": {"max_batched_tokens": 8192, "max_sequences": 256}, + "kv_cache": {"block_size": 64, "capacity": {"memory_fraction": 0.9}}, + }, + "prefill": None, + "decode": None, + }, +} + + +# ------------------------------------------------------------- fixtures + + +def trace_env(**overrides: Any) -> list[dict[str, Any]]: + """The recipe env of design §4.4.1; an override of None drops the entry.""" + + entries: dict[str, dict[str, Any]] = { + "DYN_POD_NAME": { + "name": "DYN_POD_NAME", + "valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}}, + }, + "DYN_REQUEST_TRACE": {"name": "DYN_REQUEST_TRACE", "value": "1"}, + "DYN_REQUEST_TRACE_RECORDS": { + "name": "DYN_REQUEST_TRACE_RECORDS", + "value": "request_end", + }, + "DYN_REQUEST_TRACE_SINKS": {"name": "DYN_REQUEST_TRACE_SINKS", "value": "file"}, + "DYN_REQUEST_TRACE_FILE_FORMAT": { + "name": "DYN_REQUEST_TRACE_FILE_FORMAT", + "value": "jsonl_gz", + }, + "DYN_REQUEST_TRACE_FILE_PATH": { + "name": "DYN_REQUEST_TRACE_FILE_PATH", + "value": TRACE_PATH, + }, + "DYN_REQUEST_TRACE_FILE_ROLL_BYTES": { + "name": "DYN_REQUEST_TRACE_FILE_ROLL_BYTES", + "value": "67108864", + }, + "DYN_REQUEST_TRACE_CAPACITY": { + "name": "DYN_REQUEST_TRACE_CAPACITY", + "value": "8192", + }, + } + for name, entry in overrides.items(): + if entry is None: + entries.pop(name, None) + elif isinstance(entry, dict): + entries[name] = {"name": name, **entry} + else: + entries[name] = {"name": name, "value": entry} + return list(entries.values()) + + +def frontend( + *, + args: tuple[str, ...] | list[str] | None = KV_ARGS, + command: tuple[str, ...] | list[str] | None = ("python3",), + env: list[dict[str, Any]] | None = None, + replicas: int | None = 1, + mounts: list[dict[str, Any]] | None = None, + volumes: list[dict[str, Any]] | None = None, + name: str = "Frontend", + **extra: Any, +) -> dict[str, Any]: + main: dict[str, Any] = { + "name": "main", + "image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0", + "env": trace_env() if env is None else env, + "volumeMounts": ( + [{"name": "traces", "mountPath": "/trace"}] if mounts is None else mounts + ), + } + if command is not None: + main["command"] = list(command) + if args is not None: + main["args"] = list(args) + component: dict[str, Any] = { + "name": name, + "type": "frontend", + "podTemplate": { + "spec": { + "containers": [main], + "volumes": ( + [{"name": "traces", "persistentVolumeClaim": {"claimName": CLAIM}}] + if volumes is None + else volumes + ), + } + }, + **extra, + } + if replicas is not None: + component["replicas"] = replicas + return component + + +def worker( + name: str = "agg", *, type_: str = "worker", model: str = MODEL, **extra: Any +) -> dict[str, Any]: + return { + "name": name, + "type": type_, + "replicas": 2, + "podTemplate": { + "spec": { + "containers": [ + { + "name": "main", + "command": ["python3", "-m", "dynamo.vllm"], + "args": [f"--model={model}", "--tensor-parallel-size=2"], + } + ] + } + }, + **extra, + } + + +def planner(replicas: int = 1) -> dict[str, Any]: + return { + "name": "Planner", + "type": "planner", + "replicas": replicas, + "podTemplate": {"spec": {"containers": [{"name": "main"}]}}, + } + + +def dgd(*components: dict[str, Any], **spec: Any) -> dict[str, Any]: + if not components: + components = (frontend(), worker(), planner()) + return { + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeployment", + "metadata": {"name": DGD, "namespace": NAMESPACE, "uid": "u-1"}, + "spec": {"components": list(components), **spec}, + } + + +def retune_config(router: Any = "default", **dgdr: Any) -> RetuneConfig: + return RetuneConfig( + interval_seconds=21600.0, + timeout_seconds=20400.0, + router=router, + executor="dgdr", + dgdr=DgdrExecutorConfig( + **{ + "claim_name": CLAIM, + "model": MODEL, + "sku": "h200_sxm", + "engine": copy.deepcopy(ENGINE_AGG), + **dgdr, + } + ), + ) + + +def planner_config( + mode: str = "agg", router: Any = "default", **dgdr: Any +) -> PlannerConfig: + # model_construct keeps these tests independent of the live-config + # validators (see test_live_retune_config.py); evaluate_premises reads + # only mode and retune. + return PlannerConfig.model_construct( + mode=mode, retune=retune_config(router, **dgdr) + ) + + +def parse(deployment: dict[str, Any], mode: str = "agg") -> FrontendStartup: + startup = fc.parse_frontend(deployment, planner_mode=mode) + assert isinstance(startup, FrontendStartup), startup + return startup + + +def parse_error(deployment: dict[str, Any], mode: str = "agg") -> str: + result = fc.parse_frontend(deployment, planner_mode=mode) + assert isinstance(result, str), result + return result + + +def evaluate( + deployment: dict[str, Any], + *, + config: PlannerConfig | None = None, + planner_replicas: int | None = None, +) -> PremiseReport: + config = config or planner_config() + result = fc.parse_frontend(deployment, planner_mode=str(config.mode)) + return fc.evaluate_premises( + deployment, + planner_config=config, + startup=result if isinstance(result, FrontendStartup) else None, + parse_error=result if isinstance(result, str) else None, + planner_replicas=planner_replicas, + ) + + +def reasons_matching(report: PremiseReport, prefix: str) -> list[str]: + return [reason for reason in report.reasons if reason.startswith(prefix)] + + +def with_frontend(**kwargs: Any) -> dict[str, Any]: + return dgd(frontend(**kwargs), worker(), planner()) + + +# --------------------------------------------------------- find_frontend + + +def test_find_frontend_v1beta1(): + name, spec = fc.find_frontend(dgd()) + assert name == "Frontend" + assert spec["type"] == "frontend" + + +def test_find_frontend_v1alpha1_services_map(): + deployment = { + "apiVersion": "nvidia.com/v1alpha1", + "kind": "DynamoGraphDeployment", + "metadata": {"name": DGD, "namespace": NAMESPACE}, + "spec": { + "envs": [{"name": "DYN_ROUTER_TEMPERATURE", "value": "0.5"}], + "services": { + "Frontend": { + "componentType": "frontend", + "replicas": 1, + "envs": trace_env(), + "volumeMounts": [{"name": CLAIM, "mountPoint": "/trace"}], + "extraPodSpec": { + "mainContainer": { + "command": ["python3"], + "args": list(KV_ARGS), + } + }, + }, + "VllmWorker": { + "componentType": "worker", + "replicas": 2, + "extraPodSpec": {"mainContainer": {"args": [f"--model {MODEL}"]}}, + }, + "Planner": {"componentType": "planner", "replicas": 1}, + }, + }, + } + name, spec = fc.find_frontend(deployment) + assert name == "Frontend" + assert spec["podTemplate"]["spec"]["containers"][0]["args"] == list(KV_ARGS) + + startup = parse(deployment) + assert startup.router_mode == "kv_router" + assert startup.router_config == {"router_temperature": 0.5} + report = evaluate(deployment) + assert report.spec_ok, report.reasons + assert fc.served_model_name(deployment) == MODEL + + +@pytest.mark.parametrize( + ("components", "match"), + [ + ((worker(), planner()), "no frontend component"), + ( + (frontend(), frontend(name="Frontend2"), worker()), + "2 frontend components", + ), + ((worker(frontendSidecar="sidecar-frontend"),), "frontendSidecar of agg"), + (({"name": "Epp", "type": "epp"}, worker()), "EPP component"), + ], +) +def test_find_frontend_rejects(components, match): + with pytest.raises(fc.FrontendParseError, match=match): + fc.find_frontend(dgd(*components)) + assert parse_error(dgd(*components)).startswith("P7:") + + +# ----------------------------------------------------- argv and parsing + + +def test_parse_recipe_frontend(): + startup = parse(dgd()) + assert startup.router_mode == "kv_router" + assert startup.router_config == {} + assert startup.startup_maps == {"aggregated": DEFAULT_AGG_MAP} + assert startup.frontend_component == "Frontend" + assert startup.replicas == 1 + assert startup.replica_sync is False + assert json.loads(startup.fingerprint_source) == { + "router_mode": "kv_router", + "router_config": {}, + "startup_maps": {"aggregated": DEFAULT_AGG_MAP}, + } + assert startup.fingerprint_source == fc.canonical_json( + json.loads(startup.fingerprint_source) + ) + + +@pytest.mark.parametrize( + ("command", "args", "mode"), + [ + # Operator default: python3 -m dynamo.frontend (round-robin). + (None, None, "round_robin"), + # Authored args keep the default python3 command. + (None, KV_ARGS, "kv_router"), + (("python3",), KV_ARGS, "kv_router"), + (("python3", "-m", "dynamo.frontend"), ("--router-mode", "kv"), "kv_router"), + ( + ("/usr/bin/python3", "-u", "-X", "dev", "-m", "dynamo.frontend"), + ("--router-mode=kv",), + "kv_router", + ), + (("python", "-mdynamo.frontend", "--router-mode", "kv"), None, "kv_router"), + (("python3",), ("-m", "dynamo.frontend", "--router-mode", "random"), "random"), + ( + ("python3",), + ("-m", "dynamo.frontend", "--router-mode", "power-of-two"), + "power_of_two", + ), + ], +) +def test_effective_argv(command, args, mode): + startup = parse(with_frontend(command=command, args=args)) + assert startup.router_mode == mode + + +def test_authored_command_drops_default_args(): + # graph.go: an authored frontend command clears the default args, so the + # command alone must name the module. + error = parse_error(with_frontend(command=("python3",), args=())) + assert "does not run python -m dynamo.frontend" in error + + +@pytest.mark.parametrize( + "command,args", + [ + (("/bin/sh", "-c"), ("python3 -m dynamo.frontend --router-mode kv",)), + (("bash", "-lc", "exec python3 -m dynamo.frontend --router-mode kv"), None), + (("python3", "-c", "import runpy"), None), + ], +) +def test_shell_wrapped_command_is_unparseable(command, args): + error = parse_error(with_frontend(command=command, args=args)) + assert error.startswith("P4:") + assert "shell-wrapped" in error + + +@pytest.mark.parametrize( + "command,args,match", + [ + (("dynamo-frontend",), ("--router-mode", "kv"), "not a python interpreter"), + (("python3",), ("-m", "dynamo.vllm"), "does not run python -m dynamo.frontend"), + (("python3",), ("serve.py",), "does not run python -m dynamo.frontend"), + ], +) +def test_non_frontend_command_is_unparseable(command, args, match): + assert match in parse_error(with_frontend(command=command, args=args)) + + +def test_frontend_roles_are_unparseable(): + error = parse_error(dgd(frontend(roles=[{"name": "leader"}]), worker())) + assert "uses roles" in error + + +def test_env_sets_router_mode_and_knobs(): + env = trace_env( + DYN_ROUTER_MODE="kv", + DYN_ROUTER_TEMPERATURE="0.5", + DYN_ROUTER_KV_OVERLAP_SCORE_CREDIT="0.25", + ) + startup = parse(with_frontend(args=("-m", "dynamo.frontend"), env=env)) + assert startup.router_mode == "kv_router" + assert startup.router_config == { + "router_temperature": 0.5, + "overlap_score_credit": 0.25, + } + assert startup.startup_maps["aggregated"]["router_temperature"] == 0.5 + assert startup.startup_maps["aggregated"]["overlap_score_credit"] == 0.25 + + +def test_cli_wins_over_env_and_container_env_over_graph_env(): + env = trace_env(DYN_ROUTER_TEMPERATURE="0.5", DYN_ROUTER_PREFILL_LOAD_SCALE="2") + deployment = dgd( + frontend(args=(*KV_ARGS, "--router-temperature", "0.2"), env=env), + worker(), + env=[ + {"name": "DYN_ROUTER_PREFILL_LOAD_SCALE", "value": "3"}, + {"name": "DYN_ROUTER_KV_OVERLAP_SCORE_CREDIT", "value": "0.5"}, + ], + ) + assert parse(deployment).router_config == { + "router_temperature": 0.2, + "prefill_load_scale": 2.0, + "overlap_score_credit": 0.5, + } + + +def test_planner_process_env_is_ignored(monkeypatch): + monkeypatch.setenv("DYN_ROUTER_MODE", "kv") + monkeypatch.setenv("DYN_ROUTER_TEMPERATURE", "0.9") + monkeypatch.setenv("DYN_ROUTER_REPLICA_SYNC", "1") + before = dict(os.environ) + startup = parse(with_frontend(args=("-m", "dynamo.frontend"))) + assert startup.router_mode == "round_robin" + assert startup.router_config == {} + assert startup.replica_sync is False + assert dict(os.environ) == before + + +def test_abbreviated_flag_matches_frontend_parser(): + # The frontend parses with argparse abbreviations enabled. + startup = parse(with_frontend(args=(*KV_ARGS, "--router-temp", "0.4"))) + assert startup.router_config == {"router_temperature": 0.4} + + +def test_unknown_frontend_args_are_ignored(): + args = ( + *KV_ARGS, + "--http-port", + "8000", + "--kv-cache-block-size", + "64", + "--trust-remote-code", + "--some-vllm-processor-flag", + "x", + ) + assert parse(with_frontend(args=args)).router_config == {} + + +def test_legacy_and_removed_flags_parse_quietly(caplog): + args = ( + *KV_ARGS, + "--no-kv-events", + "--admission-control", + "none", + "--enforce-disagg", + ) + startup = parse(with_frontend(args=args)) + assert startup.router_config == {"use_kv_events": False} + assert not [r for r in caplog.records if r.levelname == "WARNING"] + + +def test_replica_sync_and_disagg_maps(): + startup = parse( + with_frontend( + args=(*KV_ARGS, "--router-replica-sync", "--router-temperature", "0.3") + ), + mode="disagg", + ) + assert startup.replica_sync is True + assert startup.router_config == { + "router_replica_sync": True, + "router_temperature": 0.3, + } + assert set(startup.startup_maps) == {"prefill", "decode"} + prefill, decode = startup.startup_maps["prefill"], startup.startup_maps["decode"] + assert prefill["overlap_score_credit"] == 1.0 + assert prefill["host_cache_hit_weight"] == 0.75 + # Plain decode routers report zero cache credits. + assert decode["overlap_score_credit"] == 0.0 + assert decode["host_cache_hit_weight"] == 0.0 + assert decode["disk_cache_hit_weight"] == 0.0 + assert decode["router_temperature"] == 0.3 + + +def test_zero_credit_links_cache_weights(): + startup = parse( + with_frontend(args=(*KV_ARGS, "--router-kv-overlap-score-credit", "0")) + ) + agg = startup.startup_maps["aggregated"] + assert agg["overlap_score_credit"] == 0.0 + assert agg["host_cache_hit_weight"] == 0.0 + assert agg["disk_cache_hit_weight"] == 0.0 + + +def test_load_aware_implies_kv_mode(): + startup = parse(with_frontend(args=("-m", "dynamo.frontend", "--load-aware"))) + assert startup.router_mode == "kv_router" + assert startup.router_config["use_kv_events"] is False + assert startup.router_config["overlap_score_credit"] == 0.0 + + +def test_fingerprint_ignores_replicas_but_not_router_config(): + base = parse(dgd(frontend(replicas=1), worker())) + scaled = parse(dgd(frontend(replicas=3), worker())) + tuned = parse( + dgd(frontend(args=(*KV_ARGS, "--router-temperature", "0.1")), worker()) + ) + assert scaled.replicas == 3 + assert base.fingerprint_source == scaled.fingerprint_source + assert base.fingerprint_source != tuned.fingerprint_source + + +def test_omitted_replicas_default_to_one(): + assert parse(dgd(frontend(replicas=None), worker())).replicas == 1 + + +# -------------------------------------------- env references and sources + + +def test_arg_expands_literal_env(): + env = trace_env(TEMP="0.3") + args = (*KV_ARGS, "--router-temperature", "$(TEMP)") + assert parse(with_frontend(args=args, env=env)).router_config == { + "router_temperature": 0.3 + } + + +def test_escaped_reference_stays_literal(): + # $$(TEMP) is the literal "$(TEMP)", which is not a float. + env = trace_env(TEMP="0.3") + args = (*KV_ARGS, "--router-temperature", "$$(TEMP)") + error = parse_error(with_frontend(args=args, env=env)) + assert "cannot parse the frontend args" in error + + +def test_arg_referencing_value_from_is_unparseable(): + env = trace_env(TEMP={"valueFrom": {"configMapKeyRef": {"name": "c", "key": "t"}}}) + args = (*KV_ARGS, "--router-temperature", "$(TEMP)") + error = parse_error(with_frontend(args=args, env=env)) + assert "$(TEMP)" in error and "cannot read" in error + + +def test_opaque_string_router_arg_is_unparseable(): + env = trace_env(MODE={"valueFrom": {"secretKeyRef": {"name": "s", "key": "k"}}}) + args = ("-m", "dynamo.frontend", "--router-tracking-key-id", "id-$(MODE)") + error = parse_error(with_frontend(args=args, env=env)) + assert "router_tracking_key_id" in error + + +def test_arg_that_is_an_opaque_reference_is_unparseable(): + env = trace_env(EXTRA={"valueFrom": {"secretKeyRef": {"name": "s", "key": "k"}}}) + error = parse_error(with_frontend(args=(*KV_ARGS, "$(EXTRA)"), env=env)) + assert "$(EXTRA)" in error + + +def test_opaque_reference_in_a_non_router_arg_is_fine(): + args = (*KV_ARGS, "--metrics-prefix", "fe-$(POD_NAME)") + assert parse(with_frontend(args=args)).router_config == {} + + +OPAQUE_PORT = {"valueFrom": {"configMapKeyRef": {"name": "c", "key": "port"}}} + + +@pytest.mark.parametrize( + "args", + [ + # Typed non-router flags keep the unreadable value unconverted. + (*KV_ARGS, "--http-port", "$(HTTP_PORT)"), + (*KV_ARGS, "--http-port=$(HTTP_PORT)"), + (*KV_ARGS, "--grpc-metrics-port", "$(POD_UID)"), + # A non-router choices flag. + (*KV_ARGS, "--dyn-chat-processor", "$(HTTP_PORT)"), + # The value of a flag the frontend hands to its chat processor. + ( + *KV_ARGS, + "--dyn-chat-processor", + "vllm", + "--tool-call-parser", + "$(HTTP_PORT)", + ), + (*KV_ARGS, "--dyn-chat-processor", "vllm", "--tool-call-parser=$(HTTP_PORT)"), + ], +) +def test_opaque_reference_in_a_typed_non_router_arg_is_fine(args): + startup = parse(with_frontend(args=args, env=trace_env(HTTP_PORT=OPAQUE_PORT))) + assert startup.router_mode == "kv_router" + assert startup.router_config == {} + + +@pytest.mark.parametrize( + ("args", "match"), + [ + # Router flags whose value is unreadable. + ( + ("-m", "dynamo.frontend", "--router-mode", "$(HTTP_PORT)"), + "router arg references $(HTTP_PORT)", + ), + ( + (*KV_ARGS, "--router-temp", "$(HTTP_PORT)"), + "router arg references $(HTTP_PORT)", + ), + ( + (*KV_ARGS, "--router-ttl-secs=$(HTTP_PORT)"), + "router arg references $(HTTP_PORT)", + ), + # Left over where a flag could stand. + ( + (*KV_ARGS, "--router-kv-events", "$(HTTP_PORT)"), + "could set a router flag", + ), + ((*KV_ARGS, "--$(HTTP_PORT)"), "could set a router flag"), + ((*KV_ARGS, "--$(HTTP_PORT)=1"), "could set a router flag"), + ( + (*KV_ARGS, "--tool-call-parser", "--$(HTTP_PORT)", "$(HTTP_PORT)"), + "could set a router flag", + ), + ], +) +def test_opaque_reference_that_could_set_a_router_flag_is_unparseable(args, match): + error = parse_error(with_frontend(args=args, env=trace_env(HTTP_PORT=OPAQUE_PORT))) + assert error.startswith("P4:") + assert match in error + + +def test_router_env_from_value_from_is_unparseable(): + env = trace_env( + DYN_ROUTER_TEMPERATURE={ + "valueFrom": {"configMapKeyRef": {"name": "c", "key": "t"}} + } + ) + error = parse_error(with_frontend(env=env)) + assert "DYN_ROUTER_TEMPERATURE" in error and "valueFrom" in error + + +def test_router_env_referencing_value_from_is_unparseable(): + env = trace_env( + A_TEMP={"valueFrom": {"configMapKeyRef": {"name": "c", "key": "t"}}}, + DYN_ROUTER_TEMPERATURE="$(A_TEMP)", + ) + assert "DYN_ROUTER_TEMPERATURE" in parse_error(with_frontend(env=env)) + + +def test_router_env_references_earlier_literal(): + env = trace_env(A_TEMP="0.7", DYN_ROUTER_TEMPERATURE="$(A_TEMP)") + assert parse(with_frontend(env=env)).router_config == {"router_temperature": 0.7} + + +def test_non_router_value_from_is_fine(): + env = trace_env( + HF_TOKEN={"valueFrom": {"secretKeyRef": {"name": "hf", "key": "t"}}} + ) + assert parse(with_frontend(env=env)).router_mode == "kv_router" + + +@pytest.mark.parametrize( + ("source", "warned"), + [ + ({"secretRef": {"name": "hf-token-secret"}}, True), + ({"configMapRef": {"name": "router"}, "prefix": "DYN_"}, True), + ({"configMapRef": {"name": "router"}, "prefix": "D"}, True), + ({"secretRef": {"name": "hf-token-secret"}, "prefix": "HF_"}, False), + ], +) +def test_env_from_is_parsed_and_warned_when_it_may_set_router_env(source, warned): + deployment = with_frontend() + deployment["spec"]["components"][0]["podTemplate"]["spec"]["containers"][0][ + "envFrom" + ] = [source] + assert isinstance( + fc.parse_frontend(deployment, planner_mode="agg"), FrontendStartup + ) + report = evaluate(deployment) + envfrom_warnings = [w for w in report.warnings if "envFrom" in w] + assert bool(envfrom_warnings) == warned + assert not reasons_matching(report, "P4") + + +def test_router_policy_config_is_unparseable(): + error = parse_error( + with_frontend(args=(*KV_ARGS, "--router-policy-config", "/etc/policy.yaml")) + ) + assert "--router-policy-config" in error + + +@pytest.mark.parametrize( + "args,env,match", + [ + ((*KV_ARGS, "--router-kv-overlap-score-credit", "-1"), {}, "invalid"), + ((*KV_ARGS, "--router-temperature", "hot"), {}, "cannot parse"), + ((*KV_ARGS,), {"DYN_ROUTER_TEMPERATURE": "hot"}, "invalid frontend env"), + ( + (*KV_ARGS, "--router-approximate-cache-policy", "lru"), + {}, + "invalid", + ), + ], +) +def test_invalid_router_config_is_unparseable(args, env, match): + error = parse_error(with_frontend(args=args, env=trace_env(**env))) + assert error.startswith("P4:") + assert match in error + + +# ------------------------------------------------------ parser fidelity + + +GOLDEN_CASES = [ + ((*KV_ARGS,), {}), + ((*KV_ARGS, "--router-temperature", "0.5", "--router-replica-sync"), {}), + ( + ("-m", "dynamo.frontend"), + {"DYN_ROUTER_MODE": "kv", "DYN_ROUTER_USE_KV_EVENTS": "0"}, + ), + ((*KV_ARGS, "--no-kv-events", "--router-ttl-secs", "30"), {}), + ( + (*KV_ARGS, "--router-kv-overlap-score-credit", "0"), + {"DYN_ROUTER_TEMPERATURE": "0.1"}, + ), + (("-m", "dynamo.frontend", "--load-aware"), {}), + ((*KV_ARGS,), {"DYN_OVERLAP_SCORE_WEIGHT": "2"}), + ( + (*KV_ARGS,), + {"DYN_ROUTER_KV_OVERLAP_SCORE_WEIGHT": "0", "DYN_OVERLAP_SCORE_WEIGHT": "2"}, + ), + ((*KV_ARGS, "--router-temp", "0.25", "--router-event-threads", "8"), {}), + ( + (*KV_ARGS, "--router-tracking-hash", "keyed-xxh3-v1"), + {"DYN_ROUTER_TRACKING_KEY_FILE": "/k", "DYN_ROUTER_TRACKING_KEY_ID": "e1"}, + ), +] + + +@pytest.mark.parametrize(("args", "env"), GOLDEN_CASES) +def test_matches_real_frontend_parser(monkeypatch, args, env): + """The frontend's own parser, run with its env in os.environ, agrees.""" + + for name in list(os.environ): + if name.startswith("DYN_"): + monkeypatch.delenv(name) + for name, value in env.items(): + monkeypatch.setenv(name, value) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + parser = argparse.ArgumentParser() + FrontendArgGroup().add_arguments(parser) + namespace, _ = parser.parse_known_args(list(args[2:])) + config = FrontendConfig.from_cli_args(namespace) + if config.load_aware: + config.router_mode = "kv" + config.apply_load_aware_preset() + config.apply_conditional_disagg_config() + expected = json.loads( + KvRouterConfig(**{f: getattr(config, f) for f in _KV_ROUTER_FIELDS}).to_json() + ) + + for name in env: + monkeypatch.delenv(name) + startup = parse(with_frontend(args=args, env=trace_env(**env))) + actual = json.loads( + KvRouterConfig.from_json(json.dumps(startup.router_config)).to_json() + ) + assert actual == expected + assert startup.router_mode == fc._router_mode(config.router_mode) + + +def test_router_fields_are_all_covered(): + """Every frontend router field is in the JSON or checked on the flags.""" + + serialized = set( + json.loads( + KvRouterConfig( + router_tracking_hash="keyed-xxh3-v1", + router_tracking_key_file="/k", + router_tracking_key_id="e", + decode_active_request_weight=1.0, + enable_session_prefix_index=True, + shared_cache_multiplier=0.5, + conditional_disagg_enabled=True, + conditional_disagg_policy="isl_or_load", + conditional_disagg_eff_isl_threshold=1, + conditional_disagg_eff_isl_ratio_threshold=0.5, + conditional_disagg_prefill_busy_threshold=0.5, + conditional_disagg_decode_busy_threshold=0.5, + ).to_json() + ) + ) + covered = serialized | set(fc._SERDE_SKIPPED_DEFAULTS) | {"overlap_score_weight"} + assert set(_KV_ROUTER_FIELDS) <= covered + + +# --------------------------------------------------------- P4 premises + + +def test_recipe_dgd_satisfies_every_premise(): + report = evaluate(dgd()) + assert report.p4_ok and report.spec_ok, report.reasons + assert report.reasons == () + assert report.startup is not None + assert not [w for w in report.warnings if w.startswith("P6")] + + +def test_validated_planner_config_satisfies_every_premise(): + config = PlannerConfig( + namespace="test-ns", + environment="kubernetes", + mode="agg", + enable_load_scaling=False, + enable_throughput_scaling=False, + retune=retune_config().model_dump(), + ) + report = evaluate(dgd(), config=config) + assert report.spec_ok, report.reasons + + +def test_parse_error_fails_p4_only(): + deployment = with_frontend( + command=("/bin/sh", "-c"), args=("python3 -m dynamo.frontend",) + ) + report = evaluate(deployment) + assert not report.p4_ok and not report.spec_ok + assert len(report.reasons) == 1 and "shell-wrapped" in report.reasons[0] + assert report.startup is None + + +def test_caller_parse_error_is_reported(): + report = fc.evaluate_premises( + dgd(), + planner_config=planner_config(), + startup=None, + parse_error="P4: injected", + planner_replicas=None, + ) + assert not report.p4_ok + assert report.reasons == ("P4: injected",) + + +@pytest.mark.parametrize( + ("args", "env", "match"), + [ + (("-m", "dynamo.frontend"), {}, "routes with 'round_robin'"), + ( + ("-m", "dynamo.frontend", "--router-mode", "random"), + {}, + "routes with 'random'", + ), + ((*KV_ARGS, "--no-router-kv-events"), {}, "use_kv_events=False"), + ( + (*KV_ARGS, "--router-host-cache-hit-weight", "0.5"), + {}, + "host_cache_hit_weight=0.5", + ), + ( + (*KV_ARGS, "--router-queue-threshold", "0.5"), + {}, + "router_queue_threshold=0.5", + ), + ( + (*KV_ARGS, "--router-kv-overlap-score-credit-decay", "1"), + {}, + "overlap_score_credit_decay", + ), + ( + (*KV_ARGS, "--router-decode-active-request-weight", "1"), + {}, + "decode_active_request_weight", + ), + ( + (*KV_ARGS, "--router-prefill-load-model", "ais"), + {}, + "router_prefill_load_model is 'aic'", + ), + ( + (*KV_ARGS, "--router-conditional-disagg"), + {}, + "conditional disaggregation", + ), + ( + ( + *KV_ARGS, + "--no-router-kv-events", + "--router-approximate-cache-policy", + "lru", + ), + {}, + "router_approximate_cache_policy='lru'", + ), + ((*KV_ARGS, "--router-prefill-policy", "default"), {}, "router_prefill_policy"), + ((*KV_ARGS, "--router-decode-policy", "default"), {}, "router_decode_policy"), + ((*KV_ARGS,), {"DYN_OVERLAP_SCORE_WEIGHT": "2"}, "overlap_score_weight"), + ( + ("-m", "dynamo.frontend"), + {"DYN_ROUTER_LOAD_AWARE": "1"}, + "load-aware preset", + ), + ( + (*KV_ARGS,), + {"DYN_ROUTER_WORKER_SELECTION_POLICY": "mine"}, + "worker-selection", + ), + ( + (*KV_ARGS,), + {"DYN_ROUTER_POSITIONAL_SEARCH_MODE": "x"}, + "DYN_ROUTER_POSITIONAL_SEARCH_MODE", + ), + ( + (*KV_ARGS,), + {"DYN_ROUTER_SESSION_AFFINITY_TTL_SECS": "60"}, + "session_affinity_ttl_secs=60", + ), + ( + (*KV_ARGS, "--active-decode-blocks-threshold", "0.9"), + {}, + "active_decode_blocks_threshold", + ), + ( + (*KV_ARGS, "--active-prefill-tokens-threshold", "100"), + {}, + "active_prefill_tokens_threshold", + ), + ], +) +def test_p4_rejects(args, env, match): + report = evaluate(with_frontend(args=args, env=trace_env(**env))) + assert not report.p4_ok and not report.spec_ok + assert any( + match in reason for reason in reasons_matching(report, "P4:") + ), report.reasons + + +@pytest.mark.parametrize( + ("args", "env"), + [ + # Searched knobs may start anywhere. + ( + ( + *KV_ARGS, + "--router-temperature", + "0.5", + "--router-prefill-load-scale", + "2", + ), + {}, + ), + ((*KV_ARGS, "--router-kv-overlap-score-credit", "0.5"), {}), + # Tolerated fields. + ((*KV_ARGS, "--router-event-threads", "8", "--router-replica-sync"), {}), + ((*KV_ARGS, "--enable-session-prefix-index", "--serve-indexer"), {}), + ( + (*KV_ARGS, "--router-tracking-hash", "keyed-xxh3-v1"), + {"DYN_ROUTER_TRACKING_KEY_FILE": "/k", "DYN_ROUTER_TRACKING_KEY_ID": "e"}, + ), + # Frontend-only and transport settings. + ( + (*KV_ARGS,), + { + "DYN_ROUTER_MIN_INITIAL_WORKERS": "2", + "DYN_ROUTER_ZMQ_ENDPOINTS_PER_SUB": "4", + }, + ), + ((*KV_ARGS, "--router-session-affinity-mode", "soft"), {}), + ], +) +def test_p4_accepts(args, env): + report = evaluate(with_frontend(args=args, env=trace_env(**env))) + assert report.p4_ok, report.reasons + + +def test_p4_unsearched_knob_when_router_domain_is_pinned(): + # A scalar still searches (pins) the knob: it stays in the domain. + config = planner_config( + router={ + "overlap_score_credit": 1.0, + "prefill_load_scale": 1.0, + "temperature": 0.0, + } + ) + report = evaluate( + with_frontend(args=(*KV_ARGS, "--router-temperature", "0.5")), config=config + ) + assert report.p4_ok, report.reasons + + +def test_p4_without_router_search_requires_all_defaults(): + config = planner_config(router=None) + ok = evaluate(dgd(), config=config) + assert ok.p4_ok, ok.reasons + report = evaluate( + with_frontend(args=(*KV_ARGS, "--router-temperature", "0.5")), config=config + ) + assert any("router_temperature=0.5" in r for r in report.reasons) + assert "(none)" in report.reasons[0] + + +def test_p4_zero_startup_credit_rejects_positive_candidates(): + zero = (*KV_ARGS, "--router-kv-overlap-score-credit", "0") + # The default domain collapses the omitted credit to [0.0]. + assert evaluate(with_frontend(args=zero)).p4_ok + config = planner_config( + router={"overlap_score_credit": {"choices": [0.0, 0.5]}, "temperature": 0.0} + ) + report = evaluate(with_frontend(args=zero), config=config) + assert not report.p4_ok + assert "overlap_score_credit > 0" in report.reasons[0] + + +def test_p4_disagg_decode_credit_zero_is_not_a_zero_credit_startup(): + # Plain decode routers always report credit 0; only prefill counts. + config = planner_config(mode="disagg") + report = evaluate( + dgd( + frontend(), + worker("prefill", type_="prefill"), + worker("decode", type_="decode"), + planner(), + ), + config=config, + ) + assert report.p4_ok, report.reasons + assert report.startup is not None + assert set(report.startup.startup_maps) == {"prefill", "decode"} + + +def test_p4_written_load_model_must_match_startup(): + config = planner_config(router={"prefill_load_model": {"type": "ais"}}) + report = evaluate(dgd(), config=config) + assert not report.p4_ok + assert "prefill_load_model" in report.reasons[0] + + +@pytest.mark.parametrize( + ("declared", "match"), + [ + ({}, None), + ({"router_temperature": 0.5}, None), + ({"router_temperature": 0.5, "use_kv_events": True}, None), + ({"router_temperature": 0.4}, "router_temperature: declared 0.4, frontend 0.5"), + ({}, "router_temperature: declared 0.0"), + ({"not_a_field": 1}, "not a KvRouterConfig JSON"), + ], +) +def test_startup_router_config_must_match(declared, match): + config = planner_config(startup_router_config=declared) + args = (*KV_ARGS, "--router-temperature", "0.5") + if declared == {} and match is None: + args = KV_ARGS + report = evaluate(with_frontend(args=args), config=config) + if match is None: + assert report.p4_ok, report.reasons + else: + assert not report.p4_ok + assert any(match in r for r in report.reasons), report.reasons + + +# ------------------------------------------- P4: worker router advertisement + + +SECRET = {"valueFrom": {"secretKeyRef": {"name": "s", "key": "k"}}} + + +def router_worker( + *args: str, + env: list[dict[str, Any]] | None = None, + name: str = "agg", + type_: str = "worker", +) -> dict[str, Any]: + component = worker(name, type_=type_) + main = component["podTemplate"]["spec"]["containers"][0] + main["args"] = [*main["args"], *args] + if env is not None: + main["env"] = env + return component + + +def shell_worker(script: str) -> dict[str, Any]: + component = worker() + component["podTemplate"]["spec"]["containers"][0].update( + command=["/bin/sh", "-c"], args=[script] + ) + return component + + +def minimax_prefill_worker(*router_args: str) -> dict[str, Any]: + """The PrefillWorker of recipes/minimax-m3/vllm/disagg-gb200-agentic.""" + + return { + "name": "PrefillWorker", + "type": "prefill", + "replicas": 3, + "podTemplate": { + "spec": { + "containers": [ + { + "name": "main", + "command": ["python3"], + "args": [ + "-m", + "dynamo.vllm", + *router_args, + "--model", + "$(MODEL_NAME)", + "--served-model-name", + "$(SERVED_MODEL_NAME)", + "--tensor-parallel-size", + "4", + "--speculative-config", + "$(SPECULATIVE_CONFIG)", + "--disaggregation-mode", + "prefill", + "--kv-transfer-config", + "$(KV_TRANSFER_CONFIG)", + "--kv-events-config", + '{"enable_kv_cache_events":true,"publisher":"zmq"}', + ], + "env": [ + { + "name": "KV_TRANSFER_CONFIG", + "value": '{"kv_connector":"NixlConnector",' + '"kv_role":"kv_both"}', + }, + { + "name": "SPECULATIVE_CONFIG", + "valueFrom": { + "configMapKeyRef": { + "name": "minimax-m3-config", + "key": "speculative-config", + } + }, + }, + {"name": "MODEL_NAME", "value": "nvidia/MiniMax-M3-NVFP4"}, + { + "name": "SERVED_MODEL_NAME", + "value": "nvidia/MiniMax-M3-NVFP4", + }, + ], + } + ] + } + }, + } + + +@pytest.mark.parametrize( + ("args", "env", "match"), + [ + (("--router-mode", "kv"), None, "advertises its own router configuration"), + (("--router-mode=round-robin",), None, "router mode 'round-robin'"), + (("--router-m", "kv", "--no-router-kv-events"), None, "router mode 'kv'"), + ((), [{"name": "DYN_ROUTER_MODE", "value": "kv"}], "router mode 'kv'"), + ((), [{"name": "DYN_ROUTER_MODE", "value": ""}], "router mode ''"), + ( + (), + [{"name": "DYN_ROUTER_MODE", **SECRET}], + "DYN_ROUTER_MODE through valueFrom", + ), + ( + (), + [ + {"name": "A_MODE", **SECRET}, + {"name": "DYN_ROUTER_MODE", "value": "$(A_MODE)"}, + ], + "DYN_ROUTER_MODE through valueFrom", + ), + ( + ("--router-mode", "$(A_MODE)"), + [{"name": "A_MODE", **SECRET}], + "router flag that references $(A_MODE)", + ), + ( + ("--router-mode", "$(A_MODE)"), + [{"name": "A_MODE", "value": "kv"}], + "router mode 'kv'", + ), + (("--router-mode", "fastest"), None, "cannot parse"), + ( + (), + [{"name": "DYN_ROUTER_TEMPERATURE", "value": "hot"}], + "invalid router env", + ), + ], +) +def test_p4_rejects_a_worker_that_advertises_a_router_config(args, env, match): + report = evaluate(dgd(frontend(), router_worker(*args, env=env), planner())) + assert not report.p4_ok and not report.spec_ok + assert report.startup is not None # the frontend itself parses + rejected = reasons_matching(report, "P4: worker component agg ") + assert any(match in reason for reason in rejected), report.reasons + assert all("on the frontend only" in reason for reason in rejected) + + +@pytest.mark.parametrize( + ("args", "env"), + [ + # Router flags and env without a mode advertise nothing. + (("--no-router-kv-events", "--router-temperature", "0.5"), None), + ((), [{"name": "DYN_ROUTER_TEMPERATURE", "value": "0.5"}]), + ((), [{"name": "DYN_ROUTER_MODE_HINT", "value": "kv"}]), + # Unreadable env in a non-mode argument. + (("--router-temperature", "$(A_SECRET)"), [{"name": "A_SECRET", **SECRET}]), + ( + ("--speculative-config", "$(A_SECRET)", "$(A_SECRET)"), + [{"name": "A_SECRET", **SECRET}], + ), + ], +) +def test_p4_accepts_workers_that_advertise_nothing(args, env): + report = evaluate(dgd(frontend(), router_worker(*args, env=env), planner())) + assert report.p4_ok and report.spec_ok, report.reasons + + +def test_p4_rejects_graph_env_router_mode_reaching_the_workers(): + # recipes/solar-open2-250b: a graph-level DYN_ROUTER_MODE reaches the + # frontend and every worker. + deployment = dgd( + frontend(args=("-m", "dynamo.frontend")), + worker("prefill", type_="prefill"), + worker("decode", type_="decode"), + planner(), + env=[{"name": "DYN_ROUTER_MODE", "value": "kv"}], + ) + report = evaluate(deployment, config=planner_config("disagg")) + assert report.startup is not None and report.startup.router_mode == "kv_router" + assert not report.p4_ok + assert [r.split(" advertises")[0] for r in reasons_matching(report, "P4:")] == [ + "P4: worker component prefill", + "P4: worker component decode", + ] + + +def test_p4_checks_the_real_recipe_prefill_worker(): + config = planner_config("disagg", model="nvidia/MiniMax-M3-NVFP4") + advertising = dgd( + frontend(), + minimax_prefill_worker( + "--router-mode", "kv", "--no-router-track-output-blocks" + ), + worker("DecodeWorker", type_="decode", model="nvidia/MiniMax-M3-NVFP4"), + planner(), + ) + report = evaluate(advertising, config=config) + assert not report.p4_ok + assert reasons_matching( + report, + "P4: worker component PrefillWorker advertises its own router " + "configuration (router mode 'kv'", + ) + + # Without the router flags the worker inherits the frontend's routing; + # its unreadable --speculative-config value does not matter, and its + # $(MODEL_NAME) / $(SERVED_MODEL_NAME) expand. + inheriting = dgd( + frontend(), + minimax_prefill_worker(), + worker("DecodeWorker", type_="decode", model="nvidia/MiniMax-M3-NVFP4"), + planner(), + ) + report = evaluate(inheriting, config=config) + assert report.p4_ok and report.spec_ok, report.reasons + assert not [w for w in report.warnings if w.startswith("P7:")], report.warnings + assert fc.served_model_name(inheriting) == "nvidia/MiniMax-M3-NVFP4" + + +@pytest.mark.parametrize( + ("script", "match"), + [ + (f"python3 -m dynamo.vllm --model {MODEL} --router-mode kv", "router mode"), + ( + f"export DYN_ROUTER_MODE=kv && exec python3 -m dynamo.vllm --model {MODEL}", + "sets DYN_ROUTER_MODE in its shell command", + ), + ( + f"DYN_ROUTER_MODE=kv python3 -m dynamo.vllm --model {MODEL}", + "sets DYN_ROUTER_MODE in its shell command", + ), + ('python3 -m dynamo.vllm --router-mode "$MODE"', "cannot parse"), + (f"python3 -m dynamo.vllm --model {MODEL} --no-router-kv-events", None), + ], +) +def test_p4_shell_wrapped_worker(script, match): + report = evaluate(dgd(frontend(), shell_worker(script), planner())) + rejected = reasons_matching(report, "P4: worker component agg ") + if match is None: + assert report.p4_ok, report.reasons + else: + assert not report.p4_ok + assert any(match in reason for reason in rejected), report.reasons + + +def test_p4_rejects_a_v1alpha1_worker_router_mode(): + deployment = { + "apiVersion": "nvidia.com/v1alpha1", + "kind": "DynamoGraphDeployment", + "metadata": {"name": DGD, "namespace": NAMESPACE}, + "spec": { + "services": { + "Frontend": { + "componentType": "frontend", + "envs": trace_env(), + "volumeMounts": [{"name": CLAIM, "mountPoint": "/trace"}], + "extraPodSpec": { + "mainContainer": {"command": ["python3"], "args": list(KV_ARGS)} + }, + }, + "VllmWorker": { + "componentType": "worker", + "envs": [{"name": "DYN_ROUTER_MODE", "value": "kv"}], + "extraPodSpec": { + "mainContainer": { + "command": ["python3", "-m", "dynamo.vllm"], + "args": ["--model", MODEL], + } + }, + }, + "Planner": {"componentType": "planner", "replicas": 1}, + }, + }, + } + report = evaluate(deployment) + assert not report.p4_ok + assert reasons_matching(report, "P4: worker component VllmWorker advertises") + + +def test_worker_check_ignores_the_planner_env(monkeypatch): + monkeypatch.setenv("DYN_ROUTER_MODE", "kv") + fc._worker_router_cached.cache_clear() + try: + report = evaluate(dgd()) + finally: + fc._worker_router_cached.cache_clear() + assert report.p4_ok, report.reasons + assert os.environ["DYN_ROUTER_MODE"] == "kv" + + +WORKER_GOLDEN_CASES = [ + ((), {}), + (("--router-mode", "kv"), {}), + (("--router-m", "random"), {}), + (("--router-mode=power-of-two",), {}), + ((), {"DYN_ROUTER_MODE": "kv"}), + (("--router-mode", "least-loaded"), {"DYN_ROUTER_MODE": "kv"}), + (("--no-router-kv-events", "--router-temperature", "0.5"), {}), + ((), {"DYN_ROUTER_TEMPERATURE": "0.5"}), +] + + +@pytest.mark.parametrize(("args", "env"), WORKER_GOLDEN_CASES) +def test_worker_router_mode_matches_the_real_worker_parser(monkeypatch, args, env): + """Backends parse their left-over argv with parse_worker_router_config.""" + + from dynamo.common.configuration.groups.router_args import ( + parse_worker_router_config, + ) + + for name in list(os.environ): + if name.startswith("DYN_"): + monkeypatch.delenv(name) + for name, value in env.items(): + monkeypatch.setenv(name, value) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + config, _ = parse_worker_router_config([f"--model={MODEL}", *args]) + for name in env: + monkeypatch.delenv(name) + + worker_env = [{"name": name, "value": value} for name, value in env.items()] + report = evaluate(dgd(frontend(), router_worker(*args, env=worker_env), planner())) + rejected = reasons_matching(report, "P4: worker component agg ") + if config.router_mode is None: + assert rejected == [], rejected + else: + assert any( + f"router mode {config.router_mode!r}" in reason for reason in rejected + ), rejected + + +def test_worker_env_from_is_warned_once(): + deployment = dgd(frontend(), worker("a"), worker("b"), planner()) + for component in deployment["spec"]["components"]: + component["podTemplate"]["spec"]["containers"][0]["envFrom"] = [ + {"secretRef": {"name": "hf-token-secret"}} + ] + report = evaluate(deployment) + assert report.p4_ok and report.spec_ok, report.reasons + env_from = sorted(w for w in report.warnings if "envFrom" in w) + assert env_from == [ + "P4: the frontend container uses envFrom secretRef 'hf-token-secret'; " + "the planner cannot read it and assumes it sets no router env (P14 " + "still checks the routers' policy parameters)", + "P4: worker components a, b use envFrom; the planner cannot read it and " + "assumes it sets no router env (a DYN_ROUTER_MODE there would make the " + "worker advertise its own router configuration)", + ] + + +# ---------------------------------------------------------- P6-P10 + + +def test_p6_warns_on_unknown_model(): + report = evaluate(dgd(), config=planner_config(model="acme/private-model")) + assert report.spec_ok + assert any(w.startswith("P6:") for w in report.warnings) + + +def test_p7_served_models(): + two = dgd(frontend(), worker("a"), worker("b", model="meta/llama"), planner()) + report = evaluate(two) + assert reasons_matching(report, "P7:") == [ + f"P7: the DGD serves 2 models ({MODEL}, meta/llama); live retune needs one" + ] + assert fc.served_model_name(two) is None + assert fc.served_model_name(dgd()) == MODEL + + +def test_p7_served_model_prefers_current_status_and_warns_when_unknown(): + unknown = worker("b") + unknown["podTemplate"]["spec"]["containers"][0]["args"] = ["--model", "$(MODEL)"] + deployment = dgd(frontend(), worker("a"), unknown, planner()) + report = evaluate(deployment) + assert report.spec_ok + assert any("served model of b" in w for w in report.warnings) + + deployment["metadata"]["generation"] = 3 + deployment["status"] = { + "observedGeneration": 3, + "components": {"b": {"servedModelName": "other/model"}}, + } + assert fc.served_model_names(deployment) == ({"a": MODEL, "b": "other/model"}, ()) + deployment["status"]["observedGeneration"] = 2 + assert fc.served_model_names(deployment)[1] == ("b",) + + +def test_p7_shell_wrapped_worker_model(): + shell = worker("a") + shell["podTemplate"]["spec"]["containers"][0].update( + command=["/bin/sh", "-c"], + args=[ + f"python3 -m dynamo.sglang --model-path {MODEL} --served-model-name qwen" + ], + ) + assert fc.served_model_names(dgd(frontend(), shell)) == ({"a": "qwen"}, ()) + + +def p7_model_warnings(report: PremiseReport) -> list[str]: + return [w for w in report.warnings if w.startswith("P7: retune.dgdr.model")] + + +def test_p7_warns_when_dgdr_model_is_not_what_the_workers_serve(): + deployment = dgd(frontend(), worker(model="meta-llama/Llama-3.1-8B"), planner()) + report = evaluate(deployment) + assert report.p4_ok and report.spec_ok, report.reasons + assert p7_model_warnings(report) == [ + f"P7: retune.dgdr.model {MODEL!r} is not among the workers' " + "--model/--model-path/--served-model-name values " + "(meta-llama/Llama-3.1-8B); every DGDR Run simulates retune.dgdr.model, " + "so make sure it is the model the DGD serves" + ] + assert not p7_model_warnings(evaluate(dgd())) + + +@pytest.mark.parametrize( + "args", + [ + # An alias: the HF id is still among the values. + ["--model", "/models/qwen", "--served-model-name", MODEL], + ["--model-path", MODEL, "--served-model-name", "qwen"], + ["--model", "/models/qwen", f"--served-model-name=qwen,{MODEL}"], + # Expanded against the worker's env. + ["--model", "$(MODEL_ID)"], + ], +) +def test_p7_dgdr_model_matches_any_worker_model_value(args): + component = worker() + component["podTemplate"]["spec"]["containers"][0].update( + args=args, env=[{"name": "MODEL_ID", "value": MODEL}] + ) + report = evaluate(dgd(frontend(), component, planner())) + assert not p7_model_warnings(report), report.warnings + + +def test_p7_dgdr_model_counts_the_current_served_model_status(): + component = worker() + component["podTemplate"]["spec"]["containers"][0]["args"] = ["--model", "$(X)"] + deployment = dgd(frontend(), component, planner()) + deployment["metadata"]["generation"] = 1 + deployment["status"] = { + "observedGeneration": 1, + "components": {"agg": {"servedModelName": "other/model"}}, + } + assert p7_model_warnings(evaluate(deployment)) + deployment["status"]["components"]["agg"]["servedModelName"] = MODEL + assert not p7_model_warnings(evaluate(deployment)) + # Nothing known: only the undetermined-model warning. + del deployment["status"] + report = evaluate(deployment) + assert not p7_model_warnings(report) + assert any("served model of agg" in w for w in report.warnings) + + +@pytest.mark.parametrize( + ("extra", "match"), + [ + ({"name": "Epp", "type": "epp"}, "EPP component"), + ( + worker("side", frontendSidecar="sidecar-frontend"), + "side run a frontendSidecar", + ), + ], +) +def test_p7_gaie_layouts_with_a_frontend(extra, match): + report = evaluate(dgd(frontend(), worker(), planner(), extra)) + assert report.p4_ok + assert any(match in r for r in reasons_matching(report, "P7:")), report.reasons + + +def test_p7_no_frontend(): + report = evaluate(dgd(worker(frontendSidecar="sidecar-frontend"), planner())) + assert not report.p4_ok + assert all(r.startswith("P7:") for r in report.reasons) + assert len(report.reasons) == 1 + + +@pytest.mark.parametrize( + ("args", "env"), + [ + ((*KV_ARGS, "--tls-cert-path", "/c.pem", "--tls-key-path", "/k.pem"), {}), + ((*KV_ARGS,), {"DYN_TLS_CERT_PATH": "/c.pem", "DYN_TLS_KEY_PATH": "/k.pem"}), + ], +) +def test_p7_tls(args, env): + report = evaluate(with_frontend(args=args, env=trace_env(**env))) + assert report.p4_ok + assert reasons_matching(report, "P7:") == [ + "P7: the frontend serves TLS (--tls-cert-path / DYN_TLS_CERT_PATH)" + ] + + +@pytest.mark.parametrize( + ("replicas", "args", "ok"), + [ + (1, KV_ARGS, True), + (0, KV_ARGS, True), + (2, KV_ARGS, False), + (2, (*KV_ARGS, "--router-replica-sync"), True), + ], +) +def test_p8_frontend_replicas(replicas, args, ok): + report = evaluate(dgd(frontend(replicas=replicas, args=args), worker(), planner())) + assert bool(reasons_matching(report, "P8:")) is not ok + assert report.p4_ok + + +def test_p9_scaling_adapter_and_planner_replicas(): + adapter = dgd(frontend(), worker(scalingAdapter={}), planner()) + assert reasons_matching(evaluate(adapter), "P9:") == [ + "P9: worker component agg has a scalingAdapter; its replicas could change during a window" + ] + two_planners = dgd(frontend(), worker(), planner(replicas=2)) + assert "2 replicas" in reasons_matching(evaluate(two_planners), "P9:")[0] + # The caller's own replica count wins over the DGD. + assert not reasons_matching(evaluate(two_planners, planner_replicas=1), "P9:") + assert reasons_matching(evaluate(dgd(), planner_replicas=3), "P9:") + no_planner = evaluate(dgd(frontend(), worker())) + assert no_planner.spec_ok + assert any(w.startswith("P9:") for w in no_planner.warnings) + + +def test_p9_v1alpha1_scaling_adapter_needs_enabled(): + def deployment(adapter): + return { + "metadata": {"name": DGD}, + "spec": { + "services": { + "Frontend": {"componentType": "frontend"}, + "w": {"componentType": "worker", "scalingAdapter": adapter}, + } + }, + } + + disabled = fc._components(deployment({})) + enabled = fc._components(deployment({"enabled": True})) + assert not fc._scaling_adapter(disabled[1]) + assert fc._scaling_adapter(enabled[1]) + + +P10_CASES = [ + ({"DYN_REQUEST_TRACE": None}, None, "DYN_REQUEST_TRACE=1"), + ({"DYN_REQUEST_TRACE": "0"}, None, "DYN_REQUEST_TRACE=1"), + ({"DYN_REQUEST_TRACE": "true"}, None, None), + ({"DYN_REQUEST_TRACE_SINKS": None}, None, "DYN_REQUEST_TRACE_SINKS=file"), + ({"DYN_REQUEST_TRACE_SINKS": "stderr"}, None, "DYN_REQUEST_TRACE_SINKS=file"), + ({"DYN_REQUEST_TRACE_SINKS": "jsonl_gz"}, None, "DYN_REQUEST_TRACE_SINKS=file"), + ({"DYN_REQUEST_TRACE_SINKS": "file,stderr"}, None, None), + ({"DYN_REQUEST_TRACE_FILE_FORMAT": None}, None, None), + ({"DYN_REQUEST_TRACE_FILE_FORMAT": "jsonl"}, None, "DYN_REQUEST_TRACE_FILE_FORMAT"), + ( + {"DYN_REQUEST_TRACE_RECORDS": None}, + None, + "DYN_REQUEST_TRACE_RECORDS=request_end", + ), + ( + {"DYN_REQUEST_TRACE_RECORDS": "request_end,tool"}, + None, + "DYN_REQUEST_TRACE_RECORDS", + ), + ({"DYN_REQUEST_TRACE_RECORDS": " Request_End "}, None, None), + ( + {"DYN_REQUEST_TRACE_FILE_PATH": None}, + None, + "must set DYN_REQUEST_TRACE_FILE_PATH", + ), + ( + {"DYN_REQUEST_TRACE_FILE_PATH": f"/trace/segments/{DGD}/$(DYN_POD_NAME)/req"}, + None, + None, + ), + # The reconciler prunes only the top level of the pod's directory. + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/a/req" + }, + None, + "no further '/'", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req-$(DYN_POD_NAME)" + }, + None, + None, + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/" + }, + None, + "file prefix", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(POD_NAME)/req" + }, + None, + "file prefix", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/other/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" + }, + None, + "file prefix", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/data/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" + }, + None, + "file prefix", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": { + "valueFrom": {"configMapKeyRef": {"name": "c", "key": "p"}} + } + }, + None, + "must be a literal value", + ), + ({"DYN_POD_NAME": None}, None, "DYN_POD_NAME from fieldRef"), + ({"DYN_POD_NAME": "fixed"}, None, "DYN_POD_NAME from fieldRef"), + ( + {"DYN_POD_NAME": {"valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}}}, + None, + "DYN_POD_NAME from fieldRef", + ), + ({"DYN_PARENT_DGD_K8S_NAME": "other"}, None, "overrides DYN_PARENT_DGD_K8S_NAME"), + ({}, {"volumes": []}, "no volume for claim"), + ( + {}, + {"volumes": [{"name": "traces", "persistentVolumeClaim": {"claimName": "x"}}]}, + "no volume for claim", + ), + ({}, {"mounts": []}, "does not mount claim"), + ( + {}, + {"mounts": [{"name": "traces", "mountPath": "/trace", "readOnly": True}]}, + "read-only or through a subPath", + ), + ( + {}, + {"mounts": [{"name": "traces", "mountPath": "/trace", "subPath": "segments"}]}, + "read-only or through a subPath", + ), + ({}, {"mounts": [{"name": "traces", "mountPath": "/trace/"}]}, None), + ( + {}, + { + "mounts": [ + {"name": "traces", "mountPath": "/ro", "readOnly": True}, + {"name": "traces", "mountPath": "/trace"}, + ] + }, + None, + ), +] + + +@pytest.mark.parametrize(("env", "frontend_kwargs", "match"), P10_CASES) +def test_p10_trace_recipe(env, frontend_kwargs, match): + deployment = dgd( + frontend(env=trace_env(**env), **(frontend_kwargs or {})), worker(), planner() + ) + report = evaluate(deployment) + assert report.p4_ok, report.reasons + p10 = reasons_matching(report, "P10:") + if match is None: + assert p10 == [], p10 + assert report.spec_ok + else: + assert any(match in r for r in p10), p10 + assert not report.spec_ok + + +def test_p10_custom_segments_dir_and_graph_env(): + path = "/trace/rt/segs/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" + deployment = dgd( + frontend(env=trace_env(DYN_REQUEST_TRACE_FILE_PATH=None)), + worker(), + planner(), + env=[{"name": "DYN_REQUEST_TRACE_FILE_PATH", "value": path}], + ) + report = evaluate(deployment, config=planner_config(segments_dir="/rt/segs/")) + assert report.spec_ok, report.reasons + + +def test_p10_extra_sinks_warn(): + report = evaluate( + with_frontend(env=trace_env(DYN_REQUEST_TRACE_SINKS="file,stderr")) + ) + assert report.spec_ok + assert any("also enables stderr" in w for w in report.warnings) + + +def test_p10_is_checked_when_parsing_fails(): + deployment = with_frontend( + command=("/bin/sh", "-c"), + args=("python3 -m dynamo.frontend",), + env=trace_env(DYN_REQUEST_TRACE=None), + ) + report = evaluate(deployment) + assert reasons_matching(report, "P4:") + assert reasons_matching(report, "P10:") + + +def test_evaluate_requires_dgdr(): + config = PlannerConfig.model_construct(mode="agg", retune=None) + with pytest.raises(ValueError, match="retune.dgdr"): + fc.evaluate_premises( + dgd(), + planner_config=config, + startup=None, + parse_error=None, + planner_replicas=None, + ) + + +def test_malformed_dgd_never_raises(): + odd_frontend = { + "name": "f", + "type": "frontend", + "replicas": "many", + "podTemplate": "not-a-template", + } + for deployment in ( + {}, + {"spec": None}, + {"spec": {"components": [None, 3]}}, + {"spec": {"components": [odd_frontend]}}, + {"spec": {"components": [frontend(volumes=["x"], mounts=[None])]}}, + ): + report = evaluate(deployment) + assert not report.spec_ok + + +def test_unexpected_parser_failure_fails_closed(monkeypatch, caplog): + def broken(env): + raise RuntimeError("frontend arg groups changed") + + monkeypatch.setattr(fc, "_frontend_parser", broken) + fc._parse_cached.cache_clear() + try: + error = parse_error(dgd()) + finally: + fc._parse_cached.cache_clear() + assert error.startswith("P4: the planner failed to parse") + assert "RuntimeError: frontend arg groups changed" in error + assert any(r.levelname == "ERROR" for r in caplog.records) + + +# ------------------------------------------------------------- helpers + + +def test_worker_types_and_canonical_json(): + assert fc.worker_types("disagg") == ("prefill", "decode") + assert fc.worker_types("agg") == ("aggregated",) + assert fc.canonical_json({"b": 1, "a": [1.5]}) == '{"a":[1.5],"b":1}' + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("plain", "plain"), + ("$(A)/x", "1/x"), + ("$$(A)", "$(A)"), + ("$(MISSING)", "$(MISSING)"), + ("$(A", "$(A"), + ("$", "$"), + ("a$$b", "a$b"), + ("$(SECRET)", None), + ], +) +def test_kubernetes_expansion(value, expected): + assert fc._expand(value, {"A": "1"}, {"SECRET"}) == expected + + +def test_parse_is_cached_per_spec(): + fc._parse_cached.cache_clear() + parse(dgd()) + parse(dgd()) + info = fc._parse_cached.cache_info() + assert info.hits >= 1 and info.misses == 1 + parse(with_frontend(args=(*KV_ARGS, "--router-temperature", "0.5"))) + assert fc._parse_cached.cache_info().misses == 2 diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py new file mode 100644 index 000000000000..6fe507868881 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py @@ -0,0 +1,530 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for ``KubernetesRetuneKube`` and the shared ``FakeRetuneKube``. + +The client tests run the real ``kubernetes`` API classes and replace only +the REST transport, so they check the URLs, query parameters, timeouts and +bodies the client library actually sends, and the error mapping. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any, Optional + +import pytest +from kubernetes import client +from kubernetes.client import ApiException + +from dynamo.planner.retune.live.contracts import ( + KubeAlreadyExists, + KubeConflict, + KubeError, + KubeInvalid, + KubeNotFound, + PodInfo, + RetuneKube, +) +from dynamo.planner.retune.live.kube import KubernetesRetuneKube, request_timeout +from dynamo.planner.tests.live_retune_fakes import ( + FakeRetuneKube, + lower_request_input, + match_labels, +) + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +NS = "test-ns" + + +class FakeResponse: + def __init__(self, body: Any, status: int = 200) -> None: + self.status = status + self.reason = "OK" + self.data = body if isinstance(body, (str, bytes)) else json.dumps(body) + + def getheaders(self) -> dict: + return {"content-type": "application/json"} + + def getheader(self, name: str, default: Optional[str] = None) -> Optional[str]: + return self.getheaders().get(name.lower(), default) + + +class Transport: + """Stands in for ``RESTClientObject.request``: records and replies.""" + + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + self.replies: list[Any] = [] + + def reply(self, value: Any) -> None: + self.replies.append(value) + + def __call__( + self, + method, + url, + query_params=None, + headers=None, + post_params=None, + body=None, + _preload_content=True, + _request_timeout=None, + ): + self.calls.append( + { + "method": method, + "url": url, + "query": dict(query_params or []), + "body": body, + "preload": _preload_content, + "timeout": _request_timeout, + } + ) + value = self.replies.pop(0) if self.replies else {} + if isinstance(value, Exception): + raise value + if isinstance(value, FakeResponse): + return value + return FakeResponse(value) + + +def api_error(status: int, message: str = "nope") -> ApiException: + error = ApiException(status=status, reason="Error") + error.body = json.dumps({"kind": "Status", "message": message}) + return error + + +@pytest.fixture +def kube(): + api_client = client.ApiClient(client.Configuration()) + transport = Transport() + api_client.rest_client.request = transport + retune_kube = KubernetesRetuneKube(NS, api_client=api_client) + yield retune_kube, transport + api_client.close() + + +def path(call: dict) -> str: + return call["url"].split("://", 1)[-1].split("/", 1)[-1] + + +# ---------------------------------------------------------------- client + + +def test_request_timeout_is_a_tuple_of_whole_seconds(): + assert request_timeout(10.0) == (10, 10) + assert request_timeout(2.5) == (3, 3) + assert request_timeout(0.2) == (1, 1) + assert request_timeout(0.0) == (1, 1) + assert request_timeout(float("nan")) == (1, 1) + + +def test_satisfies_the_protocol(kube): + retune_kube, _ = kube + assert isinstance(retune_kube, KubernetesRetuneKube) + _: RetuneKube = retune_kube + assert retune_kube.namespace == NS + with pytest.raises(ValueError): + KubernetesRetuneKube("") + + +def test_get_dgd(kube): + retune_kube, transport = kube + dgd = {"metadata": {"name": "llm", "uid": "u1"}, "spec": {"services": {}}} + transport.reply(dgd) + assert retune_kube.get_dgd("llm", timeout_s=4) == dgd + (call,) = transport.calls + assert call["method"] == "GET" + assert ( + path(call) + == f"apis/nvidia.com/v1beta1/namespaces/{NS}/dynamographdeployments/llm" + ) + assert call["timeout"] == (4, 4) + + transport.reply(api_error(404)) + with pytest.raises(KubeNotFound) as raised: + retune_kube.get_dgd("gone") + assert raised.value.status == 404 + + +def test_create_request_is_strict_and_dry_run_on_request(kube): + retune_kube, transport = kube + body = {"metadata": {"name": "r1"}, "spec": {"model": {"name": "m"}}} + transport.reply({**body, "metadata": {"name": "r1", "uid": "x"}}) + created = retune_kube.create_request(body, timeout_s=2.5) + assert created["metadata"]["uid"] == "x" + call = transport.calls[-1] + assert call["method"] == "POST" + assert ( + path(call) + == f"apis/nvidia.com/v1beta2/namespaces/{NS}/dynamographdeploymentrequests" + ) + assert call["query"]["fieldValidation"] == "Strict" + assert call["query"]["fieldManager"] == "dynamo-planner-retune" + assert "dryRun" not in call["query"] + assert call["body"] == body + assert call["timeout"] == (3, 3) + + transport.reply(body) + retune_kube.create_request(body, dry_run=True) + assert transport.calls[-1]["query"]["dryRun"] == "All" + assert transport.calls[-1]["query"]["fieldValidation"] == "Strict" + + +@pytest.mark.parametrize( + "status, error_type", + [ + (409, KubeAlreadyExists), + (422, KubeInvalid), + (400, KubeInvalid), + (403, KubeError), + (500, KubeError), + ], +) +def test_create_request_maps_api_errors(kube, status, error_type): + retune_kube, transport = kube + transport.reply(api_error(status, "strict decoding error: unknown field")) + with pytest.raises(error_type) as raised: + retune_kube.create_request({"metadata": {"name": "r1"}}) + assert raised.value.status == status + assert f"HTTP {status}" in str(raised.value) + assert "unknown field" in str(raised.value) + if error_type is KubeError: + assert type(raised.value) is KubeError + + +def test_network_errors_become_kube_errors(kube): + retune_kube, transport = kube + # Imported here: the marker-report hook collects tests without urllib3. + from urllib3.exceptions import ReadTimeoutError + + transport.reply(ReadTimeoutError(None, "url", "timed out")) + with pytest.raises(KubeError) as raised: + retune_kube.get_request("r1") + assert raised.value.status is None + assert "ReadTimeoutError" in str(raised.value) + + +def test_requests_runs_and_candidates(kube): + retune_kube, transport = kube + transport.reply({"metadata": {"name": "r1"}}) + assert retune_kube.get_request("r1") == {"metadata": {"name": "r1"}} + transport.reply(api_error(404)) + assert retune_kube.get_request("missing") is None + + transport.reply({"items": [{"metadata": {"name": "r1"}}]}) + assert retune_kube.list_requests("a=b,c=d") == [{"metadata": {"name": "r1"}}] + call = transport.calls[-1] + assert ( + path(call) + == f"apis/nvidia.com/v1beta2/namespaces/{NS}/dynamographdeploymentrequests" + ) + assert call["query"]["labelSelector"] == "a=b,c=d" + + transport.reply({"metadata": {"name": "r1-1"}}) + assert retune_kube.get_run("r1-1") == {"metadata": {"name": "r1-1"}} + assert path(transport.calls[-1]).endswith("dynamographdeploymentruns/r1-1") + transport.reply(api_error(404)) + assert retune_kube.get_run("gone") is None + + transport.reply({"items": []}) + assert retune_kube.list_candidates("nvidia.com/dgdr-run-uid=u") == [] + call = transport.calls[-1] + assert path(call).endswith("dynamographdeploymentcandidates") + assert call["query"]["labelSelector"] == "nvidia.com/dgdr-run-uid=u" + + transport.reply({}) + retune_kube.delete_request("r1", propagation="Foreground", timeout_s=7) + call = transport.calls[-1] + assert call["method"] == "DELETE" + assert path(call).endswith("dynamographdeploymentrequests/r1") + assert call["query"]["propagationPolicy"] == "Foreground" + assert call["timeout"] == (7, 7) + transport.reply(api_error(404)) + retune_kube.delete_request("gone") # missing is not an error + transport.reply(api_error(500)) + with pytest.raises(KubeError): + retune_kube.delete_request("r1") + + +def _pod( + name: str, + *, + ready: bool = True, + deleting: bool = False, + containers: Optional[list] = None, + ip: Optional[str] = "10.0.0.5", +) -> dict: + return { + "metadata": { + "name": name, + "uid": f"uid-{name}", + "labels": {"nvidia.com/dynamo-component-type": "frontend"}, + **({"deletionTimestamp": "2026-10-28T00:00:00Z"} if deleting else {}), + }, + "spec": { + "containers": containers + if containers is not None + else [ + {"name": "sidecar", "ports": [{"name": "http", "containerPort": 9999}]}, + { + "name": "main", + "ports": [ + {"name": "metrics", "containerPort": 9090}, + {"name": "http", "containerPort": 8000}, + ], + }, + ] + }, + "status": { + "podIP": ip, + "conditions": [ + {"type": "PodScheduled", "status": "True"}, + {"type": "Ready", "status": "True" if ready else "False"}, + ], + }, + } + + +def test_list_pods_builds_pod_info(kube): + retune_kube, transport = kube + transport.reply( + { + "kind": "PodList", + "apiVersion": "v1", + "metadata": {}, + "items": [ + _pod("a"), + _pod("b", ready=False, deleting=True, ip=None), + _pod( + "c", + containers=[ + { + "name": "other", + "ports": [{"name": "http", "containerPort": 8080}], + } + ], + ), + _pod("d", containers=[{"name": "main"}]), + ], + } + ) + pods = retune_kube.list_pods("x=y", timeout_s=3) + call = transport.calls[-1] + assert path(call) == f"api/v1/namespaces/{NS}/pods" + assert call["query"]["labelSelector"] == "x=y" + assert call["timeout"] == (3, 3) + assert pods[0] == PodInfo( + name="a", + uid="uid-a", + ip="10.0.0.5", + ready=True, + terminating=False, + labels={"nvidia.com/dynamo-component-type": "frontend"}, + http_port=8000, # the main container's port wins + ) + assert (pods[1].ready, pods[1].terminating, pods[1].ip) == (False, True, None) + assert pods[2].http_port == 8080 + assert pods[3].http_port is None + + +def test_leases_round_trip_raw_json(kube): + retune_kube, transport = kube + lease = { + "apiVersion": "coordination.k8s.io/v1", + "kind": "Lease", + "metadata": {"name": "llm-planner-retune", "resourceVersion": "5"}, + "spec": { + "holderIdentity": "pod/abc", + "leaseDurationSeconds": 60, + "renewTime": "2026-10-28T12:00:00.000000Z", + }, + } + transport.reply(FakeResponse(json.dumps(lease).encode())) + # MicroTime strings come back verbatim, not re-serialized datetimes. + assert retune_kube.get_lease("llm-planner-retune") == lease + call = transport.calls[-1] + assert call["preload"] is False + assert ( + path(call) + == f"apis/coordination.k8s.io/v1/namespaces/{NS}/leases/llm-planner-retune" + ) + transport.reply(api_error(404)) + assert retune_kube.get_lease("gone") is None + + body = {"metadata": {"name": "l2"}, "spec": {"holderIdentity": "me"}} + transport.reply({**body, "metadata": {"name": "l2", "resourceVersion": "1"}}) + created = retune_kube.create_lease(body) + assert created["metadata"]["resourceVersion"] == "1" + call = transport.calls[-1] + assert call["method"] == "POST" + assert call["body"]["apiVersion"] == "coordination.k8s.io/v1" + assert call["body"]["kind"] == "Lease" + assert "apiVersion" not in body # the caller's dict is not mutated + transport.reply(api_error(409)) + with pytest.raises(KubeAlreadyExists): + retune_kube.create_lease(body) + + transport.reply(lease) + retune_kube.replace_lease("llm-planner-retune", lease, timeout_s=2) + call = transport.calls[-1] + assert call["method"] == "PUT" + assert call["body"]["metadata"]["resourceVersion"] == "5" + assert call["timeout"] == (2, 2) + transport.reply(api_error(409, "the object has been modified")) + with pytest.raises(KubeConflict) as raised: + retune_kube.replace_lease("llm-planner-retune", lease) + assert raised.value.status == 409 + transport.reply(api_error(404)) + with pytest.raises(KubeNotFound): + retune_kube.replace_lease("gone", lease) + + transport.reply({"kind": "Status", "status": "Success"}) + retune_kube.delete_lease("l2") + assert transport.calls[-1]["method"] == "DELETE" + transport.reply(api_error(404)) + retune_kube.delete_lease("l2") + + +def test_from_kube_api_shares_the_connector_client(): + api_client = client.ApiClient(client.Configuration()) + kube_api = SimpleNamespace( + current_namespace="ns2", + custom_api=client.CustomObjectsApi(api_client), + ) + retune_kube = KubernetesRetuneKube.from_kube_api(kube_api) + assert retune_kube.namespace == "ns2" + assert retune_kube.api_client is api_client + api_client.close() + + +# ------------------------------------------------------------------ fake + + +def test_fake_label_selectors(): + labels = {"a": "1", "b": "2"} + assert match_labels(labels, "") + assert match_labels(labels, "a=1,b==2") + assert match_labels(labels, "a,!c") + assert not match_labels(labels, "a!=1") + assert not match_labels(labels, "c") + assert not match_labels(labels, "!a") + with pytest.raises(ValueError): + match_labels(labels, "a in (1,2)") + + +def test_fake_lease_compare_and_swap_and_faults(): + kube = FakeRetuneKube(NS, now=lambda: 1_793_000_000.0) + created = kube.create_lease( + {"metadata": {"name": "l"}, "spec": {"holderIdentity": ""}} + ) + with pytest.raises(KubeAlreadyExists): + kube.create_lease({"metadata": {"name": "l"}}) + stale = dict(created, spec={"holderIdentity": "me"}) + updated = kube.replace_lease("l", stale) + assert updated["spec"]["holderIdentity"] == "me" + assert ( + updated["metadata"]["resourceVersion"] != created["metadata"]["resourceVersion"] + ) + with pytest.raises(KubeConflict): + kube.replace_lease("l", stale) # resourceVersion is stale now + with pytest.raises(KubeNotFound): + kube.replace_lease("missing", stale) + + # A write that times out but lands. + kube.fail("replace_lease", KubeError("timeout"), landed=True) + with pytest.raises(KubeError): + kube.replace_lease("l", dict(updated, spec={"holderIdentity": "other"})) + assert kube.get_lease("l")["spec"]["holderIdentity"] == "other" + # A write that fails before landing. + current = kube.get_lease("l") + kube.fail("replace_lease", KubeError("503")) + with pytest.raises(KubeError): + kube.replace_lease("l", dict(current, spec={"holderIdentity": "x"})) + assert kube.get_lease("l")["spec"]["holderIdentity"] == "other" + kube.delete_lease("l") + kube.delete_lease("l") + assert kube.get_lease("l") is None + assert [call.method for call in kube.calls].count("replace_lease") == 5 + + +def test_fake_requests_cascade_and_copies(): + kube = FakeRetuneKube(NS) + body = {"metadata": {"name": "r", "labels": {"w": "1"}}, "spec": {"x": 1}} + stored = kube.create_request(body) + stored["spec"]["x"] = 2 # copies never alias the store + assert kube.get_request("r")["spec"]["x"] == 1 + assert kube.create_request({"metadata": {"name": "dry"}}, dry_run=True) + assert kube.get_request("dry") is None + run = kube.accept_request("r") + kube.start_run(run["metadata"]["name"]) + kube.complete_run(run["metadata"]["name"], candidates=[({"p": 1}, {"m": 2})]) + request = kube.get_request("r") + assert request["status"]["recentRunRefs"] == [{"name": "r-1"}] + assert "activeRunRef" not in request["status"] + assert ( + len(kube.list_candidates(f"nvidia.com/dgdr-run-uid={run['metadata']['uid']}")) + == 1 + ) + assert kube.list_requests("w=1")[0]["metadata"]["name"] == "r" + kube.delete_request("r", propagation="Foreground") + assert kube.runs == {} and kube.candidates == {} + assert kube.deleted_requests == [("r", "Foreground")] + + +def test_fake_lowering_applies_overrides_and_operator_fields(): + request = { + "metadata": {"name": "r", "namespace": NS}, + "spec": { + "model": {"name": "m"}, + "backends": ["vllm"], + "hardware": {"gpu": {"sku": "h200_sxm", "budget": 4}}, + "workload": { + "trace": { + "format": "dynamo", + "source": {"pvc": {"claimName": "c", "path": "w/1.jsonl.gz"}}, + } + }, + "objective": { + "mode": "Optimize", + "metric": "GoodputPerGPU", + "sla": {"ttftMs": 1.0}, + }, + "recommendation": {"maxCandidates": 3}, + "aisimulate": { + "inputOverrides": { + "engine": {"backend": "vllm", "workers": {"prefill": None}}, + "dgdr_run": {"generator": "direct"}, + } + }, + }, + } + lowered = lower_request_input(request, run_name="r-1") + assert lowered["engine"]["backend"] == "vllm" + assert "prefill" not in lowered["engine"]["workers"] + assert lowered["engine"]["workers"]["decode"] == {} + assert lowered["optimization"] == { + "constraints": {"max_candidate_gpus": 4}, + "target": "goodput_per_gpu", + "strict_sla": True, + } + assert lowered["traffic"] == { + "source": { + "type": "trace", + "format": "dynamo", + "paths": ["/var/run/aisimulate/trace/trace.jsonl.gz"], + }, + "load": {"type": "trace_timestamps"}, + } + assert lowered["optimizer"] == {"max_trials": 320.0} + assert lowered["dgdr_run"]["generator"] == "direct" + assert lowered["dgdr_run"]["max_candidates"] == 3 + assert lowered["dgdr_run"]["namespace"] == NS diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py b/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py new file mode 100644 index 000000000000..08e4b95d9009 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py @@ -0,0 +1,277 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``dynamo_planner_retune_*`` metrics: names, labels and values per design +§4.10, collectors shared per registry, and methods that never raise.""" + +from __future__ import annotations + +import math + +import pytest +from prometheus_client import REGISTRY, CollectorRegistry, generate_latest + +from dynamo.planner.core.types import RETUNE_OUTCOMES +from dynamo.planner.retune.live.contracts import NullMetrics, RetuneMetricsSink +from dynamo.planner.retune.live.metrics import PREFIX, RetuneMetrics + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + + +@pytest.fixture +def registry() -> CollectorRegistry: + return CollectorRegistry() + + +def _value(registry: CollectorRegistry, name: str, **labels: str) -> float | None: + return registry.get_sample_value(f"{PREFIX}_{name}", labels) + + +def test_metrics_implement_the_sink_protocol(registry): + metrics: RetuneMetricsSink = RetuneMetrics(registry) + null: RetuneMetricsSink = NullMetrics() + for name in ( + "decision", + "failure", + "skipped", + "supervisor_state", + "scores", + "router_converged", + "patch_error", + "trace_sink_missing", + "dropped_members", + ): + assert callable(getattr(metrics, name)) + assert callable(getattr(null, name)) + + +def test_counters_count_by_label(registry): + metrics = RetuneMetrics(registry) + + metrics.decision("applied") + metrics.decision("applied") + metrics.decision("kept") + metrics.failure("crn_mismatch") + metrics.skipped("not_leader") + metrics.patch_error("409") + metrics.patch_error("timeout") + metrics.patch_error("409") + + assert _value(registry, "decisions_total", outcome="applied") == 2 + assert _value(registry, "decisions_total", outcome="kept") == 1 + assert _value(registry, "failures_total", reason="crn_mismatch") == 1 + assert _value(registry, "skipped_total", reason="not_leader") == 1 + assert _value(registry, "router_patch_errors_total", code="409") == 2 + assert _value(registry, "router_patch_errors_total", code="timeout") == 1 + + +def test_known_outcomes_and_skip_reasons_start_at_zero(registry): + RetuneMetrics(registry) + + for outcome in (*RETUNE_OUTCOMES, "promoted", "rolled_back"): + assert _value(registry, "decisions_total", outcome=outcome) == 0 + for reason in ("not_leader", "not_ready", "already_decided"): + assert _value(registry, "skipped_total", reason=reason) == 0 + # Gauges of the latest decision read NaN until one is scored. + for name in ("baseline_score", "best_score", "sweep_seconds"): + value = _value(registry, name) + assert value is not None and math.isnan(value) + + +def test_supervisor_state_gauges(registry): + metrics = RetuneMetrics(registry) + + metrics.supervisor_state( + leader=True, + ready=False, + initialized=True, + rollback_pending=True, + generation=3, + ) + assert _value(registry, "leader") == 1 + assert _value(registry, "ready") == 0 + assert _value(registry, "initialized") == 1 + assert _value(registry, "rollback_pending") == 1 + assert _value(registry, "controller_generation") == 3 + + metrics.supervisor_state( + leader=False, + ready=True, + initialized=False, + rollback_pending=False, + generation=4, + ) + assert _value(registry, "leader") == 0 + assert _value(registry, "ready") == 1 + assert _value(registry, "initialized") == 0 + assert _value(registry, "rollback_pending") == 0 + assert _value(registry, "controller_generation") == 4 + + +def test_scores_set_nan_for_unknown_values(registry): + metrics = RetuneMetrics(registry) + + metrics.scores(baseline=1.5, best=2.25, sweep_seconds=3600.0, missing_candidates=2) + assert _value(registry, "baseline_score") == 1.5 + assert _value(registry, "best_score") == 2.25 + assert _value(registry, "sweep_seconds") == 3600.0 + assert _value(registry, "missing_candidates") == 2 + + metrics.scores(baseline=None, best=None, sweep_seconds=None, missing_candidates=0) + for name in ("baseline_score", "best_score", "sweep_seconds"): + value = _value(registry, name) + assert value is not None and math.isnan(value) + assert _value(registry, "missing_candidates") == 0 + + +def test_router_converged_gauge(registry): + metrics = RetuneMetrics(registry) + + metrics.router_converged(True) + assert _value(registry, "router_converged") == 1 + metrics.router_converged(False) + assert _value(registry, "router_converged") == 0 + + +def test_trace_sink_missing_drops_the_series_once_the_sink_is_back(registry): + metrics = RetuneMetrics(registry) + + metrics.trace_sink_missing("frontend-0", True) + metrics.trace_sink_missing("frontend-1", True) + assert _value(registry, "trace_sink_missing", pod="frontend-0") == 1 + assert _value(registry, "trace_sink_missing", pod="frontend-1") == 1 + + metrics.trace_sink_missing("frontend-0", False) + assert _value(registry, "trace_sink_missing", pod="frontend-0") is None + assert _value(registry, "trace_sink_missing", pod="frontend-1") == 1 + # A pod that never had a series is not an error. + metrics.trace_sink_missing("frontend-9", False) + assert _value(registry, "trace_sink_missing", pod="frontend-9") is None + + +def test_dropped_members_counts_only_positive_counts(registry): + metrics = RetuneMetrics(registry) + + metrics.dropped_members(3) + metrics.dropped_members(0) + metrics.dropped_members(-1) + metrics.dropped_members(2) + assert _value(registry, "trace_dropped_members_total") == 5 + + +def test_exposition_uses_the_design_names(registry): + metrics = RetuneMetrics(registry) + metrics.trace_sink_missing("frontend-0", True) + text = generate_latest(registry).decode() + + for name in ( + "decisions_total", + "failures_total", + "skipped_total", + "leader", + "ready", + "initialized", + "rollback_pending", + "controller_generation", + "baseline_score", + "best_score", + "sweep_seconds", + "missing_candidates", + "router_converged", + "router_patch_errors_total", + "trace_sink_missing", + "trace_dropped_members_total", + ): + assert f"\n# HELP {PREFIX}_{name} " in f"\n{text}", name + + +def test_instances_on_one_registry_share_collectors(registry): + first = RetuneMetrics(registry) + second = RetuneMetrics(registry) + + first.decision("applied") + second.decision("applied") + assert _value(registry, "decisions_total", outcome="applied") == 2 + + +def test_default_registry_tolerates_reinstantiation(): + # The planner serves prometheus_client.REGISTRY; a controller rebuild or + # a second test must not hit "Duplicated timeseries". + first = RetuneMetrics() + second = RetuneMetrics() + before = ( + REGISTRY.get_sample_value(f"{PREFIX}_failures_total", {"reason": "probe"}) + or 0.0 + ) + first.failure("probe") + second.failure("probe") + assert ( + REGISTRY.get_sample_value(f"{PREFIX}_failures_total", {"reason": "probe"}) + == before + 2 + ) + + +def test_empty_labels_read_unknown(registry): + metrics = RetuneMetrics(registry) + + metrics.failure("") + metrics.patch_error("") + assert _value(registry, "failures_total", reason="unknown") == 1 + assert _value(registry, "router_patch_errors_total", code="unknown") == 1 + + +class _Broken: + def labels(self, *args, **kwargs): + raise RuntimeError("broken") + + def set(self, *args, **kwargs): + raise RuntimeError("broken") + + def inc(self, *args, **kwargs): + raise RuntimeError("broken") + + def remove(self, *args, **kwargs): + raise RuntimeError("broken") + + +def test_methods_never_raise(registry, monkeypatch): + metrics = RetuneMetrics(registry) + for name in vars(metrics._m): + monkeypatch.setattr(metrics._m, name, _Broken()) + + metrics.decision("applied") + metrics.failure("k8s_error") + metrics.skipped("not_ready") + metrics.supervisor_state( + leader=True, ready=True, initialized=True, rollback_pending=False, generation=1 + ) + metrics.scores(baseline=1.0, best=2.0, sweep_seconds=3.0, missing_candidates=0) + metrics.router_converged(True) + metrics.patch_error("503") + metrics.trace_sink_missing("frontend-0", True) + metrics.trace_sink_missing("frontend-0", False) + metrics.dropped_members(1) + + +def test_bad_values_do_not_raise(registry): + metrics = RetuneMetrics(registry) + + metrics.scores( + baseline="not a number", # type: ignore[arg-type] + best=None, + sweep_seconds=None, + missing_candidates=0, + ) + metrics.dropped_members(None) # type: ignore[arg-type] + metrics.supervisor_state( + leader=True, + ready=True, + initialized=True, + rollback_pending=False, + generation="x", # type: ignore[arg-type] + ) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py new file mode 100644 index 000000000000..a86953ff1516 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py @@ -0,0 +1,1031 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the live retune reconciler (segment, window, temp-file and +DGDR Request garbage collection) and its ``SweepRegistry``.""" + +from __future__ import annotations + +import os +import stat +import threading +from collections.abc import Callable +from pathlib import Path +from typing import Any, Optional + +import pytest + +from dynamo.planner.config.planner_config import DgdrExecutorConfig, RetuneConfig +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD, + LABEL_DGD_UID, + LABEL_WINDOW, + DgdRef, + KubeError, + PodInfo, +) +from dynamo.planner.retune.live.reconciler import ( + JOB_NAME_LABEL, + RUN_GENERATION_ANNOTATION, + Reconciler, + window_file_name, + window_relpath, +) + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +DGD = DgdRef(name="dgd", namespace="ns", uid="uid-now", dynamo_namespace="dyn-ns") +NOW = 1_800_000_000.0 +INTERVAL = 3600.0 +GRACE = 120.0 +TIMEOUT = 1800.0 +MARGIN = 600.0 +SEGMENT_AGE = INTERVAL + GRACE + TIMEOUT + MARGIN # 6120 +WINDOW_AGE = INTERVAL + TIMEOUT # 5400 +TEMP_AGE = 900.0 + 300.0 # cut_budget + temp margin +OLD = 100_000.0 # older than every threshold + + +def window_ms(k: int) -> int: + """End of the k-th window before NOW's current one, in epoch ms.""" + + return int((NOW - k * INTERVAL) * 1000) + + +def deadline_for(window_end_ms: int) -> float: + return window_end_ms / 1000 + GRACE + TIMEOUT + + +# ------------------------------------------------------------------ fakes + + +class FakeKube: + namespace = DGD.namespace + + def __init__(self) -> None: + self.requests: dict[str, dict[str, Any]] = {} + self.runs: dict[str, dict[str, Any]] = {} + self.job_pods: dict[str, list[PodInfo]] = {} + self.deleted: list[str] = [] + self.selectors: list[str] = [] + self.get_run_calls: list[str] = [] + self.fail_list = False + self.fail_get_run = False + self.fail_delete: set[str] = set() + self.on_delete: Optional[Callable[[str], None]] = None + + def list_requests(self, label_selector: str, *, timeout_s: float = 10.0): + self.selectors.append(label_selector) + if self.fail_list: + raise KubeError("list failed") + return [dict(obj) for obj in self.requests.values()] + + def delete_request( + self, name: str, *, propagation: str = "Background", timeout_s: float = 10.0 + ): + if name in self.fail_delete: + raise KubeError("delete failed") + if self.on_delete is not None: + self.on_delete(name) + self.deleted.append(name) + self.requests.pop(name, None) + + def get_run(self, name: str, *, timeout_s: float = 10.0): + self.get_run_calls.append(name) + if self.fail_get_run: + raise KubeError("get failed") + return self.runs.get(name) + + def list_pods(self, label_selector: str, *, timeout_s: float = 10.0): + key, _, value = label_selector.partition("=") + assert key == JOB_NAME_LABEL + return list(self.job_pods.get(value, [])) + + # -- builders + + def add_request( + self, + name: str, + window: int, + *, + dgd_uid: Optional[str] = DGD.uid, + state: str = "completed", # completed | running | creating | invalid + job_pods: int = 0, + job_pod_phase: Optional[str] = None, + claim: str = "traces", + path: Optional[str] = None, + generation: int = 1, + ) -> None: + labels = {LABEL_DGD: DGD.name, LABEL_WINDOW: str(window)} + if dgd_uid is not None: + labels[LABEL_DGD_UID] = dgd_uid + status: dict[str, Any] = {"conditions": []} + run_name = f"{name}-{generation}" + job = f"{run_name}-job" + if state in ("completed", "running"): + completed = state == "completed" + self.runs[run_name] = { + "metadata": { + "name": run_name, + "annotations": {RUN_GENERATION_ANNOTATION: str(generation)}, + }, + "status": { + "jobRef": {"name": job}, + "conditions": [ + { + "type": "Completed", + "status": "True" if completed else "False", + "reason": "SearchSucceeded" if completed else "JobRunning", + } + ], + }, + } + if completed: + status["recentRunRefs"] = [{"name": run_name}] + else: + status["activeRunRef"] = {"name": run_name} + self.job_pods[job] = [ + PodInfo( + name=f"{job}-{i}", + uid=str(i), + ip=None, + ready=False, + terminating=False, + phase=job_pod_phase, + ) + for i in range(job_pods) + ] + accepted = ("True", "RunCreated") + elif state == "invalid": + accepted = ("False", "InvalidSpec") + else: + accepted = ("False", "RunCreating") + status["activeRunRef"] = {"name": run_name} + status["conditions"].append( + { + "type": "Accepted", + "status": accepted[0], + "reason": accepted[1], + "observedGeneration": generation, + } + ) + self.requests[name] = { + "metadata": { + "name": name, + "uid": f"{name}-uid", + "generation": generation, + "labels": labels, + }, + "spec": { + "workload": { + "trace": { + "format": "dynamo", + "source": { + "pvc": { + "claimName": claim, + "path": path or f"windows/dgd/{window}.jsonl.gz", + } + }, + } + } + }, + "status": status, + } + + def finish_job(self, name: str) -> None: + for run in self.runs.values(): + if run["metadata"]["name"].startswith(name): + self.job_pods[run["status"]["jobRef"]["name"]] = [] + + +class RecordingMetrics: + def __init__(self) -> None: + self.sinks: list[tuple[str, bool]] = [] + self.fail_sinks = False + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + if self.fail_sinks: + raise RuntimeError("metrics backend failed") + self.sinks.append((pod, missing)) + + def __getattr__(self, name: str): + return lambda *args, **kwargs: None + + +def make( + tmp_path: Path, *, keep_requests: int = 2 +) -> tuple[Reconciler, FakeKube, RecordingMetrics]: + cfg = DgdrExecutorConfig( + claim_name="traces", + model="Qwen/Qwen3-0.6B", + sku="h200_sxm", + engine={}, + cut_budget_seconds=900, + keep_requests=keep_requests, + ) + retune = RetuneConfig( + interval_seconds=INTERVAL, + grace_seconds=GRACE, + timeout_seconds=TIMEOUT, + executor="dgdr", + dgdr=cfg, + ) + kube = FakeKube() + metrics = RecordingMetrics() + reconciler = Reconciler( + kube, + DGD, + cfg, + retune, + str(tmp_path), + metrics, + now=lambda: NOW, + visibility_margin_s=MARGIN, + ) + return reconciler, kube, metrics + + +def touch(path: Path, age_s: float, content: bytes = b"x") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + os.utime(path, (NOW - age_s, NOW - age_s)) + return path + + +def age_dir(path: Path, age_s: float) -> None: + os.utime(path, (NOW - age_s, NOW - age_s)) + + +def frontend(name: str, *, ready: bool = True, terminating: bool = False) -> PodInfo: + return PodInfo( + name=name, + uid=f"{name}-uid", + ip="10.0.0.1", + ready=ready, + terminating=terminating, + ) + + +def run_once( + reconciler: Reconciler, + *, + last: Optional[int] = None, + due: Optional[int] = None, + pods: Optional[list[PodInfo]] = None, + should_continue: Optional[Callable[[], bool]] = None, +): + return reconciler.run_once( + last_window_end_ms=last, + due_window_end_ms=due, + deadline_for=deadline_for, + frontend_pods=pods, + should_continue=should_continue, + ) + + +# ----------------------------------------------------------------- segments + + +def test_segments_of_a_live_pod_keep_the_newest_and_recent_ones(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + pod_dir = Path(reconciler.segments_root) / "fe-0" + old0 = touch(pod_dir / "req.000000.jsonl.gz", OLD) + recent = touch(pod_dir / "req.000001.jsonl.gz", SEGMENT_AGE - 60) + old2 = touch(pod_dir / "req.000002.jsonl.gz", OLD) + newest = touch(pod_dir / "req.000010.jsonl.gz", OLD) # numeric, not lexical + other = touch(pod_dir / "notes.txt", OLD) + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_segments == 2 + assert not old0.exists() and not old2.exists() + assert recent.exists() and newest.exists() and other.exists() + assert report.errors == {} + + +def test_directory_of_a_gone_pod_goes_only_when_everything_is_old( + tmp_path: Path, +) -> None: + reconciler, _, _ = make(tmp_path) + root = Path(reconciler.segments_root) + touch(root / "gone-old" / "req.000000.jsonl.gz", OLD) + touch(root / "gone-old" / "req.000001.jsonl.gz", SEGMENT_AGE + 1) + age_dir(root / "gone-old", OLD) + touch(root / "gone-recent" / "req.000000.jsonl.gz", OLD) + touch(root / "gone-recent" / "req.000001.jsonl.gz", 10) + age_dir(root / "gone-recent", OLD) + touch(root / "fe-0" / "req.000000.jsonl.gz", 10) + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_pod_dirs == ("gone-old",) + assert not (root / "gone-old").exists() + assert (root / "gone-recent").exists() + assert (root / "fe-0").exists() + + +@pytest.mark.parametrize("pods", [None, []]) +def test_no_segment_is_deleted_without_a_known_pod_set(tmp_path: Path, pods) -> None: + reconciler, _, _ = make(tmp_path) + root = Path(reconciler.segments_root) + seg = touch(root / "fe-0" / "req.000000.jsonl.gz", OLD) + touch(root / "fe-0" / "req.000001.jsonl.gz", OLD) + age_dir(root / "fe-0", OLD) + + report = run_once(reconciler, pods=pods) + + assert report.deleted_segments == 0 and report.deleted_pod_dirs == () + assert seg.exists() + + +def test_trace_sink_missing_is_reported_and_cleared(tmp_path: Path) -> None: + reconciler, _, metrics = make(tmp_path) + root = Path(reconciler.segments_root) + touch(root / "fe-0" / "req.000000.jsonl.gz", 0) + pods = [ + frontend("fe-0"), + frontend("fe-1"), + frontend("starting", ready=False), + frontend("leaving", terminating=True), + ] + + report = run_once(reconciler, pods=pods) + assert report.sink_missing == ("fe-1",) + assert sorted(metrics.sinks) == [("fe-0", False), ("fe-1", True)] + + metrics.sinks.clear() + (root / "fe-1").mkdir() + report = run_once(reconciler, pods=pods) + assert report.sink_missing == () + assert ("fe-1", False) in metrics.sinks + + # A pod that leaves has its gauge cleared. + metrics.sinks.clear() + run_once(reconciler, pods=[frontend("fe-0")]) + assert sorted(metrics.sinks) == [("fe-0", False), ("fe-1", False)] + + # An unknown pod set reports nothing. + metrics.sinks.clear() + assert run_once(reconciler, pods=None).sink_missing == () + assert metrics.sinks == [] + + +@pytest.mark.skipif( + hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads a directory whatever its mode", +) +def test_an_unreadable_pod_directory_does_not_stop_the_others(tmp_path: Path) -> None: + reconciler, kube, metrics = make(tmp_path, keep_requests=0) + root = Path(reconciler.segments_root) + names = [f"fe-{i}" for i in range(6)] + for name in names: + touch(root / name / "req.000000.jsonl.gz", OLD) + touch(root / name / "req.000001.jsonl.gz", OLD) + touch(root / name / "req.000002.jsonl.gz", OLD) + kube.add_request("rt-old", window_ms(6), state="completed") + # Created by a frontend running as another uid, or with umask 077. + locked = root / "fe-3" + locked.chmod(0) + try: + report = run_once( + reconciler, due=window_ms(0), pods=[frontend(name) for name in names] + ) + finally: + locked.chmod(stat.S_IRWXU) + + # Every other pod lost its two old segments, whatever the scandir order. + assert report.deleted_segments == 2 * (len(names) - 1) + assert len(list(locked.iterdir())) == 3 + assert report.errors == {"segments": 1} + # The sink check and the Request GC still ran. + assert sorted(metrics.sinks) == [(name, False) for name in names] + assert report.deleted_requests == ("rt-old",) + + +def test_a_failing_sink_check_keeps_the_segment_counts(tmp_path: Path) -> None: + reconciler, _, metrics = make(tmp_path) + pod_dir = Path(reconciler.segments_root) / "fe-0" + touch(pod_dir / "req.000000.jsonl.gz", OLD) + touch(pod_dir / "req.000001.jsonl.gz", OLD) + metrics.fail_sinks = True + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_segments == 1 + assert report.errors == {"sinks": 1} + + +def test_only_the_top_level_segments_of_a_live_pod_are_collected( + tmp_path: Path, +) -> None: + # P10 requires the sink's file prefix directly under the pod directory, + # so the reconciler does not walk subdirectories of a live pod. + reconciler, _, _ = make(tmp_path) + pod_dir = Path(reconciler.segments_root) / "fe-0" + nested = [touch(pod_dir / "sub" / f"req.00000{i}.jsonl.gz", OLD) for i in range(3)] + top = [touch(pod_dir / f"req.00000{i}.jsonl.gz", OLD) for i in range(3)] + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_segments == 2 + assert [path.exists() for path in top] == [False, False, True] + assert all(path.exists() for path in nested) + + +# ------------------------------------------------------------------ windows + + +def window_path(reconciler: Reconciler, window: int) -> Path: + return Path(reconciler.windows_root) / window_file_name(window) + + +def test_protected_windows_are_never_deleted(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + last, due = window_ms(2), int((NOW - 60) * 1000) + assert NOW >= deadline_for(last) and NOW < deadline_for(due) + last_path = touch(window_path(reconciler, last), OLD) + due_path = touch(window_path(reconciler, due), OLD) + stale = window_ms(5) + stale_path = touch(window_path(reconciler, stale), OLD) + registry_window = window_ms(7) + registry_path = touch(window_path(reconciler, registry_window), OLD) + reconciler.protect(registry_window, []) + + report = run_once(reconciler, last=last, due=due, pods=[frontend("fe-0")]) + + assert due_path.exists() # undecided and before its deadline + assert not last_path.exists() # in the range, but past its deadline + assert registry_path.exists() # this process's sweep uses it + assert not stale_path.exists() + assert report.deleted_windows == tuple(sorted((last, stale))) + + reconciler.release(registry_window) + report = run_once(reconciler, last=last, due=due) + assert report.deleted_windows == (registry_window,) + assert due_path.exists() + + +def test_range_protection_uses_the_window_deadline(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + # Two undecided windows, both before their deadline. + young = int((NOW - 60) * 1000) + older = int((NOW - 1000) * 1000) + paths = [touch(window_path(reconciler, w), OLD) for w in (young, older)] + + run_once(reconciler, last=older, due=young) + assert all(path.exists() for path in paths) + + # Without a persisted last window, every window before the due one with + # a live deadline is protected too. + run_once(reconciler, last=None, due=young) + assert all(path.exists() for path in paths) + + +def test_referenced_window_goes_once_every_referrer_is_finished(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + # A window outside the protected range whose deadline has not passed, + # so only the referrer rule decides. + window = int((NOW - 60) * 1000) + path = touch(window_path(reconciler, window), 10) # young: no fallback + kube.add_request("rt-a", window, state="completed") + kube.add_request("rt-b", window, state="running", job_pods=1) + + def round_(): + return run_once(reconciler, last=window_ms(0), due=window_ms(0)) + + round_() + assert path.exists() # rt-b's Run is still active + + # Completed, but the Job pod is still there. + kube.add_request("rt-b", window, state="completed", job_pods=1) + round_() + assert path.exists() + + kube.finish_job("rt-b") + report = round_() + assert report.deleted_windows == (window,) + assert not path.exists() + assert kube.deleted == [] # the Requests stay (keep_requests) + + +@pytest.mark.parametrize( + ("phase", "deleted"), + [("Succeeded", True), ("Failed", True), ("Running", False), (None, False)], +) +def test_finished_job_pods_of_a_completed_run_release_its_window( + tmp_path: Path, phase: Optional[str], deleted: bool +) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = int((NOW - 60) * 1000) + path = touch(window_path(reconciler, window), 10) # young: no fallback + kube.add_request("rt-a", window, state="completed", job_pods=1, job_pod_phase=phase) + + report = run_once(reconciler, last=window_ms(0), due=window_ms(0)) + assert (report.deleted_windows == (window,)) == deleted + assert path.exists() != deleted + + +def test_invalid_spec_request_does_not_hold_its_window(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = window_ms(4) + path = touch(window_path(reconciler, window), 10) + kube.add_request("rt-a", window, state="invalid") + + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_windows == (window,) + assert not path.exists() + + +def test_request_waiting_for_its_run_holds_its_window(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = window_ms(4) + path = touch(window_path(reconciler, window), OLD) + kube.add_request("rt-a", window, state="creating") + assert path.exists() + # The deadline has passed, so the Request itself is deleted; the window + # file then has no referrer and is old. + report = run_once(reconciler, due=window_ms(0)) + assert kube.deleted == ["rt-a"] + assert report.deleted_windows == (window,) + assert not path.exists() + + reconciler2, kube2, _ = make(tmp_path / "other", keep_requests=10) + young_window = int((NOW - 60) * 1000) + young_path = touch(window_path(reconciler2, young_window), OLD) + kube2.add_request("rt-b", young_window, state="creating") + report = run_once(reconciler2, due=None, last=None) + assert young_path.exists() + assert kube2.deleted == [] + + +def test_unreferenced_window_needs_the_fallback_age(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + young = touch(window_path(reconciler, window_ms(5)), WINDOW_AGE - 60) + old = touch(window_path(reconciler, window_ms(6)), WINDOW_AGE + 60) + unknown = touch(Path(reconciler.windows_root) / "README", OLD) + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_windows == (window_ms(6),) + assert young.exists() and not old.exists() and unknown.exists() + + +def test_windows_are_kept_when_requests_cannot_be_listed(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path) + path = touch(window_path(reconciler, window_ms(6)), OLD) + temp = touch(Path(reconciler.windows_root) / f".tmp-{window_ms(6)}-abc", OLD) + kube.fail_list = True + + report = run_once(reconciler, due=window_ms(0)) + + assert path.exists() + assert not temp.exists() # temp files do not depend on references + assert report.errors == {"list_requests": 1} + + +def test_a_request_of_another_claim_does_not_reference_the_window( + tmp_path: Path, +) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = window_ms(6) + path = touch(window_path(reconciler, window), OLD) + kube.add_request("rt-a", window, state="running", claim="other-claim") + kube.add_request( + "rt-b", window, state="running", path=f"windows/other-dgd/{window}.jsonl.gz" + ) + # Both Requests are past their deadline and get deleted anyway; the + # window had no referrer and is old. + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_windows == (window,) + assert not path.exists() + + +def test_temp_files(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + root = Path(reconciler.windows_root) + protected = window_ms(1) + reconciler.protect(protected, []) + old = touch(root / f".tmp-{window_ms(3)}-a1", TEMP_AGE + 1) + young = touch(root / f".tmp-{window_ms(3)}-b2", TEMP_AGE - 1) + in_use = touch(root / f".tmp-{protected}-c3", OLD) + odd = touch(root / ".tmp-garbage", TEMP_AGE + 1) + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_temp_files == 2 + assert not old.exists() and not odd.exists() + assert young.exists() and in_use.exists() + + +# ----------------------------------------------------------------- requests + + +def test_terminal_requests_outside_the_newest_kept_windows_go(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=2) + windows = [window_ms(k) for k in (5, 4, 3)] + for i, window in enumerate(windows): + kube.add_request(f"rt-{i}-a", window, state="completed") + kube.add_request(f"rt-{i}-b", window, state="invalid") + + report = run_once(reconciler, due=window_ms(0)) + + assert kube.selectors == [f"{LABEL_DGD}=dgd"] + assert report.deleted_requests == ("rt-0-a", "rt-0-b") + assert set(kube.requests) == {"rt-1-a", "rt-1-b", "rt-2-a", "rt-2-b"} + + +def test_keep_zero_still_keeps_requests_of_an_undecided_window(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + due = int((NOW - 60) * 1000) + kube.add_request("rt-due", due, state="completed") + kube.add_request("rt-old", window_ms(4), state="completed") + + report = run_once(reconciler, last=window_ms(4), due=due) + + # rt-due's window is undecided and before its deadline: the next sweep + # adopts its Succeeded Request instead of recreating it. + assert report.deleted_requests == ("rt-old",) + + +def test_non_terminal_request_goes_after_its_deadline(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + current = int((NOW - 60) * 1000) + kube.add_request("rt-live", current, state="running", job_pods=1) + kube.add_request("rt-late", window_ms(2), state="running", job_pods=1) + + report = run_once(reconciler, due=current) + + assert report.deleted_requests == ("rt-late",) + + +def test_requests_of_a_recreated_dgd_are_deleted(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + current = int((NOW - 60) * 1000) + kube.add_request("rt-old-uid", current, dgd_uid="uid-before", state="running") + kube.add_request("rt-no-uid", current, dgd_uid=None, state="running") + kube.add_request("rt-ours", current, state="running") + + report = run_once(reconciler, due=current) + + assert report.deleted_requests == ("rt-old-uid",) + + +def test_delete_queue(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + current = int((NOW - 60) * 1000) + kube.add_request("rt-a", current, state="running") + kube.add_request("rt-b", current, state="running") + reconciler.enqueue_delete(["rt-a", "rt-b", "rt-missing"]) + kube.fail_delete = {"rt-b"} + + report = run_once(reconciler, due=current) + + assert report.deleted_requests == ("rt-a", "rt-missing") + assert report.errors == {"delete_request": 1} + assert reconciler.pending_deletes() == frozenset({"rt-b"}) + + kube.fail_delete = set() + report = run_once(reconciler, due=current) + assert report.deleted_requests == ("rt-b",) + assert reconciler.pending_deletes() == frozenset() + + +def test_the_delete_queue_is_processed_when_listing_fails(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path) + reconciler.enqueue_delete(["rt-a"]) + kube.fail_list = True + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_requests == ("rt-a",) + + +def test_protected_requests_are_never_deleted(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + old = window_ms(6) + kube.add_request("rt-terminal", old, state="completed") + kube.add_request("rt-late", old, state="running") + kube.add_request("rt-other-uid", old, dgd_uid="uid-before", state="running") + reconciler.protect(old, ["rt-terminal", "rt-late", "rt-other-uid"]) + reconciler.enqueue_delete(["rt-late"]) + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_requests == () + assert kube.deleted == [] + assert reconciler.pending_deletes() == frozenset({"rt-late"}) + assert reconciler.protected_requests() == frozenset( + {"rt-terminal", "rt-late", "rt-other-uid"} + ) + + reconciler.release(old) + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_requests == ("rt-late", "rt-other-uid", "rt-terminal") + + +def test_unreadable_run_state_keeps_the_request(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + kube.add_request("rt-a", window_ms(6), state="completed") + window = touch(window_path(reconciler, window_ms(6)), 10) + kube.fail_get_run = True + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_requests == () + assert report.deleted_windows == () + assert window.exists() + assert report.errors == {"get_run": 1} + + +def test_terminal_and_finished_verdicts_are_cached(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = window_ms(6) + kube.add_request("rt-a", window, state="completed") + touch(window_path(reconciler, window), 10) + + run_once(reconciler, due=window_ms(0)) + calls = len(kube.get_run_calls) + assert calls > 0 + touch(window_path(reconciler, window), 10) # recreated: still finished + report = run_once(reconciler, due=window_ms(0)) + assert len(kube.get_run_calls) == calls + assert report.deleted_windows == (window,) + + +def test_deleted_requests_stop_referencing_their_window(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + old_window, young_window = window_ms(6), window_ms(7) + old = touch(window_path(reconciler, old_window), WINDOW_AGE + 60) + young = touch(window_path(reconciler, young_window), WINDOW_AGE - 60) + kube.add_request("rt-old", old_window, state="running") + kube.add_request("rt-young", young_window, state="running") + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_requests == ("rt-old", "rt-young") + assert report.deleted_windows == (old_window,) + assert not old.exists() and young.exists() + + +# ------------------------------------------------------------ early stop + + +def one_of_each(tmp_path: Path) -> tuple[Reconciler, FakeKube]: + """Six deletable things: 2 segments, a pod dir, a temp file, a Request + and the window file it referenced.""" + + reconciler, kube, _ = make(tmp_path, keep_requests=0) + root = Path(reconciler.segments_root) + for i in range(3): + touch(root / "fe-0" / f"req.00000{i}.jsonl.gz", OLD) + touch(root / "gone" / "req.000000.jsonl.gz", OLD) + age_dir(root / "gone", OLD) + touch(Path(reconciler.windows_root) / f".tmp-{window_ms(9)}-a1", OLD) + touch(window_path(reconciler, window_ms(6)), OLD) + kube.add_request("rt-a", window_ms(6), state="completed") + return reconciler, kube + + +def deletions(report) -> int: + return ( + report.deleted_segments + + len(report.deleted_pod_dirs) + + len(report.deleted_windows) + + report.deleted_temp_files + + len(report.deleted_requests) + ) + + +def test_should_continue_is_asked_before_every_delete(tmp_path: Path) -> None: + reconciler, _ = one_of_each(tmp_path) + calls: list[int] = [] + + def should_continue() -> bool: + calls.append(1) + return True + + report = run_once( + reconciler, + due=window_ms(0), + pods=[frontend("fe-0")], + should_continue=should_continue, + ) + + assert deletions(report) == 6 + assert len(calls) == 6 + assert not report.stopped + + +def test_a_no_from_should_continue_ends_the_round(tmp_path: Path) -> None: + reconciler, kube = one_of_each(tmp_path) + answers = [True, True] + + def should_continue() -> bool: + return answers.pop(0) if answers else False + + report = run_once( + reconciler, + due=window_ms(0), + pods=[frontend("fe-0")], + should_continue=should_continue, + ) + + # Two segment deletions, then nothing: no Request was even listed. + assert deletions(report) == 2 + assert report.stopped + assert kube.selectors == [] and kube.deleted == [] + assert window_path(reconciler, window_ms(6)).exists() + + report = run_once(reconciler, due=window_ms(0), pods=[frontend("fe-0")]) + assert deletions(report) == 4 + assert not report.stopped + + +def test_a_failing_should_continue_ends_the_round(tmp_path: Path) -> None: + reconciler, kube = one_of_each(tmp_path) + + def should_continue() -> bool: + raise RuntimeError("lease state unknown") + + report = run_once( + reconciler, + due=window_ms(0), + pods=[frontend("fe-0")], + should_continue=should_continue, + ) + + assert deletions(report) == 0 + assert report.stopped + assert kube.deleted == [] + + +# --------------------------------------------------------------- set_dgd + + +RECREATED = DgdRef( + name=DGD.name, namespace=DGD.namespace, uid="uid-new", dynamo_namespace="dyn-ns" +) + + +def test_set_dgd_follows_a_recreated_dgd(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=2) + current = int((NOW - 60) * 1000) + # The new DGD's Requests: one kept as a recent terminal window, one + # still running before its deadline. + kube.add_request("new-w1-done", window_ms(1), dgd_uid="uid-new") + kube.add_request("new-w0-running", current, dgd_uid="uid-new", state="running") + kube.add_request("old-w2", window_ms(2), dgd_uid=DGD.uid, state="running") + + reconciler.set_dgd(RECREATED) + report = run_once(reconciler, due=current) + + assert reconciler.dgd == RECREATED + assert report.deleted_requests == ("old-w2",) + assert set(kube.requests) == {"new-w1-done", "new-w0-running"} + + +def test_set_dgd_clears_the_request_state_caches(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + kube.add_request("rt-a", window_ms(3), state="completed") + + run_once(reconciler, due=window_ms(0)) + calls = len(kube.get_run_calls) + assert calls > 0 + run_once(reconciler, due=window_ms(0)) + assert len(kube.get_run_calls) == calls # cached terminal verdict + + reconciler.set_dgd(DGD) + run_once(reconciler, due=window_ms(0)) + assert len(kube.get_run_calls) > calls + + +def test_set_dgd_during_a_round_stops_it(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path) + reconciler.enqueue_delete(["rt-a", "rt-b", "rt-c"]) + kube.on_delete = lambda name: reconciler.set_dgd(RECREATED) + + report = run_once(reconciler, due=window_ms(0)) + + # It classified Requests by the old uid: nothing after the change. + assert report.deleted_requests == ("rt-a",) + assert report.stopped + assert reconciler.pending_deletes() == frozenset({"rt-b", "rt-c"}) + + +def test_set_dgd_with_a_new_name_moves_the_roots(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + renamed = DgdRef(name="other", namespace="ns", uid="u2", dynamo_namespace="d") + + reconciler.set_dgd(renamed) + + assert reconciler.segments_root == str(tmp_path / "segments" / "other") + assert reconciler.windows_root == str(tmp_path / "windows" / "other") + + +# ------------------------------------------------------------------ registry + + +def test_registry_is_reference_counted_per_window(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + reconciler.protect(5, []) + reconciler.protect(5, ["rt-a"]) + reconciler.protect(5, ["rt-b"]) + reconciler.protect(6, ["rt-c"]) + assert reconciler.protected_windows() == frozenset({5, 6}) + assert reconciler.protected_requests() == frozenset({"rt-a", "rt-b", "rt-c"}) + + # Every protect call holds one reference; the names stay the union of + # everything given to the window while any reference is left. + reconciler.release(5) + reconciler.release(5) + assert reconciler.protected_windows() == frozenset({5, 6}) + assert reconciler.protected_requests() == frozenset({"rt-a", "rt-b", "rt-c"}) + reconciler.release(5) + assert reconciler.protected_windows() == frozenset({6}) + assert reconciler.protected_requests() == frozenset({"rt-c"}) + + reconciler.release(5) # already released: no-op, never negative + reconciler.release(99) # unknown: no-op + assert reconciler.protected_windows() == frozenset({6}) + reconciler.protect(5, ["rt-d"]) + assert reconciler.protected_requests() == frozenset({"rt-c", "rt-d"}) + reconciler.release(5) + assert reconciler.protected_windows() == frozenset({6}) + assert reconciler.protected_requests() == frozenset({"rt-c"}) + + +def test_an_old_sweeps_release_keeps_a_relaunched_sweeps_protection( + tmp_path: Path, +) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + window = window_ms(6) + path = touch(window_path(reconciler, window), OLD) + kube.add_request("rt-old", window, state="running") + kube.add_request("rt-new", window, state="running") + # A rebuilt controller relaunches the window while the aborted sweep + # of the previous generation is still winding down. + reconciler.protect(window, ["rt-old"]) + reconciler.protect(window, ["rt-new"]) + reconciler.release(window) # the aborted sweep's finally + + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_requests == () + assert report.deleted_windows == () + assert path.exists() + + reconciler.release(window) # the relaunched sweep ends + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_requests == ("rt-new", "rt-old") + assert report.deleted_windows == (window,) + + +def test_registry_is_thread_safe_during_rounds(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + for k in range(10): + kube.add_request(f"rt-{k}", window_ms(20 + k), state="completed") + errors: list[BaseException] = [] + stop = threading.Event() + + def churn(offset: int) -> None: + try: + i = 0 + while not stop.is_set(): + window = 1_000 * offset + i % 50 + reconciler.protect(window, [f"x-{offset}-{i}"]) + reconciler.enqueue_delete([]) + reconciler.release(window) + i += 1 + except BaseException as exc: # surfaced below + errors.append(exc) + + threads = [threading.Thread(target=churn, args=(n,)) for n in range(4)] + for thread in threads: + thread.start() + try: + for _ in range(20): + run_once(reconciler, due=window_ms(0)) + finally: + stop.set() + for thread in threads: + thread.join() + assert errors == [] + assert reconciler.protected_windows() == frozenset() + assert kube.requests == {} + + +# ------------------------------------------------------------------ helpers + + +def test_window_relpath_matches_the_reconciler_layout(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + cfg = DgdrExecutorConfig( + claim_name="traces", model="m", sku="s", engine={}, windows_dir="/w/x/" + ) + assert window_relpath(cfg, "dgd", 1234) == "w/x/dgd/1234.jsonl.gz" + assert reconciler.windows_root == str(tmp_path / "windows" / "dgd") + assert reconciler.segments_root == str(tmp_path / "segments" / "dgd") diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py new file mode 100644 index 000000000000..78ba876a2d6c --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py @@ -0,0 +1,562 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the live retune state snapshot, write rules and Lease leadership.""" + +from __future__ import annotations + +import dataclasses +import json + +import pytest + +from dynamo.planner.config.planner_config import PlannerConfig +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD, + LABEL_DGD_UID, + DgdRef, + FrontendStartup, + KubeConflict, + KubeError, +) +from dynamo.planner.retune.live.state import ( + HISTORY_ANNOTATION, + HISTORY_LIMIT, + STATE_ANNOTATION, + LeaseLeadership, + RetuneState, + StateFormatError, + StateSnapshot, + StateStore, + append_history, + claim, + deploy, + deployed_off, + fingerprint, + has_decision, + leader_term, + leadership_lease_name, + maps_equal, + merge_initials, + micro_time, + owner_reference, + record_decision, + reset, + rollback_complete, + rollback_start, + state_lease_name, +) +from dynamo.planner.tests.live_retune_fakes import FakeRetuneKube + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +DGD = DgdRef(name="graph", namespace="test-ns", uid="uid-1", dynamo_namespace="dyn") +STARTUP = {"aggregated": {"overlap_score_credit": 1.0, "router_temperature": 0.0}} +DEPLOYED = {"aggregated": {"overlap_score_credit": 0.5, "router_temperature": 0.2}} + + +class FakeClock: + """Shared wall and monotonic clocks a test advances by hand.""" + + def __init__(self, wall: float = 1_800_000_000.0, mono: float = 1000.0) -> None: + self.wall = wall + self.mono = mono + + def time(self) -> float: + return self.wall + + def monotonic(self) -> float: + return self.mono + + def advance(self, seconds: float) -> None: + self.wall += seconds + self.mono += seconds + + +def live_config(**overrides) -> PlannerConfig: + """A planner config that passes the live retune configuration premises.""" + + retune = { + "interval_seconds": 21600, + "grace_seconds": 120, + "timeout_seconds": 20400, + "router": { + "overlap_score_credit": {"choices": [0.5, 1.0]}, + "prefill_load_scale": 1.0, + "temperature": {"choices": [0.0, 0.2]}, + }, + "executor": "dgdr", + "dgdr": { + "claim_name": "request-traces", + "model": "Qwen/Qwen3-0.6B", + "sku": "h200_sxm", + "engine": { + "mode": "aggregated", + "backend": "vllm", + "workers": { + "aggregated": { + "parallelism": { + "preset": [ + { + "tensor": 1, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, + } + ] + }, + "scheduler": {"max_batched_tokens": 8192, "max_sequences": 256}, + }, + "prefill": None, + "decode": None, + }, + }, + }, + } + retune.update(overrides.pop("retune", {})) + return PlannerConfig( + mode="agg", + environment="kubernetes", + enable_load_scaling=False, + enable_throughput_scaling=False, + retune=retune, + **overrides, + ) + + +def startup(source: str = "a") -> FrontendStartup: + return FrontendStartup( + router_mode="kv_router", + router_config={}, + startup_maps=STARTUP, + frontend_component="Frontend", + replicas=1, + replica_sync=False, + fingerprint_source=source, + ) + + +def deployed_state(**changes) -> RetuneState: + state = RetuneState( + dgd_uid="uid-1", + leader_term="1/me/1", + fingerprint="fp", + mode="active", + deploy_seq=3, + flat={"overlap_score_credit": 0.5}, + by_wt=DEPLOYED, + initials={"r1": dict(STARTUP["aggregated"])}, + last_window_end_ms=1_000, + ) + return dataclasses.replace(state, **changes) + + +# --------------------------------------------------------------- helpers + + +def test_names_term_owner_and_time() -> None: + assert leadership_lease_name("graph") == "graph-planner-retune" + assert state_lease_name("graph") == "graph-planner-retune-state" + assert leader_term(2, "pod/abc", 7) == "2/pod/abc/7" + assert owner_reference(DGD) == { + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeployment", + "name": "graph", + "uid": "uid-1", + "blockOwnerDeletion": False, + } + assert micro_time(0.5) == "1970-01-01T00:00:00.500000Z" + + +def test_maps_equal() -> None: + assert maps_equal({"a": 1, "b": {"c": 0.1 + 0.2}}, {"a": 1.0, "b": {"c": 0.3}}) + assert not maps_equal({"a": 1}, {"a": 1, "b": 2}) + assert not maps_equal({"a": {"c": 1.0}}, {"a": 1.0}) + assert not maps_equal({"a": 1.0}, {"a": 1.1}) + assert not maps_equal(None, {}) + assert maps_equal(None, None) + + +def test_fingerprint_tracks_what_the_point_was_chosen_under() -> None: + config = live_config() + base = fingerprint(config, startup("a")) + assert base == fingerprint(live_config(), startup("a")) + assert len(base) == 16 + # The parsed frontend startup is covered. + assert fingerprint(config, startup("b")) != base + assert fingerprint(config, None) != base + # So are the objective and the scoring SLA ... + assert ( + fingerprint(live_config(retune={"objective": "Throughput"}), startup()) != base + ) + assert fingerprint(live_config(retune={"sla_ttft_ms": 99.0}), startup()) != base + # ... but not advisory, which the state's mode records. + assert fingerprint(live_config(advisory=True), startup()) == base + + +# ----------------------------------------------------------------- state + + +def test_state_round_trips_through_annotations() -> None: + state = append_history(deployed_state(rollback_pending=True), {"decision_id": "d1"}) + annotations = state.to_annotations() + assert set(annotations) == {STATE_ANNOTATION, HISTORY_ANNOTATION} + body = json.loads(annotations[STATE_ANNOTATION]) + assert body["leader_term"] == "1/me/1" and body["deploy_seq"] == 3 + assert RetuneState.from_annotations(annotations) == state + + +def test_state_parsing_edge_cases() -> None: + assert RetuneState.from_annotations(None) is None + assert RetuneState.from_annotations({"other": "x"}) is None + claimed = RetuneState.from_annotations( + {STATE_ANNOTATION: json.dumps({"dgd_uid": "u", "leader_term": "t"})} + ) + assert claimed == RetuneState(dgd_uid="u", leader_term="t") + assert not claimed.has_deployment + for bad in ( + "{", + "[]", + json.dumps({"leader_term": "t"}), + json.dumps({"dgd_uid": "u", "mode": "bogus"}), + json.dumps({"dgd_uid": "u", "flat": {"a": "x"}}), + json.dumps({"dgd_uid": "u", "deploy_seq": "3"}), + ): + with pytest.raises(StateFormatError): + RetuneState.from_annotations({STATE_ANNOTATION: bad}) + # A damaged history never blocks the state. + parsed = RetuneState.from_annotations( + { + STATE_ANNOTATION: json.dumps({"dgd_uid": "u"}), + HISTORY_ANNOTATION: "not json", + } + ) + assert parsed is not None and parsed.history == () + + +def test_deployed_off() -> None: + assert not deployed_off(None, STARTUP) + assert not deployed_off(RetuneState(dgd_uid="u"), STARTUP) + assert deployed_off(deployed_state(), STARTUP) + assert not deployed_off(deployed_state(by_wt=STARTUP), STARTUP) + assert deployed_off(deployed_state(by_wt=STARTUP, rollback_pending=True), STARTUP) + # Unknown startup: an active record may have deployed something. + assert deployed_off(deployed_state(by_wt=STARTUP), None) + # An advisory record only observed: its by_wt (an older startup) is no + # deployment, unless a rollback is still pending. + assert not deployed_off(deployed_state(mode="advisory"), STARTUP) + assert not deployed_off(deployed_state(mode="advisory", by_wt=STARTUP), None) + assert deployed_off(deployed_state(mode="advisory", rollback_pending=True), STARTUP) + + +# ----------------------------------------------------------- write rules + + +def test_claim_changes_only_the_term() -> None: + fresh = claim(None, dgd_uid="uid-1", term="t2") + assert fresh == RetuneState(dgd_uid="uid-1", leader_term="t2") + state = deployed_state() + claimed = claim(state, dgd_uid="ignored", term="t9") + assert claimed == dataclasses.replace(state, leader_term="t9") + + +def test_deploy_rollback_and_reset_replace_the_deployment() -> None: + state = deployed_state(rollback_pending=True) + applied = deploy( + state, + fingerprint="fp2", + mode="active", + flat={"k": 1.0}, + by_wt={"w": {"k": 1.0}}, + ) + assert (applied.fingerprint, applied.mode, applied.flat, applied.by_wt) == ( + "fp2", + "active", + {"k": 1.0}, + {"w": {"k": 1.0}}, + ) + assert applied.rollback_pending is False and applied.deploy_seq == 4 + assert applied.initials == state.initials and applied.history == state.history + + started = rollback_start( + deployed_state(), fingerprint="fp", mode="advisory", flat={}, by_wt=STARTUP + ) + assert started.rollback_pending and started.deploy_seq == 4 + assert started.by_wt == STARTUP and started.mode == "advisory" + + for pending in (True, False): + reset_state = reset( + deployed_state(), + fingerprint="fp3", + mode="active", + flat={}, + by_wt=STARTUP, + rollback_pending=pending, + ) + assert reset_state.rollback_pending is pending + assert reset_state.deploy_seq == 4 and reset_state.fingerprint == "fp3" + + +def test_rollback_complete_is_conditional_on_deploy_seq() -> None: + pending = deployed_state(rollback_pending=True, by_wt=STARTUP) + entry = {"decision_id": "rolled_back-3", "outcome": "rolled_back"} + done = rollback_complete(pending, expected_deploy_seq=3, entry=entry) + assert done is not None + # Only the flag flips (plus the history record). + assert dataclasses.replace(done, history=pending.history) == dataclasses.replace( + pending, rollback_pending=False + ) + assert done.history[-1] == entry + # An applied (deploy_seq moved) or an earlier completion: discard. + assert rollback_complete(pending, expected_deploy_seq=2) is None + assert rollback_complete(done, expected_deploy_seq=3) is None + + +def test_merge_initials_merges_and_prunes() -> None: + state = deployed_state(initials={"r1": {"a": 1.0}, "r2": {"a": 2.0}}) + merged = merge_initials(state, {"r3": {"a": 3}}, keep={"r1", "r3"}) + assert merged.initials == {"r1": {"a": 1.0}, "r3": {"a": 3.0}} + assert merge_initials(state, {}).initials == state.initials + + +def test_history_is_deduplicated_and_bounded() -> None: + state = deployed_state(last_window_end_ms=None) + for index in range(HISTORY_LIMIT + 5): + state = record_decision( + state, {"decision_id": f"d{index}", "outcome": "kept"}, 1000 + index + ) + assert len(state.history) == HISTORY_LIMIT + assert state.history[-1]["decision_id"] == f"d{HISTORY_LIMIT + 4}" + assert state.last_window_end_ms == 1000 + HISTORY_LIMIT + 4 + # Same id again: replaced in place at the end; the window never moves back. + again = record_decision(state, {"decision_id": "d10", "outcome": "failed"}, 5) + assert again.last_window_end_ms == state.last_window_end_ms + assert [item["decision_id"] for item in again.history].count("d10") == 1 + assert again.history[-1] == {"decision_id": "d10", "outcome": "failed"} + assert has_decision(again, "d10") and not has_decision(again, "nope") + assert record_decision(state, {"decision_id": "x"}, None).last_window_end_ms == ( + state.last_window_end_ms + ) + + +# ----------------------------------------------------------------- store + + +def test_store_creates_reads_and_conditionally_replaces() -> None: + clock = FakeClock() + kube = FakeRetuneKube(now=clock.time) + store = StateStore(kube, DGD, clock=clock.time) + empty = store.read() + assert empty == StateSnapshot(state=None, resource_version=None, lease=None) + + created = store.write(RetuneState(dgd_uid="uid-1", leader_term="t1"), empty) + lease = kube.leases[store.name] + assert lease["metadata"]["ownerReferences"] == [owner_reference(DGD)] + assert lease["metadata"]["labels"] == {LABEL_DGD: "graph", LABEL_DGD_UID: "uid-1"} + assert created.state.updated_ms == int(clock.wall * 1000) + + read = store.read() + assert ( + read.state == created.state + and read.resource_version == created.resource_version + ) + + updated = store.write(dataclasses.replace(read.state, leader_term="t2"), read) + assert store.read().state.leader_term == "t2" + # The first snapshot is stale now. + with pytest.raises(KubeConflict): + store.write(dataclasses.replace(read.state, leader_term="t3"), read) + # A create that loses the race is a conflict too. + with pytest.raises(KubeConflict): + store.write(RetuneState(dgd_uid="uid-1"), empty) + assert store.read().state.leader_term == "t2" + assert updated.resource_version != read.resource_version + + store.delete() + assert store.read().lease is None + + +def test_store_reports_a_corrupt_snapshot() -> None: + kube = FakeRetuneKube() + store = StateStore(kube, DGD) + kube.create_lease( + { + "metadata": { + "name": store.name, + "annotations": {STATE_ANNOTATION: "{broken"}, + }, + "spec": {}, + } + ) + snapshot = store.read() + assert snapshot.corrupt and snapshot.state is None and snapshot.lease is not None + # The claim overwrites it, conditional on its resourceVersion. + written = store.write(RetuneState(dgd_uid="uid-1", leader_term="t"), snapshot) + assert written.state.leader_term == "t" and not store.read().corrupt + + +# ------------------------------------------------------------- leadership + + +def leadership(kube, clock, identity="a/1", **kwargs) -> LeaseLeadership: + return LeaseLeadership( + kube, + DGD, + identity=identity, + lease_duration_s=60.0, + margin_s=15.0, + monotonic=clock.monotonic, + clock=clock.time, + **kwargs, + ) + + +def holder(kube) -> str | None: + return kube.leases[leadership_lease_name("graph")]["spec"].get("holderIdentity") + + +def test_leadership_create_renew_and_expire_locally() -> None: + clock = FakeClock() + kube = FakeRetuneKube() + a = leadership(kube, clock) + assert not a.is_leader() + assert a.step() == "acquired" + assert a.is_leader() and a.transitions == 0 + lease = kube.leases[a.name] + assert lease["spec"]["leaseDurationSeconds"] == 60 + assert lease["metadata"]["ownerReferences"] == [owner_reference(DGD)] + assert a.leader_until == clock.mono + 45.0 + + clock.advance(20) + assert a.step() == "renewed" + assert a.leader_until == clock.mono + 45.0 + # Renewal failures keep leadership only until leader_until. + kube.fail("get_lease", times=10) + clock.advance(20) + assert a.step() == "error" and a.is_leader() + clock.advance(25) + assert a.step() == "error" and not a.is_leader() + kube.clear_faults() + # Its own lease again after losing track: acquired (a new term). + assert a.step() == "acquired" and a.is_leader() + + +def test_leadership_takes_over_only_after_an_unchanged_lease_duration() -> None: + clock = FakeClock() + kube = FakeRetuneKube() + a = leadership(kube, clock, identity="a/1") + b = leadership(kube, clock, identity="b/1") + assert a.step() == "acquired" + assert b.step() == "standby" + # a keeps renewing: b never takes over. + for _ in range(6): + clock.advance(20) + assert a.step() == "renewed" + assert b.step() == "standby" + # a dies (SIGKILL) after its last renewal, which b saw: b takes over + # once it has watched the lease unchanged for 60s. + clock.advance(30) + assert b.step() == "standby" + clock.advance(29) + assert b.step() == "standby" + assert not a.is_leader() # a's local leadership expired long before + clock.advance(1) + assert b.step() == "acquired" + assert b.is_leader() and b.transitions == 1 and holder(kube) == "b/1" + # a comes back: it has lost, then stands by. + assert a.step() == "lost" and not a.is_leader() + assert a.step() == "standby" + + +def test_leadership_never_overlaps() -> None: + """a stops renewing at an arbitrary point; b never leads while a does.""" + + for stop_after in range(0, 60, 7): + clock = FakeClock() + kube = FakeRetuneKube() + a = leadership(kube, clock, identity="a/1") + b = leadership(kube, clock, identity="b/1") + a.step() + b.step() + for second in range(0, 400): + clock.advance(1) + if second < stop_after and second % 20 == 0: + a.step() + if second % 7 == 0: + b.step() + assert not (a.is_leader() and b.is_leader()) + assert b.is_leader() + + +def test_leadership_lost_to_another_holder() -> None: + clock = FakeClock() + kube = FakeRetuneKube() + a = leadership(kube, clock, identity="a/1") + assert a.step() == "acquired" + lease = kube.leases[a.name] + lease["spec"]["holderIdentity"] = "intruder" + kube.bump(lease) + assert a.step() == "lost" + assert not a.is_leader() + + +def test_empty_holder_is_taken_over_at_once_and_release_is_conditional() -> None: + clock = FakeClock() + kube = FakeRetuneKube() + a = leadership(kube, clock, identity="a/1") + b = leadership(kube, clock, identity="b/1") + assert a.step() == "acquired" + assert b.step() == "standby" + assert a.release() + assert holder(kube) is None and not a.is_leader() + # Released: b takes over without waiting a lease duration. + assert b.step() == "acquired" and b.transitions == 1 + # a no longer holds it: release is a no-op that never clears b. + assert not a.release() + assert holder(kube) == "b/1" + + +def test_leadership_conflicts_and_errors() -> None: + clock = FakeClock() + kube = FakeRetuneKube() + a = leadership(kube, clock, identity="a/1") + # Creating the lease races another creator. + kube.fail("create_lease", KubeConflict("raced")) + assert a.step() == "standby" + kube.fail("create_lease", KubeError("timeout")) + assert a.step() == "error" + assert a.step() == "acquired" + clock.advance(20) + kube.fail("replace_lease", KubeConflict("stale")) + assert a.step() == "error" and a.is_leader() + kube.fail("replace_lease", KubeError("timeout")) + assert a.step() == "error" and a.is_leader() + assert a.step() == "renewed" + # Releasing tolerates API errors. + kube.fail("get_lease", KubeError("down")) + assert not a.release() and not a.is_leader() + + +def test_leadership_validates_arguments_and_caps_the_margin() -> None: + kube = FakeRetuneKube() + clock = FakeClock() + with pytest.raises(ValueError): + LeaseLeadership(kube, DGD, identity="") + with pytest.raises(ValueError): + LeaseLeadership(kube, DGD, identity="a", lease_duration_s=0) + capped = leadership(kube, clock) + assert capped.margin_s == 15.0 + tiny = LeaseLeadership( + kube, DGD, identity="a", lease_duration_s=15.0, margin_s=100.0 + ) + assert tiny.margin_s == 10.0 + tiny.set_owner(dataclasses.replace(DGD, uid="uid-2")) + assert tiny.step() == "acquired" + owner = kube.leases[tiny.name]["metadata"]["ownerReferences"][0] + assert owner["uid"] == "uid-2" diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py new file mode 100644 index 000000000000..c071a10c1186 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -0,0 +1,2915 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for ``LiveRetuneSupervisor`` and its bootstrap, plus the §6 +interleaving harness that guards the §4.7 invariants. + +Everything runs on fakes: ``FakeRetuneKube`` (shared Leases), a fake frontend +whose routers follow the ``/router_policy`` PATCH/reset/revision semantics, +a fake actuator over it, a fake controller that drives the supervisor's +hooks as ``RetuneController.on_tick`` does, a fake planner host and clocks +the tests advance by hand. Blocking calls run inline. +""" + +from __future__ import annotations + +import asyncio +import copy +import dataclasses +import itertools +import json +import logging +import math +import random +import threading +import time +from collections import deque +from types import SimpleNamespace +from typing import Any, Optional + +import pytest + +from dynamo.planner.core.types import RetuneDecision, RetuneParams +from dynamo.planner.retune.live import bootstrap +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD_UID, + ActuationReport, + ActuationTarget, + DgdRef, + FrontendStartup, + KubeConflict, + KubeError, + NullMetrics, + PremiseReport, +) +from dynamo.planner.retune.live.state import ( + STATE_ANNOTATION, + RetuneState, + StateStore, + append_history, + leadership_lease_name, + state_lease_name, +) +from dynamo.planner.retune.live.supervisor import ( + LiveRetuneSupervisor, + SpecEvaluation, + SupervisorSettings, + current_window_end_ms, + due_window_end_ms, + plan_initialization, + window_index, +) +from dynamo.planner.tests.live_retune_fakes import FakeRetuneKube +from dynamo.planner.tests.unit.test_live_retune_state import live_config + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +INTERVAL = 600.0 +GRACE = 30.0 +TIMEOUT = 540.0 +LEASE = 60.0 +START_WALL = 1_800_000_100.0 +DGD_NAME = "graph" +STATE_NAME = state_lease_name(DGD_NAME) +LEADER_NAME = leadership_lease_name(DGD_NAME) +CREDIT = "overlap_score_credit" +SCALE = "prefill_load_scale" +TEMP = "router_temperature" +KEYS = (CREDIT, SCALE, TEMP) +DOMAIN = {CREDIT: [0.5, 1.0], SCALE: [1.0], TEMP: [0.0, 0.2]} +VARIANTS = { + "a": {"aggregated": {CREDIT: 1.0, SCALE: 1.0, TEMP: 0.0, "host": 0.75}}, + "b": {"aggregated": {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.0, "host": 0.75}}, +} +# The due window at START_WALL. +DUE0 = math.floor((START_WALL - GRACE) / INTERVAL) - 1 +# The window in progress at START_WALL (the one after DUE0). First enable +# records it as decided: neither it (its trace may have begun mid-way) nor +# the due window is swept. +K0 = math.floor(START_WALL / INTERVAL) +assert K0 == DUE0 + 1 + + +def window(k: int) -> tuple[float, float, float]: + start = k * INTERVAL + return start, start + INTERVAL, start + INTERVAL + GRACE + + +def end_ms(k: int) -> int: + return int(round(window(k)[1] * 1000)) + + +def by_wt(variant: str, flat: dict[str, float]) -> dict[str, dict[str, float]]: + return {wt: {**params, **flat} for wt, params in VARIANTS[variant].items()} + + +def startup_flat(variant: str) -> dict[str, float]: + return {key: VARIANTS[variant]["aggregated"][key] for key in KEYS} + + +# ------------------------------------------------------------------ fakes + + +class Clock: + def __init__(self) -> None: + self.wall = START_WALL + self.mono = 10_000.0 + + def time(self) -> float: + return self.wall + + def monotonic(self) -> float: + return self.mono + + def advance(self, seconds: float) -> None: + self.wall += seconds + self.mono += seconds + + +async def inline(call): + return call() + + +async def no_sleep(_seconds: float) -> None: + await asyncio.sleep(0) + + +class RecordingMetrics(NullMetrics): + def __init__(self) -> None: + self.events: list[tuple[str, Any]] = [] + self.state: dict[str, Any] = {} + + def decision(self, outcome: str) -> None: + self.events.append(("decision", outcome)) + + def failure(self, reason: str) -> None: + self.events.append(("failure", reason)) + + def skipped(self, reason: str) -> None: + self.events.append(("skipped", reason)) + + def supervisor_state(self, **kwargs: Any) -> None: + self.state = kwargs + + def scores(self, **kwargs: Any) -> None: + self.events.append(("scores", kwargs)) + + def dropped_members(self, count: int) -> None: + self.events.append(("dropped_members", count)) + + def of(self, kind: str) -> list[Any]: + return [value for event, value in self.events if event == kind] + + +@dataclasses.dataclass +class Router: + router_id: str + worker_type: str + revision: int + config: dict[str, float] + initial: dict[str, float] + manual: bool = False + + +class FakeFrontend: + """Routers with the runtime.rs semantics: PATCH merges and bumps the + revision, reset restores the initial and bumps the revision.""" + + def __init__(self, maps: dict[str, dict[str, float]]) -> None: + self._ids = itertools.count() + self.down = False + # (writer, advisory, kind, target mode, router_id) + self.writes: list[tuple[str, bool, str, str, str]] = [] + self.manual_patches = 0 + self.routers: dict[str, Router] = {} + self.restart(maps) + + def restart(self, maps: dict[str, dict[str, float]]) -> None: + self.routers = {} + for worker_type, params in maps.items(): + router_id = f"r{next(self._ids)}" + self.routers[router_id] = Router( + router_id, worker_type, 0, dict(params), dict(params) + ) + + def patch(self, router: Router, params, writer, advisory, mode) -> None: + router.config.update(params) + router.revision += 1 + router.manual = False + self.writes.append((writer, advisory, "patch", mode, router.router_id)) + + def reset(self, router: Router, writer, advisory, mode) -> None: + router.config = dict(router.initial) + router.revision += 1 + router.manual = False + self.writes.append((writer, advisory, "reset", mode, router.router_id)) + + def manual_patch(self, key: str, value: float) -> None: + self.manual_patches += 1 + for router in self.routers.values(): + router.config[key] = value + router.revision += 1 + router.manual = True + + +class FakeActuator: + """§4.8 over a ``FakeFrontend``: latch, apply/rollback/observe, parity.""" + + def __init__( + self, frontend: FakeFrontend, owner: str, advisory: bool, maps + ) -> None: + self.frontend = frontend + self.owner = owner + self.advisory = advisory + self.startup_maps = {wt: dict(p) for wt, p in maps.items()} + self.latched: dict[str, dict[str, float]] = {} + self.targets: list[ActuationTarget] = [] + self.last_pods = () + + def set_startup_maps(self, maps) -> None: + self.startup_maps = {wt: dict(p) for wt, p in maps.items()} + + async def reconcile(self, target, *, initials, expected_frontends): + self.targets.append(target) + frontend = self.frontend + if frontend.down: + return ActuationReport( + converged=False, + seen_router_ids=frozenset(initials) | frozenset(self.latched), + errors={"list_pods": 1}, + ) + routers = list(frontend.routers.values()) + + def initial(router: Router) -> Optional[dict[str, float]]: + return self.latched.get(router.router_id) or initials.get(router.router_id) + + for router in routers: + if router.revision == 0 and initial(router) is None: + self.latched[router.router_id] = dict(router.config) + for router in routers: + known = initial(router) + if target.mode == "rollback": + if known is None and not target.learn_unknown: + continue # advisory: never written, never waited for + if known is None or router.config != known: + frontend.reset(router, self.owner, self.advisory, target.mode) + self.latched[router.router_id] = dict(router.config) + elif target.mode == "apply": + if known is None: + frontend.reset(router, self.owner, self.advisory, target.mode) + self.latched[router.router_id] = dict(router.config) + known = router.config + if known != self.startup_maps.get(router.worker_type): + continue + desired = target.desired_by_wt[router.worker_type] + diff = {k: v for k, v in desired.items() if router.config.get(k) != v} + if diff: + frontend.patch(router, diff, self.owner, self.advisory, target.mode) + + def expected(router: Router): + if target.mode == "apply": + return target.desired_by_wt[router.worker_type] + return initial(router) + + converged = bool(routers) and all( + expected(router) is not None and router.config == expected(router) + for router in routers + if not ( + target.mode == "rollback" + and not target.learn_unknown + and initial(router) is None + ) + ) + known = [router for router in routers if initial(router) is not None] + mismatched = [ + router + for router in known + if initial(router) != self.startup_maps.get(router.worker_type) + ] + if mismatched: + parity: Optional[bool] = False + elif known and len(known) == len(routers): + parity = True + else: + parity = None + seen = frozenset(router.router_id for router in routers) + self.latched = {rid: cfg for rid, cfg in self.latched.items() if rid in seen} + new_initials = { + rid: dict(cfg) + for rid, cfg in self.latched.items() + if initials.get(rid) != cfg + } + return ActuationReport( + converged=converged, + new_initials=new_initials, + seen_router_ids=seen, + parity_ok=parity, + ) + + +class FakeResolver: + def __init__(self, maps) -> None: + self.maps = maps + + def startup_flat(self, keys): + return {key: float(self.maps["aggregated"][key]) for key in keys} + + def by_wt(self, flat): + return {wt: {**params, **flat} for wt, params in self.maps.items()} + + +class FakeController: + """The hook-facing surface of ``RetuneController`` plus a tick model.""" + + def __init__(self, *, generation: int, variant: str) -> None: + self.generation = generation + self.variant = variant + self.router_keys = KEYS + self.router_domain = {key: list(values) for key, values in DOMAIN.items()} + self.desired = startup_flat(variant) + self.last_k = -1 + self.pending: Optional[dict[str, Any]] = None + self.closed = False + self.dropped: list[int] = [] + self.rewinds: list[int] = [] + self.restored: list[dict[str, float]] = [] + self.started: list[tuple[float, Optional[float]]] = [] + + def start(self, start_s: float, phase_s: Optional[float] = None) -> None: + self.started.append((start_s, phase_s)) + self.last_k = -1 + + def drop_pending(self) -> None: + if self.pending is not None: + if self.pending["abort"] is not None: + self.pending["abort"].set() + self.dropped.append(self.pending["k"]) + self.pending = None + + def rewind(self, last_decided_k: int) -> None: + self.rewinds.append(last_decided_k) + if self.pending is None: + self.last_k = min(self.last_k, last_decided_k) + + def restore_desired(self, flat) -> None: + missing = set(self.router_keys) - set(flat) + if missing: + raise ValueError(f"missing {missing}") + self.desired = {key: float(flat[key]) for key in self.router_keys} + self.restored.append(dict(self.desired)) + + def close(self) -> None: + self.closed = True + + def current(self) -> RetuneParams: + return RetuneParams(router=dict(self.desired)) + + # -- tick model (what on_tick does with the hooks) -- + + def tick(self, sup: LiveRetuneSupervisor, now_s: float, rng, advisory: bool): + decisions: list[RetuneDecision] = [] + pending = self.pending + if pending is not None and pending["done"]: + self.pending = None + k = pending["k"] + if pending["abort"] is not None and pending["abort"].is_set(): + # failed(lost_leadership): no decision, rewind. + self.dropped.append(k) + self.last_k = min(self.last_k, k - 1) + elif not sup.harvest_allowed(pending["generation"]): + self.dropped.append(k) + self.last_k = min(self.last_k, k - 1) + else: + decisions.append(self._decide(k, now_s, rng, advisory)) + due = math.floor((now_s - GRACE) / INTERVAL + 1e-9) - 1 + if due > self.last_k: + gate = sup.launch_gate(due, window(due)) + if gate.reason == "already_decided": + self.last_k = due + elif gate.reason is None: + assert gate.spec_replicas and gate.abort is not None + self.last_k = due + if self.pending is not None: + abort = self.pending["abort"] + # A busy behind a sweep the supervisor abandoned (abort + # set, not dropped) consumes the window for nothing. + assert ( + abort is None or not abort.is_set() + ), "busy behind an aborted sweep" + decisions.append(make_decision(due, "busy", self.current())) + else: + self.pending = { + "k": due, + "generation": self.generation, + "abort": gate.abort, + "done": False, + } + return SimpleNamespace( + retune_decisions=decisions, + router_parameters=by_wt(self.variant, self.desired), + ) + + def _decide(self, k, now_s, rng, advisory) -> RetuneDecision: + previous = self.current() + chosen = {key: rng.choice(values) for key, values in DOMAIN.items()} + roll = rng.random() + if roll < 0.15: + return make_decision( + k, "failed", previous, failure_reason="replicas_changed" + ) + if roll < 0.35 or chosen == self.desired: + return make_decision(k, "kept", previous) + if advisory: + return make_decision(k, "advisory", previous, chosen=chosen) + self.desired = dict(chosen) + return make_decision(k, "applied", previous, chosen=chosen) + + +def make_decision( + k: int, + outcome: str, + previous: Optional[RetuneParams] = None, + *, + chosen: Optional[dict[str, float]] = None, + failure_reason: Optional[str] = None, + details: Optional[dict[str, Any]] = None, +) -> RetuneDecision: + previous = previous or RetuneParams(router=startup_flat("a")) + start_s, end_s, as_of_s = window(k) + return RetuneDecision( + decision_id=f"retune-w{end_ms(k)}", + outcome=outcome, + window_index=k, + window_start_s=start_s, + window_end_s=end_s, + as_of_s=as_of_s, + launched_at_s=None if outcome == "busy" else as_of_s, + decided_at_s=as_of_s + 60.0, + num_requests=10, + num_candidates=4, + previous=previous, + chosen=previous if chosen is None else RetuneParams(router=chosen), + baseline_score=1.0, + best_score=1.5, + sweep_wall_s=50.0, + failure_reason=failure_reason, + details=details or {}, + ) + + +class FakeHost: + """``NativePlannerBase`` as the supervisor sees it.""" + + def __init__(self, spec: Spec, clock: Clock, controller: Optional[FakeController]): + self.spec = spec + self.clock = clock + self._engine = SimpleNamespace(retune_controller=controller) + self._config_lock = asyncio.Lock() + self.replacements: list[tuple[Any, int, Optional[int]]] = [] + self.fail_replace = False + self.controllers: list[FakeController] = ( + [] if controller is None else [controller] + ) + + @property + def controller(self) -> Optional[FakeController]: + return self._engine.retune_controller + + async def replace_live_retune(self, inputs, *, generation, last_decided_k): + async with self._config_lock: + self.replacements.append((inputs, generation, last_decided_k)) + old = self._engine.retune_controller + self._engine.retune_controller = None + if old is not None: + old.drop_pending() + old.close() + if inputs is None: + return + if self.fail_replace: + raise ValueError("controller construction failed") + new = FakeController(generation=generation, variant=inputs.variant) + new.start(self.clock.time(), 0.0) + if last_decided_k is not None: + new.rewind(last_decided_k) + self._engine.retune_controller = new + self.controllers.append(new) + + +class Spec: + """The DGD spec as the premise evaluator sees it (tests mutate it).""" + + def __init__(self) -> None: + self.variant = "a" + self.spec_ok = True + self.p4_ok = True + self.replicas: Optional[dict[str, int]] = {"aggregated": 2} + self.uid = "uid-1" + self.dynamo_namespace = "dyn" + self.frontend_replicas = 1 + + def startup(self) -> FrontendStartup: + return FrontendStartup( + router_mode="kv_router", + router_config={}, + startup_maps=copy.deepcopy(VARIANTS[self.variant]), + frontend_component="Frontend", + replicas=self.frontend_replicas, + replica_sync=False, + fingerprint_source=self.variant, + ) + + def evaluation(self, _dgd: Optional[dict] = None) -> SpecEvaluation: + reasons = [] if self.p4_ok else ["P4: router mode round-robin"] + if not self.spec_ok: + reasons.append("P8: two frontends without replica sync") + return SpecEvaluation( + report=PremiseReport( + p4_ok=self.p4_ok, + spec_ok=self.p4_ok and self.spec_ok, + reasons=tuple(reasons), + ), + startup=self.startup(), + inputs=SimpleNamespace(variant=self.variant) if self.p4_ok else None, + spec_replicas=None if self.replicas is None else dict(self.replicas), + dgd=DgdRef(DGD_NAME, "test-ns", self.uid, self.dynamo_namespace), + ) + + +class World: + """Shared kube, frontend, spec and clock of one test.""" + + def __init__(self, kube: Optional[FakeRetuneKube] = None) -> None: + self.clock = Clock() + self.kube = kube or FakeRetuneKube(now=self.clock.time) + self.spec = Spec() + self.frontend = FakeFrontend(VARIANTS["a"]) + self.kube.put_dgd( + {"metadata": {"name": DGD_NAME, "uid": "uid-1"}, "spec": {"components": []}} + ) + + def shared_state(self) -> Optional[RetuneState]: + lease = self.kube.leases.get(STATE_NAME) + if lease is None: + return None + return RetuneState.from_annotations(lease["metadata"].get("annotations")) + + def holder(self) -> Optional[str]: + lease = self.kube.leases.get(LEADER_NAME) + return None if lease is None else lease["spec"].get("holderIdentity") + + +@dataclasses.dataclass +class Process: + name: str + sup: LiveRetuneSupervisor + host: FakeHost + metrics: RecordingMetrics + advisory: bool + kube: Any + alive: bool = True + + @property + def controller(self) -> Optional[FakeController]: + return self.host.controller + + +def settings_for( + *, advisory: bool = False, promote: bool = False, **changes: Any +) -> SupervisorSettings: + values: dict[str, Any] = dict( + interval_s=INTERVAL, + grace_s=GRACE, + timeout_s=TIMEOUT, + advisory=advisory, + promote_last_advisory=promote, + router_keys=KEYS, + lease_duration_s=LEASE, + actuator_interval_s=15.0, + k8s_timeout_s=5.0, + write_failure_limit=3, + ) + values.update(changes) + return SupervisorSettings(**values) + + +def make_process( + world: World, + name: str = "p1", + *, + advisory: bool = False, + promote: bool = False, + kube: Any = None, + canary=None, + reconciler=None, + sweep_runner=None, + context_factory=None, + settings: Optional[SupervisorSettings] = None, + blocking=None, +) -> Process: + kube = kube or world.kube + metrics = RecordingMetrics() + evaluation = world.spec.evaluation() + controller = ( + FakeController(generation=0, variant=world.spec.variant) + if evaluation.inputs is not None + else None + ) + host = FakeHost(world.spec, world.clock, controller) + settings = settings or settings_for(advisory=advisory, promote=promote) + sup = LiveRetuneSupervisor( + settings=settings, + kube=kube, + evaluation=evaluation, + identity=f"{name}/{len(world.kube.calls)}", + premise_evaluator=world.spec.evaluation, + actuator_factory=lambda dgd, maps: FakeActuator( + world.frontend, name, advisory, maps + ), + fingerprint_fn=lambda startup: f"fp-{None if startup is None else startup.fingerprint_source}", + resolver_factory=lambda startup: FakeResolver(startup.startup_maps), + reconciler=reconciler, + metrics=metrics, + canary=canary, + sweep_runner=sweep_runner, + context_factory=context_factory, + clock=world.clock.time, + monotonic=world.clock.monotonic, + sleep=no_sleep, + blocking=blocking or inline, + ) + return Process(name, sup, host, metrics, advisory, kube) + + +async def started(world: World, name: str = "p1", **kwargs) -> Process: + process = make_process(world, name, **kwargs) + await process.sup.start(process.host, background=False) + return process + + +async def leading(world: World, name: str = "p1", **kwargs) -> Process: + """A started process that leads, is initialized and is ready.""" + + process = await started(world, name, **kwargs) + await process.sup.lease_round() + assert process.sup.is_leader() and process.sup.initialized + await process.sup.control_round() + return process + + +async def settle(processes: list[Process], world: World, rounds: int = 6) -> None: + for _ in range(rounds): + world.clock.advance(20) + for process in processes: + if process.alive: + await process.sup.lease_round() + await process.sup.outbox_round() + await process.sup.control_round() + + +def enqueue(process: Process, *decisions: RetuneDecision) -> None: + controller = process.controller + variant = controller.variant if controller is not None else "a" + desired = controller.desired if controller is not None else startup_flat("a") + process.sup.on_effects( + SimpleNamespace( + retune_decisions=list(decisions), + router_parameters=by_wt(variant, desired), + ) + ) + + +def frontend_configs(world: World) -> list[dict[str, float]]: + return [router.config for router in world.frontend.routers.values()] + + +# ------------------------------------------------------- pure functions + + +def claimed(**changes) -> RetuneState: + base = RetuneState(dgd_uid="uid-1", leader_term="1/p/2") + return dataclasses.replace(base, **changes) + + +def deployed(**changes) -> RetuneState: + values = dict( + fingerprint="fp", + mode="active", + deploy_seq=4, + flat={CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2}, + by_wt=by_wt("a", {CREDIT: 0.5, TEMP: 0.2}), + last_window_end_ms=777, + ) + values.update(changes) + return claimed(**values) + + +def plan(state: RetuneState, **overrides): + kwargs = dict( + spec_ok=True, + advisory=False, + fingerprint="fp", + startup_flat=startup_flat("a"), + startup_by_wt=VARIANTS["a"], + router_keys=KEYS, + promote=False, + now_ms=10_000_000, + interval_ms=int(INTERVAL * 1000), + first_window_end_ms=9_999, + ) + kwargs.update(overrides) + return plan_initialization(state, **kwargs) + + +def test_init_table_first_enable_persists_the_due_window() -> None: + result = plan(claimed()) + assert result.row == "active_reset" and result.write + assert result.state.mode == "active" and not result.state.rollback_pending + assert result.state.by_wt == VARIANTS["a"] and result.state.flat == startup_flat( + "a" + ) + assert result.state.last_window_end_ms == 9_999 + assert result.state.deploy_seq == 1 + assert result.restore_flat == startup_flat("a") + + +def test_init_table_spec_not_ok_is_the_first_row() -> None: + # Deployed X: roll back (startup, pending), controller untouched. + result = plan(deployed(), spec_ok=False) + assert result.row == "spec_not_ok" and result.restore_flat is None + assert result.state.rollback_pending and result.state.by_wt == VARIANTS["a"] + assert result.state.deploy_seq == 5 + # Nothing deployed and the record current: nothing to write. + clean = deployed(by_wt=VARIANTS["a"], flat=startup_flat("a")) + assert not plan(clean, spec_ok=False).write + # Even an advisory planner takes this row (with its own mode). + advisory = plan(deployed(), spec_ok=False, advisory=True) + assert advisory.row == "spec_not_ok" and advisory.state.mode == "advisory" + assert advisory.state.rollback_pending + # Unknown startup and an active record: roll back to the latched initials. + unknown = plan( + deployed(by_wt=VARIANTS["a"]), + spec_ok=False, + startup_flat=None, + startup_by_wt=None, + ) + assert unknown.state.rollback_pending and unknown.state.by_wt == {} + + +def test_init_table_advisory_rows() -> None: + rollback = plan(deployed(), advisory=True) + assert rollback.row == "advisory_rollback" and rollback.restore_flat is None + assert rollback.state.mode == "advisory" and rollback.state.rollback_pending + assert rollback.state.by_wt == VARIANTS["a"] + + observe = plan(deployed(by_wt=VARIANTS["a"], flat=startup_flat("a")), advisory=True) + assert observe.row == "advisory_observe" and observe.write + assert observe.state.mode == "advisory" and not observe.state.rollback_pending + # Already advisory and current: no write. + again = plan(observe.state, advisory=True) + assert again.row == "advisory_observe" and not again.write + # An advisory record of an older startup is not a deployment. + older = plan(observe.state, advisory=True, startup_by_wt=VARIANTS["b"]) + assert older.row == "advisory_observe" and not older.state.rollback_pending + + +def test_init_table_active_rows() -> None: + restore = plan(deployed()) + assert restore.row == "active_restore" and not restore.write + assert restore.restore_flat == {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + + for state, pending in ( + (deployed(fingerprint="old"), True), + (deployed(mode="advisory"), False), + (deployed(rollback_pending=True), True), + (deployed(flat={CREDIT: 0.5}), True), + (deployed(fingerprint="old", by_wt=VARIANTS["a"]), False), + ): + result = plan(state) + assert result.row == "active_reset", state + assert result.state.rollback_pending is pending, state + assert result.state.by_wt == VARIANTS["a"] and result.state.mode == "active" + assert result.restore_flat == startup_flat("a") + assert result.state.last_window_end_ms == 777 + + +def test_init_table_promotes_a_recent_matching_advisory() -> None: + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + entry = { + "decision_id": "retune-w9000000", + "outcome": "advisory", + "window_end_ms": 9_900_000, + "fingerprint": "fp", + "flat": chosen, + "by_wt": by_wt("a", chosen), + } + state = append_history( + deployed(mode="advisory", by_wt=VARIANTS["a"], flat=startup_flat("a")), entry + ) + result = plan(state, promote=True) + assert result.row == "promote" and result.restore_flat == chosen + assert result.state.mode == "active" and result.state.by_wt == by_wt("a", chosen) + assert not result.state.rollback_pending and result.state.deploy_seq == 5 + assert result.state.history[-1]["outcome"] == "promoted" + # Not promoted: disabled, stale fingerprint, too old, or not the latest. + assert plan(state).row == "active_reset" + assert plan(state, promote=True, fingerprint="new").row == "active_reset" + assert plan(state, promote=True, now_ms=9_900_000 + 600_001).row == "active_reset" + later = append_history(state, {"decision_id": "x", "outcome": "kept"}) + assert plan(later, promote=True).row == "active_reset" + + +def test_window_arithmetic_matches_the_controller() -> None: + now = START_WALL + end = due_window_end_ms(now, phase_s=0.0, interval_s=INTERVAL, grace_s=GRACE) + assert end == end_ms(DUE0) + assert window_index(end, phase_s=0.0, interval_s=INTERVAL) == DUE0 + # Exactly at L_k the window is due. + at_launch = window(DUE0 + 1)[2] + assert due_window_end_ms( + at_launch, phase_s=0.0, interval_s=INTERVAL, grace_s=GRACE + ) == end_ms(DUE0 + 1) + phased = due_window_end_ms(1000.0, phase_s=100.0, interval_s=300.0, grace_s=0.0) + assert phased == 1_000_000 + assert window_index(phased, phase_s=100.0, interval_s=300.0) == 2 + + +def test_current_window_is_after_the_due_and_the_not_yet_due_windows() -> None: + def current(now_s: float, phase_s: float = 0.0) -> int: + return current_window_end_ms(now_s, phase_s=phase_s, interval_s=INTERVAL) + + # Mid-window: the window in progress, one after the due window. + assert current(START_WALL) == end_ms(K0) == end_ms(DUE0 + 1) + # Within the grace after a window ended: that window is not due yet and + # the one in progress is two after the due window; both are skipped. + start = window(K0)[0] + within_grace = start + GRACE / 2 + due = due_window_end_ms( + within_grace, phase_s=0.0, interval_s=INTERVAL, grace_s=GRACE + ) + assert due == end_ms(K0 - 2) and current(within_grace) == end_ms(K0) + # A window boundary belongs to the window starting there. + assert current(start) == end_ms(K0) + # Phased: windows [100 + k T, 100 + (k + 1) T); 1000 is in [700, 1300). + assert current(1000.0, phase_s=100.0) == 1_300_000 + + +# ----------------------------------------------------------- gate/harvest + + +async def test_gate_reasons_and_the_skip_metric() -> None: + world = World() + process = await started(world) + sup = process.sup + assert sup.launch_gate(K0 + 1, window(K0 + 1)).reason == "not_leader" + assert sup.launch_gate(K0 + 1, window(K0 + 1)).reason == "not_leader" + await sup.lease_round() + # Initialized but no health yet (no actuator round). + assert sup.initialized + assert sup.launch_gate(K0 + 1, window(K0 + 1)).reason == "not_ready" + await sup.control_round() + assert sup.ready() + # First enable recorded the window in progress at start as decided: it + # and the due window before it are skipped. + assert world.shared_state().last_window_end_ms == end_ms(K0) + assert sup.launch_gate(DUE0, window(DUE0)).reason == "already_decided" + assert sup.launch_gate(K0, window(K0)).reason == "already_decided" + gate = sup.launch_gate(K0 + 1, window(K0 + 1)) + assert gate.reason is None and gate.spec_replicas == {"aggregated": 2} + assert isinstance(gate.abort, threading.Event) and not gate.abort.is_set() + # Each (k, reason) counts once. + assert process.metrics.of("skipped") == [ + "not_leader", + "not_ready", + "already_decided", + "already_decided", + ] + # Health and spec gate only launches. + world.spec.replicas = None + await sup.control_round() + assert sup.launch_gate(K0 + 2, window(K0 + 2)).reason == "not_ready" + + +async def test_harvest_allowed_needs_leader_initialized_and_generation() -> None: + world = World() + process = await leading(world) + sup = process.sup + assert sup.harvest_allowed(0) + assert not sup.harvest_allowed(1) + world.spec.spec_ok = False + await sup.control_round() # rebuild: generation 1 + assert sup.generation == 1 + assert not sup.harvest_allowed(0) and sup.harvest_allowed(1) + world.clock.advance(LEASE) # leadership lapses without renewal + assert not sup.harvest_allowed(1) + + +# ---------------------------------------------------------------- outbox + + +async def test_applied_decision_is_persisted_then_deployed() -> None: + world = World() + process = await leading(world) + sup, controller = process.sup, process.controller + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + controller.desired = dict(chosen) + enqueue(process, make_decision(K0 + 1, "applied", chosen=chosen)) + assert sup.outbox_size == 1 + assert process.metrics.of("decision") == [] # counted only once persisted + await sup.outbox_round() + state = world.shared_state() + assert sup.outbox_size == 0 + assert state.flat == chosen and state.by_wt == by_wt("a", chosen) + assert state.mode == "active" and state.deploy_seq == 2 + assert state.last_window_end_ms == end_ms(K0 + 1) + entry = state.history[-1] + assert entry["decision_id"] == f"retune-w{end_ms(K0 + 1)}" + assert entry["outcome"] == "applied" and entry["by_wt"] == by_wt("a", chosen) + assert entry["fingerprint"] == "fp-a" and entry["baseline"] == 1.0 + assert process.metrics.of("decision") == ["applied"] + assert sup.target() == ActuationTarget( + mode="apply", desired_by_wt=by_wt("a", chosen) + ) + await sup.control_round() + assert frontend_configs(world) == [by_wt("a", chosen)["aggregated"]] + # The next window launches; the decided one does not. + assert sup.launch_gate(K0 + 1, window(K0 + 1)).reason == "already_decided" + + +async def test_advisory_decision_records_the_recommendation_only() -> None: + world = World() + process = await leading(world, advisory=True) + sup = process.sup + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + enqueue( + process, + make_decision( + K0 + 1, + "advisory", + chosen=chosen, + details={"runtime_image": "img:1", "missing_candidates": 2}, + ), + ) + await sup.outbox_round() + state = world.shared_state() + assert state.mode == "advisory" and state.by_wt == VARIANTS["a"] + entry = state.history[-1] + assert entry["outcome"] == "advisory" and entry["flat"] == chosen + assert entry["by_wt"] == by_wt("a", chosen) + assert entry["image"] == {"runtime_image": "img:1"} + scores = process.metrics.of("scores")[-1] + assert scores["missing_candidates"] == 2 and scores["best"] == 1.5 + assert sup.target() == ActuationTarget(mode="observe") + world.frontend.manual_patch(TEMP, 0.7) + await settle([process], world) + assert world.frontend.writes == [] + + +async def test_failed_decision_counts_its_reason() -> None: + world = World() + process = await leading(world) + enqueue( + process, + make_decision( + K0 + 1, + "failed", + failure_reason="crn_mismatch", + details={"window": {"dropped_members": 3, "cut": True}}, + ), + ) + await process.sup.outbox_round() + assert process.metrics.of("failure") == ["crn_mismatch"] + assert process.metrics.of("dropped_members") == [3] + assert world.shared_state().history[-1]["failure_reason"] == "crn_mismatch" + + +async def test_outbox_timeout_that_landed_and_conflict() -> None: + world = World() + process = await leading(world) + sup = process.sup + # A write that timed out but landed: the re-read finds it. + world.kube.fail( + "replace_lease", + KubeError("timeout"), + landed=True, + match=lambda target: target[0] == STATE_NAME, + ) + enqueue(process, make_decision(K0 + 1, "kept")) + await sup.outbox_round() + ids = [item["decision_id"] for item in world.shared_state().history] + assert ids.count(f"retune-w{end_ms(K0 + 1)}") == 1 + assert sup.outbox_size == 0 and process.metrics.of("decision") == ["kept"] + # A stale cached resourceVersion: conflict, re-read, retry. + world.kube.bump(world.kube.leases[STATE_NAME]) + enqueue(process, make_decision(K0 + 2, "kept")) + await sup.outbox_round() + ids = [item["decision_id"] for item in world.shared_state().history] + assert ids[-2:] == [f"retune-w{end_ms(K0 + 1)}", f"retune-w{end_ms(K0 + 2)}"] + assert sup.outbox_size == 0 and process.metrics.of("decision") == ["kept", "kept"] + + +async def test_consecutive_write_failures_reinitialize_and_rewind() -> None: + world = World() + process = await leading(world) + sup, controller = process.sup, process.controller + term_seq = sup.term_seq + world.kube.fail( + "replace_lease", + KubeError("timeout"), + times=3, + match=lambda target: target[0] == STATE_NAME, + ) + controller.pending = {"k": K0 + 2, "generation": 0, "abort": None, "done": False} + controller.last_k = K0 + 2 + enqueue(process, make_decision(K0 + 1, "kept")) + for _ in range(3): + await sup.outbox_round() + assert not sup.initialized # re-init requested + assert process.metrics.of("failure") == ["state_write"] * 3 + await sup.lease_round() + assert sup.initialized and sup.term_seq == term_seq + 1 + assert sup.outbox_size == 0 + # The sweep was dropped and the controller rewound to the last decided + # window, so both windows can still run. + assert controller.pending is None and controller.dropped == [K0 + 2] + assert controller.last_k == K0 + assert world.shared_state().leader_term == sup.term + + +async def test_every_failed_state_write_counts_as_state_write() -> None: + world = World() + process = await started(world) + sup = process.sup + + def state_lease(target) -> bool: + return target[0] == STATE_NAME + + def failures() -> list[str]: + return process.metrics.of("failure") + + # First enable: the claim creates the state, the plan write fails. + world.kube.fail("replace_lease", KubeError("quota"), match=state_lease) + await sup.lease_round() + assert not sup.initialized and failures() == ["state_write"] + await sup.lease_round() + assert sup.initialized + # The claim write of a re-initialization. + world.kube.fail("replace_lease", KubeError("forbidden"), match=state_lease) + sup._request_reinit("test") + await sup.lease_round() + assert not sup.initialized and failures() == ["state_write"] * 2 + await sup.lease_round() + assert sup.initialized + # The initials of a restarted frontend (retried next round). + world.frontend.restart(VARIANTS["a"]) + world.kube.fail("replace_lease", KubeError("too large"), match=state_lease) + await sup.control_round() + assert failures() == ["state_write"] * 3 + await sup.control_round() + assert set(world.shared_state().initials) == set(world.frontend.routers) + # A lost term is not a failed write. + store = StateStore(world.kube, sup.dgd) + snapshot = store.read() + store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) + enqueue(process, make_decision(K0 + 1, "kept")) + await sup.outbox_round() + assert not sup.initialized and failures() == ["state_write"] * 3 + + +async def test_decisions_are_dropped_and_rewound_without_leadership() -> None: + world = World() + process = await started(world) + controller = process.controller + controller.last_k = K0 + 1 + enqueue(process, make_decision(K0 + 1, "kept")) + assert process.sup.outbox_size == 0 + assert controller.rewinds and controller.last_k == -1 + + +async def test_a_foreign_term_forces_reinitialization() -> None: + world = World() + process = await leading(world) + sup = process.sup + store = StateStore(world.kube, sup.dgd) + snapshot = store.read() + store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) + enqueue(process, make_decision(K0 + 1, "kept")) + await sup.outbox_round() + assert not sup.initialized and sup.outbox_size == 1 + await sup.lease_round() + assert sup.initialized and world.shared_state().leader_term == sup.term + assert sup.outbox_size == 0 + + +async def test_a_late_write_of_an_earlier_term_fails() -> None: + world = World() + process = await leading(world) + sup = process.sup + store = StateStore(world.kube, sup.dgd) + stale = store.read() + # Same-process re-initialization (term_seq + 1) moves the term on. + sup._request_reinit("test") + await sup.lease_round() + assert world.shared_state().leader_term == sup.term != stale.state.leader_term + with pytest.raises(KubeConflict): + store.write(append_history(stale.state, {"decision_id": "late"}), stale) + + +def steal_lease(world: World, identity: str) -> None: + """The leadership Lease changes hands without expiring (§4.7.9).""" + + lease = world.kube.leases[LEADER_NAME] + lease["spec"]["holderIdentity"] = identity + lease["spec"]["leaseTransitions"] = lease["spec"].get("leaseTransitions", 0) + 1 + world.kube.bump(lease) + + +async def test_a_stolen_lease_rejects_the_old_leaders_writes() -> None: + """Two processes believe they lead; the state term precondition keeps + the old one from writing once the new one has claimed.""" + + world = World() + old = await leading(world) + new = await started(world, "p2") + steal_lease(world, new.sup.identity) + await new.sup.lease_round() + assert new.sup.initialized and old.sup.is_leader() # both believe + enqueue(old, make_decision(K0 + 1, "kept")) + await old.sup.outbox_round() + state = world.shared_state() + assert state.leader_term == new.sup.term + assert all( + item["decision_id"] != f"retune-w{end_ms(K0 + 1)}" for item in state.history + ) + assert not old.sup.initialized + await old.sup.lease_round() # sees the new holder: local loss + assert not old.sup.is_leader() and old.sup.term is None + + +async def test_a_stolen_lease_aborts_the_old_leaders_claim() -> None: + world = World() + old = await leading(world) + term = world.shared_state().leader_term + steal_lease(world, "intruder/0") + assert old.sup.is_leader() # not seen yet + world.spec.spec_ok = False + await old.sup.control_round() # a rebuild re-initializes: no claim + assert world.shared_state().leader_term == term + assert not old.sup.is_leader() and not old.sup.initialized + + +# ----------------------------------------------------- initialization + + +async def test_restart_restores_the_persisted_point() -> None: + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.desired = dict(chosen) + enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.stop() + assert world.holder() is None # released + second = await leading(world, "p2") + assert second.controller.restored == [chosen] + assert second.controller.rewinds[-1] == K0 + 1 + state = world.shared_state() + assert state.leader_term == second.sup.term and state.deploy_seq == 2 + assert second.sup.target().desired_by_wt == by_wt("a", chosen) + + +async def test_fingerprint_change_resets_with_a_two_phase_rollback() -> None: + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.desired = dict(chosen) + enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.control_round() + assert frontend_configs(world) == [by_wt("a", chosen)["aggregated"]] + # The frontend args change (new startup): restart under it. + await first.sup.stop() + world.spec.variant = "b" + world.frontend.restart(VARIANTS["b"]) + world.frontend.manual_patch(TEMP, 0.9) + second = await leading(world, "p2") + state = world.shared_state() + assert state.rollback_pending and state.by_wt == VARIANTS["b"] + assert state.fingerprint == "fp-b" and state.deploy_seq == 3 + assert second.controller.restored == [startup_flat("b")] + assert second.sup.target().mode == "rollback" + # Reset once, then converged: completes after a lease duration. + resets = len(world.frontend.writes) + await second.sup.control_round() + assert len(world.frontend.writes) == resets + assert world.shared_state().rollback_pending + await settle([second], world, rounds=2) + assert world.shared_state().rollback_pending # 40s: not yet + await settle([second], world, rounds=1) + state = world.shared_state() + assert not state.rollback_pending and state.deploy_seq == 3 + assert state.history[-1]["outcome"] == "rolled_back" + assert second.sup.target().desired_by_wt == VARIANTS["b"] + await settle([second], world, rounds=3) + assert len(world.frontend.writes) == resets # no further writes + + +async def test_applied_during_a_pending_rollback_wins() -> None: + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.desired = dict(chosen) + enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.stop() + world.spec.variant = "b" + world.frontend.restart(VARIANTS["b"]) + second = await leading(world, "p2") + assert world.shared_state().rollback_pending + seq = world.shared_state().deploy_seq + # An applied decision lands while the rollback is pending. + second.controller.desired = {CREDIT: 1.0, SCALE: 1.0, TEMP: 0.2} + enqueue(second, make_decision(K0 + 2, "applied", chosen=second.controller.desired)) + await second.sup.outbox_round() + state = world.shared_state() + assert not state.rollback_pending and state.deploy_seq == seq + 1 + assert second.sup.target().mode == "apply" + # A rollback-complete write racing it is discarded, never retried. + stale_seq = seq + from dynamo.planner.retune.live.state import rollback_complete + + assert rollback_complete(state, expected_deploy_seq=stale_seq) is None + + +@pytest.mark.parametrize("racing", ["deploy", "rollback_start"]) +async def test_a_stale_rollback_complete_is_discarded(racing: str) -> None: + """Right before the rollback-complete write of deploy_seq S, a write of + the same term moves deploy_seq on (an applied, or a new rollback): the + completion of S is discarded, never retried, never applied to S+1.""" + + from dynamo.planner.retune.live.state import deploy, rollback_start + + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.desired = dict(chosen) + enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.stop() + world.spec.variant = "b" + world.frontend.restart(VARIANTS["b"]) + second = await leading(world, "p2") + sup = second.sup + seq = world.shared_state().deploy_seq + assert world.shared_state().rollback_pending + store = sup._store + original = store.write + raced = [] + + def racing_write(state, snapshot): + if not raced: + raced.append(1) + current = store.read() + rule = deploy if racing == "deploy" else rollback_start + original( + rule( + current.state, + fingerprint="fp-b", + mode="active", + flat=startup_flat("b"), + by_wt=VARIANTS["b"], + ), + current, + ) + return original(state, snapshot) + + store.write = racing_write + for _ in range(6): + if raced: + break + await settle([second], world, rounds=1) + assert raced + state = world.shared_state() + assert state.deploy_seq == seq + 1 + assert state.rollback_pending is (racing == "rollback_start") + assert all(item.get("outcome") != "rolled_back" for item in state.history) + # The new rollback completes on its own deploy_seq. + if racing == "rollback_start": + await settle([second], world, rounds=4) + state = world.shared_state() + assert not state.rollback_pending + assert state.history[-1]["decision_id"] == f"rolled_back-{seq + 1}" + + +async def test_advisory_switch_rolls_back_only_what_was_deployed() -> None: + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.desired = dict(chosen) + enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.control_round() + await first.sup.stop() + second = await leading(world, "p2", advisory=True) + state = world.shared_state() + assert state.mode == "advisory" and state.rollback_pending + assert second.controller.restored == [] # advisory never restores + await settle([second], world, rounds=5) + state = world.shared_state() + assert not state.rollback_pending + assert frontend_configs(world) == [VARIANTS["a"]["aggregated"]] + planner_writes = [w for w in world.frontend.writes if w[0] == "p2"] + assert planner_writes and all(w[2] == "reset" for w in planner_writes) + # Now advisory only observes: a manual change is left alone. + world.frontend.manual_patch(TEMP, 0.9) + before = len(world.frontend.writes) + await settle([second], world, rounds=3) + assert len(world.frontend.writes) == before + + +async def test_advisory_rollback_leaves_unlatched_routers_alone() -> None: + """§4.8: a router changed at revision > 0 before anyone latched it is + reset to learn its initial only in active mode.""" + + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.desired = dict(chosen) + enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.control_round() + target = first.sup.target() + assert target.mode == "apply" and target.learn_unknown + await first.sup.stop() + # The frontend restarts and an operator patches its router before any + # planner latched it. + world.frontend.restart(VARIANTS["a"]) + world.frontend.manual_patch(TEMP, 0.7) + (manual,) = world.frontend.routers.values() + second = await leading(world, "p2", advisory=True) + target = second.sup.target() + assert target.mode == "rollback" and not target.learn_unknown + await settle([second], world, rounds=5) + # Never written, and the rollback still completes without it. + assert not [w for w in world.frontend.writes if w[0] == "p2"] + assert manual.config[TEMP] == 0.7 and manual.revision == 1 + assert not world.shared_state().rollback_pending + await second.sup.stop() + + +def test_rollback_targets_learn_unknown_routers_only_when_active() -> None: + world = World() + for advisory, decommission, learn in ( + (False, False, True), + (True, False, False), + (True, True, False), + ): + process = make_process( + world, + settings=settings_for(advisory=advisory, decommission=decommission), + ) + sup = process.sup + sup._initialized = True + sup._snapshot = SimpleNamespace(state=claimed(rollback_pending=True)) + target = sup.target() + assert target.mode == "rollback" and target.include_not_ready + assert target.learn_unknown is learn + # A failed spec premise rolls back an active planner's own deployment. + sup = make_process(world).sup + sup._initialized = True + sup._snapshot = SimpleNamespace(state=claimed()) + sup._spec_ok = False + target = sup.target() + assert target.mode == "rollback" and target.learn_unknown + + +async def test_promote_applies_the_last_advisory_recommendation() -> None: + world = World() + first = await leading(world, advisory=True) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + enqueue(first, make_decision(K0 + 1, "advisory", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.stop() + world.clock.advance(100) + second = await leading(world, "p2", promote=True) + state = world.shared_state() + assert state.mode == "active" and state.by_wt == by_wt("a", chosen) + assert state.history[-1]["outcome"] == "promoted" + assert second.controller.restored == [chosen] + assert frontend_configs(world) == [by_wt("a", chosen)["aggregated"]] + + +async def test_a_state_of_another_dgd_uid_is_recreated() -> None: + world = World() + store = StateStore(world.kube, DgdRef(DGD_NAME, "test-ns", "old-uid", "dyn")) + snapshot = store.read() + store.write( + RetuneState( + dgd_uid="old-uid", + leader_term="0/x/1", + fingerprint="fp-a", + mode="active", + last_window_end_ms=123, + ), + snapshot, + ) + world.kube.create_request( + {"metadata": {"name": "old-req", "labels": {LABEL_DGD_UID: "old-uid"}}} + ) + reconciler = SimpleNamespace( + deleted=[], + protect=lambda *a: None, + release=lambda *a: None, + run_once=lambda **kwargs: None, + ) + reconciler.enqueue_delete = lambda names: reconciler.deleted.extend(names) + process = await leading(world, reconciler=reconciler) + state = world.shared_state() + assert state.dgd_uid == "uid-1" and state.last_window_end_ms == end_ms(K0) + owner = world.kube.leases[STATE_NAME]["metadata"]["ownerReferences"][0] + assert owner["uid"] == "uid-1" + assert reconciler.deleted == ["old-req"] + assert process.sup.initialized + + +# ----------------------------------------------------------- revalidation + + +async def test_spec_flip_rebuilds_rolls_back_and_recovers() -> None: + world = World() + process = await leading(world) + sup, host = process.sup, process.host + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + process.controller.desired = dict(chosen) + enqueue(process, make_decision(K0 + 1, "applied", chosen=chosen)) + await sup.outbox_round() + await sup.control_round() + old_controller = process.controller + # P8 breaks (a second frontend without replica sync). + world.spec.spec_ok = False + await sup.control_round() + assert host.replacements[-1][1:] == (1, K0 + 1) + assert old_controller.closed and sup.generation == 1 + state = world.shared_state() + assert state.rollback_pending and state.by_wt == VARIANTS["a"] + assert sup.target().mode == "rollback" and not sup.ready() + assert frontend_configs(world) == [VARIANTS["a"]["aggregated"]] + assert sup.launch_gate(K0 + 2, window(K0 + 2)).reason == "not_ready" + # Fixed: rebuilt again, back to ready. + world.spec.spec_ok = True + await settle([process], world, rounds=5) + assert sup.generation == 2 and sup.ready() + assert not world.shared_state().rollback_pending + assert sup.launch_gate(K0 + 2, window(K0 + 2)).reason is None + + +async def test_p4_failure_disables_until_it_recovers() -> None: + world = World() + world.spec.p4_ok = False + process = await leading(world) + sup, host = process.sup, process.host + assert sup.retune_inputs() is None and process.controller is None + assert not sup.ready() + world.spec.p4_ok = True + await sup.control_round() + assert host.replacements[-1][0].variant == "a" + assert process.controller is not None and sup.generation == 1 + await sup.control_round() + assert sup.ready() + + +async def test_controller_construction_failure_keeps_not_ready() -> None: + world = World() + process = await leading(world) + process.host.fail_replace = True + world.spec.variant = "b" + world.frontend.restart(VARIANTS["b"]) + await process.sup.control_round() # no exception escapes + assert process.controller is None and not process.sup.ready() + assert process.sup.initialized # still leads, rolls back, persists + + +async def test_p14_parity_failure_is_a_spec_failure() -> None: + world = World() + process = await leading(world) + sup = process.sup + # A frontend restarts with startup values the planner did not parse. + world.frontend.restart(VARIANTS["b"]) + await sup.control_round() + assert sup._p14_ok is False + await sup.control_round() # revalidation sees spec_ok flip + assert not sup.spec_ok and sup.generation == 1 + assert sup.target().mode == "rollback" + world.frontend.restart(VARIANTS["a"]) + await settle([process], world, rounds=3) + assert sup.spec_ok and sup.ready() + + +async def test_namespace_change_swaps_the_actuator_only() -> None: + world = World() + process = await leading(world) + old = process.sup.actuator + world.spec.dynamo_namespace = "dyn-v2" + await process.sup.control_round() + assert process.sup.actuator is not old + assert process.sup.dgd.dynamo_namespace == "dyn-v2" + assert process.sup.generation == 0 and process.host.replacements == [] + + +async def test_health_premises_only_gate_launches() -> None: + world = World() + verdict = {"ok": False} + process = await leading(world, canary=lambda sup: verdict["ok"]) + sup = process.sup + rv = world.kube.leases[STATE_NAME]["metadata"]["resourceVersion"] + assert not sup.ready() + world.frontend.down = True + await sup.control_round() + world.frontend.down = False + verdict["ok"] = True + assert world.kube.leases[STATE_NAME]["metadata"]["resourceVersion"] == rv + await sup.control_round() + assert sup.ready() + + +async def test_an_actuator_round_overtaken_by_a_local_loss_is_discarded() -> None: + world = World() + process = await leading(world) + sup = process.sup + report_before = sup.last_report + rv = world.kube.leases[STATE_NAME]["metadata"]["resourceVersion"] + + class OvertakenActuator(FakeActuator): + async def reconcile(self, target, *, initials, expected_frontends): + sup._epoch += 1 # a local loss ran while this round awaited I/O + return ActuationReport( + converged=False, + new_initials={"stale-router": {CREDIT: 9.0}}, + seen_router_ids=frozenset({"stale-router"}), + parity_ok=False, + ) + + sup._actuator = OvertakenActuator(world.frontend, "p1", False, VARIANTS["a"]) + await sup.control_round() + assert sup.last_report is report_before and sup._p14_ok is not False + assert world.kube.leases[STATE_NAME]["metadata"]["resourceVersion"] == rv + + +async def test_initials_are_persisted_and_pruned() -> None: + world = World() + process = await leading(world) + first_ids = set(world.frontend.routers) + assert set(world.shared_state().initials) == first_ids + world.frontend.restart(VARIANTS["a"]) + await process.sup.control_round() + assert set(world.shared_state().initials) == set(world.frontend.routers) + assert not first_ids & set(world.shared_state().initials) + + +# ------------------------------------------------------------ reconciler + + +class FakeReconciler: + """Records each round's ``should_continue``; a round can hang (a + hard-mounted NFS call) until ``unblock`` is set.""" + + def __init__(self, *, hang: bool = False) -> None: + self.rounds: list[Any] = [] + self.answers_after: list[bool] = [] + self.entered = threading.Event() + self.unblock = threading.Event() + if not hang: + self.unblock.set() + self.events: list[tuple[str, Any]] = [] + + def protect(self, window_end_ms, names) -> None: + pass + + def release(self, window_end_ms) -> None: + pass + + def enqueue_delete(self, names) -> None: + self.events.append(("enqueue_delete", sorted(names))) + + def set_dgd(self, dgd: DgdRef) -> None: + self.events.append(("set_dgd", dgd.uid)) + + def run_once( + self, + *, + last_window_end_ms, + due_window_end_ms, + deadline_for, + frontend_pods, + should_continue=None, + ): + self.rounds.append(should_continue) + self.entered.set() + assert self.unblock.wait(10), "never unblocked" + self.answers_after.append(should_continue()) + + +def reconciler_on_a_thread(reconciler: FakeReconciler): + """Run only the reconciler's round on a worker thread (as live).""" + + async def blocking(call): + if getattr(call, "func", None) == reconciler.run_once: + return await asyncio.to_thread(call) + return call() + + return blocking + + +async def reconciled(sup: LiveRetuneSupervisor) -> None: + task = sup.reconcile_task + if task is not None: + await task + + +async def test_a_hung_reconciler_never_blocks_revalidation_or_actuation() -> None: + world = World() + reconciler = FakeReconciler(hang=True) + process = await leading( + world, + reconciler=reconciler, + blocking=reconciler_on_a_thread(reconciler), + settings=settings_for(reconcile_budget_s=0.05), + ) + sup = process.sup + try: + assert await asyncio.to_thread(reconciler.entered.wait, 5) + # The round runs in its own task; the control loop goes on and never + # starts a second round while the first one's call still runs. + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + process.controller.desired = dict(chosen) + enqueue(process, make_decision(K0 + 1, "applied", chosen=chosen)) + await sup.outbox_round() + await asyncio.wait_for(sup.control_round(), 5) + assert frontend_configs(world) == [by_wt("a", chosen)["aggregated"]] + assert len(reconciler.rounds) == 1 and sup.reconcile_running() + # Past its budget the round's task ends and the reconciler is told + # to stop before its next delete; its call still runs. + await asyncio.wait_for(reconciled(sup), 5) + assert sup.reconcile_running() and reconciler.rounds[0]() is False + # A spec break is still revalidated and rolled back meanwhile. + world.spec.spec_ok = False + await asyncio.wait_for(sup.control_round(), 5) + await asyncio.wait_for(sup.control_round(), 5) + assert not sup.spec_ok and sup.target().mode == "rollback" + assert frontend_configs(world) == [VARIANTS["a"]["aggregated"]] + assert len(reconciler.rounds) == 1 + finally: + reconciler.unblock.set() + work = sup._reconcile_work + assert work is not None + await asyncio.wait_for(work, 5) + assert reconciler.answers_after == [False] + # Once the call returned, the next control round starts a new round. + assert not sup.reconcile_running() + await sup.control_round() + await reconciled(sup) + await asyncio.wait_for(sup._reconcile_work, 5) + assert len(reconciler.rounds) == 2 and reconciler.answers_after[-1] is True + await sup.stop() + + +async def test_reconciler_runs_only_for_the_leader_of_its_epoch() -> None: + world = World() + reconciler = FakeReconciler() + process = await leading(world, reconciler=reconciler) + sup = process.sup + await reconciled(sup) + assert len(reconciler.rounds) == 1 and reconciler.rounds[0]() is True + # Another process took the term; the actuator round finds out while + # persisting new initials: no reconciler round starts after it. + store = StateStore(world.kube, sup.dgd) + snapshot = store.read() + store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) + world.frontend.restart(VARIANTS["a"]) + await sup.control_round() + await reconciled(sup) + assert not sup.initialized + assert len(reconciler.rounds) == 1 and reconciler.rounds[0]() is False + # Re-initialized (a new epoch): the old round's should_continue stays + # False, the new round's is True until leadership lapses. + await sup.lease_round() + await sup.control_round() + await reconciled(sup) + assert len(reconciler.rounds) == 2 + assert reconciler.rounds[0]() is False and reconciler.rounds[1]() is True + world.clock.advance(LEASE) + assert not sup.is_leader() and reconciler.rounds[1]() is False + await sup.control_round() + assert len(reconciler.rounds) == 2 + + +async def test_recreated_dgd_moves_the_reconciler_first() -> None: + world = World() + reconciler = FakeReconciler() + process = await leading(world, reconciler=reconciler) + world.kube.create_request( + {"metadata": {"name": "old-req", "labels": {LABEL_DGD_UID: "uid-1"}}} + ) + world.spec.uid = "uid-2" # deleted and recreated under the same name + await process.sup.control_round() + assert process.sup.dgd.uid == "uid-2" and process.sup.initialized + assert world.shared_state().dgd_uid == "uid-2" + # The reconciler follows the new uid before the old uid's Requests are + # queued for deletion, so the new DGD's Requests stay its own. + assert reconciler.events == [("set_dgd", "uid-2"), ("enqueue_delete", ["old-req"])] + # A namespace-only change keeps the reconciler's identity. + world.spec.dynamo_namespace = "dyn-2" + await process.sup.control_round() + assert reconciler.events[-1] == ("enqueue_delete", ["old-req"]) + + +# ----------------------------------------------------------- sweeps, stop + + +async def test_sweep_fn_binds_the_context_and_never_raises() -> None: + world = World() + seen = {} + + def runner(request, *, cancel, abort, context): + seen.update(request=request, cancel=cancel, abort=abort, context=context) + return "result" + + process = await started( + world, sweep_runner=runner, context_factory=lambda sup: ("ctx", sup.dgd) + ) + cancel, abort = threading.Event(), threading.Event() + request = SimpleNamespace(window_index=5) + assert process.sup.sweep_fn(request, cancel=cancel, abort=abort) == "result" + assert seen["context"] == ("ctx", process.sup.dgd) and seen["abort"] is abort + + def broken(*args, **kwargs): + raise RuntimeError("boom") + + process.sup._sweep_runner = broken + result = process.sup.sweep_fn(request, cancel=cancel, abort=abort) + assert result.status == "failed" and result.failure_reason == "sweep_error" + assert "boom" in result.error + no_context = await started(world, "p2", sweep_runner=runner) + result = no_context.sup.sweep_fn(request, cancel=cancel, abort=abort) + assert result.status == "failed" + + +async def test_stop_aborts_waits_for_the_sweep_and_releases() -> None: + world = World() + entered = threading.Event() + + def runner(request, *, cancel, abort, context): + entered.set() + assert abort.wait(5.0) + return "aborted" + + process = await leading( + world, sweep_runner=runner, context_factory=lambda sup: None + ) + sup = process.sup + gate = sup.launch_gate(K0 + 1, window(K0 + 1)) + thread = threading.Thread( + target=sup.sweep_fn, + args=(SimpleNamespace(window_index=K0 + 1),), + kwargs={"cancel": threading.Event(), "abort": gate.abort}, + ) + thread.start() + assert entered.wait(5.0) + process.sup._sleep = lambda s: asyncio.sleep(0.01) + process.sup._monotonic = time.monotonic + await sup.stop() + thread.join(5.0) + assert gate.abort.is_set() and sup._sweeps_in_flight == 0 + assert world.holder() is None and not sup.is_leader() + await sup.stop() # idempotent + + +async def test_background_loops_run_and_stop() -> None: + world = World() + process = make_process(world) + sup = process.sup + sup._sleep = lambda s: asyncio.sleep(0.001) + await sup.start(process.host) + for _ in range(500): + await asyncio.sleep(0.001) + if sup.initialized and sup.ready(): + break + assert sup.initialized and sup.ready() + enqueue(process, make_decision(K0 + 1, "kept")) + for _ in range(500): + await asyncio.sleep(0.001) + if sup.outbox_size == 0: + break + assert world.shared_state().history[-1]["outcome"] == "kept" + await sup.stop() + assert world.holder() is None + assert process.metrics.state["leader"] is False + + +async def test_hooks_are_the_adapter_hooks() -> None: + world = World() + process = await started(world) + hooks = process.sup.hooks() + from dynamo.planner.plugins.orchestrator.engine_adapter import RetuneHooks + + assert isinstance(hooks, RetuneHooks) + assert hooks.launch_gate == process.sup.launch_gate + assert hooks.harvest_allowed == process.sup.harvest_allowed + assert hooks.sweep_fn == process.sup.sweep_fn and hooks.phase_s == 0.0 + + +async def test_decommission_rolls_back_then_deletes_everything() -> None: + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.desired = dict(chosen) + enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.control_round() + await first.sup.stop() + world.kube.create_request( + {"metadata": {"name": "rt-1", "labels": {LABEL_DGD_UID: "uid-1"}}} + ) + frontend = SimpleNamespace( + parse_frontend=lambda dgd, planner_mode: world.spec.startup() + ) + sup = bootstrap.build_decommission_supervisor( + live_config(), + kube=world.kube, + dgd=world.kube.get_dgd(DGD_NAME), + dynamo_namespace="dyn", + metrics=NullMetrics(), + actuator_factory=lambda dgd, maps: FakeActuator( + world.frontend, "gc", True, maps + ), + frontend=frontend, + clock=world.clock.time, + monotonic=world.clock.monotonic, + sleep=no_sleep, + blocking=inline, + ) + await sup.start(SimpleNamespace(), background=False) + for _ in range(6): + if sup.decommissioned: + break + await sup.lease_round() + await sup.control_round() + world.clock.advance(30) + assert sup.decommissioned + assert frontend_configs(world) == [VARIANTS["a"]["aggregated"]] + assert world.kube.requests == {} and world.kube.leases == {} + assert sup.launch_gate(K0 + 1, window(K0 + 1)).reason == "not_leader" + await sup.stop() + + +# -------------------------------------------------------------- bootstrap + + +def dgd_object(mode: str = "agg", **worker) -> dict: + components = [{"name": "Frontend", "type": "frontend", "replicas": 1}] + if mode == "disagg": + components += [ + {"name": "prefill", "type": "prefill", "replicas": 2}, + {"name": "decode", "type": "decode", **worker}, + ] + else: + components.append({"name": "worker", "type": "worker", **worker}) + return { + "metadata": { + "name": DGD_NAME, + "namespace": "test-ns", + "uid": "uid-1", + "generation": 3, + }, + "spec": {"components": components}, + } + + +def test_worker_spec_replicas() -> None: + assert bootstrap.worker_spec_replicas(dgd_object(replicas=3), "agg") == { + "aggregated": 3 + } + # Omitted replicas are the workload default of 1. + assert bootstrap.worker_spec_replicas(dgd_object(), "agg") == {"aggregated": 1} + assert bootstrap.worker_spec_replicas(dgd_object(replicas=0), "agg") is None + assert bootstrap.worker_spec_replicas( + dgd_object("disagg", replicas=4), "disagg" + ) == {"prefill": 2, "decode": 4} + assert bootstrap.worker_spec_replicas({"spec": {"components": []}}, "agg") is None + + +def test_build_retune_inputs_agg_and_disagg() -> None: + config = live_config() + startup = Spec().startup() + inputs = bootstrap.build_retune_inputs( + config, startup, dgd_name="graph", model_name="m" + ) + assert inputs.trace_paths == ("/trace/segments/graph",) + assert (inputs.trace_origin_ms, inputs.arrival_speedup_ratio) == (0.0, 1.0) + assert inputs.engine.router_mode == "kv_router" and inputs.engine.model_name == "m" + assert not inputs.engine.is_disagg and inputs.engine.extra_engine_args is None + disagg = bootstrap.build_retune_inputs( + config.model_copy(update={"mode": "disagg"}), + startup, + dgd_name="graph", + model_name=None, + ) + assert disagg.engine.is_disagg and disagg.engine.prefill_engine_args == {} + + +def test_kv_router_resolver_matches_the_router_config() -> None: + llm = pytest.importorskip("dynamo.llm") + resolver = bootstrap.KvRouterResolver({}, ("prefill", "decode")) + flat = resolver.startup_flat(KEYS) + default = llm.KvRouterConfig.from_json("{}") + serialized = json.loads(default.to_json()) + resolved = dict(default.policy_parameters("aggregated")) + assert flat == { + key: float(resolved[key] if serialized.get(key) is None else serialized[key]) + for key in KEYS + } + resolved = resolver.by_wt({CREDIT: 0.5}) + config = llm.KvRouterConfig.from_json(json.dumps({CREDIT: 0.5})) + assert resolved == { + wt: {k: float(v) for k, v in dict(config.policy_parameters(wt)).items()} + for wt in ("prefill", "decode") + } + + +class FakeFrontendModule: + def __init__(self, startup: Optional[FrontendStartup], p4_ok=True, spec_ok=True): + self.startup = startup + self.p4_ok = p4_ok + self.spec_ok = spec_ok + self.calls: list[dict] = [] + + def parse_frontend(self, dgd, *, planner_mode): + return self.startup if self.startup is not None else "P4: shell-wrapped" + + def evaluate_premises( + self, dgd, *, planner_config, startup, parse_error, planner_replicas + ): + self.calls.append({"startup": startup, "parse_error": parse_error}) + return PremiseReport( + p4_ok=self.p4_ok and startup is not None, + spec_ok=self.spec_ok and self.p4_ok and startup is not None, + reasons=() if startup is not None else (parse_error,), + ) + + def served_model_name(self, dgd): + return "served" + + +def test_evaluate_spec() -> None: + config = live_config() + frontend = FakeFrontendModule(Spec().startup()) + evaluation = bootstrap.evaluate_spec( + dgd_object(replicas=2), + config=config, + k8s_namespace="ns", + dynamo_namespace="dyn", + frontend=frontend, + ) + assert evaluation.dgd == DgdRef(DGD_NAME, "test-ns", "uid-1", "dyn") + assert evaluation.spec_replicas == {"aggregated": 2} and evaluation.generation == 3 + assert evaluation.inputs.engine.model_name == "served" + assert evaluation.report.p4_ok and evaluation.startup is not None + + unparsed = FakeFrontendModule(None) + evaluation = bootstrap.evaluate_spec( + dgd_object(), + config=config, + k8s_namespace="ns", + dynamo_namespace="dyn", + frontend=unparsed, + ) + assert evaluation.inputs is None and evaluation.startup is None + assert unparsed.calls[0]["parse_error"] == "P4: shell-wrapped" + with pytest.raises(ValueError): + bootstrap.evaluate_spec( + {"metadata": {"name": "x"}}, + config=config, + k8s_namespace="ns", + dynamo_namespace="dyn", + frontend=frontend, + ) + + +async def test_build_supervisor_wires_the_collaborators() -> None: + world = World() + world.kube.put_dgd(dgd_object(replicas=2)) + frontend = FakeFrontendModule(Spec().startup()) + reconciler = SimpleNamespace( + protect=lambda *a: None, + release=lambda *a: None, + enqueue_delete=lambda *a: None, + run_once=lambda **kwargs: None, + ) + sup = await bootstrap.build_supervisor( + live_config(), + kube=world.kube, + dgd_name=DGD_NAME, + namespace_fn=lambda: "dyn", + metrics=NullMetrics(), + identity="pod/1", + frontend=frontend, + actuator_factory=lambda dgd, maps: FakeActuator( + world.frontend, "x", False, maps + ), + reconciler=reconciler, + canary=lambda s: True, + ) + assert sup.identity == "pod/1" and sup.registry is reconciler + assert sup.retune_inputs() is not None + assert sup.evaluation.spec_replicas == {"aggregated": 2} + assert sup._settings.router_keys == KEYS + assert sup._settings.interval_s == 21600 and sup._settings.lease_duration_s == 60 + assert len(sup.fingerprint) == 16 + + +async def test_build_live_retune_requires_the_dgdr_executor() -> None: + host = SimpleNamespace( + config=live_config( + retune={"executor": "local", "dgdr": None, "timeout_seconds": None} + ) + ) + with pytest.raises(ValueError, match="replay-only"): + await bootstrap.build_live_retune(host) + + +async def test_build_live_retune_uses_the_connector(monkeypatch) -> None: + world = World() + world.kube.put_dgd(dgd_object(replicas=2)) + from dynamo.planner.retune.live import kube as kube_module + + monkeypatch.setattr( + kube_module.KubernetesRetuneKube, + "from_kube_api", + classmethod(lambda cls, api: world.kube), + ) + captured = {} + + async def fake_build(config, **kwargs): + captured.update(kwargs, config=config) + return "supervisor" + + monkeypatch.setattr(bootstrap, "build_supervisor", fake_build) + host = SimpleNamespace( + config=live_config(), + environment=SimpleNamespace( + controller=SimpleNamespace(kube_api=object(), parent_dgd_name=DGD_NAME), + runtime_namespace=lambda: "dyn-hash", + ), + ) + assert await bootstrap.build_live_retune(host) == "supervisor" + assert captured["kube"] is world.kube and captured["dgd_name"] == DGD_NAME + assert captured["namespace_fn"]() == "dyn-hash" + + +async def test_build_legacy_cleanup(monkeypatch) -> None: + world = World() + from dynamo.planner.retune.live import kube as kube_module + + monkeypatch.setattr( + kube_module.KubernetesRetuneKube, + "from_kube_api", + classmethod(lambda cls, api: world.kube), + ) + config = live_config().model_copy(update={"retune": None}) + host = SimpleNamespace( + config=config, + environment=SimpleNamespace( + controller=SimpleNamespace(kube_api=object(), parent_dgd_name=DGD_NAME), + ), + ) + assert await bootstrap.build_legacy_cleanup(host) is None # nothing left over + StateStore(world.kube, DgdRef(DGD_NAME, "test-ns", "uid-1", "dyn")).write( + RetuneState(dgd_uid="uid-1"), + StateStore(world.kube, DgdRef(DGD_NAME, "test-ns", "uid-1", "dyn")).read(), + ) + monkeypatch.setattr( + "dynamo.planner.retune.live.frontend_config.parse_frontend", + lambda dgd, planner_mode: "P4: no frontend", + ) + monkeypatch.setattr( + "dynamo.planner.retune.live.metrics.RetuneMetrics", lambda: NullMetrics() + ) + sup = await bootstrap.build_legacy_cleanup(host) + assert sup is not None and sup._settings.decommission + world.kube.fail("get_lease", KubeError("forbidden")) + assert await bootstrap.build_legacy_cleanup(host) is None + host.config = live_config() + assert await bootstrap.build_legacy_cleanup(host) is None + + +class PlannerEnvironment: + """The ``PlannerEnvironment`` surface ``NativePlannerBase`` startup uses.""" + + def __init__(self, controller: Any) -> None: + from dynamo.planner.environment.state import DeploymentState + + self.controller = controller + self.state = DeploymentState() + self.state.decode.num_gpus = 1 + self.calls: list[str] = [] + + def deployment_state(self): + return self.state + + async def initialize(self) -> None: + self.calls.append("environment.initialize") + + async def refresh(self) -> None: + pass + + async def shutdown(self) -> None: + self.calls.append("environment.shutdown") + + +class FakeCleanup: + def __init__(self, calls: list[str], start_error: Optional[Exception] = None): + self.calls = calls + self.start_error = start_error + self.host = None + + async def start(self, host) -> None: + self.calls.append("cleanup.start") + self.host = host + if self.start_error is not None: + raise self.start_error + + async def stop(self) -> None: + self.calls.append("cleanup.stop") + + +def planner_without_retune(monkeypatch, controller: Any, build): + """A ``NativePlannerBase`` without retune config whose legacy clean-up + is built by ``build(host)``; the engine bootstrap is a no-op.""" + + from unittest.mock import patch + + from dynamo.planner.core.base import NativePlannerBase + + config = live_config( + metric_reporting_prometheus_port=0, live_dashboard_port=0, control_api_port=0 + ).model_copy(update={"retune": None}) + environment = PlannerEnvironment(controller) + + async def build_legacy_cleanup(host): + environment.calls.append("cleanup.build") + return await build(host) + + monkeypatch.setattr(bootstrap, "build_legacy_cleanup", build_legacy_cleanup) + with patch("dynamo.planner.core.base.PlannerPrometheusMetrics"): + planner = NativePlannerBase(None, config, environment) + + async def bootstrap_engine(): + environment.calls.append("engine") + + async def nothing(): + pass + + planner._bootstrap_regression = bootstrap_engine # type: ignore[method-assign] + planner._bootstrap_engine_plugins_if_needed = nothing # type: ignore[method-assign] + return planner, environment + + +def kubernetes_connector() -> Any: + from unittest.mock import MagicMock + + from dynamo.planner.connectors.kubernetes import KubernetesConnector + + return MagicMock(spec=KubernetesConnector) + + +async def test_a_planner_without_retune_runs_the_legacy_cleanup(monkeypatch) -> None: + cleanups: list[FakeCleanup] = [] + + async def build(host): + cleanups.append(FakeCleanup(environment.calls)) + return cleanups[-1] + + planner, environment = planner_without_retune( + monkeypatch, kubernetes_connector(), build + ) + await planner._async_init() + try: + # Built before the engine, started after it. + assert environment.calls == [ + "environment.initialize", + "cleanup.build", + "engine", + "cleanup.start", + ] + assert cleanups[0].host is planner and planner._live_retune is None + finally: + await planner._shutdown_runtime() + assert environment.calls[-2:] == ["cleanup.stop", "environment.shutdown"] + assert planner._legacy_retune_cleanup is None + + +@pytest.mark.parametrize( + "outcome", + ["nothing_left", "forbidden", "not_found", "setup_error", "start_error"], +) +async def test_the_legacy_cleanup_never_fails_startup( + monkeypatch, caplog, outcome +) -> None: + from dynamo.planner.retune.live.contracts import KubeNotFound + + async def build(host): + if outcome == "nothing_left": + return None + if outcome == "forbidden": + raise KubeError("403 Forbidden: leases is forbidden") + if outcome == "not_found": + raise KubeNotFound("404 Not Found") + if outcome == "setup_error": + raise RuntimeError("no kube config") + return FakeCleanup(environment.calls, start_error=RuntimeError("boom")) + + planner, environment = planner_without_retune( + monkeypatch, kubernetes_connector(), build + ) + with caplog.at_level("INFO", logger="dynamo.planner.core.base"): + await planner._async_init() + try: + assert "engine" in environment.calls + assert planner._legacy_retune_cleanup is None + assert not [r for r in caplog.records if r.levelno >= logging.WARNING] + if outcome == "start_error": + assert environment.calls[-2:] == ["cleanup.start", "cleanup.stop"] + assert "legacy clean-up did not start" in caplog.text + elif outcome != "nothing_left": + assert "legacy clean-up skipped" in caplog.text + finally: + await planner._shutdown_runtime() + assert environment.calls.count("cleanup.stop") == (outcome == "start_error") + + +async def test_the_legacy_cleanup_is_kubernetes_only(monkeypatch) -> None: + async def build(host): + raise AssertionError("not a Kubernetes planner") + + for controller in (None, SimpleNamespace(kube_api=object())): + planner, environment = planner_without_retune(monkeypatch, controller, build) + await planner._async_init() + try: + assert "cleanup.build" not in environment.calls + assert planner._legacy_retune_cleanup is None + finally: + await planner._shutdown_runtime() + + +async def test_dgdr_canary_and_context() -> None: + world = World() + world.kube.put_dgd(dgd_object(replicas=2)) + config = live_config() + process = await leading(world) + canary = bootstrap.dgdr_canary(config) + assert canary(process.sup) is True + created = world.kube.calls_of("create_request")[-1] + assert created.kwargs["dry_run"] is True and world.kube.requests == {} + world.kube.reject_create = ( + "strict decoding error: unknown field spec.workload.trace" + ) + assert canary(process.sup) is False + world.spec.replicas = None + await process.sup.control_round() + assert canary(process.sup) is False + + context = bootstrap.dgdr_context_factory(config, cut_window="cutter")(process.sup) + assert context.dgd == process.sup.dgd and context.kube is world.kube + assert context.cut_window == "cutter" and context.backend == config.backend + assert context.is_leader() is True + + +# ------------------------------------------- with the real retune controller + + +async def test_real_controller_launches_harvests_and_persists() -> None: + """The supervisor's hooks drive ``RetuneController`` end to end: gate, + launch with the spec replicas and the abort event, sweep_fn, harvest + gating, on_effects, outbox and the persisted deployment.""" + + pytest.importorskip("aisimulate.sweeper") + from dynamo.planner.core.types import EngineCapabilities, WorkerCapabilities + from dynamo.planner.plugins.clock import WallClock + from dynamo.planner.retune.controller import RetuneController + from dynamo.planner.retune.sweep import SweepResult + + config = live_config() + retune = config.retune + interval = float(retune.interval_seconds) + grace = float(retune.grace_seconds) + world = World() + startup = FrontendStartup( + router_mode="kv_router", + router_config={}, + startup_maps={}, + frontend_component="Frontend", + replicas=1, + replica_sync=False, + fingerprint_source="real", + ) + inputs = bootstrap.build_retune_inputs( + config, startup, dgd_name=DGD_NAME, model_name=None + ) + resolver = bootstrap.KvRouterResolver({}, ("aggregated",)) + keys = bootstrap.searched_router_keys(config) + startup = dataclasses.replace( + startup, startup_maps=resolver.by_wt(resolver.startup_flat(keys)) + ) + world.frontend.restart(startup.startup_maps) + evaluation = SpecEvaluation( + report=PremiseReport(p4_ok=True, spec_ok=True), + startup=startup, + inputs=inputs, + spec_replicas={"aggregated": 2}, + dgd=DgdRef(DGD_NAME, "test-ns", "uid-1", "dyn"), + ) + requests = [] + best = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + + def runner(request, *, cancel, abort, context): + requests.append((request, abort)) + return SweepResult( + status="ok", + num_requests=100, + scores={ + candidate.key(): (2.0 if candidate.router == best else 1.0) + for candidate in request.candidates + }, + wall_s=1.0, + details={"runtime_image": "img:1"}, + ) + + settings = SupervisorSettings.from_config(config, router_keys=keys) + sup = LiveRetuneSupervisor( + settings=settings, + kube=world.kube, + evaluation=evaluation, + identity="p1/1", + premise_evaluator=lambda dgd: evaluation, + actuator_factory=lambda dgd, maps: FakeActuator( + world.frontend, "p1", False, maps + ), + fingerprint_fn=lambda s: "fp-real", + resolver_factory=lambda s: resolver, + metrics=RecordingMetrics(), + sweep_runner=runner, + context_factory=lambda supervisor: None, + clock=world.clock.time, + monotonic=world.clock.monotonic, + sleep=no_sleep, + blocking=inline, + ) + hooks = sup.hooks() + controller = RetuneController( + config, + inputs, + WorkerCapabilities( + decode=EngineCapabilities( + num_gpu=1, max_num_batched_tokens=2048, max_kv_tokens=16384 + ) + ), + WallClock(), + float(config.scheduling.scale_interval_seconds), + sweep_fn=hooks.sweep_fn, + launch_gate=hooks.launch_gate, + harvest_allowed=hooks.harvest_allowed, + generation=0, + ) + host = SimpleNamespace( + _engine=SimpleNamespace(retune_controller=controller), + _config_lock=asyncio.Lock(), + ) + try: + await sup.start(host, background=False) + controller.start(world.clock.time(), hooks.phase_s) + await sup.lease_round() + await sup.control_round() + assert sup.ready() and controller.router_keys == keys + + async def tick(now_s): + result = await controller.on_tick(now_s, None) + sup.on_effects( + SimpleNamespace( + retune_decisions=result.decisions, + router_parameters=result.router_parameters, + ) + ) + await sup.outbox_round() + return result + + # First enable recorded the window in progress at start as decided: + # neither it (its trace began mid-way) nor the due window launches. + now = world.clock.time() + first_due = math.floor((now - grace) / interval + 1e-9) - 1 + in_progress = math.floor(now / interval) + assert in_progress == first_due + 1 + await tick(now) + assert requests == [] + world.clock.wall = (in_progress + 1) * interval + grace + await tick(world.clock.wall) + assert requests == [] + # The next window launches at its L_k with the spec replicas. + launch_at = (in_progress + 2) * interval + grace + world.clock.wall = launch_at + await tick(launch_at) + for _ in range(200): + if requests: + break + await asyncio.sleep(0.01) + request, abort = requests[0] + assert request.window_index == in_progress + 1 + assert request.spec_replicas == {"aggregated": 2} and abort is not None + assert request.generation == 0 + # Harvested once done: applied, persisted, then deployed. + decided = None + for step in range(1, 200): + result = await tick(launch_at + step * 0.05 + 1000.0) + if result.decisions: + decided = result.decisions[0] + break + await asyncio.sleep(0.01) + assert decided is not None and decided.outcome == "applied" + assert decided.decision_id == f"retune-w{int(request.window_end_ms)}" + state = world.shared_state() + assert state.flat == best and state.deploy_seq == 2 + assert state.last_window_end_ms == int(request.window_end_ms) + assert state.by_wt == resolver.by_wt(best) + assert state.history[-1]["image"] == {"runtime_image": "img:1"} + await sup.control_round() + assert frontend_configs(world) == [resolver.by_wt(best)["aggregated"]] + # A restart restores the persisted point into a fresh controller. + controller.restore_desired(resolver.startup_flat(keys)) + sup._request_reinit("test") + await sup.lease_round() + assert controller.current_point().router == best + finally: + controller.close() + await sup.stop() + + +# =================================================== interleaving harness +# +# Two planner processes share a FakeRetuneKube, a frontend and a clock. A +# seeded random schedule interleaves lease renewals and failures, takeovers, +# state writes that succeed, conflict, time out, time out but land, or land +# late, ticks that launch and harvest sweeps, frontend restarts and manual +# patches, spec changes (startup, spec_ok, P4), advisory switches, SIGTERM +# and SIGKILL. After every step the §6 invariants are asserted; at the end +# the system settles with no infinite resets or PATCHes. + + +class CheckedKube(FakeRetuneKube): + """Asserts at every successful state write that a term change (a claim) + is made by the leadership Lease holder, and that a write never changes + the term otherwise (an earlier term's write cannot land after a claim).""" + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.state_writes = 0 + self.late = False + # Identity of the process whose call is executing (set by ProcessKube). + self.writer: Optional[str] = None + + def _holder(self) -> Optional[str]: + lease = self.leases.get(LEADER_NAME) + return None if lease is None else lease["spec"].get("holderIdentity") + + def _check(self, before: Optional[dict], after: dict) -> None: + def term(obj: Optional[dict]) -> Optional[str]: + if obj is None: + return None + raw = (obj["metadata"].get("annotations") or {}).get(STATE_ANNOTATION) + return None if raw is None else json.loads(raw).get("leader_term") + + new = term(after) + identity = None if not new else new.split("/", 1)[1].rsplit("/", 1)[0] + if self.writer is not None and new: + # At most one leader writes: a write lands only under its + # writer's own term (an earlier term's writer must fail). + assert identity == self.writer, (identity, self.writer) + if new != term(before): + assert not self.late, "a late write changed the leader_term" + assert identity == self._holder(), (identity, self._holder()) + self.state_writes += 1 + + def replace_lease(self, name, body, *, timeout_s=10.0): + before = copy.deepcopy(self.leases.get(name)) + result = super().replace_lease(name, body, timeout_s=timeout_s) + if name == STATE_NAME: + self._check(before, result) + return result + + def create_lease(self, body, *, timeout_s=10.0): + result = super().create_lease(body, timeout_s=timeout_s) + if result["metadata"]["name"] == STATE_NAME: + self._check(None, result) + return result + + +class ProcessKube: + """One process's view of the shared kube, with its own faults.""" + + def __init__(self, shared: CheckedKube) -> None: + self._shared = shared + self.namespace = shared.namespace + self.identity: Optional[str] = None + self.lease_get_failures = 0 + self.state_faults: deque[str] = deque() + self.deferred: list[tuple[str, dict]] = [] + + def __getattr__(self, attribute: str) -> Any: + return getattr(self._shared, attribute) + + def _as_writer(self, call): + self._shared.writer = self.identity + try: + return call() + finally: + self._shared.writer = None + + def create_lease(self, body, *, timeout_s=10.0): + return self._as_writer( + lambda: self._shared.create_lease(body, timeout_s=timeout_s) + ) + + def get_lease(self, name, *, timeout_s=10.0): + if name == LEADER_NAME and self.lease_get_failures > 0: + self.lease_get_failures -= 1 + raise KubeError("injected renewal failure") + return self._shared.get_lease(name, timeout_s=timeout_s) + + def replace_lease(self, name, body, *, timeout_s=10.0): + if name == STATE_NAME and self.state_faults: + fault = self.state_faults.popleft() + current = self._shared.leases.get(name) + same_term = current is not None and ( + json.loads(current["metadata"]["annotations"][STATE_ANNOTATION]).get( + "leader_term" + ) + == json.loads(body["metadata"]["annotations"][STATE_ANNOTATION]).get( + "leader_term" + ) + ) + if fault == "timeout": + raise KubeError("injected timeout") + if fault == "conflict": + raise KubeConflict("injected conflict") + if fault == "landed": + self._as_writer( + lambda: self._shared.replace_lease(name, body, timeout_s=timeout_s) + ) + raise KubeError("injected timeout (landed)") + if fault == "defer" and same_term: + self.deferred.append((name, copy.deepcopy(body))) + raise KubeError("injected timeout (still in flight)") + return self._as_writer( + lambda: self._shared.replace_lease(name, body, timeout_s=timeout_s) + ) + + def land_deferred(self) -> None: + while self.deferred: + name, body = self.deferred.pop(0) + self._shared.late = True + try: + self._as_writer(lambda: self._shared.replace_lease(name, body)) + except KubeConflict: + pass + finally: + self._shared.late = False + + +class Harness: + def __init__(self, seed: int, *, advisory_only: bool = False) -> None: + self.rng = random.Random(seed) + self.advisory_only = advisory_only + self.world = World(CheckedKube()) + self.world.kube.now = self.world.clock.time + self.processes: dict[str, Process] = {} + self.spawned = 0 + self.steps = 0 + self.transients: dict[str, int] = {} + # Monotonic time until which two processes may both believe they + # lead (the lease was taken over without expiring, §4.7.9). + self.stolen_until = -math.inf + + # -- processes -- + + async def spawn(self, slot: str) -> None: + self.spawned += 1 + kube = ProcessKube(self.world.kube) + advisory = True if self.advisory_only else self.rng.random() < 0.3 + promote = self.rng.random() < 0.5 + process = make_process( + self.world, + f"{slot}-{self.spawned}", + advisory=advisory, + promote=promote, + kube=kube, + ) + kube.identity = process.sup.identity + await process.sup.start(process.host, background=False) + self.processes[slot] = process + + def live(self) -> list[Process]: + return [p for p in self.processes.values() if p.alive] + + # -- events -- + + def _restore_transients(self) -> None: + """Faults of the frontend and the spec last a few steps, then heal.""" + + world, now = self.world, self.steps + for name, attribute, healthy in ( + ("frontend_down", "down", False), + ("spec_broken", "spec_ok", True), + ("p4_broken", "p4_ok", True), + ): + until = self.transients.get(name) + if until is not None and now >= until: + target = world.frontend if attribute == "down" else world.spec + setattr(target, attribute, healthy) + del self.transients[name] + + def _transient(self, name: str, target: Any, attribute: str, broken: Any) -> None: + if name not in self.transients: + setattr(target, attribute, broken) + self.transients[name] = self.steps + self.rng.randint(2, 25) + + async def step(self) -> str: + rng = self.rng + self.steps += 1 + self._restore_transients() + slot = rng.choice(["p1", "p2"]) + process = self.processes.get(slot) + if process is None or not process.alive: + if rng.random() < 0.5: + await self.spawn(slot) + return f"spawn {slot}" + slot = "p2" if slot == "p1" else "p1" + process = self.processes.get(slot) + if process is None or not process.alive: + return "idle" + sup = process.sup + roll = rng.random() + if roll < 0.14: + await sup.lease_round() + return f"lease {slot}" + if roll < 0.28: + await sup.control_round() + return f"control {slot}" + if roll < 0.38: + await sup.outbox_round() + return f"outbox {slot}" + if roll < 0.56: + controller = process.controller + if controller is not None and not controller.closed: + effects = controller.tick( + sup, self.world.clock.time(), rng, process.advisory + ) + sup.on_effects(effects) + return f"tick {slot}" + if roll < 0.64: + controller = process.controller + if controller is not None and controller.pending is not None: + controller.pending["done"] = True + return f"sweep_done {slot}" + if roll < 0.78: + await self.advance(rng.choice([5, 20, 20, 40, 60, 120, 240])) + return "advance" + if roll < 0.82: + kind = rng.choice(["timeout", "conflict", "landed", "defer"]) + process.kube.state_faults.extend([kind] * rng.choice([1, 1, 2, 3])) + return f"fault {kind} {slot}" + if roll < 0.84: + process.kube.land_deferred() + return f"land {slot}" + if roll < 0.855: + process.kube.lease_get_failures += rng.choice([1, 2, 4]) + return f"renew_fail {slot}" + if roll < 0.87: + self.world.frontend.restart(VARIANTS[self.world.spec.variant]) + return "frontend_restart" + if roll < 0.89: + self.world.frontend.manual_patch(TEMP, rng.choice([0.3, 0.7])) + return "manual_patch" + if roll < 0.905: + self._transient("frontend_down", self.world.frontend, "down", True) + return "frontend_down" + if roll < 0.915: + spec = self.world.spec + spec.variant = "b" if spec.variant == "a" else "a" + self.world.frontend.restart(VARIANTS[spec.variant]) + return "startup_change" + if roll < 0.93: + self._transient("spec_broken", self.world.spec, "spec_ok", False) + return "spec_break" + if roll < 0.937: + self._transient("p4_broken", self.world.spec, "p4_ok", False) + return "p4_break" + if roll < 0.945: + controller = process.controller + if controller is not None: + controller.close() + await sup.stop() + process.alive = False + return f"sigterm {slot}" + if roll < 0.951: + process.alive = False + return f"sigkill {slot}" + if roll < 0.957: + self.steal() + return "steal" + return "noop" + + def steal(self) -> None: + """The leadership Lease changes hands without expiring (an admin edit, + or a contender with a skewed clock): to the other live process or + to an intruder that never renews.""" + + lease = self.world.kube.leases.get(LEADER_NAME) + if lease is None: + return + holder = lease["spec"].get("holderIdentity") + others = [p.sup.identity for p in self.live() if p.sup.identity != holder] + lease["spec"]["holderIdentity"] = ( + self.rng.choice(others) + if others and self.rng.random() < 0.7 + else "intruder/0" + ) + lease["spec"]["leaseTransitions"] = lease["spec"].get("leaseTransitions", 0) + 1 + self.world.kube.bump(lease) + self.stolen_until = self.world.clock.monotonic() + LEASE + + async def advance(self, seconds: float) -> None: + """Time passes in chunks of at most 20s while the background loops + mostly keep running (each may miss a chunk).""" + + rng = self.rng + remaining = seconds + while remaining > 0: + chunk = min(20.0, remaining) + remaining -= chunk + self.world.clock.advance(chunk) + self._restore_transients() + for process in self.live(): + sup = process.sup + if rng.random() < 0.85: + await sup.lease_round() + if rng.random() < 0.6: + await sup.outbox_round() + if rng.random() < 0.6: + await sup.control_round() + controller = process.controller + if controller is not None and not controller.closed: + if controller.pending is not None and rng.random() < 0.3: + controller.pending["done"] = True + if rng.random() < 0.6: + sup.on_effects( + controller.tick( + sup, self.world.clock.time(), rng, process.advisory + ) + ) + self.check("advance") + + # -- invariants -- + + def check(self, event: str) -> None: + world = self.world + live = self.live() + leaders = [p.name for p in live if p.sup.is_leader()] + if world.clock.monotonic() >= self.stolen_until: + assert len(leaders) <= 1, (event, leaders) + shared = world.shared_state() + if shared is not None and shared.rollback_pending: + # A pending rollback always records the startup (or nothing). + assert shared.by_wt in (VARIANTS["a"], VARIANTS["b"], {}), ( + event, + shared.by_wt, + ) + for process in live: + sup, controller = process.sup, process.controller + # No window is wasted: a dropped sweep or decision leaves its + # window launchable. + if controller is not None: + decided = ( + -1 + if shared is None or shared.last_window_end_ms is None + else window_index( + shared.last_window_end_ms, phase_s=0.0, interval_s=INTERVAL + ) + ) + for k in controller.dropped: + assert ( + controller.last_k < k + or controller.pending is not None + or k <= decided + ), (event, process.name, k, controller.last_k) + controller.dropped.clear() + # An advisory planner never PATCHes, and resets only to finish a + # rollback of what was deployed. + for writer, advisory, kind, mode, _ in world.frontend.writes: + if advisory: + assert kind == "reset" and mode == "rollback", (event, writer, kind) + if not (sup.is_leader() and sup.initialized and sup.outbox_size == 0): + continue + snapshot = sup._snapshot + if snapshot is None or shared is None: + continue + lease = world.kube.leases[STATE_NAME] + if snapshot.resource_version != lease["metadata"]["resourceVersion"]: + continue # a late write landed; the next write re-reads + assert shared.leader_term == sup.term, event + # State, controller baseline and target agree. + if ( + shared.mode == "active" + and not shared.rollback_pending + and sup.spec_ok + and controller is not None + ): + assert controller.desired == shared.flat, ( + event, + controller.desired, + shared.flat, + ) + target = sup.target() + if shared.rollback_pending: + assert target.mode == "rollback", event + elif process.advisory: + assert target.mode == "observe", event + elif not sup.spec_ok: + assert target.mode == "rollback", event + else: + assert target.desired_by_wt == shared.by_wt, event + + async def settle(self) -> None: + world = self.world + world.frontend.down = False + world.spec.spec_ok = True + world.spec.p4_ok = True + self.transients.clear() + for process in self.live(): + process.kube.state_faults.clear() + process.kube.lease_get_failures = 0 + process.kube.deferred.clear() + if not self.live(): + await self.spawn("p1") + live = self.live() + for _ in range(12): + world.clock.advance(20) + for process in live: + await process.sup.lease_round() + await process.sup.outbox_round() + await process.sup.control_round() + self.check("settle") + writes = len(world.frontend.writes) + for _ in range(4): + world.clock.advance(20) + for process in live: + await process.sup.lease_round() + await process.sup.outbox_round() + await process.sup.control_round() + self.check("settle") + # No infinite reset or PATCH. + assert len(world.frontend.writes) == writes, world.frontend.writes[writes:] + leader = next((p for p in live if p.sup.is_leader()), None) + assert leader is not None and leader.sup.initialized + state = world.shared_state() + # Two-phase rollbacks complete once converged. + assert not state.rollback_pending + target = leader.sup.target() + configs = frontend_configs(world) + if target.mode == "apply": + assert configs == [state.by_wt["aggregated"]] + elif target.mode == "rollback": + assert configs == [VARIANTS[world.spec.variant]["aggregated"]] + + async def run(self, steps: int) -> None: + await self.spawn("p1") + await self.spawn("p2") + for _ in range(steps): + event = await self.step() + self.check(event) + await self.settle() + + +@pytest.mark.parametrize("seed", range(24)) +async def test_interleavings_keep_the_invariants(seed: int) -> None: + harness = Harness(seed) + await harness.run(steps=300) + assert harness.world.kube.state_writes > 0 + + +@pytest.mark.parametrize("seed", range(6)) +async def test_advisory_only_never_writes_to_routers(seed: int) -> None: + harness = Harness(1000 + seed, advisory_only=True) + await harness.run(steps=250) + assert harness.world.frontend.writes == [] diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py new file mode 100644 index 000000000000..0dfe1ee34c76 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -0,0 +1,1253 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the live retune window cutter. + +The golden tests compare the cut window, decompressed, byte for byte with +``read_trace_window`` over the same clean segments. A few tests spawn the +real child process; they carry a timeout. +""" + +import errno +import gzip +import io +import json +import os +import pickle +import random +import stat +import threading +import time + +import pytest + +from dynamo.planner.retune.live import window_cut +from dynamo.planner.retune.live.contracts import WindowCutResult +from dynamo.planner.retune.live.window_cut import ( + MAX_DROPPED_MEMBER_RATIO, + WINDOW_CUT_FAILURES, + WindowCutError, + _decode_exact_member, + _Garbage, + _GzipMemberReader, + _Member, + _run_in_child, + _segment_files, + cut_window, + cut_window_inprocess, + segment_index, + temp_file_name, + temp_file_window_end, +) +from dynamo.planner.retune.window import ROW_COUNT_KEYS, read_trace_window + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +SCHEMA = "dynamo.request.trace.v1" +START, END, AS_OF = 5_000, 15_000, 15_500 +_UNSET = object() + + +# ------------------------------------------------------------------ helpers + + +def _row( + request_id, + received_ms=_UNSET, + *, + total_ms=10.0, + event_ms=None, + isl=128, + osl=8, + block_size=64, + model="m", + agent_context=None, + output_tokens=_UNSET, +): + request = {"request_id": request_id} + if model is not None: + request["model"] = model + if received_ms is not _UNSET and received_ms is not None: + request["request_received_ms"] = received_ms + if total_ms is not None: + request["total_time_ms"] = total_ms + if output_tokens is not None: + request["output_tokens"] = osl if output_tokens is _UNSET else output_tokens + request["replay"] = { + "trace_block_size": block_size, + "input_length": isl, + "input_sequence_hashes": [len(request_id), 7, 11_400_714_819_323_198_485], + } + if event_ms is None: + base = received_ms if isinstance(received_ms, int) else 0 + event_ms = base + (round(total_ms) if total_ms is not None else 0) + record = { + "schema": SCHEMA, + "event_type": "request_end", + "event_time_unix_ms": event_ms, + "request": request, + } + if agent_context is not None: + record["agent_context"] = agent_context + return record + + +def _envelope(record, timestamp=0): + # The frontend's gzip sink wraps every record like this. + return {"timestamp": timestamp, "event": record} + + +def _lines(records): + return b"".join(json.dumps(record).encode("utf-8") + b"\n" for record in records) + + +def _member(records): + return gzip.compress(_lines(records)) + + +def _write(path, *chunks): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"".join(chunks)) + return path + + +def _out(tmp_path, name="w.jsonl.gz"): + return tmp_path / "windows" / "dgd" / name + + +def _cut(root, tmp_path, start=START, end=END, as_of=AS_OF, *, max_rows=10**6): + out = _out(tmp_path) + return cut_window_inprocess(root, start, end, as_of, out, max_rows=max_rows), out + + +def _golden(root, tmp_path, start=START, end=END, as_of=AS_OF): + out = tmp_path / "golden" / "window.jsonl" + window = read_trace_window([str(root)], start, end, as_of, out) + return window, out.read_bytes() + + +def _assert_golden(result, out, window, golden_bytes): + assert isinstance(result, WindowCutResult) + assert result.num_requests == window.num_requests + assert result.trace_block_size == window.trace_block_size + assert result.row_counts == window.row_counts + assert set(result.row_counts) == set(ROW_COUNT_KEYS) + if window.num_requests: + assert result.path == str(out) + assert gzip.decompress(out.read_bytes()) == golden_bytes + else: + assert result.path is None + assert not out.exists() + + +def _temp_files(directory): + if not directory.exists(): + return [] + return [name for name in os.listdir(directory) if name.startswith(".tmp-")] + + +def _random_trace(root, seed, *, pods=3, segments=3, members=5, plain_pod=False): + """Segments with every row shape ``read_trace_window`` handles. + + Arrivals and durations are coarse so (arrival, completion) pairs tie + across files; some request IDs repeat (first in file order wins), some + are not ASCII. + """ + + rng = random.Random(seed) + issued = [] + serial = 0 + for pod in range(pods): + for segment in range(segments): + chunks = [] + for _ in range(members): + records = [] + for _ in range(rng.randint(0, 14)): + roll = rng.random() + if roll < 0.04: + records.append( + { + "schema": SCHEMA, + "event_type": rng.choice(["tool_start", "tool_end"]), + "event_time_unix_ms": rng.randrange(20_000), + "agent_context": {"session_id": "s"}, + } + ) + continue + if issued and roll < 0.16: + request_id = rng.choice(issued) + else: + serial += 1 + prefix = rng.choice(["req-", "req-", "réq-", "請求-", "Req-"]) + request_id = f"{prefix}{serial % 97}-{serial}" + issued.append(request_id) + received = rng.randrange(0, 400) * 50 + total = rng.choice([10.0, 10.5, 49.5, 99.4, 600.0, 1_200.25]) + kwargs = {} + if rng.random() < 0.08: + kwargs["received_ms"] = None + kwargs["event_ms"] = received + round(total) + else: + kwargs["received_ms"] = received + if rng.random() < 0.05: + kwargs["total_ms"] = None + kwargs["event_ms"] = received + rng.randrange(0, 900) + else: + kwargs["total_ms"] = total + if rng.random() < 0.03: + kwargs["output_tokens"] = None + if rng.random() < 0.05: + kwargs["agent_context"] = {"session_id": f"s{serial}"} + if rng.random() < 0.1: + kwargs["model"] = None + received_arg = kwargs.pop("received_ms") + record = _row( + request_id, + received_arg, + isl=rng.randrange(1, 4096), + osl=rng.randrange(1, 512), + **kwargs, + ) + if rng.random() < 0.5: + record = _envelope(record, timestamp=rng.randrange(10**6)) + records.append(record) + payload = _lines(records) + if rng.random() < 0.1: + payload += b"\n \n" # blank lines are skipped + chunks.append(payload) + directory = root / f"frontend-{pod}" + if plain_pod and pod == 0: + _write(directory / f"req.{segment:06d}.jsonl", *chunks) + else: + _write( + directory / f"req.{segment:06d}.jsonl.gz", + *(gzip.compress(chunk) for chunk in chunks), + ) + return root + + +def _items(data): + return list(_GzipMemberReader(io.BytesIO(data))) + + +# ----------------------------------------------------------- gzip members + + +@pytest.fixture(params=["default", "tiny", "small"]) +def reader_chunks(request, monkeypatch): + """Run reader tests with reads and decode steps that split headers.""" + + sizes = {"default": None, "tiny": (7, 5, 16), "small": (64, 3, 128)}[request.param] + if sizes is not None: + read_chunk, decode_step, compact_at = sizes + monkeypatch.setattr(window_cut, "_READ_CHUNK", read_chunk) + monkeypatch.setattr(window_cut, "_DECODE_STEP", decode_step) + monkeypatch.setattr(window_cut, "_COMPACT_AT", compact_at) + return request.param + + +def _spans(*members): + offsets = [0] + for member in members: + offsets.append(offsets[-1] + len(member)) + return offsets + + +def test_clean_multi_member_file(reader_chunks): + payloads = [b'{"a":1}\n', b"", b'{"b":2}\n{"c":3}\n', b"x" * 5000 + b"\n"] + members = [gzip.compress(payload) for payload in payloads] + offsets = _spans(*members) + + items = _items(b"".join(members)) + + assert items == [ + _Member(offsets[i], offsets[i + 1], payloads[i]) for i in range(len(payloads)) + ] + + +@pytest.mark.parametrize("keep", ["header", "middle", "trailer"]) +def test_truncated_last_member_is_dropped(reader_chunks, keep): + first, last = gzip.compress(b"a\n" * 50), gzip.compress(b"b\n" * 50) + cut = {"header": 5, "middle": len(last) // 2, "trailer": len(last) - 3}[keep] + + items = _items(first + last[:cut]) + + assert items == [ + _Member(0, len(first), b"a\n" * 50), + _Garbage(len(first), len(first) + cut, at_eof=True), + ] + + +def test_partial_member_followed_by_complete_members(reader_chunks): + first = gzip.compress(b"first\n" * 40) + full = gzip.compress(b"".join(b"partial %d\n" % i for i in range(400))) + partial = full[: len(full) // 2] + third = gzip.compress(b"third\n") + fourth = gzip.compress(b"fourth\n") + offsets = _spans(first, partial, third, fourth) + + items = _items(first + partial + third + fourth) + + assert items == [ + _Member(offsets[0], offsets[1], b"first\n" * 40), + _Garbage(offsets[1], offsets[2], at_eof=False), + _Member(offsets[2], offsets[3], b"third\n"), + _Member(offsets[3], offsets[4], b"fourth\n"), + ] + + +@pytest.mark.parametrize("where", ["crc", "isize", "data"]) +def test_flipped_member_is_dropped_and_reader_resyncs(reader_chunks, where): + first = gzip.compress(b"first\n") + middle = bytearray(gzip.compress(b"middle row\n" * 30)) + index = {"crc": len(middle) - 8, "isize": len(middle) - 2, "data": 14}[where] + middle[index] ^= 0x40 + last = gzip.compress(b"last\n") + offsets = _spans(first, middle, last) + + items = _items(first + bytes(middle) + last) + + assert items == [ + _Member(offsets[0], offsets[1], b"first\n"), + _Garbage(offsets[1], offsets[2], at_eof=False), + _Member(offsets[2], offsets[3], b"last\n"), + ] + + +def test_zero_filled_hole_between_members(reader_chunks): + first, last = gzip.compress(b"first\n"), gzip.compress(b"last\n") + hole = bytes(4096) + + items = _items(first + hole + last) + + assert items == [ + _Member(0, len(first), b"first\n"), + _Garbage(len(first), len(first) + len(hole), at_eof=False), + _Member(len(first) + len(hole), len(first) + len(hole) + len(last), b"last\n"), + ] + + +def test_false_gzip_magic_inside_garbage_is_one_run(reader_chunks): + first, last = gzip.compress(b"first\n"), gzip.compress(b"last\n") + junk = b"\x1f\x8b\x08\x00junk" * 20 + b"\x1f\x8b" + + items = _items(first + junk + last) + + assert items == [ + _Member(0, len(first), b"first\n"), + _Garbage(len(first), len(first) + len(junk), at_eof=False), + _Member(len(first) + len(junk), len(first) + len(junk) + len(last), b"last\n"), + ] + + +def test_garbage_only_and_empty_files(reader_chunks): + assert _items(b"") == [] + assert _items(b"plain text, not gzip\n") == [_Garbage(0, 21, at_eof=True)] + member = gzip.compress(b"a\n") + assert _items(member + bytes(10)) == [ + _Member(0, len(member), b"a\n"), + _Garbage(len(member), len(member) + 10, at_eof=True), + ] + + +def test_reader_starts_at_an_offset(): + first, second = gzip.compress(b"first\n"), gzip.compress(b"second\n") + source = io.BytesIO(first + second) + + items = list(_GzipMemberReader(source, len(first))) + + assert items == [_Member(len(first), len(first) + len(second), b"second\n")] + + +def test_decode_exact_member(): + member = gzip.compress(b"row\n") + assert _decode_exact_member(member) == b"row\n" + assert _decode_exact_member(member + b"x") is None + assert _decode_exact_member(member[:-1]) is None + flipped = bytearray(member) + flipped[-6] ^= 1 + assert _decode_exact_member(bytes(flipped)) is None + assert _decode_exact_member(b"") is None + + +# ------------------------------------------------------------ golden cuts + + +@pytest.mark.parametrize("seed", [0, 1, 2]) +def test_cut_matches_read_trace_window(tmp_path, seed): + root = _random_trace(tmp_path / "segments" / "dgd", seed) + window, golden_bytes = _golden(root, tmp_path) + assert window.num_requests > 50 + assert window.row_counts["duplicate"] > 0 + assert window.row_counts["censored"] > 0 + + result, out = _cut(root, tmp_path) + + _assert_golden(result, out, window, golden_bytes) + assert result.dropped_members == 0 + assert result.dropped_bytes == 0 + assert not result.reused + assert stat.S_IMODE(out.stat().st_mode) == 0o644 + assert _temp_files(out.parent) == [] + + +def test_cut_matches_golden_with_plain_jsonl_files(tmp_path): + root = _random_trace(tmp_path / "segments" / "dgd", 3, plain_pod=True) + window, golden_bytes = _golden(root, tmp_path) + + result, out = _cut(root, tmp_path) + + _assert_golden(result, out, window, golden_bytes) + + +def test_cut_matches_golden_under_eviction_and_small_reads(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments" / "dgd", 4, members=8) + window, golden_bytes = _golden(root, tmp_path) + monkeypatch.setattr(window_cut, "_PASS2_CACHE_BYTES", 1) + monkeypatch.setattr(window_cut, "_PASS2_BLOCK_ROWS", 7) + monkeypatch.setattr(window_cut, "_READ_CHUNK", 97) + monkeypatch.setattr(window_cut, "_DECODE_STEP", 13) + monkeypatch.setattr(window_cut, "_COMPACT_AT", 64) + monkeypatch.setattr(window_cut, "_WRITE_BUFFER", 10) + monkeypatch.setattr(window_cut, "_OPEN_FILES", 2) + loads = [] + original_load = window_cut._MemberLines._load + + def counting_load(self, member_id): + loads.append(member_id) + return original_load(self, member_id) + + monkeypatch.setattr(window_cut._MemberLines, "_load", counting_load) + + result, out = _cut(root, tmp_path) + + _assert_golden(result, out, window, golden_bytes) + # Every member was evicted after each row and decoded again. + assert len(loads) == window.num_requests + assert len(set(loads)) < len(loads) + + +def test_dedup_compaction_keeps_first_rows_and_bounds_rows(tmp_path, monkeypatch): + rows = [_row(f"r{i}", START + 10 * i) for i in range(10)] + # Many repeats of the same IDs, in later files: the first row wins. + repeats = [_row(f"r{i % 10}", START + 5 + i, isl=999) for i in range(40)] + root = tmp_path / "segments" + _write(root / "a" / "req.000000.jsonl.gz", _member(rows)) + _write(root / "b" / "req.000000.jsonl.gz", *(_member([row]) for row in repeats)) + window, golden_bytes = _golden(root, tmp_path) + assert window.num_requests == 10 + monkeypatch.setattr(window_cut, "_DEDUP_SLACK_MIN", 1) + monkeypatch.setattr(window_cut, "_DEDUP_SLACK_RATIO", 0.0) + compactions = [] + original_dedup = window_cut._Selection._dedup + + def counting_dedup(self): + compactions.append(self.num_rows) + original_dedup(self) + + monkeypatch.setattr(window_cut._Selection, "_dedup", counting_dedup) + + result, out = _cut(root, tmp_path, max_rows=10) + + _assert_golden(result, out, window, golden_bytes) + assert result.row_counts["duplicate"] == 40 + assert len(compactions) > 2 # during pass 1, not only at the end + + +def test_too_many_rows_is_window_too_large(tmp_path, monkeypatch): + root = tmp_path / "segments" + rows = [_row(f"r{i}", START + i) for i in range(30)] + _write(root / "a" / "req.000000.jsonl.gz", _member(rows)) + monkeypatch.setattr(window_cut, "_DEDUP_SLACK_MIN", 1) + monkeypatch.setattr(window_cut, "_DEDUP_SLACK_RATIO", 0.0) + + result, _ = _cut(root, tmp_path, max_rows=30) + assert result.num_requests == 30 + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path, max_rows=29) + assert caught.value.reason == "window_too_large" + + +def test_empty_window_writes_no_file(tmp_path): + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("early", START - 100), _row("late", END + 1)]), + _member([_row("no-output", START + 1, output_tokens=None)]), + ) + window, golden_bytes = _golden(root, tmp_path) + + result, out = _cut(root, tmp_path) + + assert golden_bytes == b"" + _assert_golden(result, out, window, golden_bytes) + assert result.row_counts["missing_output_tokens"] == 1 + assert not out.parent.exists() + + +def test_empty_segments_root_is_an_empty_window(tmp_path): + root = tmp_path / "segments" + (root / "frontend-0").mkdir(parents=True) + + result, out = _cut(root, tmp_path) + + assert result.path is None + assert result.num_requests == 0 + assert result.trace_block_size is None + + +def test_missing_segments_root_fails(tmp_path): + with pytest.raises(WindowCutError) as caught: + _cut(tmp_path / "nowhere", tmp_path) + assert caught.value.reason == "window_cut_failed" + assert "does not exist" in str(caught.value) + + +def test_mixed_block_sizes_fail(tmp_path): + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("a", START, block_size=64), _row("b", START, block_size=16)]), + ) + with pytest.raises(ValueError, match="mixes trace_block_size"): + _golden(root, tmp_path) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "window_cut_failed" + assert "mixes trace_block_size values [16, 64]" in str(caught.value) + + +@pytest.mark.parametrize( + "line, message", + [ + (b"{not json", "invalid JSON"), + (b"[1, 2]", "must be a JSON object"), + (b'{"schema": "other", "event_type": "request_end"}', "expected"), + ( + json.dumps( + { + "schema": SCHEMA, + "event_type": "request_end", + "event_time_unix_ms": 1, + "request": { + "request_id": "x", + "output_tokens": 1, + "replay": {"input_length": 1, "trace_block_size": 64}, + }, + } + ).encode(), + "input_sequence_hashes", + ), + ], +) +def test_malformed_row_in_a_valid_member_is_trace_corrupt(tmp_path, line, message): + # The member's CRC is good: the writer wrote this, as replay would see it. + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("good", START)]), + gzip.compress(line + b"\n"), + ) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "trace_corrupt" + assert message in str(caught.value) + + +def test_out_of_range_value_is_trace_corrupt(tmp_path): + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("huge", START, block_size=2**70)]), + ) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "trace_corrupt" + + +# --------------------------------------------------- dropped gzip members + + +def _tiny_members(prefix, count, first_ms=START): + return [ + _member([_envelope(_row(f"{prefix}{i}", first_ms + i), timestamp=i)]) + for i in range(count) + ] + + +def _cut_and_twin(tmp_path, build): + """Cut ``build(bad=True)`` and the golden of ``build(bad=False)``. + + ``build(root, bad)`` writes the segments with or without the bad bytes; + the rows of a dropped member never reach the window. + """ + + clean = tmp_path / "clean" + build(clean, bad=False) + window, golden_bytes = _golden(clean, tmp_path) + damaged = tmp_path / "damaged" + build(damaged, bad=True) + result, out = _cut(damaged, tmp_path) + _assert_golden(result, out, window, golden_bytes) + return result + + +def test_in_flight_tail_of_the_newest_segment_is_not_corruption(tmp_path): + lost = _member([_row("in-flight", START + 900)]) + + def build(root, bad): + _write(root / "a" / "req.000000.jsonl.gz", *_tiny_members("old", 3)) + members = _tiny_members("new", 2, START + 100) + _write( + root / "a" / "req.000001.jsonl.gz", + *members, + *([lost[: len(lost) // 2]] if bad else []), + ) + + result = _cut_and_twin(tmp_path, build) + + # 1 of 6 members is far above the ratio, but it is the writer's tail. + assert result.dropped_members == 1 + assert result.dropped_bytes == len(lost) // 2 + + +def test_truncated_tail_of_an_older_segment_is_corruption(tmp_path): + root = tmp_path / "segments" + lost = _member([_row("lost", START + 900)]) + _write(root / "a" / "req.000000.jsonl.gz", *_tiny_members("old", 3), lost[:20]) + _write(root / "a" / "req.000001.jsonl.gz", *_tiny_members("new", 3)) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "trace_corrupt" + assert "1 of 7 gzip members" in str(caught.value) + assert "req.000000.jsonl.gz@" in str(caught.value) + + +@pytest.mark.parametrize("damage", ["partial", "crc", "hole"]) +def test_damaged_members_within_the_ratio_are_dropped(tmp_path, damage): + # 1 bad member among 100 is exactly the limit: the cut goes on. Its bytes + # are at most one member's, below the limit too. + count = int(1 / MAX_DROPPED_MEMBER_RATIO) - 1 + bad_row = _member([_row("bad", START + 50)]) + if damage == "partial": + damaged = bad_row[: len(bad_row) - 12] + elif damage == "crc": + flipped = bytearray(bad_row) + flipped[-7] ^= 0x10 + damaged = bytes(flipped) + else: + damaged = bytes(len(bad_row)) + assert len(damaged) <= min(map(len, _tiny_members("m", count))) + + def build(root, bad): + members = _tiny_members("m", count) + if bad: + members.insert(count // 2, damaged) + _write(root / "a" / "req.000000.jsonl.gz", *members) + + result = _cut_and_twin(tmp_path, build) + + assert result.dropped_members == 1 + assert result.dropped_bytes == len(damaged) + + +def test_damaged_members_above_the_ratio_are_trace_corrupt(tmp_path): + root = tmp_path / "segments" + members = _tiny_members("m", 98) + members.insert(10, bytes(64)) + members.insert(50, bytes(64)) + _write(root / "a" / "req.000000.jsonl.gz", *members) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "trace_corrupt" + assert "2 of 100 gzip members" in str(caught.value) + + +@pytest.mark.parametrize("newest", [True, False]) +def test_hole_spanning_many_members_is_trace_corrupt(tmp_path, newest): + # A zero-filled hole over 20 of 200 members is one run of bad bytes: 1 of + # 181 members is within the ratio, but 10% of the bytes is not. + root = tmp_path / "segments" + members = _tiny_members("m", 200) + hole = sum(map(len, members[50:70])) + data = b"".join([*members[:50], bytes(hole), *members[70:]]) + _write(root / "a" / "req.000000.jsonl.gz", data) + if not newest: + _write(root / "a" / "req.000001.jsonl.gz", *_tiny_members("next", 3)) + total_members = 181 + (0 if newest else 3) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + + assert caught.value.reason == "trace_corrupt" + message = str(caught.value) + assert f"1 of {total_members} gzip members" in message + assert f"({hole} of " in message + assert f"req.000000.jsonl.gz@{sum(map(len, members[:50]))}+{hole}" in message + + +def test_hole_within_the_byte_ratio_is_dropped(tmp_path): + # A hole over 1 of 200 members: 1 run, under 1% of the bytes. + members = _tiny_members("m", 200) + + def build(root, bad): + kept = [*members[:50], *members[51:]] + if bad: + kept.insert(50, bytes(len(members[50]))) + _write(root / "a" / "req.000000.jsonl.gz", *kept) + + result = _cut_and_twin(tmp_path, build) + + assert result.dropped_members == 1 + assert result.dropped_bytes == len(members[50]) + + +@pytest.mark.parametrize("plain_block", [None, 64]) +def test_plain_jsonl_unterminated_tail(tmp_path, monkeypatch, plain_block): + if plain_block is not None: + # Blocks of one line each: the tail is in a later block. + monkeypatch.setattr(window_cut, "_PLAIN_BLOCK", plain_block) + good = [_row("a", START), _row("b", START + 1)] + + def build(root, bad): + _write( + root / "a" / "req.000000.jsonl", + _lines(good), + *([b'{"schema": "dynamo.req'] if bad else []), + ) + + result = _cut_and_twin(tmp_path, build) + assert result.dropped_bytes == len(b'{"schema": "dynamo.req') + assert result.dropped_members == 0 + + # An unterminated last line that parses is a row, as replay reads it. + root = tmp_path / "parses" + _write( + root / "a" / "req.000000.jsonl", + _lines(good) + json.dumps(_row("c", START + 2)).encode(), + ) + window, golden_bytes = _golden(root, tmp_path) + assert window.num_requests == 3 + out = _out(tmp_path, "parses.jsonl.gz") + result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10) + _assert_golden(result, out, window, golden_bytes) + + +@pytest.mark.parametrize("tail", ["partial", "parses"]) +def test_plain_jsonl_stops_at_the_unterminated_tail(tmp_path, tail): + # The writer finishes the last line (and writes another) right after the + # first read: reading on would start inside that line. + path = tmp_path / "req.000000.jsonl" + first = _lines([_row("a", START)]) + if tail == "partial": + line = _lines([_row("b", START + 1)]) + written, rest = line[:20], line[20:] + else: + written, rest = json.dumps(_row("b", START + 1)).encode(), b"\n" + _write(path, first, written) + + class Appending: + def __init__(self, source): + self.source = source + self.reads = 0 + + def readlines(self, hint): + lines = self.source.readlines(hint) + self.reads += 1 + if self.reads == 1: + with open(path, "ab") as writer: + writer.write(rest + _lines([_row("c", START + 2)])) + return lines + + with open(path, "rb") as source: + reader = Appending(source) + blocks = list(window_cut._plain_blocks(reader)) + + assert reader.reads == 1 + assert len(blocks) == 1 + if tail == "partial": + assert blocks[0].payload == first + assert blocks[0].dropped == len(written) + else: + assert blocks[0].payload == first + written + assert blocks[0].dropped == 0 + assert blocks[0].end == len(blocks[0].payload) + selection = window_cut._Selection(START, END, AS_OF, 10) + selection.scan((0, str(path), True, block) for block in blocks) + assert selection.num_rows == (1 if tail == "partial" else 2) + + +def test_segment_files_order_and_newest(tmp_path): + root = tmp_path / "segments" + for name in ( + "b/req.000999.jsonl.gz", + "b/req.1000000.jsonl.gz", + "a/req.000000.jsonl.gz", + "a/req.000001.jsonl", + "a/notes.txt", + "c/single.jsonl.gz", + ): + _write(root / name, b"") + + files = _segment_files(str(root)) + + assert files == [ + (str(root / "a/req.000000.jsonl.gz"), False), + (str(root / "a/req.000001.jsonl"), True), + (str(root / "b/req.000999.jsonl.gz"), False), + (str(root / "b/req.1000000.jsonl.gz"), True), + (str(root / "c/single.jsonl.gz"), True), + ] + + +def test_compressed_and_plain_copies_of_one_segment_fail(tmp_path): + root = tmp_path / "segments" + _write(root / "a" / "req.000000.jsonl.gz", _member([_row("a", START)])) + _write(root / "a" / "req.000000.jsonl", _lines([_row("a", START)])) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "window_cut_failed" + + +def test_vanished_file_is_skipped(tmp_path, monkeypatch): + root = tmp_path / "segments" + _write(root / "a" / "req.000000.jsonl.gz", _member([_row("a", START)])) + listed = _segment_files(str(root)) + gone = str(root / "a" / "req.000001.jsonl.gz") + monkeypatch.setattr(window_cut, "_segment_files", lambda _: [*listed, (gone, True)]) + + result, _ = _cut(root, tmp_path) + + assert result.num_requests == 1 + + +def _two_pods(root): + for pod in ("a", "b"): + _write( + root / pod / "req.000000.jsonl.gz", + _member([_row(f"{pod}-{i}", START + i) for i in range(5)]), + ) + return root + + +def _skip_as_root(): + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root ignores directory permissions") + + +@pytest.mark.parametrize("mode", [0o000, 0o600]) +def test_unreadable_pod_directory_fails(tmp_path, mode): + # 0o000 cannot be listed; 0o600 can, but its files cannot be stat-ed. + # Either way that pod's rows would be missing from the window. + _skip_as_root() + root = _two_pods(tmp_path / "segments") + pod = root / "b" + pod.chmod(mode) + try: + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + finally: + pod.chmod(0o755) + + assert caught.value.reason == "window_cut_failed" + assert str(pod) in str(caught.value) + assert "PermissionError" in str(caught.value) + + +def test_unreadable_segments_root_fails(tmp_path): + # Not an empty window: the cut cannot see any segment. + _skip_as_root() + root = _two_pods(tmp_path / "segments") + root.chmod(0o000) + try: + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + finally: + root.chmod(0o755) + + assert caught.value.reason == "window_cut_failed" + assert str(root) in str(caught.value) + + +@pytest.mark.parametrize("where", ["pod-list", "file-stat", "root-list", "root-stat"]) +@pytest.mark.parametrize( + "code", [errno.ENOENT, errno.ESTALE, errno.EIO, errno.ETIMEDOUT, errno.EACCES] +) +def test_segment_walk_skips_only_vanished_entries(tmp_path, monkeypatch, where, code): + root = _two_pods(tmp_path / "segments") + target = { + "pod-list": root / "b", + "file-stat": root / "b" / "req.000000.jsonl.gz", + "root-list": root, + "root-stat": root, + }[where] + name = "scandir" if where.endswith("list") else "stat" + real = getattr(os, name) + + def failing(path, *args, **kwargs): + if os.fspath(path) == str(target): + raise OSError(code, os.strerror(code), os.fspath(path)) + return real(path, *args, **kwargs) + + monkeypatch.setattr(os, name, failing) + vanished = code in (errno.ENOENT, errno.ESTALE) + + if vanished and not where.startswith("root"): + # The reconciler deleted pod b's segments under the walk. + assert _segment_files(str(root)) == [ + (str(root / "a" / "req.000000.jsonl.gz"), True) + ] + return + with pytest.raises(WindowCutError) as caught: + _segment_files(str(root)) + assert caught.value.reason == "window_cut_failed" + if vanished and where == "root-stat": + assert "does not exist" in str(caught.value) + else: + assert str(target) in str(caught.value) + assert os.strerror(code) in str(caught.value) + + +# -------------------------------------------------------- atomic output + + +def test_existing_window_file_is_reused(tmp_path): + root = _random_trace(tmp_path / "segments", 5) + window, _ = _golden(root, tmp_path) + out = _out(tmp_path) + _write(out, b"published by another process") + + result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + + assert result.reused + assert result.path == str(out) + assert result.num_requests == window.num_requests + assert out.read_bytes() == b"published by another process" + assert _temp_files(out.parent) == [] + + +def test_link_race_reuses_the_winner(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments", 6) + out = _out(tmp_path) + real_link = os.link + linked = [] + + def racing_link(source, target): + linked.append(os.path.basename(source)) + _write(out, b"winner") + return real_link(source, target) + + monkeypatch.setattr(os, "link", racing_link) + + result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + + assert result.reused + assert out.read_bytes() == b"winner" + assert temp_file_window_end(linked[0]) == END + assert _temp_files(out.parent) == [] + + +def test_link_enoent_is_window_cut_failed(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments", 7) + out = _out(tmp_path) + + def vanished_link(source, target): + raise FileNotFoundError(2, "No such file or directory", source) + + monkeypatch.setattr(os, "link", vanished_link) + + with pytest.raises(WindowCutError) as caught: + cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + assert caught.value.reason == "window_cut_failed" + assert not out.exists() + assert _temp_files(out.parent) == [] + + +def test_output_is_deterministic_and_0644_under_any_umask(tmp_path): + root = _random_trace(tmp_path / "segments", 8) + first, second = _out(tmp_path, "a.jsonl.gz"), _out(tmp_path, "b.jsonl.gz") + previous = os.umask(0o077) + try: + cut_window_inprocess(root, START, END, AS_OF, first, max_rows=10**6) + cut_window_inprocess(root, START, END, AS_OF, second, max_rows=10**6) + finally: + os.umask(previous) + + assert first.read_bytes() == second.read_bytes() + assert stat.S_IMODE(first.stat().st_mode) == 0o644 + with gzip.open(first, "rb") as source: + assert source.read(1) + # No file name or mtime in the gzip header. + header = first.read_bytes()[:10] + assert header[3] & 0x08 == 0 + assert header[4:8] == b"\x00\x00\x00\x00" + + +@pytest.mark.parametrize("change", ["rewritten", "truncated", "deleted"]) +def test_segment_changed_between_passes_fails(tmp_path, monkeypatch, change): + root = tmp_path / "segments" + segment = _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("a", START), _row("b", START + 1)]), + _member([_row("c", START + 2)]), + ) + original_write_rows = window_cut._write_rows + + def mutate_then_write(selection, paths, sink): + if change == "rewritten": + segment.write_bytes( + _member([_row("x", START), _row("y", START + 1)]) + + _member([_row("z", START + 2)]) + ) + elif change == "truncated": + segment.write_bytes(segment.read_bytes()[:40]) + else: + segment.unlink() + return original_write_rows(selection, paths, sink) + + monkeypatch.setattr(window_cut, "_write_rows", mutate_then_write) + out = _out(tmp_path) + + with pytest.raises(WindowCutError) as caught: + cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10) + assert caught.value.reason == "window_cut_failed" + assert not out.exists() + assert _temp_files(out.parent) == [] + + +def test_line_writer_writes_in_order_and_reraises_sink_errors(monkeypatch): + monkeypatch.setattr(window_cut, "_WRITE_BUFFER", 8) + sink = io.BytesIO() + writer = window_cut._LineWriter(sink) + for index in range(100): + writer.write(f"row {index}") + writer.close() + assert sink.getvalue() == b"".join(b"row %d\n" % i for i in range(100)) + + class FullDisk(io.BytesIO): + def write(self, data): + raise OSError(28, "No space left on device") + + writer = window_cut._LineWriter(FullDisk()) + with pytest.raises(OSError, match="No space left"): + for index in range(100): + writer.write(f"row {index}") + writer.close() + writer.abort() # idempotent after a failure + + aborted = io.BytesIO() + writer = window_cut._LineWriter(aborted) + writer.write("x" * 100) + writer.abort() + writer.abort() + assert writer._thread.is_alive() is False + + +def test_write_failure_is_window_cut_failed_and_leaves_no_files(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments", 10) + out = _out(tmp_path) + + def failing_write(selection, paths, sink): + raise OSError(28, "No space left on device") + + monkeypatch.setattr(window_cut, "_write_rows", failing_write) + + with pytest.raises(WindowCutError) as caught: + cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + assert caught.value.reason == "window_cut_failed" + assert "No space left" in str(caught.value) + assert not out.exists() + assert _temp_files(out.parent) == [] + + +# ------------------------------------------------- arguments and helpers + + +@pytest.mark.parametrize( + "start, end, as_of, out, max_rows", + [ + (10, 10, 20, "w.jsonl.gz", 1), + (10, 20, 19, "w.jsonl.gz", 1), + (10, 20, 20, "w.jsonl", 1), + (10, 20, 20, "w.jsonl.gz", 0), + (10, 20, 20, "w.jsonl.gz", True), + (float("nan"), 20, 20, "w.jsonl.gz", 1), + ("10", 20, 20, "w.jsonl.gz", 1), + ], +) +def test_invalid_arguments_raise_value_error( + tmp_path, start, end, as_of, out, max_rows +): + with pytest.raises(ValueError): + cut_window_inprocess( + tmp_path, start, end, as_of, tmp_path / out, max_rows=max_rows + ) + with pytest.raises(ValueError): + cut_window( + tmp_path, + start, + end, + as_of, + tmp_path / out, + max_rows=max_rows, + deadline_s=time.time() + 60, + ) + + +def test_temp_path_must_be_next_to_out_path(tmp_path): + with pytest.raises(ValueError, match="not next to"): + cut_window_inprocess( + tmp_path, + START, + END, + AS_OF, + tmp_path / "a" / "w.jsonl.gz", + max_rows=1, + temp_path=tmp_path / "b" / temp_file_name(END), + ) + + +def test_temp_and_segment_names(): + name = temp_file_name(1_700_000_000_000.0, "ab12") + assert name == ".tmp-1700000000000-ab12" + assert temp_file_window_end(name) == 1_700_000_000_000 + assert temp_file_window_end(temp_file_name(5)) == 5 + assert temp_file_name(5) != temp_file_name(5) + assert temp_file_window_end("1700000000000.jsonl.gz") is None + assert temp_file_window_end(".tmp-12-") is None + with pytest.raises(ValueError): + temp_file_name(5, "../x") + assert segment_index("req.000042.jsonl.gz") == 42 + assert segment_index("req.1000000.jsonl") == 1_000_000 + assert segment_index("req.jsonl.gz") is None + + +def test_window_cut_error_reason_and_pickle(): + error = WindowCutError("trace_corrupt", "bad member") + copy = pickle.loads(pickle.dumps(error)) + assert (copy.reason, str(copy)) == ("trace_corrupt", "bad member") + assert set(WINDOW_CUT_FAILURES) == { + "window_cut_timeout", + "window_too_large", + "trace_corrupt", + "window_cut_failed", + "cancelled", + } + with pytest.raises(ValueError): + WindowCutError("nope", "x") + + +# ------------------------------------------------------- child process + + +def test_interrupted_before_spawn_does_not_start_a_child(tmp_path, monkeypatch): + def no_spawn(*_): + raise AssertionError("a child was started") + + monkeypatch.setattr(window_cut.multiprocessing, "get_context", no_spawn) + out = _out(tmp_path) + + with pytest.raises(WindowCutError) as caught: + cut_window( + tmp_path, START, END, AS_OF, out, max_rows=1, deadline_s=time.time() - 1 + ) + assert caught.value.reason == "window_cut_timeout" + + cancel = threading.Event() + cancel.set() + with pytest.raises(WindowCutError) as caught: + cut_window( + tmp_path, + START, + END, + AS_OF, + out, + max_rows=1, + deadline_s=time.time() + 60, + cancel=cancel, + ) + assert caught.value.reason == "cancelled" + + +@pytest.mark.timeout(120) +def test_cut_window_in_child_matches_golden(tmp_path): + root = _random_trace(tmp_path / "segments" / "dgd", 9) + window, golden_bytes = _golden(root, tmp_path) + out = _out(tmp_path) + + result = cut_window( + root, START, END, AS_OF, out, max_rows=10**6, deadline_s=time.time() + 100 + ) + + _assert_golden(result, out, window, golden_bytes) + assert _temp_files(out.parent) == [] + + +@pytest.mark.timeout(120) +def test_cut_window_in_child_reports_the_failure_reason(tmp_path): + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("a", START), _row("b", START + 1)]), + ) + + with pytest.raises(WindowCutError) as caught: + cut_window( + root, + START, + END, + AS_OF, + _out(tmp_path), + max_rows=1, + deadline_s=time.time() + 100, + ) + assert caught.value.reason == "window_too_large" + assert "more than 1 requests" in str(caught.value) + + +@pytest.mark.timeout(120) +def test_child_that_dies_is_window_cut_failed(): + with pytest.raises(WindowCutError) as caught: + _run_in_child(os._exit, (3,), {}, deadline=time.monotonic() + 100, cancel=None) + assert caught.value.reason == "window_cut_failed" + assert "exited with code 3" in str(caught.value) + + +@pytest.mark.timeout(60) +@pytest.mark.parametrize("interrupt", ["cancel", "deadline"]) +def test_interrupt_kills_child_and_removes_its_temp_file(tmp_path, interrupt): + temp = tmp_path / temp_file_name(END) + temp.write_bytes(b"partial window") + cancel = threading.Event() + timer = threading.Timer(0.5, cancel.set) + deadline = time.monotonic() + (100 if interrupt == "cancel" else 0.5) + started = time.monotonic() + timer.start() + try: + with pytest.raises(WindowCutError) as caught: + _run_in_child( + time.sleep, + (600,), + {}, + deadline=deadline, + cancel=cancel if interrupt == "cancel" else None, + cleanup=(str(temp),), + ) + finally: + timer.cancel() + timer.join() + expected = "cancelled" if interrupt == "cancel" else "window_cut_timeout" + assert caught.value.reason == expected + assert time.monotonic() - started < 20 + assert not temp.exists() diff --git a/components/src/dynamo/replay/simulation.py b/components/src/dynamo/replay/simulation.py index 2002c172a161..c3f06c0959a4 100644 --- a/components/src/dynamo/replay/simulation.py +++ b/components/src/dynamo/replay/simulation.py @@ -395,6 +395,13 @@ def _run_trace( # embedded value instead of imposing the synthetic-workload default. if trace_format not in {"weka", "dynamo"} and trace_block_size is None: trace_block_size = self.trace_block_size + # Common random numbers: every candidate of a trace sweep replays the + # same trace with ordinal request ids and the canonical router seed, + # so candidates differ only by their configuration and a score does + # not depend on the other candidates, the Run or the parallelism. + # Live planner retune compares candidates scored by different DGDR + # Runs of one window on that basis. The synthetic path has no such + # option. if deployment.deployment_mode == "agg": return run_trace_replay( trace_files=trace_files, @@ -409,6 +416,7 @@ def _run_trace( execution_model=execution_model, extra_engine_args=self._engine_args(deployment, "aggregated"), num_workers=deployment.num_workers, + _deterministic=True, **common, ) return run_trace_replay( @@ -426,6 +434,7 @@ def _run_trace( decode_engine_args=self._engine_args(deployment, "decode"), num_prefill_workers=deployment.num_prefill_workers, num_decode_workers=deployment.num_decode_workers, + _deterministic=True, **common, ) diff --git a/components/src/dynamo/replay/tests/test_simulation_crn.py b/components/src/dynamo/replay/tests/test_simulation_crn.py new file mode 100644 index 000000000000..ea45aeabdf05 --- /dev/null +++ b/components/src/dynamo/replay/tests/test_simulation_crn.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the simulation imports. + +"""Common random numbers in the Sweeper replay runner. + +Trace replays (what a DGDR trace Run evaluates) are deterministic, so live +planner retune can compare candidates scored by different Runs of one +window; synthetic replays have no such option and must not receive it. +""" + +from __future__ import annotations + +import inspect + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from aisimulate.sweeper.replay import BackendDeploymentSpec, ReplaySpec + +from dynamo.replay import ReplayReport +from dynamo.replay import api as replay_api +from dynamo.replay import simulation + +pytestmark = [ + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.planner, +] + +_REPORT = ReplayReport( + summary={"completed_requests": 1, "output_throughput_tok_s": 1.0}, + per_request=None, + coverage={}, + planner=None, +) + + +def _deployment(mode: str) -> BackendDeploymentSpec: + if mode == "agg": + return BackendDeploymentSpec( + deployment_mode="agg", + backend="vllm", + backend_version="0.11.0", + agg_engine_args={"timing_model": {"type": "polynomial"}}, + num_workers=2, + ) + return BackendDeploymentSpec( + deployment_mode="disagg", + backend="vllm", + backend_version="0.11.0", + prefill_engine_args={ + "worker_type": "prefill", + "timing_model": {"type": "polynomial"}, + }, + decode_engine_args={ + "worker_type": "decode", + "timing_model": {"type": "polynomial"}, + }, + num_prefill_workers=1, + num_decode_workers=2, + ) + + +def _strict_fake(real, seen: dict): + """A replay entry point that binds its kwargs to ``real``'s signature.""" + + signature = inspect.signature(real) + + def fake(*args, **kwargs): + signature.bind(*args, **kwargs) + seen.update(kwargs) + return _REPORT + + return fake + + +@pytest.mark.parametrize("mode", ["agg", "disagg"]) +@pytest.mark.parametrize( + "workload", + [ + {"trace_path": "window.jsonl.gz", "trace_format": "dynamo"}, + {"trace_paths": ["a.jsonl", "b.jsonl"], "trace_format": "mooncake"}, + ], +) +def test_trace_replays_are_deterministic(monkeypatch, mode, workload): + seen: dict = {} + monkeypatch.setattr( + simulation, + "run_trace_replay", + _strict_fake(replay_api.run_trace_replay, seen), + ) + spec = ReplaySpec( + backend_deployment=_deployment(mode), + workload=workload, + goal={"target": "throughput"}, + ) + + simulation.DynamoReplayRunnerFactory().create(0).run(spec) + + assert seen["_deterministic"] is True + + +@pytest.mark.parametrize("mode", ["agg", "disagg"]) +def test_synthetic_replays_do_not_receive_the_flag(monkeypatch, mode): + # The synthetic entry point has no such parameter; the strict fake fails + # if the flag leaks into the shared kwargs. + assert ( + "_deterministic" + not in inspect.signature(replay_api.run_synthetic_trace_replay).parameters + ) + seen: dict = {} + monkeypatch.setattr( + simulation, + "run_synthetic_trace_replay", + _strict_fake(replay_api.run_synthetic_trace_replay, seen), + ) + spec = ReplaySpec( + backend_deployment=_deployment(mode), + workload={ + "trace_path": None, + "isl": 512, + "osl": 128, + "num_request_ratio": 10.0, + "concurrency": None, + "request_rate": 20.0, + }, + goal={"target": "throughput"}, + ) + + simulation.DynamoReplayRunnerFactory().create(0).run(spec) + + assert seen # the synthetic path ran + assert "_deterministic" not in seen diff --git a/deploy/helm/charts/platform/components/operator/templates/planner.yaml b/deploy/helm/charts/platform/components/operator/templates/planner.yaml index 2c9ef2bb73d0..b6d69b7620fc 100644 --- a/deploy/helm/charts/platform/components/operator/templates/planner.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/planner.yaml @@ -57,6 +57,21 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["list"] +{{- if .Values.dgdrV1Beta2.enabled }} +# EXPERIMENTAL live retune (planner retune.executor: dgdr): sweep through DGDR +# v1beta2 Requests, read their Runs and Candidates, and hold the leadership +# and state Leases (read, create, replace and delete; never patch). Lease names +# are per DGD, so the Lease rule covers every Lease in the namespace. +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentrequests"] + verbs: ["create", "get", "list", "watch", "delete"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentruns", "dynamographdeploymentcandidates"] + verbs: ["get", "list", "watch"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update", "delete"] +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -78,6 +93,9 @@ roleRef: # Even though this is a ClusterRole, the operator binds it via a namespace-scoped # RoleBinding (internal/rbac/manager.go EnsureServiceAccountWithRBAC), so all # permissions — including pods/list — are effective only in the DGD's namespace. +# RBAC escalation prevention lets the operator bind only permissions it holds, +# so every rule here must also be in the operator's role (with dgdrV1Beta2, the +# DGDR v1beta2 Run and Candidate rules of the operator's v1beta2 integration). --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -107,4 +125,19 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["list"] +{{- if .Values.dgdrV1Beta2.enabled }} +# EXPERIMENTAL live retune (planner retune.executor: dgdr): sweep through DGDR +# v1beta2 Requests, read their Runs and Candidates, and hold the leadership +# and state Leases (read, create, replace and delete; never patch). Lease names +# are per DGD, so the Lease rule covers every Lease in the namespace. +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentrequests"] + verbs: ["create", "get", "list", "watch", "delete"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentruns", "dynamographdeploymentcandidates"] + verbs: ["get", "list", "watch"] +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update", "delete"] +{{- end }} {{- end }} diff --git a/deploy/helm/charts/platform/tests/planner_rbac_test.yaml b/deploy/helm/charts/platform/tests/planner_rbac_test.yaml index 389d2332c571..833743ebc58e 100644 --- a/deploy/helm/charts/platform/tests/planner_rbac_test.yaml +++ b/deploy/helm/charts/platform/tests/planner_rbac_test.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -suite: planner RBAC — Pod lifecycle observation +suite: planner RBAC — Pod lifecycle observation and live retune templates: - charts/dynamo-operator/templates/planner.yaml @@ -114,3 +114,186 @@ tests: - get - patch documentIndex: 1 + + # ── EXPERIMENTAL live retune (dgdrV1Beta2.enabled) ──────────────────────────── + + - it: omits live retune rules from the ClusterRole when dgdrV1Beta2 is disabled (default) + asserts: + - notContains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentrequests + verbs: + - create + - get + - list + - watch + - delete + documentIndex: 0 + - notContains: + path: rules + content: + apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - delete + documentIndex: 0 + + - it: grants DGDR v1beta2 and Lease access in the ClusterRole when dgdrV1Beta2 is enabled + set: + dynamo-operator.dgdrV1Beta2.enabled: true + asserts: + - contains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentrequests + verbs: + - create + - get + - list + - watch + - delete + documentIndex: 0 + - contains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentruns + - dynamographdeploymentcandidates + verbs: + - get + - list + - watch + documentIndex: 0 + - contains: + path: rules + content: + apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - delete + documentIndex: 0 + - contains: + path: rules + content: + apiGroups: + - "" + resources: + - pods + verbs: + - list + documentIndex: 0 + # The planner reads, creates, replaces and deletes Leases; it never patches them. + - notContains: + path: rules + content: + apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - patch + - delete + documentIndex: 0 + + - it: omits live retune rules from the Role when dgdrV1Beta2 is disabled in namespace-restricted mode + set: + dynamo-operator.namespaceRestriction.enabled: true + dynamo-operator.upgradeCRD: false + asserts: + - notContains: + path: rules + content: + apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - delete + documentIndex: 1 + + - it: grants DGDR v1beta2 and Lease access in the Role when dgdrV1Beta2 is enabled in namespace-restricted mode + set: + dynamo-operator.namespaceRestriction.enabled: true + dynamo-operator.upgradeCRD: false + dynamo-operator.dgdrV1Beta2.enabled: true + asserts: + - contains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentrequests + verbs: + - create + - get + - list + - watch + - delete + documentIndex: 1 + - contains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentruns + - dynamographdeploymentcandidates + verbs: + - get + - list + - watch + documentIndex: 1 + - contains: + path: rules + content: + apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - delete + documentIndex: 1 + # The planner reads, creates, replaces and deletes Leases; it never patches them. + - notContains: + path: rules + content: + apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - patch + - delete + documentIndex: 1 diff --git a/docs/fern/index.yml b/docs/fern/index.yml index 113c6cc803bf..bdfeba8d2e44 100644 --- a/docs/fern/index.yml +++ b/docs/fern/index.yml @@ -224,6 +224,8 @@ navigation: path: pages/kubernetes/auto-deployment/dynamo-profiler.md - page: Dynamo Planner path: pages/kubernetes/auto-deployment/dynamo-planner.mdx + - page: Planner Live Retune + path: pages/kubernetes/auto-deployment/planner-live-retune.md - section: Operations icon: screwdriver-wrench diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx index c0d92b5b97af..ed528edd6aaa 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx @@ -471,16 +471,18 @@ searches from the point it applied. To predict what an active Planner does, repl ## Simulate Retune (Experimental) -**Experimental.** Retune runs only in offline simulation in this release. A live Planner rejects a -configuration that sets `retune` at startup. +**Experimental.** This section covers replay retune, `retune.executor: local` (the default), which +runs only in offline simulation; a live Planner rejects it at startup. To retune a live deployment +on Kubernetes, which uses `executor: dgdr` and is for development clusters only, see +[Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). Retune makes the Planner retune its router and Planner runtime parameters from the traffic of the last window. For the concept and the two scenarios, see [Periodic Retune](planner-guide.md#periodic-retune-experimental). For windows, scoring, and decision -timing, see [Planner Design](planner-design.md#retune-experimental-simulation-only). For every +timing, see [Planner Design](planner-design.md#retune-experimental). For every `retune` field, its default, and the search-domain syntax, see the -[`retune` reference](../../../../reference/components/planner-configuration.mdx#retune-experimental-simulation-only). +[`retune` reference](../../../../reference/components/planner-configuration.mdx#retune-experimental). ### Requirements @@ -507,7 +509,7 @@ With `dynamo.replay.run_trace_replay`: `agent_context` (an agentic Dynamo trace). Retune windows drop tool rows. `execution_model` is accepted, because it applies only to agentic requests. - `router_mode="kv_router"` and a router configuration that meets the - [`retune.router` requirements](../../../../reference/components/planner-configuration.mdx#retune-experimental-simulation-only) + [`retune.router` requirements](../../../../reference/components/planner-configuration.mdx#retune-experimental) when `retune.router` is set. - With throughput scaling, retune reads `load_predictor_warmup_trace` as recorded history. It must then be a Dynamo request trace that ends at or before the first replayed request, on the same @@ -712,7 +714,7 @@ decision lands. Choose the value by what the run is for: - A fixed number of seconds gives the same decision ticks on every run. Use it to compare configurations, and set it to the sweep time you expect on the host that will run sweeps. - `measured` charges the sweep's own wall-clock time on the replay host, as the - [`decision_latency_seconds` field](../../../../reference/components/planner-configuration.mdx#retune-experimental-simulation-only) + [`decision_latency_seconds` field](../../../../reference/components/planner-configuration.mdx#retune-experimental) defines it. Decision ticks then depend on the speed and load of the replay host, so two runs of the same config can switch at different ticks and report different results. Set `decision_latency_scale` above 1 to model a slower sweep host, or below 1 for a faster one. Above @@ -911,7 +913,9 @@ Scenario 2 at about 4.5 GB with `max_parallel: 8`. ### Limitations -- **Simulation only.** A live Planner rejects `retune` at startup. +- **Simulation only.** A live Planner rejects `executor: local` at startup. Live retune uses + `executor: dgdr`; see + [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). - **Open-loop Dynamo request traces only.** See [Requirements](#requirements). - **Cold window replays.** Each candidate replay starts with an empty KV cache and the replica counts at the window start, so prefix-cache reuse from before the window does not carry over. diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md index 62674dfe6ef8..96d84fce49a1 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-design.md @@ -289,18 +289,26 @@ When both modes are enabled, throughput-based scaling (longer interval) sets a l In aggregated mode (`mode: agg`), engines handle both prefill and decode via chunked prefill. The planner maintains both TTFT and ITL regression models but uses per-worker time-averaged metrics (not instantaneous) for regression training to smooth out chunked prefill noise. Scale up if either prefill or decode signals overload; scale down only if both signal underload. -## Retune (Experimental, Simulation Only) + + +## Retune (Experimental) > [!WARNING] -> **Experimental.** Retune runs only in planner-in-the-loop offline replay in this release. A live -> Planner rejects a configuration that sets `retune` at startup. +> **Experimental.** With `retune.executor: local`, the default, retune runs only in +> planner-in-the-loop offline replay, and a live Planner rejects it at startup. With +> `executor: dgdr`, a live Planner on Kubernetes retunes three KV router knobs with fixed replicas +> through DGDR v1beta2 Requests; it is for development clusters only. See +> [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). Retune periodically replays the last window of recorded traffic against a grid of router and Planner parameters and applies the best point. The [Planner Guide](planner-guide.md#periodic-retune-experimental) covers the concept, the -[`retune` reference](../../../../reference/components/planner-configuration.mdx#retune-experimental-simulation-only) -the fields, and [Simulate Retune](aisimulate-planner-replay.mdx#simulate-retune-experimental) how to -run it. This section describes the mechanics. +[`retune` reference](../../../../reference/components/planner-configuration.mdx#retune-experimental) +the fields, [Simulate Retune](aisimulate-planner-replay.mdx#simulate-retune-experimental) how to +run it in replay, and +[Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md) +how to run it on a live deployment. This section describes the mechanics of replay retune, +executor `local`. ### Engine Integration @@ -326,10 +334,12 @@ Retune lives in the Planner engine (`OrchestratorEngineAdapter`), not in a plugi type, and `PlannerEffects.retune_decisions`, which carry the `planner_changes` for the host to apply. Router parameters are level-triggered desired state: the host applies the difference. - In retune-only mode, `retune` set with both scaling flags explicitly `false`, the adapter returns - no `scale_to`. The [`retune` field](../../../../reference/components/planner-configuration.mdx#retune-experimental-simulation-only) + no `scale_to`. The [`retune` field](../../../../reference/components/planner-configuration.mdx#retune-experimental) lists what else that mode turns off. -- The live runtime base, `NativePlannerBase`, raises `planner.retune is replay-only in this release` - at construction. +- The live runtime base, `NativePlannerBase`, raises + `planner.retune is replay-only with executor 'local'; use executor: dgdr` at construction when + `retune.executor` is `local`. With `dgdr`, it starts the live retune supervisor of + [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md) instead. ### Windows @@ -483,7 +493,7 @@ With `D` = `decision_latency_seconds` and `Δ` = `scheduling.scale_interval_seco launched at tick time `t` is ready at `t + max(D, Δ)`, and the decision lands on the first tick at or after that time at which the sweep is done, never in the launch tick. With `measured`, `D` is the modeled sweep time that the -[`decision_latency_seconds` field](../../../../reference/components/planner-configuration.mdx#retune-experimental-simulation-only) +[`decision_latency_seconds` field](../../../../reference/components/planner-configuration.mdx#retune-experimental) defines. Under a `VirtualClock`, the replay host, the controller waits for the sweep inside the launch tick, @@ -547,15 +557,14 @@ request trace, and its error names the AISimulate setting. 1. **Adjustment interval vs scaling latency**: If a plugin interval is shorter than the time to scale, later ticks may observe an in-progress transition and hold rather than stacking new replica changes. 2. **Average-based prediction**: Throughput-based scaling uses average ISL/OSL, which may not represent bimodal or heavy-tailed distributions well. 3. **Local Planner scope**: Each local Planner instance directly manages one DGD. Use GlobalPlanner for shared GPU budgets and coordination across multiple DGDs. -4. **Retune is experimental and simulation-only**: See the retune [limitations](aisimulate-planner-replay.mdx#limitations). +4. **Retune is experimental**: executor `local` runs only in simulation; see the retune [limitations](aisimulate-planner-replay.mdx#limitations). Executor `dgdr` runs on a live Planner with the limitations in [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md#limitations). ## Future Work - Distribution-aware interpolation (beyond mean ISL/OSL) - Adaptive adjustment interval based on observed scaling latency -- Retune on a live Planner: reading the frontend request traces as they are written, patching - `/router_policy` on the frontends, and restoring the previous point when a deployment rejects an - applied change +- Live retune of Planner parameters, with load or throughput scaling, and of the router knobs + that a DGDR v1beta2 Request cannot express ## File Map @@ -576,6 +585,7 @@ request trace, and its error names the AISimulate setting. | `components/src/dynamo/planner/retune/sweep.py` | Sweep worker processes, candidate replays, scoring, and the load-predictor pre-sweep. | | `components/src/dynamo/planner/retune/window.py` | Request-trace window and history reader for sweeps. | | `components/src/dynamo/planner/retune/domains.py` | Expansion of `retune.router` and `retune.planner` into the candidate grid. | +| `components/src/dynamo/planner/retune/live/` | Live retune (`executor: dgdr`): deployment rules, window cuts, DGDR v1beta2 sweeps, leadership and state Leases, router actuation, and clean-up. | | `components/src/dynamo/planner/offline/replay_adapter.py` | Replay host: tick records, scaling events, router parameter updates, and retune actuation. | | `components/src/dynamo/planner/simulation/` | AISimulate Planner adapter: the `planner` section of `aisimulate predict` and `recommend`. | | `components/src/dynamo/router/simulation/config.py` | AISimulate router section schemas, including `retune.router`. | diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md index b52f84272098..3ac7b5e8eb8d 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md @@ -69,7 +69,7 @@ spec: advisory: true ``` -Advisory mode is suggestion-only. The Planner computes recommended replica counts, logs them, exports them as diagnostics, and shows them in HTML reports. The recommendations are not applied as scaling decisions: the Planner does not execute scaling actions or change the deployment. Planner-in-the-loop replay honors it too: the replay keeps the simulated deployment fixed, reports the recommendations, and records [retune](#periodic-retune-experimental) decisions, which run only in simulation, without applying them; see [Replay in Advisory Mode](aisimulate-planner-replay.mdx#replay-in-advisory-mode). +Advisory mode is suggestion-only. The Planner computes recommended replica counts, logs them, exports them as diagnostics, and shows them in HTML reports. The recommendations are not applied as scaling decisions: the Planner does not execute scaling actions or change the deployment. Planner-in-the-loop replay honors it too: the replay keeps the simulated deployment fixed, reports the recommendations, and records [retune](#periodic-retune-experimental) decisions without applying them; see [Replay in Advisory Mode](aisimulate-planner-replay.mdx#replay-in-advisory-mode). ### Optimization Target @@ -328,9 +328,12 @@ spec: ## Periodic Retune (Experimental) > [!WARNING] -> **Experimental.** In this release, retune runs only in offline simulation: -> `aisimulate predict --stack dynamo` and `dynamo.replay.run_trace_replay`. A live Planner rejects a -> configuration that sets `retune` at startup. +> **Experimental.** `retune.executor` selects where retune runs. With `local`, the default, it runs +> only in offline simulation: `aisimulate predict --stack dynamo` and +> `dynamo.replay.run_trace_replay`, and a live Planner rejects it at startup. With `dgdr`, a live +> Planner on Kubernetes retunes three KV router knobs with fixed replicas through DGDR v1beta2 +> Requests; it is for development clusters only. See +> [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). Retune lets the Planner tune its own runtime parameters from the traffic it served. Every `retune.interval_seconds`, it takes the window of the request trace that just ended and replays it @@ -377,8 +380,11 @@ A candidate that changes decisions is judged on one replay of one window, so This minimal fixed-replica configuration searches the 108-point default router domain every six hours. It is a `PlannerConfig` for the `planner_config` argument of `dynamo.replay.run_trace_replay` (see [Run Retune from Python](aisimulate-planner-replay.mdx#run-retune-from-python)), not a DGDR -`spec.features.planner` fragment: a live Planner rejects it. In `aisimulate predict`, the `planner` -section takes `retune` and the two scaling flags, and `mode` comes from `engine.mode`; see +`spec.features.planner` fragment: a live Planner rejects it, because it uses the default +`executor: local`. For a live Planner, see +[Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). +In `aisimulate predict`, the `planner` section takes `retune` and the two scaling flags, and `mode` +comes from `engine.mode`; see [Scenario 1](aisimulate-planner-replay.mdx#scenario-1-fixed-replicas-router-only). ```yaml @@ -391,11 +397,12 @@ retune: ``` To run retune and read its decisions, see -[Simulate Retune](aisimulate-planner-replay.mdx#simulate-retune-experimental). For windows, -candidates, scoring, and decision timing, see -[Planner Design](planner-design.md#retune-experimental-simulation-only). For every field and the +[Simulate Retune](aisimulate-planner-replay.mdx#simulate-retune-experimental), and to run it on a +live deployment, see [Planner Live Retune](../../../../kubernetes/auto-deployment/planner-live-retune.md). +For windows, candidates, scoring, and decision timing, see +[Planner Design](planner-design.md#retune-experimental). For every field and the search-domain syntax, see the -[`retune` reference](../../../../reference/components/planner-configuration.mdx#retune-experimental-simulation-only). +[`retune` reference](../../../../reference/components/planner-configuration.mdx#retune-experimental). ## Integration with Profiler diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md new file mode 100644 index 000000000000..ea10836c8160 --- /dev/null +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -0,0 +1,377 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Planner Live Retune +subtitle: Let the Planner tune the KV router of a running DynamoGraphDeployment from its own recorded traffic, through DGDR v1beta2 search runs. +--- + +> [!CAUTION] +> **Experimental, development clusters only.** Live retune builds on DGDR v1beta2, which the +> operator serves only with the experimental `dgdrV1Beta2.enabled` Helm value, and it writes +> router parameters through the frontend admin API, which has no authentication. Do not enable it +> on a cluster that serves production traffic. + +Live retune is the Kubernetes form of [Periodic Retune](../../developer-guide/knowledge-base/modular-components/planner/planner-guide.md#periodic-retune-experimental). +Every `retune.interval_seconds`, the Planner cuts the window of request traces that just ended, +creates DynamoGraphDeploymentRequests (DGDR) v1beta2 that replay the window for a grid of KV +router parameters, scores the candidates the Runs report on `retune.objective`, and keeps the +current parameters unless a candidate beats them by more than `retune.min_improvement_percentage`. +Outside advisory mode it then sets the winner on every frontend router with +`PATCH /router_policy`. + +Live retune searches the three router knobs a DGDR Request can express: +`overlap_score_credit`, `prefill_load_scale` and `temperature`. Replicas stay fixed: the Planner +runs in retune-only mode and never scales. + +## How it works + +1. **Record.** Each frontend writes a `request_end` record per request to its own directory on a + shared ReadWriteMany claim (`///`). +2. **Cut a window.** At `L_k = window end + grace_seconds`, the Planner merges the records of the + window into one canonical trace file, `//.jsonl.gz`, in a + subprocess bounded by `cut_budget_seconds`. +3. **Search.** The Planner splits the router grid into at most `run.max_shards` Cartesian shards of + at most 100 points and creates one DGDR v1beta2 Request per shard. Each Request replays the + window file on the pinned `retune.dgdr.engine` with the replica counts of the DGD spec. One more + single-point Request replays a grid point again as a common-random-numbers probe: its score + must match the grid's within a relative error of 1e-9. +4. **Decide.** The Planner reads each Run's Candidates, scores them on `retune.objective`, and + decides with the same rules as replay retune. A decision is persisted in a Lease before it + reaches any router. +5. **Apply.** The Planner sets all eight router policy parameters of every frontend router to + the decided values, and keeps them there: a manual change to them is reverted. + +Windows are aligned to `retune.dgdr.phase_seconds + k × interval_seconds` in epoch time, so a +restarted Planner continues the same windows. Leadership and state live in two Leases owned by +the DGD, `-planner-retune` and `-planner-retune-state`. The first window after retune is +first enabled is skipped, since its trace is incomplete. + +Every DGDR trace Run replays with common random numbers: ordinal request ids and a fixed router +seed. A Run's score therefore does not depend on the other candidates or the Run's parallelism. +This applies to all DGDR trace Runs, not only those the Planner creates. + +## Prerequisites + +- **Operator with DGDR v1beta2 traces.** Install the platform chart with + `dynamo-operator.dgdrV1Beta2.enabled=true` and an operator that accepts `spec.workload.trace`. + The same Helm value grants the Planner ServiceAccount the access it needs (see [RBAC](#rbac)). + In a cluster-wide install, the operator must also hold the DGDR v1beta2 Run and Candidate + permissions of [ai-dynamo/dynamo#17308](https://github.com/ai-dynamo/dynamo/pull/17308); see + [RBAC](#rbac). +- **Run Job scheduling.** The namespace needs a LimitRange for the Run Jobs, and the operator + must discover GPUs, through DCGM or GPU discovery with labeled nodes. See + [Auto Deploy with DGDR](auto-deploy-with-dgdr.md#prerequisites). +- **A public model.** The Run Job has no Hugging Face credentials and no model mount, so + `retune.dgdr.model` must be a public Hugging Face repository id that AISimulate can resolve. +- **A ReadWriteMany claim** in the DGD namespace, dedicated to traces. Create `segments/` and + `windows/` at its root, writable by the user the frontend and Planner containers run as. +- **A KV-router frontend with default routing fields**, and workers that do not set their own + router mode. See [Deployment rules](#deployment-rules). +- **One Planner replica.** + +## Configure the frontend + +Add the trace environment and the claim to the frontend container of the DGD. The planner checks +these values against `retune.dgdr` at startup and whenever the DGD changes. + +```yaml +spec: + components: + - name: Frontend + type: frontend + replicas: 1 + podTemplate: + spec: + containers: + - name: main + command: [python3] + args: [-m, dynamo.frontend, --router-mode, kv] + env: + - name: DYN_POD_NAME + valueFrom: {fieldRef: {fieldPath: metadata.name}} + - {name: DYN_REQUEST_TRACE, value: "1"} + - {name: DYN_REQUEST_TRACE_RECORDS, value: request_end} + - {name: DYN_REQUEST_TRACE_SINKS, value: file} + - {name: DYN_REQUEST_TRACE_FILE_FORMAT, value: jsonl_gz} + - name: DYN_REQUEST_TRACE_FILE_PATH + value: /trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req + - {name: DYN_REQUEST_TRACE_FILE_ROLL_BYTES, value: "67108864"} + - {name: DYN_REQUEST_TRACE_CAPACITY, value: "8192"} + volumeMounts: + - {name: request-traces, mountPath: /trace} + volumes: + - name: request-traces + persistentVolumeClaim: {claimName: request-traces} +``` + +- `DYN_REQUEST_TRACE_FILE_PATH` is a file prefix directly under + `//$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/`, such as `req`: the + part after `$(DYN_POD_NAME)/` must not contain `/`. `` is where the frontend mounts + `retune.dgdr.claim_name`. The operator injects `DYN_PARENT_DGD_K8S_NAME`. +- Records are dropped without a count for `prompt_embeds`, multimodal, `n > 1` and `best_of > 1` + requests, and when the writer lags or a batch write fails. +- The trace holds unkeyed xxh3 hash chains of the prompts. Anyone who can create pods in the + namespace can mount the claim and read them. + +See [Agent Tracing](../../use-cases/agents/agent-tracing.md) for the `request_end` record. + +## Configure the Planner + +Mount the same claim in the Planner container at `retune.dgdr.mount_path`, and pass a +configuration like the one below with `--config`: + +```yaml +environment: kubernetes +backend: vllm +mode: agg +advisory: true # start advisory; see Roll out +enable_load_scaling: false # retune-only: both flags false +enable_throughput_scaling: false +retune: + interval_seconds: 21600 # T: window length and decision cadence + grace_seconds: 120 # covers p99 request time + 1 s flush + claim visibility + timeout_seconds: 20400 # required; at most T - 600; includes the window cut + decision_latency_seconds: 0 + objective: GoodputPerGPU + sla_ttft_ms: 500 + sla_itl_ms: 30 + router: default # 3 x 9 x 4 = 108 points + executor: dgdr + dgdr: + phase_seconds: 0 + claim_name: request-traces + mount_path: /trace + segments_dir: segments + windows_dir: windows + model: Qwen/Qwen3-32B + sku: h200_sxm + engine: + mode: aggregated + backend: vllm + backend_version: "0.11.0" + context_length: 32768 + workers: + aggregated: + parallelism: + preset: + - {tensor: 2, pipeline: 1, attention_data: 1, moe_tensor: 1, moe_expert: 1} + scheduler: {max_batched_tokens: 8192, max_sequences: 256} + prefill: null + decode: null + cut_budget_seconds: 900 + job_startup_seconds: 300 + run: + parallelism: 8 + candidate_timeout_seconds: 1800 + max_shards: 3 + keep_requests: 5 + max_window_rows: 5000000 + promote_last_advisory: false +``` + +`retune.dgdr` fields: + +| Field | Default | Meaning | +|---|---|---| +| `claim_name` | required | ReadWriteMany claim, in the DGD namespace, that holds the traces | +| `mount_path` | `/trace` | Where the Planner container mounts the claim; absolute | +| `segments_dir`, `windows_dir` | `segments`, `windows` | Claim-relative directories of the frontend segments and the window files; neither may contain the other | +| `phase_seconds` | `0` | Window phase in epoch seconds | +| `model` | required | Hugging Face repository id the workers serve; the Requests' `spec.model.name` | +| `sku` | required | The Requests' `spec.hardware.gpu.sku` | +| `engine` | required | The pinned AISimulate engine section; see [Engine section](#engine-section) | +| `startup_router_config` | `null` | Optional startup `KvRouterConfig` JSON; when set it must equal what the Planner parses from the frontend | +| `cut_budget_seconds` | `900` | Deadline of the window cut subprocess | +| `job_startup_seconds` | `300` | Run Job start-up time reserved when deciding whether a shard can still finish | +| `run.parallelism` | `8` | AISimulate `optimizer.parallelism` of each Run; Job resources may lower it | +| `run.candidate_timeout_seconds` | `1800` | AISimulate `optimizer.candidate_timeout_seconds` of each Run | +| `run.max_shards` | `3` | Most grid shards (Requests) per window; the probe and off-grid points do not count | +| `keep_requests` | `5` | Windows whose finished Requests are kept for audit | +| `max_window_rows` | `5000000` | A larger window fails with `window_too_large` | +| `promote_last_advisory` | `false` | On switching to active, apply the last advisory recommendation if it is at most one interval old and the configuration is unchanged | +| `lease_duration_seconds` | `60` | Leadership Lease duration, renewed every third of it | +| `actuator_interval_seconds` | `15` | Period of the router and clean-up rounds | +| `trace_incomplete_threshold` | `0.05` | With `metric_pulling_prometheus_endpoint`, a window with this fraction fewer requests than the frontend served fails with `trace_incomplete` | + +For the other `retune` fields and the limits that `executor: dgdr` adds to them, see the +[Retune field reference](../../reference/components/planner-configuration.mdx#retune-experimental). + +### Engine section + +`retune.dgdr.engine` becomes `spec.aisimulate.inputOverrides.engine` of every Request, so every +value in it must be a scalar. A `{choices: ...}` or `{range: ...}` domain, or an omitted knob that +AISimulate searches by default, would make the Run search engine settings as well. + +- `mode` is `aggregated` for `mode: agg` and `disaggregated` for `mode: disagg`. +- Each worker role of the mode (`aggregated`, or `prefill` and `decode`) sets + `parallelism.preset` to a list of exactly one entry with `tensor`, `pipeline`, `attention_data`, + `moe_tensor` and `moe_expert`, and sets `scheduler.max_batched_tokens` and + `scheduler.max_sequences`. +- Do not set `replicas` anywhere: the Planner fills them in from the DGD spec at each launch and + sets `spec.hardware.gpu.budget` to their GPU total. +- Write roles the mode does not use as `null`, or leave them out. +- Set the model and GPU with `retune.dgdr.model` and `retune.dgdr.sku`, not in the engine. + +### Time budget + +Each shard must finish before the window's deadline, `L_k + timeout_seconds`. A shard of `n` +points needs + +```text +need(n) = cut_budget_seconds + job_startup_seconds + + (1 + ceil((n - 1) / run.parallelism)) x run.candidate_timeout_seconds +``` + +AISimulate evaluates the first candidate alone, hence the `1 +`. The Planner rejects a +configuration whose largest grid shard needs more than `timeout_seconds`, because the baseline +can fall in any shard and a window without a baseline score fails. With the defaults, the 108-point +grid splits into 3 shards of 36 points and needs 900 + 300 + 6 × 1800 = 12000 seconds. Measure +replay time and memory on your trace before relying on these defaults. + +## Deployment rules + +The Planner checks three kinds of rules. None of them stops a running Planner except a +configuration error at startup. + +**Configuration**, checked when the Planner starts: + +| Rule | Requirement | +|---|---| +| P1 | `environment: kubernetes`; `decision_latency_seconds` is `0`, or `measured` with `decision_latency_scale: 1`; `timeout_seconds` is set and at most `interval_seconds - 600` | +| P2 | `enable_load_scaling` and `enable_throughput_scaling` are both `false`; `retune.planner` is not set | +| P3 | `retune.router` writes only `overlap_score_credit`, `prefill_load_scale`, `temperature`, `policy: kv_router` and `prefill_load_model: {type: none}` | +| P5 | `retune.dgdr.engine` is all scalars, as in [Engine section](#engine-section) | +| P13 | The largest grid shard has at most 100 points and fits the [time budget](#time-budget) | + +**Deployment**, checked at startup, when the DGD changes and before every router round. While one +fails, the Planner launches no windows, resets the routers to their startup parameters, and +retries: + +| Rule | Requirement | +|---|---| +| P4 | The frontend runs `--router-mode kv` with every routing field the Request cannot express at its default, `kv_event_lag_ms` unset or 0, no `DYN_ROUTER_WORKER_SELECTION_POLICY`, a `none` prefill load model, router env given as plain values (no `valueFrom`), and a command that is not wrapped in a shell. The Planner cannot read `envFrom` sources: it assumes they set no router env, and only warns about a source whose `prefix` could produce a `DYN_` name. Keep router env out of `envFrom`: P14 catches only a changed policy parameter. No worker advertises its own router configuration: worker components set neither `--router-mode` nor `DYN_ROUTER_MODE`, directly or through the graph-level `spec.env`, because a worker's advertised configuration replaces the frontend's for its workers | +| P7 | The DGD serves one model; the frontend is not an EPP, not a `frontendSidecar`, and has no TLS | +| P8 | One frontend replica, or `--router-replica-sync` | +| P9 | No `scalingAdapter` on worker components; one Planner replica | +| P10 | The frontend trace environment and claim mount match [Configure the frontend](#configure-the-frontend), with the trace file prefix directly under `$(DYN_POD_NAME)/` | +| P14 | Every router's startup parameters equal the ones the Planner parsed from the frontend | + +**Health**, checked every round. While one fails, windows do not launch; nothing is reset: + +| Rule | Requirement | +|---|---| +| P11 | The API server accepts a dry-run Request with `spec.workload.trace` | +| P12 | At least `max(1, replicas)` frontends are Ready, each with a router per worker type, and `/router_policy` answers | + +The Planner only warns when AISimulate does not list the model among its default models (P6). + +## Roll out + +1. Prepare the claim and its directories, then add the frontend trace configuration and the + Planner mount. +2. Start with `advisory: true`. In advisory mode the Planner records the decision it would make + and changes no router value it has not set itself. Watch `dynamo_planner_retune_ready` and the + decision history: + + ```bash + kubectl get lease -planner-retune-state -n \ + -o jsonpath='{.metadata.annotations.nvidia\.com/planner-retune-history}' + ``` + +3. Set `advisory: false`. With `retune.dgdr.promote_last_advisory: true`, the Planner applies the + last advisory recommendation right away if it is at most one interval old and the + configuration has not changed since. + +## Disable + +1. Set `advisory: true` and wait until `dynamo_planner_retune_router_converged` is 1 and + `dynamo_planner_retune_rollback_pending` is 0. The routers are then back at their startup + parameters. The mode stored in the state alone does not show that. +2. Remove `retune` from the Planner configuration. +3. Remove the trace environment from the frontend. +4. Delete the Planner's Requests and both Leases: + + ```bash + kubectl delete dynamographdeploymentrequests.nvidia.com -n \ + -l nvidia.com/planner-retune-dgd= + kubectl delete lease -n -planner-retune -planner-retune-state + ``` + +5. Delete `/` and `/` from the claim. + +Before upgrading the operator or its CRDs, switch to advisory or disable retune, then delete the +retune Requests. + +## Security + +- **Frontend admin API.** `/router_policy` is served on the frontend HTTP port, next to the + OpenAI API, without authentication, and is on by default. An Ingress that forwards `/` exposes + it. Deny the admin paths at L7 and allow in-cluster access only from the ingress controller and + the Planner. Do not set `DYN_DISABLE_FRONTEND_ADMIN_API` on the frontend: the Planner reports a + frontend whose `/router_policy` answers 404 as not ready. +- **Ownership.** Outside advisory mode the Planner owns the eight router policy parameters and + reverts manual changes. +- **Trace data.** See the privacy note in [Configure the frontend](#configure-the-frontend). + +### RBAC + +With `dynamo-operator.dgdrV1Beta2.enabled=true`, the platform chart adds these rules to the Planner Role +(namespace-restricted installs) and ClusterRole (cluster-wide installs): + +| API group | Resources | Verbs | +|---|---|---| +| `nvidia.com` | `dynamographdeploymentrequests` | create, get, list, watch, delete | +| `nvidia.com` | `dynamographdeploymentruns`, `dynamographdeploymentcandidates` | get, list, watch | +| `coordination.k8s.io` | `leases` | get, create, update, delete | + +The Planner also lists pods with its existing permission. These rules are namespace-wide: a +Role or a RoleBinding of the ClusterRole has no `resourceNames`, because the Planner creates its +Leases and Requests under per-DGD names. As a result, any workload that runs as the Planner +ServiceAccount can: + +- create DGDR Requests, which start Run Jobs, in its namespace; +- read, update or delete any Lease in its namespace, not only the two retune Leases. In a + namespace-restricted install, the Planner Role lives in the release namespace, which by default + also holds the operator's leader-election Lease, so such a workload can disrupt operator leader + election. + +In a cluster-wide install, the operator binds the Planner ClusterRole into each DGD namespace with +a RoleBinding, and Kubernetes lets it bind only permissions it holds itself. Set +`dynamo-operator.dgdrV1Beta2.enabled=true` there only with an operator whose own role grants +`get`, `list` and `watch` on `dynamographdeploymentruns` and `dynamographdeploymentcandidates`, +which the DGDR v1beta2 operator change in +[ai-dynamo/dynamo#17308](https://github.com/ai-dynamo/dynamo/pull/17308) adds. +Without it, the API server rejects the Planner RoleBinding in every namespace that does not have +one yet, and the DGDs there fail to reconcile their Planner RBAC, with or without live retune. + +## Metrics + +The Planner exports these on `metric_reporting_prometheus_port`: + +| Metric | Type | Meaning | +|---|---|---| +| `dynamo_planner_retune_decisions_total{outcome}` | counter | Decisions, counted after the state write persisted them | +| `dynamo_planner_retune_failures_total{reason}` | counter | Failed windows by reason, such as `launch_too_late`, `replicas_changed`, `window_cut_failed`, `trace_corrupt`, `crn_mismatch` or `k8s_error` | +| `dynamo_planner_retune_skipped_total{reason}` | counter | Due windows held back: `not_leader`, `not_ready` or `already_decided` | +| `dynamo_planner_retune_leader`, `_ready`, `_initialized`, `_rollback_pending` | gauge | Supervisor state, 0 or 1 | +| `dynamo_planner_retune_controller_generation` | gauge | Generation of the retune controller; it grows when a DGD change rebuilds it | +| `dynamo_planner_retune_baseline_score`, `_best_score` | gauge | Scores of the latest decision; NaN when not scored | +| `dynamo_planner_retune_sweep_seconds`, `_missing_candidates` | gauge | Duration of the latest sweep and its candidates without a score | +| `dynamo_planner_retune_router_converged` | gauge | 1 when every frontend router holds the target parameters | +| `dynamo_planner_retune_router_patch_errors_total{code}` | counter | Failed router updates by HTTP status or error kind | +| `dynamo_planner_retune_trace_sink_missing{pod}` | gauge | 1 for a Ready frontend without its own trace directory | +| `dynamo_planner_retune_trace_dropped_members_total` | counter | Corrupt or truncated trace gzip members skipped by window cuts | + +## Limitations + +| Not supported | Reason | +|---|---| +| Retune with load or throughput scaling, or of Planner parameters | A DGDR Run has no base Planner configuration, warm-up or predictor pre-sweep | +| Frontends with non-default routing fields beyond the three knobs, or `kv_event_lag_ms` other than 0 | The DGDR router section expresses only the three knobs, and the Run does not pass a lag | +| Workers that set their own router mode (`--router-mode` or `DYN_ROUTER_MODE`) | A worker's advertised router configuration replaces the frontend's for its workers, and the Run replays only the frontend's | +| Private, fine-tuned or local-path models | The Run Job has no Hugging Face credentials or model mount | +| Several frontends without router replica sync | AISimulate simulates one router | +| EPP or Gateway API Inference Extension frontends, `frontendSidecar`, TLS frontends | No `/router_policy` there, different addressing, or certificates | +| Run Job resources, deadline and tolerations | Fixed by the DGDR Run Job | +| Time-based segment rotation and a segment count limit | The frontend trace writer rotates by size only | +| A separate or authenticated admin listener | Not available in the frontend yet | diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index c0c292879f95..3886ee52e5f5 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -381,7 +381,7 @@ These fields default from environment variables and are excluded from serialized ## Advisory mode - Suggestion-only mode. The Planner computes, logs, exports, and reports recommended replica counts and [retune](#retune-experimental-simulation-only) decisions without executing any of them or changing the deployment. Use it to evaluate a new configuration or validate SLA targets against production traffic. The runtime configuration APIs, such as `PATCH /v1/min-endpoints`, still change the Planner's own settings. + Suggestion-only mode. The Planner computes, logs, exports, and reports recommended replica counts and [retune](#retune-experimental) decisions without executing any of them or changing the deployment. Use it to evaluate a new configuration or validate SLA targets against production traffic. The runtime configuration APIs, such as `PATCH /v1/min-endpoints`, still change the Planner's own settings. Planner-in-the-loop replay honors it too: an advisory replay keeps the simulated deployment at its starting replicas and router parameters, reports the recommendations in `planner.recommended_scaling_events`, and records a retune decision that would apply with outcome `advisory`. See [Replay in Advisory Mode](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#replay-in-advisory-mode). @@ -627,12 +627,17 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p -## Retune (Experimental, Simulation Only) + -`retune` makes the Planner periodically replay the last window of recorded traffic against a grid of router and Planner parameters and apply the best point by `objective`, SLA goodput per GPU by default. This section is the field contract. For the concept, see [Periodic Retune](../../developer-guide/knowledge-base/modular-components/planner/planner-guide.md#periodic-retune-experimental); for windows, scoring, and decision timing, see [Planner Design](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#retune-experimental-simulation-only); to run it, see [Simulate Retune](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#simulate-retune-experimental). +## Retune (Experimental) + +`retune` makes the Planner periodically replay the last window of recorded traffic against a grid of router and Planner parameters and apply the best point by `objective`, SLA goodput per GPU by default. This section is the field contract. For the concept, see [Periodic Retune](../../developer-guide/knowledge-base/modular-components/planner/planner-guide.md#periodic-retune-experimental); for windows, scoring, and decision timing, see [Planner Design](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#retune-experimental); to run it in replay, see [Simulate Retune](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#simulate-retune-experimental); to run it on a live deployment, see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). -**Experimental.** In this release, retune runs only inside planner-in-the-loop offline replay: `dynamo.replay.run_trace_replay` with `trace_format="dynamo"`, or `aisimulate predict --stack dynamo` on a Dynamo request trace. A live Planner rejects a config that sets `retune` at startup. +**Experimental.** Where retune runs depends on `executor`: + +- `local`, the default, runs only inside planner-in-the-loop offline replay: `dynamo.replay.run_trace_replay` with `trace_format="dynamo"`, or `aisimulate predict --stack dynamo` on a Dynamo request trace. A live Planner rejects it at startup. +- `dgdr` runs on a live Planner on Kubernetes, through DGDR v1beta2 Requests. It retunes three KV router knobs with fixed replicas and is meant for development clusters only; see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). @@ -656,13 +661,13 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p Trace history before each window that warms the load predictors of candidate planners and of a newly selected predictor. With throughput scaling, replay reads this history from the replayed trace and, before its first request, from `load_predictor_warmup_trace`, which must then be a Dynamo request trace that ends at or before the first replayed request; without throughput scaling no history is read. `null` resolves to the minimum the predictors need: `throughput_adjustment_interval_seconds` × max(`prophet_window_size`, `kalman_min_points`) over the current predictor and the load-predictor candidates, or 0 without throughput scaling. An explicit value must be 0 or greater, and a value smaller than that minimum is rejected. - Delay from sweep launch to decision. A number is a fixed delay, 0 or greater. `measured` uses the sweep's own wall-clock time, capped at `timeout_seconds`, times `decision_latency_scale`; it depends on the replay host, so it is not reproducible across hosts or runs. The decision lands no earlier than one Planner tick after launch. See [Decision Timing](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#decision-timing) for the mechanics and [Decision Latency](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#decision-latency) for choosing a value. + Delay from sweep launch to decision. A number is a fixed delay, 0 or greater. `measured` uses the sweep's own wall-clock time, capped at `timeout_seconds`, times `decision_latency_scale`; it depends on the replay host, so it is not reproducible across hosts or runs. The decision lands no earlier than one Planner tick after launch. See [Decision Timing](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#decision-timing) for the mechanics and [Decision Latency](../../developer-guide/knowledge-base/modular-components/planner/aisimulate-planner-replay.mdx#decision-latency) for choosing a value. With executor `dgdr`, the decision lands when the sweep returns, so this must be 0, or `measured` with `decision_latency_scale: 1`. Multiplier on the measured sweep time, for example to model a production sweep host that is slower or faster than the replay host. Must be greater than 0. A value other than 1 requires `decision_latency_seconds: measured`. Above 1, set `timeout_seconds` ≤ `interval_seconds` / scale (the default, `interval_seconds`, is rejected); the modeled deadline is then scale × `timeout_seconds`. - Wall-clock deadline for one sweep; its workers are killed after it and the decision is `timeout`. `null` means `interval_seconds`. Must be greater than 0. Requires `timeout_seconds` ≤ `interval_seconds`. A numeric `decision_latency_seconds` must also be ≤ `timeout_seconds`; with `measured`, `decision_latency_scale` × `timeout_seconds` must also be ≤ `interval_seconds`. This keeps at most one sweep in flight. + Wall-clock deadline for one sweep; its workers are killed after it and the decision is `timeout`. `null` means `interval_seconds`. Must be greater than 0. Requires `timeout_seconds` ≤ `interval_seconds`. A numeric `decision_latency_seconds` must also be ≤ `timeout_seconds`; with `measured`, `decision_latency_scale` × `timeout_seconds` must also be ≤ `interval_seconds`. This keeps at most one sweep in flight. With executor `dgdr`, it is required, must be ≤ `interval_seconds` − 600, and includes the window cut. Metric each candidate replay is scored on, written as the DGDR v1beta2 `spec.objective.metric` value. `TTFT` and `E2ELatency` are minimized; the others are maximized. See [Objectives](#objectives) for what each one scores. With load or throughput scaling enabled, only `GoodputPerGPU` and `ThroughputPerGPU` are accepted; retune-only mode accepts every value. @@ -682,7 +687,7 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p Upper bound on the expanded grid size (the product of all domain sizes), on the values of each knob, and separately on the number of load-predictor candidates. Must be at least 1. A grid that exceeds it is rejected with the size of each dimension. For the candidate set a sweep replays, see [Candidates and the Sweep](../../developer-guide/knowledge-base/modular-components/planner/planner-design.md#candidates-and-the-sweep). - Number of sweep worker processes. Must be at least 1. Each worker replays one candidate at a time, so memory grows with this value and the window size. + Number of sweep worker processes of executor `local`. Must be at least 1. Each worker replays one candidate at a time, so memory grows with this value and the window size. Executor `dgdr` ignores it and uses `dgdr.run.parallelism`. Router search domain, written like the `router` section of `aisimulate recommend --stack dynamo` (see [Search domains](#search-domains)). A knob omitted from the map searches its AISimulate default domain; write a scalar to pin it. `default` searches the whole AISimulate default domain. `null` (or no `router` field) keeps every router parameter at its startup value; an empty map `{}` is rejected. A searched router requires `router_mode="kv_router"` (`router.policy: kv_router` in `aisimulate predict`) with the default worker-selection policy, no `overlap_score_weight`, and conditional disaggregation disabled. @@ -690,6 +695,83 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p Planner search domain, written like the `planner` section of `aisimulate recommend --stack dynamo` and limited to what a running Planner can change (see [Search domains](#search-domains)). `default` searches the parts of the AISimulate default domain that apply to this Planner. `null` (or no `planner` field) keeps every Planner field unchanged; an empty map `{}` is rejected. A written knob or group requires load or throughput scaling, and the scaling mode that uses it: for example, `load_sensitivity` needs load scaling, and `load_predictor` needs `optimization_target: sla` with throughput scaling. A searched field must have a value in the Planner config. `ttft_ms` and `itl_ms` are never searched. + + Where sweeps run. `local` replays the candidates in worker processes of the Planner and runs only inside planner-in-the-loop replay; a live Planner rejects it at startup with `planner.retune is replay-only with executor 'local'; use executor: dgdr`. `dgdr` runs each sweep as DGDR v1beta2 Requests from a live Planner on Kubernetes and requires the `dgdr` section below. **Experimental**, development clusters only: see [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md). + + With `dgdr`, the Planner also rejects at startup a config that breaks a [configuration rule](../../kubernetes/auto-deployment/planner-live-retune.md#deployment-rules) of live retune: it requires `environment: kubernetes`, retune-only mode with no `planner` domain, a `router` domain that writes only `overlap_score_credit`, `prefill_load_scale`, `temperature`, `policy: kv_router` and `prefill_load_model: {type: none}`, the `timeout_seconds` and `decision_latency_seconds` limits above, a `dgdr.engine` that follows the rules below, and a largest grid shard of at most 100 points that fits the [time budget](../../kubernetes/auto-deployment/planner-live-retune.md#time-budget). + + Allowed values: local dgdr + + + Settings of executor `dgdr`. Required when `executor` is `dgdr`, and rejected otherwise. Rejects unknown fields. [Planner Live Retune](../../kubernetes/auto-deployment/planner-live-retune.md) shows a complete example. + + + + ReadWriteMany PersistentVolumeClaim, in the DGD namespace, that holds the frontend traces and the window files. The frontends and the Planner mount it. + + + Where the Planner container mounts `claim_name`. Must be absolute. + + + Claim-relative directory of the frontend trace segments, written as `///`. `/`-separated names without empty, `.` or `..` segments. `segments_dir` and `windows_dir` must not contain one another. + + + Claim-relative directory of the window files, written as `//.jsonl.gz`. Same rules as `segments_dir`. + + + Window phase. Window `k` is `[phase_seconds + k × interval_seconds, phase_seconds + (k+1) × interval_seconds)` in epoch seconds, so a restarted Planner continues the same windows. Must be 0 or greater. + + + Hugging Face repository id the workers serve, written as each Request's `spec.model.name`. The Run Job has no Hugging Face credentials or model mount, so it must be a public repository that AISimulate can resolve. + + + GPU SKU, written as each Request's `spec.hardware.gpu.sku`. + + + The pinned AISimulate `engine` section, written as each Request's `spec.aisimulate.inputOverrides.engine`. Every value must be a scalar, each worker role of `mode` sets one `parallelism.preset` entry and both scheduler knobs, and `replicas` must not be set: the Planner fills them in from the DGD spec at each launch. See [Engine section](../../kubernetes/auto-deployment/planner-live-retune.md#engine-section). + + + Optional startup `KvRouterConfig` JSON. When set, it must equal what the Planner parses from the frontend's arguments and environment; the parsed value is the source of truth. + + + Deadline of the subprocess that cuts a window file from the segments. Must be greater than 0. + + + Run Job start-up time reserved when deciding whether a shard can still finish before the window's deadline. Must be 0 or greater. + + + How each DGDR Run evaluates its points. Rejects unknown fields. + + + + AISimulate `optimizer.parallelism` of each Run. Must be at least 1. The Run Job's CPU and memory may lower the effective value. + + + AISimulate `optimizer.candidate_timeout_seconds` of each Run. Must be greater than 0. + + + Most grid shards, one Request each, per window. A shard is a Cartesian sub-grid of at most 100 points. The common-random-numbers probe and off-grid points are extra single-point Requests and do not count. Must be in [1, 16]. + + + + Windows whose finished Requests are kept for audit; older ones are deleted. Must be 0 or greater. + + + A window with more requests fails with `window_too_large`. Must be at least 1. + + + On switching from `advisory: true` to `false`, apply the last advisory recommendation if it is at most one interval old and the configuration is unchanged. + + + Duration of the leadership Lease, renewed every third of it. Must be at least 15. + + + Period of the router and clean-up rounds. Must be greater than 0. + + + With `metric_pulling_prometheus_endpoint`, a window whose request count is below the frontend's by more than this fraction fails with `trace_incomplete`. Must be in [0, 1]. + + ### Objectives @@ -954,11 +1036,11 @@ The Planner enforces these cross-field rules at startup and rejects a config tha - `fpm_sample_bucket_size` must be a perfect square. - `global_planner_namespace` is required when `environment` is `global-planner`. - Under `optimization_target: load`, prefill modes require `prefill_scale_up_queue_tokens` > `prefill_scale_down_queue_tokens`, and decode modes require `decode_scale_up_kv_rate` > `decode_scale_down_kv_rate`. -- At least one scaling mode must be enabled. Under any `optimization_target` other than `sla`, the Planner forces load scaling on and throughput scaling off and ignores `ttft_ms` / `itl_ms`. The only exception is retune-only mode: `retune` set with both `enable_load_scaling` and `enable_throughput_scaling` explicitly `false` (see [Retune](#retune-experimental-simulation-only)). +- At least one scaling mode must be enabled. Under any `optimization_target` other than `sla`, the Planner forces load scaling on and throughput scaling off and ignores `ttft_ms` / `itl_ms`. The only exception is retune-only mode: `retune` set with both `enable_load_scaling` and `enable_throughput_scaling` explicitly `false` (see [Retune](#retune-experimental)). - When `ais_perf_model` is set, its `roles` must cover the deployment: `prefill` and `decode` for `disagg`, the matching role for `prefill`/`decode`, or `aggregated` for `agg`. Role keys must agree with `worker_type`. Unknown canonical fields and the retired `aic_perf_model` input are rejected. - `scheduling.scale_interval_seconds` must evenly divide `load_adjustment_interval_seconds` and, when throughput scaling is enabled, `throughput_adjustment_interval_seconds`. - When both scaling modes are enabled, `load_adjustment_interval_seconds` must be shorter than `throughput_adjustment_interval_seconds`. -- `retune` has its own cross-field rules, listed with its fields in [Retune](#retune-experimental-simulation-only). A live Planner rejects any config that sets `retune`. +- `retune` has its own cross-field rules, listed with its fields in [Retune](#retune-experimental). A live Planner rejects a `retune` with executor `local`; executor `dgdr` adds the [configuration rules](../../kubernetes/auto-deployment/planner-live-retune.md#deployment-rules) of live retune. ## Related pages diff --git a/docs/fern/translations/zh-CN/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md b/docs/fern/translations/zh-CN/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md index 8a5b477a8673..42729c33194d 100644 --- a/docs/fern/translations/zh-CN/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md +++ b/docs/fern/translations/zh-CN/pages/developer-guide/knowledge-base/modular-components/planner/planner-guide.md @@ -60,7 +60,7 @@ spec: advisory: true ``` -advisory 模式仅提供建议。Planner 会计算建议副本数、记录日志、将其导出为诊断信息，并显示在 HTML 报告中。这些建议不会作为扩缩容决策应用：Planner 不会执行扩缩容操作，也不会更改部署。planner-in-the-loop replay 也遵循此设置：模拟部署保持不变，只报告建议，retune 决策（retune 目前仅在离线仿真中运行）也只记录、不应用。 +advisory 模式仅提供建议。Planner 会计算建议副本数、记录日志、将其导出为诊断信息，并显示在 HTML 报告中。这些建议不会作为扩缩容决策应用：Planner 不会执行扩缩容操作，也不会更改部署。planner-in-the-loop replay 也遵循此设置：模拟部署保持不变，只报告建议，retune 决策（`executor: local` 的 retune 仅在离线仿真中运行；线上部署使用 `executor: dgdr`，目前为实验功能）也只记录、不应用。 ### 优化目标 From 8815c8af0dc89a9b2122940c6cd6059e29a74694 Mon Sep 17 00:00:00 2001 From: Pranjul Gupta Date: Mon, 26 Oct 2026 12:27:07 +0100 Subject: [PATCH 003/100] pr-17262: squash aggregate GPU discovery Squashed from https://github.com/ai-dynamo/dynamo/pull/17262. Signed-off-by: Pranjul Gupta --- .../profiler/utils/dgdr_v1beta1_types.py | 4 +- .../dynamographdeploymentrequest_types.go | 8 +- ...dia.com_dynamographdeploymentrequests.yaml | 8 +- ...dynamographdeploymentrequest_controller.go | 8 +- ...ploymentrequest_controller_envtest_test.go | 15 +- .../controller/enrich_hardware_test.go | 43 ++++++ deploy/operator/internal/gpu/discovery.go | 40 +++-- .../operator/internal/gpu/discovery_test.go | 138 ++++++++++++++++++ .../additional-resources/api-reference-k8s.md | 4 +- .../dynamo-graph-deployment-request.mdx | 4 +- .../kubernetes-api/full-api-reference.mdx | 4 +- 11 files changed, 241 insertions(+), 35 deletions(-) diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py index d4ce2ad16f3e..425b031feb66 100644 --- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py +++ b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py @@ -229,7 +229,7 @@ class HardwareSpec(BaseModel): gpuSku: Optional[GPUSKUType] = Field( default=None, - description="GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.", + description="GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.", ) vramMb: Optional[float] = Field( default=None, @@ -237,7 +237,7 @@ class HardwareSpec(BaseModel): ) totalGpus: Optional[int] = Field( default=None, - description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.", + description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.", ) numGpusPerNode: Optional[int] = Field( default=None, diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go index bbdc617b187e..fadaef947842 100644 --- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go @@ -407,8 +407,8 @@ type FeaturesSpec struct { // If all four fields are set, discovery is skipped. type HardwareSpec struct { // GPUSKU selects the GPU type to target. - // When omitted, auto-detected by selecting the GPU with the highest - // node count, then highest VRAM. In mixed-GPU clusters, set this to + // When omitted, auto-detected by selecting the node with the most GPUs, + // then highest VRAM per GPU. In mixed-GPU clusters, set this to // choose which GPU type to use. Discovery and totalGpus are then // restricted to nodes matching this SKU. // The legacy value gb200_sxm is deprecated; use gb200 instead. @@ -423,8 +423,8 @@ type HardwareSpec struct { // TotalGPUs is the GPU budget for profiling and deployment. // The profiler uses this to determine parallelism and replica count. - // When omitted, computed by counting GPUs on discovered nodes - // (filtered by gpuSku when set), temporarily capped at 32 to + // When omitted, computed by summing GPUs on discovered nodes with the + // selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to // limit profiler search space. This cap may be removed in a future // release. Set this field explicitly to override. // +optional diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 404cc63ec777..1363f9e54924 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -663,8 +663,8 @@ spec: - mi300 description: |- GPUSKU selects the GPU type to target. - When omitted, auto-detected by selecting the GPU with the highest - node count, then highest VRAM. In mixed-GPU clusters, set this to + When omitted, auto-detected by selecting the node with the most GPUs, + then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. @@ -721,8 +721,8 @@ spec: description: |- TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. - When omitted, computed by counting GPUs on discovered nodes - (filtered by gpuSku when set), temporarily capped at 32 to + When omitted, computed by summing GPUs on discovered nodes with the + selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. format: int32 diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index 7577d49df281..e93462cdf5e7 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -1900,7 +1900,7 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx logger.Info("GPU discovery completed successfully", "gpusPerNode", gpuInfo.GPUsPerNode, "nodesWithGPUs", gpuInfo.NodesWithGPUs, - "totalGpus", gpuInfo.GPUsPerNode*gpuInfo.NodesWithGPUs, + "totalGpus", gpuInfo.TotalGPUs, "model", gpuInfo.Model, "vramMiB", gpuInfo.VRAMPerGPU, "system", gpuInfo.System, @@ -1935,14 +1935,14 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx if hw.TotalGPUs == nil && gpuInfo != nil { // TODO: This is a temporary limit to prevent the profiler from using too many GPUs. // Will be removed once a fix is in the Profiler/AIC. - const defaultMaxAutoGPUs = int32(32) - total := int32(gpuInfo.GPUsPerNode * gpuInfo.NodesWithGPUs) + const defaultMaxAutoGPUs = 32 + total := gpuInfo.TotalGPUs if total > defaultMaxAutoGPUs { logger.Info("Capping auto-discovered TotalGPUs at default limit; set hardware.totalGpus to override", "discovered", total, "cap", defaultMaxAutoGPUs) total = defaultMaxAutoGPUs } - hw.TotalGPUs = &total + hw.TotalGPUs = new(int32(total)) changed = true } if hw.Interconnect == "" && gpuInfo != nil && gpuInfo.Interconnect != "" { diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go index 7658c07a3b0b..ba02c3b2fdb7 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "slices" "strings" "time" @@ -3007,13 +3008,13 @@ spec: Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed()) defer func() { _ = k8sClient.Delete(ctx, dgdr) }() - GinkgoT().Log("creating two eight-GPU nodes for node-label discovery") + GinkgoT().Log("creating eight-GPU and four-GPU nodes for node-label discovery") gpuNodes := make([]*corev1.Node, 0, 2) - for _, nodeName := range []string{"gpu-worker-budget-1", "gpu-worker-budget-2"} { + for i, count := range []string{"8", "4"} { node := &corev1.Node{ - Name: nodeName, + Name: fmt.Sprintf("gpu-worker-budget-%d", i), Labels: map[string]string{ - gpu.LabelGPUCount: "8", + gpu.LabelGPUCount: count, gpu.LabelGPUProduct: "H200-SXM5-141GB", gpu.LabelGPUMemory: "141312", }, @@ -3042,7 +3043,7 @@ spec: Expect(k8sClient.Get(ctx, request.NamespacedName, &persisted)).Should(Succeed()) Expect(persisted.Spec.Hardware).NotTo(BeNil()) Expect(persisted.Spec.Hardware.TotalGPUs).NotTo(BeNil()) - Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(16))) + Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(12))) GinkgoT().Log("creating the profiling Job from the persisted DGDR") _, err = reconciler.Reconcile(ctx, request) @@ -3110,6 +3111,7 @@ spec: VRAMPerGPU: 81920, System: "h100_sxm", NodesWithGPUs: 1, + TotalGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3224,6 +3226,7 @@ spec: VRAMPerGPU: 81920, System: "h100_sxm", NodesWithGPUs: 1, + TotalGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3339,6 +3342,7 @@ spec: VRAMPerGPU: 81920, System: "h100_sxm", NodesWithGPUs: 1, + TotalGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3961,6 +3965,7 @@ spec: VRAMPerGPU: 81920, System: "a100_sxm", NodesWithGPUs: 1, + TotalGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) diff --git a/deploy/operator/internal/controller/enrich_hardware_test.go b/deploy/operator/internal/controller/enrich_hardware_test.go index ce7b90a4225f..4d365ca62e29 100644 --- a/deploy/operator/internal/controller/enrich_hardware_test.go +++ b/deploy/operator/internal/controller/enrich_hardware_test.go @@ -273,6 +273,46 @@ func TestEnrichHardwareFromDiscovery(t *testing.T) { } } +func TestEnrichHardwareFromDiscovery_TotalGPUs(t *testing.T) { + tests := []struct { + name string + counts []int + explicitTotal *int32 + wantTotal int32 + }{ + {name: "heterogeneous nodes", counts: []int{8, 4}, wantTotal: 12}, + {name: "aggregate below cap despite larger maximum times nodes", counts: []int{8, 8, 8, 4, 2}, wantTotal: 30}, + {name: "aggregate above cap", counts: []int{8, 8, 8, 8, 4}, wantTotal: 32}, + {name: "explicit total above cap", counts: []int{8, 4}, explicitTotal: new(int32(64)), wantTotal: 64}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("creating H100 nodes and an unrelated A100 node") + objects := []client.Object{gpuNode("a100-node", "A100-SXM4-40GB", 8, 40960)} + for i, count := range tt.counts { + objects = append(objects, gpuNode(fmt.Sprintf("h100-node-%d", i), "H100-SXM5-80GB", count, 81920)) + } + r := newFakeReconciler(objects...) + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ + Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ + Hardware: &nvidiacomv1beta1.HardwareSpec{ + GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, + TotalGPUs: tt.explicitTotal, + }, + }, + } + + t.Log("enriching missing hardware fields while preserving an explicit budget") + changed, err := r.enrichHardwareFromDiscovery(t.Context(), dgdr) + require.NoError(t, err) + require.True(t, changed) + require.NotNil(t, dgdr.Spec.Hardware.TotalGPUs) + assert.Equal(t, tt.wantTotal, *dgdr.Spec.Hardware.TotalGPUs) + assert.Equal(t, int32(8), *dgdr.Spec.Hardware.NumGPUsPerNode) + }) + } +} + func TestEnrichHardwareFromDiscovery_WritesOptionalHardwareMetadata(t *testing.T) { r := newFakeReconciler() r.GPUDiscovery = gpupkg.NewGPUDiscovery(nil) @@ -281,6 +321,7 @@ func TestEnrichHardwareFromDiscovery_WritesOptionalHardwareMetadata(t *testing.T NodeName: "n1", GPUsPerNode: 8, NodesWithGPUs: 2, + TotalGPUs: 16, Model: "NVIDIA H100 80GB HBM3", VRAMPerGPU: 81079, System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, @@ -313,6 +354,7 @@ func TestEnrichHardwareFromDiscovery_FillsOptionalMetadataWhenRequiredFieldsSet( NodeName: "n1", GPUsPerNode: 8, NodesWithGPUs: 4, + TotalGPUs: 32, Model: "NVIDIA H100 80GB HBM3", VRAMPerGPU: 81079, System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, @@ -367,6 +409,7 @@ func TestCreateProfilingJobPersistsDiscoveredHardware(t *testing.T) { NodeName: "n1", GPUsPerNode: 8, NodesWithGPUs: 2, + TotalGPUs: 16, Model: "NVIDIA H100 80GB HBM3", VRAMPerGPU: 81079, System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go index d431fdcce5c9..e553776ba575 100644 --- a/deploy/operator/internal/gpu/discovery.go +++ b/deploy/operator/internal/gpu/discovery.go @@ -167,8 +167,9 @@ var gpuRules = []gpuRule{ // GPUInfo contains discovered GPU configuration from cluster nodes type GPUInfo struct { NodeName string // Name of the node with this GPU configuration - GPUsPerNode int // Maximum GPUs per node found in the cluster - NodesWithGPUs int // Number of nodes that have GPUs + GPUsPerNode int // Maximum GPUs per node with the selected SKU + NodesWithGPUs int // Number of discovered nodes with the selected SKU + TotalGPUs int // Sum of GPUs across discovered nodes with the selected SKU, before any budget cap Model string // GPU product name (e.g., "H100-SXM5-80GB") VRAMPerGPU int // VRAM in MiB per GPU System nvidiacomv1beta1.GPUSKUType // AIC hardware system identifier (e.g., "h100_sxm", "h200_sxm"), empty if unknown @@ -259,7 +260,8 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // // When filterSKU is non-empty, only nodes whose inferred SKU matches are // considered. When empty, the best node is selected first (highest GPU count, -// then VRAM) and then only nodes with the same SKU are counted. +// then VRAM). Each node with the selected SKU contributes once to NodesWithGPUs, +// and its GPU count is summed into TotalGPUs. // // The function performs the following: // @@ -271,8 +273,7 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // 6. Selects the "best" GPU node (filtered by SKU when set) based on: // - Highest GPU count // - Highest VRAM per GPU (tie-breaker) -// 7. Counts only nodes matching the selected SKU for NodesWithGPUs. -// 8. Caches the result per SKU for a short duration to avoid repeated scraping. +// 7. Caches the result per SKU for a short duration to avoid repeated scraping. // // Behavior Notes: // @@ -280,7 +281,8 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // load-balancing ambiguity in multi-node clusters. // - If at least one pod is successfully scraped, partial failures are tolerated. // - If all pods fail to scrape, an aggregated error is returned. -// - Assumes DCGM exporter runs as a DaemonSet (one pod per GPU node). +// - Multiple exporters on one node use the report with the highest GPU count, +// then highest VRAM, for both node selection and aggregation. // // Returns: // - *GPUInfo for the selected node @@ -353,6 +355,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, nodeName string } allNodes := make([]nodeInfo, 0, len(dcgmPods)) + nodeIndexes := make(map[string]int) var scrapeErrors []error for _, pod := range dcgmPods { @@ -366,7 +369,18 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, continue } - allNodes = append(allNodes, nodeInfo{info: info, sku: InferHardwareSystem(info.Model), nodeName: pod.Spec.NodeName}) + // Running pods have a scheduled node name; keep its strongest report in first-seen node order. + n := nodeInfo{info: info, sku: InferHardwareSystem(info.Model), nodeName: pod.Spec.NodeName} + if index, seen := nodeIndexes[n.nodeName]; seen { + previous := allNodes[index].info + if info.GPUsPerNode > previous.GPUsPerNode || + (info.GPUsPerNode == previous.GPUsPerNode && info.VRAMPerGPU > previous.VRAMPerGPU) { + allNodes[index] = n + } + continue + } + nodeIndexes[n.nodeName] = len(allNodes) + allNodes = append(allNodes, n) } if len(allNodes) == 0 { @@ -399,11 +413,12 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, return nil, fmt.Errorf("no GPU metrics could be parsed from any DCGM pod") } - // Count only nodes with the same SKU as the selected best node, + // Count nodes and GPUs with the same SKU as the selected best node, // and detect RDMA on matching nodes only. On a cold cache and a no-RDMA // cluster, this performs one Node read per matching node; that keeps a // single negative node from masking RDMA on another node. nodesWithGPUs := 0 + totalGPUs := 0 var rdmaDetected bool var rdmaType string for _, n := range allNodes { @@ -411,6 +426,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, continue } nodesWithGPUs++ + totalGPUs += n.info.GPUsPerNode if !rdmaDetected { rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.nodeName) if rdma { @@ -435,6 +451,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, bestNode.System = bestSKU bestNode.CloudProvider = cloudProvider bestNode.NodesWithGPUs = nodesWithGPUs + bestNode.TotalGPUs = totalGPUs bestNode.RDMAEnabled = rdmaDetected bestNode.RDMAType = rdmaType @@ -868,16 +885,18 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK len(nodeList.Items), LabelGPUCount, LabelGPUProduct, LabelGPUMemory) } - // Count only nodes with the same SKU as the selected best node, + // Count nodes and GPUs with the same SKU as the selected best node, // and detect RDMA on matching nodes only. On a cold cache and a no-RDMA // cluster, this performs one Node read per matching node; that keeps a // single negative node from masking RDMA on another node. nodesWithGPUs := 0 + totalGPUs := 0 var rdmaDetected bool var rdmaType string for _, n := range allNodes { if n.sku == bestSKU { nodesWithGPUs++ + totalGPUs += n.info.GPUsPerNode if !rdmaDetected { rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.info.NodeName) if rdma { @@ -898,12 +917,13 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK bestNode.System = bestSKU bestNode.NodesWithGPUs = nodesWithGPUs + bestNode.TotalGPUs = totalGPUs bestNode.RDMAEnabled = rdmaDetected bestNode.RDMAType = rdmaType logger.Info("GPU discovery completed", "gpusPerNode", bestNode.GPUsPerNode, "nodesWithGPUs", bestNode.NodesWithGPUs, - "totalGpus", bestNode.GPUsPerNode*bestNode.NodesWithGPUs, + "totalGpus", bestNode.TotalGPUs, "model", bestNode.Model, "vram", bestNode.VRAMPerGPU, "system", bestNode.System, diff --git a/deploy/operator/internal/gpu/discovery_test.go b/deploy/operator/internal/gpu/discovery_test.go index 10f0c84a91dd..ce15c4effcef 100644 --- a/deploy/operator/internal/gpu/discovery_test.go +++ b/deploy/operator/internal/gpu/discovery_test.go @@ -51,6 +51,83 @@ func newFakeClient(objs ...client.Object) client.Reader { Build() } +func TestGPUDiscovery_TotalGPUs(t *testing.T) { + for _, source := range []string{"node labels", "DCGM"} { + t.Run(source, func(t *testing.T) { + tests := []struct { + name string + h100Counts []int + a100Counts []int + filterSKU nvidiacomv1beta1.GPUSKUType + wantSKU nvidiacomv1beta1.GPUSKUType + wantPerNode int + wantNodes int + wantTotal int + }{ + {name: "heterogeneous nodes", h100Counts: []int{8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, + {name: "maximum selected after smaller node", h100Counts: []int{4, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, + {name: "unfiltered excludes other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, + {name: "filter selects other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, filterSKU: "a100_sxm", wantSKU: "a100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 10}, + {name: "discovery does not cap the total", h100Counts: []int{8, 8, 8, 8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 5, wantTotal: 36}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("creating GPU nodes and DCGM exporters for each SKU") + var objects []client.Object + metrics := make(map[string]GPUInfo) + for _, group := range []struct { + model string + vram int + counts []int + }{ + {model: "H100-SXM5-80GB", vram: 81920, counts: tt.h100Counts}, + {model: "A100-SXM4-40GB", vram: 40960, counts: tt.a100Counts}, + } { + for _, count := range group.counts { + name := fmt.Sprintf("gpu-node-%d", len(metrics)) + ip := fmt.Sprintf("10.0.0.%d", len(metrics)+1) + objects = append(objects, &corev1.Node{ + Name: name, + Labels: map[string]string{ + LabelGPUCount: fmt.Sprint(count), LabelGPUProduct: group.model, LabelGPUMemory: fmt.Sprint(group.vram), + }, + }, &corev1.Pod{ + Name: name, Namespace: "gpu-operator", + Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, + Spec: corev1.PodSpec{NodeName: name}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: ip}, + }) + metrics[buildDCGMEndpoint(ip)] = GPUInfo{NodeName: name, GPUsPerNode: count, Model: group.model, VRAMPerGPU: group.vram} + } + } + k8sClient := newFakeClient(objects...) + + t.Log("discovering the selected SKU's actual GPU total") + var info *GPUInfo + var err error + if source == "DCGM" { + discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) { + scraped, ok := metrics[endpoint] + if !ok { + return nil, fmt.Errorf("unexpected DCGM endpoint %q", endpoint) + } + return &scraped, nil + }) + info, err = discovery.DiscoverGPUsFromDCGMFiltered(t.Context(), k8sClient, nil, tt.filterSKU) + } else { + info, err = DiscoverGPUsFiltered(t.Context(), k8sClient, tt.filterSKU) + } + require.NoError(t, err) + assert.Equal(t, tt.wantSKU, info.System) + assert.Equal(t, tt.wantPerNode, info.GPUsPerNode) + assert.Equal(t, tt.wantNodes, info.NodesWithGPUs) + assert.Equal(t, tt.wantTotal, info.TotalGPUs) + }) + } + }) + } +} + func TestDiscoverGPUs_SingleNode(t *testing.T) { ctx := context.Background() @@ -70,6 +147,7 @@ func TestDiscoverGPUs_SingleNode(t *testing.T) { require.NotNil(t, gpuInfo) assert.Equal(t, 8, gpuInfo.GPUsPerNode) + assert.Equal(t, 8, gpuInfo.TotalGPUs) assert.Equal(t, "H100-SXM5-80GB", gpuInfo.Model) assert.Equal(t, 81920, gpuInfo.VRAMPerGPU) assert.Equal(t, "h100_sxm", string(gpuInfo.System)) @@ -103,6 +181,7 @@ func TestDiscoverGPUs_MultipleNodesHomogeneous(t *testing.T) { require.NotNil(t, gpuInfo) assert.Equal(t, 8, gpuInfo.GPUsPerNode) + assert.Equal(t, 16, gpuInfo.TotalGPUs) assert.Equal(t, "H100-SXM5-80GB", gpuInfo.Model) assert.Equal(t, 81920, gpuInfo.VRAMPerGPU) } @@ -1061,6 +1140,62 @@ func TestScrapeMetricsEndpoint_ConnectionReuse(t *testing.T) { }) } +func TestDiscoverGPUsFromDCGM_DeduplicatesNodes(t *testing.T) { + tests := []struct { + name string + counts [2]int + vram [2]int + failFirstExporter bool + wantVRAM int + }{ + {name: "matching reports", counts: [2]int{8, 8}, vram: [2]int{81920, 81920}, wantVRAM: 81920}, + {name: "first exporter fails", counts: [2]int{8, 8}, vram: [2]int{81920, 81920}, failFirstExporter: true, wantVRAM: 81920}, + {name: "larger count last", counts: [2]int{4, 8}, vram: [2]int{81920, 40960}, wantVRAM: 40960}, + {name: "larger count first", counts: [2]int{8, 4}, vram: [2]int{40960, 81920}, wantVRAM: 40960}, + {name: "higher VRAM last", counts: [2]int{8, 8}, vram: [2]int{40960, 81920}, wantVRAM: 81920}, + {name: "higher VRAM first", counts: [2]int{8, 8}, vram: [2]int{81920, 40960}, wantVRAM: 81920}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("creating two exporters on one node and one on another node") + var objects []client.Object + metrics := make(map[string]GPUInfo) + for i, node := range []string{"node-a", "node-a", "node-b"} { + ip := fmt.Sprintf("10.0.0.%d", i+1) + count, vram := 4, 81920 + if i < 2 { + count, vram = tt.counts[i], tt.vram[i] + } + metrics[buildDCGMEndpoint(ip)] = GPUInfo{GPUsPerNode: count, Model: "H100-SXM5-80GB", VRAMPerGPU: vram} + objects = append(objects, &corev1.Pod{ + Name: fmt.Sprintf("dcgm-%d", i), Namespace: "gpu-operator", + Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: ip}, + }) + } + discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) { + if tt.failFirstExporter && endpoint == buildDCGMEndpoint("10.0.0.1") { + return nil, errors.New("exporter unavailable") + } + scraped, ok := metrics[endpoint] + if !ok { + return nil, fmt.Errorf("unexpected DCGM endpoint %q", endpoint) + } + return &scraped, nil + }) + + t.Log("counting each successfully discovered node once") + info, err := discovery.DiscoverGPUsFromDCGM(t.Context(), newFakeClient(objects...), nil) + require.NoError(t, err) + assert.Equal(t, 8, info.GPUsPerNode) + assert.Equal(t, tt.wantVRAM, info.VRAMPerGPU) + assert.Equal(t, 2, info.NodesWithGPUs) + assert.Equal(t, 12, info.TotalGPUs) + }) + } +} + func TestDiscoverGPUsFromDCGM_CacheHit(t *testing.T) { ctx := context.Background() @@ -1118,6 +1253,7 @@ func TestDiscoverGPUsFromDCGM_CacheHit(t *testing.T) { require.Equal(t, 1, callCount) require.Equal(t, info1, info2) + require.Equal(t, 4, info2.TotalGPUs) } func TestDiscoverGPUsFromDCGM_SharesConcurrentScrape(t *testing.T) { @@ -1216,11 +1352,13 @@ func TestDiscoverGPUsFromDCGMFiltered_MixedSKU(t *testing.T) { &corev1.Pod{ Name: "dcgm-h100", Namespace: "gpu-operator", Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, + Spec: corev1.PodSpec{NodeName: "node-h100"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.1"}, }, &corev1.Pod{ Name: "dcgm-a100", Namespace: "gpu-operator", Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, + Spec: corev1.PodSpec{NodeName: "node-a100"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.2"}, }, } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 74670f7dbc4a..c416c42c527f 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2543,9 +2543,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected by selecting the GPU with the highest
node count, then highest VRAM. In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| +| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected by selecting the node with the most GPUs,
then highest VRAM per GPU. In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| | `vramMb` _float_ | VRAMMB is the VRAM per GPU in MiB.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
| -| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by counting GPUs on discovered nodes
(filtered by gpuSku when set), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
| +| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by summing GPUs on discovered nodes with the
selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
| | `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
| | `interconnect` _string_ | Interconnect describes the primary GPU-to-GPU interconnect *within a node*.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT configure or enable any GPU interconnect; it only describes what is available/assumed.
- When omitted, the operator may attempt best-effort discovery (currently distinguishes "nvlink"
vs "pcie" based on DCGM NVLink link count). If discovery is unavailable, it may remain empty.
Impact of wrong / missing values:
- If set more optimistically than reality (e.g., "nvlink" when only PCIe is present), performance
models may overestimate intra-node bandwidth and choose overly aggressive parallelism or layouts,
resulting in degraded performance compared to expectations.
- If set more pessimistically than reality (e.g., "pcie" when NVLink is present), the system may
choose conservative plans and leave performance on the table.
- If unset and undiscovered, consumers should treat the interconnect as unknown and fall back to
conservative assumptions.
Example values: "pcie", "nvlink". Other values may be accepted but may not be auto-detected. | | Optional: \{\}
| | `rdma` _boolean_ | RDMA indicates whether the cluster has RDMA-capable networking available for Dynamo data movement.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT install, enable, or configure RDMA (e.g., drivers, SR-IOV, NVIDIA network operator,
GPUDirect settings). It only expresses availability/intent.
- When omitted, the operator may attempt best-effort discovery (e.g., via node labels indicating
RDMA/SR-IOV capability and/or presence of NVIDIA network-operator RDMA components). If discovery
is unavailable, it may remain unset.
Impact of wrong / missing values:
- False positive (set true when RDMA is not actually usable end-to-end) may cause plans or
deployments to assume RDMA is available; depending on the runtime transport selection and
fallback behavior, this can lead to connection/setup failures or performance regressions.
- False negative (set false when RDMA is available) will typically avoid RDMA-optimized paths and
fall back to non-RDMA transports, usually remaining functional but potentially slower.
- If unset and undiscovered, consumers should treat RDMA availability as unknown and use
conservative defaults / fallback transports. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index 286a33378170..e2dd516d6571 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -142,13 +142,13 @@ service. - GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes. + GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes. VRAM per GPU in MiB. Auto-detected from cluster GPU nodes when omitted. - GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by `gpuSku` when set), temporarily capped at 32 to limit the profiler search space. Set explicitly to override the cap. + GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by `gpuSku` when set), temporarily capped at 32 to limit the profiler search space. For example, matching nodes with 8 and 4 GPUs yield a budget of 12, while `numGpusPerNode` remains 8. Set explicitly to override the cap. GPUs per node. Auto-detected when omitted. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 4b345d6e0604..1baae47908d9 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -3799,7 +3799,7 @@ If all four fields are set, discovery is skipped. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta1-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta1-dynamographdeploymentrequestspec) -GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. +GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. See [GPUSKUType](#gpuskutype). **Validation:** Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300] Optional: \{\} @@ -3810,7 +3810,7 @@ VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU -TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. +TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. **Validation:** Optional: \{\} From 3669d629aa2113944f7779cea0031068ebd4c60d Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 20:06:04 +0100 Subject: [PATCH 004/100] pr-17262: pr-1: separate selected-shape GPU capacity [upstream: Pranjulcr7/dynamo#1] Signed-off-by: Dr. Stefan Schimanski --- .../profiler/utils/dgdr_v1beta1_types.py | 8 +- .../dynamographdeploymentrequest_types.go | 24 ++-- ...dia.com_dynamographdeploymentrequests.yaml | 24 ++-- ...dynamographdeploymentrequest_controller.go | 25 +++- ...ploymentrequest_controller_envtest_test.go | 46 ++++---- .../controller/enrich_hardware_test.go | 107 ++++++++++-------- deploy/operator/internal/gpu/discovery.go | 69 +++++++---- .../operator/internal/gpu/discovery_test.go | 52 ++++++--- .../additional-resources/api-reference-k8s.md | 8 +- .../dynamo-graph-deployment-request.mdx | 8 +- .../kubernetes-api/full-api-reference.mdx | 8 +- 11 files changed, 237 insertions(+), 142 deletions(-) diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py index 425b031feb66..b686ebb2b46f 100644 --- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py +++ b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py @@ -229,19 +229,19 @@ class HardwareSpec(BaseModel): gpuSku: Optional[GPUSKUType] = Field( default=None, - description="GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.", + description="GPUSKU selects the GPU type to target. When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.", ) vramMb: Optional[float] = Field( default=None, - description="VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes.", + description="VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes. When this value differs from the auto-selected shape and totalGpus is omitted, the request is rejected.", ) totalGpus: Optional[int] = Field( default=None, - description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.", + description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on nodes matching the selected shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily capped at 32 to limit profiler search space. For example, same-type nodes with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU shape yields an automatic budget of 8. This cap may be removed in a future release. Set this field explicitly to override the derived budget or use a numGpusPerNode or vramMb value that differs from the selected shape. A shape override without an explicit totalGpus is rejected.", ) numGpusPerNode: Optional[int] = Field( default=None, - description="NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected from cluster GPU nodes.", + description="NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on. When this value differs from the auto-selected shape and totalGpus is omitted, the request is rejected.", ) interconnect: Optional[str] = Field( default=None, diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go index fadaef947842..645ab6643bae 100644 --- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go @@ -407,8 +407,8 @@ type FeaturesSpec struct { // If all four fields are set, discovery is skipped. type HardwareSpec struct { // GPUSKU selects the GPU type to target. - // When omitted, auto-detected by selecting the node with the most GPUs, - // then highest VRAM per GPU. In mixed-GPU clusters, set this to + // When omitted, auto-detected as the GPU type of the node with the most GPUs + // (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to // choose which GPU type to use. Discovery and totalGpus are then // restricted to nodes matching this SKU. // The legacy value gb200_sxm is deprecated; use gb200 instead. @@ -417,21 +417,29 @@ type HardwareSpec struct { GPUSKU GPUSKUType `json:"gpuSku,omitempty"` // VRAMMB is the VRAM per GPU in MiB. - // When omitted, auto-detected from cluster GPU nodes. + // When omitted, auto-detected from cluster GPU nodes. When this value differs + // from the auto-selected shape and totalGpus is omitted, the request is rejected. // +optional VRAMMB *float64 `json:"vramMb,omitempty"` // TotalGPUs is the GPU budget for profiling and deployment. // The profiler uses this to determine parallelism and replica count. - // When omitted, computed by summing GPUs on discovered nodes with the - // selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to - // limit profiler search space. This cap may be removed in a future - // release. Set this field explicitly to override. + // When omitted, computed by summing GPUs on nodes matching the selected + // shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily + // capped at 32 to limit profiler search space. For example, same-type nodes + // with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU + // shape yields an automatic budget of 8. This cap may be removed in a future + // release. Set this field explicitly to override the derived budget or use a + // numGpusPerNode or vramMb value that differs from the selected shape. A shape + // override without an explicit totalGpus is rejected. // +optional TotalGPUs *int32 `json:"totalGpus,omitempty"` // NumGPUsPerNode is the number of GPUs per node. - // When omitted, auto-detected from cluster GPU nodes. + // When omitted, auto-detected as the highest GPU count on any discovered node + // of the selected GPU type. In clusters whose nodes have different GPU counts, + // set this explicitly to the node size you intend to schedule on. When this value + // differs from the auto-selected shape and totalGpus is omitted, the request is rejected. // +optional NumGPUsPerNode *int32 `json:"numGpusPerNode,omitempty"` // Interconnect describes the primary GPU-to-GPU interconnect *within a node*. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 1363f9e54924..66b313d02c50 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -663,8 +663,8 @@ spec: - mi300 description: |- GPUSKU selects the GPU type to target. - When omitted, auto-detected by selecting the node with the most GPUs, - then highest VRAM per GPU. In mixed-GPU clusters, set this to + When omitted, auto-detected as the GPU type of the node with the most GPUs + (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. @@ -693,7 +693,10 @@ spec: numGpusPerNode: description: |- NumGPUsPerNode is the number of GPUs per node. - When omitted, auto-detected from cluster GPU nodes. + When omitted, auto-detected as the highest GPU count on any discovered node + of the selected GPU type. In clusters whose nodes have different GPU counts, + set this explicitly to the node size you intend to schedule on. When this value + differs from the auto-selected shape and totalGpus is omitted, the request is rejected. format: int32 type: integer rdma: @@ -721,16 +724,21 @@ spec: description: |- TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. - When omitted, computed by summing GPUs on discovered nodes with the - selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to - limit profiler search space. This cap may be removed in a future - release. Set this field explicitly to override. + When omitted, computed by summing GPUs on nodes matching the selected + shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily + capped at 32 to limit profiler search space. For example, same-type nodes + with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU + shape yields an automatic budget of 8. This cap may be removed in a future + release. Set this field explicitly to override the derived budget or use a + numGpusPerNode or vramMb value that differs from the selected shape. A shape + override without an explicit totalGpus is rejected. format: int32 type: integer vramMb: description: |- VRAMMB is the VRAM per GPU in MiB. - When omitted, auto-detected from cluster GPU nodes. + When omitted, auto-detected from cluster GPU nodes. When this value differs + from the auto-selected shape and totalGpus is omitted, the request is rejected. type: number type: object image: diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index e93462cdf5e7..4282b75f8f53 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -1901,6 +1901,7 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx "gpusPerNode", gpuInfo.GPUsPerNode, "nodesWithGPUs", gpuInfo.NodesWithGPUs, "totalGpus", gpuInfo.TotalGPUs, + "selectedShapeGpus", gpuInfo.SelectedShapeGPUs, "model", gpuInfo.Model, "vramMiB", gpuInfo.VRAMPerGPU, "system", gpuInfo.System, @@ -1910,6 +1911,11 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx "rdma", gpuInfo.RDMAEnabled, "rdmaType", gpuInfo.RDMAType) + // An automatically derived budget is valid only for discovery's selected node shape. + if err := validateAutomaticGPUShape(hw, gpuInfo); err != nil { + return changed, err + } + if hw.GPUSKU == "" { inferred := gpu.InferHardwareSystem(gpuInfo.Model) switch { @@ -1936,10 +1942,10 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx // TODO: This is a temporary limit to prevent the profiler from using too many GPUs. // Will be removed once a fix is in the Profiler/AIC. const defaultMaxAutoGPUs = 32 - total := gpuInfo.TotalGPUs + total := gpuInfo.SelectedShapeGPUs if total > defaultMaxAutoGPUs { - logger.Info("Capping auto-discovered TotalGPUs at default limit; set hardware.totalGpus to override", - "discovered", total, "cap", defaultMaxAutoGPUs) + logger.Info("Capping auto-discovered GPU budget at default limit; set hardware.totalGpus to override", + "selectedShapeGpus", total, "cap", defaultMaxAutoGPUs) total = defaultMaxAutoGPUs } hw.TotalGPUs = new(int32(total)) @@ -1962,6 +1968,19 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx return changed, nil } +func validateAutomaticGPUShape(hw *nvidiacomv1beta1.HardwareSpec, gpuInfo *gpu.GPUInfo) error { + if hw.TotalGPUs != nil { + return nil + } + if hw.NumGPUsPerNode != nil && int(*hw.NumGPUsPerNode) != gpuInfo.GPUsPerNode { + return fmt.Errorf("cannot auto-detect hardware.totalGpus: hardware.numGpusPerNode=%d does not match the selected discovered shape with %d GPUs per node; set hardware.totalGpus explicitly", *hw.NumGPUsPerNode, gpuInfo.GPUsPerNode) + } + if hw.VRAMMB != nil && *hw.VRAMMB != float64(gpuInfo.VRAMPerGPU) { + return fmt.Errorf("cannot auto-detect hardware.totalGpus: hardware.vramMb=%g does not match the selected discovered shape with %d MiB per GPU; set hardware.totalGpus explicitly", *hw.VRAMMB, gpuInfo.VRAMPerGPU) + } + return nil +} + func (r *DynamoGraphDeploymentRequestReconciler) discoverHardwareForEnrichment(ctx context.Context, hw *nvidiacomv1beta1.HardwareSpec, discoveryRequired bool) (*gpu.GPUInfo, error) { logger := log.FromContext(ctx) logger.Info("Attempting GPU discovery for profiling job") diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go index ba02c3b2fdb7..57dc16fc030e 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go @@ -3043,7 +3043,7 @@ spec: Expect(k8sClient.Get(ctx, request.NamespacedName, &persisted)).Should(Succeed()) Expect(persisted.Spec.Hardware).NotTo(BeNil()) Expect(persisted.Spec.Hardware.TotalGPUs).NotTo(BeNil()) - Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(12))) + Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(8))) GinkgoT().Log("creating the profiling Job from the persisted DGDR") _, err = reconciler.Reconcile(ctx, request) @@ -3107,11 +3107,12 @@ spec: defer func() { _ = k8sClient.Delete(ctx, dgdr) }() mockGPU := &gpu.GPUInfo{ - GPUsPerNode: 8, - VRAMPerGPU: 81920, - System: "h100_sxm", - NodesWithGPUs: 1, - TotalGPUs: 8, + GPUsPerNode: 8, + VRAMPerGPU: 81920, + System: "h100_sxm", + NodesWithGPUs: 1, + TotalGPUs: 8, + SelectedShapeGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3222,11 +3223,12 @@ spec: defer func() { _ = k8sClient.Delete(ctx, dgdr) }() mockGPU := &gpu.GPUInfo{ - GPUsPerNode: 8, - VRAMPerGPU: 81920, - System: "h100_sxm", - NodesWithGPUs: 1, - TotalGPUs: 8, + GPUsPerNode: 8, + VRAMPerGPU: 81920, + System: "h100_sxm", + NodesWithGPUs: 1, + TotalGPUs: 8, + SelectedShapeGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3338,11 +3340,12 @@ spec: defer func() { _ = k8sClient.Delete(ctx, dgdr) }() mockGPU := &gpu.GPUInfo{ - GPUsPerNode: 8, - VRAMPerGPU: 81920, - System: "h100_sxm", - NodesWithGPUs: 1, - TotalGPUs: 8, + GPUsPerNode: 8, + VRAMPerGPU: 81920, + System: "h100_sxm", + NodesWithGPUs: 1, + TotalGPUs: 8, + SelectedShapeGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3961,11 +3964,12 @@ spec: // Mock GPU discovery so validation and enrichment succeed. mockGPU := &gpu.GPUInfo{ - GPUsPerNode: 8, - VRAMPerGPU: 81920, - System: "a100_sxm", - NodesWithGPUs: 1, - TotalGPUs: 8, + GPUsPerNode: 8, + VRAMPerGPU: 81920, + System: "a100_sxm", + NodesWithGPUs: 1, + TotalGPUs: 8, + SelectedShapeGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) diff --git a/deploy/operator/internal/controller/enrich_hardware_test.go b/deploy/operator/internal/controller/enrich_hardware_test.go index 4d365ca62e29..f24f4f9fc918 100644 --- a/deploy/operator/internal/controller/enrich_hardware_test.go +++ b/deploy/operator/internal/controller/enrich_hardware_test.go @@ -177,17 +177,17 @@ func TestEnrichHardwareFromDiscovery(t *testing.T) { }, { name: "nothing set, full discovery", - discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, Model: "H100-SXM5-80GB", VRAMPerGPU: 81920}, + discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, TotalGPUs: 8, SelectedShapeGPUs: 8, Model: "H100-SXM5-80GB", VRAMPerGPU: 81920}, wantGPUSKU: "h100_sxm", wantVRAM: 81920, wantGPUsNode: 8, wantTotalGPUs: 8, wantChanged: true, }, { name: "nothing set, V100 discovered", - discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, Model: "Tesla-V100-SXM2-16GB", VRAMPerGPU: 16384}, + discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, TotalGPUs: 8, SelectedShapeGPUs: 8, Model: "Tesla-V100-SXM2-16GB", VRAMPerGPU: 16384}, wantGPUSKU: "v100_sxm", wantVRAM: 16384, wantGPUsNode: 8, wantTotalGPUs: 8, wantChanged: true, }, { name: "nothing set, unknown GPU falls back to model name", - discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 4, Model: "FutureGPU-X1000", VRAMPerGPU: 65536}, + discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 4, TotalGPUs: 4, SelectedShapeGPUs: 4, Model: "FutureGPU-X1000", VRAMPerGPU: 65536}, wantGPUSKU: "FutureGPU-X1000", wantVRAM: 65536, wantGPUsNode: 4, wantTotalGPUs: 4, wantChanged: true, }, { @@ -195,7 +195,7 @@ func TestEnrichHardwareFromDiscovery(t *testing.T) { hardware: &nvidiacomv1beta1.HardwareSpec{ GPUSKU: "b200_sxm", VRAMMB: new(141312.0), NumGPUsPerNode: new(int32(8)), }, - discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, Model: "B200-SXM-180GB", VRAMPerGPU: 141312}, + discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, TotalGPUs: 8, SelectedShapeGPUs: 8, Model: "B200-SXM-180GB", VRAMPerGPU: 141312}, wantGPUSKU: "b200_sxm", wantVRAM: 141312, wantGPUsNode: 8, wantTotalGPUs: 8, wantChanged: true, }, { @@ -203,16 +203,24 @@ func TestEnrichHardwareFromDiscovery(t *testing.T) { hardware: &nvidiacomv1beta1.HardwareSpec{ VRAMMB: new(81920.0), NumGPUsPerNode: new(int32(8)), TotalGPUs: new(int32(16)), }, - discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, Model: "H200-SXM5-141GB", VRAMPerGPU: 141312}, + discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, TotalGPUs: 8, SelectedShapeGPUs: 8, Model: "H200-SXM5-141GB", VRAMPerGPU: 141312}, wantGPUSKU: "h200_sxm", wantVRAM: 81920, wantGPUsNode: 8, wantTotalGPUs: 16, wantChanged: true, // user overrides win }, { - name: "vramMb and numGpusPerNode override discovery", + name: "numGpusPerNode override requires an explicit totalGpus", hardware: &nvidiacomv1beta1.HardwareSpec{ - GPUSKU: "a100_sxm", VRAMMB: new(40960.0), NumGPUsPerNode: new(int32(4)), + GPUSKU: "a100_sxm", VRAMMB: new(81920.0), NumGPUsPerNode: new(int32(4)), }, - discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, Model: "A100-SXM4-80GB", VRAMPerGPU: 81920}, - wantGPUSKU: "a100_sxm", wantVRAM: 40960, wantGPUsNode: 4, wantTotalGPUs: 8, wantChanged: true, + discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, TotalGPUs: 8, SelectedShapeGPUs: 8, Model: "A100-SXM4-80GB", VRAMPerGPU: 81920}, + wantErr: "set hardware.totalGpus explicitly", + }, + { + name: "vramMb override requires an explicit totalGpus", + hardware: &nvidiacomv1beta1.HardwareSpec{ + GPUSKU: "a100_sxm", VRAMMB: new(40960.0), NumGPUsPerNode: new(int32(8)), + }, + discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, TotalGPUs: 8, SelectedShapeGPUs: 8, Model: "A100-SXM4-80GB", VRAMPerGPU: 81920}, + wantErr: "set hardware.totalGpus explicitly", }, { name: "no fields set, discovery fails", @@ -280,9 +288,9 @@ func TestEnrichHardwareFromDiscovery_TotalGPUs(t *testing.T) { explicitTotal *int32 wantTotal int32 }{ - {name: "heterogeneous nodes", counts: []int{8, 4}, wantTotal: 12}, - {name: "aggregate below cap despite larger maximum times nodes", counts: []int{8, 8, 8, 4, 2}, wantTotal: 30}, - {name: "aggregate above cap", counts: []int{8, 8, 8, 8, 4}, wantTotal: 32}, + {name: "different node sizes use only the selected shape", counts: []int{8, 4}, wantTotal: 8}, + {name: "matching selected shapes are aggregated", counts: []int{8, 8, 8, 4, 2}, wantTotal: 24}, + {name: "selected shape aggregate is capped", counts: []int{8, 8, 8, 8, 4}, wantTotal: 32}, {name: "explicit total above cap", counts: []int{8, 4}, explicitTotal: new(int32(64)), wantTotal: 64}, } for _, tt := range tests { @@ -318,16 +326,17 @@ func TestEnrichHardwareFromDiscovery_WritesOptionalHardwareMetadata(t *testing.T r.GPUDiscovery = gpupkg.NewGPUDiscovery(nil) r.GPUDiscoveryCache = gpupkg.NewGPUDiscoveryCache() r.GPUDiscoveryCache.Set("", &gpupkg.GPUInfo{ - NodeName: "n1", - GPUsPerNode: 8, - NodesWithGPUs: 2, - TotalGPUs: 16, - Model: "NVIDIA H100 80GB HBM3", - VRAMPerGPU: 81079, - System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, - Interconnect: "pcie", - RDMAEnabled: true, - RDMAType: "rdma", + NodeName: "n1", + GPUsPerNode: 8, + NodesWithGPUs: 2, + TotalGPUs: 16, + SelectedShapeGPUs: 16, + Model: "NVIDIA H100 80GB HBM3", + VRAMPerGPU: 81079, + System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, + Interconnect: "pcie", + RDMAEnabled: true, + RDMAType: "rdma", }, time.Minute) dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ @@ -351,16 +360,17 @@ func TestEnrichHardwareFromDiscovery_FillsOptionalMetadataWhenRequiredFieldsSet( r.GPUDiscovery = gpupkg.NewGPUDiscovery(nil) r.GPUDiscoveryCache = gpupkg.NewGPUDiscoveryCache() r.GPUDiscoveryCache.Set(nvidiacomv1beta1.GPUSKUTypeH100PCIe, &gpupkg.GPUInfo{ - NodeName: "n1", - GPUsPerNode: 8, - NodesWithGPUs: 4, - TotalGPUs: 32, - Model: "NVIDIA H100 80GB HBM3", - VRAMPerGPU: 81079, - System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, - Interconnect: "pcie", - RDMAEnabled: true, - RDMAType: "rdma", + NodeName: "n1", + GPUsPerNode: 8, + NodesWithGPUs: 4, + TotalGPUs: 32, + SelectedShapeGPUs: 32, + Model: "NVIDIA H100 80GB HBM3", + VRAMPerGPU: 81079, + System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, + Interconnect: "pcie", + RDMAEnabled: true, + RDMAType: "rdma", }, time.Minute) dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ @@ -406,16 +416,17 @@ func TestCreateProfilingJobPersistsDiscoveredHardware(t *testing.T) { cache := gpupkg.NewGPUDiscoveryCache() cache.Set("", &gpupkg.GPUInfo{ - NodeName: "n1", - GPUsPerNode: 8, - NodesWithGPUs: 2, - TotalGPUs: 16, - Model: "NVIDIA H100 80GB HBM3", - VRAMPerGPU: 81079, - System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, - Interconnect: "pcie", - RDMAEnabled: true, - RDMAType: "rdma", + NodeName: "n1", + GPUsPerNode: 8, + NodesWithGPUs: 2, + TotalGPUs: 16, + SelectedShapeGPUs: 16, + Model: "NVIDIA H100 80GB HBM3", + VRAMPerGPU: 81079, + System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, + Interconnect: "pcie", + RDMAEnabled: true, + RDMAType: "rdma", }, time.Minute) r := &DynamoGraphDeploymentRequestReconciler{ @@ -555,11 +566,13 @@ func TestEnrichHardwareFromDiscovery_NormalizesBareModelFromDCGM(t *testing.T) { // DCGM metrics lack a form factor suffix (e.g. "NVIDIA H200"). mockScraper := func(_ context.Context, _ string) (*gpupkg.GPUInfo, error) { return &gpupkg.GPUInfo{ - NodeName: "gpu-node", - GPUsPerNode: 8, - Model: tt.dcgmModel, - VRAMPerGPU: 143770, - System: "", + NodeName: "gpu-node", + GPUsPerNode: 8, + TotalGPUs: 8, + SelectedShapeGPUs: 8, + Model: tt.dcgmModel, + VRAMPerGPU: 143770, + System: "", }, nil } diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go index e553776ba575..57e762506642 100644 --- a/deploy/operator/internal/gpu/discovery.go +++ b/deploy/operator/internal/gpu/discovery.go @@ -166,21 +166,22 @@ var gpuRules = []gpuRule{ // GPUInfo contains discovered GPU configuration from cluster nodes type GPUInfo struct { - NodeName string // Name of the node with this GPU configuration - GPUsPerNode int // Maximum GPUs per node with the selected SKU - NodesWithGPUs int // Number of discovered nodes with the selected SKU - TotalGPUs int // Sum of GPUs across discovered nodes with the selected SKU, before any budget cap - Model string // GPU product name (e.g., "H100-SXM5-80GB") - VRAMPerGPU int // VRAM in MiB per GPU - System nvidiacomv1beta1.GPUSKUType // AIC hardware system identifier (e.g., "h100_sxm", "h200_sxm"), empty if unknown - MIGEnabled bool // True if MIG is enabled (inferred from model or additional labels, not implemented in this version) - MIGProfiles map[string]int // Optional: map of MIG profile name to count (requires additional label parsing, not implemented in this version) - CloudProvider string // aws | gcp | aks | other | unknown - RDMAEnabled bool // Indicates whether RDMA is enabled for this node (e.g., via InfiniBand, RoCE, or similar high-speed networking) - RDMAType string // Type of RDMA transport detected (e.g., "infiniband", "roce", "rdma", "sriov", or "none") - Interconnect string // Primary GPU-to-GPU interconnect technology used within the node (e.g., "nvlink" for high-bandwidth links or "pcie" for standard bus-based communication) - InterconnectTier string // Qualitative or platform-specific classification of the interconnect (e.g., NVLink generation, topology tier, or vendor-defined performance level) - NVLinkLinks int // Number of NVLink connections per GPU (0 if NVLink is not present or interconnect is PCIe-only) + NodeName string // Name of the node with this GPU configuration + GPUsPerNode int // Maximum GPUs per node with the selected SKU + NodesWithGPUs int // Number of discovered nodes with the selected SKU + TotalGPUs int // Sum of GPUs across discovered nodes with the selected SKU, before any budget cap + SelectedShapeGPUs int // Sum of GPUs on nodes matching the selected SKU, GPU count, and VRAM per GPU + Model string // GPU product name (e.g., "H100-SXM5-80GB") + VRAMPerGPU int // VRAM in MiB per GPU + System nvidiacomv1beta1.GPUSKUType // AIC hardware system identifier (e.g., "h100_sxm", "h200_sxm"), empty if unknown + MIGEnabled bool // True if MIG is enabled (inferred from model or additional labels, not implemented in this version) + MIGProfiles map[string]int // Optional: map of MIG profile name to count (requires additional label parsing, not implemented in this version) + CloudProvider string // aws | gcp | aks | other | unknown + RDMAEnabled bool // Indicates whether RDMA is enabled for this node (e.g., via InfiniBand, RoCE, or similar high-speed networking) + RDMAType string // Type of RDMA transport detected (e.g., "infiniband", "roce", "rdma", "sriov", or "none") + Interconnect string // Primary GPU-to-GPU interconnect technology used within the node (e.g., "nvlink" for high-bandwidth links or "pcie" for standard bus-based communication) + InterconnectTier string // Qualitative or platform-specific classification of the interconnect (e.g., NVLink generation, topology tier, or vendor-defined performance level) + NVLinkLinks int // Number of NVLink connections per GPU (0 if NVLink is not present or interconnect is PCIe-only) } type ScrapeMetricsFunc func(ctx context.Context, endpoint string) (*GPUInfo, error) @@ -260,8 +261,10 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // // When filterSKU is non-empty, only nodes whose inferred SKU matches are // considered. When empty, the best node is selected first (highest GPU count, -// then VRAM). Each node with the selected SKU contributes once to NodesWithGPUs, -// and its GPU count is summed into TotalGPUs. +// then VRAM). Each node with the selected GPU type contributes once to NodesWithGPUs, +// and its GPU count is summed into TotalGPUs. Known types are grouped by SKU; unknown +// types are grouped by normalized model name. SelectedShapeGPUs counts only the nodes +// that also match the selected GPU count and VRAM per GPU. // // The function performs the following: // @@ -419,14 +422,15 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, // single negative node from masking RDMA on another node. nodesWithGPUs := 0 totalGPUs := 0 + selectedShapeGPUs := 0 var rdmaDetected bool var rdmaType string for _, n := range allNodes { - if n.sku != bestSKU { + if !matchesSelectedGPUType(n.sku, bestSKU, n.info.Model, bestNode.Model) { continue } nodesWithGPUs++ - totalGPUs += n.info.GPUsPerNode + totalGPUs, selectedShapeGPUs = accumulateGPUCapacity(totalGPUs, selectedShapeGPUs, n.info, bestNode) if !rdmaDetected { rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.nodeName) if rdma { @@ -452,6 +456,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, bestNode.CloudProvider = cloudProvider bestNode.NodesWithGPUs = nodesWithGPUs bestNode.TotalGPUs = totalGPUs + bestNode.SelectedShapeGPUs = selectedShapeGPUs bestNode.RDMAEnabled = rdmaDetected bestNode.RDMAType = rdmaType @@ -817,7 +822,9 @@ func DiscoverGPUs(ctx context.Context, k8sClient client.Reader) (*GPUInfo, error // // When filterSKU is non-empty, only nodes whose inferred SKU matches are // considered for selection and counting. When empty, the best node is selected -// first and then only nodes with the same SKU are counted. +// first and then only nodes of the same GPU type are counted. Known types are grouped +// by SKU; unknown types are grouped by normalized model name. SelectedShapeGPUs counts +// only nodes that also match the selected GPU count and VRAM per GPU. // // This function requires cluster-wide node read permissions and expects nodes // to have GFD labels. If no nodes with GPU labels are found, it returns an error. @@ -891,12 +898,13 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK // single negative node from masking RDMA on another node. nodesWithGPUs := 0 totalGPUs := 0 + selectedShapeGPUs := 0 var rdmaDetected bool var rdmaType string for _, n := range allNodes { - if n.sku == bestSKU { + if matchesSelectedGPUType(n.sku, bestSKU, n.info.Model, bestNode.Model) { nodesWithGPUs++ - totalGPUs += n.info.GPUsPerNode + totalGPUs, selectedShapeGPUs = accumulateGPUCapacity(totalGPUs, selectedShapeGPUs, n.info, bestNode) if !rdmaDetected { rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.info.NodeName) if rdma { @@ -918,12 +926,14 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK bestNode.System = bestSKU bestNode.NodesWithGPUs = nodesWithGPUs bestNode.TotalGPUs = totalGPUs + bestNode.SelectedShapeGPUs = selectedShapeGPUs bestNode.RDMAEnabled = rdmaDetected bestNode.RDMAType = rdmaType logger.Info("GPU discovery completed", "gpusPerNode", bestNode.GPUsPerNode, "nodesWithGPUs", bestNode.NodesWithGPUs, "totalGpus", bestNode.TotalGPUs, + "selectedShapeGpus", bestNode.SelectedShapeGPUs, "model", bestNode.Model, "vram", bestNode.VRAMPerGPU, "system", bestNode.System, @@ -932,6 +942,21 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK return bestNode, nil } +func accumulateGPUCapacity(total, selected int, node, selectedShape *GPUInfo) (int, int) { + total += node.GPUsPerNode + if node.GPUsPerNode == selectedShape.GPUsPerNode && node.VRAMPerGPU == selectedShape.VRAMPerGPU { + selected += node.GPUsPerNode + } + return total, selected +} + +func matchesSelectedGPUType(discoveredSKU, selectedSKU nvidiacomv1beta1.GPUSKUType, discoveredModel, selectedModel string) bool { + if discoveredSKU != selectedSKU { + return false + } + return selectedSKU != "" || normalize(discoveredModel) == normalize(selectedModel) +} + // extractGPUInfoFromNode extracts GPU information from a single node's labels. // Returns error if required labels are missing or invalid. func extractGPUInfoFromNode(node *corev1.Node) (*GPUInfo, error) { diff --git a/deploy/operator/internal/gpu/discovery_test.go b/deploy/operator/internal/gpu/discovery_test.go index ce15c4effcef..437e5f11ab0f 100644 --- a/deploy/operator/internal/gpu/discovery_test.go +++ b/deploy/operator/internal/gpu/discovery_test.go @@ -55,20 +55,26 @@ func TestGPUDiscovery_TotalGPUs(t *testing.T) { for _, source := range []string{"node labels", "DCGM"} { t.Run(source, func(t *testing.T) { tests := []struct { - name string - h100Counts []int - a100Counts []int - filterSKU nvidiacomv1beta1.GPUSKUType - wantSKU nvidiacomv1beta1.GPUSKUType - wantPerNode int - wantNodes int - wantTotal int + name string + h100Counts []int + h100VRAM []int + h100Models []string + a100Counts []int + filterSKU nvidiacomv1beta1.GPUSKUType + wantSKU nvidiacomv1beta1.GPUSKUType + wantPerNode int + wantNodes int + wantTotal int + wantSelected int }{ - {name: "heterogeneous nodes", h100Counts: []int{8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, - {name: "maximum selected after smaller node", h100Counts: []int{4, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, - {name: "unfiltered excludes other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, - {name: "filter selects other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, filterSKU: "a100_sxm", wantSKU: "a100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 10}, - {name: "discovery does not cap the total", h100Counts: []int{8, 8, 8, 8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 5, wantTotal: 36}, + {name: "different node sizes", h100Counts: []int{8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12, wantSelected: 8}, + {name: "maximum selected after smaller node", h100Counts: []int{4, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12, wantSelected: 8}, + {name: "same selected shape", h100Counts: []int{8, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 16, wantSelected: 16}, + {name: "different VRAM per GPU", h100Counts: []int{8, 8}, h100VRAM: []int{81920, 40960}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 16, wantSelected: 8}, + {name: "unknown models remain separate GPU types", h100Counts: []int{8, 8}, h100Models: []string{"FutureGPU-A", "FutureGPU-B"}, wantPerNode: 8, wantNodes: 1, wantTotal: 8, wantSelected: 8}, + {name: "unfiltered excludes other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12, wantSelected: 8}, + {name: "filter selects other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, filterSKU: "a100_sxm", wantSKU: "a100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 10, wantSelected: 8}, + {name: "discovery does not cap totals", h100Counts: []int{8, 8, 8, 8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 5, wantTotal: 36, wantSelected: 32}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -79,17 +85,27 @@ func TestGPUDiscovery_TotalGPUs(t *testing.T) { model string vram int counts []int + vrams []int + models []string }{ - {model: "H100-SXM5-80GB", vram: 81920, counts: tt.h100Counts}, + {model: "H100-SXM5-80GB", vram: 81920, counts: tt.h100Counts, vrams: tt.h100VRAM, models: tt.h100Models}, {model: "A100-SXM4-40GB", vram: 40960, counts: tt.a100Counts}, } { - for _, count := range group.counts { + for index, count := range group.counts { + vram := group.vram + if index < len(group.vrams) { + vram = group.vrams[index] + } + model := group.model + if index < len(group.models) { + model = group.models[index] + } name := fmt.Sprintf("gpu-node-%d", len(metrics)) ip := fmt.Sprintf("10.0.0.%d", len(metrics)+1) objects = append(objects, &corev1.Node{ Name: name, Labels: map[string]string{ - LabelGPUCount: fmt.Sprint(count), LabelGPUProduct: group.model, LabelGPUMemory: fmt.Sprint(group.vram), + LabelGPUCount: fmt.Sprint(count), LabelGPUProduct: model, LabelGPUMemory: fmt.Sprint(vram), }, }, &corev1.Pod{ Name: name, Namespace: "gpu-operator", @@ -97,7 +113,7 @@ func TestGPUDiscovery_TotalGPUs(t *testing.T) { Spec: corev1.PodSpec{NodeName: name}, Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: ip}, }) - metrics[buildDCGMEndpoint(ip)] = GPUInfo{NodeName: name, GPUsPerNode: count, Model: group.model, VRAMPerGPU: group.vram} + metrics[buildDCGMEndpoint(ip)] = GPUInfo{NodeName: name, GPUsPerNode: count, Model: model, VRAMPerGPU: vram} } } k8sClient := newFakeClient(objects...) @@ -122,6 +138,7 @@ func TestGPUDiscovery_TotalGPUs(t *testing.T) { assert.Equal(t, tt.wantPerNode, info.GPUsPerNode) assert.Equal(t, tt.wantNodes, info.NodesWithGPUs) assert.Equal(t, tt.wantTotal, info.TotalGPUs) + assert.Equal(t, tt.wantSelected, info.SelectedShapeGPUs) }) } }) @@ -1192,6 +1209,7 @@ func TestDiscoverGPUsFromDCGM_DeduplicatesNodes(t *testing.T) { assert.Equal(t, tt.wantVRAM, info.VRAMPerGPU) assert.Equal(t, 2, info.NodesWithGPUs) assert.Equal(t, 12, info.TotalGPUs) + assert.Equal(t, 8, info.SelectedShapeGPUs) }) } } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index c416c42c527f..9bc8620b9140 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2543,10 +2543,10 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected by selecting the node with the most GPUs,
then highest VRAM per GPU. In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| -| `vramMb` _float_ | VRAMMB is the VRAM per GPU in MiB.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
| -| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by summing GPUs on discovered nodes with the
selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
| -| `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
| +| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected as the GPU type of the node with the most GPUs
(ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| +| `vramMb` _float_ | VRAMMB is the VRAM per GPU in MiB.
When omitted, auto-detected from cluster GPU nodes. When this value differs
from the auto-selected shape and totalGpus is omitted, the request is rejected. | | Optional: \{\}
| +| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by summing GPUs on nodes matching the selected
shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily
capped at 32 to limit profiler search space. For example, same-type nodes
with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU
shape yields an automatic budget of 8. This cap may be removed in a future
release. Set this field explicitly to override the derived budget or use a
numGpusPerNode or vramMb value that differs from the selected shape. A shape
override without an explicit totalGpus is rejected. | | Optional: \{\}
| +| `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected as the highest GPU count on any discovered node
of the selected GPU type. In clusters whose nodes have different GPU counts,
set this explicitly to the node size you intend to schedule on. When this value
differs from the auto-selected shape and totalGpus is omitted, the request is rejected. | | Optional: \{\}
| | `interconnect` _string_ | Interconnect describes the primary GPU-to-GPU interconnect *within a node*.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT configure or enable any GPU interconnect; it only describes what is available/assumed.
- When omitted, the operator may attempt best-effort discovery (currently distinguishes "nvlink"
vs "pcie" based on DCGM NVLink link count). If discovery is unavailable, it may remain empty.
Impact of wrong / missing values:
- If set more optimistically than reality (e.g., "nvlink" when only PCIe is present), performance
models may overestimate intra-node bandwidth and choose overly aggressive parallelism or layouts,
resulting in degraded performance compared to expectations.
- If set more pessimistically than reality (e.g., "pcie" when NVLink is present), the system may
choose conservative plans and leave performance on the table.
- If unset and undiscovered, consumers should treat the interconnect as unknown and fall back to
conservative assumptions.
Example values: "pcie", "nvlink". Other values may be accepted but may not be auto-detected. | | Optional: \{\}
| | `rdma` _boolean_ | RDMA indicates whether the cluster has RDMA-capable networking available for Dynamo data movement.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT install, enable, or configure RDMA (e.g., drivers, SR-IOV, NVIDIA network operator,
GPUDirect settings). It only expresses availability/intent.
- When omitted, the operator may attempt best-effort discovery (e.g., via node labels indicating
RDMA/SR-IOV capability and/or presence of NVIDIA network-operator RDMA components). If discovery
is unavailable, it may remain unset.
Impact of wrong / missing values:
- False positive (set true when RDMA is not actually usable end-to-end) may cause plans or
deployments to assume RDMA is available; depending on the runtime transport selection and
fallback behavior, this can lead to connection/setup failures or performance regressions.
- False negative (set false when RDMA is available) will typically avoid RDMA-optimized paths and
fall back to non-RDMA transports, usually remaining functional but potentially slower.
- If unset and undiscovered, consumers should treat RDMA availability as unknown and use
conservative defaults / fallback transports. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index e2dd516d6571..b45b6b214a0d 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -142,16 +142,16 @@ service. - GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes. + GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes. - VRAM per GPU in MiB. Auto-detected from cluster GPU nodes when omitted. + VRAM per GPU in MiB. Auto-detected from cluster GPU nodes when omitted. If this value differs from the auto-selected shape and `totalGpus` is omitted, the request is rejected. - GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by `gpuSku` when set), temporarily capped at 32 to limit the profiler search space. For example, matching nodes with 8 and 4 GPUs yield a budget of 12, while `numGpusPerNode` remains 8. Set explicitly to override the cap. + GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by summing GPUs on nodes matching the auto-selected shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily capped at 32 to limit the profiler search space. This reflects discovered GPUs, not currently free capacity. For example, same-type nodes with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU shape yields an automatic budget of 8. Set explicitly to override the derived budget or use a `numGpusPerNode` or `vramMb` value that differs from the selected shape; a shape override without `totalGpus` is rejected. - GPUs per node. Auto-detected when omitted. + GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. If this value differs from the auto-selected shape and `totalGpus` is omitted, the request is rejected. Primary GPU-to-GPU interconnect within a node, for example `nvlink` or `pcie`. Capability metadata for profiling and planning only — it does not enable any interconnect. Best-effort discovery distinguishes `nvlink` from `pcie` when omitted. An over-optimistic value can lead the planner to overestimate intra-node bandwidth and pick overly aggressive parallelism. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 1baae47908d9..2126898ae33b 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -3799,23 +3799,23 @@ If all four fields are set, discovery is skipped. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta1-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta1-dynamographdeploymentrequestspec) -GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. +GPUSKU selects the GPU type to target. When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. See [GPUSKUType](#gpuskutype). **Validation:** Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300] Optional: \{\} -VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes. +VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes. When this value differs from the auto-selected shape and totalGpus is omitted, the request is rejected. **Validation:** Optional: \{\} -TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. +TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on nodes matching the selected shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily capped at 32 to limit profiler search space. For example, same-type nodes with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU shape yields an automatic budget of 8. This cap may be removed in a future release. Set this field explicitly to override the derived budget or use a numGpusPerNode or vramMb value that differs from the selected shape. A shape override without an explicit totalGpus is rejected. **Validation:** Optional: \{\} -NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected from cluster GPU nodes. +NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on. When this value differs from the auto-selected shape and totalGpus is omitted, the request is rejected. **Validation:** Optional: \{\} From f4a03e2582104f8f5c771035a52c7fa3ba23db39 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 23:00:28 +0100 Subject: [PATCH 005/100] pr-17262: pr-1: tolerate nominal VRAM discovery [upstream: Pranjulcr7/dynamo#1] Signed-off-by: Dr. Stefan Schimanski --- .../src/dynamo/profiler/utils/dgdr_v1beta1_types.py | 2 +- .../api/v1beta1/dynamographdeploymentrequest_types.go | 5 +++-- .../bases/nvidia.com_dynamographdeploymentrequests.yaml | 5 +++-- .../controller/dynamographdeploymentrequest_controller.go | 8 +++++++- .../operator/internal/controller/enrich_hardware_test.go | 8 ++++++++ .../additional-resources/api-reference-k8s.md | 2 +- .../pages/reference/kubernetes-api/full-api-reference.mdx | 2 +- 7 files changed, 24 insertions(+), 8 deletions(-) diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py index b686ebb2b46f..d86747414952 100644 --- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py +++ b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py @@ -233,7 +233,7 @@ class HardwareSpec(BaseModel): ) vramMb: Optional[float] = Field( default=None, - description="VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes. When this value differs from the auto-selected shape and totalGpus is omitted, the request is rejected.", + description="VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes. Nominal values within 10% of the measured value identify the same shape. When this value differs by more than 10% from the auto-selected shape and totalGpus is omitted, the request is rejected.", ) totalGpus: Optional[int] = Field( default=None, diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go index 645ab6643bae..a4e1196af6b9 100644 --- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go @@ -417,8 +417,9 @@ type HardwareSpec struct { GPUSKU GPUSKUType `json:"gpuSku,omitempty"` // VRAMMB is the VRAM per GPU in MiB. - // When omitted, auto-detected from cluster GPU nodes. When this value differs - // from the auto-selected shape and totalGpus is omitted, the request is rejected. + // When omitted, auto-detected from cluster GPU nodes. Nominal values within 10% + // of the measured value identify the same shape. When this value differs by more + // than 10% from the auto-selected shape and totalGpus is omitted, the request is rejected. // +optional VRAMMB *float64 `json:"vramMb,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 66b313d02c50..af2a4fb488ec 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -737,8 +737,9 @@ spec: vramMb: description: |- VRAMMB is the VRAM per GPU in MiB. - When omitted, auto-detected from cluster GPU nodes. When this value differs - from the auto-selected shape and totalGpus is omitted, the request is rejected. + When omitted, auto-detected from cluster GPU nodes. Nominal values within 10% + of the measured value identify the same shape. When this value differs by more + than 10% from the auto-selected shape and totalGpus is omitted, the request is rejected. type: number type: object image: diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index 4282b75f8f53..342e5a436d7f 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -1975,12 +1975,18 @@ func validateAutomaticGPUShape(hw *nvidiacomv1beta1.HardwareSpec, gpuInfo *gpu.G if hw.NumGPUsPerNode != nil && int(*hw.NumGPUsPerNode) != gpuInfo.GPUsPerNode { return fmt.Errorf("cannot auto-detect hardware.totalGpus: hardware.numGpusPerNode=%d does not match the selected discovered shape with %d GPUs per node; set hardware.totalGpus explicitly", *hw.NumGPUsPerNode, gpuInfo.GPUsPerNode) } - if hw.VRAMMB != nil && *hw.VRAMMB != float64(gpuInfo.VRAMPerGPU) { + if hw.VRAMMB != nil && !sameVRAMShape(*hw.VRAMMB, gpuInfo.VRAMPerGPU) { return fmt.Errorf("cannot auto-detect hardware.totalGpus: hardware.vramMb=%g does not match the selected discovered shape with %d MiB per GPU; set hardware.totalGpus explicitly", *hw.VRAMMB, gpuInfo.VRAMPerGPU) } return nil } +// sameVRAMShape accepts nominal memory sizes while rejecting a materially different GPU variant. +func sameVRAMShape(requestedMiB float64, discoveredMiB int) bool { + discovered := float64(discoveredMiB) + return max(requestedMiB-discovered, discovered-requestedMiB) <= discovered/10 +} + func (r *DynamoGraphDeploymentRequestReconciler) discoverHardwareForEnrichment(ctx context.Context, hw *nvidiacomv1beta1.HardwareSpec, discoveryRequired bool) (*gpu.GPUInfo, error) { logger := log.FromContext(ctx) logger.Info("Attempting GPU discovery for profiling job") diff --git a/deploy/operator/internal/controller/enrich_hardware_test.go b/deploy/operator/internal/controller/enrich_hardware_test.go index f24f4f9fc918..8f751efd0a38 100644 --- a/deploy/operator/internal/controller/enrich_hardware_test.go +++ b/deploy/operator/internal/controller/enrich_hardware_test.go @@ -198,6 +198,14 @@ func TestEnrichHardwareFromDiscovery(t *testing.T) { discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, TotalGPUs: 8, SelectedShapeGPUs: 8, Model: "B200-SXM-180GB", VRAMPerGPU: 141312}, wantGPUSKU: "b200_sxm", wantVRAM: 141312, wantGPUsNode: 8, wantTotalGPUs: 8, wantChanged: true, }, + { + name: "nominal vramMb near measured VRAM derives totalGpus", + hardware: &nvidiacomv1beta1.HardwareSpec{ + GPUSKU: "h100_sxm", VRAMMB: new(81920.0), NumGPUsPerNode: new(int32(8)), + }, + discoveredGPU: &gpupkg.GPUInfo{NodeName: "n1", GPUsPerNode: 8, TotalGPUs: 8, SelectedShapeGPUs: 8, Model: "NVIDIA H100 80GB HBM3", VRAMPerGPU: 81079}, + wantGPUSKU: "h100_sxm", wantVRAM: 81920, wantGPUsNode: 8, wantTotalGPUs: 8, wantChanged: true, + }, { name: "only gpuSku missing, discovery fills it", hardware: &nvidiacomv1beta1.HardwareSpec{ diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 9bc8620b9140..3a476e4ae4fc 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2544,7 +2544,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected as the GPU type of the node with the most GPUs
(ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| -| `vramMb` _float_ | VRAMMB is the VRAM per GPU in MiB.
When omitted, auto-detected from cluster GPU nodes. When this value differs
from the auto-selected shape and totalGpus is omitted, the request is rejected. | | Optional: \{\}
| +| `vramMb` _float_ | VRAMMB is the VRAM per GPU in MiB.
When omitted, auto-detected from cluster GPU nodes. Nominal values within 10%
of the measured value identify the same shape. When this value differs by more
than 10% from the auto-selected shape and totalGpus is omitted, the request is rejected. | | Optional: \{\}
| | `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by summing GPUs on nodes matching the selected
shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily
capped at 32 to limit profiler search space. For example, same-type nodes
with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU
shape yields an automatic budget of 8. This cap may be removed in a future
release. Set this field explicitly to override the derived budget or use a
numGpusPerNode or vramMb value that differs from the selected shape. A shape
override without an explicit totalGpus is rejected. | | Optional: \{\}
| | `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected as the highest GPU count on any discovered node
of the selected GPU type. In clusters whose nodes have different GPU counts,
set this explicitly to the node size you intend to schedule on. When this value
differs from the auto-selected shape and totalGpus is omitted, the request is rejected. | | Optional: \{\}
| | `interconnect` _string_ | Interconnect describes the primary GPU-to-GPU interconnect *within a node*.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT configure or enable any GPU interconnect; it only describes what is available/assumed.
- When omitted, the operator may attempt best-effort discovery (currently distinguishes "nvlink"
vs "pcie" based on DCGM NVLink link count). If discovery is unavailable, it may remain empty.
Impact of wrong / missing values:
- If set more optimistically than reality (e.g., "nvlink" when only PCIe is present), performance
models may overestimate intra-node bandwidth and choose overly aggressive parallelism or layouts,
resulting in degraded performance compared to expectations.
- If set more pessimistically than reality (e.g., "pcie" when NVLink is present), the system may
choose conservative plans and leave performance on the table.
- If unset and undiscovered, consumers should treat the interconnect as unknown and fall back to
conservative assumptions.
Example values: "pcie", "nvlink". Other values may be accepted but may not be auto-detected. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 2126898ae33b..e262b3c7644f 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -3805,7 +3805,7 @@ See [GPUSKUType](#gpuskutype). -VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes. When this value differs from the auto-selected shape and totalGpus is omitted, the request is rejected. +VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes. Nominal values within 10% of the measured value identify the same shape. When this value differs by more than 10% from the auto-selected shape and totalGpus is omitted, the request is rejected. **Validation:** Optional: \{\} From 99cb26e4d6ba7e4c450f6a16fa329b0a1580b787 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:27:53 +0100 Subject: [PATCH 006/100] pr-16661: squash v1beta2 DGDR admission Squashed from https://github.com/ai-dynamo/dynamo/pull/16661. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .../operator/templates/deployment.yaml | 6 + .../templates/webhook-configuration.yaml | 2 + .../platform/components/operator/values.yaml | 10 + .../namespace_restriction_webhooks_test.yaml | 20 + .../dynamographdeploymentrequest_types.go | 43 +- deploy/operator/cmd/main.go | 46 +- ...dia.com_dynamographdeploymentrequests.yaml | 49 +- .../nvidia.com_dynamographdeploymentruns.yaml | 49 +- .../controller/aisimulate/defaults.go | 66 ++ .../internal/controller/aisimulate/doc.go | 38 + .../internal/controller/aisimulate/image.go | 62 ++ .../controller/aisimulate/image_test.go | 52 ++ .../internal/controller/aisimulate/input.go | 250 ++++++ .../controller/aisimulate/input_test.go | 433 ++++++++++ .../internal/controller/aisimulate/paths.go | 59 ++ .../aisimulate/release_contract_test.go | 78 ++ ...dynamographdeploymentrequest_controller.go | 90 +- .../internal/controller/suite_envtest_test.go | 18 + .../internal/dynamo/component_common.go | 4 +- .../dynamo/component_frontend_test.go | 10 +- .../dynamo/component_namespace_test.go | 12 +- .../internal/dynamo/component_worker_test.go | 18 +- deploy/operator/internal/dynamo/graph.go | 10 +- deploy/operator/internal/dynamo/hash.go | 6 +- .../internal/features/runtime/gate.go | 6 +- .../internal/features/runtime/gate_test.go | 12 +- .../internal/features/runtime/gates.go | 13 +- .../internal/gpu/dcgm_phase_context_test.go | 41 + deploy/operator/internal/gpu/discovery.go | 10 + .../internal/gpu/hardware_discoverer.go | 162 ++++ .../internal/gpu/hardware_discoverer_test.go | 90 ++ deploy/operator/internal/gpu/heterogeneous.go | 51 ++ .../internal/gpu/heterogeneous_test.go | 56 ++ .../versionoverride.go} | 8 +- .../versionoverride_test.go} | 2 +- .../dynamographdeploymentrequest_handler.go | 41 + ...namographdeploymentrequest_handler_test.go | 7 +- ...namographdeploymentrequest_routing_test.go | 146 ++++ ...ographdeploymentrequest_v1beta1_options.go | 13 + .../dynamographdeploymentrequest_v1beta2.go | 263 ++++++ ...graphdeploymentrequest_v1beta2_artifact.go | 57 ++ ...deploymentrequest_v1beta2_artifact_test.go | 120 +++ ...deploymentrequest_v1beta2_hardware_test.go | 117 +++ ...ographdeploymentrequest_v1beta2_helpers.go | 201 +++++ ...namographdeploymentrequest_v1beta2_test.go | 312 +++++++ .../internal/webhook/request_version.go | 35 + .../internal/webhook/request_version_test.go | 57 ++ .../operator/internal/webhook/setup/setup.go | 12 +- .../dynamographdeploymentrequest.go | 74 +- ...amographdeploymentrequest_boundary_test.go | 39 + ...hdeploymentrequest_carrier_envtest_test.go | 101 +++ .../dynamographdeploymentrequest_routing.go | 67 ++ .../dynamographdeploymentrequest_v1beta2.go | 82 ++ ...namographdeploymentrequest_v1beta2_test.go | 162 ++++ ...ploymentrequest_validation_envtest_test.go | 796 +++++++++++++++++- .../validation/planner_runtime_features.go | 4 +- .../webhook/validation/shared_helpers.go | 6 +- .../webhook/validation/suite_envtest_test.go | 43 + .../additional-resources/api-reference-k8s.md | 30 +- .../kubernetes-api/full-api-reference.mdx | 36 +- 60 files changed, 4436 insertions(+), 267 deletions(-) create mode 100644 deploy/operator/internal/controller/aisimulate/defaults.go create mode 100644 deploy/operator/internal/controller/aisimulate/doc.go create mode 100644 deploy/operator/internal/controller/aisimulate/image.go create mode 100644 deploy/operator/internal/controller/aisimulate/image_test.go create mode 100644 deploy/operator/internal/controller/aisimulate/input.go create mode 100644 deploy/operator/internal/controller/aisimulate/input_test.go create mode 100644 deploy/operator/internal/controller/aisimulate/paths.go create mode 100644 deploy/operator/internal/controller/aisimulate/release_contract_test.go create mode 100644 deploy/operator/internal/gpu/dcgm_phase_context_test.go create mode 100644 deploy/operator/internal/gpu/hardware_discoverer.go create mode 100644 deploy/operator/internal/gpu/hardware_discoverer_test.go create mode 100644 deploy/operator/internal/gpu/heterogeneous.go create mode 100644 deploy/operator/internal/gpu/heterogeneous_test.go rename deploy/operator/internal/{runtimeversion/runtimeversion.go => versionoverride/versionoverride.go} (91%) rename deploy/operator/internal/{runtimeversion/runtimeversion_test.go => versionoverride/versionoverride_test.go} (99%) create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact_test.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go create mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go create mode 100644 deploy/operator/internal/webhook/request_version.go create mode 100644 deploy/operator/internal/webhook/request_version_test.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index 8a4ea7dd9080..483d9e381207 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -92,6 +92,12 @@ spec: {{- with .Values.controllerManager.manager.dgdrDefaultImage }} - {{ printf "--dgdr-default-image=%s" . | quote }} {{- end }} + {{- with .Values.controllerManager.manager.dgdrAISimulateDefaultImage }} + - {{ printf "--dgdr-aisimulate-default-image=%s" . | quote }} + {{- end }} + {{- with .Values.controllerManager.manager.dgdrAISimulateDefaultVersionOverride }} + - {{ printf "--dgdr-aisimulate-default-version-override=%s" . | quote }} + {{- end }} command: - /manager env: diff --git a/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml b/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml index 5f6880fe46e6..770627f942b3 100644 --- a/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/webhook-configuration.yaml @@ -150,6 +150,7 @@ webhooks: namespace: {{ .Release.Namespace }} path: /validate-nvidia-com-v1beta1-dynamographdeploymentrequest failurePolicy: {{ .Values.webhook.failurePolicy }} + matchPolicy: Equivalent name: vdynamographdeploymentrequest.kb.io {{- if .Values.webhook.namespaceSelector }} namespaceSelector: @@ -274,6 +275,7 @@ webhooks: namespace: {{ .Release.Namespace }} path: /mutate-nvidia-com-v1beta1-dynamographdeploymentrequest failurePolicy: {{ .Values.webhook.failurePolicy }} + matchPolicy: Equivalent name: mdynamographdeploymentrequestv1beta1.kb.io {{- if .Values.webhook.namespaceSelector }} namespaceSelector: diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index c526a93bebde..007019cb491e 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -115,6 +115,16 @@ controllerManager: # derives dynamo-planner:, which only exists for GA # releases. dgdrDefaultImage: "" + # AISimulate runner image put into v1beta2 DGDR spec.aisimulate.image when unset. + # Empty derives nvcr.io/nvidia/ai-dynamo/dynamo-planner:. + dgdrAISimulateDefaultImage: "" + # AISimulate version bundled in the default image (for example 0.13.0), put into v1beta2 DGDR + # spec.aisimulate.versionOverride together with that image. Empty uses the AISimulate version + # the operator maps to the image's Dynamo release. Required when the image is not a mapped + # Dynamo release (for example nightly); the operator refuses to start without it. It also + # applies to the derived planner image when dgdrAISimulateDefaultImage is empty. A Request + # that names its own image never receives it. + dgdrAISimulateDefaultVersionOverride: "" resources: limits: cpu: 1024m diff --git a/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml b/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml index af335066ea7a..42edcb584537 100644 --- a/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml +++ b/deploy/helm/charts/platform/tests/namespace_restriction_webhooks_test.yaml @@ -97,3 +97,23 @@ tests: asserts: - hasDocuments: count: 4 + + - it: matches DGDR requests through every served API version + template: charts/dynamo-operator/templates/webhook-configuration.yaml + asserts: + - equal: + path: webhooks[3].name + value: vdynamographdeploymentrequest.kb.io + documentIndex: 0 + - equal: + path: webhooks[3].matchPolicy + value: Equivalent + documentIndex: 0 + - equal: + path: webhooks[2].name + value: mdynamographdeploymentrequestv1beta1.kb.io + documentIndex: 1 + - equal: + path: webhooks[2].matchPolicy + value: Equivalent + documentIndex: 1 diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 39887c2d6d17..d7413eb3936a 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -70,6 +70,7 @@ type HardwareSpec struct { // +kubebuilder:validation:MaxItems=2 // +kubebuilder:validation:items:Minimum=0 // +kubebuilder:validation:items:Maximum=1 +// +kubebuilder:validation:XValidation:rule="size(self) != 2 || self[0] < self[1]",message="a range needs min < max" type KVLoadRatio []float64 // StaticWorkloadSpec defines one of AISimulate's synthetic workload shapes. @@ -110,11 +111,10 @@ type StaticWorkloadSpec struct { // WorkloadSpec defines the synthetic traffic model used for every candidate evaluation. type WorkloadSpec struct { - // Static defines one of AISimulate's synthetic workload shapes. When omitted, - // it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - // An explicit empty object opts out of this underlay. + // Static defines one of AISimulate's synthetic workload shapes. Its fields are not + // defaulted individually: the default belongs to spec.workload as a whole, so an + // explicit workload object keeps exactly the fields it names. // +optional - // +kubebuilder:default={isl: 4000, osl: 1000, kvLoadRatio: {0.0, 1.0}} Static *StaticWorkloadSpec `json:"static,omitempty"` } @@ -161,11 +161,10 @@ type ObjectiveSLASpec struct { type ObjectiveSpec struct { // Mode selects scalar optimization or a Pareto search. // +optional - // +kubebuilder:default=Optimize Mode ObjectiveMode `json:"mode,omitempty"` // Metric is the scalar target when mode is Optimize. - // When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + // When omitted in Optimize mode, admission defaults it to Throughput. // +optional Metric *ObjectiveMetric `json:"metric,omitempty"` @@ -176,24 +175,33 @@ type ObjectiveSpec struct { // +listType=set Metrics []ObjectiveMetric `json:"metrics,omitempty"` - // SLA defines per-request latency bounds used to calculate goodput. + // SLA defines per-request latency bounds used to calculate goodput. When omitted, no + // bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. // +optional - // +kubebuilder:default={ttftMs: 2000, itlMs: 30} SLA *ObjectiveSLASpec `json:"sla,omitempty"` } // AISimulateSpec selects the AISimulate execution artifact and supplies native input. type AISimulateSpec struct { // Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - // and execution defaults are resolved outside this API. + // persists the operator's configured AISimulate image, or the Dynamo planner image of + // the operator's release, together with the AISimulate version it bundles. // +optional // +kubebuilder:validation:MinLength=1 Image string `json:"image,omitempty"` - // VersionOverride declares the AISimulate compatibility version when it - // cannot be inferred from the effective image. + // VersionOverride declares the AISimulate version the image bundles as three numeric parts + + // (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. + + // When omitted, the version is the one the operator maps to the Dynamo release in the + + // image tag. It is required when the tag is not a Dynamo release the operator maps (a + + // nightly or custom image), and it wins over the mapping. // +optional // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$` VersionOverride string `json:"versionOverride,omitempty"` // Parameters contains native AISimulate input. Field names and nesting use @@ -210,7 +218,6 @@ type RecommendationSpec struct { // candidates are ordered best first; Pareto candidates have no scalar rank. // This field does not limit internal trials. // +optional - // +kubebuilder:default=1 // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=100 MaxCandidates int32 `json:"maxCandidates,omitempty"` @@ -226,6 +233,7 @@ type NativeSearchInput struct { // Backends lists one or more inference backends searched by AISimulate and used by generated candidates. // +optional // +kubebuilder:validation:MinItems=1 + // +listType=set Backends []Backend `json:"backends,omitempty"` // Hardware bounds the accelerator configurations evaluated by the search. @@ -234,12 +242,19 @@ type NativeSearchInput struct { Hardware *HardwareSpec `json:"hardware,omitempty"` // Workload defines the synthetic traffic model used for every candidate evaluation. + // When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + // an explicit empty workload receives the same shape from admission. An explicit + // workload.static: {} opts out of that shape, leaving the workload to + // spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + // fields it names. + // The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + // KV-load ratio search. // +optional Workload *WorkloadSpec `json:"workload,omitempty"` // Objective defines scalar optimization or a multi-objective Pareto search. - // When omitted, the schema supplies the default Optimize mode and SLA; - // admission supplies the conditional scalar metric. + // When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + // supplies the conditional scalar metric. // +optional Objective *ObjectiveSpec `json:"objective,omitempty"` diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 9d9248628aa8..fab7f7d378f6 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -68,11 +68,13 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/crdmigrator" lpxv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx/scheduler/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/namespace_scope" "github.com/ai-dynamo/dynamo/deploy/operator/internal/observability" "github.com/ai-dynamo/dynamo/deploy/operator/internal/podcache" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secrets" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" snapshotv1alpha1 "github.com/ai-dynamo/snapshot/api/v1alpha1" @@ -115,7 +117,6 @@ func initCRDSchemes() { utilruntime.Must(nvidiacomv1alpha1.AddToScheme(crdScheme)) utilruntime.Must(nvidiacomv1beta1.AddToScheme(crdScheme)) - utilruntime.Must(nvidiacomv1beta2.AddToScheme(crdScheme)) utilruntime.Must(lwsscheme.AddToScheme(crdScheme)) @@ -156,6 +157,8 @@ func main() { var operatorImage string var operatorImagePullPolicy string var dgdrDefaultImage string + var dgdrAISimulateDefaultImage string + var dgdrAISimulateDefaultVersionOverride string flag.StringVar(&configFile, "config", "", "Path to operator configuration file (required)") flag.StringVar(&operatorVersion, "operator-version", "unknown", "Version of the operator (used in lease holder identity)") @@ -169,6 +172,13 @@ func main() { "Image pull policy for operator helper init containers") flag.StringVar(&dgdrDefaultImage, "dgdr-default-image", "", "Default DGDR profiler image, put into DGDR spec.image when unset; empty derives dynamo-planner:") + flag.StringVar(&dgdrAISimulateDefaultImage, "dgdr-aisimulate-default-image", "", + "AISimulate runner image, put into v1beta2 DGDR spec.aisimulate.image when unset; "+ + "empty derives dynamo-planner:") + flag.StringVar(&dgdrAISimulateDefaultVersionOverride, "dgdr-aisimulate-default-version-override", "", + "AISimulate version bundled in the AISimulate default image, put into v1beta2 DGDR "+ + "spec.aisimulate.versionOverride together with that image; empty uses the version the release "+ + "contract maps to the image's Dynamo release, and is required for an image without one") opts := zap.Options{ Development: true, } @@ -176,6 +186,13 @@ func main() { flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + if err := webhookdefaulting.ValidateAISimulateDefault( + operatorVersion, dgdrAISimulateDefaultImage, dgdrAISimulateDefaultVersionOverride, + ); err != nil { + setupLog.Error(err, "invalid AISimulate default") + os.Exit(1) + } + if configFile == "" { setupLog.Error(nil, "--config flag is required") os.Exit(1) @@ -501,7 +518,8 @@ func main() { } if err := registerWebhookHandlers( - mgr, operatorCfg, runtimeConfig, operatorVersion, dgdrDefaultImage, gates, + mgr, operatorCfg, runtimeConfig, operatorVersion, dgdrDefaultImage, dgdrAISimulateDefaultImage, + dgdrAISimulateDefaultVersionOverride, gates, ); err != nil { setupLog.Error(err, "failed to register webhooks") os.Exit(1) @@ -637,6 +655,8 @@ func registerWebhookHandlers( runtimeConfig *commonController.RuntimeConfig, operatorVersion string, dgdrDefaultImage string, + dgdrAISimulateDefaultImage string, + dgdrAISimulateDefaultVersionOverride string, gate features.Gate, ) error { var operatorPrincipal string @@ -648,10 +668,24 @@ func registerWebhookHandlers( } if err := webhooksetup.Setup(mgr, webhooksetup.Options{ - Config: operatorCfg, - RuntimeConfig: runtimeConfig, - OperatorVersion: operatorVersion, - DGDRDefaultImage: dgdrDefaultImage, + Config: operatorCfg, + RuntimeConfig: runtimeConfig, + OperatorVersion: operatorVersion, + DGDRV1beta1: webhookdefaulting.DGDRV1beta1Options{ + DefaultImage: dgdrDefaultImage, + }, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + AISimulateImage: dgdrAISimulateDefaultImage, + AISimulateVersionOverride: dgdrAISimulateDefaultVersionOverride, + // The same discovery order as the DGDR reconciler: DCGM first, then node labels when + // the GPUDiscovery gate is on. The webhook keeps its own short-lived cache. + Hardware: &gpu.Discoverer{ + Reader: mgr.GetAPIReader(), + DCGM: gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint), + Cache: gpu.NewGPUDiscoveryCache(), + NodeLabels: func() bool { return gate.Enabled(features.GPUDiscovery) }, + }, + }, OperatorPrincipal: operatorPrincipal, Gate: gate, }); err != nil { diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index af2a4fb488ec..98a82b7d21d6 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9603,7 +9603,8 @@ spec: image: description: |- Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - and execution defaults are resolved outside this API. + persists the operator's configured AISimulate image, or the Dynamo planner image of + the operator's release, together with the AISimulate version it bundles. minLength: 1 type: string parameters: @@ -9613,10 +9614,9 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true versionOverride: - description: |- - VersionOverride declares the AISimulate compatibility version when it - cannot be inferred from the effective image. + description: nightly or custom image), and it wins over the mapping. minLength: 1 + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ type: string type: object backends: @@ -9630,6 +9630,7 @@ spec: type: string minItems: 1 type: array + x-kubernetes-list-type: set hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. @@ -9668,13 +9669,13 @@ spec: objective: description: |- Objective defines scalar optimization or a multi-objective Pareto search. - When omitted, the schema supplies the default Optimize mode and SLA; - admission supplies the conditional scalar metric. + When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + supplies the conditional scalar metric. properties: metric: description: |- Metric is the scalar target when mode is Optimize. - When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + When omitted in Optimize mode, admission defaults it to Throughput. enum: - Throughput - ThroughputPerGPU @@ -9702,17 +9703,15 @@ spec: type: array x-kubernetes-list-type: set mode: - default: Optimize description: Mode selects scalar optimization or a Pareto search. enum: - Optimize - Pareto type: string sla: - default: - itlMs: 30 - ttftMs: 2000 - description: SLA defines per-request latency bounds used to calculate goodput. + description: |- + SLA defines per-request latency bounds used to calculate goodput. When omitted, no + bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. @@ -9745,7 +9744,6 @@ spec: description: Recommendation controls bounded projection into DGDC resources. properties: maxCandidates: - default: 1 description: |- MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first; Pareto candidates have no scalar rank. @@ -9764,19 +9762,21 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true workload: - description: Workload defines the synthetic traffic model used for every candidate evaluation. + description: |- + Workload defines the synthetic traffic model used for every candidate evaluation. + When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + an explicit empty workload receives the same shape from admission. An explicit + workload.static: {} opts out of that shape, leaving the workload to + spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + fields it names. + The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + KV-load ratio search. properties: static: - default: - isl: 4000 - kvLoadRatio: - - 0 - - 1 - osl: 1000 description: |- - Static defines one of AISimulate's synthetic workload shapes. When omitted, - it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - An explicit empty object opts out of this underlay. + Static defines one of AISimulate's synthetic workload shapes. Its fields are not + defaulted individually: the default belongs to spec.workload as a whole, so an + explicit workload object keeps exactly the fields it names. properties: concurrency: description: Concurrency is a fixed closed-loop in-flight request cap. @@ -9799,6 +9799,9 @@ spec: maxItems: 2 minItems: 1 type: array + x-kubernetes-validations: + - message: a range needs min < max + rule: size(self) != 2 || self[0] < self[1] numRequestRatio: description: NumRequestRatio sets request count relative to the selected load. exclusiveMinimum: true diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 70a8e6f5e736..8218fea7e98f 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -66,7 +66,8 @@ spec: image: description: |- Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - and execution defaults are resolved outside this API. + persists the operator's configured AISimulate image, or the Dynamo planner image of + the operator's release, together with the AISimulate version it bundles. minLength: 1 type: string parameters: @@ -76,10 +77,9 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true versionOverride: - description: |- - VersionOverride declares the AISimulate compatibility version when it - cannot be inferred from the effective image. + description: nightly or custom image), and it wins over the mapping. minLength: 1 + pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ type: string type: object backends: @@ -93,6 +93,7 @@ spec: type: string minItems: 1 type: array + x-kubernetes-list-type: set hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. @@ -131,13 +132,13 @@ spec: objective: description: |- Objective defines scalar optimization or a multi-objective Pareto search. - When omitted, the schema supplies the default Optimize mode and SLA; - admission supplies the conditional scalar metric. + When omitted, admission supplies the object. Its mode defaults to Optimize, and admission + supplies the conditional scalar metric. properties: metric: description: |- Metric is the scalar target when mode is Optimize. - When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. + When omitted in Optimize mode, admission defaults it to Throughput. enum: - Throughput - ThroughputPerGPU @@ -165,17 +166,15 @@ spec: type: array x-kubernetes-list-type: set mode: - default: Optimize description: Mode selects scalar optimization or a Pareto search. enum: - Optimize - Pareto type: string sla: - default: - itlMs: 30 - ttftMs: 2000 - description: SLA defines per-request latency bounds used to calculate goodput. + description: |- + SLA defines per-request latency bounds used to calculate goodput. When omitted, no + bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. @@ -208,7 +207,6 @@ spec: description: Recommendation controls bounded projection into DGDC resources. properties: maxCandidates: - default: 1 description: |- MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first; Pareto candidates have no scalar rank. @@ -219,19 +217,21 @@ spec: type: integer type: object workload: - description: Workload defines the synthetic traffic model used for every candidate evaluation. + description: |- + Workload defines the synthetic traffic model used for every candidate evaluation. + When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; + an explicit empty workload receives the same shape from admission. An explicit + workload.static: {} opts out of that shape, leaving the workload to + spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the + fields it names. + The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a + KV-load ratio search. properties: static: - default: - isl: 4000 - kvLoadRatio: - - 0 - - 1 - osl: 1000 description: |- - Static defines one of AISimulate's synthetic workload shapes. When omitted, - it searches the full KV-load ratio range at ISL 4000 and OSL 1000. - An explicit empty object opts out of this underlay. + Static defines one of AISimulate's synthetic workload shapes. Its fields are not + defaulted individually: the default belongs to spec.workload as a whole, so an + explicit workload object keeps exactly the fields it names. properties: concurrency: description: Concurrency is a fixed closed-loop in-flight request cap. @@ -254,6 +254,9 @@ spec: maxItems: 2 minItems: 1 type: array + x-kubernetes-validations: + - message: a range needs min < max + rule: size(self) != 2 || self[0] < self[1] numRequestRatio: description: NumRequestRatio sets request count relative to the selected load. exclusiveMinimum: true diff --git a/deploy/operator/internal/controller/aisimulate/defaults.go b/deploy/operator/internal/controller/aisimulate/defaults.go new file mode 100644 index 000000000000..8aecb2d8d569 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/defaults.go @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + +// The derived defaults. No API field carries them: each depends on the typed underlay, so +// no single typed field can supply it. +const ( + // DefaultRequestCount is workload.request_count when the typed underlay does not + // choose num_request_ratio. + DefaultRequestCount int64 = 1000 + + // DefaultMaxTrials bounds a KV-load range search that has no explicit sweep.max_trials. + // It matches the trial budget of AISimulate's unified CLI (optimizer.max_trials). + DefaultMaxTrials int64 = 320 +) + +// applyDefaults adds the derived defaults to the typed underlay, before the native +// parameters are applied. spec.aisimulate.parameters is the final overlay: it overrides or +// deletes any derived value, and nothing is derived in response to what it introduces. +// A native change that needs a matching change elsewhere (a request_count next to +// num_request_ratio, a strict_sla next to a new SLA bound) must make it itself. +func applyDefaults(underlay map[string]any) error { + b := builder{root: underlay} + + if has(underlay, workloadPath...) { + if !has(underlay, workloadRequestCount...) && !has(underlay, workloadNumRequestRatio...) { + b.set(float64(DefaultRequestCount), workloadRequestCount) + } + kvLoadRatio, _, _ := unstructured.NestedFieldNoCopy(underlay, workloadKVLoadRatio...) + if _, isRange := kvLoadRatio.([]any); isRange && !has(underlay, sweepMaxTrials...) { + b.set(float64(DefaultMaxTrials), sweepMaxTrials) + } + } + + if hasSLABound(underlay) && !has(underlay, goalStrictSLA...) { + b.set(true, goalStrictSLA) + } + return b.err +} + +// hasSLABound reports whether the input bounds at least one latency. +func hasSLABound(input map[string]any) bool { + for _, path := range [][]string{goalSLATTFTMs, goalSLAITLMs, goalSLAE2EMs} { + if bound, _, _ := unstructured.NestedFieldNoCopy(input, path...); bound != nil { + return true + } + } + return false +} diff --git a/deploy/operator/internal/controller/aisimulate/doc.go b/deploy/operator/internal/controller/aisimulate/doc.go new file mode 100644 index 000000000000..a15cb22952e7 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/doc.go @@ -0,0 +1,38 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package aisimulate lowers a v1beta2 DynamoGraphDeploymentRequest spec into the native +// AISimulate input that a Run executes. +// +// Lowering sits below the Request reconciler. Admission does not call it: spec.aisimulate.parameters +// is opaque there, and the validation of the lowered input belongs to the schema of the AISimulate +// release the Request selects. The reconciler lowers the Request once and freezes the output into the +// Run, which executes that object as it is. +// +// Lower works in three steps. The typed spec (model, backends, hardware, workload, objective) +// is first translated into the native input, the "underlay". The derived defaults that no API +// field carries are then added to it: the request count, a bounded trial budget for a +// KV-load range search, and goal.strict_sla when an SLA bound is set. Last, +// spec.aisimulate.parameters is applied on top as a JSON Merge Patch (RFC 7386): native +// objects merge recursively, arrays and scalars replace the value, and null deletes it. The +// parameters are authoritative: they win on every path they name, including the derived +// defaults, and are applied once, last. +// +// The document stays unstructured (map[string]any) because its schema belongs to the +// AISimulate release the Request selects, not to this package. Reads and writes go +// through the helpers of k8s.io/apimachinery/pkg/apis/meta/v1/unstructured. +package aisimulate diff --git a/deploy/operator/internal/controller/aisimulate/image.go b/deploy/operator/internal/controller/aisimulate/image.go new file mode 100644 index 000000000000..60025699e1af --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/image.go @@ -0,0 +1,62 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "errors" + "fmt" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" +) + +// aisimulateVersionByDynamoMinor is the release contract between a Dynamo release and the AISimulate +// package its planner image bundles, keyed by the Dynamo major.minor. The image tag is the Dynamo +// version, not the version of the AISimulate package inside the image, and the AISimulate version +// selects the version-specific input schema, so the two are never inferred from one tag. Every +// Dynamo minor bump adds its entry; release_contract_test.go fails until it does. +var aisimulateVersionByDynamoMinor = map[string]string{ + "1.6": "0.13.0", +} + +// ResolveVersion returns the AISimulate version a Request selects: the explicit override when set, +// otherwise the AISimulate version bundled in the Dynamo release named by the image tag. The +// override is the escape hatch for images the contract does not cover (a nightly or custom planner +// image) and names the AISimulate version itself. An image whose tag is not a Dynamo release with a +// mapping entry, or a missing image, is an error; the caller reports it on the field that fixes it. +func ResolveVersion(image, versionOverride string) (string, error) { + if versionOverride != "" { + version, err := versionoverride.Parse(versionOverride) + if err != nil { + return "", fmt.Errorf("AISimulate version override %q: %w", versionOverride, err) + } + return version.String(), nil + } + if image == "" { + return "", errors.New("no AISimulate image") + } + release, err := versionoverride.ParseImageVersion(image) + if err != nil { + return "", fmt.Errorf("resolve AISimulate version: %w", err) + } + minor := fmt.Sprintf("%d.%d", release.Major, release.Minor) + version, ok := aisimulateVersionByDynamoMinor[minor] + if !ok { + return "", fmt.Errorf("image %q is Dynamo %s, which has no bundled AISimulate version in the release contract", image, minor) + } + return version, nil +} diff --git a/deploy/operator/internal/controller/aisimulate/image_test.go b/deploy/operator/internal/controller/aisimulate/image_test.go new file mode 100644 index 000000000000..2a05392112a7 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/image_test.go @@ -0,0 +1,52 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "testing" + +func TestResolveVersion(t *testing.T) { + const mapped = "registry.example.com/aisimulate:1.6.0" + want := aisimulateVersionByDynamoMinor["1.6"] + cases := []struct { + name, image, override, want string + wantErr bool + }{ + {name: "bundled version of the Dynamo release in the tag", image: mapped, want: want}, + {name: "v-prefixed tag", image: "registry.example.com/aisimulate:v1.6.0", want: want}, + {name: "patch release of the same minor", image: "registry.example.com/aisimulate:1.6.3", want: want}, + {name: "prerelease of the same minor", image: "registry.example.com/aisimulate:1.6.0-rc.1", want: want}, + {name: "release without a mapping entry", image: "registry.example.com/aisimulate:1.2.3", wantErr: true}, + {name: "override wins over the mapping", image: mapped, override: "0.99.0", want: "0.99.0"}, + {name: "override rescues a release without an entry", image: "registry.example.com/aisimulate:1.2.3", override: "0.9.0", want: "0.9.0"}, + {name: "override rescues a custom tag", image: "registry.example.com/aisimulate:custom", override: "0.13.1", want: "0.13.1"}, + {name: "override that is not a version", image: mapped, override: "bad", wantErr: true}, + {name: "custom tag needs an override", image: "registry.example.com/aisimulate:custom", wantErr: true}, + {name: "untagged image needs an override", image: "registry.example.com/aisimulate", wantErr: true}, + {name: "registry port is not a tag", image: "registry.example.com:5000/aisimulate", wantErr: true}, + {name: "no image", wantErr: true}, + {name: "override without an image", override: "0.13.0", want: "0.13.0"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := ResolveVersion(tc.image, tc.override) + if (err != nil) != tc.wantErr || got != tc.want { + t.Fatalf("ResolveVersion(%q, %q) = %q, %v; want %q (error %v)", tc.image, tc.override, got, err, tc.want, tc.wantErr) + } + }) + } +} diff --git a/deploy/operator/internal/controller/aisimulate/input.go b/deploy/operator/internal/controller/aisimulate/input.go new file mode 100644 index 000000000000..fbfee0514080 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/input.go @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "fmt" + "strings" + "unicode" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Lower returns the effective native input: the typed underlay, merged with +// spec.aisimulate.parameters, plus the derived defaults. The spec must already carry its +// structural defaults; lowering repeats none of them. The only error is malformed +// parameters (not a JSON object); semantic problems are the caller's to validate. +func Lower(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + parameters, err := Parameters(spec) + if err != nil { + return nil, err + } + underlay, err := Underlay(spec) + if err != nil { + return nil, err + } + if underlay, err = normalize(underlay); err != nil { + return nil, err + } + if err := applyDefaults(underlay); err != nil { + return nil, err + } + input, _ := mergePatch(underlay, parameters).(map[string]any) + if input == nil { + input = map[string]any{} + } + return input, nil +} + +// Parameters returns spec.aisimulate.parameters as a JSON object, or nil when absent. +func Parameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + if spec.AISimulate == nil || spec.AISimulate.Parameters == nil { + return nil, nil + } + raw := spec.AISimulate.Parameters.Raw + if len(raw) == 0 && spec.AISimulate.Parameters.Object != nil { + var err error + if raw, err = json.Marshal(spec.AISimulate.Parameters.Object); err != nil { + return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) + } + } + if len(raw) == 0 { + return nil, nil + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) + } + switch typed := value.(type) { + case nil: + return nil, nil + case map[string]any: + return typed, nil + default: + return nil, fmt.Errorf("spec.aisimulate.parameters: must be a JSON object, got %T", value) + } +} + +// Underlay translates the typed spec into native input. Typed fields that are unset +// produce no key, so the native defaults and the parameters decide them. +func Underlay(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { + b := builder{root: map[string]any{}} + + if spec.Model != nil { + b.set(spec.Model.Name, searchSpaceModelName) + } + if len(spec.Backends) > 0 { + backends := make([]string, len(spec.Backends)) + for i, backend := range spec.Backends { + backends[i] = string(backend) + } + b.setStrings(backends, searchSpaceBackend) + } + if spec.Hardware != nil && spec.Hardware.GPU != nil { + if sku := spec.Hardware.GPU.SKU; sku != nil && *sku != "" { + b.set(*sku, searchSpaceHardwareSKU) + } + if budget := spec.Hardware.GPU.Budget; budget != nil { + b.set(int64(*budget), searchSpaceGPUBudget) + } + } + + if spec.Workload != nil && spec.Workload.Static != nil { + static := spec.Workload.Static + if static.ISL != nil { + b.set(int64(*static.ISL), workloadISL) + } + if static.OSL != nil { + b.set(int64(*static.OSL), workloadOSL) + } + if static.NumRequestRatio != nil { + b.set(*static.NumRequestRatio, workloadNumRequestRatio) + } + if static.RequestRate != nil { + b.set(*static.RequestRate, workloadRequestRate) + } + if static.Concurrency != nil { + b.set(int64(*static.Concurrency), workloadConcurrency) + } + switch len(static.KVLoadRatio) { + case 0: + case 1: + b.set(static.KVLoadRatio[0], workloadKVLoadRatio) + default: + ratios := make([]any, len(static.KVLoadRatio)) + for i, ratio := range static.KVLoadRatio { + ratios[i] = ratio + } + b.set(ratios, workloadKVLoadRatio) + } + } + + if objective := spec.Objective; objective != nil { + if objective.Mode == v1beta2.ObjectiveModePareto { + b.set(TargetPareto, goalTarget) + if len(objective.Metrics) > 0 { + b.setStrings(nativeMetrics(objective.Metrics), goalParetoObjectives) + } + } else if objective.Metric != nil { + b.set(nativeMetric(*objective.Metric), goalTarget) + } + if sla := objective.SLA; sla != nil { + if sla.TTFTMs != nil { + b.set(*sla.TTFTMs, goalSLATTFTMs) + } + if sla.ITLMs != nil { + b.set(*sla.ITLMs, goalSLAITLMs) + } + if sla.E2EMs != nil { + b.set(*sla.E2EMs, goalSLAE2EMs) + } + } + } + return b.root, b.err +} + +// builder writes native input values, remembering the first failure so the translation reads +// as a straight list of assignments. Every path is written under a map the caller created +// and every value is a JSON scalar or a list of them, so a failure means a bad path. +type builder struct { + root map[string]any + err error +} + +func (b *builder) set(value any, path []string) { + if b.err != nil { + return + } + if err := unstructured.SetNestedField(b.root, value, path...); err != nil { + b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) + } +} + +func (b *builder) setStrings(values []string, path []string) { + if b.err != nil { + return + } + if err := unstructured.SetNestedStringSlice(b.root, values, path...); err != nil { + b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) + } +} + +// nativeMetric maps an API metric to AISimulate's goal.target by converting its CamelCase +// name to snake_case: ThroughputPerGPU becomes throughput_per_gpu and E2ELatency becomes +// e2e_latency. A run of capitals stays together, a digit does not start a word, and the last +// capital of a run starts the next word when a lowercase letter follows it. +func nativeMetric(metric v1beta2.ObjectiveMetric) string { + name := []rune(string(metric)) + var out []rune + for i, r := range name { + if i > 0 && unicode.IsUpper(r) { + prev := name[i-1] + nextLower := i+1 < len(name) && unicode.IsLower(name[i+1]) + if unicode.IsLower(prev) || (unicode.IsUpper(prev) && nextLower) { + out = append(out, '_') + } + } + out = append(out, unicode.ToLower(r)) + } + return string(out) +} + +func nativeMetrics(metrics []v1beta2.ObjectiveMetric) []string { + out := make([]string, len(metrics)) + for i, metric := range metrics { + out[i] = nativeMetric(metric) + } + return out +} + +// normalize round-trips through JSON so every number is a float64, the type the +// parameters decode to. Merge and validation then see one representation. +func normalize(value map[string]any) (map[string]any, error) { + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} + +// mergePatch applies patch to target with RFC 7386 semantics. It returns a new value for +// the merged parts and does not retain the patch's maps for null members. +func mergePatch(target, patch any) any { + patchMap, ok := patch.(map[string]any) + if !ok { + return patch + } + targetMap, ok := target.(map[string]any) + if !ok { + targetMap = map[string]any{} + } + for key, value := range patchMap { + if value == nil { + delete(targetMap, key) + continue + } + targetMap[key] = mergePatch(targetMap[key], value) + } + return targetMap +} diff --git a/deploy/operator/internal/controller/aisimulate/input_test.go b/deploy/operator/internal/controller/aisimulate/input_test.go new file mode 100644 index 000000000000..dfeff5a150a0 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/input_test.go @@ -0,0 +1,433 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "strings" + "testing" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +func ptrTo[T any](v T) *T { return &v } + +// defaultedSpec is a model-only Request as it looks after the CRD's structural defaults and +// admission: the fixed-concurrency workload, the scalar metric and discovered hardware. It +// has no SLA bounds and no KV-load range; withSLA and withKVRange add them. +func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { + metric := v1beta2.ObjectiveMetric("Throughput") + return &v1beta2.DynamoGraphDeploymentRequestSpec{ + Model: &v1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, + Backends: []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang, v1beta2.BackendTRTLLM}, + Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("h200_sxm"), Budget: ptrTo(int32(8))}}, + Workload: &v1beta2.WorkloadSpec{Static: &v1beta2.StaticWorkloadSpec{ + ISL: ptrTo(int32(1024)), OSL: ptrTo(int32(128)), Concurrency: ptrTo(int32(10)), + }}, + Objective: &v1beta2.ObjectiveSpec{Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric}, + Recommendation: &v1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +// withSLA adds the latency bounds the earlier defaults used to supply. +func withSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)} + return spec +} + +// withKVRange replaces the fixed concurrency with a KV-load range search. +func withKVRange(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ + ISL: ptrTo(int32(4000)), OSL: ptrTo(int32(1000)), KVLoadRatio: v1beta2.KVLoadRatio{0, 1}, + } + return spec +} + +func withParameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec, parameters string) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.AISimulate = &v1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(parameters)}} + return spec +} + +func lower(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { + t.Helper() + input, err := Lower(spec) + if err != nil { + t.Fatalf("Lower: %v", err) + } + return input +} + +func encode(t *testing.T, input map[string]any) string { + t.Helper() + raw, err := json.Marshal(input) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func underlay(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { + t.Helper() + input, err := Underlay(spec) + if err != nil { + t.Fatalf("Underlay: %v", err) + } + return input +} + +func nested(t *testing.T, input map[string]any, path ...string) any { + t.Helper() + value, _, err := unstructured.NestedFieldNoCopy(input, path...) + if err != nil { + t.Fatal(err) + } + return value +} + +func TestLowerModelOnlyRequest(t *testing.T) { + want := `{"goal":{"target":"throughput"},` + + `"search_space":{"backend":["vllm","sglang","trtllm"],"gpu_budget":8,"hardware_sku":"h200_sxm","model_name":"meta-llama/Llama-3.1-8B-Instruct"},` + + `"workload":{"concurrency":10,"isl":1024,"osl":128,"request_count":1000}}` + if got := encode(t, lower(t, defaultedSpec())); got != want { + t.Fatalf("effective input\n got: %s\nwant: %s", got, want) + } +} + +func TestLowerIsDeterministicAndDoesNotMutateTheSpec(t *testing.T) { + const parameters = `{"sweep":{"max_trials":100},"workload":{"request_count":5}}` + spec := withParameters(defaultedSpec(), parameters) + first, second := encode(t, lower(t, spec)), encode(t, lower(t, spec)) + if first != second { + t.Fatalf("not deterministic:\n%s\n%s", first, second) + } + if string(spec.AISimulate.Parameters.Raw) != parameters { + t.Fatalf("parameters mutated: %s", spec.AISimulate.Parameters.Raw) + } +} + +func TestLowerExplicitRequest(t *testing.T) { + metric := v1beta2.ObjectiveMetric("Throughput") + spec := withSLA(defaultedSpec()) + spec.Backends = []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang} + spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ISL: ptrTo(int32(4000)), OSL: ptrTo(int32(1000)), RequestRate: ptrTo(10.0)} + spec.Objective.Metric = &metric + spec = withParameters(spec, `{"workload":{"request_count":1000},"sweep":{"max_trials":100}}`) + + want := `{"goal":{"sla":{"itl_ms":30,"ttft_ms":2000},"strict_sla":true,"target":"throughput"},` + + `"search_space":{"backend":["vllm","sglang"],"gpu_budget":8,"hardware_sku":"h200_sxm","model_name":"meta-llama/Llama-3.1-8B-Instruct"},` + + `"sweep":{"max_trials":100},` + + `"workload":{"isl":4000,"osl":1000,"request_count":1000,"request_rate":10}}` + if got := encode(t, lower(t, spec)); got != want { + t.Fatalf("effective input\n got: %s\nwant: %s", got, want) + } +} + +func TestMergePatchSemantics(t *testing.T) { + cases := []struct{ name, target, patch, want string }{ + {"scalar replaces", `{"a":1}`, `{"a":2}`, `{"a":2}`}, + {"object merges recursively", `{"a":{"b":1,"c":2}}`, `{"a":{"b":3}}`, `{"a":{"b":3,"c":2}}`}, + {"null deletes", `{"a":1,"b":2}`, `{"a":null}`, `{"b":2}`}, + {"array replaces", `{"a":[1,2,3]}`, `{"a":[4]}`, `{"a":[4]}`}, + {"object replaces scalar and drops nulls", `{"a":1}`, `{"a":{"b":null,"c":1}}`, `{"a":{"c":1}}`}, + {"null on absent key is a no-op", `{"a":1}`, `{"z":null}`, `{"a":1}`}, + {"empty patch", `{"a":1}`, `{}`, `{"a":1}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var target, patch any + if err := json.Unmarshal([]byte(tc.target), &target); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(tc.patch), &patch); err != nil { + t.Fatal(err) + } + got, _ := json.Marshal(mergePatch(target, patch)) + if string(got) != tc.want { + t.Fatalf("got %s want %s", got, tc.want) + } + }) + } + t.Run("nil patch map", func(t *testing.T) { + target := map[string]any{"a": 1.0} + if got := mergePatch(target, map[string]any(nil)); got == nil { + t.Fatal("a nil patch must keep the target") + } + }) +} + +func TestNativeValuesWinOverTheTypedUnderlay(t *testing.T) { + t.Run("a native null deletes a typed field", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"isl":null}}`)) + if nested(t, input, "workload", "isl") != nil { + t.Fatalf("isl survived a native null: %s", encode(t, input)) + } + }) + t.Run("a native value overrides a typed field", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"isl":8000},"search_space":{"gpu_budget":16}}`)) + if nested(t, input, "workload", "isl") != 8000.0 || nested(t, input, "search_space", "gpu_budget") != 16.0 { + t.Fatalf("native values did not win: %s", encode(t, input)) + } + }) + t.Run("a native target overrides the typed metric", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"goal":{"target":"ttft"}}`)) + if nested(t, input, "goal", "target") != "ttft" { + t.Fatalf("typed metric beat the native target: %s", encode(t, input)) + } + }) + t.Run("a native SLA bound overrides only the bound it names", func(t *testing.T) { + input := lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "goal", "sla", "ttft_ms") != 500.0 || nested(t, input, "goal", "sla", "itl_ms") != 30.0 { + t.Fatalf("unexpected SLA: %s", encode(t, input)) + } + }) +} + +func TestRequestCountDefault(t *testing.T) { + t.Run("derived from the typed underlay", func(t *testing.T) { + if nested(t, lower(t, defaultedSpec()), "workload", "request_count") != 1000.0 { + t.Fatal("request_count not derived") + } + }) + t.Run("a native ratio deletes the derived count itself", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"num_request_ratio":10,"request_count":null}}`)) + if nested(t, input, "workload", "request_count") != nil || nested(t, input, "workload", "num_request_ratio") != 10.0 { + t.Fatalf("unexpected workload: %s", encode(t, input)) + } + }) + t.Run("skipped when typed numRequestRatio is set", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.NumRequestRatio = ptrTo(5.0) + if nested(t, lower(t, spec), "workload", "request_count") != nil { + t.Fatal("request_count defaulted next to typed numRequestRatio") + } + }) + t.Run("native null deletes the default", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"request_count":null}}`)) + if nested(t, input, "workload", "request_count") != nil { + t.Fatal("request_count re-added after a native null") + } + }) + t.Run("native deleting the typed ratio does not derive a count", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.NumRequestRatio = ptrTo(5.0) + input := lower(t, withParameters(spec, `{"workload":{"num_request_ratio":null}}`)) + if nested(t, input, "workload", "request_count") != nil { + t.Fatalf("a count was derived after the native overlay: %s", encode(t, input)) + } + }) + t.Run("not added without a workload", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload = &v1beta2.WorkloadSpec{} + if nested(t, lower(t, spec), "workload") != nil { + t.Fatal("a workload appeared from nothing") + } + }) +} + +func TestMaxTrialsDefault(t *testing.T) { + t.Run("added for a KV-load range", func(t *testing.T) { + if nested(t, lower(t, withKVRange(defaultedSpec())), "sweep", "max_trials") != 320.0 { + t.Fatal("no default max_trials for a range search") + } + }) + t.Run("not derived for the model-only fixed concurrency", func(t *testing.T) { + if input := lower(t, defaultedSpec()); nested(t, input, "sweep") != nil { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("not added for a pinned load", func(t *testing.T) { + spec := withKVRange(defaultedSpec()) + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.5} + input := lower(t, spec) + if nested(t, input, "sweep") != nil { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + if nested(t, input, "workload", "kv_load_ratio") != 0.5 { + t.Fatal("a one-value ratio should be a scalar") + } + }) + t.Run("native value or null wins", func(t *testing.T) { + if nested(t, lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":7}}`)), "sweep", "max_trials") != 7.0 { + t.Fatal("native max_trials ignored") + } + if nested(t, lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":null}}`)), "sweep", "max_trials") != nil { + t.Fatal("max_trials re-added after a native null") + } + }) +} + +func TestStrictSLADerivation(t *testing.T) { + t.Run("enabled when the typed SLA has a bound", func(t *testing.T) { + if nested(t, lower(t, withSLA(defaultedSpec())), "goal", "strict_sla") != true { + t.Fatal("strict_sla not derived") + } + }) + t.Run("model-only derives nothing", func(t *testing.T) { + input := lower(t, defaultedSpec()) + if nested(t, input, "goal", "strict_sla") != nil || nested(t, input, "goal", "sla") != nil { + t.Fatalf("model-only input has SLA keys: %s", encode(t, input)) + } + }) + t.Run("an empty SLA derives nothing", func(t *testing.T) { + spec := defaultedSpec() + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{} + input := lower(t, spec) + if nested(t, input, "goal", "strict_sla") != nil || nested(t, input, "goal", "sla") != nil { + t.Fatalf("an empty SLA produced SLA keys: %s", encode(t, input)) + } + }) + t.Run("a native bound alone does not derive it", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "goal", "sla", "ttft_ms") != 500.0 || nested(t, input, "goal", "strict_sla") != nil { + t.Fatalf("unexpected goal: %s", encode(t, input)) + } + }) + t.Run("native value is kept", func(t *testing.T) { + if nested(t, lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"strict_sla":false}}`)), "goal", "strict_sla") != false { + t.Fatal("native strict_sla overridden") + } + }) + t.Run("is not revisited after the native overlay", func(t *testing.T) { + input := lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"sla":{"ttft_ms":null,"itl_ms":null}}}`)) + if nested(t, input, "goal", "strict_sla") != true { + t.Fatalf("the derived strict_sla changed after native bounds were deleted: %s", encode(t, input)) + } + }) +} + +func TestParametersApplyOnceAfterDerivedDefaults(t *testing.T) { + t.Run("sweep null removes the complete derived sweep", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":null}`)) + if nested(t, input, "sweep") != nil { + t.Fatalf("sweep survived a native null: %s", encode(t, input)) + } + }) + t.Run("max_trials null removes only max_trials", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":null,"seed":3}}`)) + if nested(t, input, "sweep", "max_trials") != nil || nested(t, input, "sweep", "seed") != 3.0 { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("a native max_trials replaces the derived value", func(t *testing.T) { + input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":7}}`)) + if nested(t, input, "sweep", "max_trials") != 7.0 { + t.Fatalf("unexpected sweep: %s", encode(t, input)) + } + }) + t.Run("nothing is derived from values only parameters introduce", func(t *testing.T) { + input := lower(t, withParameters(defaultedSpec(), `{"workload":{"kv_load_ratio":[0,1]},"goal":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "sweep") != nil || nested(t, input, "goal", "strict_sla") != nil { + t.Fatalf("a field was derived from native-only values: %s", encode(t, input)) + } + }) +} + +func TestUnderlayParetoObjective(t *testing.T) { + spec := defaultedSpec() + spec.Objective.Mode = v1beta2.ObjectiveModePareto + spec.Objective.Metric = nil + spec.Objective.Metrics = []v1beta2.ObjectiveMetric{"ThroughputPerGPU", "ThroughputPerUser", "Unknown"} + input := underlay(t, spec) + if nested(t, input, "goal", "target") != TargetPareto { + t.Fatalf("not a pareto goal: %v", input["goal"]) + } + objectives, _, _ := unstructured.NestedSlice(input, "goal", "pareto_objectives") + got, _ := json.Marshal(objectives) + if string(got) != `["throughput_per_gpu","throughput_per_user","unknown"]` { + t.Fatalf("pareto_objectives = %s", got) + } +} + +func TestUnderlayOmitsUnsetTypedFields(t *testing.T) { + spec := &v1beta2.DynamoGraphDeploymentRequestSpec{Model: &v1beta2.ModelSpec{Name: "m"}} + if got := encode(t, underlay(t, spec)); got != `{"search_space":{"model_name":"m"}}` { + t.Fatalf("got %s", got) + } + spec.Hardware = &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("")}} + if got := encode(t, underlay(t, spec)); got != `{"search_space":{"model_name":"m"}}` { + t.Fatalf("an empty SKU must not lower: %s", got) + } +} + +func TestUnderlayKVLoadRatioShapes(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.25} + if nested(t, underlay(t, spec), "workload", "kv_load_ratio") != 0.25 { + t.Fatal("a single ratio lowers to a scalar") + } + spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.1, 0.2, 0.3} + if ratios, _ := nested(t, underlay(t, spec), "workload", "kv_load_ratio").([]any); len(ratios) != 3 { + t.Fatal("longer lists lower unchanged so validation can report them") + } +} + +func TestParametersMustBeAnObject(t *testing.T) { + for _, parameters := range []string{`[1]`, `"x"`, `3`, `{`} { + if _, err := Lower(withParameters(defaultedSpec(), parameters)); err == nil { + t.Errorf("Lower accepted parameters %s", parameters) + } + } + for _, parameters := range []string{``, `null`, `{}`} { + if _, err := Lower(withParameters(defaultedSpec(), parameters)); err != nil { + t.Errorf("Lower rejected parameters %q: %v", parameters, err) + } + } +} + +func TestHasPath(t *testing.T) { + parameters := map[string]any{ + "goal": map[string]any{"target": nil, "sla": map[string]any{"ttft_ms": 1.0}}, + "workload": map[string]any{"isl": 8.0}, + } + for path, want := range map[string]bool{ + "goal.target": true, // a null still names the key + "goal.sla": true, + "goal.sla.ttft_ms": true, + "goal.sla.itl_ms": false, + "workload.isl": true, + "workload.osl": false, + "sweep.max_trials": false, + "goal.target.deeper": false, + "workload.isl.deeper": false, + } { + if got := has(parameters, strings.Split(path, ".")...); got != want { + t.Errorf("has(%s) = %v, want %v", path, got, want) + } + } + if has(nil, "goal") { + t.Error("nothing is named in nil parameters") + } +} + +func TestNativeMetricNames(t *testing.T) { + for metric, want := range map[v1beta2.ObjectiveMetric]string{ + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", + } { + if got := nativeMetric(metric); got != want { + t.Errorf("nativeMetric(%s) = %q, want %q", metric, got, want) + } + } +} diff --git a/deploy/operator/internal/controller/aisimulate/paths.go b/deploy/operator/internal/controller/aisimulate/paths.go new file mode 100644 index 000000000000..bd2a9d16b7d8 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/paths.go @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + +// Native input paths, in the snake_case spelling of the upstream AISimulate contract. They +// address the same native key when the underlay is built and when derived defaults are read, +// and are passed to the unstructured helpers, which must not modify them. +var ( + searchSpaceModelName = []string{"search_space", "model_name"} + searchSpaceBackend = []string{"search_space", "backend"} + searchSpaceHardwareSKU = []string{"search_space", "hardware_sku"} + searchSpaceGPUBudget = []string{"search_space", "gpu_budget"} + + workloadPath = []string{"workload"} + workloadISL = []string{"workload", "isl"} + workloadOSL = []string{"workload", "osl"} + workloadNumRequestRatio = []string{"workload", "num_request_ratio"} + workloadRequestRate = []string{"workload", "request_rate"} + workloadConcurrency = []string{"workload", "concurrency"} + workloadKVLoadRatio = []string{"workload", "kv_load_ratio"} + workloadRequestCount = []string{"workload", "request_count"} + + goalTarget = []string{"goal", "target"} + goalParetoObjectives = []string{"goal", "pareto_objectives"} + goalSLATTFTMs = []string{"goal", "sla", "ttft_ms"} + goalSLAITLMs = []string{"goal", "sla", "itl_ms"} + goalSLAE2EMs = []string{"goal", "sla", "e2e_ms"} + goalStrictSLA = []string{"goal", "strict_sla"} + + sweepMaxTrials = []string{"sweep", "max_trials"} +) + +// TargetPareto is the goal.target of a multi-objective search. +const TargetPareto = "pareto" + +// has reports whether obj contains the path, even with a null value. Applied to +// spec.aisimulate.parameters it tells whether the caller named a key, including to delete +// it. +func has(obj map[string]any, path ...string) bool { + _, found, _ := unstructured.NestedFieldNoCopy(obj, path...) + return found +} diff --git a/deploy/operator/internal/controller/aisimulate/release_contract_test.go b/deploy/operator/internal/controller/aisimulate/release_contract_test.go new file mode 100644 index 000000000000..a5cf0464b8f8 --- /dev/null +++ b/deploy/operator/internal/controller/aisimulate/release_contract_test.go @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "os" + "path/filepath" + "regexp" + "testing" +) + +// The repository root, relative to this package: deploy/operator/internal/controller/aisimulate. +const repositoryRoot = "../../../../.." + +var ( + pinPattern = regexp.MustCompile(`aisimulate==(\d+\.\d+\.\d+)`) + releasePattern = regexp.MustCompile(`(?m)^version\s*=\s*"(\d+\.\d+)\.`) +) + +func readRepositoryFile(t *testing.T, path string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(repositoryRoot, path)) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(raw) +} + +// pinnedAISimulate returns the released core (major.minor.patch) of the aisimulate pin in a file, +// so 0.13.0.dev202610160000000073 normalizes to 0.13.0. +func pinnedAISimulate(t *testing.T, path string) string { + t.Helper() + match := pinPattern.FindStringSubmatch(readRepositoryFile(t, path)) + if match == nil { + t.Fatalf("%s pins no aisimulate==", path) + } + return match[1] +} + +// TestBundledAISimulateVersionMatchesTheDynamoRelease ties the release contract to the repository: +// the AISimulate version the current Dynamo minor maps to must be the one the repository pins. It +// fails when a Dynamo minor bump has no mapping entry, or when the pin moves without the entry. +func TestBundledAISimulateVersionMatchesTheDynamoRelease(t *testing.T) { + canonical := pinnedAISimulate(t, "container/deps/requirements.aisimulate.txt") + for _, path := range []string{"pyproject.toml", "benchmarks/pyproject.toml"} { + if got := pinnedAISimulate(t, path); got != canonical { + t.Errorf("%s pins aisimulate %s, but container/deps/requirements.aisimulate.txt pins %s", path, got, canonical) + } + } + + release := releasePattern.FindStringSubmatch(readRepositoryFile(t, "pyproject.toml")) + if release == nil { + t.Fatal("pyproject.toml has no project version") + } + minor := release[1] + mapped, ok := aisimulateVersionByDynamoMinor[minor] + if !ok { + t.Fatalf("Dynamo %s has no entry in aisimulateVersionByDynamoMinor; add %q: %q with the release", minor, minor, canonical) + } + if mapped != canonical { + t.Fatalf("aisimulateVersionByDynamoMinor[%q] = %q, but the repository pins aisimulate %s", minor, mapped, canonical) + } +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index 342e5a436d7f..67eae58feae0 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -27,7 +27,6 @@ import ( "maps" "reflect" "sort" - "strings" "text/template" "time" @@ -1492,52 +1491,11 @@ func (r *DynamoGraphDeploymentRequestReconciler) validateGPUHardwareInfo(ctx con "or set spec.hardware.{gpuSku,vramMb,numGpusPerNode} explicitly.") } -// GetGPUDiscoveryFailureReason classifies a GPU discovery error and -// returns a stable, actionable reason string suitable for structured logging. -// -// The classification is based on known error message patterns produced during: -// - DCGM exporter pod discovery -// - Helm-based GPU operator and DCGM discovery -// - Metrics scraping -// - Prometheus parsing -// -// If the error does not match any known category, "unknown" is returned. +// GetGPUDiscoveryFailureReason classifies a GPU discovery error into a stable reason string +// for structured logging. The classification lives in the gpu package so the v1beta2 +// admission path reports the same reasons. func GetGPUDiscoveryFailureReason(err error) string { - const unknownReason = "unknown" - - if err == nil { - return unknownReason - } - errMsg := strings.ToLower(err.Error()) - - switch { - case strings.Contains(errMsg, "list pods"): - return "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)" - case strings.Contains(errMsg, "gpu operator is not installed"): - return "GPU Operator not installed in expected namespace" - case strings.Contains(errMsg, "helm init failed"): - return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" - case strings.Contains(errMsg, "timeout waiting for dcgm exporter pods"): - return "timeout while waiting for DCGM exporter pods to become ready" - case strings.Contains(errMsg, "http get"): - return "failed to reach DCGM metrics endpoint on pod (network/port issue)" - case strings.Contains(errMsg, "metrics endpoint") && - strings.Contains(errMsg, "status"): - return "DCGM pod metrics endpoint returned non-200 status" - case strings.Contains(errMsg, "parse prometheus metrics"): - return "failed to parse dcgm Prometheus metrics (invalid format)" - case strings.Contains(errMsg, "no gpus detected"): - return "no GPUs detected in dcgm metrics (GPU model or metrics missing)" - case strings.Contains(errMsg, "dcgm is not enabled in the GPU Operator"): - return "DCGM is not enabled in the GPU Operator (check GPU Operator configuration and permissions)" - case strings.Contains(errMsg, "failed to scrape any dcgm exporter pod"): - return "failed to scrape any dcgm exporter pod (check DCGM exporter pod status and network connectivity)" - case strings.Contains(errMsg, "no gpu metrics could be parsed from any dcgm pod"): - return "no GPU metrics could be parsed from any DCGM pod (check DCGM exporter pod status and network connectivity)" - case strings.Contains(errMsg, "failed to create helm path"): - return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" - } - return unknownReason + return gpu.DiscoveryFailureReason(err) } // createProfilingJob creates a Kubernetes Job for profiling using SyncResource. @@ -2000,41 +1958,21 @@ func (r *DynamoGraphDeploymentRequestReconciler) discoverHardwareForEnrichment(c return nil, nil } - var dcgmErr error - if r.GPUDiscovery != nil { - discoveredInfo, err := r.GPUDiscovery.DiscoverGPUsFromDCGMFiltered(ctx, reader, r.GPUDiscoveryCache, hw.GPUSKU) - if err == nil { - return discoveredInfo, nil - } - dcgmErr = err - - reason := GetGPUDiscoveryFailureReason(err) - logger.Info("DCGM discovery failed, falling back to node-label discovery", - "reason", reason, "error", err.Error()) + discoverer := &gpu.Discoverer{ + Reader: reader, + DCGM: r.GPUDiscovery, + Cache: r.GPUDiscoveryCache, + NodeLabels: r.gpuDiscoveryEnabled, } - - if !r.gpuDiscoveryEnabled() { + info, err := discoverer.Discover(ctx, hw.GPUSKU) + if err != nil { if discoveryRequired { - if dcgmErr != nil { - return nil, fmt.Errorf("auto-discovery failed: %w", dcgmErr) - } - return nil, fmt.Errorf("auto-discovery failed: node-label discovery is disabled") + return nil, err } - logger.Info("Optional hardware metadata discovery skipped because node-label discovery is disabled") + logger.Info("Optional hardware metadata discovery unavailable; leaving unset fields unchanged", "error", err.Error()) return nil, nil } - - discoveredInfo, err := gpu.DiscoverGPUsFiltered(ctx, reader, hw.GPUSKU) - if err == nil { - return discoveredInfo, nil - } - - logger.Info("Node-label discovery also failed", "error", err.Error()) - if discoveryRequired { - return nil, fmt.Errorf("auto-discovery failed: %w", err) - } - logger.Info("Optional hardware metadata discovery unavailable; leaving unset fields unchanged") - return nil, nil + return info, nil } // extractModelCachePVCConfig reads model cache PVC settings from the typed v1beta1 spec. diff --git a/deploy/operator/internal/controller/suite_envtest_test.go b/deploy/operator/internal/controller/suite_envtest_test.go index 45651e30d0f0..0a6c4633cc74 100644 --- a/deploy/operator/internal/controller/suite_envtest_test.go +++ b/deploy/operator/internal/controller/suite_envtest_test.go @@ -9,10 +9,14 @@ package controller import ( + "context" "os" "testing" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,6 +28,16 @@ import ( const admissionBypassUsername = "operatorenv-controller-admission-bypass" +// The controller specs create native v1beta2 Requests without an image or hardware. Admission +// completes them from the operator's configuration, as it does in production. +const controllerAISimulateImage = "registry.example/aisimulate:1.2.0" + +type fixedHardwareDiscoverer struct{} + +func (fixedHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, nil +} + var ( sharedEnv = operatorenv.New(operatorenv.Options{ Admission: operatorenv.AdmissionWebhooks{ @@ -44,6 +58,10 @@ func setupProductionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOpti RuntimeConfig: opts.RuntimeConfig, OperatorVersion: opts.OperatorVersion, OperatorPrincipal: opts.OperatorPrincipal, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + AISimulateImage: controllerAISimulateImage, + Hardware: fixedHardwareDiscoverer{}, + }, }) } diff --git a/deploy/operator/internal/dynamo/component_common.go b/deploy/operator/internal/dynamo/component_common.go index 3f09b3400d60..cfa186de1ac4 100644 --- a/deploy/operator/internal/dynamo/component_common.go +++ b/deploy/operator/internal/dynamo/component_common.go @@ -10,7 +10,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" controller_common "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" corev1 "k8s.io/api/core/v1" ) @@ -69,7 +69,7 @@ type ComponentContext struct { Infrastructure configv1alpha1.InfrastructureConfiguration EPPConfig *v1beta1.EPPConfig WorkerHashSuffix string - RuntimeVersion *runtimeversion.Version + RuntimeVersion *versionoverride.Version Annotations map[string]string } diff --git a/deploy/operator/internal/dynamo/component_frontend_test.go b/deploy/operator/internal/dynamo/component_frontend_test.go index d51d4fd3d021..2c38ee903090 100644 --- a/deploy/operator/internal/dynamo/component_frontend_test.go +++ b/deploy/operator/internal/dynamo/component_frontend_test.go @@ -11,7 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,13 +21,13 @@ func TestFrontendDefaultsStartupProbeVersionGate(t *testing.T) { tests := []struct { name string origin string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantEnabled bool }{ {name: "unknown runtime on new graph", origin: "1.7.0"}, - {name: "older runtime on new graph", origin: "1.7.0", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 9}}, - {name: "released 1.6 graph preserves existing defaults", origin: "1.6.0", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 6}}, - {name: "new graph with minimum supported runtime", origin: "1.7.0", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 6}, wantEnabled: true}, + {name: "older runtime on new graph", origin: "1.7.0", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 9}}, + {name: "released 1.6 graph preserves existing defaults", origin: "1.6.0", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 6}}, + {name: "new graph with minimum supported runtime", origin: "1.7.0", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 6}, wantEnabled: true}, } for _, tt := range tests { diff --git a/deploy/operator/internal/dynamo/component_namespace_test.go b/deploy/operator/internal/dynamo/component_namespace_test.go index 3ea504f8d441..cc2741d95370 100644 --- a/deploy/operator/internal/dynamo/component_namespace_test.go +++ b/deploy/operator/internal/dynamo/component_namespace_test.go @@ -11,7 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -22,13 +22,13 @@ func TestComponentNamespacePrefixRuntimeCompatibility(t *testing.T) { t.Log("render frontend and EPP defaults across the strict-discovery release boundary") versions := []struct { name string - version *runtimeversion.Version + version *versionoverride.Version strict bool }{ {name: "unknown runtime"}, - {name: "older runtime", version: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 9}}, - {name: "supported runtime", version: &runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, strict: true}, - {name: "newer runtime", version: &runtimeversion.Version{Major: 1, Minor: 7, Patch: 0}, strict: true}, + {name: "older runtime", version: &versionoverride.Version{Major: 1, Minor: 5, Patch: 9}}, + {name: "supported runtime", version: &versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, strict: true}, + {name: "newer runtime", version: &versionoverride.Version{Major: 1, Minor: 7, Patch: 0}, strict: true}, } for _, componentType := range []string{commonconsts.ComponentTypeFrontend, commonconsts.ComponentTypeEPP} { @@ -59,7 +59,7 @@ func TestLegacyEPPDoesNotEnableStrictNamespacePrefix(t *testing.T) { container, err := NewEPPDefaults().GetBaseContainer(ComponentContext{ DynamoNamespace: "default-foo", ComponentType: commonconsts.ComponentTypeEPP, - RuntimeVersion: &runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + RuntimeVersion: &versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, EPPConfig: &v1beta1.EPPConfig{}, }) require.NoError(t, err) diff --git a/deploy/operator/internal/dynamo/component_worker_test.go b/deploy/operator/internal/dynamo/component_worker_test.go index aa39447c5b40..2c399d60724e 100644 --- a/deploy/operator/internal/dynamo/component_worker_test.go +++ b/deploy/operator/internal/dynamo/component_worker_test.go @@ -9,14 +9,14 @@ import ( "testing" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "github.com/stretchr/testify/require" ) func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantEnabled string }{ { @@ -25,17 +25,17 @@ func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { }, { name: "older runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 4, Patch: 9}, wantEnabled: "false", }, { name: "minimum supported runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, wantEnabled: "true", }, { name: "newer runtime", - runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, wantEnabled: "true", }, } @@ -64,7 +64,7 @@ func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { func TestWorkerDefaultsLivenessFailureThresholdVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *runtimeversion.Version + runtimeVersion *versionoverride.Version wantFailureCount int32 }{ { @@ -73,17 +73,17 @@ func TestWorkerDefaultsLivenessFailureThresholdVersionGate(t *testing.T) { }, { name: "older runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 4, Patch: 9}, wantFailureCount: 1, }, { name: "minimum supported runtime", - runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, wantFailureCount: 3, }, { name: "newer runtime", - runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + runtimeVersion: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, wantFailureCount: 3, }, } diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index 09692c41b4ff..30cb0f0ebb5e 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -44,7 +44,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/compatibility" gms "github.com/ai-dynamo/dynamo/deploy/operator/internal/gms" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/imdario/mergo" "google.golang.org/protobuf/types/known/wrapperspb" @@ -2163,8 +2163,8 @@ func mergeFrontendSidecarDefaults(podSpec *corev1.PodSpec, sidecarName string, p } // Resolve the frontend image independently of the component runtime. - var resolvedRuntimeVersion *runtimeversion.Version - if version, err := runtimeversion.ParseImageVersion(podSpec.Containers[i].Image); err == nil { + var resolvedRuntimeVersion *versionoverride.Version + if version, err := versionoverride.ParseImageVersion(podSpec.Containers[i].Image); err == nil { resolvedRuntimeVersion = &version } @@ -2256,8 +2256,8 @@ func generateComponentContext(component *v1beta1.DynamoComponentDeploymentShared workerHashSuffix = workerHash } - var resolvedRuntimeVersion *runtimeversion.Version - if version, err := runtimeversion.Resolve(runtimeImage, component.RuntimeVersionOverride); err == nil { + var resolvedRuntimeVersion *versionoverride.Version + if version, err := versionoverride.Resolve(runtimeImage, component.RuntimeVersionOverride); err == nil { resolvedRuntimeVersion = &version } else if component.RuntimeVersionOverride != "" { return ComponentContext{}, fmt.Errorf("resolve runtime version override: %w", err) diff --git a/deploy/operator/internal/dynamo/hash.go b/deploy/operator/internal/dynamo/hash.go index 9f165f36ebaf..e7e7f44ea2f6 100644 --- a/deploy/operator/internal/dynamo/hash.go +++ b/deploy/operator/internal/dynamo/hash.go @@ -27,7 +27,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/utils/ptr" ) @@ -35,7 +35,7 @@ import ( const dgdWorkerHashPlaceholderValue = "worker-hash-placeholder" // Preserve hashes for runtimes released before version hashing. -var minimumHashedRuntimeVersion = runtimeversion.Version{Major: 1, Minor: 5, Patch: 0} +var minimumHashedRuntimeVersion = versionoverride.Version{Major: 1, Minor: 5, Patch: 0} // ComputeDGDWorkersSpecHash computes the v2 worker hash from the worker DCDs // generated by the same DGD-to-DCD path used by reconciliation. The active @@ -228,7 +228,7 @@ func resolvedRuntimeVersionForHash(component *v1beta1.DynamoComponentDeploymentS if runtime := GetDynamoContainer(component); runtime != nil { image = runtime.Image } - version, err := runtimeversion.Resolve(image, component.RuntimeVersionOverride) + version, err := versionoverride.Resolve(image, component.RuntimeVersionOverride) if err != nil || version.Compare(minimumHashedRuntimeVersion) < 0 { return "" } diff --git a/deploy/operator/internal/features/runtime/gate.go b/deploy/operator/internal/features/runtime/gate.go index 805099b09e95..66a3aedab213 100644 --- a/deploy/operator/internal/features/runtime/gate.go +++ b/deploy/operator/internal/features/runtime/gate.go @@ -5,16 +5,16 @@ package runtime -import "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" +import "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" // Gate controls runtime-version-dependent operator behavior. type Gate struct { Name string - MinRuntimeVersion runtimeversion.Version + MinRuntimeVersion versionoverride.Version } // Enabled reports whether a known runtime version meets the feature threshold. -func (g Gate) Enabled(version *runtimeversion.Version) bool { +func (g Gate) Enabled(version *versionoverride.Version) bool { if version == nil { return false } diff --git a/deploy/operator/internal/features/runtime/gate_test.go b/deploy/operator/internal/features/runtime/gate_test.go index c3919a34b988..b9ea9db53738 100644 --- a/deploy/operator/internal/features/runtime/gate_test.go +++ b/deploy/operator/internal/features/runtime/gate_test.go @@ -8,20 +8,20 @@ package runtime import ( "testing" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" ) func TestGateEnabled(t *testing.T) { t.Log("define a feature introduced by Dynamo runtime 1.4.0") gate := Gate{ Name: "TestFeature", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 4, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 4, Patch: 0}, } t.Log("define versions below, at, and above the feature threshold") tests := []struct { name string - version *runtimeversion.Version + version *versionoverride.Version want bool }{ { @@ -29,16 +29,16 @@ func TestGateEnabled(t *testing.T) { }, { name: "older runtime", - version: &runtimeversion.Version{Major: 1, Minor: 3, Patch: 9}, + version: &versionoverride.Version{Major: 1, Minor: 3, Patch: 9}, }, { name: "minimum supported runtime", - version: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 0}, + version: &versionoverride.Version{Major: 1, Minor: 4, Patch: 0}, want: true, }, { name: "newer runtime", - version: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, + version: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, want: true, }, } diff --git a/deploy/operator/internal/features/runtime/gates.go b/deploy/operator/internal/features/runtime/gates.go index 91970f5d0cf9..ee6b782562c1 100644 --- a/deploy/operator/internal/features/runtime/gates.go +++ b/deploy/operator/internal/features/runtime/gates.go @@ -5,22 +5,21 @@ package runtime -import "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" +import "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" var ( // StrictNamespacePrefix gates operator-managed discovery isolation. StrictNamespacePrefix = Gate{ Name: "StrictNamespacePrefix", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, } - // CanaryHealthChecks gates the canary health-check rendering defaults. // Runtime 1.5.0 is the first version whose resolved runtime version is // included in the worker hash, so enabling the feature cannot silently // change an existing worker generation. CanaryHealthChecks = Gate{ Name: "CanaryHealthChecks", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // IncreasedWorkerFailureThreshold gates the worker liveness default. @@ -29,7 +28,7 @@ var ( // change an existing worker generation. IncreasedWorkerFailureThreshold = Gate{ Name: "IncreasedWorkerFailureThreshold", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // NativeRustEPP gates the native Rust EPP requirement: runtime versions at @@ -38,7 +37,7 @@ var ( // 1.5.0, the first release shipping the native Rust EPP image. NativeRustEPP = Gate{ Name: "NativeRustEPP", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, } // PlannerDGDComponentStatus gates Planner discovery of model, runtime, @@ -48,6 +47,6 @@ var ( // PodTemplates. PlannerDGDComponentStatus = Gate{ Name: "PlannerDGDComponentStatus", - MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, } ) diff --git a/deploy/operator/internal/gpu/dcgm_phase_context_test.go b/deploy/operator/internal/gpu/dcgm_phase_context_test.go new file mode 100644 index 000000000000..ba4a86fe6fb6 --- /dev/null +++ b/deploy/operator/internal/gpu/dcgm_phase_context_test.go @@ -0,0 +1,41 @@ +package gpu + +import ( + "context" + "testing" + "time" +) + +func TestDCGMPhaseContextReservesTimeForTheFallback(t *testing.T) { + outer, cancelOuter := context.WithTimeout(t.Context(), 4*time.Second) + defer cancelOuter() + outerDeadline, _ := outer.Deadline() + + t.Log("With the fallback enabled the DCGM phase ends well before the outer deadline") + phase, cancel := dcgmPhaseContext(outer, func() bool { return true }) + defer cancel() + phaseDeadline, ok := phase.Deadline() + if !ok { + t.Fatal("phase context has no deadline") + } + if remaining := outerDeadline.Sub(phaseDeadline); remaining < time.Second { + t.Fatalf("only %v is left for the fallback, want at least 1s", remaining) + } + + t.Log("Without the fallback the whole context is used") + same, cancelSame := dcgmPhaseContext(outer, func() bool { return false }) + defer cancelSame() + if same != outer { + t.Fatal("context was narrowed although there is no fallback to reserve time for") + } + + t.Log("A context without a deadline is left alone") + bare, cancelBare := dcgmPhaseContext(t.Context(), func() bool { + t.Fatal("the fallback callback was asked although the context has no deadline") + return true + }) + defer cancelBare() + if _, hasDeadline := bare.Deadline(); hasDeadline { + t.Fatal("a deadline was invented for a context that had none") + } +} diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go index 57e762506642..97233ca7d81f 100644 --- a/deploy/operator/internal/gpu/discovery.go +++ b/deploy/operator/internal/gpu/discovery.go @@ -182,6 +182,12 @@ type GPUInfo struct { Interconnect string // Primary GPU-to-GPU interconnect technology used within the node (e.g., "nvlink" for high-bandwidth links or "pcie" for standard bus-based communication) InterconnectTier string // Qualitative or platform-specific classification of the interconnect (e.g., NVLink generation, topology tier, or vendor-defined performance level) NVLinkLinks int // Number of NVLink connections per GPU (0 if NVLink is not present or interconnect is PCIe-only) + + // OtherModels lists the GPU models of the nodes that unfiltered discovery left out because + // their SKU differs from System. Discovery selects the node with the most GPUs, so a non-empty + // list means System is a choice among several GPU types, not a fact about the cluster. It is + // empty for a filtered discovery and for a cluster of one SKU. + OtherModels []string } type ScrapeMetricsFunc func(ctx context.Context, endpoint string) (*GPUInfo, error) @@ -452,6 +458,8 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, if err != nil { cloudProvider = CloudProviderUnknown } + bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, len(allNodes), + func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return allNodes[i].sku, allNodes[i].info.Model }) bestNode.System = bestSKU bestNode.CloudProvider = cloudProvider bestNode.NodesWithGPUs = nodesWithGPUs @@ -923,6 +931,8 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK rdmaDetected = true } + bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, len(allNodes), + func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return allNodes[i].sku, allNodes[i].info.Model }) bestNode.System = bestSKU bestNode.NodesWithGPUs = nodesWithGPUs bestNode.TotalGPUs = totalGPUs diff --git a/deploy/operator/internal/gpu/hardware_discoverer.go b/deploy/operator/internal/gpu/hardware_discoverer.go new file mode 100644 index 000000000000..988ffb428d72 --- /dev/null +++ b/deploy/operator/internal/gpu/hardware_discoverer.go @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gpu + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// Discoverer finds the cluster's GPUs the way the DGDR controller does: DCGM first, then +// node labels when node-label discovery is enabled. It is a plain value so callers other +// than the v1beta1 reconciler (the v1beta2 admission webhook) share the same order and the +// same error wording. +type Discoverer struct { + Reader client.Reader + // DCGM scrapes the DCGM exporter. Nil skips it. + DCGM *GPUDiscovery + Cache *GPUDiscoveryCache + // NodeLabels reports whether the node-label fallback is enabled (the GPUDiscovery feature + // gate). It is a func so the gate is only read when DCGM did not answer, as the v1beta1 + // reconciler always did. Nil means disabled. + NodeLabels func() bool +} + +// Discover returns the discovered GPUs, restricted to filterSKU when it is set. An error +// means no backend produced a result. +func (d *Discoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*GPUInfo, error) { + logger := log.FromContext(ctx) + if d == nil || d.Reader == nil { + return nil, errors.New("auto-discovery failed: APIReader is not configured") + } + + var dcgmErr error + if d.DCGM != nil { + // A slow exporter must not leave the node-label fallback with an expired context. + dcgmCtx, cancel := dcgmPhaseContext(ctx, d.NodeLabels) + info, err := d.DCGM.DiscoverGPUsFromDCGMFiltered(dcgmCtx, d.Reader, d.Cache, filterSKU) + cancel() + if err == nil { + return info, nil + } + dcgmErr = err + logger.Info("DCGM discovery failed, falling back to node-label discovery", + "reason", DiscoveryFailureReason(err), "error", err.Error()) + } + + if d.NodeLabels == nil || !d.NodeLabels() { + if dcgmErr != nil { + return nil, fmt.Errorf("auto-discovery failed: %w", dcgmErr) + } + return nil, errors.New("auto-discovery failed: node-label discovery is disabled") + } + + info, err := DiscoverGPUsFiltered(ctx, d.Reader, filterSKU) + if err != nil { + logger.Info("Node-label discovery also failed", "error", err.Error()) + return nil, fmt.Errorf("auto-discovery failed: %w", err) + } + return info, nil +} + +// KnownSKU returns the hardware catalog name of the discovered GPU, or empty when the model +// is not one the catalog knows. Unlike the v1beta1 reconciler it never falls back to the raw +// model string: a v1beta2 SKU is an AISimulate catalog name, and an unknown name would only +// fail later inside the search. +func KnownSKU(info *GPUInfo) nvidiacomv1beta1.GPUSKUType { + if info == nil { + return "" + } + if info.System != "" { + return info.System + } + return InferHardwareSystem(info.Model) +} + +// DiscoveryFailureReason classifies a GPU discovery error and +// returns a stable, actionable reason string suitable for structured logging. +// +// The classification is based on known error message patterns produced during: +// - DCGM exporter pod discovery +// - Helm-based GPU operator and DCGM discovery +// - Metrics scraping +// - Prometheus parsing +// +// If the error does not match any known category, "unknown" is returned. +func DiscoveryFailureReason(err error) string { + const unknownReason = "unknown" + + if err == nil { + return unknownReason + } + errMsg := strings.ToLower(err.Error()) + + switch { + case strings.Contains(errMsg, "list pods"): + return "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)" + case strings.Contains(errMsg, "gpu operator is not installed"): + return "GPU Operator not installed in expected namespace" + case strings.Contains(errMsg, "helm init failed"): + return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" + case strings.Contains(errMsg, "timeout waiting for dcgm exporter pods"): + return "timeout while waiting for DCGM exporter pods to become ready" + case strings.Contains(errMsg, "http get"): + return "failed to reach DCGM metrics endpoint on pod (network/port issue)" + case strings.Contains(errMsg, "metrics endpoint") && + strings.Contains(errMsg, "status"): + return "DCGM pod metrics endpoint returned non-200 status" + case strings.Contains(errMsg, "parse prometheus metrics"): + return "failed to parse dcgm Prometheus metrics (invalid format)" + case strings.Contains(errMsg, "no gpus detected"): + return "no GPUs detected in dcgm metrics (GPU model or metrics missing)" + case strings.Contains(errMsg, "dcgm is not enabled in the gpu operator"): + return "DCGM is not enabled in the GPU Operator (check GPU Operator configuration and permissions)" + case strings.Contains(errMsg, "failed to scrape any dcgm exporter pod"): + return "failed to scrape any dcgm exporter pod (check DCGM exporter pod status and network connectivity)" + case strings.Contains(errMsg, "no gpu metrics could be parsed from any dcgm pod"): + return "no GPU metrics could be parsed from any DCGM pod (check DCGM exporter pod status and network connectivity)" + case strings.Contains(errMsg, "failed to create helm path"): + return "failed to initialize Helm client (RBAC, kubeconfig, or Helm driver issue)" + } + return unknownReason +} + +// dcgmPhaseContext bounds the DCGM phase of discovery. When ctx has a deadline and fallbackEnabled +// reports that the node-label fallback is enabled, the DCGM phase gets half of what remains, so a slow exporter cannot +// consume the whole admission budget and leave the fallback with an expired context. Otherwise +// ctx is returned unchanged. The returned cancel function must be called. +func dcgmPhaseContext(ctx context.Context, fallbackEnabled func() bool) (context.Context, context.CancelFunc) { + deadline, ok := ctx.Deadline() + // fallbackEnabled is only asked when there is a deadline to share: without one nothing is reserved, + // and the callback may depend on state that a caller without a deadline never configured. + if !ok || fallbackEnabled == nil || !fallbackEnabled() { + return ctx, func() {} + } + remaining := time.Until(deadline) + if remaining <= 0 { + return ctx, func() {} + } + return context.WithTimeout(ctx, remaining/2) +} diff --git a/deploy/operator/internal/gpu/hardware_discoverer_test.go b/deploy/operator/internal/gpu/hardware_discoverer_test.go new file mode 100644 index 000000000000..ef0493c42197 --- /dev/null +++ b/deploy/operator/internal/gpu/hardware_discoverer_test.go @@ -0,0 +1,90 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gpu + +import ( + "context" + "errors" + "strings" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestDiscovererWithoutReader(t *testing.T) { + var nilDiscoverer *Discoverer + for _, d := range []*Discoverer{nilDiscoverer, {}} { + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "APIReader is not configured") { + t.Fatalf("got %v", err) + } + } +} + +func TestDiscovererWithNoBackend(t *testing.T) { + d := &Discoverer{Reader: fake.NewClientBuilder().Build()} + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "node-label discovery is disabled") { + t.Fatalf("got %v", err) + } +} + +func TestDiscovererNodeLabelsFindNothing(t *testing.T) { + d := &Discoverer{Reader: fake.NewClientBuilder().Build(), NodeLabels: func() bool { return true }} + if _, err := d.Discover(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "auto-discovery failed") { + t.Fatalf("an empty cluster must be an error, got %v", err) + } +} + +func TestKnownSKU(t *testing.T) { + cases := []struct { + name string + info *GPUInfo + want nvidiacomv1beta1.GPUSKUType + }{ + {"nil", nil, ""}, + {"discovered system wins", &GPUInfo{System: "h100_sxm", Model: "NVIDIA H200"}, "h100_sxm"}, + {"inferred from the model", &GPUInfo{Model: "NVIDIA H200"}, "h200_sxm"}, + {"unknown model is not passed through", &GPUInfo{Model: "NVIDIA RTX 6000 Ada"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := KnownSKU(tc.info); got != tc.want { + t.Fatalf("got %q want %q", got, tc.want) + } + }) + } +} + +func TestDiscoveryFailureReason(t *testing.T) { + cases := map[string]string{ + "list pods: forbidden": "failed to list DCGM exporter pods (RBAC/cluster connectivity issue)", + "GPU Operator is not installed": "GPU Operator not installed in expected namespace", + "http get failed": "failed to reach DCGM metrics endpoint on pod (network/port issue)", + "metrics endpoint returned status": "DCGM pod metrics endpoint returned non-200 status", + "no GPUs detected": "no GPUs detected in dcgm metrics (GPU model or metrics missing)", + "something else": "unknown", + } + for msg, want := range cases { + if got := DiscoveryFailureReason(errors.New(msg)); got != want { + t.Errorf("%q: got %q, want %q", msg, got, want) + } + } + if got := DiscoveryFailureReason(nil); got != "unknown" { + t.Errorf("nil: got %q", got) + } +} diff --git a/deploy/operator/internal/gpu/heterogeneous.go b/deploy/operator/internal/gpu/heterogeneous.go new file mode 100644 index 000000000000..312f9e362695 --- /dev/null +++ b/deploy/operator/internal/gpu/heterogeneous.go @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package gpu + +import ( + "sort" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" +) + +// collectOtherModels returns, sorted and without repeats, the models of the nodes whose SKU +// differs from the selected one. A filtered discovery asked for one SKU and returns nothing: +// the nodes of other SKUs were never candidates. at yields the SKU and the model of node i. +func collectOtherModels( + filterSKU, selected nvidiacomv1beta1.GPUSKUType, + nodes int, + at func(i int) (nvidiacomv1beta1.GPUSKUType, string), +) []string { + if filterSKU != "" { + return nil + } + seen := map[string]struct{}{} + for i := 0; i < nodes; i++ { + if sku, model := at(i); sku != selected { + seen[model] = struct{}{} + } + } + if len(seen) == 0 { + return nil + } + models := make([]string, 0, len(seen)) + for model := range seen { + models = append(models, model) + } + sort.Strings(models) + return models +} diff --git a/deploy/operator/internal/gpu/heterogeneous_test.go b/deploy/operator/internal/gpu/heterogeneous_test.go new file mode 100644 index 000000000000..71380d227077 --- /dev/null +++ b/deploy/operator/internal/gpu/heterogeneous_test.go @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package gpu + +import ( + "reflect" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" +) + +func TestCollectOtherModels(t *testing.T) { + type node struct { + sku nvidiacomv1beta1.GPUSKUType + model string + } + tests := []struct { + name string + filter nvidiacomv1beta1.GPUSKUType + selected nvidiacomv1beta1.GPUSKUType + nodes []node + want []string + }{ + {"a cluster of one SKU", "", "h200_sxm", []node{{"h200_sxm", "H200"}, {"h200_sxm", "H200"}}, nil}, + {"a second SKU is reported once, sorted", "", "h200_sxm", + []node{{"h200_sxm", "H200"}, {"h100_sxm", "H100"}, {"b200_sxm", "B200"}, {"h100_sxm", "H100"}}, + []string{"B200", "H100"}}, + {"a model the catalog does not know still counts", "", "h200_sxm", + []node{{"h200_sxm", "H200"}, {"", "Mystery GPU"}}, []string{"Mystery GPU"}}, + {"a filtered discovery never reports", "h200_sxm", "h200_sxm", + []node{{"h200_sxm", "H200"}, {"h100_sxm", "H100"}}, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := collectOtherModels(tt.filter, tt.selected, len(tt.nodes), + func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return tt.nodes[i].sku, tt.nodes[i].model }) + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} diff --git a/deploy/operator/internal/runtimeversion/runtimeversion.go b/deploy/operator/internal/versionoverride/versionoverride.go similarity index 91% rename from deploy/operator/internal/runtimeversion/runtimeversion.go rename to deploy/operator/internal/versionoverride/versionoverride.go index eab91c6c48d8..38c4682d3116 100644 --- a/deploy/operator/internal/runtimeversion/runtimeversion.go +++ b/deploy/operator/internal/versionoverride/versionoverride.go @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package runtimeversion +package versionoverride import ( "cmp" @@ -16,7 +16,7 @@ import ( var imageTagPattern = regexp.MustCompile(`^[vV]?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-((?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$`) -// Version identifies a runtime version by semver core. +// Version identifies a normalized compatibility version. type Version struct { Major uint64 Minor uint64 @@ -38,7 +38,7 @@ func (v Version) Compare(other Version) int { ) } -// Parse returns the runtime version represented by an explicit override. +// Parse returns the compatibility version represented by an explicit override. func Parse(value string) (Version, error) { version, err := semver.StrictNewVersion(value) if err != nil { @@ -47,7 +47,7 @@ func Parse(value string) (Version, error) { return fromSemver(version), nil } -// ParseImageVersion returns a normalized runtime version from an image tag. +// ParseImageVersion returns a normalized compatibility version from an image tag. func ParseImageVersion(image string) (Version, error) { tag := imageTag(image) if tag == "" { diff --git a/deploy/operator/internal/runtimeversion/runtimeversion_test.go b/deploy/operator/internal/versionoverride/versionoverride_test.go similarity index 99% rename from deploy/operator/internal/runtimeversion/runtimeversion_test.go rename to deploy/operator/internal/versionoverride/versionoverride_test.go index edc26679c343..8cccb8005eef 100644 --- a/deploy/operator/internal/runtimeversion/runtimeversion_test.go +++ b/deploy/operator/internal/versionoverride/versionoverride_test.go @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package runtimeversion +package versionoverride import "testing" diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index 07f059d7c9d0..b51ba6fd85e9 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -46,6 +46,9 @@ type DGDRDefaulter struct { // dynamo-planner: tag does not exist (pre-release charts: // rc, nightly); empty keeps the derived default. DefaultImage string + // v1beta2 configures the defaults of Requests submitted as v1beta2, which reach this + // handler as a v1beta1 object carrying the spec.v1beta2 envelope. + v1beta2 DGDRV1beta2Options } // NewDGDRDefaulter creates a new DGDRDefaulter with the given operator version @@ -54,6 +57,21 @@ func NewDGDRDefaulter(operatorVersion, defaultImage string) *DGDRDefaulter { return &DGDRDefaulter{OperatorVersion: operatorVersion, DefaultImage: defaultImage} } +// WithV1beta2 sets the defaults of v1beta2 Requests and returns the defaulter. +func (d *DGDRDefaulter) WithV1beta2(options DGDRV1beta2Options) *DGDRDefaulter { + // The operator refuses to start with a configuration that does not resolve, so a failure here + // only happens to a caller that skipped ValidateAISimulateDefault: keep the image and let + // validation report what the Request lacks. + if image, version, err := resolveAISimulateDefault( + d.OperatorVersion, options.AISimulateImage, options.AISimulateVersionOverride); err == nil { + options.AISimulateImage, options.AISimulateVersionOverride = image, version + } else { + options.AISimulateImage = defaultPlannerImageFor(d.OperatorVersion, options.AISimulateImage) + } + d.v1beta2 = options + return d +} + // Default implements admission.CustomDefaulter. // Native API defaults apply on CREATE and UPDATE. If spec.image is not set on // CREATE, derives a default image from the operator version. @@ -71,6 +89,23 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { if err != nil { return fmt.Errorf("get admission request from context: %w", err) } + requestVersion := internalwebhook.RequestedVersion(req) + + // The original request version decides what spec.v1beta2 means. A native v1beta2 Request + // is defaulted from its carrier on CREATE and UPDATE; only CREATE + // and an UPDATE that changes spec.hardware read the cluster. + if requestVersion == internalwebhook.VersionV1Beta2 && dgdr.Spec.V1Beta2 != nil { + oldSpec, err := decodeOldSpecV1beta2(req.Operation, req.OldObject.Raw) + if err != nil { + return err + } + return defaultDynamoGraphDeploymentRequestV1beta2(ctx, dgdr, req.Operation, oldSpec, d.v1beta2) + } + // A foreign-version view (a carrier written or updated through the wrong API version, or + // a v1beta1 payload written through v1beta2) is read-only; validation enforces that. + if requestVersion == internalwebhook.VersionV1Beta2 || dgdr.Spec.V1Beta2 != nil { + return nil + } defaultDynamoGraphDeploymentRequest(dgdr, dynamoGraphDeploymentRequestDefaultOptions{ operation: req.Operation, @@ -79,6 +114,12 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { return nil } +// defaultPlannerImageFor returns explicitImage when set, else the planner image with a +// canonical operator-version tag, or an empty string when the version cannot be parsed. +func defaultPlannerImageFor(operatorVersion, explicitImage string) string { + return NewDGDRDefaulter(operatorVersion, explicitImage).defaultImageFor() +} + // RegisterWithManager registers the DGDR defaulting webhook with the manager. func (d *DGDRDefaulter) RegisterWithManager(mgr manager.Manager) error { defaulter := internalwebhook.NewLeaseAwareDefaulter(d, internalwebhook.GetExcludedNamespaces()) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go index 0c867c434e89..9d45a2543a8f 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) -func TestDGDRDefaulter_defaultImageFor(t *testing.T) { +func TestDefaultPlannerImageFor(t *testing.T) { // Note: the default planner image is only published starting from Dynamo 1.1.0, // so these tests use 1.1.0 as the earliest known valid version. tests := []struct { @@ -77,10 +77,9 @@ func TestDGDRDefaulter_defaultImageFor(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := NewDGDRDefaulter(tt.operatorVersion, tt.defaultImage) - got := d.defaultImageFor() + got := defaultPlannerImageFor(tt.operatorVersion, tt.defaultImage) if got != tt.expectedImage { - t.Errorf("defaultImageFor() = %q, want %q", got, tt.expectedImage) + t.Errorf("defaultPlannerImageFor() = %q, want %q", got, tt.expectedImage) } }) } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go new file mode 100644 index 000000000000..416fe004f280 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go @@ -0,0 +1,146 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "bytes" + "context" + "encoding/json" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// ctxForRequest builds the admission context of a request the API server converted to +// v1beta1. requestVersion is the version the client used; empty means no conversion. +func ctxForRequest(op admissionv1.Operation, requestVersion string) context.Context { + gvk := nvidiacomv1beta1.DynamoGraphDeploymentRequestGVK + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: op, + Kind: metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}, + }} + if requestVersion != "" { + req.RequestKind = &metav1.GroupVersionKind{Group: gvk.Group, Version: requestVersion, Kind: gvk.Kind} + } + return admission.NewContextWithRequest(context.Background(), req) +} + +const modelOnlyCarrier = `{"model":{"name":"Qwen/Qwen3-32B"}}` + +func carrierRequest() *nvidiacomv1beta1.DynamoGraphDeploymentRequest { + return &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "r", Namespace: "default"}, + Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ + V1Beta2: &apiextensionsv1.JSON{Raw: []byte(modelOnlyCarrier)}, + }, + } +} + +func aisimulateImageOf(t *testing.T, raw []byte) string { + t.Helper() + var spec map[string]any + if err := json.Unmarshal(raw, &spec); err != nil { + t.Fatalf("decode carrier: %v", err) + } + aisimulate, _ := spec["aisimulate"].(map[string]any) + image, _ := aisimulate["image"].(string) + return image +} + +func TestDGDRDefaulter_RoutesByRequestedVersion(t *testing.T) { + for _, tt := range []struct { + name string + options DGDRV1beta2Options + want string + }{ + { + name: "an omitted image is derived from the operator version", + want: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.6.0", + }, + { + name: "an explicit default image overrides derivation", + options: DGDRV1beta2Options{AISimulateImage: "registry.example/aisimulate:custom"}, + want: "registry.example/aisimulate:custom", + }, + } { + t.Run(tt.name, func(t *testing.T) { + d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(tt.options) + dgdr := carrierRequest() + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if got := aisimulateImageOf(t, dgdr.Spec.V1Beta2.Raw); got != tt.want { + t.Fatalf("aisimulate.image = %q, want %q", got, tt.want) + } + }) + } + + d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(DGDRV1beta2Options{}) + + t.Run("RequestKind decides, not the converted Kind", func(t *testing.T) { + dgdr := carrierRequest() + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image != "" { + t.Fatalf("v1beta1 image defaulting touched a v1beta2 request: %q", dgdr.Spec.Image) + } + }) + + for _, version := range []string{"", "v1beta1", "v1alpha1"} { + t.Run("a foreign carrier is never mutated, requested as "+version, func(t *testing.T) { + for _, op := range []admissionv1.Operation{admissionv1.Create, admissionv1.Update} { + dgdr := carrierRequest() + before := append([]byte(nil), dgdr.Spec.V1Beta2.Raw...) + if err := d.Default(ctxForRequest(op, version), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if !bytes.Equal(dgdr.Spec.V1Beta2.Raw, before) { + t.Fatalf("%s: carrier changed from %s to %s", op, before, dgdr.Spec.V1Beta2.Raw) + } + if dgdr.Spec.Image != "" { + t.Fatalf("%s: native v1beta1 defaults reached a foreign carrier", op) + } + } + }) + } + + t.Run("a v1beta1 payload written through v1beta2 is left for validation", func(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{Model: "m"}} + if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image != "" { + t.Fatalf("a foreign v1beta1 payload was defaulted: image %q", dgdr.Spec.Image) + } + }) + + t.Run("a native v1beta1 request keeps its defaults", func(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{Model: "m"}} + if err := d.Default(ctxForRequest(admissionv1.Create, ""), dgdr); err != nil { + t.Fatalf("Default() error = %v", err) + } + if dgdr.Spec.Image == "" { + t.Fatal("native v1beta1 image defaulting stopped working") + } + }) +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go new file mode 100644 index 000000000000..f5140ee376a0 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta1_options.go @@ -0,0 +1,13 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package defaulting + +// DGDRV1beta1Options configures the defaults of v1beta1 Requests. +type DGDRV1beta1Options struct { + // DefaultImage is the profiler image put into spec.image when a Request is created + // without one; empty derives dynamo-planner:. + DefaultImage string +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go new file mode 100644 index 000000000000..37e7098880b3 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -0,0 +1,263 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/utils/ptr" +) + +// defaultMetricV1beta2 is the scalar target an Optimize objective receives when it names none. +const defaultMetricV1beta2 = nvidiacomv1beta2.ObjectiveMetric("Throughput") + +// Defaults of the objects admission creates. Admission is their only owner: the API server +// defaults a mutated object in the webhook's version (v1beta1, whose v1beta2 carrier is +// opaque), so the v1beta2 schema carries no default markers for them. +const ( + defaultStaticISLV1beta2 int32 = 1024 + defaultStaticOSLV1beta2 int32 = 128 + defaultStaticConcurrencyV1beta2 int32 = 10 + defaultMaxCandidatesV1beta2 int32 = 1 +) + +// This file defaults the v1beta2 shape of a DynamoGraphDeploymentRequest. It is separate +// from the canonical defaulting because the v1beta1 hub carries a v1beta2 Request as an +// opaque spec.v1beta2 envelope: conversion cannot apply field-level defaults to it, so the +// handler decodes the envelope, runs this traversal and encodes the result. Names carry the +// API version because the canonical names belong to v1beta1. +// +// Admission owns every v1beta2 default, and each default has this one owner: backends, +// objective.mode, the scalar metric, recommendation.maxCandidates, workload.static, the +// cluster hardware and the AISimulate image. workload.static: {} opts out of the static +// workload. objective.sla has no default: an omitted SLA stays absent. +// spec.aisimulate.parameters never changes a default. It overrides the defaulted typed +// values later, when the input is lowered. + +// defaultDynamoGraphDeploymentRequestV1beta2 defaults the v1beta2 spec that dgdr carries in +// spec.v1beta2. The environment is read once, before any mutation, and only for a CREATE or +// an UPDATE that changes spec.hardware (see resolveHardware). oldSpec is the stored spec on +// UPDATE and nil otherwise. The envelope is rewritten +// only when defaulting changed the spec, so a defaulted object stays byte-identical across +// repeated admissions. +func defaultDynamoGraphDeploymentRequestV1beta2( + ctx context.Context, + dgdr *nvidiacomv1beta1.DynamoGraphDeploymentRequest, + operation admissionv1.Operation, + oldSpec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + options DGDRV1beta2Options, +) error { + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &spec); err != nil { + return fmt.Errorf("decode spec.v1beta2: %w", err) + } + before, err := json.Marshal(&spec) + if err != nil { + return fmt.Errorf("encode spec.v1beta2: %w", err) + } + + defaults, err := options.resolveDefaults(ctx, operation, oldSpec, &spec) + if err != nil { + return admissionVerdictV1beta2(dgdr, err) + } + defaultDynamoGraphDeploymentRequestSpecV1beta2(&spec, defaults) + + after, err := json.Marshal(&spec) + if err != nil { + return fmt.Errorf("encode spec.v1beta2: %w", err) + } + if !bytes.Equal(before, after) { + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: after} + } + return nil +} + +// decodeOldSpecV1beta2 returns the stored v1beta2 spec an UPDATE replaces. The API server +// converts the old object to the same v1beta1 carrier view as the new one, so the spec sits +// in the same spec.v1beta2 envelope. It returns nil when there is nothing to compare with. +func decodeOldSpecV1beta2( + operation admissionv1.Operation, + oldObject []byte, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, error) { + if operation != admissionv1.Update || len(oldObject) == 0 { + return nil, nil + } + var old nvidiacomv1beta1.DynamoGraphDeploymentRequest + if err := json.Unmarshal(oldObject, &old); err != nil { + return nil, fmt.Errorf("decode old object: %w", err) + } + if old.Spec.V1Beta2 == nil { + return nil, nil + } + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(old.Spec.V1Beta2.Raw, &spec); err != nil { + return nil, fmt.Errorf("decode old spec.v1beta2: %w", err) + } + return &spec, nil +} + +// dynamoGraphDeploymentRequestV1beta2DefaultOptions carries the request-scoped values the +// traversal needs. An empty field means the environment had nothing to offer. +type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { + // hardware is the GPU identity and budget discovered from the cluster. + hardware hardwareDefaultsV1beta2 + // aisimulateImage is the operator's resolved AISimulate image. + aisimulateImage string + // aisimulateVersionOverride belongs to aisimulateImage. + aisimulateVersionOverride string +} + +// hardwareDefaultsV1beta2 is the hardware the cluster can offer for the values a Request omits. +type hardwareDefaultsV1beta2 struct { + sku string + budget int32 +} + +// defaultDynamoGraphDeploymentRequestSpecV1beta2 defaults spec in place. It applies on +// CREATE and UPDATE and is idempotent; opts carries hardware only when the environment was +// asked (see resolveHardware). spec must not be nil. Explicit values, including empty +// objects, are kept. The functions below mirror the Go API tree and visit children in +// declaration order. +func defaultDynamoGraphDeploymentRequestSpecV1beta2( + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + defaultDynamoGraphDeploymentRequestNativeSpecV1beta2(&spec.DynamoGraphDeploymentRequestNativeSpec, opts) +} + +func defaultDynamoGraphDeploymentRequestNativeSpecV1beta2( + native *nvidiacomv1beta2.DynamoGraphDeploymentRequestNativeSpec, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + defaultNativeSearchInputV1beta2(&native.NativeSearchInput, opts) +} + +func defaultNativeSearchInputV1beta2( + input *nvidiacomv1beta2.NativeSearchInput, + opts dynamoGraphDeploymentRequestV1beta2DefaultOptions, +) { + if input.Backends == nil { + input.Backends = []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"} + } + + // Hardware has no static default: it exists only when the cluster supplied some. + if input.Hardware == nil && opts.hardware != (hardwareDefaultsV1beta2{}) { + input.Hardware = &nvidiacomv1beta2.HardwareSpec{} + } + if input.Hardware != nil { + defaultHardwareSpecV1beta2(input.Hardware, opts.hardware) + } + + if input.Workload == nil { + input.Workload = &nvidiacomv1beta2.WorkloadSpec{} + } + defaultWorkloadSpecV1beta2(input.Workload) + + if input.Objective == nil { + input.Objective = &nvidiacomv1beta2.ObjectiveSpec{} + } + defaultObjectiveSpecV1beta2(input.Objective) + + // The operator image is persisted with the Request, so spec.aisimulate is created to + // carry it. + if input.AISimulate == nil && opts.aisimulateImage != "" { + input.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} + } + if input.AISimulate != nil { + defaultAISimulateSpecV1beta2(input.AISimulate, opts.aisimulateImage, opts.aisimulateVersionOverride) + } + + if input.Recommendation == nil { + input.Recommendation = &nvidiacomv1beta2.RecommendationSpec{} + } + defaultRecommendationSpecV1beta2(input.Recommendation) +} + +func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, defaults hardwareDefaultsV1beta2) { + if hardware.GPU == nil && defaults != (hardwareDefaultsV1beta2{}) { + hardware.GPU = &nvidiacomv1beta2.GPUHardwareSpec{} + } + if hardware.GPU != nil { + defaultGPUHardwareSpecV1beta2(hardware.GPU, defaults) + } +} + +// defaultGPUHardwareSpecV1beta2 fills each omitted value on its own, so a caller can pin the +// SKU and let admission size the budget. +func defaultGPUHardwareSpecV1beta2(gpu *nvidiacomv1beta2.GPUHardwareSpec, defaults hardwareDefaultsV1beta2) { + if gpu.SKU == nil && defaults.sku != "" { + gpu.SKU = ptr.To(defaults.sku) + } + if gpu.Budget == nil && defaults.budget > 0 { + gpu.Budget = ptr.To(defaults.budget) + } +} + +func defaultWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec) { + // An explicit static: {} is the opt-out, so the static defaults apply only to the object + // created here. + if workload.Static == nil { + workload.Static = &nvidiacomv1beta2.StaticWorkloadSpec{} + defaultStaticWorkloadSpecV1beta2(workload.Static) + } +} + +func defaultStaticWorkloadSpecV1beta2(static *nvidiacomv1beta2.StaticWorkloadSpec) { + static.ISL = ptr.To(defaultStaticISLV1beta2) + static.OSL = ptr.To(defaultStaticOSLV1beta2) + static.Concurrency = ptr.To(defaultStaticConcurrencyV1beta2) +} + +// defaultObjectiveSpecV1beta2 defaults the mode and gives an Optimize objective without a +// metric the default scalar target. It reads the typed objective alone: a goal.target in +// spec.aisimulate.parameters overrides the completed typed value when the input is lowered, +// so it never suppresses this default. +func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { + if objective.Mode == "" { + objective.Mode = nvidiacomv1beta2.ObjectiveModeOptimize + } + if objective.Mode == nvidiacomv1beta2.ObjectiveModeOptimize && objective.Metric == nil { + objective.Metric = ptr.To(defaultMetricV1beta2) + } +} + +// defaultAISimulateSpecV1beta2 persists the operator's AISimulate artifact as one pair. The +// override follows the image: it is applied only when admission supplies the image. A caller's +// image derives its own version or brings its own override, and a caller's override always wins. +func defaultAISimulateSpecV1beta2(aisimulate *nvidiacomv1beta2.AISimulateSpec, image, versionOverride string) { + if aisimulate.Image != "" || image == "" { + return + } + aisimulate.Image = image + if aisimulate.VersionOverride == "" { + aisimulate.VersionOverride = versionOverride + } +} + +func defaultRecommendationSpecV1beta2(recommendation *nvidiacomv1beta2.RecommendationSpec) { + if recommendation.MaxCandidates == 0 { + recommendation.MaxCandidates = defaultMaxCandidatesV1beta2 + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact.go new file mode 100644 index 000000000000..613a3f9d0ca2 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact.go @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "fmt" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" +) + +// resolveAISimulateDefault returns the AISimulate artifact admission persists as one pair: the +// configured planner image, or the official one derived from the operator version when none is +// configured, and the AISimulate version it carries. The explicit override wins, the derived image +// included; otherwise the version is the one bundled in the Dynamo release named by the image tag +// (aisimulate.ResolveVersion). Both are empty when there is nothing to persist: no image is +// configured and the operator version names no release. An override then has no image to pair with. +func resolveAISimulateDefault(operatorVersion, image, versionOverride string) (resolvedImage, version string, err error) { + resolvedImage = defaultPlannerImageFor(operatorVersion, image) + if resolvedImage == "" { + if versionOverride != "" { + return "", "", fmt.Errorf("the AISimulate default version override needs an image: operator version %q "+ + "names no release to derive the planner image from; set the AISimulate default image", operatorVersion) + } + return "", "", nil + } + version, err = aisimulate.ResolveVersion(resolvedImage, versionOverride) + if err != nil { + return "", "", fmt.Errorf("the AISimulate default image %q cannot be used as it is: %w; "+ + "set the AISimulate default version override to the AISimulate version it bundles", resolvedImage, err) + } + return resolvedImage, version, nil +} + +// ValidateAISimulateDefault rejects an operator configuration whose default AISimulate artifact +// admission could not persist: an image, configured or derived from the operator version, whose +// Dynamo release has no bundled AISimulate version in the release contract needs the override that +// names it, and an override without any image to pair with is rejected too. An override with an +// empty image configuration is valid: it pairs with the official planner image. +func ValidateAISimulateDefault(operatorVersion, image, versionOverride string) error { + _, _, err := resolveAISimulateDefault(operatorVersion, image, versionOverride) + return err +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact_test.go new file mode 100644 index 000000000000..9ec9339eee89 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact_test.go @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" +) + +func TestDefaultAISimulateFollowsTheOwnershipOfTheImage(t *testing.T) { + const image = "registry.example/aisimulate:nightly" + tests := []struct { + name string + spec nvidiacomv1beta2.AISimulateSpec + wantImage string + wantOverride string + }{ + {"the operator's pair is persisted together", nvidiacomv1beta2.AISimulateSpec{}, image, "1.2.0"}, + {"a caller's image gets no operator override", + nvidiacomv1beta2.AISimulateSpec{Image: "example/own:v1.0.0"}, "example/own:v1.0.0", ""}, + {"a caller's override wins next to the operator's image", + nvidiacomv1beta2.AISimulateSpec{VersionOverride: "2.0.0"}, image, "2.0.0"}, + {"a caller's pair is kept", + nvidiacomv1beta2.AISimulateSpec{Image: "example/own:x", VersionOverride: "3.0.0"}, "example/own:x", "3.0.0"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := tt.spec + defaultAISimulateSpecV1beta2(&spec, image, "1.2.0") + if spec.Image != tt.wantImage || spec.VersionOverride != tt.wantOverride { + t.Fatalf("got %q/%q, want %q/%q", spec.Image, spec.VersionOverride, tt.wantImage, tt.wantOverride) + } + }) + } +} + +func TestValidateAISimulateDefault(t *testing.T) { + const ( + mapped = "registry.example/aisimulate:1.6.0" + unmapped = "registry.example/aisimulate:1.2.3" + nightly = "registry.example/aisimulate:nightly" + bundled = "0.13.0" + unmappedV = "0.9.0" + ) + tests := []struct { + name, operatorVersion, image, override string + wantErr bool + }{ + {"the derived planner image of a mapped release", "1.6.0", "", "", false}, + {"the derived planner image of a prerelease", "1.6.0-rc.1", "", "", false}, + {"the derived planner image of a release without an entry", "1.99.0", "", "", true}, + {"an override rescues the derived image", "1.99.0", "", unmappedV, false}, + {"an override with an empty image configuration", "1.6.0", "", bundled, false}, + {"an operator version that names no release", "unknown", "", "", false}, + {"an override with nothing to pair with", "unknown", "", bundled, true}, + {"a mapped tag", "1.6.0", mapped, "", false}, + {"a mapped tag with an override", "1.6.0", mapped, "0.13.1", false}, + {"a release without an entry", "1.6.0", unmapped, "", true}, + {"a release without an entry, with its override", "1.6.0", unmapped, unmappedV, false}, + {"a non-release tag with its override", "1.6.0", nightly, bundled, false}, + {"a non-release tag without an override", "1.6.0", nightly, "", true}, + {"an override that is not a version", "1.6.0", nightly, "bad", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := ValidateAISimulateDefault(tt.operatorVersion, tt.image, tt.override); (err != nil) != tt.wantErr { + t.Fatalf("error = %v, want error: %v", err, tt.wantErr) + } + }) + } +} + +func TestWithV1beta2PersistsThePlannerImageWithItsBundledAISimulateVersion(t *testing.T) { + derived := defaultImage + ":1.6.0" + bundled, err := aisimulate.ResolveVersion(derived, "") + if err != nil { + t.Fatalf("the release contract must map the operator version used here: %v", err) + } + tests := []struct { + name string + options DGDRV1beta2Options + wantImage string + wantOverride string + }{ + {"the derived image gets the bundled version", DGDRV1beta2Options{}, derived, bundled}, + {"the explicit override wins with the derived image", + DGDRV1beta2Options{AISimulateVersionOverride: "0.13.1"}, derived, "0.13.1"}, + {"a configured mapped image gets the bundled version", + DGDRV1beta2Options{AISimulateImage: "registry.example/aisimulate:1.6.0"}, + "registry.example/aisimulate:1.6.0", bundled}, + {"a configured nightly image keeps its override", + DGDRV1beta2Options{AISimulateImage: "registry.example/aisimulate:nightly", AISimulateVersionOverride: "0.13.0"}, + "registry.example/aisimulate:nightly", "0.13.0"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := NewDGDRDefaulter("1.6.0", "").WithV1beta2(tt.options).v1beta2 + if got.AISimulateImage != tt.wantImage || got.AISimulateVersionOverride != tt.wantOverride { + t.Fatalf("got %q/%q, want %q/%q", got.AISimulateImage, got.AISimulateVersionOverride, tt.wantImage, tt.wantOverride) + } + }) + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go new file mode 100644 index 000000000000..d4cbbc621047 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "context" + "errors" + "strings" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + admissionv1 "k8s.io/api/admission/v1" +) + +func hardwareOfV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.HardwareSpec { + return &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} +} + +func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { + str := func(s string) *string { return &s } + num := func(n int32) *int32 { return &n } + h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} + h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8} + tests := []struct { + name string + hardware *nvidiacomv1beta2.HardwareSpec + found gpu.GPUInfo + err error + wantSKU string + wantBudget int32 + wantCalls int + wantFilter string + wantErr string + }{ + {"omitted hardware discovers the SKU and the budget", nil, h200x8, nil, "h200_sxm", 8, 1, "", ""}, + {"an empty hardware object is automatic discovery", &nvidiacomv1beta2.HardwareSpec{}, h200x8, nil, "h200_sxm", 8, 1, "", ""}, + {"an empty gpu object is automatic discovery", + &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{}}, h200x8, nil, "h200_sxm", 8, 1, "", ""}, + {"the discovered budget is capped", nil, h200x64, nil, "h200_sxm", 32, 1, "", ""}, + {"a SKU alone keeps the SKU, filters the cluster and persists the capped budget", + hardwareOfV1beta2(str("h200_sxm"), nil), h200x64, nil, "", 32, 1, "h200_sxm", ""}, + {"a budget alone keeps the budget and discovers the SKU", hardwareOfV1beta2(nil, num(4)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, + {"an explicit budget above the cluster size is kept", hardwareOfV1beta2(nil, num(64)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, + {"an explicit budget above the cap is kept", hardwareOfV1beta2(str("h200_sxm"), num(64)), h200x8, nil, "", 0, 0, "", ""}, + {"a SKU and a budget never reach the cluster", hardwareOfV1beta2(str("h100_sxm"), num(4)), gpu.GPUInfo{}, errors.New("down"), "", 0, 0, "", ""}, + {"a failed discovery rejects", nil, gpu.GPUInfo{}, errors.New("no GPUs found"), "", 0, 1, "", + "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found)"}, + {"a GPU model the catalog does not know rejects", nil, + gpu.GPUInfo{Model: "Mystery GPU", GPUsPerNode: 8, NodesWithGPUs: 1}, nil, "", 0, 1, "", + `spec.hardware.gpu.sku: Required value: the cluster's GPU model "Mystery GPU" maps to no AISimulate SKU`}, + {"no GPUs of the SKU reject the missing budget", hardwareOfV1beta2(str("h100_sxm"), nil), + gpu.GPUInfo{System: "h100_sxm"}, nil, "", 0, 1, "h100_sxm", + `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h100_sxm" were found in the cluster`}, + {"a heterogeneous cluster rejects an omitted SKU", nil, + gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, + nil, "", 0, 1, "", + "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100); set spec.hardware.gpu.sku"}, + {"a heterogeneous cluster rejects a budget alone too", hardwareOfV1beta2(nil, num(4)), + gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, + nil, "", 0, 1, "", + "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100)"}, + {"a pinned SKU is not ambiguous in a heterogeneous cluster", hardwareOfV1beta2(str("h200_sxm"), nil), + gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, nil, "", 8, 1, "h200_sxm", ""}, + {"no GPUs at all reject the missing budget", nil, gpu.GPUInfo{System: "h200_sxm"}, nil, "", 0, 1, "", + `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h200_sxm" were found in the cluster`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + probe := &fakeDiscoverer{found: tt.found, err: tt.err} + spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + spec.Hardware = tt.hardware + got, err := DGDRV1beta2Options{Hardware: probe}.resolveHardware(context.Background(), admissionv1.Create, nil, spec) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error = %v, want it to contain %q", err, tt.wantErr) + } + } else if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err == nil && (got.sku != tt.wantSKU || got.budget != tt.wantBudget) { + t.Fatalf("defaults = %+v, want sku %q budget %d", got, tt.wantSKU, tt.wantBudget) + } + if probe.calls != tt.wantCalls { + t.Fatalf("discovery calls = %d, want %d", probe.calls, tt.wantCalls) + } + if tt.wantCalls > 0 && probe.filter != tt.wantFilter { + t.Fatalf("discovery filter = %q, want %q", probe.filter, tt.wantFilter) + } + }) + } +} + +func TestResolveHardwareLeavesAnUnchangedUpdateAlone(t *testing.T) { + probe := &fakeDiscoverer{err: errors.New("down")} + spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + got, err := DGDRV1beta2Options{Hardware: probe}.resolveHardware(context.Background(), admissionv1.Update, spec.DeepCopy(), spec) + if err != nil || got.sku != "" || got.budget != 0 || probe.calls != 0 { + t.Fatalf("got %+v, %v after %d calls; an update that leaves spec.hardware unchanged must not read the cluster", + got, err, probe.calls) + } +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go new file mode 100644 index 000000000000..fe7bfb6c36f9 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -0,0 +1,201 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/validation/field" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + admissionv1 "k8s.io/api/admission/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" +) + +const ( + // defaultMaxHardwareBudget caps a discovered GPU budget. It matches the limit DGDR + // v1beta1 applies to auto-discovered hardware, which keeps a search from using every GPU + // in a large cluster. Set a budget explicitly to exceed it. + defaultMaxHardwareBudget int32 = 32 + + // defaultHardwareDiscoveryTimeout bounds GPU discovery inside the admission call. Keep it + // below the webhook's timeoutSeconds, or the API server gives up first and the request + // fails under the webhook's failurePolicy instead of reaching validation. + defaultHardwareDiscoveryTimeout = 5 * time.Second +) + +// HardwareDiscoverer finds the cluster's GPUs. filterSKU, when set, restricts the result to +// GPUs of that SKU. +type HardwareDiscoverer interface { + Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) +} + +// DGDRV1beta2Options configures the defaults of v1beta2 Requests. +type DGDRV1beta2Options struct { + // AISimulateImage overrides the version-matched Dynamo planner image persisted + // into spec.aisimulate.image when the caller omits it. + AISimulateImage string + // AISimulateVersionOverride is the AISimulate version bundled in AISimulateImage, for an image + // whose Dynamo release the release contract does not map (a nightly or custom planner image). + // Empty maps the image's release. WithV1beta2 completes it, so admission persists the image + // together with its AISimulate version, and only next to an image it supplies itself, never + // next to an image the caller chose. + AISimulateVersionOverride string + // Hardware discovers the cluster's GPUs. Nil disables hardware defaulting, and + // validation then requires the hardware to be spelled out. + Hardware HardwareDiscoverer + // HardwareDiscoveryTimeout defaults to 5 seconds. + HardwareDiscoveryTimeout time.Duration + // MaxHardwareBudget caps a discovered GPU budget; zero means 32. + MaxHardwareBudget int32 +} + +// resolveDefaults reads the environment for the values a v1beta2 spec omits. It does not +// mutate spec. It fails when the typed hardware cannot be completed: the error is the +// admission verdict, so a Request is never stored with hardware that depends on AISimulate's +// implicit defaults or on a cluster that could not be read. +func (o DGDRV1beta2Options) resolveDefaults( + ctx context.Context, + operation admissionv1.Operation, + oldSpec, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, +) (dynamoGraphDeploymentRequestV1beta2DefaultOptions, error) { + hardware, err := o.resolveHardware(ctx, operation, oldSpec, spec) + if err != nil { + return dynamoGraphDeploymentRequestV1beta2DefaultOptions{}, err + } + return dynamoGraphDeploymentRequestV1beta2DefaultOptions{ + hardware: hardware, + aisimulateImage: o.AISimulateImage, + aisimulateVersionOverride: o.AISimulateVersionOverride, + }, nil +} + +// resolveHardware returns the SKU and budget the typed spec omits. It asks the cluster only +// when something is missing, restricted to the typed SKU when there is one. +// +// Discovery runs on CREATE and on an UPDATE that changes spec.hardware. A Request stays +// mutable and every Run snapshots it, so a caller may change or remove typed hardware before +// the next Run. An UPDATE that leaves spec.hardware unchanged, metadata-only ones included, +// never asks the cluster: a Request created while discovery had nothing to offer must not +// pick hardware up once discovery recovers, which would change its spec and generation behind +// the writer's back. oldSpec is the stored spec on UPDATE and nil otherwise. The decision reads +// the typed hardware alone, never spec.aisimulate.parameters. +// +// After a successful return the typed hardware is complete. An omitted SKU becomes the +// cluster's AISimulate SKU, and an omitted budget becomes min(matching GPUs, cap). An explicit +// budget is authoritative: it is never capped, whatever the cluster holds. What the cluster +// cannot supply is rejected on the field that fixes it: a failed discovery, a GPU model the +// AISimulate catalog does not know, and a SKU without any matching GPU. +func (o DGDRV1beta2Options) resolveHardware( + ctx context.Context, + operation admissionv1.Operation, + oldSpec, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, +) (hardwareDefaultsV1beta2, error) { + changed := operation == admissionv1.Update && oldSpec != nil && + !apiequality.Semantic.DeepEqual(oldSpec.Hardware, spec.Hardware) + if o.Hardware == nil || (operation != admissionv1.Create && !changed) { + return hardwareDefaultsV1beta2{}, nil + } + var sku string + var hasBudget bool + if spec.Hardware != nil && spec.Hardware.GPU != nil { + if spec.Hardware.GPU.SKU != nil { + sku = *spec.Hardware.GPU.SKU + } + hasBudget = spec.Hardware.GPU.Budget != nil + } + if sku != "" && hasBudget { + return hardwareDefaultsV1beta2{}, nil + } + + timeout := o.HardwareDiscoveryTimeout + if timeout <= 0 { + timeout = defaultHardwareDiscoveryTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + gpuPath := field.NewPath("spec", "hardware", "gpu") + found, err := o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) + if err != nil { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath, fmt.Sprintf( + "the cluster's GPUs could not be discovered (%v); set spec.hardware.gpu.sku and spec.hardware.gpu.budget", err))) + } + + // The cluster facts are mapped into the v1beta2 API here: the AISimulate catalog name of + // the GPU model (empty when the catalog does not know it) and the total GPU count. + var defaults hardwareDefaultsV1beta2 + if sku == "" { + if found != nil && len(found.OtherModels) > 0 { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("sku"), fmt.Sprintf( + "the cluster has several GPU types (%s); set spec.hardware.gpu.sku", + strings.Join(append([]string{found.Model}, found.OtherModels...), ", ")))) + } + sku = string(gpu.KnownSKU(found)) + if sku == "" { + detail := "the cluster's GPUs map to no AISimulate SKU; set spec.hardware.gpu.sku" + if found != nil && found.Model != "" { + detail = fmt.Sprintf("the cluster's GPU model %q maps to no AISimulate SKU; set spec.hardware.gpu.sku", found.Model) + } + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("sku"), detail)) + } + defaults.sku = sku + } + if !hasBudget { + var gpus int + if found != nil { + // TODO(#17192): Use the discovered aggregate GPU count so mixed node sizes of one SKU are counted correctly. + gpus = found.GPUsPerNode * found.NodesWithGPUs + } + if gpus <= 0 { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("budget"), fmt.Sprintf( + "no GPUs of SKU %q were found in the cluster; set spec.hardware.gpu.budget", sku))) + } + maxBudget := o.MaxHardwareBudget + if maxBudget <= 0 { + maxBudget = defaultMaxHardwareBudget + } + defaults.budget = int32(min(gpus, int(maxBudget))) + } + return defaults, nil +} + +// hardwareRejectionV1beta2 is hardware the cluster cannot complete, with the field that fixes it. +type hardwareRejectionV1beta2 struct{ err *field.Error } + +func (e *hardwareRejectionV1beta2) Error() string { return e.err.Error() } + +func rejectHardwareV1beta2(err *field.Error) error { return &hardwareRejectionV1beta2{err: err} } + +// admissionVerdictV1beta2 turns a rejection into the invalid-object verdict validation uses, so a +// client sees the field and its cause. Any other error is returned as it is. +func admissionVerdictV1beta2(dgdr *nvidiacomv1beta1.DynamoGraphDeploymentRequest, err error) error { + var rejection *hardwareRejectionV1beta2 + if errors.As(err, &rejection) { + return apierrors.NewInvalid(nvidiacomv1beta1.DynamoGraphDeploymentRequestGVK.GroupKind(), dgdr.Name, + field.ErrorList{rejection.err}) + } + return err +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go new file mode 100644 index 000000000000..25b6dfbf90b5 --- /dev/null +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -0,0 +1,312 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package defaulting + +import ( + "context" + "encoding/json" + "testing" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + admissionv1 "k8s.io/api/admission/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +func v1beta2Ptr[T any](v T) *T { return &v } + +type fakeDiscoverer struct { + found gpu.GPUInfo + err error + calls int + filter string + timeout time.Duration +} + +func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.calls++ + f.filter = string(filterSKU) + if deadline, ok := ctx.Deadline(); ok { + f.timeout = time.Until(deadline) + } + found := f.found + return &found, f.err +} + +func TestResolveHardwareV1beta2(t *testing.T) { + spec := func(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + s := modelOnlySpecV1beta2() + if sku != nil || budget != nil { + s.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + } + return s + } + + t.Run("no discoverer", func(t *testing.T) { + if got, err := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || !(got == hardwareDefaultsV1beta2{}) { + t.Fatalf("got %+v", got) + } + }) + t.Run("complete hardware never asks the cluster", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) + if err != nil { + t.Fatal(err) + } + if d.calls != 0 || !(got == hardwareDefaultsV1beta2{}) { + t.Fatalf("calls=%d got=%+v", d.calls, got) + } + }) + t.Run("everything missing", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 16, NodesWithGPUs: 1}} + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) + if err != nil { + t.Fatal(err) + } + if got.sku != "h200_sxm" || got.budget != 16 || d.filter != "" { + t.Fatalf("got %+v, filter %q", got, d.filter) + } + if d.timeout <= 0 || d.timeout > defaultHardwareDiscoveryTimeout { + t.Fatalf("discovery timeout = %v", d.timeout) + } + }) + t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), nil)) + if err != nil { + t.Fatal(err) + } + if d.filter != "b200_sxm" || got.sku != "" || got.budget != 4 { + t.Fatalf("got %+v, filter %q", got, d.filter) + } + }) + t.Run("a pinned budget keeps its value", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 64, NodesWithGPUs: 1}} + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, v1beta2Ptr(int32(3)))) + if err != nil { + t.Fatal(err) + } + if got.sku != "h200_sxm" || got.budget != 0 { + t.Fatalf("got %+v", got) + } + }) + t.Run("the budget is capped", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 1024, NodesWithGPUs: 1}} + if got, err := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || got.budget != 32 { + t.Fatalf("default cap: %+v", got) + } + if got, err := (DGDRV1beta2Options{Hardware: d, MaxHardwareBudget: 8}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || got.budget != 8 { + t.Fatalf("explicit cap: %+v", got) + } + }) + t.Run("the timeout is configurable", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{GPUsPerNode: 1, NodesWithGPUs: 1}} + _, _ = DGDRV1beta2Options{Hardware: d, HardwareDiscoveryTimeout: time.Hour}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) + if d.timeout < time.Minute { + t.Fatalf("timeout = %v", d.timeout) + } + }) +} + +func TestResolveDefaultsV1beta2(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + got, err := DGDRV1beta2Options{Hardware: d, AISimulateImage: "registry.example.com/aisimulate:1.2.0"}. + resolveDefaults(context.Background(), admissionv1.Create, nil, modelOnlySpecV1beta2()) + if err != nil { + t.Fatal(err) + } + if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { + t.Fatalf("got %+v", got) + } +} + +func v1beta2Envelope(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) *nvidiacomv1beta1.DynamoGraphDeploymentRequest { + t.Helper() + raw, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: raw} + return dgdr +} + +func TestDefaultEnvelopeV1beta2(t *testing.T) { + options := DGDRV1beta2Options{ + AISimulateImage: "registry.example.com/aisimulate:1.2.0", + Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}, + } + dgdr := v1beta2Envelope(t, modelOnlySpecV1beta2()) + + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { + t.Fatal(err) + } + var got nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &got); err != nil { + t.Fatal(err) + } + if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 { + t.Fatalf("hardware not defaulted: %+v", got.Hardware) + } + if got.AISimulate == nil || got.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { + t.Fatalf("image not defaulted: %+v", got.AISimulate) + } + if got.Objective == nil || got.Objective.Metric == nil { + t.Fatalf("objective metric not defaulted: %+v", got.Objective) + } + + // A second admission of the defaulted object changes nothing, byte for byte. + first := string(dgdr.Spec.V1Beta2.Raw) + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { + t.Fatal(err) + } + if string(dgdr.Spec.V1Beta2.Raw) != first { + t.Fatalf("not idempotent:\n%s\n%s", first, dgdr.Spec.V1Beta2.Raw) + } +} + +func TestDefaultEnvelopeV1beta2RejectsMalformedSpec(t *testing.T) { + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: []byte(`{"model": 3}`)} + if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err == nil { + t.Fatal("want a decode error") + } +} + +func hardwareSpecV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + spec := modelOnlySpecV1beta2() + if sku != nil || budget != nil { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + } + return spec +} + +func TestHardwareIsNeverDiscoveredOnOtherOperations(t *testing.T) { + for _, op := range []admissionv1.Operation{admissionv1.Delete, admissionv1.Connect, ""} { + t.Run("operation "+string(op), func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + old := hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), op, old, hardwareSpecV1beta2(nil, nil)) + if err != nil { + t.Fatal(err) + } + if got != (hardwareDefaultsV1beta2{}) || d.calls != 0 { + t.Fatalf("hardware resolved on %q: %+v, calls %d", op, got, d.calls) + } + }) + } +} + +func TestResolveHardwareByOperationV1beta2(t *testing.T) { + full := func() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + return hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + } + tests := []struct { + name string + op admissionv1.Operation + old, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + want hardwareDefaultsV1beta2 + wantCalls int + wantFilter string + }{ + {"create discovers everything", admissionv1.Create, nil, hardwareSpecV1beta2(nil, nil), + hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, + {"metadata-only update without hardware does not discover", admissionv1.Update, + hardwareSpecV1beta2(nil, nil), hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, + {"update with unchanged incomplete hardware does not discover", admissionv1.Update, + hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + hardwareDefaultsV1beta2{}, 0, ""}, + {"update without a stored spec does not discover", admissionv1.Update, + nil, hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, + {"removing only the budget discovers only the budget", admissionv1.Update, + full(), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + hardwareDefaultsV1beta2{budget: 8}, 1, "h100_sxm"}, + {"removing only the SKU discovers only the SKU", admissionv1.Update, + full(), hardwareSpecV1beta2(nil, v1beta2Ptr(int32(4))), + hardwareDefaultsV1beta2{sku: "h200_sxm"}, 1, ""}, + {"removing all typed hardware discovers both", admissionv1.Update, + full(), hardwareSpecV1beta2(nil, nil), + hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, + {"changing to complete hardware does not discover", admissionv1.Update, + full(), hardwareSpecV1beta2(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2))), + hardwareDefaultsV1beta2{}, 0, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), tt.op, tt.old, tt.spec) + if err != nil { + t.Fatal(err) + } + if got != tt.want || d.calls != tt.wantCalls || (tt.wantCalls > 0 && d.filter != tt.wantFilter) { + t.Fatalf("got %+v, calls %d, filter %q; want %+v, calls %d, filter %q", + got, d.calls, d.filter, tt.want, tt.wantCalls, tt.wantFilter) + } + }) + } +} + +func TestDecodeOldSpecV1beta2(t *testing.T) { + old, err := json.Marshal(v1beta2Envelope(t, hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil))) + if err != nil { + t.Fatal(err) + } + got, err := decodeOldSpecV1beta2(admissionv1.Update, old) + if err != nil || got == nil || got.Hardware == nil || *got.Hardware.GPU.SKU != "h100_sxm" { + t.Fatalf("got %+v, err %v", got, err) + } + if got, err := decodeOldSpecV1beta2(admissionv1.Create, old); got != nil || err != nil { + t.Fatalf("create has no old spec: %+v, %v", got, err) + } + if got, err := decodeOldSpecV1beta2(admissionv1.Update, []byte(`{}`)); got != nil || err != nil { + t.Fatalf("an old object without the envelope has no old spec: %+v, %v", got, err) + } + if _, err := decodeOldSpecV1beta2(admissionv1.Update, []byte(`{`)); err == nil { + t.Fatal("want a decode error") + } +} + +// modelOnlySpecV1beta2 is a Request as a client sends it: only the model is named. +func modelOnlySpecV1beta2() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} + spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"} + return spec +} + +// An envelope that admission has nothing to add to keeps its exact bytes, even when they are +// not in canonical form. +func TestDefaultEnvelopeV1beta2LeavesUnchangedEnvelopeUntouched(t *testing.T) { + spec := modelOnlySpecV1beta2() + defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) + raw, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + dgdr := v1beta2Envelope(t, spec) + dgdr.Spec.V1Beta2.Raw = append([]byte(" "), raw...) + original := string(dgdr.Spec.V1Beta2.Raw) + if err := defaultDynamoGraphDeploymentRequestV1beta2( + context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err != nil { + t.Fatal(err) + } + if string(dgdr.Spec.V1Beta2.Raw) != original { + t.Fatalf("envelope was rewritten without a change: %q", dgdr.Spec.V1Beta2.Raw) + } +} diff --git a/deploy/operator/internal/webhook/request_version.go b/deploy/operator/internal/webhook/request_version.go new file mode 100644 index 000000000000..982ecc9c3558 --- /dev/null +++ b/deploy/operator/internal/webhook/request_version.go @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package webhook + +import "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + +// VersionV1Beta2 is the API version whose Requests reach the webhooks as a converted v1beta1 +// object carrying spec.v1beta2. +const VersionV1Beta2 = "v1beta2" + +// RequestedVersion is the API version the client used. With matchPolicy: Equivalent the API +// server converts the object to the version the webhook registered for; RequestKind then +// carries the original kind and Kind the converted one. RequestKind is absent when no +// conversion happened. +func RequestedVersion(req admission.Request) string { + if req.RequestKind != nil { + return req.RequestKind.Version + } + return req.Kind.Version +} diff --git a/deploy/operator/internal/webhook/request_version_test.go b/deploy/operator/internal/webhook/request_version_test.go new file mode 100644 index 000000000000..837bf346d0a4 --- /dev/null +++ b/deploy/operator/internal/webhook/request_version_test.go @@ -0,0 +1,57 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package webhook + +import ( + "testing" + + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// requestWithKinds returns a request whose stored kind is v1beta1, the version every webhook +// receives after conversion, and whose original request kind is requestKind (empty for none). +func requestWithKinds(requestKind string) admission.Request { + req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + Kind: metav1.GroupVersionKind{Group: "nvidia.com", Version: "v1beta1", Kind: "DynamoGraphDeploymentRequest"}, + }} + if requestKind != "" { + req.RequestKind = &metav1.GroupVersionKind{Group: "nvidia.com", Version: requestKind, Kind: "DynamoGraphDeploymentRequest"} + } + return req +} + +func TestRequestedVersion(t *testing.T) { + tests := []struct { + name string + req admission.Request + want string + }{ + {"no conversion uses Kind", requestWithKinds(""), "v1beta1"}, + {"RequestKind wins after conversion", requestWithKinds("v1beta2"), "v1beta2"}, + {"RequestKind wins over a matching Kind", requestWithKinds("v1alpha1"), "v1alpha1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RequestedVersion(tt.req); got != tt.want { + t.Fatalf("RequestedVersion() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/deploy/operator/internal/webhook/setup/setup.go b/deploy/operator/internal/webhook/setup/setup.go index 26a48c20e8ad..e25f1f0b5429 100644 --- a/deploy/operator/internal/webhook/setup/setup.go +++ b/deploy/operator/internal/webhook/setup/setup.go @@ -23,10 +23,11 @@ type Options struct { Config *configv1alpha1.OperatorConfiguration RuntimeConfig *commoncontroller.RuntimeConfig OperatorVersion string - // DGDRDefaultImage is the default DGDR profiler image, put into - // DGDR spec.image when unset; empty derives - // dynamo-planner:. - DGDRDefaultImage string + // DGDRV1beta1 configures the defaults of v1beta1 Requests. + DGDRV1beta1 webhookdefaulting.DGDRV1beta1Options + // DGDRV1beta2 configures the defaults of v1beta2 Requests: the AISimulate runner + // image and the cluster hardware discovery. + DGDRV1beta2 webhookdefaulting.DGDRV1beta2Options OperatorPrincipal string Gate features.Gate } @@ -104,7 +105,8 @@ func Setup(mgr ctrl.Manager, opts Options) error { return fmt.Errorf("unable to register DynamoGraphDeployment defaulting webhook: %w", err) } - dgdrDefaulter := webhookdefaulting.NewDGDRDefaulter(opts.OperatorVersion, opts.DGDRDefaultImage) + dgdrDefaulter := webhookdefaulting.NewDGDRDefaulter(opts.OperatorVersion, opts.DGDRV1beta1.DefaultImage). + WithV1beta2(opts.DGDRV1beta2) if err := dgdrDefaulter.RegisterWithManager(mgr); err != nil { return fmt.Errorf("unable to register DynamoGraphDeploymentRequest defaulting webhook: %w", err) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index aaad2a91d819..8835e790abe6 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -27,7 +27,8 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdoverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + internalwebhook "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" k8sruntime "k8s.io/apimachinery/pkg/runtime" @@ -46,41 +47,51 @@ func NewDynamoGraphDeploymentRequestValidator() *DynamoGraphDeploymentRequestVal // dynamoGraphDeploymentRequestValidation carries DGDR-specific request state. // API values, paths, and accumulated errors remain explicit validator arguments. type dynamoGraphDeploymentRequestValidation struct { - ctx context.Context - warnings admission.Warnings + ctx context.Context + // requestVersion is the API version the client used, resolved once from the admission + // request. Every object delivered to the webhook is a converted v1beta1 object; + // spec.v1beta2 says which represented spec it carries. + requestVersion string + warnings admission.Warnings } func (v *dynamoGraphDeploymentRequestValidation) warn(message string) { v.warnings = append(v.warnings, message) } -// Validate performs stateless validation on request. ctx and request must not be nil. +// Validate performs stateless validation on request. ctx must carry the admission request and +// request must not be nil. func (v *DynamoGraphDeploymentRequestValidator) Validate( ctx context.Context, request *nvidiacomv1beta1.DynamoGraphDeploymentRequest, ) (admission.Warnings, error) { - // Legacy v1beta1 rules do not apply to a converted v1beta2 carrier. - if request.Spec.V1Beta2 != nil { - return nil, nil + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("get admission request: %w", err) + } + validation := &dynamoGraphDeploymentRequestValidation{ + ctx: ctx, + requestVersion: internalwebhook.RequestedVersion(req), } - - validation := &dynamoGraphDeploymentRequestValidation{ctx: ctx} allErrs := validation.validateDynamoGraphDeploymentRequest(request) return validation.warnings, invalidDynamoGraphDeploymentRequestError(request, allErrs) } -// ValidateUpdate validates newRequest against oldRequest. ctx, oldRequest, and newRequest must not be nil. +// ValidateUpdate validates newRequest against oldRequest. ctx must carry the admission request; +// oldRequest and newRequest must not be nil. func (v *DynamoGraphDeploymentRequestValidator) ValidateUpdate( ctx context.Context, oldRequest *nvidiacomv1beta1.DynamoGraphDeploymentRequest, newRequest *nvidiacomv1beta1.DynamoGraphDeploymentRequest, ) (admission.Warnings, error) { - // Legacy v1beta1 rules do not apply to a converted v1beta2 carrier. - if newRequest.Spec.V1Beta2 != nil { - return nil, nil + req, err := admission.RequestFromContext(ctx) + if err != nil { + return nil, fmt.Errorf("get admission request: %w", err) + } + validation := &dynamoGraphDeploymentRequestValidation{ + ctx: ctx, + requestVersion: internalwebhook.RequestedVersion(req), } - - validation := &dynamoGraphDeploymentRequestValidation{ctx: ctx} allErrs := validation.validateDynamoGraphDeploymentRequestUpdate(newRequest, oldRequest) validation.warnDeprecatedDGDOverrideTargets(&newRequest.Spec) validation.warnDeprecatedGPUSKU(&newRequest.Spec) @@ -99,6 +110,22 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe spec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, fldPath *field.Path, ) field.ErrorList { + // The requested API version decides what spec.v1beta2 means. A spec that only conversion + // may produce is rejected on CREATE. + if v.requestVersion == internalwebhook.VersionV1Beta2 && spec.V1Beta2 == nil { + // A hub/v1beta1 representation was submitted through v1beta2. + return foreignCreateErrors(fldPath, false) + } + if v.requestVersion != internalwebhook.VersionV1Beta2 && spec.V1Beta2 != nil { + // A v1beta2 carrier was submitted through v1alpha1 or v1beta1. + return foreignCreateErrors(fldPath, true) + } + // A v1beta2 Request reaches this webhook as a v1beta1 object carrying spec.v1beta2; + // the v1beta1 rules below do not apply to it. + if spec.V1Beta2 != nil { + return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(spec.V1Beta2, fldPath.Child("v1beta2")) + } + allErrs := field.ErrorList{} if !features.MustGateFrom(v.ctx).Enabled(features.GPUDiscovery) && !hasManualDGDRHardware(spec.Hardware) { @@ -251,6 +278,21 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe fldPath *field.Path, oldPhase nvidiacomv1beta1.DGDRPhase, ) field.ErrorList { + // A spec viewed through the API version that does not own it is read-only: an unchanged + // spec round-trips, any other write is rejected. + if v.requestVersion == internalwebhook.VersionV1Beta2 && newSpec.V1Beta2 == nil { + return foreignUpdateVerdict(fldPath, newSpec, oldSpec) + } + if v.requestVersion != internalwebhook.VersionV1Beta2 && newSpec.V1Beta2 != nil { + return foreignUpdateVerdict(fldPath, newSpec, oldSpec) + } + // A native v1beta2 spec is unconstrained by phase. Stateless validation of the new typed + // spec runs on every CREATE and UPDATE: an unchanged spec is no reason to skip it. Ratcheting + // belongs to the generated AISimulate schema validation, which will ratchet its own result. + if newSpec.V1Beta2 != nil { + return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(newSpec.V1Beta2, fldPath.Child("v1beta2")) + } + allErrs := field.ErrorList{} gpuDiscoveryEnabled := features.MustGateFrom(v.ctx).Enabled(features.GPUDiscovery) @@ -304,7 +346,7 @@ func dgdrRuntimeVersionOverrideRequired(spec *nvidiacomv1beta1.DynamoGraphDeploy if spec.Image == "" || spec.RuntimeVersionOverride != "" { return false } - _, err := runtimeversion.ParseImageVersion(spec.Image) + _, err := versionoverride.ParseImageVersion(spec.Image) return err != nil } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go new file mode 100644 index 000000000000..e8fdc85c88c2 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_boundary_test.go @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "context" + "testing" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" +) + +// The webhook boundary protects raw conversion carriers, so a missing admission request must +// fail the call instead of counting as a native write. +func TestDGDRValidatorRequiresTheAdmissionRequest(t *testing.T) { + validator := NewDynamoGraphDeploymentRequestValidator() + request := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} + + if _, err := validator.Validate(context.Background(), request); err == nil { + t.Fatal("Validate must fail without an admission request in the context") + } + if _, err := validator.ValidateUpdate(context.Background(), request, request); err == nil { + t.Fatal("ValidateUpdate must fail without an admission request in the context") + } +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go new file mode 100644 index 000000000000..f8f8eac32574 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -0,0 +1,101 @@ +package validation_test + +import ( + "bytes" + "encoding/json" + "sort" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" +) + +// A request created through its owning API version has to stay writable through every other +// served version. A metadata-only patch through the foreign view must not be rejected, and it +// must neither rewrite the spec nor bump the generation. A default that materialises a native +// field next to a conversion carrier breaks this, so it is covered at the API-server boundary. +func TestDynamoGraphDeploymentRequestCarrierObjectsStayWritable(t *testing.T) { + t.Run("native v1beta1 written through v1beta2", func(t *testing.T) { + carrierRoundTrip(t, betaDGDRForAdmission(nil), beta2DGDRForAdmission(nil), "v1beta1") + }) + t.Run("native v1beta2 written through v1beta1", func(t *testing.T) { + carrierRoundTrip(t, beta2DGDRForAdmission(setBeta2AdmissionHardware), betaDGDRForAdmission(nil), "v1beta2") + }) +} + +func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, carrierKey string) { + t.Helper() + env := admissionEnv.ForTest(t) + admissionGate.set(features.Gates{GPUDiscovery: true}) + warnings := &warningRecorder{} + ownerClient := newAdmissionResourceClient(t, env, ownerFixture, "", warnings) + otherClient := newAdmissionResourceClient(t, env, otherFixture, "", warnings) + + t.Log("Create the request through the version that owns its representation") + object, _ := admissionObject(t, ownerFixture, env.Namespace(), nil) + created, err := ownerClient.Create(t.Context(), object, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create through the owning version: %v", err) + } + name := created.GetName() + + t.Log("Read it through the other version and expect the conversion carrier") + foreign, err := otherClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read through the other version: %v", err) + } + spec, _ := foreign.Object["spec"].(map[string]any) + if _, ok := spec[carrierKey]; !ok { + keys := make([]string, 0, len(spec)) + for key := range spec { + keys = append(keys, key) + } + sort.Strings(keys) + t.Fatalf("foreign view has spec keys %v, want the %s carrier", keys, carrierKey) + } + + t.Log("Patch metadata through each version") + for label, client := range map[string]dynamic.ResourceInterface{"foreign": otherClient, "owner": ownerClient} { + patch := []byte(`{"metadata":{"labels":{"carrier-probe-` + label + `":"1"}}}`) + if _, err := client.Patch(t.Context(), name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatalf("metadata-only patch through the %s version: %v", label, err) + } + } + + t.Log("The spec and generation must be untouched") + after, err := ownerClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read back through the owning version: %v", err) + } + if after.GetGeneration() != created.GetGeneration() { + t.Fatalf("generation = %d, want %d: a metadata-only patch rewrote the spec", after.GetGeneration(), created.GetGeneration()) + } + + t.Log("Write the unchanged foreign view back with a full update; the spec must be byte-stable") + beforeSpec, err := json.Marshal(foreign.Object["spec"]) + if err != nil { + t.Fatalf("encode the foreign spec: %v", err) + } + current, err := otherClient.Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("read the foreign view before the update: %v", err) + } + updated, err := otherClient.Update(t.Context(), current, metav1.UpdateOptions{}) + if err != nil { + t.Fatalf("full update of the unchanged foreign view: %v", err) + } + afterSpec, err := json.Marshal(updated.Object["spec"]) + if err != nil { + t.Fatalf("encode the updated spec: %v", err) + } + if !bytes.Equal(beforeSpec, afterSpec) { + t.Fatalf("the foreign spec changed across an unchanged update:\nbefore: %s\nafter: %s", beforeSpec, afterSpec) + } + if updated.GetGeneration() != created.GetGeneration() { + t.Fatalf("generation = %d, want %d: an unchanged update rewrote the spec", updated.GetGeneration(), created.GetGeneration()) + } +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go new file mode 100644 index 000000000000..a0c10bc6fc31 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_routing.go @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// foreignCreateErrors rejects a spec that conversion alone may produce. The carrier is raw +// JSON that bypasses structural pruning; it is safe only when conversion built it from a +// native request that was validated against its own versioned schema. +func foreignCreateErrors(fldPath *field.Path, hasV1Beta2Carrier bool) field.ErrorList { + if hasV1Beta2Carrier { + return field.ErrorList{field.Forbidden( + fldPath.Child("v1beta2"), + "spec.v1beta2 is a conversion carrier that only conversion may write; create the Request through the v1beta2 API", + )} + } + return field.ErrorList{field.Forbidden( + fldPath, + "a v1beta1 representation cannot be written through the v1beta2 API; spec.v1beta1 is a conversion carrier that only conversion may write", + )} +} + +// foreignUpdateVerdict decides an UPDATE whose spec is viewed through the API version that +// does not own it: an unchanged spec round-trips, any other write is rejected. +func foreignUpdateVerdict( + fldPath *field.Path, + newSpec, oldSpec *nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec, +) field.ErrorList { + if apiequality.Semantic.DeepEqual(newSpec, oldSpec) { + return nil + } + return foreignUpdateErrors(fldPath, newSpec.V1Beta2 != nil) +} + +// foreignUpdateErrors rejects a change to a spec viewed through the API version that does +// not own it. An unchanged spec never reaches this function. +func foreignUpdateErrors(fldPath *field.Path, hasV1Beta2Carrier bool) field.ErrorList { + if hasV1Beta2Carrier { + return field.ErrorList{field.Forbidden( + fldPath.Child("v1beta2"), + "spec.v1beta2 is a read-only view through this API version; update the Request through the v1beta2 API", + )} + } + return field.ErrorList{field.Forbidden( + fldPath, + "this Request is owned by v1beta1; its spec is read-only through the v1beta2 API, update it through v1beta1", + )} +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go new file mode 100644 index 000000000000..557129b380e4 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "encoding/json" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// validateDynamoGraphDeploymentRequestEnvelopeV1beta2 validates the v1beta2 spec carried in +// spec.v1beta2 of a v1beta1 object. raw is that field and fldPath is its path; a spec that +// does not decode is reported there. Errors about the decoded spec use the v1beta2 spec's own +// field paths (spec.model, spec.aisimulate, ...), which are what the user wrote, so they are +// rooted at spec rather than at fldPath. +func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( + raw *apiextensionsv1.JSON, + fldPath *field.Path, +) field.ErrorList { + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if err := json.Unmarshal(raw.Raw, &spec); err != nil { + return field.ErrorList{field.Invalid(fldPath, "", "must be a valid v1beta2 spec: "+err.Error())} + } + return validateDynamoGraphDeploymentRequestV1beta2(&spec) +} + +// validateDynamoGraphDeploymentRequestV1beta2 validates a defaulted v1beta2 Request spec: the +// typed fields and the image and version selection. spec.aisimulate.parameters is opaque here: the +// CRD types it as an object, and the validation of its content belongs to the schema of the +// AISimulate release the Request selects. +func validateDynamoGraphDeploymentRequestV1beta2(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) field.ErrorList { + return validateDynamoGraphDeploymentRequestSpecV1beta2(spec, field.NewPath("spec")) +} + +// validateDynamoGraphDeploymentRequestSpecV1beta2 validates the typed rules that the source +// schema and CEL cannot express. The generated OpenAPI schema and CEL of the v1beta2 API have +// already run before conversion reaches this webhook, so they are not repeated here: native +// model presence, the model name length, backend uniqueness and the kvLoadRatio range order. +// spec and fldPath must not be nil. +func validateDynamoGraphDeploymentRequestSpecV1beta2( + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + fldPath *field.Path, +) field.ErrorList { + return validateAISimulateSpecV1beta2(spec.AISimulate, fldPath.Child("aisimulate")) +} + +// validateAISimulateSpecV1beta2 validates the execution artifact. Admission persists the +// operator's default image, so a Request that still has none was created while the operator +// had no default image configured. spec may be nil. +func validateAISimulateSpecV1beta2(spec *nvidiacomv1beta2.AISimulateSpec, fldPath *field.Path) field.ErrorList { + if spec == nil || spec.Image == "" { + return field.ErrorList{field.Required(fldPath.Child("image"), + "an AISimulate image is required; set it, or configure the operator's default AISimulate image")} + } + if _, err := aisimulate.ResolveVersion(spec.Image, spec.VersionOverride); err != nil { + if spec.VersionOverride != "" { + return field.ErrorList{field.Invalid(fldPath.Child("versionOverride"), spec.VersionOverride, + "must be a semantic version such as \"0.13.0\"")} + } + return field.ErrorList{field.Required(fldPath.Child("versionOverride"), + "is required when the tag of spec.aisimulate.image is not a Dynamo release with a bundled AISimulate version")} + } + return nil +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go new file mode 100644 index 000000000000..9ad45a24a0b9 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go @@ -0,0 +1,162 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "encoding/json" + "strings" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +func v1beta2Ptr[T any](v T) *T { return &v } + +// v1beta2DefaultedSpec is a Request as it looks once the schema and admission have defaulted it. +func v1beta2DefaultedSpec() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + metric := nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") + return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, + Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendSGLang, nvidiacomv1beta2.BackendTRTLLM}, + Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: v1beta2Ptr("h200_sxm"), Budget: v1beta2Ptr(int32(8))}}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ + ISL: v1beta2Ptr(int32(4000)), OSL: v1beta2Ptr(int32(1000)), KVLoadRatio: nvidiacomv1beta2.KVLoadRatio{0, 1}, + }}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, + Metric: &metric, + SLA: &nvidiacomv1beta2.ObjectiveSLASpec{TTFTMs: v1beta2Ptr(2000.0), ITLMs: v1beta2Ptr(30.0)}, + }, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:1.6.0"}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +func v1beta2WithParameters(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, parameters string) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + if spec.AISimulate == nil { + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:1.6.0"} + } + spec.AISimulate.Parameters = &runtime.RawExtension{Raw: []byte(parameters)} + return spec +} + +func v1beta2Details(errs field.ErrorList) string { + var out []string + for _, err := range errs { + out = append(out, err.Field+": "+err.Detail) + } + return strings.Join(out, "\n") +} + +func v1beta2RequireError(t *testing.T, errs field.ErrorList, substr string) { + t.Helper() + if !strings.Contains(v1beta2Details(errs), substr) { + t.Fatalf("want an error containing %q, got:\n%s", substr, v1beta2Details(errs)) + } +} + +func v1beta2RequireNoError(t *testing.T, errs field.ErrorList) { + t.Helper() + if len(errs) > 0 { + t.Fatalf("unexpected errors:\n%s", v1beta2Details(errs)) + } +} + +func TestValidateV1beta2AcceptsADefaultedRequest(t *testing.T) { + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(v1beta2DefaultedSpec())) +} + +// The webhook does not repeat what the generated schema and CEL enforce before conversion: +// native model presence, the model name length, backend uniqueness and the kvLoadRatio range +// order. The operatorenv table proves those errors end to end. +func TestValidateV1beta2LeavesSourceSchemaRulesToTheSchema(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.Model = nil + spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendVLLM} + spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) +} + +func TestValidateV1beta2ParametersAreOpaque(t *testing.T) { + for _, parameters := range []string{ + `{"workload":{"isl":0},"goal":{"target":"nope"}}`, + `[1]`, // the CRD types parameters as an object; admission does not interpret it + } { + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2( + v1beta2WithParameters(v1beta2DefaultedSpec(), parameters))) + } +} + +func TestValidateV1beta2AISimulateImage(t *testing.T) { + t.Run("no AISimulate section", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate = nil + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.image: an AISimulate image is required") + }) + t.Run("parameters without an image", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.image") + }) + t.Run("custom tag needs a version", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate.Image = "registry.example.com/aisimulate:custom" + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.versionOverride: is required when the tag of spec.aisimulate.image is not a Dynamo release with a bundled AISimulate version") + }) + t.Run("custom tag with a version", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate.Image = "registry.example.com/aisimulate:custom" + spec.AISimulate.VersionOverride = "0.13.1" + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) + }) + t.Run("a Dynamo release without a bundled version needs one", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate.Image = "registry.example.com/aisimulate:1.2.3" + spec.AISimulate.VersionOverride = "" + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.versionOverride") + }) + t.Run("an override that is not a version", func(t *testing.T) { + spec := v1beta2DefaultedSpec() + spec.AISimulate.VersionOverride = "bad" + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.versionOverride: must be a semantic version") + }) +} + +func v1beta2Envelope(raw string) *apiextensionsv1.JSON { + return &apiextensionsv1.JSON{Raw: []byte(raw)} +} + +func TestValidateEnvelopeV1beta2(t *testing.T) { + path := field.NewPath("spec", "v1beta2") + + good, err := json.Marshal(v1beta2DefaultedSpec()) + if err != nil { + t.Fatal(err) + } + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(good)), path)) + + bad := v1beta2DefaultedSpec() + bad.AISimulate = nil + raw, _ := json.Marshal(bad) + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(raw)), path), "aisimulate") + + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index d783bdde1cb3..506958890cd0 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -18,6 +18,9 @@ package validation_test import ( + "encoding/json" + "errors" + "reflect" "strings" "testing" @@ -26,6 +29,7 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -55,18 +59,27 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec":{"components":[{"name":"VllmPrefillWorker"}]} }` + discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} + tests := []struct { name string request runtime.Object oldRequest runtime.Object gpuDiscovery bool seedWithoutWebhook bool - wantImage string - wantSchemaErr string - wantCELErr string - wantStatusErr string - wantWebhook []string - wantWarnings []string + // discovered and discoveryErr are what cluster GPU discovery reports while the case + // runs; beforeUpdate may change them between the seed and the update. + discovered gpu.GPUInfo + discoveryErr error + beforeUpdate func(*testing.T) + // check inspects the object the API server returned. + check func(*testing.T, *unstructured.Unstructured) + wantImage string + wantSchemaErr string + wantCELErr string + wantStatusErr string + wantWebhook []string + wantWarnings []string }{ // Source-version schema, CEL, and conversion boundaries. { @@ -103,12 +116,13 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, { name: "v1beta2 create bypasses legacy v1beta1 validation", - request: beta2DGDRForAdmission(nil), + request: beta2DGDRForAdmission(setBeta2AdmissionHardware), }, { name: "v1beta2 update bypasses legacy v1beta1 validation", - oldRequest: beta2DGDRForAdmission(nil), + oldRequest: beta2DGDRForAdmission(setBeta2AdmissionHardware), request: beta2DGDRForAdmission(func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + setBeta2AdmissionHardware(request) request.Labels = map[string]string{"updated": "true"} }), }, @@ -531,10 +545,546 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { `spec.searchStrategy: Invalid value: "thorough": is incompatible with spec.backend "auto"; set spec.backend to a specific backend (sglang, trtllm, or vllm)`, }, }, + + // v1beta2: the CRD owns the unconditional defaults, admission the environment ones. + { + name: "v1beta2 model-only request gets schema and admission defaults", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2StructuralDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 explicit hardware is kept", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != "h100_sxm" { + t.Fatalf("hardware.gpu.sku = %q, want h100_sxm", got) + } + if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != 4 { + t.Fatalf("hardware.gpu.budget = %v, want 4", got) + } + }, + }, + { + name: "v1beta2 parameters are stored untouched and never decide hardware defaulting", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(4)}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 8) + if got := v1beta2String(t, result, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "h100_sxm" { + t.Fatalf("parameters.search_space.hardware_sku = %q, want h100_sxm", got) + } + }, + }, + { + name: "v1beta2 parameters null does not suppress hardware discovery", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": nil}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got == "" { + t.Fatal("hardware.gpu.sku is empty: discovery must read the typed spec only") + } + }, + }, + { + name: "v1beta2 metadata-only update does not read the cluster again", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}) + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}) + }, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discovered: discoveredH200, + beforeUpdate: func(*testing.T) { + admissionHardware.set(gpu.GPUInfo{}, errors.New("the cluster is unreadable")) + }, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 8) + if got := result.GetGeneration(); got != 1 { + t.Fatalf("generation = %d, want 1: the spec must not change", got) + } + }, + }, + { + name: "v1beta2 removing only the budget rediscovers only the budget", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h100_sxm", 8) + }, + }, + { + name: "v1beta2 removing only the SKU rediscovers only the SKU", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h200_sxm", 4) + }, + }, + { + name: "v1beta2 removing all typed hardware rediscovers the SKU and the budget", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPU(t, result, "h200_sxm", 8) + }, + }, + { + name: "v1beta2 empty workload receives the static model-only default", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { + if got := v1beta2Number(t, result, "spec", "workload", "static", key); got != want { + t.Fatalf("workload.static.%s = %v, want %v", key, got, want) + } + } + v1beta2Absent(t, result, "spec", "workload", "static", "kvLoadRatio") + }, + }, + { + name: "v1beta2 empty workload static is an opt-out and stays empty", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "workload", "static") + }, + }, + { + name: "v1beta2 empty workload static stays empty beside native parameters", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}} + spec["aisimulate"] = map[string]any{"parameters": mustJSONObject(t, `{"workload":{"isl":512,"osl":128,"concurrency":8,"request_count":100}}`)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "workload", "static") + v1beta2Absent(t, result, "spec", "objective", "sla") + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 empty sla is an opt-out and is not refilled", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{"sla": map[string]any{}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Empty(t, result, "spec", "objective", "sla") + if got := v1beta2Number(t, result, "spec", "workload", "static", "isl"); got != 1024 { + t.Fatalf("workload.static.isl = %v, want the schema default 1024", got) + } + }, + }, + { + name: "v1beta2 native parameters win without rewriting the typed spec", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"parameters": mustJSONObject(t, `{"workload":{"isl":256}}`)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2Number(t, result, "spec", "workload", "static", "isl"); got != 1024 { + t.Fatalf("typed workload.static.isl = %v, want 1024", got) + } + if got := v1beta2Number(t, result, "spec", "aisimulate", "parameters", "workload", "isl"); got != 256 { + t.Fatalf("parameters.workload.isl = %v, want 256", got) + } + }, + }, + { + name: "v1beta2 stored invalid spec is rejected again on a metadata-only update", + oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:custom"} + }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:custom"} + }, map[string]any{"updated": "true"}), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.versionOverride: Required value: is required when the tag of spec.aisimulate.image is not a Dynamo release with a bundled AISimulate version", + }, + }, + { + name: "v1beta2 model-only request stores the configured AISimulate image and its override", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 caller image does not receive the operator's version override", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:1.6.0"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "aisimulate", "image"); got != "registry.example.com/aisimulate:1.6.0" { + t.Fatalf("aisimulate.image = %q, want the caller's image", got) + } + if _, found, _ := unstructured.NestedString(result.Object, "spec", "aisimulate", "versionOverride"); found { + t.Fatalf("aisimulate.versionOverride was set next to a caller's image") + } + }, + }, + { + name: "v1beta2 caller override wins next to the configured image", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "2.0.0"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "aisimulate", "image"); got != admissionAISimulateImage { + t.Fatalf("aisimulate.image = %q, want the configured image", got) + } + if got := v1beta2String(t, result, "spec", "aisimulate", "versionOverride"); got != "2.0.0" { + t.Fatalf("aisimulate.versionOverride = %q, want the caller's 2.0.0", got) + } + }, + }, + + { + name: "v1beta2 hardware omitted is discovered and the budget capped", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8}, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 32) + }, + }, + { + name: "v1beta2 an empty hardware object is automatic discovery", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{} + }, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 8) + }, + }, + { + name: "v1beta2 a SKU alone keeps the SKU and persists the capped budget", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} + }, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 8}, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h100_sxm", 32) + }, + }, + { + name: "v1beta2 a budget alone keeps the budget and discovers the SKU", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 4) + }, + }, + { + name: "v1beta2 a SKU and a budget are kept without reading the cluster", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discoveryErr: errors.New("the cluster must not be read"), + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h100_sxm", 4) + }, + }, + { + name: "v1beta2 an explicit budget above the cap and the cluster size is kept", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(64)}} + }, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 64) + }, + }, + { + name: "v1beta2 a failed discovery is rejected", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discoveryErr: errors.New("no GPUs found"), + wantWebhook: []string{ + "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found); " + + "set spec.hardware.gpu.sku and spec.hardware.gpu.budget", + }, + }, + { + name: "v1beta2 a GPU model the catalog does not know is rejected", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{Model: "Mystery GPU", GPUsPerNode: 8, NodesWithGPUs: 1}, + wantWebhook: []string{ + `spec.hardware.gpu.sku: Required value: the cluster's GPU model "Mystery GPU" maps to no AISimulate SKU; ` + + "set spec.hardware.gpu.sku", + }, + }, + { + name: "v1beta2 a heterogeneous cluster does not pick a SKU", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{ + System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}, + }, + wantWebhook: []string{ + "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100); " + + "set spec.hardware.gpu.sku", + }, + }, + { + name: "v1beta2 a pinned SKU is accepted in a heterogeneous cluster", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h200_sxm"}} + }, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 8) + }, + }, + + { + name: "v1beta2 a SKU without any matching GPU is rejected", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} + }, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{System: "h100_sxm"}, + wantWebhook: []string{ + `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h100_sxm" were found in the cluster; ` + + "set spec.hardware.gpu.budget", + }, + }, + + { + name: "v1beta2 empty objective and recommendation receive their defaults", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{} + spec["recommendation"] = map[string]any{} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2StructuralDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 pareto objective receives no scalar metric", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"TTFT", "Throughput"}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + v1beta2Absent(t, result, "spec", "objective", "metric") + }, + }, + { + name: "v1beta2 explicit values are all kept", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"vllm"} + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + spec["workload"] = map[string]any{"static": map[string]any{"isl": int64(10)}} + spec["objective"] = map[string]any{"mode": "Optimize", "metric": "TTFT"} + spec["aisimulate"] = map[string]any{ + "image": "registry.example.com/aisimulate:custom", + "versionOverride": "1.4.0", + } + spec["recommendation"] = map[string]any{"maxCandidates": int64(3)} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: expectV1beta2ExplicitValues, + }, + { + name: "v1beta2 duplicate backends are rejected on create", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"vllm", "vllm"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.backends[1]: Duplicate value: "vllm"`, + }, + { + name: "v1beta2 missing model is rejected by the representation CEL", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { delete(spec, "model") }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec: Invalid value: exactly one of native v1beta2 fields or v1beta1 must be specified`, + }, + { + name: "v1beta2 empty model name is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["model"] = map[string]any{"name": ""} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.model.name: Invalid value: "": spec.model.name in body should be at least 1 chars long`, + }, + { + name: "v1beta2 kvLoadRatio range needs min below max", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{"kvLoadRatio": []any{0.9, 0.1}}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload.static.kvLoadRatio: Invalid value: a range needs min < max`, + }, + { + name: "v1beta2 versionOverride latest is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "latest"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.aisimulate.versionOverride: Invalid value: "latest": spec.aisimulate.versionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, + { + name: "v1beta2 versionOverride with a v prefix is rejected by the schema", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "v0.13.0"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.aisimulate.versionOverride: Invalid value: "v0.13.0": spec.aisimulate.versionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + }, + { + name: "v1beta2 versionOverride 0.13.0 is accepted", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{"versionOverride": "0.13.0"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "aisimulate", "versionOverride"); got != "0.13.0" { + t.Fatalf("versionOverride = %q, want 0.13.0", got) + } + }, + }, + + // v1beta2 UPDATE: defaults stay idempotent, hardware is never rediscovered, and any + // spec change is fully validated. + { + name: "v1beta2 update without a spec change keeps the spec and generation", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(nil, map[string]any{"updated": "true"}), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1: defaulting must not change an unchanged spec", result.GetGeneration()) + } + expectV1beta2StructuralDefaults(t, result) + expectV1beta2AdmissionDefaults(t, result) + }, + }, + { + name: "v1beta2 native hardware survives a metadata-only update when the cluster is unreadable", + oldRequest: v1beta2NativeHardwareDGDR(nil), + request: v1beta2NativeHardwareDGDR(map[string]any{"updated": "true"}), + gpuDiscovery: true, + discovered: discoveredH200, + beforeUpdate: func(*testing.T) { + admissionHardware.set(gpu.GPUInfo{}, errors.New("the cluster is unreadable")) + }, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1", result.GetGeneration()) + } + expectV1beta2GPUValues(t, result, "h200_sxm", 8) + if got := v1beta2String(t, result, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "h100_sxm" { + t.Fatalf("parameters.search_space.hardware_sku = %q, want h100_sxm", got) + } + }, + }, + { + name: "v1beta2 spec change is accepted and bumps the generation", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["model"] = map[string]any{"name": alternateAdmissionModel} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if result.GetGeneration() != 2 { + t.Fatalf("generation = %d, want 2", result.GetGeneration()) + } + if got := v1beta2String(t, result, "spec", "model", "name"); got != alternateAdmissionModel { + t.Fatalf("model.name = %q, want %q", got, alternateAdmissionModel) + } + }, + }, + { + name: "v1beta2 update to duplicate backends is rejected", + oldRequest: v1beta2DGDRForAdmission(nil, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["backends"] = []any{"sglang", "sglang"} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.backends[1]: Duplicate value: "sglang"`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + admissionHardware.set(tt.discovered, tt.discoveryErr) + t.Cleanup(func() { admissionHardware.set(gpu.GPUInfo{}, nil) }) + gates := features.Gates{GPUDiscovery: tt.gpuDiscovery} test := admissionTestCase{ object: tt.request, @@ -546,6 +1096,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { wantStatusError: tt.wantStatusErr, wantWebhookErrors: tt.wantWebhook, wantWarnings: tt.wantWarnings, + beforeUpdate: tt.beforeUpdate, } if tt.oldRequest != nil && !tt.gpuDiscovery { @@ -554,6 +1105,9 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { test.seedGates = &seedGates } actual := runAdmissionTest(t, test) + if tt.check != nil { + tt.check(t, actual) + } if tt.wantImage != "" { image, found, err := unstructured.NestedString(actual.Object, "spec", "image") if err != nil || !found { @@ -582,7 +1136,7 @@ func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing. }, { name: "v1alpha1 v1beta2 spec rejects native status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: alphaDGDRForAdmission(nil), status: map[string]any{"state": string(nvidiacomv1alpha1.DGDRStatePending)}, }, @@ -594,13 +1148,13 @@ func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing. }, { name: "v1beta1 v1beta2 spec rejects native status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: betaDGDRForAdmission(nil), status: map[string]any{"phase": string(nvidiacomv1beta1.DGDRPhasePending)}, }, { name: "v1beta2 native spec rejects v1beta1 status", - createObject: beta2DGDRForAdmission(nil), + createObject: beta2DGDRForAdmission(setBeta2AdmissionHardware), statusObject: beta2DGDRForAdmission(nil), status: map[string]any{"v1beta1": map[string]any{"phase": string(nvidiacomv1beta1.DGDRPhasePending)}}, }, @@ -651,6 +1205,7 @@ func TestDynamoGraphDeploymentRequestV1Beta2EmptyStatusSurvivesV1Alpha1(t *testi warnings := &warningRecorder{} beta2Fixture := beta2DGDRForAdmission(func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { request.Name = "empty-status-alpha-bridge" + setBeta2AdmissionHardware(request) }) beta2Client := newAdmissionResourceClient(t, env, beta2Fixture, "", warnings) object, _ := admissionObject(t, beta2Fixture, env.Namespace(), nil) @@ -757,3 +1312,222 @@ func beta2DGDRForAdmission( } return request } + +const v1beta2AdmissionModel = "Qwen/Qwen3-0.6B" + +// v1beta2NativeHardwareDGDR supplies hardware only through native AISimulate parameters. +func v1beta2NativeHardwareDGDR(labels map[string]any) *unstructured.Unstructured { + return v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["aisimulate"] = map[string]any{ + "image": admissionAISimulateImage, + "versionOverride": admissionAISimulateVersionOverride, + "parameters": map[string]any{ + "search_space": map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}, + }, + } + }, labels) +} + +// v1beta2DGDRForAdmission builds a v1beta2 Request as a client sends it, before the CRD +// defaults. It is unstructured on purpose: an omitted field, +// an explicit empty object and a null are different requests, and typed Go cannot tell them +// apart. +func v1beta2DGDRForAdmission(mutate func(spec map[string]any), labels map[string]any) *unstructured.Unstructured { + spec := map[string]any{"model": map[string]any{"name": v1beta2AdmissionModel}} + if mutate != nil { + mutate(spec) + } + metadata := map[string]any{"name": "test-v1beta2-dgdr", "namespace": "default"} + if labels != nil { + metadata["labels"] = labels + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRequest", + "metadata": metadata, + "spec": spec, + }} +} + +// mustJSONObject decodes a native AISimulate parameters document. +func mustJSONObject(t *testing.T, document string) map[string]any { + t.Helper() + var value map[string]any + if err := json.Unmarshal([]byte(document), &value); err != nil { + t.Fatalf("decode %q: %v", document, err) + } + return value +} + +func v1beta2Field(t *testing.T, obj *unstructured.Unstructured, path ...string) (any, bool) { + t.Helper() + value, found, err := unstructured.NestedFieldNoCopy(obj.Object, path...) + if err != nil { + t.Fatalf("read %v: %v", path, err) + } + return value, found +} + +func v1beta2Number(t *testing.T, obj *unstructured.Unstructured, path ...string) float64 { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found { + t.Fatalf("%v is not set", path) + } + switch n := value.(type) { + case int64: + return float64(n) + case float64: + return n + default: + t.Fatalf("%v = %#v, want a number", path, value) + return 0 + } +} + +func v1beta2String(t *testing.T, obj *unstructured.Unstructured, path ...string) string { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found { + t.Fatalf("%v is not set", path) + } + text, ok := value.(string) + if !ok { + t.Fatalf("%v = %#v, want a string", path, value) + } + return text +} + +func v1beta2Absent(t *testing.T, obj *unstructured.Unstructured, path ...string) { + t.Helper() + if value, found := v1beta2Field(t, obj, path...); found { + t.Fatalf("%v = %#v, want it absent", path, value) + } +} + +func v1beta2Empty(t *testing.T, obj *unstructured.Unstructured, path ...string) { + t.Helper() + value, found := v1beta2Field(t, obj, path...) + if !found || !reflect.DeepEqual(value, map[string]any{}) { + t.Fatalf("%v = %#v (found=%v), want an empty object", path, value, found) + } +} + +// expectV1beta2StructuralDefaults checks the defaults admission supplies to a model-only Request. +func expectV1beta2StructuralDefaults(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { + if got := v1beta2Number(t, obj, "spec", "workload", "static", key); got != want { + t.Fatalf("workload.static.%s = %v, want %v", key, got, want) + } + } + v1beta2Absent(t, obj, "spec", "workload", "static", "kvLoadRatio") + v1beta2Absent(t, obj, "spec", "objective", "sla") + if got := v1beta2String(t, obj, "spec", "objective", "mode"); got != "Optimize" { + t.Fatalf("objective.mode = %q, want Optimize", got) + } + if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm", "sglang", "trtllm"}) { + t.Fatalf("backends = %#v, want vllm, sglang and trtllm", got) + } + if got := v1beta2Number(t, obj, "spec", "recommendation", "maxCandidates"); got != 1 { + t.Fatalf("recommendation.maxCandidates = %v, want 1", got) + } +} + +// expectV1beta2ExplicitValues checks that every value the explicit-values row sent survived. +func expectV1beta2ExplicitValues(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm"}) { + t.Fatalf("backends = %#v, want only vllm", got) + } + for _, want := range []struct { + path []string + value float64 + }{ + {[]string{"spec", "hardware", "gpu", "budget"}, 4}, + {[]string{"spec", "workload", "static", "isl"}, 10}, + {[]string{"spec", "recommendation", "maxCandidates"}, 3}, + } { + if got := v1beta2Number(t, obj, want.path...); got != want.value { + t.Fatalf("%v = %v, want %v", want.path, got, want.value) + } + } + for _, want := range []struct { + path []string + value string + }{ + {[]string{"spec", "hardware", "gpu", "sku"}, "h100_sxm"}, + {[]string{"spec", "objective", "mode"}, "Optimize"}, + {[]string{"spec", "objective", "metric"}, "TTFT"}, + {[]string{"spec", "aisimulate", "image"}, "registry.example.com/aisimulate:custom"}, + {[]string{"spec", "aisimulate", "versionOverride"}, "1.4.0"}, + } { + if got := v1beta2String(t, obj, want.path...); got != want.value { + t.Fatalf("%v = %q, want %q", want.path, got, want.value) + } + } + // A static workload that names only isl is not filled in. + v1beta2Absent(t, obj, "spec", "workload", "static", "osl") + v1beta2Absent(t, obj, "spec", "workload", "static", "concurrency") +} + +// expectV1beta2AdmissionDefaults checks what admission supplies from the environment. +func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured) { + t.Helper() + if got := v1beta2String(t, obj, "spec", "objective", "metric"); got != "Throughput" { + t.Fatalf("objective.metric = %q, want Throughput", got) + } + if got := v1beta2String(t, obj, "spec", "aisimulate", "image"); got != admissionAISimulateImage { + t.Fatalf("aisimulate.image = %q, want %q", got, admissionAISimulateImage) + } + if got := v1beta2String(t, obj, "spec", "aisimulate", "versionOverride"); got != admissionAISimulateVersionOverride { + t.Fatalf("aisimulate.versionOverride = %q, want %q", got, admissionAISimulateVersionOverride) + } + if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != "h200_sxm" { + t.Fatalf("hardware.gpu.sku = %q, want h200_sxm", got) + } + if got := v1beta2Number(t, obj, "spec", "hardware", "gpu", "budget"); got != 8 { + t.Fatalf("hardware.gpu.budget = %v, want 8", got) + } +} + +// setV1beta2SearchSpace sets spec.aisimulate.parameters.search_space to searchSpace, replacing +// any parameters the fixture carried. +func setV1beta2SearchSpace(spec map[string]any, searchSpace map[string]any) { + aisimulate, _ := spec["aisimulate"].(map[string]any) + if aisimulate == nil { + aisimulate = map[string]any{} + spec["aisimulate"] = aisimulate + } + aisimulate["parameters"] = map[string]any{"search_space": searchSpace} +} + +// setBeta2AdmissionHardware names the hardware, so fixtures do not depend on GPU discovery. +func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + sku, budget := "h100_sxm", int32(4) + request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + } +} + +// expectV1beta2GPU asserts the typed GPU hardware the API server stored. +func expectV1beta2GPU(t *testing.T, result *unstructured.Unstructured, sku string, budget float64) { + t.Helper() + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != sku { + t.Fatalf("hardware.gpu.sku = %q, want %q", got, sku) + } + if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != budget { + t.Fatalf("hardware.gpu.budget = %v, want %v", got, budget) + } +} + +// expectV1beta2GPUValues checks the typed hardware a Request was stored with. +func expectV1beta2GPUValues(t *testing.T, obj *unstructured.Unstructured, sku string, budget float64) { + t.Helper() + if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != sku { + t.Fatalf("hardware.gpu.sku = %q, want %q", got, sku) + } + if got := v1beta2Number(t, obj, "spec", "hardware", "gpu", "budget"); got != budget { + t.Fatalf("hardware.gpu.budget = %v, want %v", got, budget) + } +} diff --git a/deploy/operator/internal/webhook/validation/planner_runtime_features.go b/deploy/operator/internal/webhook/validation/planner_runtime_features.go index 1e772c501f72..c61ff68ec318 100644 --- a/deploy/operator/internal/webhook/validation/planner_runtime_features.go +++ b/deploy/operator/internal/webhook/validation/planner_runtime_features.go @@ -13,7 +13,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -117,7 +117,7 @@ func rolePodTemplatesPlannerRuntimeErrorDetail(planners []plannerRuntimeContract minimum := runtimefeatures.PlannerDGDComponentStatus.MinRuntimeVersion.String() for _, planner := range planners { - version, err := runtimeversion.Resolve(planner.image, planner.runtimeVersionOverride) + version, err := versionoverride.Resolve(planner.image, planner.runtimeVersionOverride) if err != nil { return fmt.Sprintf( "role-specific PodTemplates require every planner component to use Dynamo runtime %s or later; planner component %q has no resolvable runtime version: %v", diff --git a/deploy/operator/internal/webhook/validation/shared_helpers.go b/deploy/operator/internal/webhook/validation/shared_helpers.go index a5780d059a14..2d903624d701 100644 --- a/deploy/operator/internal/webhook/validation/shared_helpers.go +++ b/deploy/operator/internal/webhook/validation/shared_helpers.go @@ -30,7 +30,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/epp" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" @@ -153,7 +153,7 @@ func runtimeVersionOverrideRequired(image, override string) bool { if override != "" { return false } - _, err := runtimeversion.ParseImageVersion(image) + _, err := versionoverride.ParseImageVersion(image) return err != nil } @@ -196,7 +196,7 @@ func eppRuntimeCompatibilityError(contract eppRuntimeContract, eppConfigPath *fi return nil } - version, err := runtimeversion.Resolve(contract.image, contract.runtimeVersionOverride) + version, err := versionoverride.Resolve(contract.image, contract.runtimeVersionOverride) if err != nil { return nil } diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 076800b18f37..5e0f2dd8fb56 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -20,8 +20,10 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -43,6 +45,10 @@ const ( frontendImage150 = "registry.example/dynamo-frontend:1.5.0" legacyEPPImage140 = "registry.example/epp-image:1.4.0" legacySeedUsername = "operatorenv-legacy-seeder" + admissionAISimulateImage = "registry.example.com/aisimulate:nightly" + // admissionAISimulateVersionOverride is the AISimulate version bundled in the configured + // default image, whose tag is not a Dynamo release. + admissionAISimulateVersionOverride = "0.13.0" ) func groveProviderOverride(target, value string) *nvidiacomv1beta1.ProviderOverride { @@ -65,6 +71,9 @@ var ( // Fixture barriers observe the same cache used by webhook validation. admissionReader client.Reader + // Hardware that the v1beta2 defaulter discovers; tests set it before each case. + admissionHardware = &fakeHardwareDiscoverer{} + // Admission cases must remain sequential because they share this gate and a cluster-scoped topology fixture. admissionGate = &mutableFeatureGate{} admissionEnv = operatorenv.New(operatorenv.Options{ @@ -91,9 +100,35 @@ func setupAdmissionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptio OperatorVersion: opts.OperatorVersion, OperatorPrincipal: opts.OperatorPrincipal, Gate: admissionGate, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + AISimulateImage: admissionAISimulateImage, + AISimulateVersionOverride: admissionAISimulateVersionOverride, + Hardware: admissionHardware, + }, }) } +// fakeHardwareDiscoverer stands in for the cluster's GPU discovery. Cases run sequentially, +// so a plain mutex-guarded value is enough. +type fakeHardwareDiscoverer struct { + mu sync.Mutex + found gpu.GPUInfo + err error +} + +func (f *fakeHardwareDiscoverer) set(found gpu.GPUInfo, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.found, f.err = found, err +} + +func (f *fakeHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + found := f.found + return &found, f.err +} + type mutableFeatureGate struct { mu sync.RWMutex gates features.Gate @@ -122,6 +157,10 @@ type admissionTestCase struct { terminating bool username string + // beforeUpdate runs after the old state is seeded and before the update is submitted. Use + // it to change what the webhooks see between the two writes, such as hardware discovery. + beforeUpdate func(*testing.T) + wantSchemaError string wantCELError string wantStatusError string @@ -166,6 +205,10 @@ func runAdmissionTest(t *testing.T, test admissionTestCase) *unstructured.Unstru old = beginTermination(t, seedClient, old) } + if test.beforeUpdate != nil { + test.beforeUpdate(t) + } + t.Log("Submit the update request through the Kubernetes API server") warnings.clear() oldFixture, _ := admissionObject(t, test.oldObject, env.Namespace(), nil) diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 3a476e4ae4fc..ee9c3e5766c5 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3242,8 +3242,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
and execution defaults are resolved outside this API. | | MinLength: 1
Optional: \{\}
| -| `versionOverride` _string_ | VersionOverride declares the AISimulate compatibility version when it
cannot be inferred from the effective image. | | MinLength: 1
Optional: \{\}
| +| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
persists the operator's configured AISimulate image, or the Dynamo planner image of
the operator's release, together with the AISimulate version it bundles. | | MinLength: 1
Optional: \{\}
| +| `versionOverride` _string_ | nightly or custom image), and it wins over the mapping. | | MinLength: 1
Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| | `parameters` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Parameters contains native AISimulate input. Field names and nesting use
the upstream AISimulate JSON contract, including snake_case names. | | Type: object
Optional: \{\}
| @@ -3438,8 +3438,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3480,8 +3480,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| | `v1beta1` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | V1Beta1 losslessly preserves a native v1beta1 spec while this object is
represented through v1beta2. It is mutually exclusive with native fields.
This field is a read-only conversion view; modify the request through the
native v1beta1 API. | | Type: object
Optional: \{\}
| @@ -3543,8 +3543,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3686,8 +3686,8 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, the schema supplies the default Optimize mode and SLA;
admission supplies the conditional scalar metric. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3759,10 +3759,10 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization or a Pareto search. | Optimize | Enum: [Optimize Pareto]
Optional: \{\}
| -| `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| +| `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization or a Pareto search. | | Enum: [Optimize Pareto]
Optional: \{\}
| +| `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to Throughput. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| | `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics lists two or more distinct scalar targets when mode is Pareto. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 7
MinItems: 2
Optional: \{\}
| -| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. | \{ itlMs:30 ttftMs:2000 \} | Optional: \{\}
| +| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. When omitted, no
bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. | | Optional: \{\}
| #### RecommendationSpec @@ -3781,7 +3781,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `maxCandidates` _integer_ | MaxCandidates is the maximum number of DGDCs owned by one run. Scalar
candidates are ordered best first; Pareto candidates have no scalar rank.
This field does not limit internal trials. | 1 | Maximum: 100
Minimum: 1
Optional: \{\}
| +| `maxCandidates` _integer_ | MaxCandidates is the maximum number of DGDCs owned by one run. Scalar
candidates are ordered best first; Pareto candidates have no scalar rank.
This field does not limit internal trials. | | Maximum: 100
Minimum: 1
Optional: \{\}
| #### ResourceReference @@ -3855,7 +3855,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines one of AISimulate's synthetic workload shapes. When omitted,
it searches the full KV-load ratio range at ISL 4000 and OSL 1000.
An explicit empty object opts out of this underlay. | \{ isl:4000 kvLoadRatio:[0 1] osl:1000 \} | Optional: \{\}
| +| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines one of AISimulate's synthetic workload shapes. Its fields are not
defaulted individually: the default belongs to spec.workload as a whole, so an
explicit workload object keeps exactly the fields it names. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index e262b3c7644f..4c073e979aee 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4531,13 +4531,13 @@ AISimulateSpec selects the AISimulate execution artifact and supplies native inp **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -Image selects a Dynamo-compatible AISimulate runner. When omitted, admission and execution defaults are resolved outside this API. +Image selects a Dynamo-compatible AISimulate runner. When omitted, admission persists the operator's configured AISimulate image, or the Dynamo planner image of the operator's release, together with the AISimulate version it bundles. **Validation:** MinLength: 1 Optional: \{\} -VersionOverride declares the AISimulate compatibility version when it cannot be inferred from the effective image. -**Validation:** MinLength: 1 Optional: \{\} +nightly or custom image), and it wins over the mapping. +**Validation:** MinLength: 1 Pattern: `^(0|[1-9][0-9]\{0,3\})\.(0|[1-9][0-9]\{0,3\})\.(0|[1-9][0-9]\{0,3\})$` Optional: \{\} @@ -4806,13 +4806,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from admission. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -4888,13 +4888,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from admission. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5010,13 +5010,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from admission. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5209,13 +5209,13 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from admission. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, the schema supplies the default Optimize mode and SLA; admission supplies the conditional scalar metric. +Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5289,14 +5289,14 @@ ObjectiveSpec defines scalar optimization or a multi-objective Pareto search. **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) - + Mode selects scalar optimization or a Pareto search. See [ObjectiveMode](#objectivemode). **Validation:** Enum: [Optimize Pareto] Optional: \{\} -Metric is the scalar target when mode is Optimize. When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. +Metric is the scalar target when mode is Optimize. When omitted in Optimize mode, admission defaults it to Throughput. See [ObjectiveMetric](#objectivemetric). **Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] Optional: \{\} @@ -5307,8 +5307,8 @@ See [ObjectiveMetric](#objectivemetric). **Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] MaxItems: 7 MinItems: 2 Optional: \{\} - -SLA defines per-request latency bounds used to calculate goodput. + +SLA defines per-request latency bounds used to calculate goodput. When omitted, no bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. See [ObjectiveSLASpec](#objectiveslaspec). **Validation:** Optional: \{\} @@ -5321,7 +5321,7 @@ RecommendationSpec controls bounded projection into DGDC resources. **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) - + MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first; Pareto candidates have no scalar rank. This field does not limit internal trials. **Validation:** Maximum: 100 Minimum: 1 Optional: \{\} @@ -5407,8 +5407,8 @@ WorkloadSpec defines the synthetic traffic model used for every candidate evalua **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) - -Static defines one of AISimulate's synthetic workload shapes. When omitted, it searches the full KV-load ratio range at ISL 4000 and OSL 1000. An explicit empty object opts out of this underlay. + +Static defines one of AISimulate's synthetic workload shapes. Its fields are not defaulted individually: the default belongs to spec.workload as a whole, so an explicit workload object keeps exactly the fields it names. See [StaticWorkloadSpec](#staticworkloadspec). **Validation:** Optional: \{\} From 9f79e53218a160ec281b5a85f2dc15e87ab89ab9 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 26 Oct 2026 15:56:52 +0100 Subject: [PATCH 007/100] pr-16661: pr-4: squash operator-owned AISimulate execution Squashed from https://github.com/devivasudevan/dynamo/pull/4. Signed-off-by: Dr. Stefan Schimanski --- .../operator/templates/deployment.yaml | 6 - .../platform/components/operator/values.yaml | 10 - deploy/operator/api/roundtrip_fuzz_test.go | 34 +- ...ographdeploymentrequest_conversion_test.go | 11 +- .../dynamographdeploymentrequest_types.go | 143 +++--- .../v1beta2/dynamographdeploymentrun_types.go | 16 + .../api/v1beta2/zz_generated.deepcopy.go | 121 +++-- deploy/operator/cmd/main.go | 23 +- ...dia.com_dynamographdeploymentrequests.yaml | 146 +++--- .../nvidia.com_dynamographdeploymentruns.yaml | 157 +++--- .../operator/internal/aisimulate/defaults.go | 82 +++ deploy/operator/internal/aisimulate/doc.go | 40 ++ deploy/operator/internal/aisimulate/input.go | 280 +++++++++++ .../internal/aisimulate/input_test.go | 471 ++++++++++++++++++ deploy/operator/internal/aisimulate/paths.go | 59 +++ .../controller/aisimulate/defaults.go | 66 --- .../internal/controller/aisimulate/doc.go | 38 -- .../internal/controller/aisimulate/image.go | 62 --- .../controller/aisimulate/image_test.go | 52 -- .../internal/controller/aisimulate/input.go | 250 ---------- .../controller/aisimulate/input_test.go | 433 ---------------- .../internal/controller/aisimulate/paths.go | 59 --- .../aisimulate/release_contract_test.go | 78 --- .../internal/controller/suite_envtest_test.go | 7 +- .../internal/dynamo/component_common.go | 4 +- .../dynamo/component_frontend_test.go | 10 +- .../dynamo/component_namespace_test.go | 12 +- .../internal/dynamo/component_worker_test.go | 18 +- deploy/operator/internal/dynamo/graph.go | 10 +- deploy/operator/internal/dynamo/hash.go | 6 +- .../internal/features/runtime/gate.go | 6 +- .../internal/features/runtime/gate_test.go | 12 +- .../internal/features/runtime/gates.go | 13 +- .../runtimeversion.go} | 8 +- .../runtimeversion_test.go} | 2 +- .../dynamographdeploymentrequest_handler.go | 9 - ...namographdeploymentrequest_routing_test.go | 42 +- .../dynamographdeploymentrequest_v1beta2.go | 67 +-- ...graphdeploymentrequest_v1beta2_artifact.go | 57 --- ...deploymentrequest_v1beta2_artifact_test.go | 120 ----- ...ographdeploymentrequest_v1beta2_helpers.go | 17 +- ...namographdeploymentrequest_v1beta2_test.go | 10 +- .../operator/internal/webhook/setup/setup.go | 3 +- .../dynamographdeploymentrequest.go | 4 +- .../dynamographdeploymentrequest_v1beta2.go | 46 +- ...namographdeploymentrequest_v1beta2_test.go | 110 +--- ...ploymentrequest_validation_envtest_test.go | 210 +++----- .../validation/planner_runtime_features.go | 4 +- .../webhook/validation/shared_helpers.go | 6 +- .../webhook/validation/suite_envtest_test.go | 8 +- .../additional-resources/api-reference-k8s.md | 112 +++-- .../kubernetes-api/full-api-reference.mdx | 151 ++++-- 52 files changed, 1685 insertions(+), 2036 deletions(-) create mode 100644 deploy/operator/internal/aisimulate/defaults.go create mode 100644 deploy/operator/internal/aisimulate/doc.go create mode 100644 deploy/operator/internal/aisimulate/input.go create mode 100644 deploy/operator/internal/aisimulate/input_test.go create mode 100644 deploy/operator/internal/aisimulate/paths.go delete mode 100644 deploy/operator/internal/controller/aisimulate/defaults.go delete mode 100644 deploy/operator/internal/controller/aisimulate/doc.go delete mode 100644 deploy/operator/internal/controller/aisimulate/image.go delete mode 100644 deploy/operator/internal/controller/aisimulate/image_test.go delete mode 100644 deploy/operator/internal/controller/aisimulate/input.go delete mode 100644 deploy/operator/internal/controller/aisimulate/input_test.go delete mode 100644 deploy/operator/internal/controller/aisimulate/paths.go delete mode 100644 deploy/operator/internal/controller/aisimulate/release_contract_test.go rename deploy/operator/internal/{versionoverride/versionoverride.go => runtimeversion/runtimeversion.go} (91%) rename deploy/operator/internal/{versionoverride/versionoverride_test.go => runtimeversion/runtimeversion_test.go} (99%) delete mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact.go delete mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact_test.go diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index 483d9e381207..8a4ea7dd9080 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -92,12 +92,6 @@ spec: {{- with .Values.controllerManager.manager.dgdrDefaultImage }} - {{ printf "--dgdr-default-image=%s" . | quote }} {{- end }} - {{- with .Values.controllerManager.manager.dgdrAISimulateDefaultImage }} - - {{ printf "--dgdr-aisimulate-default-image=%s" . | quote }} - {{- end }} - {{- with .Values.controllerManager.manager.dgdrAISimulateDefaultVersionOverride }} - - {{ printf "--dgdr-aisimulate-default-version-override=%s" . | quote }} - {{- end }} command: - /manager env: diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 007019cb491e..c526a93bebde 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -115,16 +115,6 @@ controllerManager: # derives dynamo-planner:, which only exists for GA # releases. dgdrDefaultImage: "" - # AISimulate runner image put into v1beta2 DGDR spec.aisimulate.image when unset. - # Empty derives nvcr.io/nvidia/ai-dynamo/dynamo-planner:. - dgdrAISimulateDefaultImage: "" - # AISimulate version bundled in the default image (for example 0.13.0), put into v1beta2 DGDR - # spec.aisimulate.versionOverride together with that image. Empty uses the AISimulate version - # the operator maps to the image's Dynamo release. Required when the image is not a mapped - # Dynamo release (for example nightly); the operator refuses to start without it. It also - # applies to the derived planner image when dgdrAISimulateDefaultImage is empty. A Request - # that names its own image never receives it. - dgdrAISimulateDefaultVersionOverride: "" resources: limits: cpu: 1024m diff --git a/deploy/operator/api/roundtrip_fuzz_test.go b/deploy/operator/api/roundtrip_fuzz_test.go index 7f8eecad02bf..e3dbe001fd5f 100644 --- a/deploy/operator/api/roundtrip_fuzz_test.go +++ b/deploy/operator/api/roundtrip_fuzz_test.go @@ -60,6 +60,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/conversion" "sigs.k8s.io/randfill" "sigs.k8s.io/yaml" @@ -568,22 +569,21 @@ func fuzzBeta2Workload(workload *v1beta2.WorkloadSpec, c randfill.Continue) { } static := workload.Static - static.ISL = oneOfPtr(c, int32(1), int32(4000), int32(131072)) - static.OSL = oneOfPtr(c, int32(1), int32(1000), int32(16384)) - static.NumRequestRatio = oneOfPtr(c, 0.5, 1.0, 1000.0) - static.RequestRate = nil - static.Concurrency = nil - static.KVLoadRatio = nil + static.InputTokens = oneOfPtr(c, int32(1), int32(4000), int32(131072)) + static.OutputTokens = oneOfPtr(c, int32(1), int32(1000), int32(16384)) + static.Load = nil switch c.Intn(4) { case 1: - static.RequestRate = oneOfPtr(c, 0.1, 1.0, 100.0) + static.Load = &v1beta2.StaticLoadSpec{RequestsPerSecond: oneOfPtr(c, 0.1, 1.0, 100.0)} case 2: - static.Concurrency = oneOfPtr(c, int32(1), int32(32), int32(1024)) + static.Load = &v1beta2.StaticLoadSpec{Concurrency: oneOfPtr(c, int32(1), int32(32), int32(1024))} case 3: if c.Bool() { - static.KVLoadRatio = v1beta2.KVLoadRatio{0.5} + static.Load = &v1beta2.StaticLoadSpec{KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{Value: ptr.To(0.5)}} } else { - static.KVLoadRatio = v1beta2.KVLoadRatio{0.0, 1.0} + static.Load = &v1beta2.StaticLoadSpec{KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{ + Range: &v1beta2.KVCapacityFractionRange{Min: 0.1, Max: 1.0}, + }} } } } @@ -653,7 +653,7 @@ func isGoodputMetric(metric v1beta2.ObjectiveMetric) bool { return metric == "Goodput" || metric == "GoodputPerGPU" } -type fuzzAISimulateParameters struct { +type fuzzAISimulateInputOverrides struct { Search *fuzzAISimulateSearch `json:"search,omitempty"` Workload *fuzzAISimulateWorkload `json:"workload,omitempty"` } @@ -671,20 +671,20 @@ type fuzzAISimulateWorkload struct { func fuzzAISimulateSpec(s *v1beta2.AISimulateSpec, c randfill.Continue) { c.FillNoCustom(s) - // Exercise both absent and typed object parameter payloads. + // Exercise both absent and typed object input override payloads. if c.Bool() { - s.Parameters = nil + s.InputOverrides = nil return } // Marshal a typed test input instead of generating arbitrary RawExtension bytes. - parameters := &fuzzAISimulateParameters{} - c.Fill(parameters) - raw, err := json.Marshal(parameters) + overrides := &fuzzAISimulateInputOverrides{} + c.Fill(overrides) + raw, err := json.Marshal(overrides) if err != nil { panic(err) } - s.Parameters = &runtime.RawExtension{Raw: raw} + s.InputOverrides = &runtime.RawExtension{Raw: raw} } func marshalFuzzEnvelope(value any) *apiextensionsv1.JSON { diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_conversion_test.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_conversion_test.go index 19e4f3721406..004527cb90c2 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_conversion_test.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_conversion_test.go @@ -97,10 +97,9 @@ func TestV1Beta2RoundTripUsesEnvelope(t *testing.T) { }, Workload: &WorkloadSpec{ Static: &StaticWorkloadSpec{ - ISL: new(int32(4096)), - OSL: new(int32(1024)), - NumRequestRatio: new(2.0), - RequestRate: new(12.5), + InputTokens: new(int32(4096)), + OutputTokens: new(int32(1024)), + Load: &StaticLoadSpec{RequestsPerSecond: new(12.5)}, }, }, Objective: &ObjectiveSpec{ @@ -109,9 +108,7 @@ func TestV1Beta2RoundTripUsesEnvelope(t *testing.T) { SLA: &ObjectiveSLASpec{TTFTMs: new(2000.0), ITLMs: new(30.0)}, }, AISimulate: &AISimulateSpec{ - Image: "nvcr.io/nvidia/aisimulate:latest", - VersionOverride: "1.0.0", - Parameters: &runtime.RawExtension{Raw: []byte(`{"search_method":"grid"}`)}, + InputOverrides: &runtime.RawExtension{Raw: []byte(`{"search_method":"grid"}`)}, }, Recommendation: &RecommendationSpec{MaxCandidates: 3}, }, diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index d7413eb3936a..0b28070cb360 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -64,56 +64,81 @@ type HardwareSpec struct { GPU *GPUHardwareSpec `json:"gpu,omitempty"` } -// KVLoadRatio is either a pinned load ratio or a minimum/maximum range. -// AISimulate performs mode-specific validation. -// +kubebuilder:validation:MinItems=1 -// +kubebuilder:validation:MaxItems=2 -// +kubebuilder:validation:items:Minimum=0 -// +kubebuilder:validation:items:Maximum=1 -// +kubebuilder:validation:XValidation:rule="size(self) != 2 || self[0] < self[1]",message="a range needs min < max" -type KVLoadRatio []float64 - -// StaticWorkloadSpec defines one of AISimulate's synthetic workload shapes. -// +kubebuilder:validation:XValidation:rule="(has(self.requestRate) ? 1 : 0) + (has(self.concurrency) ? 1 : 0) + (has(self.kvLoadRatio) ? 1 : 0) <= 1",message="at most one of requestRate, concurrency, or kvLoadRatio may be specified" -type StaticWorkloadSpec struct { - // ISL is the input sequence length in tokens. - // +optional - // +kubebuilder:validation:Minimum=1 - ISL *int32 `json:"isl,omitempty"` +// KVCapacityFractionRange defines the searched fraction of a candidate's KV capacity. +// +kubebuilder:validation:XValidation:rule="self.min < self.max",message="min must be less than max" +type KVCapacityFractionRange struct { + // Min is the lower end of the searched fraction range. + // +kubebuilder:validation:ExclusiveMinimum=true + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 + Min float64 `json:"min"` - // OSL is the output sequence length in tokens. - // +optional - // +kubebuilder:validation:Minimum=1 - OSL *int32 `json:"osl,omitempty"` + // Max is the upper end of the searched fraction range. + // +kubebuilder:validation:ExclusiveMinimum=true + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=1 + Max float64 `json:"max"` +} - // NumRequestRatio sets request count relative to the selected load. +// KVCapacityFractionSpec selects either one fixed fraction or a searched range. +// +kubebuilder:validation:XValidation:rule="(has(self.value) ? 1 : 0) + (has(self.range) ? 1 : 0) == 1",message="exactly one of value or range must be specified" +type KVCapacityFractionSpec struct { + // Value uses a fixed fraction of the candidate's KV capacity. A value of 1 + // uses its full estimated capacity. // +optional // +kubebuilder:validation:ExclusiveMinimum=true // +kubebuilder:validation:Minimum=0 - NumRequestRatio *float64 `json:"numRequestRatio,omitempty"` + // +kubebuilder:validation:Maximum=1 + Value *float64 `json:"value,omitempty"` + + // Range searches fractions between min and max. + // +optional + Range *KVCapacityFractionRange `json:"range,omitempty"` +} - // RequestRate is a fixed open-loop arrival rate in requests per second. +// StaticLoadSpec selects exactly one synthetic load model by field presence. +// +kubebuilder:validation:XValidation:rule="(has(self.requestsPerSecond) ? 1 : 0) + (has(self.concurrency) ? 1 : 0) + (has(self.kvCapacityFraction) ? 1 : 0) == 1",message="exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified" +type StaticLoadSpec struct { + // RequestsPerSecond is a fixed open-loop arrival rate. // +optional // +kubebuilder:validation:ExclusiveMinimum=true // +kubebuilder:validation:Minimum=0 - RequestRate *float64 `json:"requestRate,omitempty"` + RequestsPerSecond *float64 `json:"requestsPerSecond,omitempty"` // Concurrency is a fixed closed-loop in-flight request cap. // +optional // +kubebuilder:validation:Minimum=1 Concurrency *int32 `json:"concurrency,omitempty"` - // KVLoadRatio derives concurrency from candidate KV capacity. A two-value - // range requires a bounded AISimulate sweep. + // KVCapacityFraction derives concurrency from each candidate's estimated KV capacity. // +optional - KVLoadRatio KVLoadRatio `json:"kvLoadRatio,omitempty"` + KVCapacityFraction *KVCapacityFractionSpec `json:"kvCapacityFraction,omitempty"` +} + +// StaticWorkloadSpec defines AISimulate synthetic traffic. Admission completes a nonempty +// object with the omitted static defaults; an explicit empty object opts out of them. +type StaticWorkloadSpec struct { + // InputTokens is the synthetic request input length in tokens. + // +optional + // +kubebuilder:validation:Minimum=1 + InputTokens *int32 `json:"inputTokens,omitempty"` + + // OutputTokens is the synthetic request output length in tokens. + // +optional + // +kubebuilder:validation:Minimum=1 + OutputTokens *int32 `json:"outputTokens,omitempty"` + + // Load selects the synthetic load model. When omitted from a nonempty static + // workload, admission defaults it to concurrency 10. + // +optional + Load *StaticLoadSpec `json:"load,omitempty"` } // WorkloadSpec defines the synthetic traffic model used for every candidate evaluation. type WorkloadSpec struct { - // Static defines one of AISimulate's synthetic workload shapes. Its fields are not - // defaulted individually: the default belongs to spec.workload as a whole, so an - // explicit workload object keeps exactly the fields it names. + // Static defines AISimulate synthetic traffic. When omitted, admission supplies the + // complete static default. An explicit empty object opts out; any nonempty object is + // completed with omitted inputTokens, outputTokens and load defaults. // +optional Static *StaticWorkloadSpec `json:"static,omitempty"` } @@ -156,6 +181,7 @@ type ObjectiveSLASpec struct { // ObjectiveSpec defines scalar optimization or a multi-objective Pareto search. // +kubebuilder:validation:XValidation:rule="!has(self.mode) || (self.mode == 'Optimize' && !has(self.metrics)) || (self.mode == 'Pareto' && !has(self.metric) && has(self.metrics))",message="Optimize accepts only metric; Pareto requires metrics" +// +kubebuilder:validation:XValidation:rule="!has(self.metrics) || (size(self.metrics) == 2 && self.metrics.exists(m, m == 'ThroughputPerGPU') && self.metrics.exists(m, m == 'ThroughputPerUser'))",message="Pareto supports exactly ThroughputPerGPU and ThroughputPerUser" // +kubebuilder:validation:XValidation:rule="!(has(self.metric) && self.metric in ['Goodput', 'GoodputPerGPU']) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))",message="Goodput and GoodputPerGPU require at least one SLA bound" // +kubebuilder:validation:XValidation:rule="!(has(self.metrics) && self.metrics.exists(m, m in ['Goodput', 'GoodputPerGPU'])) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))",message="Pareto metrics containing Goodput or GoodputPerGPU require at least one SLA bound" type ObjectiveSpec struct { @@ -168,48 +194,33 @@ type ObjectiveSpec struct { // +optional Metric *ObjectiveMetric `json:"metric,omitempty"` - // Metrics lists two or more distinct scalar targets when mode is Pareto. + // Metrics selects AISimulate's public Pareto objective pair when mode is Pareto. + // The configured AISimulate release supports exactly ThroughputPerGPU and + // ThroughputPerUser through its public recommendation input. // +optional // +kubebuilder:validation:MinItems=2 - // +kubebuilder:validation:MaxItems=7 + // +kubebuilder:validation:MaxItems=2 // +listType=set Metrics []ObjectiveMetric `json:"metrics,omitempty"` // SLA defines per-request latency bounds used to calculate goodput. When omitted, no - // bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. + // bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. // +optional SLA *ObjectiveSLASpec `json:"sla,omitempty"` } -// AISimulateSpec selects the AISimulate execution artifact and supplies native input. +// AISimulateSpec supplies public recommendation input to the AISimulate version in the operator's configured +// planner image. type AISimulateSpec struct { - // Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - // persists the operator's configured AISimulate image, or the Dynamo planner image of - // the operator's release, together with the AISimulate version it bundles. - // +optional - // +kubebuilder:validation:MinLength=1 - Image string `json:"image,omitempty"` - - // VersionOverride declares the AISimulate version the image bundles as three numeric parts - - // (for example 0.13.0), not a semver version: no v prefix, pre-release or build suffix. - - // When omitted, the version is the one the operator maps to the Dynamo release in the - - // image tag. It is required when the tag is not a Dynamo release the operator maps (a - - // nightly or custom image), and it wins over the mapping. - // +optional - // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$` - VersionOverride string `json:"versionOverride,omitempty"` - - // Parameters contains native AISimulate input. Field names and nesting use - // the upstream AISimulate JSON contract, including snake_case names. + // InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from + // the typed fields. It uses the upstream AISimulate JSON contract, including + // snake_case names. Values here take precedence, and null deletes a lowered + // field. The operator runs the AISimulate version from its configured planner + // image; Requests and Runs cannot select a different image or version. // +optional // +kubebuilder:pruning:PreserveUnknownFields // +kubebuilder:validation:Type=object - Parameters *runtime.RawExtension `json:"parameters,omitempty"` + InputOverrides *runtime.RawExtension `json:"inputOverrides,omitempty"` } // RecommendationSpec controls bounded projection into DGDC resources. @@ -242,13 +253,13 @@ type NativeSearchInput struct { Hardware *HardwareSpec `json:"hardware,omitempty"` // Workload defines the synthetic traffic model used for every candidate evaluation. - // When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; - // an explicit empty workload receives the same shape from admission. An explicit - // workload.static: {} opts out of that shape, leaving the workload to - // spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the - // fields it names. - // The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a - // KV-load ratio search. + // When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; + // an explicit empty workload receives the same shape. workload.static: {} opts out of all + // traffic defaults. Any other static object is completed with its missing defaults. + // spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over + // every lowered traffic field it names. + // The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials + // nor a KV-capacity-fraction search. // +optional Workload *WorkloadSpec `json:"workload,omitempty"` @@ -258,7 +269,7 @@ type NativeSearchInput struct { // +optional Objective *ObjectiveSpec `json:"objective,omitempty"` - // AISimulate selects the execution artifact and supplies native AISimulate input. + // AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. // +optional AISimulate *AISimulateSpec `json:"aisimulate,omitempty"` diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go index 692520aab96b..280a487756b4 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go @@ -20,6 +20,7 @@ package v1beta2 import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) @@ -119,6 +120,17 @@ type DGDRRunProgress struct { Candidates *CandidateProgress `json:"candidates,omitempty"` } +// AISimulateRunStatus reports the exact public recommendation input used for this execution. +type AISimulateRunStatus struct { + // Input is the fully lowered AISimulate input after typed fields, derived defaults and + // spec.aisimulate.inputOverrides have been merged. Field names and nesting use the + // upstream AISimulate JSON contract. + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Type=object + Input *runtime.RawExtension `json:"input,omitempty"` +} + // DynamoGraphDeploymentRunStatus represents one search execution's observed state. type DynamoGraphDeploymentRunStatus struct { // Phase is a concise presentation state. Conditions remain authoritative. @@ -133,6 +145,10 @@ type DynamoGraphDeploymentRunStatus struct { // +optional Progress *DGDRRunProgress `json:"progress,omitempty"` + // AISimulate reports the effective input executed by AISimulate. + // +optional + AISimulate *AISimulateRunStatus `json:"aisimulate,omitempty"` + // CandidateRefs identifies the current bounded DGDC projection. List order // represents rank, best first, for scalar searches. // +optional diff --git a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go index 2188565147ed..fe68ab15bfa9 100644 --- a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go @@ -44,11 +44,31 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AISimulateRunStatus) DeepCopyInto(out *AISimulateRunStatus) { + *out = *in + if in.Input != nil { + in, out := &in.Input, &out.Input + *out = new(runtime.RawExtension) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AISimulateRunStatus. +func (in *AISimulateRunStatus) DeepCopy() *AISimulateRunStatus { + if in == nil { + return nil + } + out := new(AISimulateRunStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AISimulateSpec) DeepCopyInto(out *AISimulateSpec) { *out = *in - if in.Parameters != nil { - in, out := &in.Parameters, &out.Parameters + if in.InputOverrides != nil { + in, out := &in.InputOverrides, &out.InputOverrides *out = new(runtime.RawExtension) (*in).DeepCopyInto(*out) } @@ -458,6 +478,11 @@ func (in *DynamoGraphDeploymentRunStatus) DeepCopyInto(out *DynamoGraphDeploymen *out = new(DGDRRunProgress) (*in).DeepCopyInto(*out) } + if in.AISimulate != nil { + in, out := &in.AISimulate, &out.AISimulate + *out = new(AISimulateRunStatus) + (*in).DeepCopyInto(*out) + } if in.CandidateRefs != nil { in, out := &in.CandidateRefs, &out.CandidateRefs *out = make([]corev1.LocalObjectReference, len(*in)) @@ -555,22 +580,43 @@ func (in *HardwareSpec) DeepCopy() *HardwareSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in KVLoadRatio) DeepCopyInto(out *KVLoadRatio) { - { - in := &in - *out = make(KVLoadRatio, len(*in)) - copy(*out, *in) +func (in *KVCapacityFractionRange) DeepCopyInto(out *KVCapacityFractionRange) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KVCapacityFractionRange. +func (in *KVCapacityFractionRange) DeepCopy() *KVCapacityFractionRange { + if in == nil { + return nil + } + out := new(KVCapacityFractionRange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KVCapacityFractionSpec) DeepCopyInto(out *KVCapacityFractionSpec) { + *out = *in + if in.Value != nil { + in, out := &in.Value, &out.Value + *out = new(float64) + **out = **in + } + if in.Range != nil { + in, out := &in.Range, &out.Range + *out = new(KVCapacityFractionRange) + **out = **in } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KVLoadRatio. -func (in KVLoadRatio) DeepCopy() KVLoadRatio { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KVCapacityFractionSpec. +func (in *KVCapacityFractionSpec) DeepCopy() *KVCapacityFractionSpec { if in == nil { return nil } - out := new(KVLoadRatio) + out := new(KVCapacityFractionSpec) in.DeepCopyInto(out) - return *out + return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -744,37 +790,52 @@ func (in *RoundProgress) DeepCopy() *RoundProgress { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StaticWorkloadSpec) DeepCopyInto(out *StaticWorkloadSpec) { +func (in *StaticLoadSpec) DeepCopyInto(out *StaticLoadSpec) { *out = *in - if in.ISL != nil { - in, out := &in.ISL, &out.ISL - *out = new(int32) + if in.RequestsPerSecond != nil { + in, out := &in.RequestsPerSecond, &out.RequestsPerSecond + *out = new(float64) **out = **in } - if in.OSL != nil { - in, out := &in.OSL, &out.OSL + if in.Concurrency != nil { + in, out := &in.Concurrency, &out.Concurrency *out = new(int32) **out = **in } - if in.NumRequestRatio != nil { - in, out := &in.NumRequestRatio, &out.NumRequestRatio - *out = new(float64) - **out = **in + if in.KVCapacityFraction != nil { + in, out := &in.KVCapacityFraction, &out.KVCapacityFraction + *out = new(KVCapacityFractionSpec) + (*in).DeepCopyInto(*out) } - if in.RequestRate != nil { - in, out := &in.RequestRate, &out.RequestRate - *out = new(float64) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StaticLoadSpec. +func (in *StaticLoadSpec) DeepCopy() *StaticLoadSpec { + if in == nil { + return nil + } + out := new(StaticLoadSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StaticWorkloadSpec) DeepCopyInto(out *StaticWorkloadSpec) { + *out = *in + if in.InputTokens != nil { + in, out := &in.InputTokens, &out.InputTokens + *out = new(int32) **out = **in } - if in.Concurrency != nil { - in, out := &in.Concurrency, &out.Concurrency + if in.OutputTokens != nil { + in, out := &in.OutputTokens, &out.OutputTokens *out = new(int32) **out = **in } - if in.KVLoadRatio != nil { - in, out := &in.KVLoadRatio, &out.KVLoadRatio - *out = make(KVLoadRatio, len(*in)) - copy(*out, *in) + if in.Load != nil { + in, out := &in.Load, &out.Load + *out = new(StaticLoadSpec) + (*in).DeepCopyInto(*out) } } diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index fab7f7d378f6..e51be09f02d3 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -157,8 +157,6 @@ func main() { var operatorImage string var operatorImagePullPolicy string var dgdrDefaultImage string - var dgdrAISimulateDefaultImage string - var dgdrAISimulateDefaultVersionOverride string flag.StringVar(&configFile, "config", "", "Path to operator configuration file (required)") flag.StringVar(&operatorVersion, "operator-version", "unknown", "Version of the operator (used in lease holder identity)") @@ -172,13 +170,6 @@ func main() { "Image pull policy for operator helper init containers") flag.StringVar(&dgdrDefaultImage, "dgdr-default-image", "", "Default DGDR profiler image, put into DGDR spec.image when unset; empty derives dynamo-planner:") - flag.StringVar(&dgdrAISimulateDefaultImage, "dgdr-aisimulate-default-image", "", - "AISimulate runner image, put into v1beta2 DGDR spec.aisimulate.image when unset; "+ - "empty derives dynamo-planner:") - flag.StringVar(&dgdrAISimulateDefaultVersionOverride, "dgdr-aisimulate-default-version-override", "", - "AISimulate version bundled in the AISimulate default image, put into v1beta2 DGDR "+ - "spec.aisimulate.versionOverride together with that image; empty uses the version the release "+ - "contract maps to the image's Dynamo release, and is required for an image without one") opts := zap.Options{ Development: true, } @@ -186,13 +177,6 @@ func main() { flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - if err := webhookdefaulting.ValidateAISimulateDefault( - operatorVersion, dgdrAISimulateDefaultImage, dgdrAISimulateDefaultVersionOverride, - ); err != nil { - setupLog.Error(err, "invalid AISimulate default") - os.Exit(1) - } - if configFile == "" { setupLog.Error(nil, "--config flag is required") os.Exit(1) @@ -518,8 +502,7 @@ func main() { } if err := registerWebhookHandlers( - mgr, operatorCfg, runtimeConfig, operatorVersion, dgdrDefaultImage, dgdrAISimulateDefaultImage, - dgdrAISimulateDefaultVersionOverride, gates, + mgr, operatorCfg, runtimeConfig, operatorVersion, dgdrDefaultImage, gates, ); err != nil { setupLog.Error(err, "failed to register webhooks") os.Exit(1) @@ -655,8 +638,6 @@ func registerWebhookHandlers( runtimeConfig *commonController.RuntimeConfig, operatorVersion string, dgdrDefaultImage string, - dgdrAISimulateDefaultImage string, - dgdrAISimulateDefaultVersionOverride string, gate features.Gate, ) error { var operatorPrincipal string @@ -675,8 +656,6 @@ func registerWebhookHandlers( DefaultImage: dgdrDefaultImage, }, DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ - AISimulateImage: dgdrAISimulateDefaultImage, - AISimulateVersionOverride: dgdrAISimulateDefaultVersionOverride, // The same discovery order as the DGDR reconciler: DCGM first, then node labels when // the GPUDiscovery gate is on. The webhook keeps its own short-lived cache. Hardware: &gpu.Discoverer{ diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 98a82b7d21d6..164a5cb025d0 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9598,26 +9598,17 @@ spec: Native v1beta2 fields and the V1Beta1 conversion envelope are mutually exclusive. properties: aisimulate: - description: AISimulate selects the execution artifact and supplies native AISimulate input. + description: AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. properties: - image: + inputOverrides: description: |- - Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - persists the operator's configured AISimulate image, or the Dynamo planner image of - the operator's release, together with the AISimulate version it bundles. - minLength: 1 - type: string - parameters: - description: |- - Parameters contains native AISimulate input. Field names and nesting use - the upstream AISimulate JSON contract, including snake_case names. + InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from + the typed fields. It uses the upstream AISimulate JSON contract, including + snake_case names. Values here take precedence, and null deletes a lowered + field. The operator runs the AISimulate version from its configured planner + image; Requests and Runs cannot select a different image or version. type: object x-kubernetes-preserve-unknown-fields: true - versionOverride: - description: nightly or custom image), and it wins over the mapping. - minLength: 1 - pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ - type: string type: object backends: description: Backends lists one or more inference backends searched by AISimulate and used by generated candidates. @@ -9686,7 +9677,10 @@ spec: - GoodputPerGPU type: string metrics: - description: Metrics lists two or more distinct scalar targets when mode is Pareto. + description: |- + Metrics selects AISimulate's public Pareto objective pair when mode is Pareto. + The configured AISimulate release supports exactly ThroughputPerGPU and + ThroughputPerUser through its public recommendation input. items: description: ObjectiveMetric is a metric optimized by AISimulate. enum: @@ -9698,7 +9692,7 @@ spec: - Goodput - GoodputPerGPU type: string - maxItems: 7 + maxItems: 2 minItems: 2 type: array x-kubernetes-list-type: set @@ -9711,7 +9705,7 @@ spec: sla: description: |- SLA defines per-request latency bounds used to calculate goodput. When omitted, no - bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. + bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. @@ -9736,6 +9730,8 @@ spec: x-kubernetes-validations: - message: Optimize accepts only metric; Pareto requires metrics rule: '!has(self.mode) || (self.mode == ''Optimize'' && !has(self.metrics)) || (self.mode == ''Pareto'' && !has(self.metric) && has(self.metrics))' + - message: Pareto supports exactly ThroughputPerGPU and ThroughputPerUser + rule: '!has(self.metrics) || (size(self.metrics) == 2 && self.metrics.exists(m, m == ''ThroughputPerGPU'') && self.metrics.exists(m, m == ''ThroughputPerUser''))' - message: Goodput and GoodputPerGPU require at least one SLA bound rule: '!(has(self.metric) && self.metric in [''Goodput'', ''GoodputPerGPU'']) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))' - message: Pareto metrics containing Goodput or GoodputPerGPU require at least one SLA bound @@ -9764,63 +9760,87 @@ spec: workload: description: |- Workload defines the synthetic traffic model used for every candidate evaluation. - When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; - an explicit empty workload receives the same shape from admission. An explicit - workload.static: {} opts out of that shape, leaving the workload to - spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the - fields it names. - The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a - KV-load ratio search. + When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; + an explicit empty workload receives the same shape. workload.static: {} opts out of all + traffic defaults. Any other static object is completed with its missing defaults. + spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over + every lowered traffic field it names. + The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials + nor a KV-capacity-fraction search. properties: static: description: |- - Static defines one of AISimulate's synthetic workload shapes. Its fields are not - defaulted individually: the default belongs to spec.workload as a whole, so an - explicit workload object keeps exactly the fields it names. + Static defines AISimulate synthetic traffic. When omitted, admission supplies the + complete static default. An explicit empty object opts out; any nonempty object is + completed with omitted inputTokens, outputTokens and load defaults. properties: - concurrency: - description: Concurrency is a fixed closed-loop in-flight request cap. + inputTokens: + description: InputTokens is the synthetic request input length in tokens. format: int32 minimum: 1 type: integer - isl: - description: ISL is the input sequence length in tokens. - format: int32 - minimum: 1 - type: integer - kvLoadRatio: + load: description: |- - KVLoadRatio derives concurrency from candidate KV capacity. A two-value - range requires a bounded AISimulate sweep. - items: - maximum: 1 - minimum: 0 - type: number - maxItems: 2 - minItems: 1 - type: array + Load selects the synthetic load model. When omitted from a nonempty static + workload, admission defaults it to concurrency 10. + properties: + concurrency: + description: Concurrency is a fixed closed-loop in-flight request cap. + format: int32 + minimum: 1 + type: integer + kvCapacityFraction: + description: KVCapacityFraction derives concurrency from each candidate's estimated KV capacity. + properties: + range: + description: Range searches fractions between min and max. + properties: + max: + description: Max is the upper end of the searched fraction range. + exclusiveMinimum: true + maximum: 1 + minimum: 0 + type: number + min: + description: Min is the lower end of the searched fraction range. + exclusiveMinimum: true + maximum: 1 + minimum: 0 + type: number + required: + - max + - min + type: object + x-kubernetes-validations: + - message: min must be less than max + rule: self.min < self.max + value: + description: |- + Value uses a fixed fraction of the candidate's KV capacity. A value of 1 + uses its full estimated capacity. + exclusiveMinimum: true + maximum: 1 + minimum: 0 + type: number + type: object + x-kubernetes-validations: + - message: exactly one of value or range must be specified + rule: '(has(self.value) ? 1 : 0) + (has(self.range) ? 1 : 0) == 1' + requestsPerSecond: + description: RequestsPerSecond is a fixed open-loop arrival rate. + exclusiveMinimum: true + minimum: 0 + type: number + type: object x-kubernetes-validations: - - message: a range needs min < max - rule: size(self) != 2 || self[0] < self[1] - numRequestRatio: - description: NumRequestRatio sets request count relative to the selected load. - exclusiveMinimum: true - minimum: 0 - type: number - osl: - description: OSL is the output sequence length in tokens. + - message: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified + rule: '(has(self.requestsPerSecond) ? 1 : 0) + (has(self.concurrency) ? 1 : 0) + (has(self.kvCapacityFraction) ? 1 : 0) == 1' + outputTokens: + description: OutputTokens is the synthetic request output length in tokens. format: int32 minimum: 1 type: integer - requestRate: - description: RequestRate is a fixed open-loop arrival rate in requests per second. - exclusiveMinimum: true - minimum: 0 - type: number type: object - x-kubernetes-validations: - - message: at most one of requestRate, concurrency, or kvLoadRatio may be specified - rule: '(has(self.requestRate) ? 1 : 0) + (has(self.concurrency) ? 1 : 0) + (has(self.kvLoadRatio) ? 1 : 0) <= 1' type: object type: object x-kubernetes-validations: diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 8218fea7e98f..c0cbeafc4fe4 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -61,26 +61,17 @@ spec: Run creators resolve optional Request inputs before persisting the Run. properties: aisimulate: - description: AISimulate selects the execution artifact and supplies native AISimulate input. + description: AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. properties: - image: + inputOverrides: description: |- - Image selects a Dynamo-compatible AISimulate runner. When omitted, admission - persists the operator's configured AISimulate image, or the Dynamo planner image of - the operator's release, together with the AISimulate version it bundles. - minLength: 1 - type: string - parameters: - description: |- - Parameters contains native AISimulate input. Field names and nesting use - the upstream AISimulate JSON contract, including snake_case names. + InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from + the typed fields. It uses the upstream AISimulate JSON contract, including + snake_case names. Values here take precedence, and null deletes a lowered + field. The operator runs the AISimulate version from its configured planner + image; Requests and Runs cannot select a different image or version. type: object x-kubernetes-preserve-unknown-fields: true - versionOverride: - description: nightly or custom image), and it wins over the mapping. - minLength: 1 - pattern: ^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$ - type: string type: object backends: description: Backends lists one or more inference backends searched by AISimulate and used by generated candidates. @@ -149,7 +140,10 @@ spec: - GoodputPerGPU type: string metrics: - description: Metrics lists two or more distinct scalar targets when mode is Pareto. + description: |- + Metrics selects AISimulate's public Pareto objective pair when mode is Pareto. + The configured AISimulate release supports exactly ThroughputPerGPU and + ThroughputPerUser through its public recommendation input. items: description: ObjectiveMetric is a metric optimized by AISimulate. enum: @@ -161,7 +155,7 @@ spec: - Goodput - GoodputPerGPU type: string - maxItems: 7 + maxItems: 2 minItems: 2 type: array x-kubernetes-list-type: set @@ -174,7 +168,7 @@ spec: sla: description: |- SLA defines per-request latency bounds used to calculate goodput. When omitted, no - bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. + bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. @@ -199,6 +193,8 @@ spec: x-kubernetes-validations: - message: Optimize accepts only metric; Pareto requires metrics rule: '!has(self.mode) || (self.mode == ''Optimize'' && !has(self.metrics)) || (self.mode == ''Pareto'' && !has(self.metric) && has(self.metrics))' + - message: Pareto supports exactly ThroughputPerGPU and ThroughputPerUser + rule: '!has(self.metrics) || (size(self.metrics) == 2 && self.metrics.exists(m, m == ''ThroughputPerGPU'') && self.metrics.exists(m, m == ''ThroughputPerUser''))' - message: Goodput and GoodputPerGPU require at least one SLA bound rule: '!(has(self.metric) && self.metric in [''Goodput'', ''GoodputPerGPU'']) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))' - message: Pareto metrics containing Goodput or GoodputPerGPU require at least one SLA bound @@ -219,63 +215,87 @@ spec: workload: description: |- Workload defines the synthetic traffic model used for every candidate evaluation. - When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; - an explicit empty workload receives the same shape from admission. An explicit - workload.static: {} opts out of that shape, leaving the workload to - spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the - fields it names. - The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a - KV-load ratio search. + When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; + an explicit empty workload receives the same shape. workload.static: {} opts out of all + traffic defaults. Any other static object is completed with its missing defaults. + spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over + every lowered traffic field it names. + The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials + nor a KV-capacity-fraction search. properties: static: description: |- - Static defines one of AISimulate's synthetic workload shapes. Its fields are not - defaulted individually: the default belongs to spec.workload as a whole, so an - explicit workload object keeps exactly the fields it names. + Static defines AISimulate synthetic traffic. When omitted, admission supplies the + complete static default. An explicit empty object opts out; any nonempty object is + completed with omitted inputTokens, outputTokens and load defaults. properties: - concurrency: - description: Concurrency is a fixed closed-loop in-flight request cap. + inputTokens: + description: InputTokens is the synthetic request input length in tokens. format: int32 minimum: 1 type: integer - isl: - description: ISL is the input sequence length in tokens. - format: int32 - minimum: 1 - type: integer - kvLoadRatio: + load: description: |- - KVLoadRatio derives concurrency from candidate KV capacity. A two-value - range requires a bounded AISimulate sweep. - items: - maximum: 1 - minimum: 0 - type: number - maxItems: 2 - minItems: 1 - type: array + Load selects the synthetic load model. When omitted from a nonempty static + workload, admission defaults it to concurrency 10. + properties: + concurrency: + description: Concurrency is a fixed closed-loop in-flight request cap. + format: int32 + minimum: 1 + type: integer + kvCapacityFraction: + description: KVCapacityFraction derives concurrency from each candidate's estimated KV capacity. + properties: + range: + description: Range searches fractions between min and max. + properties: + max: + description: Max is the upper end of the searched fraction range. + exclusiveMinimum: true + maximum: 1 + minimum: 0 + type: number + min: + description: Min is the lower end of the searched fraction range. + exclusiveMinimum: true + maximum: 1 + minimum: 0 + type: number + required: + - max + - min + type: object + x-kubernetes-validations: + - message: min must be less than max + rule: self.min < self.max + value: + description: |- + Value uses a fixed fraction of the candidate's KV capacity. A value of 1 + uses its full estimated capacity. + exclusiveMinimum: true + maximum: 1 + minimum: 0 + type: number + type: object + x-kubernetes-validations: + - message: exactly one of value or range must be specified + rule: '(has(self.value) ? 1 : 0) + (has(self.range) ? 1 : 0) == 1' + requestsPerSecond: + description: RequestsPerSecond is a fixed open-loop arrival rate. + exclusiveMinimum: true + minimum: 0 + type: number + type: object x-kubernetes-validations: - - message: a range needs min < max - rule: size(self) != 2 || self[0] < self[1] - numRequestRatio: - description: NumRequestRatio sets request count relative to the selected load. - exclusiveMinimum: true - minimum: 0 - type: number - osl: - description: OSL is the output sequence length in tokens. + - message: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified + rule: '(has(self.requestsPerSecond) ? 1 : 0) + (has(self.concurrency) ? 1 : 0) + (has(self.kvCapacityFraction) ? 1 : 0) == 1' + outputTokens: + description: OutputTokens is the synthetic request output length in tokens. format: int32 minimum: 1 type: integer - requestRate: - description: RequestRate is a fixed open-loop arrival rate in requests per second. - exclusiveMinimum: true - minimum: 0 - type: number type: object - x-kubernetes-validations: - - message: at most one of requestRate, concurrency, or kvLoadRatio may be specified - rule: '(has(self.requestRate) ? 1 : 0) + (has(self.concurrency) ? 1 : 0) + (has(self.kvLoadRatio) ? 1 : 0) <= 1' type: object type: object x-kubernetes-validations: @@ -284,6 +304,17 @@ spec: status: description: DynamoGraphDeploymentRunStatus represents one search execution's observed state. properties: + aisimulate: + description: AISimulate reports the effective input executed by AISimulate. + properties: + input: + description: |- + Input is the fully lowered AISimulate input after typed fields, derived defaults and + spec.aisimulate.inputOverrides have been merged. Field names and nesting use the + upstream AISimulate JSON contract. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object candidateRefs: description: |- CandidateRefs identifies the current bounded DGDC projection. List order diff --git a/deploy/operator/internal/aisimulate/defaults.go b/deploy/operator/internal/aisimulate/defaults.go new file mode 100644 index 000000000000..e6721282e186 --- /dev/null +++ b/deploy/operator/internal/aisimulate/defaults.go @@ -0,0 +1,82 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + +// The derived defaults. No API field carries them: each depends on the typed underlay, so +// no single typed field can supply it. +const ( + // DefaultRequestCount is traffic.stop.requests for every native static workload. + // Alternative stop policies belong entirely to inputOverrides, which must delete + // this value when replacing it. + DefaultRequestCount int64 = 1000 + + // DefaultMaxTrials bounds a KV-load range search that has no explicit optimizer.max_trials. + // It matches the trial budget of AISimulate's unified CLI. + DefaultMaxTrials int64 = 320 +) + +// publicUnderlay supplies the role objects required by AISimulate's default +// aggregated-and-disaggregated engine search. Empty workers retain the upstream +// parallelism, scheduler, KV-cache and timing defaults. +func publicUnderlay() map[string]any { + return map[string]any{ + "engine": map[string]any{ + "workers": map[string]any{ + "aggregated": map[string]any{}, + "prefill": map[string]any{}, + "decode": map[string]any{}, + }, + }, + } +} + +// applyDefaults adds the derived defaults to the typed underlay, before the public-input +// input overrides are applied. spec.aisimulate.inputOverrides is the final overlay: it overrides or +// deletes any derived value, and nothing is derived in response to what it introduces. +// An override that needs a matching change elsewhere (deleting requests when adding +// requests_per_load_unit, or strict_sla next to a new SLA bound) must make that change itself. +func applyDefaults(underlay map[string]any) error { + b := builder{root: underlay} + + if has(underlay, trafficPath...) { + if !has(underlay, trafficStopRequests...) { + b.set(float64(DefaultRequestCount), trafficStopRequests) + } + fraction, _, _ := unstructured.NestedMap(underlay, trafficLoadFraction...) + if has(fraction, "range") && !has(underlay, optimizerMaxTrials...) { + b.set(float64(DefaultMaxTrials), optimizerMaxTrials) + } + } + + if hasSLABound(underlay) && !has(underlay, optimizationStrictSLA...) { + b.set(true, optimizationStrictSLA) + } + return b.err +} + +// hasSLABound reports whether the input bounds at least one latency. +func hasSLABound(input map[string]any) bool { + for _, path := range [][]string{evaluationSLATTFTMs, evaluationSLAITLMs, evaluationSLAE2EMs} { + if bound, _, _ := unstructured.NestedFieldNoCopy(input, path...); bound != nil { + return true + } + } + return false +} diff --git a/deploy/operator/internal/aisimulate/doc.go b/deploy/operator/internal/aisimulate/doc.go new file mode 100644 index 000000000000..a41ef4bee3d7 --- /dev/null +++ b/deploy/operator/internal/aisimulate/doc.go @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package aisimulate lowers a v1beta2 NativeSearchInput snapshot into the public +// AISimulate recommendation input consumed by `aisimulate recommend --stack dynamo`. +// +// Admission and reconciliation use the same lowering. Admission runs it after structural +// validation so malformed overrides and unmappable typed values fail before persistence. The +// Request reconciler snapshots the accepted input into an immutable Run. The Run reconciler +// lowers that snapshot once, records the result in status.aisimulate.input and passes the same +// document to the Job. +// +// Lower starts with the public engine-role underlay required for AISimulate's default +// aggregated-and-disaggregated search. It then translates the typed spec (model, backends, +// hardware, workload, objective) into that input. The derived defaults that no API field +// carries are added next: the request count, a bounded trial budget for a +// KV-load range search, and optimization.strict_sla when an SLA bound is set. Last, +// spec.aisimulate.inputOverrides is applied on top as a JSON Merge Patch (RFC 7386): upstream +// objects merge recursively, arrays and scalars replace the value, and null deletes it. The +// input overrides are authoritative: they win on every path they name, including the derived +// defaults, and are applied once, last. +// +// The document stays unstructured (map[string]any) because its schema belongs to the +// AISimulate release bundled in the operator's Planner image, not to this package. Reads and writes go +// through the helpers of k8s.io/apimachinery/pkg/apis/meta/v1/unstructured. +package aisimulate diff --git a/deploy/operator/internal/aisimulate/input.go b/deploy/operator/internal/aisimulate/input.go new file mode 100644 index 000000000000..1c75e9ca7c97 --- /dev/null +++ b/deploy/operator/internal/aisimulate/input.go @@ -0,0 +1,280 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "fmt" + "strings" + "unicode" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Lower returns the effective public recommendation input: the typed underlay and derived defaults, merged +// with spec.aisimulate.inputOverrides. The spec must already carry its structural defaults; +// lowering repeats none of them. +func Lower(spec *v1beta2.NativeSearchInput) (map[string]any, error) { + return LowerWithUnderlay(spec, nil) +} + +// LowerWithUnderlay returns the effective public recommendation input with a controller-owned +// static underlay beneath the typed API fields and derived defaults. InputOverrides remains the +// final merge patch, so it may override or delete fields from either underlay. +func LowerWithUnderlay(spec *v1beta2.NativeSearchInput, staticUnderlay map[string]any) (map[string]any, error) { + overrides, err := InputOverrides(spec) + if err != nil { + return nil, err + } + typedUnderlay, err := Underlay(spec) + if err != nil { + return nil, err + } + staticUnderlay, err = normalize(staticUnderlay) + if err != nil { + return nil, err + } + underlay := publicUnderlay() + underlay, _ = mergePatch(underlay, staticUnderlay).(map[string]any) + underlay, _ = mergePatch(underlay, typedUnderlay).(map[string]any) + if underlay, err = normalize(underlay); err != nil { + return nil, err + } + if err := applyDefaults(underlay); err != nil { + return nil, err + } + input, _ := mergePatch(underlay, overrides).(map[string]any) + if input == nil { + input = map[string]any{} + } + return input, nil +} + +// InputOverrides returns spec.aisimulate.inputOverrides as a JSON object, or nil when absent. +func InputOverrides(spec *v1beta2.NativeSearchInput) (map[string]any, error) { + if spec.AISimulate == nil || spec.AISimulate.InputOverrides == nil { + return nil, nil + } + raw := spec.AISimulate.InputOverrides.Raw + if len(raw) == 0 && spec.AISimulate.InputOverrides.Object != nil { + var err error + if raw, err = json.Marshal(spec.AISimulate.InputOverrides.Object); err != nil { + return nil, fmt.Errorf("spec.aisimulate.inputOverrides: %w", err) + } + } + if len(raw) == 0 { + return nil, nil + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil, fmt.Errorf("spec.aisimulate.inputOverrides: %w", err) + } + switch typed := value.(type) { + case nil: + return nil, nil + case map[string]any: + return typed, nil + default: + return nil, fmt.Errorf("spec.aisimulate.inputOverrides: must be a JSON object, got %T", value) + } +} + +// Underlay translates the typed spec into the public CoreRecommendationConfig input consumed by +// `aisimulate recommend`. Typed fields that are unset produce no key, so AISimulate defaults and +// input overrides decide them. +func Underlay(spec *v1beta2.NativeSearchInput) (map[string]any, error) { + b := builder{root: map[string]any{}} + + if spec.Model != nil { + b.set(spec.Model.Name, engineModel) + } + if len(spec.Backends) > 0 { + backends := make([]string, len(spec.Backends)) + for i, backend := range spec.Backends { + backends[i] = string(backend) + } + b.setStrings(backends, engineBackendChoices) + } + if spec.Hardware != nil && spec.Hardware.GPU != nil { + if sku := spec.Hardware.GPU.SKU; sku != nil && *sku != "" { + b.set(*sku, engineHardware) + } + if budget := spec.Hardware.GPU.Budget; budget != nil { + b.set(int64(*budget), optimizationHardwareBudget) + } + } + + if spec.Workload != nil && spec.Workload.Static != nil { + static := spec.Workload.Static + if hasStaticWorkloadFields(static) { + b.set("synthetic", trafficSourceType) + } + if static.InputTokens != nil { + b.set(int64(*static.InputTokens), trafficSourceInputTokens) + } + if static.OutputTokens != nil { + b.set(int64(*static.OutputTokens), trafficSourceOutputTokens) + } + if load := static.Load; load != nil { + switch { + case load.RequestsPerSecond != nil: + b.set("constant_rate", trafficLoadType) + b.set(*load.RequestsPerSecond, trafficLoadRequestsPerSecond) + case load.Concurrency != nil: + b.set("concurrency", trafficLoadType) + b.set(int64(*load.Concurrency), trafficLoadConcurrency) + case load.KVCapacityFraction != nil: + b.set("kv_capacity_fraction", trafficLoadType) + fraction := load.KVCapacityFraction + switch { + case fraction.Value != nil: + b.set(*fraction.Value, trafficLoadFraction) + case fraction.Range != nil: + b.set(map[string]any{"range": map[string]any{ + "min": fraction.Range.Min, + "max": fraction.Range.Max, + }}, trafficLoadFraction) + } + } + } + } + + if objective := spec.Objective; objective != nil { + if objective.Mode == v1beta2.ObjectiveModePareto { + if len(objective.Metrics) > 0 && !isDefaultParetoMetrics(objective.Metrics) { + b.err = fmt.Errorf("spec.objective.metrics: the AISimulate public recommendation input only exposes its default Pareto objectives") + } else { + b.set(TargetPareto, optimizationTarget) + } + } else if objective.Metric != nil { + b.set(publicMetric(*objective.Metric), optimizationTarget) + } + if sla := objective.SLA; sla != nil { + if sla.TTFTMs != nil { + b.set(*sla.TTFTMs, evaluationSLATTFTMs) + } + if sla.ITLMs != nil { + b.set(*sla.ITLMs, evaluationSLAITLMs) + } + if sla.E2EMs != nil { + b.set(*sla.E2EMs, evaluationSLAE2EMs) + } + } + } + return b.root, b.err +} + +func hasStaticWorkloadFields(static *v1beta2.StaticWorkloadSpec) bool { + return static.InputTokens != nil || static.OutputTokens != nil || static.Load != nil +} + +func isDefaultParetoMetrics(metrics []v1beta2.ObjectiveMetric) bool { + if len(metrics) != 2 { + return false + } + found := map[v1beta2.ObjectiveMetric]bool{} + for _, metric := range metrics { + found[metric] = true + } + return found[v1beta2.ObjectiveMetric("ThroughputPerGPU")] && found[v1beta2.ObjectiveMetric("ThroughputPerUser")] +} + +// builder writes public input values, remembering the first failure so the translation reads +// as a straight list of assignments. Every path is written under a map the caller created +// and every value is a JSON scalar or a list of them, so a failure means a bad path. +type builder struct { + root map[string]any + err error +} + +func (b *builder) set(value any, path []string) { + if b.err != nil { + return + } + if err := unstructured.SetNestedField(b.root, value, path...); err != nil { + b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) + } +} + +func (b *builder) setStrings(values []string, path []string) { + if b.err != nil { + return + } + if err := unstructured.SetNestedStringSlice(b.root, values, path...); err != nil { + b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) + } +} + +// publicMetric maps an API metric to AISimulate's optimization.target by converting its CamelCase +// name to snake_case: ThroughputPerGPU becomes throughput_per_gpu and E2ELatency becomes +// e2e_latency. A run of capitals stays together, a digit does not start a word, and the last +// capital of a run starts the next word when a lowercase letter follows it. +func publicMetric(metric v1beta2.ObjectiveMetric) string { + name := []rune(string(metric)) + var out []rune + for i, r := range name { + if i > 0 && unicode.IsUpper(r) { + prev := name[i-1] + nextLower := i+1 < len(name) && unicode.IsLower(name[i+1]) + if unicode.IsLower(prev) || (unicode.IsUpper(prev) && nextLower) { + out = append(out, '_') + } + } + out = append(out, unicode.ToLower(r)) + } + return string(out) +} + +// normalize round-trips through JSON so every number is a float64, the type the +// input overrides decode to. Merge and validation then see one representation. +func normalize(value map[string]any) (map[string]any, error) { + if value == nil { + return nil, nil + } + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} + +// mergePatch applies patch to target with RFC 7386 semantics. It returns a new value for +// the merged parts and does not retain the patch's maps for null members. +func mergePatch(target, patch any) any { + patchMap, ok := patch.(map[string]any) + if !ok { + return patch + } + targetMap, ok := target.(map[string]any) + if !ok { + targetMap = map[string]any{} + } + for key, value := range patchMap { + if value == nil { + delete(targetMap, key) + continue + } + targetMap[key] = mergePatch(targetMap[key], value) + } + return targetMap +} diff --git a/deploy/operator/internal/aisimulate/input_test.go b/deploy/operator/internal/aisimulate/input_test.go new file mode 100644 index 000000000000..714e7cb7e1e1 --- /dev/null +++ b/deploy/operator/internal/aisimulate/input_test.go @@ -0,0 +1,471 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "encoding/json" + "strings" + "testing" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" +) + +func ptrTo[T any](v T) *T { return &v } + +// defaultedSpec is a model-only Request as it looks after structural and admission defaults: +// the fixed-concurrency workload, the scalar metric and discovered hardware. It +// has no SLA bounds and no KV-load range; withSLA and withKVRange add them. +func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { + metric := v1beta2.ObjectiveMetric("Throughput") + return &v1beta2.DynamoGraphDeploymentRequestSpec{ + Model: &v1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, + Backends: []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang, v1beta2.BackendTRTLLM}, + Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("h200_sxm"), Budget: ptrTo(int32(8))}}, + Workload: &v1beta2.WorkloadSpec{Static: &v1beta2.StaticWorkloadSpec{ + InputTokens: ptrTo(int32(1024)), + OutputTokens: ptrTo(int32(128)), + Load: &v1beta2.StaticLoadSpec{Concurrency: ptrTo(int32(10))}, + }}, + Objective: &v1beta2.ObjectiveSpec{Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric}, + Recommendation: &v1beta2.RecommendationSpec{MaxCandidates: 1}, + } +} + +// withSLA adds the latency bounds the earlier defaults used to supply. +func withSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)} + return spec +} + +// withKVRange replaces the fixed concurrency with a KV-load range search. +func withKVRange(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ + InputTokens: ptrTo(int32(4000)), + OutputTokens: ptrTo(int32(1000)), + Load: &v1beta2.StaticLoadSpec{KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{ + Range: &v1beta2.KVCapacityFractionRange{Min: 0.1, Max: 1}, + }}, + } + return spec +} + +func withInputOverrides(spec *v1beta2.DynamoGraphDeploymentRequestSpec, overrides string) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.AISimulate = &v1beta2.AISimulateSpec{InputOverrides: &runtime.RawExtension{Raw: []byte(overrides)}} + return spec +} + +func lower(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { + t.Helper() + input, err := Lower(&spec.NativeSearchInput) + if err != nil { + t.Fatalf("Lower: %v", err) + } + return input +} + +func encode(t *testing.T, input map[string]any) string { + t.Helper() + raw, err := json.Marshal(input) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func underlay(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { + t.Helper() + input, err := Underlay(&spec.NativeSearchInput) + if err != nil { + t.Fatalf("Underlay: %v", err) + } + return input +} + +func nested(t *testing.T, input map[string]any, path ...string) any { + t.Helper() + value, _, err := unstructured.NestedFieldNoCopy(input, path...) + if err != nil { + t.Fatal(err) + } + return value +} + +func TestLowerModelOnlyRequest(t *testing.T) { + want := `{"engine":{"backend":{"choices":["vllm","sglang","trtllm"]},"hardware":"h200_sxm","model":"meta-llama/Llama-3.1-8B-Instruct","workers":{"aggregated":{},"decode":{},"prefill":{}}},` + + `"optimization":{"constraints":{"max_candidate_gpus":8},"target":"throughput"},` + + `"traffic":{"load":{"concurrency":10,"type":"concurrency"},"source":{"input_tokens":1024,"output_tokens":128,"type":"synthetic"},"stop":{"requests":1000}}}` + if got := encode(t, lower(t, defaultedSpec())); got != want { + t.Fatalf("effective input\n got: %s\nwant: %s", got, want) + } +} + +func TestLowerIsDeterministicAndDoesNotMutateTheSpec(t *testing.T) { + const overrides = `{"optimizer":{"max_trials":100},"traffic":{"stop":{"requests":5}}}` + spec := withInputOverrides(defaultedSpec(), overrides) + first, second := encode(t, lower(t, spec)), encode(t, lower(t, spec)) + if first != second { + t.Fatalf("not deterministic:\n%s\n%s", first, second) + } + if string(spec.AISimulate.InputOverrides.Raw) != overrides { + t.Fatalf("overrides mutated: %s", spec.AISimulate.InputOverrides.Raw) + } +} + +func TestLowerExplicitRequest(t *testing.T) { + metric := v1beta2.ObjectiveMetric("Throughput") + spec := withSLA(defaultedSpec()) + spec.Backends = []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang} + spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ + InputTokens: ptrTo(int32(4000)), + OutputTokens: ptrTo(int32(1000)), + Load: &v1beta2.StaticLoadSpec{RequestsPerSecond: ptrTo(10.0)}, + } + spec.Objective.Metric = &metric + spec = withInputOverrides(spec, `{"traffic":{"stop":{"requests":1000}},"optimizer":{"max_trials":100}}`) + + want := `{"engine":{"backend":{"choices":["vllm","sglang"]},"hardware":"h200_sxm","model":"meta-llama/Llama-3.1-8B-Instruct","workers":{"aggregated":{},"decode":{},"prefill":{}}},` + + `"evaluation":{"sla":{"itl_ms":30,"ttft_ms":2000}},` + + `"optimization":{"constraints":{"max_candidate_gpus":8},"strict_sla":true,"target":"throughput"},` + + `"optimizer":{"max_trials":100},` + + `"traffic":{"load":{"requests_per_second":10,"type":"constant_rate"},"source":{"input_tokens":4000,"output_tokens":1000,"type":"synthetic"},"stop":{"requests":1000}}}` + if got := encode(t, lower(t, spec)); got != want { + t.Fatalf("effective input\n got: %s\nwant: %s", got, want) + } +} + +func TestMergePatchSemantics(t *testing.T) { + cases := []struct{ name, target, patch, want string }{ + {"scalar replaces", `{"a":1}`, `{"a":2}`, `{"a":2}`}, + {"object merges recursively", `{"a":{"b":1,"c":2}}`, `{"a":{"b":3}}`, `{"a":{"b":3,"c":2}}`}, + {"null deletes", `{"a":1,"b":2}`, `{"a":null}`, `{"b":2}`}, + {"array replaces", `{"a":[1,2,3]}`, `{"a":[4]}`, `{"a":[4]}`}, + {"object replaces scalar and drops nulls", `{"a":1}`, `{"a":{"b":null,"c":1}}`, `{"a":{"c":1}}`}, + {"null on absent key is a no-op", `{"a":1}`, `{"z":null}`, `{"a":1}`}, + {"empty patch", `{"a":1}`, `{}`, `{"a":1}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var target, patch any + if err := json.Unmarshal([]byte(tc.target), &target); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(tc.patch), &patch); err != nil { + t.Fatal(err) + } + got, _ := json.Marshal(mergePatch(target, patch)) + if string(got) != tc.want { + t.Fatalf("got %s want %s", got, tc.want) + } + }) + } + t.Run("nil patch map", func(t *testing.T) { + target := map[string]any{"a": 1.0} + if got := mergePatch(target, map[string]any(nil)); got == nil { + t.Fatal("a nil patch must keep the target") + } + }) +} + +func TestInputOverridesWinOverTheTypedUnderlay(t *testing.T) { + t.Run("a null deletes a typed field", func(t *testing.T) { + input := lower(t, withInputOverrides(defaultedSpec(), `{"traffic":{"source":{"input_tokens":null}}}`)) + if nested(t, input, "traffic", "source", "input_tokens") != nil { + t.Fatalf("inputTokens survived an override null: %s", encode(t, input)) + } + }) + t.Run("a value overrides a typed field", func(t *testing.T) { + input := lower(t, withInputOverrides(defaultedSpec(), `{"traffic":{"source":{"input_tokens":8000}},"optimization":{"constraints":{"max_candidate_gpus":16}}}`)) + if nested(t, input, "traffic", "source", "input_tokens") != 8000.0 || nested(t, input, "optimization", "constraints", "max_candidate_gpus") != 16.0 { + t.Fatalf("input overrides did not win: %s", encode(t, input)) + } + }) + t.Run("a target overrides the typed metric", func(t *testing.T) { + input := lower(t, withInputOverrides(defaultedSpec(), `{"optimization":{"target":"ttft"}}`)) + if nested(t, input, "optimization", "target") != "ttft" { + t.Fatalf("typed metric beat the override target: %s", encode(t, input)) + } + }) + t.Run("an SLA bound overrides only the bound it names", func(t *testing.T) { + input := lower(t, withInputOverrides(withSLA(defaultedSpec()), `{"evaluation":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "evaluation", "sla", "ttft_ms") != 500.0 || nested(t, input, "evaluation", "sla", "itl_ms") != 30.0 { + t.Fatalf("unexpected SLA: %s", encode(t, input)) + } + }) +} + +func TestRequestCountDefault(t *testing.T) { + t.Run("derived from the typed underlay", func(t *testing.T) { + if nested(t, lower(t, defaultedSpec()), "traffic", "stop", "requests") != 1000.0 { + t.Fatal("traffic.stop.requests not derived") + } + }) + t.Run("an override ratio deletes the derived count itself", func(t *testing.T) { + input := lower(t, withInputOverrides(defaultedSpec(), `{"traffic":{"stop":{"requests_per_load_unit":10,"requests":null}}}`)) + if nested(t, input, "traffic", "stop", "requests") != nil || nested(t, input, "traffic", "stop", "requests_per_load_unit") != 10.0 { + t.Fatalf("unexpected workload: %s", encode(t, input)) + } + }) + t.Run("an override null deletes the default", func(t *testing.T) { + input := lower(t, withInputOverrides(defaultedSpec(), `{"traffic":{"stop":{"requests":null}}}`)) + if nested(t, input, "traffic", "stop", "requests") != nil { + t.Fatal("request count re-added after an override null") + } + }) + t.Run("not added without a workload", func(t *testing.T) { + spec := defaultedSpec() + spec.Workload = &v1beta2.WorkloadSpec{} + if nested(t, lower(t, spec), "traffic") != nil { + t.Fatal("a workload appeared from nothing") + } + }) +} + +func TestMaxTrialsDefault(t *testing.T) { + t.Run("added for a KV-load range", func(t *testing.T) { + if nested(t, lower(t, withKVRange(defaultedSpec())), "optimizer", "max_trials") != 320.0 { + t.Fatal("no default max_trials for a range search") + } + }) + t.Run("not derived for the model-only fixed concurrency", func(t *testing.T) { + if input := lower(t, defaultedSpec()); nested(t, input, "optimizer") != nil { + t.Fatalf("unexpected optimizer: %s", encode(t, input)) + } + }) + t.Run("not added for a pinned load", func(t *testing.T) { + spec := withKVRange(defaultedSpec()) + spec.Workload.Static.Load.KVCapacityFraction = &v1beta2.KVCapacityFractionSpec{Value: ptrTo(0.5)} + input := lower(t, spec) + if nested(t, input, "optimizer") != nil { + t.Fatalf("unexpected optimizer: %s", encode(t, input)) + } + if nested(t, input, "traffic", "load", "fraction") != 0.5 { + t.Fatal("a one-value ratio should be a scalar") + } + }) + t.Run("override value or null wins", func(t *testing.T) { + if nested(t, lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":7}}`)), "optimizer", "max_trials") != 7.0 { + t.Fatal("override max_trials ignored") + } + if nested(t, lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":null}}`)), "optimizer", "max_trials") != nil { + t.Fatal("max_trials re-added after an override null") + } + }) +} + +func TestStrictSLADerivation(t *testing.T) { + t.Run("enabled when the typed SLA has a bound", func(t *testing.T) { + if nested(t, lower(t, withSLA(defaultedSpec())), "optimization", "strict_sla") != true { + t.Fatal("strict_sla not derived") + } + }) + t.Run("model-only derives nothing", func(t *testing.T) { + input := lower(t, defaultedSpec()) + if nested(t, input, "optimization", "strict_sla") != nil || nested(t, input, "evaluation", "sla") != nil { + t.Fatalf("model-only input has SLA keys: %s", encode(t, input)) + } + }) + t.Run("an empty SLA derives nothing", func(t *testing.T) { + spec := defaultedSpec() + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{} + input := lower(t, spec) + if nested(t, input, "optimization", "strict_sla") != nil || nested(t, input, "evaluation", "sla") != nil { + t.Fatalf("an empty SLA produced SLA keys: %s", encode(t, input)) + } + }) + t.Run("an override bound alone does not derive it", func(t *testing.T) { + input := lower(t, withInputOverrides(defaultedSpec(), `{"evaluation":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "evaluation", "sla", "ttft_ms") != 500.0 || nested(t, input, "optimization", "strict_sla") != nil { + t.Fatalf("unexpected optimization input: %s", encode(t, input)) + } + }) + t.Run("override value is kept", func(t *testing.T) { + if nested(t, lower(t, withInputOverrides(withSLA(defaultedSpec()), `{"optimization":{"strict_sla":false}}`)), "optimization", "strict_sla") != false { + t.Fatal("override strict_sla was ignored") + } + }) + t.Run("is not revisited after the input override", func(t *testing.T) { + input := lower(t, withInputOverrides(withSLA(defaultedSpec()), `{"evaluation":{"sla":{"ttft_ms":null,"itl_ms":null}}}`)) + if nested(t, input, "optimization", "strict_sla") != true { + t.Fatalf("the derived strict_sla changed after override bounds were deleted: %s", encode(t, input)) + } + }) +} + +func TestInputOverridesApplyOnceAfterDerivedDefaults(t *testing.T) { + t.Run("optimizer null removes the complete derived optimizer", func(t *testing.T) { + input := lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":null}`)) + if nested(t, input, "optimizer") != nil { + t.Fatalf("optimizer survived an override null: %s", encode(t, input)) + } + }) + t.Run("max_trials null removes only max_trials", func(t *testing.T) { + input := lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":null,"seed":3}}`)) + if nested(t, input, "optimizer", "max_trials") != nil || nested(t, input, "optimizer", "seed") != 3.0 { + t.Fatalf("unexpected optimizer: %s", encode(t, input)) + } + }) + t.Run("an override max_trials replaces the derived value", func(t *testing.T) { + input := lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":7}}`)) + if nested(t, input, "optimizer", "max_trials") != 7.0 { + t.Fatalf("unexpected optimizer: %s", encode(t, input)) + } + }) + t.Run("nothing is derived from values only overrides introduce", func(t *testing.T) { + input := lower(t, withInputOverrides(defaultedSpec(), `{"traffic":{"load":{"fraction":{"range":{"min":0.1,"max":1}}}},"evaluation":{"sla":{"ttft_ms":500}}}`)) + if nested(t, input, "optimizer") != nil || nested(t, input, "optimization", "strict_sla") != nil { + t.Fatalf("a field was derived from override-only values: %s", encode(t, input)) + } + }) +} + +func TestUnderlayParetoObjective(t *testing.T) { + spec := defaultedSpec() + spec.Objective.Mode = v1beta2.ObjectiveModePareto + spec.Objective.Metric = nil + spec.Objective.Metrics = []v1beta2.ObjectiveMetric{"ThroughputPerGPU", "ThroughputPerUser"} + input := underlay(t, spec) + if nested(t, input, "optimization", "target") != TargetPareto { + t.Fatalf("not a Pareto optimization: %v", input["optimization"]) + } + + spec.Objective.Metrics = []v1beta2.ObjectiveMetric{"Throughput", "TTFT"} + if _, err := Underlay(&spec.NativeSearchInput); err == nil { + t.Fatal("custom Pareto objectives must fail until the public AISimulate input exposes them") + } +} + +func TestUnderlayOmitsUnsetTypedFields(t *testing.T) { + spec := &v1beta2.DynamoGraphDeploymentRequestSpec{Model: &v1beta2.ModelSpec{Name: "m"}} + if got := encode(t, underlay(t, spec)); got != `{"engine":{"model":"m"}}` { + t.Fatalf("got %s", got) + } + spec.Hardware = &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("")}} + if got := encode(t, underlay(t, spec)); got != `{"engine":{"model":"m"}}` { + t.Fatalf("an empty SKU must not lower: %s", got) + } +} + +func TestUnderlayKVCapacityFractionShapes(t *testing.T) { + spec := defaultedSpec() + spec.Workload.Static.Load = &v1beta2.StaticLoadSpec{ + KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{Value: ptrTo(0.25)}, + } + if nested(t, underlay(t, spec), "traffic", "load", "fraction") != 0.25 { + t.Fatal("a fixed fraction must lower to a scalar") + } + spec.Workload.Static.Load.KVCapacityFraction = &v1beta2.KVCapacityFractionSpec{ + Range: &v1beta2.KVCapacityFractionRange{Min: 0.1, Max: 0.9}, + } + if got := nested(t, underlay(t, spec), "traffic", "load", "fraction", "range", "min"); got != 0.1 { + t.Fatalf("range min = %v, want 0.1", got) + } + if got := nested(t, underlay(t, spec), "traffic", "load", "fraction", "range", "max"); got != 0.9 { + t.Fatalf("range max = %v, want 0.9", got) + } +} + +func TestLowerWithUnderlayKeepsInputOverridesAuthoritative(t *testing.T) { + spec := withInputOverrides(defaultedSpec(), `{"dgdr_run":{"name":"custom","runtime_image":null}}`) + input, err := LowerWithUnderlay(&spec.NativeSearchInput, map[string]any{"dgdr_run": map[string]any{ + "name": "default", "runtime_image": "planner:1.2.3", "max_candidates": int64(1), + }}) + if err != nil { + t.Fatalf("LowerWithUnderlay: %v", err) + } + if nested(t, input, "dgdr_run", "name") != "custom" { + t.Fatalf("input override did not replace the execution name: %s", encode(t, input)) + } + if nested(t, input, "dgdr_run", "runtime_image") != nil { + t.Fatalf("input override did not delete the runtime image: %s", encode(t, input)) + } + if nested(t, input, "dgdr_run", "max_candidates") != float64(1) { + t.Fatalf("static execution underlay was lost: %s", encode(t, input)) + } +} + +func TestLowerIncludesPublicEngineRoleUnderlay(t *testing.T) { + workers, found, err := unstructured.NestedMap(lower(t, defaultedSpec()), engineWorkers...) + if err != nil || !found { + t.Fatalf("engine.workers: found=%v err=%v", found, err) + } + for _, role := range []string{"aggregated", "prefill", "decode"} { + worker, ok := workers[role].(map[string]any) + if !ok || len(worker) != 0 { + t.Errorf("engine.workers.%s = %#v, want an empty object", role, workers[role]) + } + } +} + +func TestInputOverridesMustBeAnObject(t *testing.T) { + for _, overrides := range []string{`[1]`, `"x"`, `3`, `{`} { + spec := withInputOverrides(defaultedSpec(), overrides) + if _, err := Lower(&spec.NativeSearchInput); err == nil { + t.Errorf("Lower accepted overrides %s", overrides) + } + } + for _, overrides := range []string{``, `null`, `{}`} { + spec := withInputOverrides(defaultedSpec(), overrides) + if _, err := Lower(&spec.NativeSearchInput); err != nil { + t.Errorf("Lower rejected overrides %q: %v", overrides, err) + } + } +} + +func TestHasPath(t *testing.T) { + overrides := map[string]any{ + "optimization": map[string]any{"target": nil}, + "evaluation": map[string]any{"sla": map[string]any{"ttft_ms": 1.0}}, + "traffic": map[string]any{"source": map[string]any{"input_tokens": 8.0}}, + } + for path, want := range map[string]bool{ + "optimization.target": true, // a null still names the key + "evaluation.sla": true, + "evaluation.sla.ttft_ms": true, + "evaluation.sla.itl_ms": false, + "traffic.source.input_tokens": true, + "traffic.source.output_tokens": false, + "optimizer.max_trials": false, + "optimization.target.deeper": false, + "traffic.source.input_tokens.foo": false, + } { + if got := has(overrides, strings.Split(path, ".")...); got != want { + t.Errorf("has(%s) = %v, want %v", path, got, want) + } + } + if has(nil, "optimization") { + t.Error("nothing is named in nil overrides") + } +} + +func TestPublicMetricNames(t *testing.T) { + for metric, want := range map[v1beta2.ObjectiveMetric]string{ + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", + } { + if got := publicMetric(metric); got != want { + t.Errorf("publicMetric(%s) = %q, want %q", metric, got, want) + } + } +} diff --git a/deploy/operator/internal/aisimulate/paths.go b/deploy/operator/internal/aisimulate/paths.go new file mode 100644 index 000000000000..25ef533ecd6c --- /dev/null +++ b/deploy/operator/internal/aisimulate/paths.go @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + +// Public recommendation input paths, in the snake_case spelling of the upstream AISimulate +// contract. They address the same key when the underlay is built and when derived defaults are read, +// and are passed to the unstructured helpers, which must not modify them. +var ( + engineModel = []string{"engine", "model"} + engineBackendChoices = []string{"engine", "backend", "choices"} + engineHardware = []string{"engine", "hardware"} + engineWorkers = []string{"engine", "workers"} + + optimizationTarget = []string{"optimization", "target"} + optimizationHardwareBudget = []string{"optimization", "constraints", "max_candidate_gpus"} + optimizationStrictSLA = []string{"optimization", "strict_sla"} + evaluationSLATTFTMs = []string{"evaluation", "sla", "ttft_ms"} + evaluationSLAITLMs = []string{"evaluation", "sla", "itl_ms"} + evaluationSLAE2EMs = []string{"evaluation", "sla", "e2e_ms"} + optimizerMaxTrials = []string{"optimizer", "max_trials"} + + trafficPath = []string{"traffic"} + trafficSourceType = []string{"traffic", "source", "type"} + trafficSourceInputTokens = []string{"traffic", "source", "input_tokens"} + trafficSourceOutputTokens = []string{"traffic", "source", "output_tokens"} + trafficLoadType = []string{"traffic", "load", "type"} + trafficLoadConcurrency = []string{"traffic", "load", "concurrency"} + trafficLoadRequestsPerSecond = []string{"traffic", "load", "requests_per_second"} + trafficLoadFraction = []string{"traffic", "load", "fraction"} + trafficStopRequests = []string{"traffic", "stop", "requests"} +) + +// TargetPareto is optimization.target for a multi-objective search. +const TargetPareto = "pareto" + +// has reports whether obj contains the path, even with a null value. Applied to +// spec.aisimulate.inputOverrides it tells whether the caller named a key, including to delete +// it. +func has(obj map[string]any, path ...string) bool { + _, found, _ := unstructured.NestedFieldNoCopy(obj, path...) + return found +} diff --git a/deploy/operator/internal/controller/aisimulate/defaults.go b/deploy/operator/internal/controller/aisimulate/defaults.go deleted file mode 100644 index 8aecb2d8d569..000000000000 --- a/deploy/operator/internal/controller/aisimulate/defaults.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - -// The derived defaults. No API field carries them: each depends on the typed underlay, so -// no single typed field can supply it. -const ( - // DefaultRequestCount is workload.request_count when the typed underlay does not - // choose num_request_ratio. - DefaultRequestCount int64 = 1000 - - // DefaultMaxTrials bounds a KV-load range search that has no explicit sweep.max_trials. - // It matches the trial budget of AISimulate's unified CLI (optimizer.max_trials). - DefaultMaxTrials int64 = 320 -) - -// applyDefaults adds the derived defaults to the typed underlay, before the native -// parameters are applied. spec.aisimulate.parameters is the final overlay: it overrides or -// deletes any derived value, and nothing is derived in response to what it introduces. -// A native change that needs a matching change elsewhere (a request_count next to -// num_request_ratio, a strict_sla next to a new SLA bound) must make it itself. -func applyDefaults(underlay map[string]any) error { - b := builder{root: underlay} - - if has(underlay, workloadPath...) { - if !has(underlay, workloadRequestCount...) && !has(underlay, workloadNumRequestRatio...) { - b.set(float64(DefaultRequestCount), workloadRequestCount) - } - kvLoadRatio, _, _ := unstructured.NestedFieldNoCopy(underlay, workloadKVLoadRatio...) - if _, isRange := kvLoadRatio.([]any); isRange && !has(underlay, sweepMaxTrials...) { - b.set(float64(DefaultMaxTrials), sweepMaxTrials) - } - } - - if hasSLABound(underlay) && !has(underlay, goalStrictSLA...) { - b.set(true, goalStrictSLA) - } - return b.err -} - -// hasSLABound reports whether the input bounds at least one latency. -func hasSLABound(input map[string]any) bool { - for _, path := range [][]string{goalSLATTFTMs, goalSLAITLMs, goalSLAE2EMs} { - if bound, _, _ := unstructured.NestedFieldNoCopy(input, path...); bound != nil { - return true - } - } - return false -} diff --git a/deploy/operator/internal/controller/aisimulate/doc.go b/deploy/operator/internal/controller/aisimulate/doc.go deleted file mode 100644 index a15cb22952e7..000000000000 --- a/deploy/operator/internal/controller/aisimulate/doc.go +++ /dev/null @@ -1,38 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package aisimulate lowers a v1beta2 DynamoGraphDeploymentRequest spec into the native -// AISimulate input that a Run executes. -// -// Lowering sits below the Request reconciler. Admission does not call it: spec.aisimulate.parameters -// is opaque there, and the validation of the lowered input belongs to the schema of the AISimulate -// release the Request selects. The reconciler lowers the Request once and freezes the output into the -// Run, which executes that object as it is. -// -// Lower works in three steps. The typed spec (model, backends, hardware, workload, objective) -// is first translated into the native input, the "underlay". The derived defaults that no API -// field carries are then added to it: the request count, a bounded trial budget for a -// KV-load range search, and goal.strict_sla when an SLA bound is set. Last, -// spec.aisimulate.parameters is applied on top as a JSON Merge Patch (RFC 7386): native -// objects merge recursively, arrays and scalars replace the value, and null deletes it. The -// parameters are authoritative: they win on every path they name, including the derived -// defaults, and are applied once, last. -// -// The document stays unstructured (map[string]any) because its schema belongs to the -// AISimulate release the Request selects, not to this package. Reads and writes go -// through the helpers of k8s.io/apimachinery/pkg/apis/meta/v1/unstructured. -package aisimulate diff --git a/deploy/operator/internal/controller/aisimulate/image.go b/deploy/operator/internal/controller/aisimulate/image.go deleted file mode 100644 index 60025699e1af..000000000000 --- a/deploy/operator/internal/controller/aisimulate/image.go +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import ( - "errors" - "fmt" - - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" -) - -// aisimulateVersionByDynamoMinor is the release contract between a Dynamo release and the AISimulate -// package its planner image bundles, keyed by the Dynamo major.minor. The image tag is the Dynamo -// version, not the version of the AISimulate package inside the image, and the AISimulate version -// selects the version-specific input schema, so the two are never inferred from one tag. Every -// Dynamo minor bump adds its entry; release_contract_test.go fails until it does. -var aisimulateVersionByDynamoMinor = map[string]string{ - "1.6": "0.13.0", -} - -// ResolveVersion returns the AISimulate version a Request selects: the explicit override when set, -// otherwise the AISimulate version bundled in the Dynamo release named by the image tag. The -// override is the escape hatch for images the contract does not cover (a nightly or custom planner -// image) and names the AISimulate version itself. An image whose tag is not a Dynamo release with a -// mapping entry, or a missing image, is an error; the caller reports it on the field that fixes it. -func ResolveVersion(image, versionOverride string) (string, error) { - if versionOverride != "" { - version, err := versionoverride.Parse(versionOverride) - if err != nil { - return "", fmt.Errorf("AISimulate version override %q: %w", versionOverride, err) - } - return version.String(), nil - } - if image == "" { - return "", errors.New("no AISimulate image") - } - release, err := versionoverride.ParseImageVersion(image) - if err != nil { - return "", fmt.Errorf("resolve AISimulate version: %w", err) - } - minor := fmt.Sprintf("%d.%d", release.Major, release.Minor) - version, ok := aisimulateVersionByDynamoMinor[minor] - if !ok { - return "", fmt.Errorf("image %q is Dynamo %s, which has no bundled AISimulate version in the release contract", image, minor) - } - return version, nil -} diff --git a/deploy/operator/internal/controller/aisimulate/image_test.go b/deploy/operator/internal/controller/aisimulate/image_test.go deleted file mode 100644 index 2a05392112a7..000000000000 --- a/deploy/operator/internal/controller/aisimulate/image_test.go +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import "testing" - -func TestResolveVersion(t *testing.T) { - const mapped = "registry.example.com/aisimulate:1.6.0" - want := aisimulateVersionByDynamoMinor["1.6"] - cases := []struct { - name, image, override, want string - wantErr bool - }{ - {name: "bundled version of the Dynamo release in the tag", image: mapped, want: want}, - {name: "v-prefixed tag", image: "registry.example.com/aisimulate:v1.6.0", want: want}, - {name: "patch release of the same minor", image: "registry.example.com/aisimulate:1.6.3", want: want}, - {name: "prerelease of the same minor", image: "registry.example.com/aisimulate:1.6.0-rc.1", want: want}, - {name: "release without a mapping entry", image: "registry.example.com/aisimulate:1.2.3", wantErr: true}, - {name: "override wins over the mapping", image: mapped, override: "0.99.0", want: "0.99.0"}, - {name: "override rescues a release without an entry", image: "registry.example.com/aisimulate:1.2.3", override: "0.9.0", want: "0.9.0"}, - {name: "override rescues a custom tag", image: "registry.example.com/aisimulate:custom", override: "0.13.1", want: "0.13.1"}, - {name: "override that is not a version", image: mapped, override: "bad", wantErr: true}, - {name: "custom tag needs an override", image: "registry.example.com/aisimulate:custom", wantErr: true}, - {name: "untagged image needs an override", image: "registry.example.com/aisimulate", wantErr: true}, - {name: "registry port is not a tag", image: "registry.example.com:5000/aisimulate", wantErr: true}, - {name: "no image", wantErr: true}, - {name: "override without an image", override: "0.13.0", want: "0.13.0"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - got, err := ResolveVersion(tc.image, tc.override) - if (err != nil) != tc.wantErr || got != tc.want { - t.Fatalf("ResolveVersion(%q, %q) = %q, %v; want %q (error %v)", tc.image, tc.override, got, err, tc.want, tc.wantErr) - } - }) - } -} diff --git a/deploy/operator/internal/controller/aisimulate/input.go b/deploy/operator/internal/controller/aisimulate/input.go deleted file mode 100644 index fbfee0514080..000000000000 --- a/deploy/operator/internal/controller/aisimulate/input.go +++ /dev/null @@ -1,250 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import ( - "encoding/json" - "fmt" - "strings" - "unicode" - - v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" -) - -// Lower returns the effective native input: the typed underlay, merged with -// spec.aisimulate.parameters, plus the derived defaults. The spec must already carry its -// structural defaults; lowering repeats none of them. The only error is malformed -// parameters (not a JSON object); semantic problems are the caller's to validate. -func Lower(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { - parameters, err := Parameters(spec) - if err != nil { - return nil, err - } - underlay, err := Underlay(spec) - if err != nil { - return nil, err - } - if underlay, err = normalize(underlay); err != nil { - return nil, err - } - if err := applyDefaults(underlay); err != nil { - return nil, err - } - input, _ := mergePatch(underlay, parameters).(map[string]any) - if input == nil { - input = map[string]any{} - } - return input, nil -} - -// Parameters returns spec.aisimulate.parameters as a JSON object, or nil when absent. -func Parameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { - if spec.AISimulate == nil || spec.AISimulate.Parameters == nil { - return nil, nil - } - raw := spec.AISimulate.Parameters.Raw - if len(raw) == 0 && spec.AISimulate.Parameters.Object != nil { - var err error - if raw, err = json.Marshal(spec.AISimulate.Parameters.Object); err != nil { - return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) - } - } - if len(raw) == 0 { - return nil, nil - } - var value any - if err := json.Unmarshal(raw, &value); err != nil { - return nil, fmt.Errorf("spec.aisimulate.parameters: %w", err) - } - switch typed := value.(type) { - case nil: - return nil, nil - case map[string]any: - return typed, nil - default: - return nil, fmt.Errorf("spec.aisimulate.parameters: must be a JSON object, got %T", value) - } -} - -// Underlay translates the typed spec into native input. Typed fields that are unset -// produce no key, so the native defaults and the parameters decide them. -func Underlay(spec *v1beta2.DynamoGraphDeploymentRequestSpec) (map[string]any, error) { - b := builder{root: map[string]any{}} - - if spec.Model != nil { - b.set(spec.Model.Name, searchSpaceModelName) - } - if len(spec.Backends) > 0 { - backends := make([]string, len(spec.Backends)) - for i, backend := range spec.Backends { - backends[i] = string(backend) - } - b.setStrings(backends, searchSpaceBackend) - } - if spec.Hardware != nil && spec.Hardware.GPU != nil { - if sku := spec.Hardware.GPU.SKU; sku != nil && *sku != "" { - b.set(*sku, searchSpaceHardwareSKU) - } - if budget := spec.Hardware.GPU.Budget; budget != nil { - b.set(int64(*budget), searchSpaceGPUBudget) - } - } - - if spec.Workload != nil && spec.Workload.Static != nil { - static := spec.Workload.Static - if static.ISL != nil { - b.set(int64(*static.ISL), workloadISL) - } - if static.OSL != nil { - b.set(int64(*static.OSL), workloadOSL) - } - if static.NumRequestRatio != nil { - b.set(*static.NumRequestRatio, workloadNumRequestRatio) - } - if static.RequestRate != nil { - b.set(*static.RequestRate, workloadRequestRate) - } - if static.Concurrency != nil { - b.set(int64(*static.Concurrency), workloadConcurrency) - } - switch len(static.KVLoadRatio) { - case 0: - case 1: - b.set(static.KVLoadRatio[0], workloadKVLoadRatio) - default: - ratios := make([]any, len(static.KVLoadRatio)) - for i, ratio := range static.KVLoadRatio { - ratios[i] = ratio - } - b.set(ratios, workloadKVLoadRatio) - } - } - - if objective := spec.Objective; objective != nil { - if objective.Mode == v1beta2.ObjectiveModePareto { - b.set(TargetPareto, goalTarget) - if len(objective.Metrics) > 0 { - b.setStrings(nativeMetrics(objective.Metrics), goalParetoObjectives) - } - } else if objective.Metric != nil { - b.set(nativeMetric(*objective.Metric), goalTarget) - } - if sla := objective.SLA; sla != nil { - if sla.TTFTMs != nil { - b.set(*sla.TTFTMs, goalSLATTFTMs) - } - if sla.ITLMs != nil { - b.set(*sla.ITLMs, goalSLAITLMs) - } - if sla.E2EMs != nil { - b.set(*sla.E2EMs, goalSLAE2EMs) - } - } - } - return b.root, b.err -} - -// builder writes native input values, remembering the first failure so the translation reads -// as a straight list of assignments. Every path is written under a map the caller created -// and every value is a JSON scalar or a list of them, so a failure means a bad path. -type builder struct { - root map[string]any - err error -} - -func (b *builder) set(value any, path []string) { - if b.err != nil { - return - } - if err := unstructured.SetNestedField(b.root, value, path...); err != nil { - b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) - } -} - -func (b *builder) setStrings(values []string, path []string) { - if b.err != nil { - return - } - if err := unstructured.SetNestedStringSlice(b.root, values, path...); err != nil { - b.err = fmt.Errorf("set %s: %w", strings.Join(path, "."), err) - } -} - -// nativeMetric maps an API metric to AISimulate's goal.target by converting its CamelCase -// name to snake_case: ThroughputPerGPU becomes throughput_per_gpu and E2ELatency becomes -// e2e_latency. A run of capitals stays together, a digit does not start a word, and the last -// capital of a run starts the next word when a lowercase letter follows it. -func nativeMetric(metric v1beta2.ObjectiveMetric) string { - name := []rune(string(metric)) - var out []rune - for i, r := range name { - if i > 0 && unicode.IsUpper(r) { - prev := name[i-1] - nextLower := i+1 < len(name) && unicode.IsLower(name[i+1]) - if unicode.IsLower(prev) || (unicode.IsUpper(prev) && nextLower) { - out = append(out, '_') - } - } - out = append(out, unicode.ToLower(r)) - } - return string(out) -} - -func nativeMetrics(metrics []v1beta2.ObjectiveMetric) []string { - out := make([]string, len(metrics)) - for i, metric := range metrics { - out[i] = nativeMetric(metric) - } - return out -} - -// normalize round-trips through JSON so every number is a float64, the type the -// parameters decode to. Merge and validation then see one representation. -func normalize(value map[string]any) (map[string]any, error) { - raw, err := json.Marshal(value) - if err != nil { - return nil, err - } - var out map[string]any - if err := json.Unmarshal(raw, &out); err != nil { - return nil, err - } - return out, nil -} - -// mergePatch applies patch to target with RFC 7386 semantics. It returns a new value for -// the merged parts and does not retain the patch's maps for null members. -func mergePatch(target, patch any) any { - patchMap, ok := patch.(map[string]any) - if !ok { - return patch - } - targetMap, ok := target.(map[string]any) - if !ok { - targetMap = map[string]any{} - } - for key, value := range patchMap { - if value == nil { - delete(targetMap, key) - continue - } - targetMap[key] = mergePatch(targetMap[key], value) - } - return targetMap -} diff --git a/deploy/operator/internal/controller/aisimulate/input_test.go b/deploy/operator/internal/controller/aisimulate/input_test.go deleted file mode 100644 index dfeff5a150a0..000000000000 --- a/deploy/operator/internal/controller/aisimulate/input_test.go +++ /dev/null @@ -1,433 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import ( - "encoding/json" - "strings" - "testing" - - v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" -) - -func ptrTo[T any](v T) *T { return &v } - -// defaultedSpec is a model-only Request as it looks after the CRD's structural defaults and -// admission: the fixed-concurrency workload, the scalar metric and discovered hardware. It -// has no SLA bounds and no KV-load range; withSLA and withKVRange add them. -func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { - metric := v1beta2.ObjectiveMetric("Throughput") - return &v1beta2.DynamoGraphDeploymentRequestSpec{ - Model: &v1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, - Backends: []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang, v1beta2.BackendTRTLLM}, - Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("h200_sxm"), Budget: ptrTo(int32(8))}}, - Workload: &v1beta2.WorkloadSpec{Static: &v1beta2.StaticWorkloadSpec{ - ISL: ptrTo(int32(1024)), OSL: ptrTo(int32(128)), Concurrency: ptrTo(int32(10)), - }}, - Objective: &v1beta2.ObjectiveSpec{Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric}, - Recommendation: &v1beta2.RecommendationSpec{MaxCandidates: 1}, - } -} - -// withSLA adds the latency bounds the earlier defaults used to supply. -func withSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { - spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)} - return spec -} - -// withKVRange replaces the fixed concurrency with a KV-load range search. -func withKVRange(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { - spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ - ISL: ptrTo(int32(4000)), OSL: ptrTo(int32(1000)), KVLoadRatio: v1beta2.KVLoadRatio{0, 1}, - } - return spec -} - -func withParameters(spec *v1beta2.DynamoGraphDeploymentRequestSpec, parameters string) *v1beta2.DynamoGraphDeploymentRequestSpec { - spec.AISimulate = &v1beta2.AISimulateSpec{Parameters: &runtime.RawExtension{Raw: []byte(parameters)}} - return spec -} - -func lower(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { - t.Helper() - input, err := Lower(spec) - if err != nil { - t.Fatalf("Lower: %v", err) - } - return input -} - -func encode(t *testing.T, input map[string]any) string { - t.Helper() - raw, err := json.Marshal(input) - if err != nil { - t.Fatal(err) - } - return string(raw) -} - -func underlay(t *testing.T, spec *v1beta2.DynamoGraphDeploymentRequestSpec) map[string]any { - t.Helper() - input, err := Underlay(spec) - if err != nil { - t.Fatalf("Underlay: %v", err) - } - return input -} - -func nested(t *testing.T, input map[string]any, path ...string) any { - t.Helper() - value, _, err := unstructured.NestedFieldNoCopy(input, path...) - if err != nil { - t.Fatal(err) - } - return value -} - -func TestLowerModelOnlyRequest(t *testing.T) { - want := `{"goal":{"target":"throughput"},` + - `"search_space":{"backend":["vllm","sglang","trtllm"],"gpu_budget":8,"hardware_sku":"h200_sxm","model_name":"meta-llama/Llama-3.1-8B-Instruct"},` + - `"workload":{"concurrency":10,"isl":1024,"osl":128,"request_count":1000}}` - if got := encode(t, lower(t, defaultedSpec())); got != want { - t.Fatalf("effective input\n got: %s\nwant: %s", got, want) - } -} - -func TestLowerIsDeterministicAndDoesNotMutateTheSpec(t *testing.T) { - const parameters = `{"sweep":{"max_trials":100},"workload":{"request_count":5}}` - spec := withParameters(defaultedSpec(), parameters) - first, second := encode(t, lower(t, spec)), encode(t, lower(t, spec)) - if first != second { - t.Fatalf("not deterministic:\n%s\n%s", first, second) - } - if string(spec.AISimulate.Parameters.Raw) != parameters { - t.Fatalf("parameters mutated: %s", spec.AISimulate.Parameters.Raw) - } -} - -func TestLowerExplicitRequest(t *testing.T) { - metric := v1beta2.ObjectiveMetric("Throughput") - spec := withSLA(defaultedSpec()) - spec.Backends = []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang} - spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ISL: ptrTo(int32(4000)), OSL: ptrTo(int32(1000)), RequestRate: ptrTo(10.0)} - spec.Objective.Metric = &metric - spec = withParameters(spec, `{"workload":{"request_count":1000},"sweep":{"max_trials":100}}`) - - want := `{"goal":{"sla":{"itl_ms":30,"ttft_ms":2000},"strict_sla":true,"target":"throughput"},` + - `"search_space":{"backend":["vllm","sglang"],"gpu_budget":8,"hardware_sku":"h200_sxm","model_name":"meta-llama/Llama-3.1-8B-Instruct"},` + - `"sweep":{"max_trials":100},` + - `"workload":{"isl":4000,"osl":1000,"request_count":1000,"request_rate":10}}` - if got := encode(t, lower(t, spec)); got != want { - t.Fatalf("effective input\n got: %s\nwant: %s", got, want) - } -} - -func TestMergePatchSemantics(t *testing.T) { - cases := []struct{ name, target, patch, want string }{ - {"scalar replaces", `{"a":1}`, `{"a":2}`, `{"a":2}`}, - {"object merges recursively", `{"a":{"b":1,"c":2}}`, `{"a":{"b":3}}`, `{"a":{"b":3,"c":2}}`}, - {"null deletes", `{"a":1,"b":2}`, `{"a":null}`, `{"b":2}`}, - {"array replaces", `{"a":[1,2,3]}`, `{"a":[4]}`, `{"a":[4]}`}, - {"object replaces scalar and drops nulls", `{"a":1}`, `{"a":{"b":null,"c":1}}`, `{"a":{"c":1}}`}, - {"null on absent key is a no-op", `{"a":1}`, `{"z":null}`, `{"a":1}`}, - {"empty patch", `{"a":1}`, `{}`, `{"a":1}`}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - var target, patch any - if err := json.Unmarshal([]byte(tc.target), &target); err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(tc.patch), &patch); err != nil { - t.Fatal(err) - } - got, _ := json.Marshal(mergePatch(target, patch)) - if string(got) != tc.want { - t.Fatalf("got %s want %s", got, tc.want) - } - }) - } - t.Run("nil patch map", func(t *testing.T) { - target := map[string]any{"a": 1.0} - if got := mergePatch(target, map[string]any(nil)); got == nil { - t.Fatal("a nil patch must keep the target") - } - }) -} - -func TestNativeValuesWinOverTheTypedUnderlay(t *testing.T) { - t.Run("a native null deletes a typed field", func(t *testing.T) { - input := lower(t, withParameters(defaultedSpec(), `{"workload":{"isl":null}}`)) - if nested(t, input, "workload", "isl") != nil { - t.Fatalf("isl survived a native null: %s", encode(t, input)) - } - }) - t.Run("a native value overrides a typed field", func(t *testing.T) { - input := lower(t, withParameters(defaultedSpec(), `{"workload":{"isl":8000},"search_space":{"gpu_budget":16}}`)) - if nested(t, input, "workload", "isl") != 8000.0 || nested(t, input, "search_space", "gpu_budget") != 16.0 { - t.Fatalf("native values did not win: %s", encode(t, input)) - } - }) - t.Run("a native target overrides the typed metric", func(t *testing.T) { - input := lower(t, withParameters(defaultedSpec(), `{"goal":{"target":"ttft"}}`)) - if nested(t, input, "goal", "target") != "ttft" { - t.Fatalf("typed metric beat the native target: %s", encode(t, input)) - } - }) - t.Run("a native SLA bound overrides only the bound it names", func(t *testing.T) { - input := lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"sla":{"ttft_ms":500}}}`)) - if nested(t, input, "goal", "sla", "ttft_ms") != 500.0 || nested(t, input, "goal", "sla", "itl_ms") != 30.0 { - t.Fatalf("unexpected SLA: %s", encode(t, input)) - } - }) -} - -func TestRequestCountDefault(t *testing.T) { - t.Run("derived from the typed underlay", func(t *testing.T) { - if nested(t, lower(t, defaultedSpec()), "workload", "request_count") != 1000.0 { - t.Fatal("request_count not derived") - } - }) - t.Run("a native ratio deletes the derived count itself", func(t *testing.T) { - input := lower(t, withParameters(defaultedSpec(), `{"workload":{"num_request_ratio":10,"request_count":null}}`)) - if nested(t, input, "workload", "request_count") != nil || nested(t, input, "workload", "num_request_ratio") != 10.0 { - t.Fatalf("unexpected workload: %s", encode(t, input)) - } - }) - t.Run("skipped when typed numRequestRatio is set", func(t *testing.T) { - spec := defaultedSpec() - spec.Workload.Static.NumRequestRatio = ptrTo(5.0) - if nested(t, lower(t, spec), "workload", "request_count") != nil { - t.Fatal("request_count defaulted next to typed numRequestRatio") - } - }) - t.Run("native null deletes the default", func(t *testing.T) { - input := lower(t, withParameters(defaultedSpec(), `{"workload":{"request_count":null}}`)) - if nested(t, input, "workload", "request_count") != nil { - t.Fatal("request_count re-added after a native null") - } - }) - t.Run("native deleting the typed ratio does not derive a count", func(t *testing.T) { - spec := defaultedSpec() - spec.Workload.Static.NumRequestRatio = ptrTo(5.0) - input := lower(t, withParameters(spec, `{"workload":{"num_request_ratio":null}}`)) - if nested(t, input, "workload", "request_count") != nil { - t.Fatalf("a count was derived after the native overlay: %s", encode(t, input)) - } - }) - t.Run("not added without a workload", func(t *testing.T) { - spec := defaultedSpec() - spec.Workload = &v1beta2.WorkloadSpec{} - if nested(t, lower(t, spec), "workload") != nil { - t.Fatal("a workload appeared from nothing") - } - }) -} - -func TestMaxTrialsDefault(t *testing.T) { - t.Run("added for a KV-load range", func(t *testing.T) { - if nested(t, lower(t, withKVRange(defaultedSpec())), "sweep", "max_trials") != 320.0 { - t.Fatal("no default max_trials for a range search") - } - }) - t.Run("not derived for the model-only fixed concurrency", func(t *testing.T) { - if input := lower(t, defaultedSpec()); nested(t, input, "sweep") != nil { - t.Fatalf("unexpected sweep: %s", encode(t, input)) - } - }) - t.Run("not added for a pinned load", func(t *testing.T) { - spec := withKVRange(defaultedSpec()) - spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.5} - input := lower(t, spec) - if nested(t, input, "sweep") != nil { - t.Fatalf("unexpected sweep: %s", encode(t, input)) - } - if nested(t, input, "workload", "kv_load_ratio") != 0.5 { - t.Fatal("a one-value ratio should be a scalar") - } - }) - t.Run("native value or null wins", func(t *testing.T) { - if nested(t, lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":7}}`)), "sweep", "max_trials") != 7.0 { - t.Fatal("native max_trials ignored") - } - if nested(t, lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":null}}`)), "sweep", "max_trials") != nil { - t.Fatal("max_trials re-added after a native null") - } - }) -} - -func TestStrictSLADerivation(t *testing.T) { - t.Run("enabled when the typed SLA has a bound", func(t *testing.T) { - if nested(t, lower(t, withSLA(defaultedSpec())), "goal", "strict_sla") != true { - t.Fatal("strict_sla not derived") - } - }) - t.Run("model-only derives nothing", func(t *testing.T) { - input := lower(t, defaultedSpec()) - if nested(t, input, "goal", "strict_sla") != nil || nested(t, input, "goal", "sla") != nil { - t.Fatalf("model-only input has SLA keys: %s", encode(t, input)) - } - }) - t.Run("an empty SLA derives nothing", func(t *testing.T) { - spec := defaultedSpec() - spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{} - input := lower(t, spec) - if nested(t, input, "goal", "strict_sla") != nil || nested(t, input, "goal", "sla") != nil { - t.Fatalf("an empty SLA produced SLA keys: %s", encode(t, input)) - } - }) - t.Run("a native bound alone does not derive it", func(t *testing.T) { - input := lower(t, withParameters(defaultedSpec(), `{"goal":{"sla":{"ttft_ms":500}}}`)) - if nested(t, input, "goal", "sla", "ttft_ms") != 500.0 || nested(t, input, "goal", "strict_sla") != nil { - t.Fatalf("unexpected goal: %s", encode(t, input)) - } - }) - t.Run("native value is kept", func(t *testing.T) { - if nested(t, lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"strict_sla":false}}`)), "goal", "strict_sla") != false { - t.Fatal("native strict_sla overridden") - } - }) - t.Run("is not revisited after the native overlay", func(t *testing.T) { - input := lower(t, withParameters(withSLA(defaultedSpec()), `{"goal":{"sla":{"ttft_ms":null,"itl_ms":null}}}`)) - if nested(t, input, "goal", "strict_sla") != true { - t.Fatalf("the derived strict_sla changed after native bounds were deleted: %s", encode(t, input)) - } - }) -} - -func TestParametersApplyOnceAfterDerivedDefaults(t *testing.T) { - t.Run("sweep null removes the complete derived sweep", func(t *testing.T) { - input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":null}`)) - if nested(t, input, "sweep") != nil { - t.Fatalf("sweep survived a native null: %s", encode(t, input)) - } - }) - t.Run("max_trials null removes only max_trials", func(t *testing.T) { - input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":null,"seed":3}}`)) - if nested(t, input, "sweep", "max_trials") != nil || nested(t, input, "sweep", "seed") != 3.0 { - t.Fatalf("unexpected sweep: %s", encode(t, input)) - } - }) - t.Run("a native max_trials replaces the derived value", func(t *testing.T) { - input := lower(t, withParameters(withKVRange(defaultedSpec()), `{"sweep":{"max_trials":7}}`)) - if nested(t, input, "sweep", "max_trials") != 7.0 { - t.Fatalf("unexpected sweep: %s", encode(t, input)) - } - }) - t.Run("nothing is derived from values only parameters introduce", func(t *testing.T) { - input := lower(t, withParameters(defaultedSpec(), `{"workload":{"kv_load_ratio":[0,1]},"goal":{"sla":{"ttft_ms":500}}}`)) - if nested(t, input, "sweep") != nil || nested(t, input, "goal", "strict_sla") != nil { - t.Fatalf("a field was derived from native-only values: %s", encode(t, input)) - } - }) -} - -func TestUnderlayParetoObjective(t *testing.T) { - spec := defaultedSpec() - spec.Objective.Mode = v1beta2.ObjectiveModePareto - spec.Objective.Metric = nil - spec.Objective.Metrics = []v1beta2.ObjectiveMetric{"ThroughputPerGPU", "ThroughputPerUser", "Unknown"} - input := underlay(t, spec) - if nested(t, input, "goal", "target") != TargetPareto { - t.Fatalf("not a pareto goal: %v", input["goal"]) - } - objectives, _, _ := unstructured.NestedSlice(input, "goal", "pareto_objectives") - got, _ := json.Marshal(objectives) - if string(got) != `["throughput_per_gpu","throughput_per_user","unknown"]` { - t.Fatalf("pareto_objectives = %s", got) - } -} - -func TestUnderlayOmitsUnsetTypedFields(t *testing.T) { - spec := &v1beta2.DynamoGraphDeploymentRequestSpec{Model: &v1beta2.ModelSpec{Name: "m"}} - if got := encode(t, underlay(t, spec)); got != `{"search_space":{"model_name":"m"}}` { - t.Fatalf("got %s", got) - } - spec.Hardware = &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("")}} - if got := encode(t, underlay(t, spec)); got != `{"search_space":{"model_name":"m"}}` { - t.Fatalf("an empty SKU must not lower: %s", got) - } -} - -func TestUnderlayKVLoadRatioShapes(t *testing.T) { - spec := defaultedSpec() - spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.25} - if nested(t, underlay(t, spec), "workload", "kv_load_ratio") != 0.25 { - t.Fatal("a single ratio lowers to a scalar") - } - spec.Workload.Static.KVLoadRatio = v1beta2.KVLoadRatio{0.1, 0.2, 0.3} - if ratios, _ := nested(t, underlay(t, spec), "workload", "kv_load_ratio").([]any); len(ratios) != 3 { - t.Fatal("longer lists lower unchanged so validation can report them") - } -} - -func TestParametersMustBeAnObject(t *testing.T) { - for _, parameters := range []string{`[1]`, `"x"`, `3`, `{`} { - if _, err := Lower(withParameters(defaultedSpec(), parameters)); err == nil { - t.Errorf("Lower accepted parameters %s", parameters) - } - } - for _, parameters := range []string{``, `null`, `{}`} { - if _, err := Lower(withParameters(defaultedSpec(), parameters)); err != nil { - t.Errorf("Lower rejected parameters %q: %v", parameters, err) - } - } -} - -func TestHasPath(t *testing.T) { - parameters := map[string]any{ - "goal": map[string]any{"target": nil, "sla": map[string]any{"ttft_ms": 1.0}}, - "workload": map[string]any{"isl": 8.0}, - } - for path, want := range map[string]bool{ - "goal.target": true, // a null still names the key - "goal.sla": true, - "goal.sla.ttft_ms": true, - "goal.sla.itl_ms": false, - "workload.isl": true, - "workload.osl": false, - "sweep.max_trials": false, - "goal.target.deeper": false, - "workload.isl.deeper": false, - } { - if got := has(parameters, strings.Split(path, ".")...); got != want { - t.Errorf("has(%s) = %v, want %v", path, got, want) - } - } - if has(nil, "goal") { - t.Error("nothing is named in nil parameters") - } -} - -func TestNativeMetricNames(t *testing.T) { - for metric, want := range map[v1beta2.ObjectiveMetric]string{ - "Throughput": "throughput", - "ThroughputPerGPU": "throughput_per_gpu", - "ThroughputPerUser": "throughput_per_user", - "TTFT": "ttft", - "E2ELatency": "e2e_latency", - "Goodput": "goodput", - "GoodputPerGPU": "goodput_per_gpu", - } { - if got := nativeMetric(metric); got != want { - t.Errorf("nativeMetric(%s) = %q, want %q", metric, got, want) - } - } -} diff --git a/deploy/operator/internal/controller/aisimulate/paths.go b/deploy/operator/internal/controller/aisimulate/paths.go deleted file mode 100644 index bd2a9d16b7d8..000000000000 --- a/deploy/operator/internal/controller/aisimulate/paths.go +++ /dev/null @@ -1,59 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - -// Native input paths, in the snake_case spelling of the upstream AISimulate contract. They -// address the same native key when the underlay is built and when derived defaults are read, -// and are passed to the unstructured helpers, which must not modify them. -var ( - searchSpaceModelName = []string{"search_space", "model_name"} - searchSpaceBackend = []string{"search_space", "backend"} - searchSpaceHardwareSKU = []string{"search_space", "hardware_sku"} - searchSpaceGPUBudget = []string{"search_space", "gpu_budget"} - - workloadPath = []string{"workload"} - workloadISL = []string{"workload", "isl"} - workloadOSL = []string{"workload", "osl"} - workloadNumRequestRatio = []string{"workload", "num_request_ratio"} - workloadRequestRate = []string{"workload", "request_rate"} - workloadConcurrency = []string{"workload", "concurrency"} - workloadKVLoadRatio = []string{"workload", "kv_load_ratio"} - workloadRequestCount = []string{"workload", "request_count"} - - goalTarget = []string{"goal", "target"} - goalParetoObjectives = []string{"goal", "pareto_objectives"} - goalSLATTFTMs = []string{"goal", "sla", "ttft_ms"} - goalSLAITLMs = []string{"goal", "sla", "itl_ms"} - goalSLAE2EMs = []string{"goal", "sla", "e2e_ms"} - goalStrictSLA = []string{"goal", "strict_sla"} - - sweepMaxTrials = []string{"sweep", "max_trials"} -) - -// TargetPareto is the goal.target of a multi-objective search. -const TargetPareto = "pareto" - -// has reports whether obj contains the path, even with a null value. Applied to -// spec.aisimulate.parameters it tells whether the caller named a key, including to delete -// it. -func has(obj map[string]any, path ...string) bool { - _, found, _ := unstructured.NestedFieldNoCopy(obj, path...) - return found -} diff --git a/deploy/operator/internal/controller/aisimulate/release_contract_test.go b/deploy/operator/internal/controller/aisimulate/release_contract_test.go deleted file mode 100644 index a5cf0464b8f8..000000000000 --- a/deploy/operator/internal/controller/aisimulate/release_contract_test.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package aisimulate - -import ( - "os" - "path/filepath" - "regexp" - "testing" -) - -// The repository root, relative to this package: deploy/operator/internal/controller/aisimulate. -const repositoryRoot = "../../../../.." - -var ( - pinPattern = regexp.MustCompile(`aisimulate==(\d+\.\d+\.\d+)`) - releasePattern = regexp.MustCompile(`(?m)^version\s*=\s*"(\d+\.\d+)\.`) -) - -func readRepositoryFile(t *testing.T, path string) string { - t.Helper() - raw, err := os.ReadFile(filepath.Join(repositoryRoot, path)) - if err != nil { - t.Fatalf("read %s: %v", path, err) - } - return string(raw) -} - -// pinnedAISimulate returns the released core (major.minor.patch) of the aisimulate pin in a file, -// so 0.13.0.dev202610160000000073 normalizes to 0.13.0. -func pinnedAISimulate(t *testing.T, path string) string { - t.Helper() - match := pinPattern.FindStringSubmatch(readRepositoryFile(t, path)) - if match == nil { - t.Fatalf("%s pins no aisimulate==", path) - } - return match[1] -} - -// TestBundledAISimulateVersionMatchesTheDynamoRelease ties the release contract to the repository: -// the AISimulate version the current Dynamo minor maps to must be the one the repository pins. It -// fails when a Dynamo minor bump has no mapping entry, or when the pin moves without the entry. -func TestBundledAISimulateVersionMatchesTheDynamoRelease(t *testing.T) { - canonical := pinnedAISimulate(t, "container/deps/requirements.aisimulate.txt") - for _, path := range []string{"pyproject.toml", "benchmarks/pyproject.toml"} { - if got := pinnedAISimulate(t, path); got != canonical { - t.Errorf("%s pins aisimulate %s, but container/deps/requirements.aisimulate.txt pins %s", path, got, canonical) - } - } - - release := releasePattern.FindStringSubmatch(readRepositoryFile(t, "pyproject.toml")) - if release == nil { - t.Fatal("pyproject.toml has no project version") - } - minor := release[1] - mapped, ok := aisimulateVersionByDynamoMinor[minor] - if !ok { - t.Fatalf("Dynamo %s has no entry in aisimulateVersionByDynamoMinor; add %q: %q with the release", minor, minor, canonical) - } - if mapped != canonical { - t.Fatalf("aisimulateVersionByDynamoMinor[%q] = %q, but the repository pins aisimulate %s", minor, mapped, canonical) - } -} diff --git a/deploy/operator/internal/controller/suite_envtest_test.go b/deploy/operator/internal/controller/suite_envtest_test.go index 0a6c4633cc74..3d2f9edd032c 100644 --- a/deploy/operator/internal/controller/suite_envtest_test.go +++ b/deploy/operator/internal/controller/suite_envtest_test.go @@ -28,10 +28,6 @@ import ( const admissionBypassUsername = "operatorenv-controller-admission-bypass" -// The controller specs create native v1beta2 Requests without an image or hardware. Admission -// completes them from the operator's configuration, as it does in production. -const controllerAISimulateImage = "registry.example/aisimulate:1.2.0" - type fixedHardwareDiscoverer struct{} func (fixedHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { @@ -59,8 +55,7 @@ func setupProductionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOpti OperatorVersion: opts.OperatorVersion, OperatorPrincipal: opts.OperatorPrincipal, DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ - AISimulateImage: controllerAISimulateImage, - Hardware: fixedHardwareDiscoverer{}, + Hardware: fixedHardwareDiscoverer{}, }, }) } diff --git a/deploy/operator/internal/dynamo/component_common.go b/deploy/operator/internal/dynamo/component_common.go index cfa186de1ac4..3f09b3400d60 100644 --- a/deploy/operator/internal/dynamo/component_common.go +++ b/deploy/operator/internal/dynamo/component_common.go @@ -10,7 +10,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" controller_common "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" corev1 "k8s.io/api/core/v1" ) @@ -69,7 +69,7 @@ type ComponentContext struct { Infrastructure configv1alpha1.InfrastructureConfiguration EPPConfig *v1beta1.EPPConfig WorkerHashSuffix string - RuntimeVersion *versionoverride.Version + RuntimeVersion *runtimeversion.Version Annotations map[string]string } diff --git a/deploy/operator/internal/dynamo/component_frontend_test.go b/deploy/operator/internal/dynamo/component_frontend_test.go index 2c38ee903090..d51d4fd3d021 100644 --- a/deploy/operator/internal/dynamo/component_frontend_test.go +++ b/deploy/operator/internal/dynamo/component_frontend_test.go @@ -11,7 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -21,13 +21,13 @@ func TestFrontendDefaultsStartupProbeVersionGate(t *testing.T) { tests := []struct { name string origin string - runtimeVersion *versionoverride.Version + runtimeVersion *runtimeversion.Version wantEnabled bool }{ {name: "unknown runtime on new graph", origin: "1.7.0"}, - {name: "older runtime on new graph", origin: "1.7.0", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 9}}, - {name: "released 1.6 graph preserves existing defaults", origin: "1.6.0", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 6}}, - {name: "new graph with minimum supported runtime", origin: "1.7.0", runtimeVersion: &versionoverride.Version{Major: 1, Minor: 6}, wantEnabled: true}, + {name: "older runtime on new graph", origin: "1.7.0", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 9}}, + {name: "released 1.6 graph preserves existing defaults", origin: "1.6.0", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 6}}, + {name: "new graph with minimum supported runtime", origin: "1.7.0", runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 6}, wantEnabled: true}, } for _, tt := range tests { diff --git a/deploy/operator/internal/dynamo/component_namespace_test.go b/deploy/operator/internal/dynamo/component_namespace_test.go index cc2741d95370..3ea504f8d441 100644 --- a/deploy/operator/internal/dynamo/component_namespace_test.go +++ b/deploy/operator/internal/dynamo/component_namespace_test.go @@ -11,7 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -22,13 +22,13 @@ func TestComponentNamespacePrefixRuntimeCompatibility(t *testing.T) { t.Log("render frontend and EPP defaults across the strict-discovery release boundary") versions := []struct { name string - version *versionoverride.Version + version *runtimeversion.Version strict bool }{ {name: "unknown runtime"}, - {name: "older runtime", version: &versionoverride.Version{Major: 1, Minor: 5, Patch: 9}}, - {name: "supported runtime", version: &versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, strict: true}, - {name: "newer runtime", version: &versionoverride.Version{Major: 1, Minor: 7, Patch: 0}, strict: true}, + {name: "older runtime", version: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 9}}, + {name: "supported runtime", version: &runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, strict: true}, + {name: "newer runtime", version: &runtimeversion.Version{Major: 1, Minor: 7, Patch: 0}, strict: true}, } for _, componentType := range []string{commonconsts.ComponentTypeFrontend, commonconsts.ComponentTypeEPP} { @@ -59,7 +59,7 @@ func TestLegacyEPPDoesNotEnableStrictNamespacePrefix(t *testing.T) { container, err := NewEPPDefaults().GetBaseContainer(ComponentContext{ DynamoNamespace: "default-foo", ComponentType: commonconsts.ComponentTypeEPP, - RuntimeVersion: &versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, + RuntimeVersion: &runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, EPPConfig: &v1beta1.EPPConfig{}, }) require.NoError(t, err) diff --git a/deploy/operator/internal/dynamo/component_worker_test.go b/deploy/operator/internal/dynamo/component_worker_test.go index 2c399d60724e..aa39447c5b40 100644 --- a/deploy/operator/internal/dynamo/component_worker_test.go +++ b/deploy/operator/internal/dynamo/component_worker_test.go @@ -9,14 +9,14 @@ import ( "testing" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" "github.com/stretchr/testify/require" ) func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *versionoverride.Version + runtimeVersion *runtimeversion.Version wantEnabled string }{ { @@ -25,17 +25,17 @@ func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { }, { name: "older runtime", - runtimeVersion: &versionoverride.Version{Major: 1, Minor: 4, Patch: 9}, + runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9}, wantEnabled: "false", }, { name: "minimum supported runtime", - runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, + runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, wantEnabled: "true", }, { name: "newer runtime", - runtimeVersion: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, + runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, wantEnabled: "true", }, } @@ -64,7 +64,7 @@ func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) { func TestWorkerDefaultsLivenessFailureThresholdVersionGate(t *testing.T) { tests := []struct { name string - runtimeVersion *versionoverride.Version + runtimeVersion *runtimeversion.Version wantFailureCount int32 }{ { @@ -73,17 +73,17 @@ func TestWorkerDefaultsLivenessFailureThresholdVersionGate(t *testing.T) { }, { name: "older runtime", - runtimeVersion: &versionoverride.Version{Major: 1, Minor: 4, Patch: 9}, + runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9}, wantFailureCount: 1, }, { name: "minimum supported runtime", - runtimeVersion: &versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, + runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, wantFailureCount: 3, }, { name: "newer runtime", - runtimeVersion: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, + runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, wantFailureCount: 3, }, } diff --git a/deploy/operator/internal/dynamo/graph.go b/deploy/operator/internal/dynamo/graph.go index 30cb0f0ebb5e..09692c41b4ff 100644 --- a/deploy/operator/internal/dynamo/graph.go +++ b/deploy/operator/internal/dynamo/graph.go @@ -44,7 +44,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/compatibility" gms "github.com/ai-dynamo/dynamo/deploy/operator/internal/gms" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1" "github.com/imdario/mergo" "google.golang.org/protobuf/types/known/wrapperspb" @@ -2163,8 +2163,8 @@ func mergeFrontendSidecarDefaults(podSpec *corev1.PodSpec, sidecarName string, p } // Resolve the frontend image independently of the component runtime. - var resolvedRuntimeVersion *versionoverride.Version - if version, err := versionoverride.ParseImageVersion(podSpec.Containers[i].Image); err == nil { + var resolvedRuntimeVersion *runtimeversion.Version + if version, err := runtimeversion.ParseImageVersion(podSpec.Containers[i].Image); err == nil { resolvedRuntimeVersion = &version } @@ -2256,8 +2256,8 @@ func generateComponentContext(component *v1beta1.DynamoComponentDeploymentShared workerHashSuffix = workerHash } - var resolvedRuntimeVersion *versionoverride.Version - if version, err := versionoverride.Resolve(runtimeImage, component.RuntimeVersionOverride); err == nil { + var resolvedRuntimeVersion *runtimeversion.Version + if version, err := runtimeversion.Resolve(runtimeImage, component.RuntimeVersionOverride); err == nil { resolvedRuntimeVersion = &version } else if component.RuntimeVersionOverride != "" { return ComponentContext{}, fmt.Errorf("resolve runtime version override: %w", err) diff --git a/deploy/operator/internal/dynamo/hash.go b/deploy/operator/internal/dynamo/hash.go index e7e7f44ea2f6..9f165f36ebaf 100644 --- a/deploy/operator/internal/dynamo/hash.go +++ b/deploy/operator/internal/dynamo/hash.go @@ -27,7 +27,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" dynamolpx "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/lpx" "github.com/ai-dynamo/dynamo/deploy/operator/internal/provideroverride" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/utils/ptr" ) @@ -35,7 +35,7 @@ import ( const dgdWorkerHashPlaceholderValue = "worker-hash-placeholder" // Preserve hashes for runtimes released before version hashing. -var minimumHashedRuntimeVersion = versionoverride.Version{Major: 1, Minor: 5, Patch: 0} +var minimumHashedRuntimeVersion = runtimeversion.Version{Major: 1, Minor: 5, Patch: 0} // ComputeDGDWorkersSpecHash computes the v2 worker hash from the worker DCDs // generated by the same DGD-to-DCD path used by reconciliation. The active @@ -228,7 +228,7 @@ func resolvedRuntimeVersionForHash(component *v1beta1.DynamoComponentDeploymentS if runtime := GetDynamoContainer(component); runtime != nil { image = runtime.Image } - version, err := versionoverride.Resolve(image, component.RuntimeVersionOverride) + version, err := runtimeversion.Resolve(image, component.RuntimeVersionOverride) if err != nil || version.Compare(minimumHashedRuntimeVersion) < 0 { return "" } diff --git a/deploy/operator/internal/features/runtime/gate.go b/deploy/operator/internal/features/runtime/gate.go index 66a3aedab213..805099b09e95 100644 --- a/deploy/operator/internal/features/runtime/gate.go +++ b/deploy/operator/internal/features/runtime/gate.go @@ -5,16 +5,16 @@ package runtime -import "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" +import "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" // Gate controls runtime-version-dependent operator behavior. type Gate struct { Name string - MinRuntimeVersion versionoverride.Version + MinRuntimeVersion runtimeversion.Version } // Enabled reports whether a known runtime version meets the feature threshold. -func (g Gate) Enabled(version *versionoverride.Version) bool { +func (g Gate) Enabled(version *runtimeversion.Version) bool { if version == nil { return false } diff --git a/deploy/operator/internal/features/runtime/gate_test.go b/deploy/operator/internal/features/runtime/gate_test.go index b9ea9db53738..c3919a34b988 100644 --- a/deploy/operator/internal/features/runtime/gate_test.go +++ b/deploy/operator/internal/features/runtime/gate_test.go @@ -8,20 +8,20 @@ package runtime import ( "testing" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" ) func TestGateEnabled(t *testing.T) { t.Log("define a feature introduced by Dynamo runtime 1.4.0") gate := Gate{ Name: "TestFeature", - MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 4, Patch: 0}, + MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 4, Patch: 0}, } t.Log("define versions below, at, and above the feature threshold") tests := []struct { name string - version *versionoverride.Version + version *runtimeversion.Version want bool }{ { @@ -29,16 +29,16 @@ func TestGateEnabled(t *testing.T) { }, { name: "older runtime", - version: &versionoverride.Version{Major: 1, Minor: 3, Patch: 9}, + version: &runtimeversion.Version{Major: 1, Minor: 3, Patch: 9}, }, { name: "minimum supported runtime", - version: &versionoverride.Version{Major: 1, Minor: 4, Patch: 0}, + version: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 0}, want: true, }, { name: "newer runtime", - version: &versionoverride.Version{Major: 2, Minor: 0, Patch: 0}, + version: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0}, want: true, }, } diff --git a/deploy/operator/internal/features/runtime/gates.go b/deploy/operator/internal/features/runtime/gates.go index ee6b782562c1..91970f5d0cf9 100644 --- a/deploy/operator/internal/features/runtime/gates.go +++ b/deploy/operator/internal/features/runtime/gates.go @@ -5,21 +5,22 @@ package runtime -import "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" +import "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" var ( // StrictNamespacePrefix gates operator-managed discovery isolation. StrictNamespacePrefix = Gate{ Name: "StrictNamespacePrefix", - MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, } + // CanaryHealthChecks gates the canary health-check rendering defaults. // Runtime 1.5.0 is the first version whose resolved runtime version is // included in the worker hash, so enabling the feature cannot silently // change an existing worker generation. CanaryHealthChecks = Gate{ Name: "CanaryHealthChecks", - MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, } // IncreasedWorkerFailureThreshold gates the worker liveness default. @@ -28,7 +29,7 @@ var ( // change an existing worker generation. IncreasedWorkerFailureThreshold = Gate{ Name: "IncreasedWorkerFailureThreshold", - MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, } // NativeRustEPP gates the native Rust EPP requirement: runtime versions at @@ -37,7 +38,7 @@ var ( // 1.5.0, the first release shipping the native Rust EPP image. NativeRustEPP = Gate{ Name: "NativeRustEPP", - MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 5, Patch: 0}, + MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 5, Patch: 0}, } // PlannerDGDComponentStatus gates Planner discovery of model, runtime, @@ -47,6 +48,6 @@ var ( // PodTemplates. PlannerDGDComponentStatus = Gate{ Name: "PlannerDGDComponentStatus", - MinRuntimeVersion: versionoverride.Version{Major: 1, Minor: 6, Patch: 0}, + MinRuntimeVersion: runtimeversion.Version{Major: 1, Minor: 6, Patch: 0}, } ) diff --git a/deploy/operator/internal/versionoverride/versionoverride.go b/deploy/operator/internal/runtimeversion/runtimeversion.go similarity index 91% rename from deploy/operator/internal/versionoverride/versionoverride.go rename to deploy/operator/internal/runtimeversion/runtimeversion.go index 38c4682d3116..eab91c6c48d8 100644 --- a/deploy/operator/internal/versionoverride/versionoverride.go +++ b/deploy/operator/internal/runtimeversion/runtimeversion.go @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package versionoverride +package runtimeversion import ( "cmp" @@ -16,7 +16,7 @@ import ( var imageTagPattern = regexp.MustCompile(`^[vV]?(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:-((?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*))?(?:\+([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$`) -// Version identifies a normalized compatibility version. +// Version identifies a runtime version by semver core. type Version struct { Major uint64 Minor uint64 @@ -38,7 +38,7 @@ func (v Version) Compare(other Version) int { ) } -// Parse returns the compatibility version represented by an explicit override. +// Parse returns the runtime version represented by an explicit override. func Parse(value string) (Version, error) { version, err := semver.StrictNewVersion(value) if err != nil { @@ -47,7 +47,7 @@ func Parse(value string) (Version, error) { return fromSemver(version), nil } -// ParseImageVersion returns a normalized compatibility version from an image tag. +// ParseImageVersion returns a normalized runtime version from an image tag. func ParseImageVersion(image string) (Version, error) { tag := imageTag(image) if tag == "" { diff --git a/deploy/operator/internal/versionoverride/versionoverride_test.go b/deploy/operator/internal/runtimeversion/runtimeversion_test.go similarity index 99% rename from deploy/operator/internal/versionoverride/versionoverride_test.go rename to deploy/operator/internal/runtimeversion/runtimeversion_test.go index 8cccb8005eef..edc26679c343 100644 --- a/deploy/operator/internal/versionoverride/versionoverride_test.go +++ b/deploy/operator/internal/runtimeversion/runtimeversion_test.go @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package versionoverride +package runtimeversion import "testing" diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index b51ba6fd85e9..01601c6cd602 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -59,15 +59,6 @@ func NewDGDRDefaulter(operatorVersion, defaultImage string) *DGDRDefaulter { // WithV1beta2 sets the defaults of v1beta2 Requests and returns the defaulter. func (d *DGDRDefaulter) WithV1beta2(options DGDRV1beta2Options) *DGDRDefaulter { - // The operator refuses to start with a configuration that does not resolve, so a failure here - // only happens to a caller that skipped ValidateAISimulateDefault: keep the image and let - // validation report what the Request lacks. - if image, version, err := resolveAISimulateDefault( - d.OperatorVersion, options.AISimulateImage, options.AISimulateVersionOverride); err == nil { - options.AISimulateImage, options.AISimulateVersionOverride = image, version - } else { - options.AISimulateImage = defaultPlannerImageFor(d.OperatorVersion, options.AISimulateImage) - } d.v1beta2 = options return d } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go index 416fe004f280..bbf3ad192df0 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go @@ -20,7 +20,6 @@ package defaulting import ( "bytes" "context" - "encoding/json" "testing" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -55,45 +54,7 @@ func carrierRequest() *nvidiacomv1beta1.DynamoGraphDeploymentRequest { } } -func aisimulateImageOf(t *testing.T, raw []byte) string { - t.Helper() - var spec map[string]any - if err := json.Unmarshal(raw, &spec); err != nil { - t.Fatalf("decode carrier: %v", err) - } - aisimulate, _ := spec["aisimulate"].(map[string]any) - image, _ := aisimulate["image"].(string) - return image -} - func TestDGDRDefaulter_RoutesByRequestedVersion(t *testing.T) { - for _, tt := range []struct { - name string - options DGDRV1beta2Options - want string - }{ - { - name: "an omitted image is derived from the operator version", - want: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.6.0", - }, - { - name: "an explicit default image overrides derivation", - options: DGDRV1beta2Options{AISimulateImage: "registry.example/aisimulate:custom"}, - want: "registry.example/aisimulate:custom", - }, - } { - t.Run(tt.name, func(t *testing.T) { - d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(tt.options) - dgdr := carrierRequest() - if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { - t.Fatalf("Default() error = %v", err) - } - if got := aisimulateImageOf(t, dgdr.Spec.V1Beta2.Raw); got != tt.want { - t.Fatalf("aisimulate.image = %q, want %q", got, tt.want) - } - }) - } - d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(DGDRV1beta2Options{}) t.Run("RequestKind decides, not the converted Kind", func(t *testing.T) { @@ -104,6 +65,9 @@ func TestDGDRDefaulter_RoutesByRequestedVersion(t *testing.T) { if dgdr.Spec.Image != "" { t.Fatalf("v1beta1 image defaulting touched a v1beta2 request: %q", dgdr.Spec.Image) } + if !bytes.Contains(dgdr.Spec.V1Beta2.Raw, []byte(`"backends"`)) { + t.Fatalf("native v1beta2 defaults were not applied: %s", dgdr.Spec.V1Beta2.Raw) + } }) for _, version := range []string{"", "v1beta1", "v1alpha1"} { diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index 37e7098880b3..becc3fb11dff 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -37,10 +37,10 @@ const defaultMetricV1beta2 = nvidiacomv1beta2.ObjectiveMetric("Throughput") // defaults a mutated object in the webhook's version (v1beta1, whose v1beta2 carrier is // opaque), so the v1beta2 schema carries no default markers for them. const ( - defaultStaticISLV1beta2 int32 = 1024 - defaultStaticOSLV1beta2 int32 = 128 - defaultStaticConcurrencyV1beta2 int32 = 10 - defaultMaxCandidatesV1beta2 int32 = 1 + defaultStaticInputTokensV1beta2 int32 = 1024 + defaultStaticOutputTokensV1beta2 int32 = 128 + defaultStaticConcurrencyV1beta2 int32 = 10 + defaultMaxCandidatesV1beta2 int32 = 1 ) // This file defaults the v1beta2 shape of a DynamoGraphDeploymentRequest. It is separate @@ -50,10 +50,11 @@ const ( // API version because the canonical names belong to v1beta1. // // Admission owns every v1beta2 default, and each default has this one owner: backends, -// objective.mode, the scalar metric, recommendation.maxCandidates, workload.static, the -// cluster hardware and the AISimulate image. workload.static: {} opts out of the static -// workload. objective.sla has no default: an omitted SLA stays absent. -// spec.aisimulate.parameters never changes a default. It overrides the defaulted typed +// objective.mode, the scalar metric, recommendation.maxCandidates, workload.static and the +// cluster hardware. workload.static: {} opts out of the static workload; a nonempty static +// object receives any missing inputTokens, outputTokens and load defaults. objective.sla has +// no default: an omitted SLA stays absent. +// spec.aisimulate.inputOverrides never changes a default. It overrides the defaulted typed // values later, when the input is lowered. // defaultDynamoGraphDeploymentRequestV1beta2 defaults the v1beta2 spec that dgdr carries in @@ -123,10 +124,6 @@ func decodeOldSpecV1beta2( type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { // hardware is the GPU identity and budget discovered from the cluster. hardware hardwareDefaultsV1beta2 - // aisimulateImage is the operator's resolved AISimulate image. - aisimulateImage string - // aisimulateVersionOverride belongs to aisimulateImage. - aisimulateVersionOverride string } // hardwareDefaultsV1beta2 is the hardware the cluster can offer for the values a Request omits. @@ -180,15 +177,6 @@ func defaultNativeSearchInputV1beta2( } defaultObjectiveSpecV1beta2(input.Objective) - // The operator image is persisted with the Request, so spec.aisimulate is created to - // carry it. - if input.AISimulate == nil && opts.aisimulateImage != "" { - input.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} - } - if input.AISimulate != nil { - defaultAISimulateSpecV1beta2(input.AISimulate, opts.aisimulateImage, opts.aisimulateVersionOverride) - } - if input.Recommendation == nil { input.Recommendation = &nvidiacomv1beta2.RecommendationSpec{} } @@ -216,23 +204,33 @@ func defaultGPUHardwareSpecV1beta2(gpu *nvidiacomv1beta2.GPUHardwareSpec, defaul } func defaultWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec) { - // An explicit static: {} is the opt-out, so the static defaults apply only to the object - // created here. if workload.Static == nil { workload.Static = &nvidiacomv1beta2.StaticWorkloadSpec{} - defaultStaticWorkloadSpecV1beta2(workload.Static) + } else if workload.Static.InputTokens == nil && workload.Static.OutputTokens == nil && workload.Static.Load == nil { + // An explicit empty object is the one static-workload opt-out. + return } + defaultStaticWorkloadSpecV1beta2(workload.Static) } +// defaultStaticWorkloadSpecV1beta2 completes a static workload with every omitted default. func defaultStaticWorkloadSpecV1beta2(static *nvidiacomv1beta2.StaticWorkloadSpec) { - static.ISL = ptr.To(defaultStaticISLV1beta2) - static.OSL = ptr.To(defaultStaticOSLV1beta2) - static.Concurrency = ptr.To(defaultStaticConcurrencyV1beta2) + if static.InputTokens == nil { + static.InputTokens = ptr.To(defaultStaticInputTokensV1beta2) + } + if static.OutputTokens == nil { + static.OutputTokens = ptr.To(defaultStaticOutputTokensV1beta2) + } + if static.Load == nil { + static.Load = &nvidiacomv1beta2.StaticLoadSpec{ + Concurrency: ptr.To(defaultStaticConcurrencyV1beta2), + } + } } // defaultObjectiveSpecV1beta2 defaults the mode and gives an Optimize objective without a // metric the default scalar target. It reads the typed objective alone: a goal.target in -// spec.aisimulate.parameters overrides the completed typed value when the input is lowered, +// spec.aisimulate.inputOverrides overrides the completed typed value when the input is lowered, // so it never suppresses this default. func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { if objective.Mode == "" { @@ -243,19 +241,6 @@ func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { } } -// defaultAISimulateSpecV1beta2 persists the operator's AISimulate artifact as one pair. The -// override follows the image: it is applied only when admission supplies the image. A caller's -// image derives its own version or brings its own override, and a caller's override always wins. -func defaultAISimulateSpecV1beta2(aisimulate *nvidiacomv1beta2.AISimulateSpec, image, versionOverride string) { - if aisimulate.Image != "" || image == "" { - return - } - aisimulate.Image = image - if aisimulate.VersionOverride == "" { - aisimulate.VersionOverride = versionOverride - } -} - func defaultRecommendationSpecV1beta2(recommendation *nvidiacomv1beta2.RecommendationSpec) { if recommendation.MaxCandidates == 0 { recommendation.MaxCandidates = defaultMaxCandidatesV1beta2 diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact.go deleted file mode 100644 index 613a3f9d0ca2..000000000000 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact.go +++ /dev/null @@ -1,57 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package defaulting - -import ( - "fmt" - - "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" -) - -// resolveAISimulateDefault returns the AISimulate artifact admission persists as one pair: the -// configured planner image, or the official one derived from the operator version when none is -// configured, and the AISimulate version it carries. The explicit override wins, the derived image -// included; otherwise the version is the one bundled in the Dynamo release named by the image tag -// (aisimulate.ResolveVersion). Both are empty when there is nothing to persist: no image is -// configured and the operator version names no release. An override then has no image to pair with. -func resolveAISimulateDefault(operatorVersion, image, versionOverride string) (resolvedImage, version string, err error) { - resolvedImage = defaultPlannerImageFor(operatorVersion, image) - if resolvedImage == "" { - if versionOverride != "" { - return "", "", fmt.Errorf("the AISimulate default version override needs an image: operator version %q "+ - "names no release to derive the planner image from; set the AISimulate default image", operatorVersion) - } - return "", "", nil - } - version, err = aisimulate.ResolveVersion(resolvedImage, versionOverride) - if err != nil { - return "", "", fmt.Errorf("the AISimulate default image %q cannot be used as it is: %w; "+ - "set the AISimulate default version override to the AISimulate version it bundles", resolvedImage, err) - } - return resolvedImage, version, nil -} - -// ValidateAISimulateDefault rejects an operator configuration whose default AISimulate artifact -// admission could not persist: an image, configured or derived from the operator version, whose -// Dynamo release has no bundled AISimulate version in the release contract needs the override that -// names it, and an override without any image to pair with is rejected too. An override with an -// empty image configuration is valid: it pairs with the official planner image. -func ValidateAISimulateDefault(operatorVersion, image, versionOverride string) error { - _, _, err := resolveAISimulateDefault(operatorVersion, image, versionOverride) - return err -} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact_test.go deleted file mode 100644 index 9ec9339eee89..000000000000 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_artifact_test.go +++ /dev/null @@ -1,120 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package defaulting - -import ( - "testing" - - nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" -) - -func TestDefaultAISimulateFollowsTheOwnershipOfTheImage(t *testing.T) { - const image = "registry.example/aisimulate:nightly" - tests := []struct { - name string - spec nvidiacomv1beta2.AISimulateSpec - wantImage string - wantOverride string - }{ - {"the operator's pair is persisted together", nvidiacomv1beta2.AISimulateSpec{}, image, "1.2.0"}, - {"a caller's image gets no operator override", - nvidiacomv1beta2.AISimulateSpec{Image: "example/own:v1.0.0"}, "example/own:v1.0.0", ""}, - {"a caller's override wins next to the operator's image", - nvidiacomv1beta2.AISimulateSpec{VersionOverride: "2.0.0"}, image, "2.0.0"}, - {"a caller's pair is kept", - nvidiacomv1beta2.AISimulateSpec{Image: "example/own:x", VersionOverride: "3.0.0"}, "example/own:x", "3.0.0"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - spec := tt.spec - defaultAISimulateSpecV1beta2(&spec, image, "1.2.0") - if spec.Image != tt.wantImage || spec.VersionOverride != tt.wantOverride { - t.Fatalf("got %q/%q, want %q/%q", spec.Image, spec.VersionOverride, tt.wantImage, tt.wantOverride) - } - }) - } -} - -func TestValidateAISimulateDefault(t *testing.T) { - const ( - mapped = "registry.example/aisimulate:1.6.0" - unmapped = "registry.example/aisimulate:1.2.3" - nightly = "registry.example/aisimulate:nightly" - bundled = "0.13.0" - unmappedV = "0.9.0" - ) - tests := []struct { - name, operatorVersion, image, override string - wantErr bool - }{ - {"the derived planner image of a mapped release", "1.6.0", "", "", false}, - {"the derived planner image of a prerelease", "1.6.0-rc.1", "", "", false}, - {"the derived planner image of a release without an entry", "1.99.0", "", "", true}, - {"an override rescues the derived image", "1.99.0", "", unmappedV, false}, - {"an override with an empty image configuration", "1.6.0", "", bundled, false}, - {"an operator version that names no release", "unknown", "", "", false}, - {"an override with nothing to pair with", "unknown", "", bundled, true}, - {"a mapped tag", "1.6.0", mapped, "", false}, - {"a mapped tag with an override", "1.6.0", mapped, "0.13.1", false}, - {"a release without an entry", "1.6.0", unmapped, "", true}, - {"a release without an entry, with its override", "1.6.0", unmapped, unmappedV, false}, - {"a non-release tag with its override", "1.6.0", nightly, bundled, false}, - {"a non-release tag without an override", "1.6.0", nightly, "", true}, - {"an override that is not a version", "1.6.0", nightly, "bad", true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := ValidateAISimulateDefault(tt.operatorVersion, tt.image, tt.override); (err != nil) != tt.wantErr { - t.Fatalf("error = %v, want error: %v", err, tt.wantErr) - } - }) - } -} - -func TestWithV1beta2PersistsThePlannerImageWithItsBundledAISimulateVersion(t *testing.T) { - derived := defaultImage + ":1.6.0" - bundled, err := aisimulate.ResolveVersion(derived, "") - if err != nil { - t.Fatalf("the release contract must map the operator version used here: %v", err) - } - tests := []struct { - name string - options DGDRV1beta2Options - wantImage string - wantOverride string - }{ - {"the derived image gets the bundled version", DGDRV1beta2Options{}, derived, bundled}, - {"the explicit override wins with the derived image", - DGDRV1beta2Options{AISimulateVersionOverride: "0.13.1"}, derived, "0.13.1"}, - {"a configured mapped image gets the bundled version", - DGDRV1beta2Options{AISimulateImage: "registry.example/aisimulate:1.6.0"}, - "registry.example/aisimulate:1.6.0", bundled}, - {"a configured nightly image keeps its override", - DGDRV1beta2Options{AISimulateImage: "registry.example/aisimulate:nightly", AISimulateVersionOverride: "0.13.0"}, - "registry.example/aisimulate:nightly", "0.13.0"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := NewDGDRDefaulter("1.6.0", "").WithV1beta2(tt.options).v1beta2 - if got.AISimulateImage != tt.wantImage || got.AISimulateVersionOverride != tt.wantOverride { - t.Fatalf("got %q/%q, want %q/%q", got.AISimulateImage, got.AISimulateVersionOverride, tt.wantImage, tt.wantOverride) - } - }) - } -} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index fe7bfb6c36f9..c0f04cef3037 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -54,15 +54,6 @@ type HardwareDiscoverer interface { // DGDRV1beta2Options configures the defaults of v1beta2 Requests. type DGDRV1beta2Options struct { - // AISimulateImage overrides the version-matched Dynamo planner image persisted - // into spec.aisimulate.image when the caller omits it. - AISimulateImage string - // AISimulateVersionOverride is the AISimulate version bundled in AISimulateImage, for an image - // whose Dynamo release the release contract does not map (a nightly or custom planner image). - // Empty maps the image's release. WithV1beta2 completes it, so admission persists the image - // together with its AISimulate version, and only next to an image it supplies itself, never - // next to an image the caller chose. - AISimulateVersionOverride string // Hardware discovers the cluster's GPUs. Nil disables hardware defaulting, and // validation then requires the hardware to be spelled out. Hardware HardwareDiscoverer @@ -85,11 +76,7 @@ func (o DGDRV1beta2Options) resolveDefaults( if err != nil { return dynamoGraphDeploymentRequestV1beta2DefaultOptions{}, err } - return dynamoGraphDeploymentRequestV1beta2DefaultOptions{ - hardware: hardware, - aisimulateImage: o.AISimulateImage, - aisimulateVersionOverride: o.AISimulateVersionOverride, - }, nil + return dynamoGraphDeploymentRequestV1beta2DefaultOptions{hardware: hardware}, nil } // resolveHardware returns the SKU and budget the typed spec omits. It asks the cluster only @@ -101,7 +88,7 @@ func (o DGDRV1beta2Options) resolveDefaults( // never asks the cluster: a Request created while discovery had nothing to offer must not // pick hardware up once discovery recovers, which would change its spec and generation behind // the writer's back. oldSpec is the stored spec on UPDATE and nil otherwise. The decision reads -// the typed hardware alone, never spec.aisimulate.parameters. +// the typed hardware alone, never spec.aisimulate.inputOverrides. // // After a successful return the typed hardware is complete. An omitted SKU becomes the // cluster's AISimulate SKU, and an omitted budget becomes min(matching GPUs, cap). An explicit diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index 25b6dfbf90b5..60678ebe0d61 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -127,12 +127,12 @@ func TestResolveHardwareV1beta2(t *testing.T) { func TestResolveDefaultsV1beta2(t *testing.T) { d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} - got, err := DGDRV1beta2Options{Hardware: d, AISimulateImage: "registry.example.com/aisimulate:1.2.0"}. + got, err := DGDRV1beta2Options{Hardware: d}. resolveDefaults(context.Background(), admissionv1.Create, nil, modelOnlySpecV1beta2()) if err != nil { t.Fatal(err) } - if got.aisimulateImage != "registry.example.com/aisimulate:1.2.0" || got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { + if got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { t.Fatalf("got %+v", got) } } @@ -150,8 +150,7 @@ func v1beta2Envelope(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentR func TestDefaultEnvelopeV1beta2(t *testing.T) { options := DGDRV1beta2Options{ - AISimulateImage: "registry.example.com/aisimulate:1.2.0", - Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}, + Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}, } dgdr := v1beta2Envelope(t, modelOnlySpecV1beta2()) @@ -165,9 +164,6 @@ func TestDefaultEnvelopeV1beta2(t *testing.T) { if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 { t.Fatalf("hardware not defaulted: %+v", got.Hardware) } - if got.AISimulate == nil || got.AISimulate.Image != "registry.example.com/aisimulate:1.2.0" { - t.Fatalf("image not defaulted: %+v", got.AISimulate) - } if got.Objective == nil || got.Objective.Metric == nil { t.Fatalf("objective metric not defaulted: %+v", got.Objective) } diff --git a/deploy/operator/internal/webhook/setup/setup.go b/deploy/operator/internal/webhook/setup/setup.go index e25f1f0b5429..16b73dffcbf7 100644 --- a/deploy/operator/internal/webhook/setup/setup.go +++ b/deploy/operator/internal/webhook/setup/setup.go @@ -25,8 +25,7 @@ type Options struct { OperatorVersion string // DGDRV1beta1 configures the defaults of v1beta1 Requests. DGDRV1beta1 webhookdefaulting.DGDRV1beta1Options - // DGDRV1beta2 configures the defaults of v1beta2 Requests: the AISimulate runner - // image and the cluster hardware discovery. + // DGDRV1beta2 configures cluster-derived defaults of v1beta2 Requests. DGDRV1beta2 webhookdefaulting.DGDRV1beta2Options OperatorPrincipal string Gate features.Gate diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index 8835e790abe6..3e2cd01c8509 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -27,7 +27,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdoverride" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" internalwebhook "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -346,7 +346,7 @@ func dgdrRuntimeVersionOverrideRequired(spec *nvidiacomv1beta1.DynamoGraphDeploy if spec.Image == "" || spec.RuntimeVersionOverride != "" { return false } - _, err := versionoverride.ParseImageVersion(spec.Image) + _, err := runtimeversion.ParseImageVersion(spec.Image) return err != nil } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index 557129b380e4..0a7c411af443 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -21,7 +21,7 @@ import ( "encoding/json" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/aisimulate" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/aisimulate" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -39,44 +39,12 @@ func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( if err := json.Unmarshal(raw.Raw, &spec); err != nil { return field.ErrorList{field.Invalid(fldPath, "", "must be a valid v1beta2 spec: "+err.Error())} } - return validateDynamoGraphDeploymentRequestV1beta2(&spec) -} - -// validateDynamoGraphDeploymentRequestV1beta2 validates a defaulted v1beta2 Request spec: the -// typed fields and the image and version selection. spec.aisimulate.parameters is opaque here: the -// CRD types it as an object, and the validation of its content belongs to the schema of the -// AISimulate release the Request selects. -func validateDynamoGraphDeploymentRequestV1beta2(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) field.ErrorList { - return validateDynamoGraphDeploymentRequestSpecV1beta2(spec, field.NewPath("spec")) -} - -// validateDynamoGraphDeploymentRequestSpecV1beta2 validates the typed rules that the source -// schema and CEL cannot express. The generated OpenAPI schema and CEL of the v1beta2 API have -// already run before conversion reaches this webhook, so they are not repeated here: native -// model presence, the model name length, backend uniqueness and the kvLoadRatio range order. -// spec and fldPath must not be nil. -func validateDynamoGraphDeploymentRequestSpecV1beta2( - spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, - fldPath *field.Path, -) field.ErrorList { - return validateAISimulateSpecV1beta2(spec.AISimulate, fldPath.Child("aisimulate")) -} - -// validateAISimulateSpecV1beta2 validates the execution artifact. Admission persists the -// operator's default image, so a Request that still has none was created while the operator -// had no default image configured. spec may be nil. -func validateAISimulateSpecV1beta2(spec *nvidiacomv1beta2.AISimulateSpec, fldPath *field.Path) field.ErrorList { - if spec == nil || spec.Image == "" { - return field.ErrorList{field.Required(fldPath.Child("image"), - "an AISimulate image is required; set it, or configure the operator's default AISimulate image")} - } - if _, err := aisimulate.ResolveVersion(spec.Image, spec.VersionOverride); err != nil { - if spec.VersionOverride != "" { - return field.ErrorList{field.Invalid(fldPath.Child("versionOverride"), spec.VersionOverride, - "must be a semantic version such as \"0.13.0\"")} - } - return field.ErrorList{field.Required(fldPath.Child("versionOverride"), - "is required when the tag of spec.aisimulate.image is not a Dynamo release with a bundled AISimulate version")} + if _, err := aisimulate.Lower(&spec.NativeSearchInput); err != nil { + return field.ErrorList{field.Invalid( + field.NewPath("spec"), + "", + "cannot be lowered to the public AISimulate recommendation input: "+err.Error(), + )} } return nil } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go index 9ad45a24a0b9..3a65159fa1b6 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go @@ -18,46 +18,13 @@ package validation import ( - "encoding/json" "strings" "testing" - nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" ) -func v1beta2Ptr[T any](v T) *T { return &v } - -// v1beta2DefaultedSpec is a Request as it looks once the schema and admission have defaulted it. -func v1beta2DefaultedSpec() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - metric := nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") - return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - Model: &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, - Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendSGLang, nvidiacomv1beta2.BackendTRTLLM}, - Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: v1beta2Ptr("h200_sxm"), Budget: v1beta2Ptr(int32(8))}}, - Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{ - ISL: v1beta2Ptr(int32(4000)), OSL: v1beta2Ptr(int32(1000)), KVLoadRatio: nvidiacomv1beta2.KVLoadRatio{0, 1}, - }}, - Objective: &nvidiacomv1beta2.ObjectiveSpec{ - Mode: nvidiacomv1beta2.ObjectiveModeOptimize, - Metric: &metric, - SLA: &nvidiacomv1beta2.ObjectiveSLASpec{TTFTMs: v1beta2Ptr(2000.0), ITLMs: v1beta2Ptr(30.0)}, - }, - AISimulate: &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:1.6.0"}, - Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, - } -} - -func v1beta2WithParameters(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, parameters string) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - if spec.AISimulate == nil { - spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{Image: "registry.example.com/aisimulate:1.6.0"} - } - spec.AISimulate.Parameters = &runtime.RawExtension{Raw: []byte(parameters)} - return spec -} - func v1beta2Details(errs field.ErrorList) string { var out []string for _, err := range errs { @@ -80,66 +47,6 @@ func v1beta2RequireNoError(t *testing.T, errs field.ErrorList) { } } -func TestValidateV1beta2AcceptsADefaultedRequest(t *testing.T) { - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(v1beta2DefaultedSpec())) -} - -// The webhook does not repeat what the generated schema and CEL enforce before conversion: -// native model presence, the model name length, backend uniqueness and the kvLoadRatio range -// order. The operatorenv table proves those errors end to end. -func TestValidateV1beta2LeavesSourceSchemaRulesToTheSchema(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.Model = nil - spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendVLLM} - spec.Workload.Static.KVLoadRatio = nvidiacomv1beta2.KVLoadRatio{0.9, 0.1} - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) -} - -func TestValidateV1beta2ParametersAreOpaque(t *testing.T) { - for _, parameters := range []string{ - `{"workload":{"isl":0},"goal":{"target":"nope"}}`, - `[1]`, // the CRD types parameters as an object; admission does not interpret it - } { - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2( - v1beta2WithParameters(v1beta2DefaultedSpec(), parameters))) - } -} - -func TestValidateV1beta2AISimulateImage(t *testing.T) { - t.Run("no AISimulate section", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.AISimulate = nil - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.image: an AISimulate image is required") - }) - t.Run("parameters without an image", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{} - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.image") - }) - t.Run("custom tag needs a version", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.AISimulate.Image = "registry.example.com/aisimulate:custom" - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.versionOverride: is required when the tag of spec.aisimulate.image is not a Dynamo release with a bundled AISimulate version") - }) - t.Run("custom tag with a version", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.AISimulate.Image = "registry.example.com/aisimulate:custom" - spec.AISimulate.VersionOverride = "0.13.1" - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestV1beta2(spec)) - }) - t.Run("a Dynamo release without a bundled version needs one", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.AISimulate.Image = "registry.example.com/aisimulate:1.2.3" - spec.AISimulate.VersionOverride = "" - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.versionOverride") - }) - t.Run("an override that is not a version", func(t *testing.T) { - spec := v1beta2DefaultedSpec() - spec.AISimulate.VersionOverride = "bad" - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestV1beta2(spec), "spec.aisimulate.versionOverride: must be a semantic version") - }) -} - func v1beta2Envelope(raw string) *apiextensionsv1.JSON { return &apiextensionsv1.JSON{Raw: []byte(raw)} } @@ -147,16 +54,13 @@ func v1beta2Envelope(raw string) *apiextensionsv1.JSON { func TestValidateEnvelopeV1beta2(t *testing.T) { path := field.NewPath("spec", "v1beta2") - good, err := json.Marshal(v1beta2DefaultedSpec()) - if err != nil { - t.Fatal(err) - } - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(good)), path)) - - bad := v1beta2DefaultedSpec() - bad.AISimulate = nil - raw, _ := json.Marshal(bad) - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(string(raw)), path), "aisimulate") + v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2( + v1beta2Envelope(`{"model":{"name":"meta-llama/Llama-3.1-8B-Instruct"}}`), path)) v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") + + v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{ + "model":{"name":"meta-llama/Llama-3.1-8B-Instruct"}, + "objective":{"mode":"Pareto","metrics":["Throughput","TTFT"]} + }`), path), "cannot be lowered to the public AISimulate recommendation input") } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 506958890cd0..7d8cc625b934 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -574,23 +574,26 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 parameters are stored untouched and never decide hardware defaulting", + name: "v1beta2 input overrides are stored untouched and never decide hardware defaulting", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(4)}) + setV1beta2InputOverrides(spec, map[string]any{ + "engine": map[string]any{"hardware": "h100_sxm"}, + "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": int64(4)}}, + }) }, nil), gpuDiscovery: true, discovered: discoveredH200, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h200_sxm", 8) - if got := v1beta2String(t, result, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "h100_sxm" { - t.Fatalf("parameters.search_space.hardware_sku = %q, want h100_sxm", got) + if got := v1beta2String(t, result, "spec", "aisimulate", "inputOverrides", "engine", "hardware"); got != "h100_sxm" { + t.Fatalf("inputOverrides.engine.hardware = %q, want h100_sxm", got) } }, }, { - name: "v1beta2 parameters null does not suppress hardware discovery", + name: "v1beta2 input overrides null does not suppress hardware discovery", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": nil}) + setV1beta2InputOverrides(spec, map[string]any{"engine": map[string]any{"hardware": nil}}) }, nil), gpuDiscovery: true, discovered: discoveredH200, @@ -603,10 +606,16 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { { name: "v1beta2 metadata-only update does not read the cluster again", oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}) + setV1beta2InputOverrides(spec, map[string]any{ + "engine": map[string]any{"hardware": "h100_sxm"}, + "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": int64(8)}}, + }) }, nil), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2SearchSpace(spec, map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}) + setV1beta2InputOverrides(spec, map[string]any{ + "engine": map[string]any{"hardware": "h100_sxm"}, + "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": int64(8)}}, + }) }, map[string]any{"updated": "true"}), gpuDiscovery: true, discovered: discoveredH200, @@ -668,12 +677,14 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, discovered: discoveredH200, check: func(t *testing.T, result *unstructured.Unstructured) { - for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { + for key, want := range map[string]float64{"inputTokens": 1024, "outputTokens": 128} { if got := v1beta2Number(t, result, "spec", "workload", "static", key); got != want { t.Fatalf("workload.static.%s = %v, want %v", key, got, want) } } - v1beta2Absent(t, result, "spec", "workload", "static", "kvLoadRatio") + if got := v1beta2Number(t, result, "spec", "workload", "static", "load", "concurrency"); got != 10 { + t.Fatalf("workload.static.load.concurrency = %v, want 10", got) + } }, }, { @@ -688,10 +699,10 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 empty workload static stays empty beside native parameters", + name: "v1beta2 empty workload static stays empty beside public input overrides", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"static": map[string]any{}} - spec["aisimulate"] = map[string]any{"parameters": mustJSONObject(t, `{"workload":{"isl":512,"osl":128,"concurrency":8,"request_count":100}}`)} + spec["aisimulate"] = map[string]any{"inputOverrides": mustJSONObject(t, `{"traffic":{"source":{"type":"synthetic","input_tokens":512,"output_tokens":128},"load":{"type":"concurrency","concurrency":8},"stop":{"requests":100}}}`)} }, nil), gpuDiscovery: true, discovered: discoveredH200, @@ -710,84 +721,27 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { discovered: discoveredH200, check: func(t *testing.T, result *unstructured.Unstructured) { v1beta2Empty(t, result, "spec", "objective", "sla") - if got := v1beta2Number(t, result, "spec", "workload", "static", "isl"); got != 1024 { - t.Fatalf("workload.static.isl = %v, want the schema default 1024", got) + if got := v1beta2Number(t, result, "spec", "workload", "static", "inputTokens"); got != 1024 { + t.Fatalf("workload.static.inputTokens = %v, want the admission default 1024", got) } }, }, { - name: "v1beta2 native parameters win without rewriting the typed spec", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"parameters": mustJSONObject(t, `{"workload":{"isl":256}}`)} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - if got := v1beta2Number(t, result, "spec", "workload", "static", "isl"); got != 1024 { - t.Fatalf("typed workload.static.isl = %v, want 1024", got) - } - if got := v1beta2Number(t, result, "spec", "aisimulate", "parameters", "workload", "isl"); got != 256 { - t.Fatalf("parameters.workload.isl = %v, want 256", got) - } - }, - }, - { - name: "v1beta2 stored invalid spec is rejected again on a metadata-only update", - oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:custom"} - }, nil), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:custom"} - }, map[string]any{"updated": "true"}), - seedWithoutWebhook: true, - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{ - "spec.aisimulate.versionOverride: Required value: is required when the tag of spec.aisimulate.image is not a Dynamo release with a bundled AISimulate version", - }, - }, - { - name: "v1beta2 model-only request stores the configured AISimulate image and its override", - request: v1beta2DGDRForAdmission(nil, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2AdmissionDefaults(t, result) - }, - }, - { - name: "v1beta2 caller image does not receive the operator's version override", + name: "v1beta2 public input overrides win without rewriting the typed spec", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"image": "registry.example.com/aisimulate:1.6.0"} + spec["aisimulate"] = map[string]any{"inputOverrides": mustJSONObject(t, `{"traffic":{"source":{"input_tokens":256}}}`)} }, nil), gpuDiscovery: true, discovered: discoveredH200, check: func(t *testing.T, result *unstructured.Unstructured) { - if got := v1beta2String(t, result, "spec", "aisimulate", "image"); got != "registry.example.com/aisimulate:1.6.0" { - t.Fatalf("aisimulate.image = %q, want the caller's image", got) + if got := v1beta2Number(t, result, "spec", "workload", "static", "inputTokens"); got != 1024 { + t.Fatalf("typed workload.static.inputTokens = %v, want 1024", got) } - if _, found, _ := unstructured.NestedString(result.Object, "spec", "aisimulate", "versionOverride"); found { - t.Fatalf("aisimulate.versionOverride was set next to a caller's image") + if got := v1beta2Number(t, result, "spec", "aisimulate", "inputOverrides", "traffic", "source", "input_tokens"); got != 256 { + t.Fatalf("inputOverrides.traffic.source.input_tokens = %v, want 256", got) } }, }, - { - name: "v1beta2 caller override wins next to the configured image", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"versionOverride": "2.0.0"} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - if got := v1beta2String(t, result, "spec", "aisimulate", "image"); got != admissionAISimulateImage { - t.Fatalf("aisimulate.image = %q, want the configured image", got) - } - if got := v1beta2String(t, result, "spec", "aisimulate", "versionOverride"); got != "2.0.0" { - t.Fatalf("aisimulate.versionOverride = %q, want the caller's 2.0.0", got) - } - }, - }, - { name: "v1beta2 hardware omitted is discovered and the budget capped", request: v1beta2DGDRForAdmission(nil, nil), @@ -925,7 +879,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { { name: "v1beta2 pareto objective receives no scalar metric", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"TTFT", "Throughput"}} + spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"ThroughputPerGPU", "ThroughputPerUser"}} }, nil), gpuDiscovery: true, discovered: discoveredH200, @@ -933,17 +887,23 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { v1beta2Absent(t, result, "spec", "objective", "metric") }, }, + { + name: "v1beta2 pareto rejects objectives absent from the public AISimulate input", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"TTFT", "Throughput"}} + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.objective: Invalid value: Pareto supports exactly ThroughputPerGPU and ThroughputPerUser`, + }, { name: "v1beta2 explicit values are all kept", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["backends"] = []any{"vllm"} spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} - spec["workload"] = map[string]any{"static": map[string]any{"isl": int64(10)}} + spec["workload"] = map[string]any{"static": map[string]any{"inputTokens": int64(10)}} spec["objective"] = map[string]any{"mode": "Optimize", "metric": "TTFT"} - spec["aisimulate"] = map[string]any{ - "image": "registry.example.com/aisimulate:custom", - "versionOverride": "1.4.0", - } + spec["aisimulate"] = map[string]any{"inputOverrides": map[string]any{"custom": "kept"}} spec["recommendation"] = map[string]any{"maxCandidates": int64(3)} }, nil), gpuDiscovery: true, @@ -976,46 +936,47 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { wantSchemaErr: `spec.model.name: Invalid value: "": spec.model.name in body should be at least 1 chars long`, }, { - name: "v1beta2 kvLoadRatio range needs min below max", + name: "v1beta2 KV capacity fraction range needs min below max", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"static": map[string]any{"kvLoadRatio": []any{0.9, 0.1}}} + spec["workload"] = map[string]any{"static": map[string]any{"load": map[string]any{ + "kvCapacityFraction": map[string]any{"range": map[string]any{"min": 0.9, "max": 0.1}}, + }}} }, nil), gpuDiscovery: true, discovered: discoveredH200, - wantCELErr: `spec.workload.static.kvLoadRatio: Invalid value: a range needs min < max`, + wantCELErr: `spec.workload.static.load.kvCapacityFraction.range: Invalid value: min must be less than max`, }, { - name: "v1beta2 versionOverride latest is rejected by the schema", + name: "v1beta2 KV capacity fraction must be positive", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"versionOverride": "latest"} + spec["workload"] = map[string]any{"static": map[string]any{"load": map[string]any{ + "kvCapacityFraction": map[string]any{"value": 0.0}, + }}} }, nil), gpuDiscovery: true, discovered: discoveredH200, - wantSchemaErr: `spec.aisimulate.versionOverride: Invalid value: "latest": spec.aisimulate.versionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + wantSchemaErr: `spec.workload.static.load.kvCapacityFraction.value: Invalid value: 0: spec.workload.static.load.kvCapacityFraction.value in body should be greater than 0`, }, { - name: "v1beta2 versionOverride with a v prefix is rejected by the schema", + name: "v1beta2 load requires exactly one variant", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"versionOverride": "v0.13.0"} + spec["workload"] = map[string]any{"static": map[string]any{"load": map[string]any{ + "concurrency": int64(10), "requestsPerSecond": 10.0, + }}} }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - wantSchemaErr: `spec.aisimulate.versionOverride: Invalid value: "v0.13.0": spec.aisimulate.versionOverride in body should match '^(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})\.(0|[1-9][0-9]{0,3})$'`, + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload.static.load: Invalid value: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified`, }, { - name: "v1beta2 versionOverride 0.13.0 is accepted", + name: "v1beta2 empty load is not a second default opt-out", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"versionOverride": "0.13.0"} + spec["workload"] = map[string]any{"static": map[string]any{"load": map[string]any{}}} }, nil), gpuDiscovery: true, discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - if got := v1beta2String(t, result, "spec", "aisimulate", "versionOverride"); got != "0.13.0" { - t.Fatalf("versionOverride = %q, want 0.13.0", got) - } - }, + wantCELErr: `spec.workload.static.load: Invalid value: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified`, }, - // v1beta2 UPDATE: defaults stay idempotent, hardware is never rediscovered, and any // spec change is fully validated. { @@ -1046,8 +1007,8 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { t.Fatalf("generation = %d, want 1", result.GetGeneration()) } expectV1beta2GPUValues(t, result, "h200_sxm", 8) - if got := v1beta2String(t, result, "spec", "aisimulate", "parameters", "search_space", "hardware_sku"); got != "h100_sxm" { - t.Fatalf("parameters.search_space.hardware_sku = %q, want h100_sxm", got) + if got := v1beta2String(t, result, "spec", "aisimulate", "inputOverrides", "engine", "hardware"); got != "h100_sxm" { + t.Fatalf("inputOverrides.engine.hardware = %q, want h100_sxm", got) } }, }, @@ -1315,14 +1276,13 @@ func beta2DGDRForAdmission( const v1beta2AdmissionModel = "Qwen/Qwen3-0.6B" -// v1beta2NativeHardwareDGDR supplies hardware only through native AISimulate parameters. +// v1beta2NativeHardwareDGDR supplies hardware only through public AISimulate input overrides. func v1beta2NativeHardwareDGDR(labels map[string]any) *unstructured.Unstructured { return v1beta2DGDRForAdmission(func(spec map[string]any) { spec["aisimulate"] = map[string]any{ - "image": admissionAISimulateImage, - "versionOverride": admissionAISimulateVersionOverride, - "parameters": map[string]any{ - "search_space": map[string]any{"hardware_sku": "h100_sxm", "gpu_budget": int64(8)}, + "inputOverrides": map[string]any{ + "engine": map[string]any{"hardware": "h100_sxm"}, + "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": int64(8)}}, }, } }, labels) @@ -1349,7 +1309,7 @@ func v1beta2DGDRForAdmission(mutate func(spec map[string]any), labels map[string }} } -// mustJSONObject decodes a native AISimulate parameters document. +// mustJSONObject decodes a native AISimulate input override document. func mustJSONObject(t *testing.T, document string) map[string]any { t.Helper() var value map[string]any @@ -1416,12 +1376,14 @@ func v1beta2Empty(t *testing.T, obj *unstructured.Unstructured, path ...string) // expectV1beta2StructuralDefaults checks the defaults admission supplies to a model-only Request. func expectV1beta2StructuralDefaults(t *testing.T, obj *unstructured.Unstructured) { t.Helper() - for key, want := range map[string]float64{"isl": 1024, "osl": 128, "concurrency": 10} { + for key, want := range map[string]float64{"inputTokens": 1024, "outputTokens": 128} { if got := v1beta2Number(t, obj, "spec", "workload", "static", key); got != want { t.Fatalf("workload.static.%s = %v, want %v", key, got, want) } } - v1beta2Absent(t, obj, "spec", "workload", "static", "kvLoadRatio") + if got := v1beta2Number(t, obj, "spec", "workload", "static", "load", "concurrency"); got != 10 { + t.Fatalf("workload.static.load.concurrency = %v, want 10", got) + } v1beta2Absent(t, obj, "spec", "objective", "sla") if got := v1beta2String(t, obj, "spec", "objective", "mode"); got != "Optimize" { t.Fatalf("objective.mode = %q, want Optimize", got) @@ -1445,7 +1407,9 @@ func expectV1beta2ExplicitValues(t *testing.T, obj *unstructured.Unstructured) { value float64 }{ {[]string{"spec", "hardware", "gpu", "budget"}, 4}, - {[]string{"spec", "workload", "static", "isl"}, 10}, + {[]string{"spec", "workload", "static", "inputTokens"}, 10}, + {[]string{"spec", "workload", "static", "outputTokens"}, 128}, + {[]string{"spec", "workload", "static", "load", "concurrency"}, 10}, {[]string{"spec", "recommendation", "maxCandidates"}, 3}, } { if got := v1beta2Number(t, obj, want.path...); got != want.value { @@ -1459,16 +1423,13 @@ func expectV1beta2ExplicitValues(t *testing.T, obj *unstructured.Unstructured) { {[]string{"spec", "hardware", "gpu", "sku"}, "h100_sxm"}, {[]string{"spec", "objective", "mode"}, "Optimize"}, {[]string{"spec", "objective", "metric"}, "TTFT"}, - {[]string{"spec", "aisimulate", "image"}, "registry.example.com/aisimulate:custom"}, - {[]string{"spec", "aisimulate", "versionOverride"}, "1.4.0"}, + {[]string{"spec", "aisimulate", "inputOverrides", "custom"}, "kept"}, } { if got := v1beta2String(t, obj, want.path...); got != want.value { t.Fatalf("%v = %q, want %q", want.path, got, want.value) } } - // A static workload that names only isl is not filled in. - v1beta2Absent(t, obj, "spec", "workload", "static", "osl") - v1beta2Absent(t, obj, "spec", "workload", "static", "concurrency") + // A nonempty static workload receives every omitted member of the default. } // expectV1beta2AdmissionDefaults checks what admission supplies from the environment. @@ -1477,12 +1438,6 @@ func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured if got := v1beta2String(t, obj, "spec", "objective", "metric"); got != "Throughput" { t.Fatalf("objective.metric = %q, want Throughput", got) } - if got := v1beta2String(t, obj, "spec", "aisimulate", "image"); got != admissionAISimulateImage { - t.Fatalf("aisimulate.image = %q, want %q", got, admissionAISimulateImage) - } - if got := v1beta2String(t, obj, "spec", "aisimulate", "versionOverride"); got != admissionAISimulateVersionOverride { - t.Fatalf("aisimulate.versionOverride = %q, want %q", got, admissionAISimulateVersionOverride) - } if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != "h200_sxm" { t.Fatalf("hardware.gpu.sku = %q, want h200_sxm", got) } @@ -1491,15 +1446,14 @@ func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured } } -// setV1beta2SearchSpace sets spec.aisimulate.parameters.search_space to searchSpace, replacing -// any parameters the fixture carried. -func setV1beta2SearchSpace(spec map[string]any, searchSpace map[string]any) { +// setV1beta2InputOverrides replaces any input overrides the fixture carried. +func setV1beta2InputOverrides(spec map[string]any, overrides map[string]any) { aisimulate, _ := spec["aisimulate"].(map[string]any) if aisimulate == nil { aisimulate = map[string]any{} spec["aisimulate"] = aisimulate } - aisimulate["parameters"] = map[string]any{"search_space": searchSpace} + aisimulate["inputOverrides"] = overrides } // setBeta2AdmissionHardware names the hardware, so fixtures do not depend on GPU discovery. diff --git a/deploy/operator/internal/webhook/validation/planner_runtime_features.go b/deploy/operator/internal/webhook/validation/planner_runtime_features.go index c61ff68ec318..1e772c501f72 100644 --- a/deploy/operator/internal/webhook/validation/planner_runtime_features.go +++ b/deploy/operator/internal/webhook/validation/planner_runtime_features.go @@ -13,7 +13,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -117,7 +117,7 @@ func rolePodTemplatesPlannerRuntimeErrorDetail(planners []plannerRuntimeContract minimum := runtimefeatures.PlannerDGDComponentStatus.MinRuntimeVersion.String() for _, planner := range planners { - version, err := versionoverride.Resolve(planner.image, planner.runtimeVersionOverride) + version, err := runtimeversion.Resolve(planner.image, planner.runtimeVersionOverride) if err != nil { return fmt.Sprintf( "role-specific PodTemplates require every planner component to use Dynamo runtime %s or later; planner component %q has no resolvable runtime version: %v", diff --git a/deploy/operator/internal/webhook/validation/shared_helpers.go b/deploy/operator/internal/webhook/validation/shared_helpers.go index 2d903624d701..a5780d059a14 100644 --- a/deploy/operator/internal/webhook/validation/shared_helpers.go +++ b/deploy/operator/internal/webhook/validation/shared_helpers.go @@ -30,7 +30,7 @@ import ( "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo/epp" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/versionoverride" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/validation/field" @@ -153,7 +153,7 @@ func runtimeVersionOverrideRequired(image, override string) bool { if override != "" { return false } - _, err := versionoverride.ParseImageVersion(image) + _, err := runtimeversion.ParseImageVersion(image) return err != nil } @@ -196,7 +196,7 @@ func eppRuntimeCompatibilityError(contract eppRuntimeContract, eppConfigPath *fi return nil } - version, err := versionoverride.Resolve(contract.image, contract.runtimeVersionOverride) + version, err := runtimeversion.Resolve(contract.image, contract.runtimeVersionOverride) if err != nil { return nil } diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 5e0f2dd8fb56..2e15b11a10c2 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -45,10 +45,6 @@ const ( frontendImage150 = "registry.example/dynamo-frontend:1.5.0" legacyEPPImage140 = "registry.example/epp-image:1.4.0" legacySeedUsername = "operatorenv-legacy-seeder" - admissionAISimulateImage = "registry.example.com/aisimulate:nightly" - // admissionAISimulateVersionOverride is the AISimulate version bundled in the configured - // default image, whose tag is not a Dynamo release. - admissionAISimulateVersionOverride = "0.13.0" ) func groveProviderOverride(target, value string) *nvidiacomv1beta1.ProviderOverride { @@ -101,9 +97,7 @@ func setupAdmissionWebhooks(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptio OperatorPrincipal: opts.OperatorPrincipal, Gate: admissionGate, DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ - AISimulateImage: admissionAISimulateImage, - AISimulateVersionOverride: admissionAISimulateVersionOverride, - Hardware: admissionHardware, + Hardware: admissionHardware, }, }) } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index ee9c3e5766c5..9e5a5ad4e700 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3226,11 +3226,28 @@ Package v1beta2 contains API Schema definitions for the nvidia.com v1beta2 API g +#### AISimulateRunStatus + + + +AISimulateRunStatus reports the exact public recommendation input used for this execution. + + + +_Appears in:_ +- [DynamoGraphDeploymentRunStatus](#dynamographdeploymentrunstatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `input` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Input is the fully lowered AISimulate input after typed fields, derived defaults and
spec.aisimulate.inputOverrides have been merged. Field names and nesting use the
upstream AISimulate JSON contract. | | Type: object
Optional: \{\}
| + + #### AISimulateSpec -AISimulateSpec selects the AISimulate execution artifact and supplies native input. +AISimulateSpec supplies public recommendation input to the AISimulate version in the operator's configured +planner image. @@ -3242,9 +3259,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `image` _string_ | Image selects a Dynamo-compatible AISimulate runner. When omitted, admission
persists the operator's configured AISimulate image, or the Dynamo planner image of
the operator's release, together with the AISimulate version it bundles. | | MinLength: 1
Optional: \{\}
| -| `versionOverride` _string_ | nightly or custom image), and it wins over the mapping. | | MinLength: 1
Pattern: `^(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})\.(0\|[1-9][0-9]\{0,3\})$`
Optional: \{\}
| -| `parameters` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Parameters contains native AISimulate input. Field names and nesting use
the upstream AISimulate JSON contract, including snake_case names. | | Type: object
Optional: \{\}
| +| `inputOverrides` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from
the typed fields. It uses the upstream AISimulate JSON contract, including
snake_case names. Values here take precedence, and null deletes a lowered
field. The operator runs the AISimulate version from its configured planner
image; Requests and Runs cannot select a different image or version. | | Type: object
Optional: \{\}
| #### Backend @@ -3438,9 +3453,9 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| -| `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| +| `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3480,9 +3495,9 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| -| `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| +| `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| | `v1beta1` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | V1Beta1 losslessly preserves a native v1beta1 spec while this object is
represented through v1beta2. It is mutually exclusive with native fields.
This field is a read-only conversion view; modify the request through the
native v1beta1 API. | | Type: object
Optional: \{\}
| @@ -3543,9 +3558,9 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| -| `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| +| `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3565,6 +3580,7 @@ _Appears in:_ | `phase` _[DGDRRunPhase](#dgdrrunphase)_ | Phase is a concise presentation state. Conditions remain authoritative. | | Enum: [Pending Running Succeeded Failed]
Optional: \{\}
| | `jobRef` _[ResourceReference](#resourcereference)_ | JobRef identifies the controller-owned Job for this run. | | Optional: \{\}
| | `progress` _[DGDRRunProgress](#dgdrrunprogress)_ | Progress contains branch-aware optimizer and evaluation outcome counters. | | Optional: \{\}
| +| `aisimulate` _[AISimulateRunStatus](#aisimulaterunstatus)_ | AISimulate reports the effective input executed by AISimulate. | | Optional: \{\}
| | `candidateRefs` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core) array_ | CandidateRefs identifies the current bounded DGDC projection. List order
represents rank, best first, for scalar searches. | | Optional: \{\}
| | `startTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | StartTime is when execution started. | | Optional: \{\}
| | `completionTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta)_ | CompletionTime is present only after a terminal outcome. | | Optional: \{\}
| @@ -3630,23 +3646,39 @@ _Appears in:_ | `gpu` _[GPUHardwareSpec](#gpuhardwarespec)_ | GPU describes accelerator identity and the maximum allocation per candidate.
Omitted fields are resolved independently by admission. | | Optional: \{\}
| -#### KVLoadRatio +#### KVCapacityFractionRange -_Underlying type:_ _float array_ -KVLoadRatio is either a pinned load ratio or a minimum/maximum range. -AISimulate performs mode-specific validation. -_Validation:_ -- MaxItems: 2 -- MinItems: 1 -- items:Maximum: 1 -- items:Minimum: 0 +KVCapacityFractionRange defines the searched fraction of a candidate's KV capacity. + + _Appears in:_ -- [StaticWorkloadSpec](#staticworkloadspec) +- [KVCapacityFractionSpec](#kvcapacityfractionspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `min` _float_ | Min is the lower end of the searched fraction range. | | ExclusiveMinimum: true
Maximum: 1
Minimum: 0
| +| `max` _float_ | Max is the upper end of the searched fraction range. | | ExclusiveMinimum: true
Maximum: 1
Minimum: 0
| + + +#### KVCapacityFractionSpec + +KVCapacityFractionSpec selects either one fixed fraction or a searched range. + + + +_Appears in:_ +- [StaticLoadSpec](#staticloadspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `value` _float_ | Value uses a fixed fraction of the candidate's KV capacity. A value of 1
uses its full estimated capacity. | | ExclusiveMinimum: true
Maximum: 1
Minimum: 0
Optional: \{\}
| +| `range` _[KVCapacityFractionRange](#kvcapacityfractionrange)_ | Range searches fractions between min and max. | | Optional: \{\}
| + #### ModelSpec @@ -3686,9 +3718,9 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10;
an explicit empty workload receives the same shape from admission. An explicit
workload.static: \{\} opts out of that shape, leaving the workload to
spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the
fields it names.
The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a
KV-load ratio search. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| -| `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate selects the execution artifact and supplies native AISimulate input. | | Optional: \{\}
| +| `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3761,8 +3793,8 @@ _Appears in:_ | --- | --- | --- | --- | | `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization or a Pareto search. | | Enum: [Optimize Pareto]
Optional: \{\}
| | `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to Throughput. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| -| `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics lists two or more distinct scalar targets when mode is Pareto. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 7
MinItems: 2
Optional: \{\}
| -| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. When omitted, no
bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. | | Optional: \{\}
| +| `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics selects AISimulate's public Pareto objective pair when mode is Pareto.
The configured AISimulate release supports exactly ThroughputPerGPU and
ThroughputPerUser through its public recommendation input. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 2
MinItems: 2
Optional: \{\}
| +| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. When omitted, no
bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. | | Optional: \{\}
| #### RecommendationSpec @@ -3818,11 +3850,30 @@ _Appears in:_ | `total` _integer_ | Total is the number of planned rounds. | | | +#### StaticLoadSpec + + + +StaticLoadSpec selects exactly one synthetic load model by field presence. + + + +_Appears in:_ +- [StaticWorkloadSpec](#staticworkloadspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `requestsPerSecond` _float_ | RequestsPerSecond is a fixed open-loop arrival rate. | | ExclusiveMinimum: true
Minimum: 0
Optional: \{\}
| +| `concurrency` _integer_ | Concurrency is a fixed closed-loop in-flight request cap. | | Minimum: 1
Optional: \{\}
| +| `kvCapacityFraction` _[KVCapacityFractionSpec](#kvcapacityfractionspec)_ | KVCapacityFraction derives concurrency from each candidate's estimated KV capacity. | | Optional: \{\}
| + + #### StaticWorkloadSpec -StaticWorkloadSpec defines one of AISimulate's synthetic workload shapes. +StaticWorkloadSpec defines AISimulate synthetic traffic. Admission completes a nonempty +object with the omitted static defaults; an explicit empty object opts out of them. @@ -3831,12 +3882,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `isl` _integer_ | ISL is the input sequence length in tokens. | | Minimum: 1
Optional: \{\}
| -| `osl` _integer_ | OSL is the output sequence length in tokens. | | Minimum: 1
Optional: \{\}
| -| `numRequestRatio` _float_ | NumRequestRatio sets request count relative to the selected load. | | ExclusiveMinimum: true
Minimum: 0
Optional: \{\}
| -| `requestRate` _float_ | RequestRate is a fixed open-loop arrival rate in requests per second. | | ExclusiveMinimum: true
Minimum: 0
Optional: \{\}
| -| `concurrency` _integer_ | Concurrency is a fixed closed-loop in-flight request cap. | | Minimum: 1
Optional: \{\}
| -| `kvLoadRatio` _[KVLoadRatio](#kvloadratio)_ | KVLoadRatio derives concurrency from candidate KV capacity. A two-value
range requires a bounded AISimulate sweep. | | MaxItems: 2
MinItems: 1
items:Maximum: 1
items:Minimum: 0
Optional: \{\}
| +| `inputTokens` _integer_ | InputTokens is the synthetic request input length in tokens. | | Minimum: 1
Optional: \{\}
| +| `outputTokens` _integer_ | OutputTokens is the synthetic request output length in tokens. | | Minimum: 1
Optional: \{\}
| +| `load` _[StaticLoadSpec](#staticloadspec)_ | Load selects the synthetic load model. When omitted from a nonempty static
workload, admission defaults it to concurrency 10. | | Optional: \{\}
| #### v1beta2 WorkloadSpec @@ -3855,7 +3903,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines one of AISimulate's synthetic workload shapes. Its fields are not
defaulted individually: the default belongs to spec.workload as a whole, so an
explicit workload object keeps exactly the fields it names. | | Optional: \{\}
| +| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines AISimulate synthetic traffic. When omitted, admission supplies the
complete static default. An explicit empty object opts out; any nonempty object is
completed with omitted inputTokens, outputTokens and load defaults. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 4c073e979aee..6b6cfa4b921c 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -10,12 +10,12 @@ subtitle: Full Kubernetes CRD + operator-config API reference, generated from th > [!WARNING] > **Auto-generated from source.** This page is regenerated from the Dynamo operator CRDs by `docs/fern/scripts/gen_kubernetes_api.py` and covers **every** type across all three API packages: the deprecated `nvidia.com/v1alpha1` surface, the supported `nvidia.com/v1beta1` surface, and the operator's own `operator.config.dynamo.nvidia.com/v1alpha1` configuration. The trimmed [DGD](dynamo-graph-deployment.mdx), [DGDR](dynamo-graph-deployment-request.mdx), and [DCD](dynamo-component-deployment.mdx) references cover only user-facing `v1beta1` fields. To edit the surface, change the Go types under `deploy/operator/api/` and let CI regenerate this page. -Dynamo publishes 4 Kubernetes API packages with 208 typed CRD and config sections. +Dynamo publishes 4 Kubernetes API packages with 211 typed CRD and config sections. 71 types 76 types -32 types +35 types 29 types @@ -4523,25 +4523,30 @@ Package v1beta2 contains API Schema definitions for the nvidia.com v1beta2 API g - -AISimulateSpec selects the AISimulate execution artifact and supplies native input. + +AISimulateRunStatus reports the exact public recommendation input used for this execution. **Kind:** `type` -**Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) +**Appears in:** [DynamoGraphDeploymentRunStatus](#dynamographdeploymentrunstatus) - -Image selects a Dynamo-compatible AISimulate runner. When omitted, admission persists the operator's configured AISimulate image, or the Dynamo planner image of the operator's release, together with the AISimulate version it bundles. -**Validation:** MinLength: 1 Optional: \{\} + +Input is the fully lowered AISimulate input after typed fields, derived defaults and spec.aisimulate.inputOverrides have been merged. Field names and nesting use the upstream AISimulate JSON contract. +See [RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg). +**Validation:** Type: object Optional: \{\} + - -nightly or custom image), and it wins over the mapping. -**Validation:** MinLength: 1 Pattern: `^(0|[1-9][0-9]\{0,3\})\.(0|[1-9][0-9]\{0,3\})\.(0|[1-9][0-9]\{0,3\})$` Optional: \{\} - + +AISimulateSpec supplies public recommendation input to the AISimulate version in the operator's configured +planner image. - -Parameters contains native AISimulate input. Field names and nesting use the upstream AISimulate JSON contract, including snake_case names. + +**Kind:** `type` +**Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) + + +InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered field. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. See [RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg). **Validation:** Type: object Optional: \{\} @@ -4806,7 +4811,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from admission. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials nor a KV-capacity-fraction search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -4818,7 +4823,7 @@ See [ObjectiveSpec](#objectivespec). -AISimulate selects the execution artifact and supplies native AISimulate input. +AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. See [AISimulateSpec](#aisimulatespec). **Validation:** Optional: \{\} @@ -4888,7 +4893,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from admission. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials nor a KV-capacity-fraction search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -4900,7 +4905,7 @@ See [ObjectiveSpec](#objectivespec). -AISimulate selects the execution artifact and supplies native AISimulate input. +AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. See [AISimulateSpec](#aisimulatespec). **Validation:** Optional: \{\} @@ -5010,7 +5015,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from admission. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials nor a KV-capacity-fraction search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5022,7 +5027,7 @@ See [ObjectiveSpec](#objectivespec). -AISimulate selects the execution artifact and supplies native AISimulate input. +AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. See [AISimulateSpec](#aisimulatespec). **Validation:** Optional: \{\} @@ -5059,6 +5064,12 @@ See [DGDRRunProgress](#dgdrrunprogress). **Validation:** Optional: \{\} + +AISimulate reports the effective input executed by AISimulate. +See [AISimulateRunStatus](#aisimulaterunstatus). +**Validation:** Optional: \{\} + + CandidateRefs identifies the current bounded DGDC projection. List order represents rank, best first, for scalar searches. See [LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core). @@ -5158,15 +5169,41 @@ See [GPUHardwareSpec](#gpuhardwarespec). - -KVLoadRatio is either a pinned load ratio or a minimum/maximum range. -AISimulate performs mode-specific validation. + +KVCapacityFractionRange defines the searched fraction of a candidate's KV capacity. **Kind:** `type` -**Underlying type:** `float array` -**Validation:** MaxItems: 2; MinItems: 1; items:Maximum: 1; items:Minimum: 0 -**Appears in:** [StaticWorkloadSpec](#staticworkloadspec) +**Appears in:** [KVCapacityFractionSpec](#kvcapacityfractionspec) + + +Min is the lower end of the searched fraction range. +**Validation:** ExclusiveMinimum: true Maximum: 1 Minimum: 0 + + + +Max is the upper end of the searched fraction range. +**Validation:** ExclusiveMinimum: true Maximum: 1 Minimum: 0 + + + + +KVCapacityFractionSpec selects either one fixed fraction or a searched range. + + +**Kind:** `type` +**Appears in:** [StaticLoadSpec](#staticloadspec) + + +Value uses a fixed fraction of the candidate's KV capacity. A value of 1 uses its full estimated capacity. +**Validation:** ExclusiveMinimum: true Maximum: 1 Minimum: 0 Optional: \{\} + + + +Range searches fractions between min and max. +See [KVCapacityFractionRange](#kvcapacityfractionrange). +**Validation:** Optional: \{\} + @@ -5209,7 +5246,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies a static shape of ISL 1024, OSL 128 and concurrency 10; an explicit empty workload receives the same shape from admission. An explicit workload.static: \{\} opts out of that shape, leaving the workload to spec.aisimulate.parameters.workload. spec.aisimulate.parameters.workload overrides the fields it names. The default evaluates a fixed concurrency, so it creates neither sweep.max_trials nor a KV-load ratio search. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials nor a KV-capacity-fraction search. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5221,7 +5258,7 @@ See [ObjectiveSpec](#objectivespec). -AISimulate selects the execution artifact and supplies native AISimulate input. +AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. See [AISimulateSpec](#aisimulatespec). **Validation:** Optional: \{\} @@ -5302,13 +5339,13 @@ See [ObjectiveMetric](#objectivemetric). -Metrics lists two or more distinct scalar targets when mode is Pareto. +Metrics selects AISimulate's public Pareto objective pair when mode is Pareto. The configured AISimulate release supports exactly ThroughputPerGPU and ThroughputPerUser through its public recommendation input. See [ObjectiveMetric](#objectivemetric). -**Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] MaxItems: 7 MinItems: 2 Optional: \{\} +**Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] MaxItems: 2 MinItems: 2 Optional: \{\} -SLA defines per-request latency bounds used to calculate goodput. When omitted, no bounds apply. spec.aisimulate.parameters.goal.sla overrides the bounds it names. +SLA defines per-request latency bounds used to calculate goodput. When omitted, no bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. See [ObjectiveSLASpec](#objectiveslaspec). **Validation:** Optional: \{\} @@ -5361,42 +5398,52 @@ Total is the number of planned rounds. - -StaticWorkloadSpec defines one of AISimulate's synthetic workload shapes. + +StaticLoadSpec selects exactly one synthetic load model by field presence. **Kind:** `type` -**Appears in:** [WorkloadSpec](#v1beta2-workloadspec) +**Appears in:** [StaticWorkloadSpec](#staticworkloadspec) - -ISL is the input sequence length in tokens. -**Validation:** Minimum: 1 Optional: \{\} + +RequestsPerSecond is a fixed open-loop arrival rate. +**Validation:** ExclusiveMinimum: true Minimum: 0 Optional: \{\} - -OSL is the output sequence length in tokens. + +Concurrency is a fixed closed-loop in-flight request cap. **Validation:** Minimum: 1 Optional: \{\} - -NumRequestRatio sets request count relative to the selected load. -**Validation:** ExclusiveMinimum: true Minimum: 0 Optional: \{\} + +KVCapacityFraction derives concurrency from each candidate's estimated KV capacity. +See [KVCapacityFractionSpec](#kvcapacityfractionspec). +**Validation:** Optional: \{\} + - -RequestRate is a fixed open-loop arrival rate in requests per second. -**Validation:** ExclusiveMinimum: true Minimum: 0 Optional: \{\} + +StaticWorkloadSpec defines AISimulate synthetic traffic. Admission completes a nonempty +object with the omitted static defaults; an explicit empty object opts out of them. + + +**Kind:** `type` +**Appears in:** [WorkloadSpec](#v1beta2-workloadspec) + + +InputTokens is the synthetic request input length in tokens. +**Validation:** Minimum: 1 Optional: \{\} - -Concurrency is a fixed closed-loop in-flight request cap. + +OutputTokens is the synthetic request output length in tokens. **Validation:** Minimum: 1 Optional: \{\} - -KVLoadRatio derives concurrency from candidate KV capacity. A two-value range requires a bounded AISimulate sweep. -See [KVLoadRatio](#kvloadratio). -**Validation:** MaxItems: 2 MinItems: 1 items:Maximum: 1 items:Minimum: 0 Optional: \{\} + +Load selects the synthetic load model. When omitted from a nonempty static workload, admission defaults it to concurrency 10. +See [StaticLoadSpec](#staticloadspec). +**Validation:** Optional: \{\} @@ -5408,7 +5455,7 @@ WorkloadSpec defines the synthetic traffic model used for every candidate evalua **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -Static defines one of AISimulate's synthetic workload shapes. Its fields are not defaulted individually: the default belongs to spec.workload as a whole, so an explicit workload object keeps exactly the fields it names. +Static defines AISimulate synthetic traffic. When omitted, admission supplies the complete static default. An explicit empty object opts out; any nonempty object is completed with omitted inputTokens, outputTokens and load defaults. See [StaticWorkloadSpec](#staticworkloadspec). **Validation:** Optional: \{\} From 105febadca3552dd12ae34585bbe2fe7f67b27dc Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 09:13:18 +0100 Subject: [PATCH 008/100] pr-16661: pr-4: enforce v1beta2 admission boundaries Signed-off-by: Dr. Stefan Schimanski (cherry picked from commit 9ed993fc070f33cb91e84364bd3e97b75869db4b) --- .../dynamographdeploymentrequest_types.go | 38 ++-- ...dia.com_dynamographdeploymentrequests.yaml | 34 ++-- .../nvidia.com_dynamographdeploymentruns.yaml | 34 ++-- ...deploymentrequest_v1beta2_hardware_test.go | 13 +- ...ographdeploymentrequest_v1beta2_helpers.go | 36 ++-- ...namographdeploymentrequest_v1beta2_test.go | 10 +- ...hdeploymentrequest_carrier_envtest_test.go | 1 + .../dynamographdeploymentrequest_v1beta2.go | 19 +- ...ploymentrequest_v1beta2_input_overrides.go | 72 ++++++++ ...namographdeploymentrequest_v1beta2_test.go | 5 - ...ploymentrequest_validation_envtest_test.go | 168 ++++++++++++++++-- .../webhook/validation/suite_envtest_test.go | 10 ++ .../additional-resources/api-reference-k8s.md | 25 +-- .../kubernetes-api/full-api-reference.mdx | 27 +-- 14 files changed, 369 insertions(+), 123 deletions(-) create mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 0b28070cb360..9b0303947b74 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -143,13 +143,15 @@ type WorkloadSpec struct { Static *StaticWorkloadSpec `json:"static,omitempty"` } -// ObjectiveMode selects scalar optimization or a multi-objective Pareto search. -// +kubebuilder:validation:Enum=Optimize;Pareto +// ObjectiveMode selects the optimization mode. Pareto is reserved for a future +// release and is not admitted until Candidate publication supports Pareto fronts. +// +kubebuilder:validation:Enum=Optimize type ObjectiveMode string const ( ObjectiveModeOptimize ObjectiveMode = "Optimize" - ObjectiveModePareto ObjectiveMode = "Pareto" + // ObjectiveModePareto is reserved for the future Pareto API implementation. + ObjectiveModePareto ObjectiveMode = "Pareto" ) // ObjectiveMetric is a metric optimized by AISimulate. @@ -179,13 +181,11 @@ type ObjectiveSLASpec struct { E2EMs *float64 `json:"e2eMs,omitempty"` } -// ObjectiveSpec defines scalar optimization or a multi-objective Pareto search. -// +kubebuilder:validation:XValidation:rule="!has(self.mode) || (self.mode == 'Optimize' && !has(self.metrics)) || (self.mode == 'Pareto' && !has(self.metric) && has(self.metrics))",message="Optimize accepts only metric; Pareto requires metrics" -// +kubebuilder:validation:XValidation:rule="!has(self.metrics) || (size(self.metrics) == 2 && self.metrics.exists(m, m == 'ThroughputPerGPU') && self.metrics.exists(m, m == 'ThroughputPerUser'))",message="Pareto supports exactly ThroughputPerGPU and ThroughputPerUser" +// ObjectiveSpec defines scalar optimization. Pareto fields are reserved for a future release. +// +kubebuilder:validation:XValidation:rule="!has(self.metrics)",message="metrics is reserved for the future Pareto implementation" // +kubebuilder:validation:XValidation:rule="!(has(self.metric) && self.metric in ['Goodput', 'GoodputPerGPU']) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))",message="Goodput and GoodputPerGPU require at least one SLA bound" -// +kubebuilder:validation:XValidation:rule="!(has(self.metrics) && self.metrics.exists(m, m in ['Goodput', 'GoodputPerGPU'])) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))",message="Pareto metrics containing Goodput or GoodputPerGPU require at least one SLA bound" type ObjectiveSpec struct { - // Mode selects scalar optimization or a Pareto search. + // Mode selects scalar optimization. Only Optimize is currently supported. // +optional Mode ObjectiveMode `json:"mode,omitempty"` @@ -194,9 +194,7 @@ type ObjectiveSpec struct { // +optional Metric *ObjectiveMetric `json:"metric,omitempty"` - // Metrics selects AISimulate's public Pareto objective pair when mode is Pareto. - // The configured AISimulate release supports exactly ThroughputPerGPU and - // ThroughputPerUser through its public recommendation input. + // Metrics is reserved for the future Pareto implementation. // +optional // +kubebuilder:validation:MinItems=2 // +kubebuilder:validation:MaxItems=2 @@ -215,8 +213,14 @@ type AISimulateSpec struct { // InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from // the typed fields. It uses the upstream AISimulate JSON contract, including // snake_case names. Values here take precedence, and null deletes a lowered - // field. The operator runs the AISimulate version from its configured planner - // image; Requests and Runs cannot select a different image or version. + // field. The dgdr_run member must remain an object, and its operator-owned + // transport, identity, and runtime fields name, max_candidates, snapshot_dir, + // runtime_image, runtime_version_override, derive_runtime_images, + // num_gpus_per_node, and namespace are operator-owned. Request admission + // rejects them, and the Run controller enforces its values while lowering the + // immutable snapshot. The operator runs the AISimulate version from its + // configured planner image; Requests and Runs cannot select a different image + // or version. // +optional // +kubebuilder:pruning:PreserveUnknownFields // +kubebuilder:validation:Type=object @@ -226,7 +230,7 @@ type AISimulateSpec struct { // RecommendationSpec controls bounded projection into DGDC resources. type RecommendationSpec struct { // MaxCandidates is the maximum number of DGDCs owned by one run. Scalar - // candidates are ordered best first; Pareto candidates have no scalar rank. + // candidates are ordered best first. // This field does not limit internal trials. // +optional // +kubebuilder:validation:Minimum=1 @@ -263,9 +267,9 @@ type NativeSearchInput struct { // +optional Workload *WorkloadSpec `json:"workload,omitempty"` - // Objective defines scalar optimization or a multi-objective Pareto search. - // When omitted, admission supplies the object. Its mode defaults to Optimize, and admission - // supplies the conditional scalar metric. + // Objective defines scalar optimization. When omitted, + // admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults + // to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. // +optional Objective *ObjectiveSpec `json:"objective,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 164a5cb025d0..cb332d851982 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9605,8 +9605,14 @@ spec: InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered - field. The operator runs the AISimulate version from its configured planner - image; Requests and Runs cannot select a different image or version. + field. The dgdr_run member must remain an object, and its operator-owned + transport, identity, and runtime fields name, max_candidates, snapshot_dir, + runtime_image, runtime_version_override, derive_runtime_images, + num_gpus_per_node, and namespace are operator-owned. Request admission + rejects them, and the Run controller enforces its values while lowering the + immutable snapshot. The operator runs the AISimulate version from its + configured planner image; Requests and Runs cannot select a different image + or version. type: object x-kubernetes-preserve-unknown-fields: true type: object @@ -9659,9 +9665,9 @@ spec: type: object objective: description: |- - Objective defines scalar optimization or a multi-objective Pareto search. - When omitted, admission supplies the object. Its mode defaults to Optimize, and admission - supplies the conditional scalar metric. + Objective defines scalar optimization. When omitted, + admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults + to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. properties: metric: description: |- @@ -9677,10 +9683,7 @@ spec: - GoodputPerGPU type: string metrics: - description: |- - Metrics selects AISimulate's public Pareto objective pair when mode is Pareto. - The configured AISimulate release supports exactly ThroughputPerGPU and - ThroughputPerUser through its public recommendation input. + description: Metrics is reserved for the future Pareto implementation. items: description: ObjectiveMetric is a metric optimized by AISimulate. enum: @@ -9697,10 +9700,9 @@ spec: type: array x-kubernetes-list-type: set mode: - description: Mode selects scalar optimization or a Pareto search. + description: Mode selects scalar optimization. Only Optimize is currently supported. enum: - Optimize - - Pareto type: string sla: description: |- @@ -9728,21 +9730,17 @@ spec: rule: '!has(self.e2eMs) || (!has(self.ttftMs) && !has(self.itlMs))' type: object x-kubernetes-validations: - - message: Optimize accepts only metric; Pareto requires metrics - rule: '!has(self.mode) || (self.mode == ''Optimize'' && !has(self.metrics)) || (self.mode == ''Pareto'' && !has(self.metric) && has(self.metrics))' - - message: Pareto supports exactly ThroughputPerGPU and ThroughputPerUser - rule: '!has(self.metrics) || (size(self.metrics) == 2 && self.metrics.exists(m, m == ''ThroughputPerGPU'') && self.metrics.exists(m, m == ''ThroughputPerUser''))' + - message: metrics is reserved for the future Pareto implementation + rule: '!has(self.metrics)' - message: Goodput and GoodputPerGPU require at least one SLA bound rule: '!(has(self.metric) && self.metric in [''Goodput'', ''GoodputPerGPU'']) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))' - - message: Pareto metrics containing Goodput or GoodputPerGPU require at least one SLA bound - rule: '!(has(self.metrics) && self.metrics.exists(m, m in [''Goodput'', ''GoodputPerGPU''])) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))' recommendation: description: Recommendation controls bounded projection into DGDC resources. properties: maxCandidates: description: |- MaxCandidates is the maximum number of DGDCs owned by one run. Scalar - candidates are ordered best first; Pareto candidates have no scalar rank. + candidates are ordered best first. This field does not limit internal trials. format: int32 maximum: 100 diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index c0cbeafc4fe4..1192395c057e 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -68,8 +68,14 @@ spec: InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered - field. The operator runs the AISimulate version from its configured planner - image; Requests and Runs cannot select a different image or version. + field. The dgdr_run member must remain an object, and its operator-owned + transport, identity, and runtime fields name, max_candidates, snapshot_dir, + runtime_image, runtime_version_override, derive_runtime_images, + num_gpus_per_node, and namespace are operator-owned. Request admission + rejects them, and the Run controller enforces its values while lowering the + immutable snapshot. The operator runs the AISimulate version from its + configured planner image; Requests and Runs cannot select a different image + or version. type: object x-kubernetes-preserve-unknown-fields: true type: object @@ -122,9 +128,9 @@ spec: type: object objective: description: |- - Objective defines scalar optimization or a multi-objective Pareto search. - When omitted, admission supplies the object. Its mode defaults to Optimize, and admission - supplies the conditional scalar metric. + Objective defines scalar optimization. When omitted, + admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults + to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. properties: metric: description: |- @@ -140,10 +146,7 @@ spec: - GoodputPerGPU type: string metrics: - description: |- - Metrics selects AISimulate's public Pareto objective pair when mode is Pareto. - The configured AISimulate release supports exactly ThroughputPerGPU and - ThroughputPerUser through its public recommendation input. + description: Metrics is reserved for the future Pareto implementation. items: description: ObjectiveMetric is a metric optimized by AISimulate. enum: @@ -160,10 +163,9 @@ spec: type: array x-kubernetes-list-type: set mode: - description: Mode selects scalar optimization or a Pareto search. + description: Mode selects scalar optimization. Only Optimize is currently supported. enum: - Optimize - - Pareto type: string sla: description: |- @@ -191,21 +193,17 @@ spec: rule: '!has(self.e2eMs) || (!has(self.ttftMs) && !has(self.itlMs))' type: object x-kubernetes-validations: - - message: Optimize accepts only metric; Pareto requires metrics - rule: '!has(self.mode) || (self.mode == ''Optimize'' && !has(self.metrics)) || (self.mode == ''Pareto'' && !has(self.metric) && has(self.metrics))' - - message: Pareto supports exactly ThroughputPerGPU and ThroughputPerUser - rule: '!has(self.metrics) || (size(self.metrics) == 2 && self.metrics.exists(m, m == ''ThroughputPerGPU'') && self.metrics.exists(m, m == ''ThroughputPerUser''))' + - message: metrics is reserved for the future Pareto implementation + rule: '!has(self.metrics)' - message: Goodput and GoodputPerGPU require at least one SLA bound rule: '!(has(self.metric) && self.metric in [''Goodput'', ''GoodputPerGPU'']) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))' - - message: Pareto metrics containing Goodput or GoodputPerGPU require at least one SLA bound - rule: '!(has(self.metrics) && self.metrics.exists(m, m in [''Goodput'', ''GoodputPerGPU''])) || (has(self.sla) && (has(self.sla.ttftMs) || has(self.sla.itlMs) || has(self.sla.e2eMs)))' recommendation: description: Recommendation controls bounded projection into DGDC resources. properties: maxCandidates: description: |- MaxCandidates is the maximum number of DGDCs owned by one run. Scalar - candidates are ordered best first; Pareto candidates have no scalar rank. + candidates are ordered best first. This field does not limit internal trials. format: int32 maximum: 100 diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index d4cbbc621047..c797d4b6ec23 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -57,16 +57,21 @@ func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { hardwareOfV1beta2(str("h200_sxm"), nil), h200x64, nil, "", 32, 1, "h200_sxm", ""}, {"a budget alone keeps the budget and discovers the SKU", hardwareOfV1beta2(nil, num(4)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, {"an explicit budget above the cluster size is kept", hardwareOfV1beta2(nil, num(64)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, - {"an explicit budget above the cap is kept", hardwareOfV1beta2(str("h200_sxm"), num(64)), h200x8, nil, "", 0, 0, "", ""}, - {"a SKU and a budget never reach the cluster", hardwareOfV1beta2(str("h100_sxm"), num(4)), gpu.GPUInfo{}, errors.New("down"), "", 0, 0, "", ""}, + {"an explicit budget above the cap is kept", hardwareOfV1beta2(str("h200_sxm"), num(64)), h200x8, nil, "", 0, 1, "h200_sxm", ""}, + {"a SKU and a budget verify the SKU", hardwareOfV1beta2(str("h100_sxm"), num(4)), + gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + nil, "", 0, 1, "h100_sxm", ""}, + {"complete hardware rejects failed verification", hardwareOfV1beta2(str("h100_sxm"), num(4)), + gpu.GPUInfo{}, errors.New("down"), "", 0, 1, "h100_sxm", + `spec.hardware.gpu.sku: Invalid value: "h100_sxm": could not be verified against the cluster's GPUs (down)`}, {"a failed discovery rejects", nil, gpu.GPUInfo{}, errors.New("no GPUs found"), "", 0, 1, "", "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found)"}, {"a GPU model the catalog does not know rejects", nil, gpu.GPUInfo{Model: "Mystery GPU", GPUsPerNode: 8, NodesWithGPUs: 1}, nil, "", 0, 1, "", `spec.hardware.gpu.sku: Required value: the cluster's GPU model "Mystery GPU" maps to no AISimulate SKU`}, - {"no GPUs of the SKU reject the missing budget", hardwareOfV1beta2(str("h100_sxm"), nil), + {"no GPUs of the SKU reject the SKU", hardwareOfV1beta2(str("h100_sxm"), nil), gpu.GPUInfo{System: "h100_sxm"}, nil, "", 0, 1, "h100_sxm", - `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h100_sxm" were found in the cluster`}, + `spec.hardware.gpu.sku: Invalid value: "h100_sxm": no GPUs of this SKU were found in the cluster`}, {"a heterogeneous cluster rejects an omitted SKU", nil, gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, nil, "", 0, 1, "", diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index c0f04cef3037..39e1e37226e9 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -79,8 +79,8 @@ func (o DGDRV1beta2Options) resolveDefaults( return dynamoGraphDeploymentRequestV1beta2DefaultOptions{hardware: hardware}, nil } -// resolveHardware returns the SKU and budget the typed spec omits. It asks the cluster only -// when something is missing, restricted to the typed SKU when there is one. +// resolveHardware returns the SKU and budget the typed spec omits and verifies an explicit SKU +// against the cluster. Discovery is restricted to the typed SKU when there is one. // // Discovery runs on CREATE and on an UPDATE that changes spec.hardware. A Request stays // mutable and every Run snapshots it, so a caller may change or remove typed hardware before @@ -94,7 +94,7 @@ func (o DGDRV1beta2Options) resolveDefaults( // cluster's AISimulate SKU, and an omitted budget becomes min(matching GPUs, cap). An explicit // budget is authoritative: it is never capped, whatever the cluster holds. What the cluster // cannot supply is rejected on the field that fixes it: a failed discovery, a GPU model the -// AISimulate catalog does not know, and a SKU without any matching GPU. +// AISimulate catalog does not know, and an explicit SKU without any matching GPU. func (o DGDRV1beta2Options) resolveHardware( ctx context.Context, operation admissionv1.Operation, @@ -113,10 +113,6 @@ func (o DGDRV1beta2Options) resolveHardware( } hasBudget = spec.Hardware.GPU.Budget != nil } - if sku != "" && hasBudget { - return hardwareDefaultsV1beta2{}, nil - } - timeout := o.HardwareDiscoveryTimeout if timeout <= 0 { timeout = defaultHardwareDiscoveryTimeout @@ -127,6 +123,13 @@ func (o DGDRV1beta2Options) resolveHardware( gpuPath := field.NewPath("spec", "hardware", "gpu") found, err := o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) if err != nil { + if sku != "" { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Invalid( + gpuPath.Child("sku"), + sku, + fmt.Sprintf("could not be verified against the cluster's GPUs (%v)", err), + )) + } return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath, fmt.Sprintf( "the cluster's GPUs could not be discovered (%v); set spec.hardware.gpu.sku and spec.hardware.gpu.budget", err))) } @@ -150,12 +153,21 @@ func (o DGDRV1beta2Options) resolveHardware( } defaults.sku = sku } + + // A filtered discovery with no GPUs means the caller pinned a SKU the cluster cannot run. + var gpus int + if found != nil { + // TODO(#17192): Use the discovered aggregate GPU count so mixed node sizes of one SKU are counted correctly. + gpus = found.GPUsPerNode * found.NodesWithGPUs + } + if spec.Hardware != nil && spec.Hardware.GPU != nil && spec.Hardware.GPU.SKU != nil && gpus <= 0 { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Invalid( + gpuPath.Child("sku"), + sku, + "no GPUs of this SKU were found in the cluster", + )) + } if !hasBudget { - var gpus int - if found != nil { - // TODO(#17192): Use the discovered aggregate GPU count so mixed node sizes of one SKU are counted correctly. - gpus = found.GPUsPerNode * found.NodesWithGPUs - } if gpus <= 0 { return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("budget"), fmt.Sprintf( "no GPUs of SKU %q were found in the cluster; set spec.hardware.gpu.budget", sku))) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index 60678ebe0d61..75164cc1cf19 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -64,13 +64,13 @@ func TestResolveHardwareV1beta2(t *testing.T) { t.Fatalf("got %+v", got) } }) - t.Run("complete hardware never asks the cluster", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} + t.Run("complete hardware verifies its SKU against the cluster", func(t *testing.T) { + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) if err != nil { t.Fatal(err) } - if d.calls != 0 || !(got == hardwareDefaultsV1beta2{}) { + if d.calls != 1 || d.filter != "b200_sxm" || !(got == hardwareDefaultsV1beta2{}) { t.Fatalf("calls=%d got=%+v", d.calls, got) } }) @@ -240,9 +240,9 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { {"removing all typed hardware discovers both", admissionv1.Update, full(), hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, - {"changing to complete hardware does not discover", admissionv1.Update, + {"changing to complete hardware verifies the new SKU", admissionv1.Update, full(), hardwareSpecV1beta2(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2))), - hardwareDefaultsV1beta2{}, 0, ""}, + hardwareDefaultsV1beta2{}, 1, "b200_sxm"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go index f8f8eac32574..2503a0da41e2 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -31,6 +31,7 @@ func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, c t.Helper() env := admissionEnv.ForTest(t) admissionGate.set(features.Gates{GPUDiscovery: true}) + setAdmissionH100Topology() warnings := &warningRecorder{} ownerClient := newAdmissionResourceClient(t, env, ownerFixture, "", warnings) otherClient := newAdmissionResourceClient(t, env, otherFixture, "", warnings) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index 0a7c411af443..ca5093f39251 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -39,12 +39,25 @@ func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( if err := json.Unmarshal(raw.Raw, &spec); err != nil { return field.ErrorList{field.Invalid(fldPath, "", "must be a valid v1beta2 spec: "+err.Error())} } + + allErrs := validateDynamoGraphDeploymentRequestSpecV1beta2(&spec, field.NewPath("spec")) if _, err := aisimulate.Lower(&spec.NativeSearchInput); err != nil { - return field.ErrorList{field.Invalid( + allErrs = append(allErrs, field.Invalid( field.NewPath("spec"), "", "cannot be lowered to the public AISimulate recommendation input: "+err.Error(), - )} + )) + } + return allErrs +} + +// validateDynamoGraphDeploymentRequestSpecV1beta2 validates spec. spec and fldPath must not be nil. +func validateDynamoGraphDeploymentRequestSpecV1beta2( + spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, + fldPath *field.Path, +) field.ErrorList { + if spec.AISimulate == nil { + return nil } - return nil + return validateAISimulateSpec(spec.AISimulate, fldPath.Child("aisimulate")) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go new file mode 100644 index 000000000000..bd955a3e54b5 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go @@ -0,0 +1,72 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/aisimulate" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// validateAISimulateSpec validates the Request's native AISimulate overrides. +func validateAISimulateSpec( + input *nvidiacomv1beta2.AISimulateSpec, + fldPath *field.Path, +) field.ErrorList { + // Use the lowering package's decoder as the single interpretation of RawExtension. + overrides, err := aisimulate.InputOverrides(&nvidiacomv1beta2.NativeSearchInput{AISimulate: input}) + if err != nil || overrides == nil { + // The lowering validation reports malformed override documents once. + return nil + } + + dgdrRunValue, found := overrides["dgdr_run"] + if !found { + return nil + } + dgdrRun, ok := dgdrRunValue.(map[string]any) + if !ok { + return field.ErrorList{field.Forbidden( + fldPath.Child("inputOverrides", "dgdr_run"), + "must be an object because the operator owns required execution fields within it", + )} + } + + // Search overrides remain authoritative; only execution values coupled to the + // controller-owned Job and shared volume are reserved. + reserved := []string{ + "derive_runtime_images", + "max_candidates", + "name", + "namespace", + "num_gpus_per_node", + "runtime_image", + "runtime_version_override", + "snapshot_dir", + } + allErrs := field.ErrorList{} + for _, key := range reserved { + if _, found := dgdrRun[key]; found { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("inputOverrides", "dgdr_run", key), + "is managed by the operator and cannot be overridden", + )) + } + } + return allErrs +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go index 3a65159fa1b6..981129efb985 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go @@ -58,9 +58,4 @@ func TestValidateEnvelopeV1beta2(t *testing.T) { v1beta2Envelope(`{"model":{"name":"meta-llama/Llama-3.1-8B-Instruct"}}`), path)) v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") - - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{ - "model":{"name":"meta-llama/Llama-3.1-8B-Instruct"}, - "objective":{"mode":"Pareto","metrics":["Throughput","TTFT"]} - }`), path), "cannot be lowered to the public AISimulate recommendation input") } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 7d8cc625b934..bbed17e865ac 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -60,6 +60,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }` discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} + discoveredH100 := gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} tests := []struct { name string @@ -115,8 +116,9 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, }, { - name: "v1beta2 create bypasses legacy v1beta1 validation", - request: beta2DGDRForAdmission(setBeta2AdmissionHardware), + name: "v1beta2 create bypasses legacy v1beta1 validation", + request: beta2DGDRForAdmission(setBeta2AdmissionHardware), + discovered: discoveredH100, }, { name: "v1beta2 update bypasses legacy v1beta1 validation", @@ -125,6 +127,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { setBeta2AdmissionHardware(request) request.Labels = map[string]string{"updated": "true"} }), + discovered: discoveredH100, }, { name: "v1beta1 empty native representation is rejected by envelope validation", @@ -558,7 +561,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 explicit hardware is kept", + name: "v1beta2 explicit hardware is kept after admission verifies the SKU", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} }, nil), @@ -573,6 +576,17 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { } }, }, + { + name: "v1beta2 explicit hardware is rejected when the SKU cannot be verified", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} + }, nil), + gpuDiscovery: true, + discoveryErr: errors.New("no matching GPUs found"), + wantWebhook: []string{ + `spec.hardware.gpu.sku: Invalid value: "h100_sxm": could not be verified against the cluster's GPUs (no matching GPUs found)`, + }, + }, { name: "v1beta2 input overrides are stored untouched and never decide hardware defaulting", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -742,6 +756,82 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { } }, }, + { + name: "v1beta2 public input overrides may configure non-reserved dgdr run fields", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2InputOverrides(spec, map[string]any{"dgdr_run": map[string]any{ + "generator": "aic", + "snapshot_interval_seconds": int64(30), + }}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "aisimulate", "inputOverrides", "dgdr_run", "generator"); got != "aic" { + t.Fatalf("dgdr_run.generator = %q, want aic", got) + } + }, + }, + { + name: "v1beta2 operator-owned dgdr run input cannot be overridden", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2InputOverrides(spec, map[string]any{"dgdr_run": map[string]any{ + "derive_runtime_images": false, + "max_candidates": int64(3), + "name": "foreign-run", + "namespace": "other", + "num_gpus_per_node": int64(4), + "runtime_image": "example.invalid/runtime:latest", + "runtime_version_override": "1.2.3", + "snapshot_dir": "/tmp/snapshots", + }}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.inputOverrides.dgdr_run.derive_runtime_images: Forbidden: is managed by the operator and cannot be overridden", + "spec.aisimulate.inputOverrides.dgdr_run.max_candidates: Forbidden: is managed by the operator and cannot be overridden", + "spec.aisimulate.inputOverrides.dgdr_run.name: Forbidden: is managed by the operator and cannot be overridden", + "spec.aisimulate.inputOverrides.dgdr_run.namespace: Forbidden: is managed by the operator and cannot be overridden", + "spec.aisimulate.inputOverrides.dgdr_run.num_gpus_per_node: Forbidden: is managed by the operator and cannot be overridden", + "spec.aisimulate.inputOverrides.dgdr_run.runtime_image: Forbidden: is managed by the operator and cannot be overridden", + "spec.aisimulate.inputOverrides.dgdr_run.runtime_version_override: Forbidden: is managed by the operator and cannot be overridden", + "spec.aisimulate.inputOverrides.dgdr_run.snapshot_dir: Forbidden: is managed by the operator and cannot be overridden", + }, + }, + { + name: "v1beta2 dgdr run input cannot be deleted", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2InputOverrides(spec, map[string]any{"dgdr_run": nil}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.inputOverrides.dgdr_run: Forbidden: must be an object because the operator owns required execution fields within it", + }, + }, + { + name: "v1beta2 dgdr run input cannot be replaced by a scalar", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2InputOverrides(spec, map[string]any{"dgdr_run": "wrong"}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.inputOverrides.dgdr_run: Forbidden: must be an object because the operator owns required execution fields within it", + }, + }, + { + name: "v1beta2 dgdr run input cannot be replaced by a list", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2InputOverrides(spec, map[string]any{"dgdr_run": []any{"wrong"}}) + }, nil), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.inputOverrides.dgdr_run: Forbidden: must be an object because the operator owns required execution fields within it", + }, + }, { name: "v1beta2 hardware omitted is discovered and the budget capped", request: v1beta2DGDRForAdmission(nil, nil), @@ -785,12 +875,12 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 a SKU and a budget are kept without reading the cluster", + name: "v1beta2 a SKU and a budget are kept after SKU verification", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} }, nil), gpuDiscovery: true, - discoveryErr: errors.New("the cluster must not be read"), + discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h100_sxm", 4) }, @@ -858,8 +948,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, discovered: gpu.GPUInfo{System: "h100_sxm"}, wantWebhook: []string{ - `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h100_sxm" were found in the cluster; ` + - "set spec.hardware.gpu.budget", + `spec.hardware.gpu.sku: Invalid value: "h100_sxm": no GPUs of this SKU were found in the cluster`, }, }, @@ -877,24 +966,22 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, { - name: "v1beta2 pareto objective receives no scalar metric", + name: "v1beta2 pareto objective is reserved until Candidate publication supports it", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"ThroughputPerGPU", "ThroughputPerUser"}} }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - v1beta2Absent(t, result, "spec", "objective", "metric") - }, + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.objective.mode: Unsupported value: "Pareto": supported values: "Optimize"`, }, { - name: "v1beta2 pareto rejects objectives absent from the public AISimulate input", + name: "v1beta2 optimize rejects the reserved Pareto metrics field", request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"TTFT", "Throughput"}} + spec["objective"] = map[string]any{"mode": "Optimize", "metrics": []any{"ThroughputPerGPU", "ThroughputPerUser"}} }, nil), gpuDiscovery: true, discovered: discoveredH200, - wantCELErr: `spec.objective: Invalid value: Pareto supports exactly ThroughputPerGPU and ThroughputPerUser`, + wantCELErr: `spec.objective: Invalid value: metrics is reserved for the future Pareto implementation`, }, { name: "v1beta2 explicit values are all kept", @@ -1082,6 +1169,40 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { } } +func TestDynamoGraphDeploymentRunSpecIsImmutable(t *testing.T) { + tests := []struct { + name string + oldRun *nvidiacomv1beta2.DynamoGraphDeploymentRun + run *nvidiacomv1beta2.DynamoGraphDeploymentRun + wantCELErr string + }{ + { + name: "metadata-only update keeps the immutable spec", + oldRun: v1beta2RunForAdmission(v1beta2AdmissionModel, nil), + run: v1beta2RunForAdmission(v1beta2AdmissionModel, map[string]string{"updated": "true"}), + }, + { + name: "spec update is rejected by the API server", + oldRun: v1beta2RunForAdmission(v1beta2AdmissionModel, nil), + run: v1beta2RunForAdmission(alternateAdmissionModel, nil), + wantCELErr: `: Invalid value: spec is immutable`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := runAdmissionTest(t, admissionTestCase{ + object: tt.run, + oldObject: tt.oldRun, + wantCELError: tt.wantCELErr, + }) + if tt.wantCELErr == "" && actual.GetGeneration() != 1 { + t.Fatalf("generation = %d, want 1 for a metadata-only update", actual.GetGeneration()) + } + }) + } +} + func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing.T) { tests := []struct { name string @@ -1132,6 +1253,7 @@ func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing. t.Log("Create a request through the production API and conversion path") env := admissionEnv.ForTest(t) admissionGate.set(features.Gates{GPUDiscovery: true}) + setAdmissionH100Topology() warnings := &warningRecorder{} createClient := newAdmissionResourceClient(t, env, test.createObject, "", warnings) object, _ := admissionObject(t, test.createObject, env.Namespace(), nil) @@ -1163,6 +1285,7 @@ func TestDynamoGraphDeploymentRequestV1Beta2EmptyStatusSurvivesV1Alpha1(t *testi t.Log("Create a native v1beta2 request with empty status") env := admissionEnv.ForTest(t) admissionGate.set(features.Gates{GPUDiscovery: true}) + setAdmissionH100Topology() warnings := &warningRecorder{} beta2Fixture := beta2DGDRForAdmission(func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { request.Name = "empty-status-alpha-bridge" @@ -1274,6 +1397,19 @@ func beta2DGDRForAdmission( return request } +func v1beta2RunForAdmission(model string, labels map[string]string) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + TypeMeta: metav1.TypeMeta{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRun", + }, + ObjectMeta: metav1.ObjectMeta{Name: "test-v1beta2-dgdr-run", Namespace: "default", Labels: labels}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: model}, + }}, + } +} + const v1beta2AdmissionModel = "Qwen/Qwen3-0.6B" // v1beta2NativeHardwareDGDR supplies hardware only through public AISimulate input overrides. diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 2e15b11a10c2..fd9b2f56792f 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -123,6 +123,12 @@ func (f *fakeHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUS return &found, f.err } +func setAdmissionH100Topology() { + admissionHardware.set(gpu.GPUInfo{ + System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, + }, nil) +} + type mutableFeatureGate struct { mu sync.RWMutex gates features.Gate @@ -298,6 +304,8 @@ func admissionResource(t *testing.T, kind string) string { return "dynamographdeployments" case "DynamoGraphDeploymentRequest": return "dynamographdeploymentrequests" + case "DynamoGraphDeploymentRun": + return "dynamographdeploymentruns" default: t.Fatalf("unsupported admission kind %q", kind) return "" @@ -324,6 +332,8 @@ func admissionGVK(t *testing.T, object runtime.Object) schema.GroupVersionKind { return nvidiacomv1beta1.DynamoGraphDeploymentRequestGVK case *nvidiacomv1beta2.DynamoGraphDeploymentRequest: return nvidiacomv1beta2.DynamoGraphDeploymentRequestGVK + case *nvidiacomv1beta2.DynamoGraphDeploymentRun: + return nvidiacomv1beta2.DynamoGraphDeploymentRunGVK default: t.Fatalf("unsupported admission object type %T", object) return schema.GroupVersionKind{} diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 9e5a5ad4e700..76a174f6c2d1 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3259,7 +3259,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `inputOverrides` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from
the typed fields. It uses the upstream AISimulate JSON contract, including
snake_case names. Values here take precedence, and null deletes a lowered
field. The operator runs the AISimulate version from its configured planner
image; Requests and Runs cannot select a different image or version. | | Type: object
Optional: \{\}
| +| `inputOverrides` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from
the typed fields. It uses the upstream AISimulate JSON contract, including
snake_case names. Values here take precedence, and null deletes a lowered
field. The dgdr_run member must remain an object, and its operator-owned
transport, identity, and runtime fields name, max_candidates, snapshot_dir,
runtime_image, runtime_version_override, derive_runtime_images,
num_gpus_per_node, and namespace are operator-owned. Request admission
rejects them, and the Run controller enforces its values while lowering the
immutable snapshot. The operator runs the AISimulate version from its
configured planner image; Requests and Runs cannot select a different image
or version. | | Type: object
Optional: \{\}
| #### Backend @@ -3454,7 +3454,7 @@ _Appears in:_ | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3496,7 +3496,7 @@ _Appears in:_ | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| | `v1beta1` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | V1Beta1 losslessly preserves a native v1beta1 spec while this object is
represented through v1beta2. It is mutually exclusive with native fields.
This field is a read-only conversion view; modify the request through the
native v1beta1 API. | | Type: object
Optional: \{\}
| @@ -3559,7 +3559,7 @@ _Appears in:_ | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3719,7 +3719,7 @@ _Appears in:_ | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| -| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization or a multi-objective Pareto search.
When omitted, admission supplies the object. Its mode defaults to Optimize, and admission
supplies the conditional scalar metric. | | Optional: \{\}
| +| `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3742,10 +3742,11 @@ _Appears in:_ _Underlying type:_ _string_ -ObjectiveMode selects scalar optimization or a multi-objective Pareto search. +ObjectiveMode selects the optimization mode. Pareto is reserved for a future +release and is not admitted until Candidate publication supports Pareto fronts. _Validation:_ -- Enum: [Optimize Pareto] +- Enum: [Optimize] _Appears in:_ - [ObjectiveSpec](#objectivespec) @@ -3753,7 +3754,7 @@ _Appears in:_ | Field | Description | | --- | --- | | `Optimize` | | -| `Pareto` | | +| `Pareto` | ObjectiveModePareto is reserved for the future Pareto API implementation.
| #### ObjectiveSLASpec @@ -3779,7 +3780,7 @@ _Appears in:_ -ObjectiveSpec defines scalar optimization or a multi-objective Pareto search. +ObjectiveSpec defines scalar optimization. Pareto fields are reserved for a future release. @@ -3791,9 +3792,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization or a Pareto search. | | Enum: [Optimize Pareto]
Optional: \{\}
| +| `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization. Only Optimize is currently supported. | | Enum: [Optimize]
Optional: \{\}
| | `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to Throughput. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| -| `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics selects AISimulate's public Pareto objective pair when mode is Pareto.
The configured AISimulate release supports exactly ThroughputPerGPU and
ThroughputPerUser through its public recommendation input. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 2
MinItems: 2
Optional: \{\}
| +| `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics is reserved for the future Pareto implementation. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 2
MinItems: 2
Optional: \{\}
| | `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. When omitted, no
bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. | | Optional: \{\}
| @@ -3813,7 +3814,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `maxCandidates` _integer_ | MaxCandidates is the maximum number of DGDCs owned by one run. Scalar
candidates are ordered best first; Pareto candidates have no scalar rank.
This field does not limit internal trials. | | Maximum: 100
Minimum: 1
Optional: \{\}
| +| `maxCandidates` _integer_ | MaxCandidates is the maximum number of DGDCs owned by one run. Scalar
candidates are ordered best first.
This field does not limit internal trials. | | Maximum: 100
Minimum: 1
Optional: \{\}
| #### ResourceReference diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 6b6cfa4b921c..97cf71ca53d4 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4546,7 +4546,7 @@ planner image. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered field. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. +InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered field. The dgdr_run member must remain an object, and its operator-owned transport, identity, and runtime fields name, max_candidates, snapshot_dir, runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the immutable snapshot. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. See [RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg). **Validation:** Type: object Optional: \{\} @@ -4817,7 +4817,7 @@ See [WorkloadSpec](#v1beta2-workloadspec). -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. +Objective defines scalar optimization. When omitted, admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -4899,7 +4899,7 @@ See [WorkloadSpec](#v1beta2-workloadspec). -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. +Objective defines scalar optimization. When omitted, admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5021,7 +5021,7 @@ See [WorkloadSpec](#v1beta2-workloadspec). -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. +Objective defines scalar optimization. When omitted, admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5252,7 +5252,7 @@ See [WorkloadSpec](#v1beta2-workloadspec). -Objective defines scalar optimization or a multi-objective Pareto search. When omitted, admission supplies the object. Its mode defaults to Optimize, and admission supplies the conditional scalar metric. +Objective defines scalar optimization. When omitted, admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. See [ObjectiveSpec](#objectivespec). **Validation:** Optional: \{\} @@ -5281,18 +5281,19 @@ ObjectiveMetric is a metric optimized by AISimulate. -ObjectiveMode selects scalar optimization or a multi-objective Pareto search. +ObjectiveMode selects the optimization mode. Pareto is reserved for a future +release and is not admitted until Candidate publication supports Pareto fronts. **Kind:** `enum` **Underlying type:** `string` -**Validation:** Enum: [Optimize Pareto] +**Validation:** Enum: [Optimize] **Appears in:** [ObjectiveSpec](#objectivespec) **Allowed values** - Optimize -- Pareto +- Pareto ObjectiveModePareto is reserved for the future Pareto API implementation. @@ -5320,16 +5321,16 @@ E2EMs is an alternative end-to-end bound in milliseconds. -ObjectiveSpec defines scalar optimization or a multi-objective Pareto search. +ObjectiveSpec defines scalar optimization. Pareto fields are reserved for a future release. **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -Mode selects scalar optimization or a Pareto search. +Mode selects scalar optimization. Only Optimize is currently supported. See [ObjectiveMode](#objectivemode). -**Validation:** Enum: [Optimize Pareto] Optional: \{\} +**Validation:** Enum: [Optimize] Optional: \{\} @@ -5339,7 +5340,7 @@ See [ObjectiveMetric](#objectivemetric). -Metrics selects AISimulate's public Pareto objective pair when mode is Pareto. The configured AISimulate release supports exactly ThroughputPerGPU and ThroughputPerUser through its public recommendation input. +Metrics is reserved for the future Pareto implementation. See [ObjectiveMetric](#objectivemetric). **Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] MaxItems: 2 MinItems: 2 Optional: \{\} @@ -5359,7 +5360,7 @@ RecommendationSpec controls bounded projection into DGDC resources. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first; Pareto candidates have no scalar rank. This field does not limit internal trials. +MaxCandidates is the maximum number of DGDCs owned by one run. Scalar candidates are ordered best first. This field does not limit internal trials. **Validation:** Maximum: 100 Minimum: 1 Optional: \{\} From 9d90eb096b9b392791014b78d4bad084b96d6718 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 17:46:53 +0100 Subject: [PATCH 009/100] pr-16661: pr-4: preserve AISimulate override integers Signed-off-by: Dr. Stefan Schimanski --- deploy/operator/internal/aisimulate/input.go | 57 ++++++++++++------- .../internal/aisimulate/input_test.go | 34 ++++++++--- 2 files changed, 60 insertions(+), 31 deletions(-) diff --git a/deploy/operator/internal/aisimulate/input.go b/deploy/operator/internal/aisimulate/input.go index 1c75e9ca7c97..54d42bc1c679 100644 --- a/deploy/operator/internal/aisimulate/input.go +++ b/deploy/operator/internal/aisimulate/input.go @@ -18,10 +18,11 @@ package aisimulate import ( + "bytes" "encoding/json" "fmt" + "io" "strings" - "unicode" v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -82,9 +83,14 @@ func InputOverrides(spec *v1beta2.NativeSearchInput) (map[string]any, error) { return nil, nil } var value any - if err := json.Unmarshal(raw, &value); err != nil { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := decoder.Decode(&value); err != nil { return nil, fmt.Errorf("spec.aisimulate.inputOverrides: %w", err) } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, fmt.Errorf("spec.aisimulate.inputOverrides: trailing JSON value") + } switch typed := value.(type) { case nil: return nil, nil @@ -163,7 +169,12 @@ func Underlay(spec *v1beta2.NativeSearchInput) (map[string]any, error) { b.set(TargetPareto, optimizationTarget) } } else if objective.Metric != nil { - b.set(publicMetric(*objective.Metric), optimizationTarget) + target, err := publicMetric(*objective.Metric) + if err != nil { + b.err = err + } else { + b.set(target, optimizationTarget) + } } if sla := objective.SLA; sla != nil { if sla.TTFTMs != nil { @@ -221,28 +232,30 @@ func (b *builder) setStrings(values []string, path []string) { } } -// publicMetric maps an API metric to AISimulate's optimization.target by converting its CamelCase -// name to snake_case: ThroughputPerGPU becomes throughput_per_gpu and E2ELatency becomes -// e2e_latency. A run of capitals stays together, a digit does not start a word, and the last -// capital of a run starts the next word when a lowercase letter follows it. -func publicMetric(metric v1beta2.ObjectiveMetric) string { - name := []rune(string(metric)) - var out []rune - for i, r := range name { - if i > 0 && unicode.IsUpper(r) { - prev := name[i-1] - nextLower := i+1 < len(name) && unicode.IsLower(name[i+1]) - if unicode.IsLower(prev) || (unicode.IsUpper(prev) && nextLower) { - out = append(out, '_') - } - } - out = append(out, unicode.ToLower(r)) +// publicMetric maps each public API metric to its explicit AISimulate optimization target. +func publicMetric(metric v1beta2.ObjectiveMetric) (string, error) { + switch metric { + case v1beta2.ObjectiveMetric("Throughput"): + return "throughput", nil + case v1beta2.ObjectiveMetric("ThroughputPerGPU"): + return "throughput_per_gpu", nil + case v1beta2.ObjectiveMetric("ThroughputPerUser"): + return "throughput_per_user", nil + case v1beta2.ObjectiveMetric("TTFT"): + return "ttft", nil + case v1beta2.ObjectiveMetric("E2ELatency"): + return "e2e_latency", nil + case v1beta2.ObjectiveMetric("Goodput"): + return "goodput", nil + case v1beta2.ObjectiveMetric("GoodputPerGPU"): + return "goodput_per_gpu", nil + default: + return "", fmt.Errorf("spec.objective.metric: unsupported value %q", metric) } - return string(out) } -// normalize round-trips through JSON so every number is a float64, the type the -// input overrides decode to. Merge and validation then see one representation. +// normalize round-trips operator-owned maps through JSON so their Go-specific numeric types +// become JSON-compatible values before merging them with the user-owned input overrides. func normalize(value map[string]any) (map[string]any, error) { if value == nil { return nil, nil diff --git a/deploy/operator/internal/aisimulate/input_test.go b/deploy/operator/internal/aisimulate/input_test.go index 714e7cb7e1e1..19011da7c30f 100644 --- a/deploy/operator/internal/aisimulate/input_test.go +++ b/deploy/operator/internal/aisimulate/input_test.go @@ -192,7 +192,7 @@ func TestInputOverridesWinOverTheTypedUnderlay(t *testing.T) { }) t.Run("a value overrides a typed field", func(t *testing.T) { input := lower(t, withInputOverrides(defaultedSpec(), `{"traffic":{"source":{"input_tokens":8000}},"optimization":{"constraints":{"max_candidate_gpus":16}}}`)) - if nested(t, input, "traffic", "source", "input_tokens") != 8000.0 || nested(t, input, "optimization", "constraints", "max_candidate_gpus") != 16.0 { + if nested(t, input, "traffic", "source", "input_tokens") != json.Number("8000") || nested(t, input, "optimization", "constraints", "max_candidate_gpus") != json.Number("16") { t.Fatalf("input overrides did not win: %s", encode(t, input)) } }) @@ -204,7 +204,7 @@ func TestInputOverridesWinOverTheTypedUnderlay(t *testing.T) { }) t.Run("an SLA bound overrides only the bound it names", func(t *testing.T) { input := lower(t, withInputOverrides(withSLA(defaultedSpec()), `{"evaluation":{"sla":{"ttft_ms":500}}}`)) - if nested(t, input, "evaluation", "sla", "ttft_ms") != 500.0 || nested(t, input, "evaluation", "sla", "itl_ms") != 30.0 { + if nested(t, input, "evaluation", "sla", "ttft_ms") != json.Number("500") || nested(t, input, "evaluation", "sla", "itl_ms") != 30.0 { t.Fatalf("unexpected SLA: %s", encode(t, input)) } }) @@ -218,7 +218,7 @@ func TestRequestCountDefault(t *testing.T) { }) t.Run("an override ratio deletes the derived count itself", func(t *testing.T) { input := lower(t, withInputOverrides(defaultedSpec(), `{"traffic":{"stop":{"requests_per_load_unit":10,"requests":null}}}`)) - if nested(t, input, "traffic", "stop", "requests") != nil || nested(t, input, "traffic", "stop", "requests_per_load_unit") != 10.0 { + if nested(t, input, "traffic", "stop", "requests") != nil || nested(t, input, "traffic", "stop", "requests_per_load_unit") != json.Number("10") { t.Fatalf("unexpected workload: %s", encode(t, input)) } }) @@ -260,7 +260,7 @@ func TestMaxTrialsDefault(t *testing.T) { } }) t.Run("override value or null wins", func(t *testing.T) { - if nested(t, lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":7}}`)), "optimizer", "max_trials") != 7.0 { + if nested(t, lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":7}}`)), "optimizer", "max_trials") != json.Number("7") { t.Fatal("override max_trials ignored") } if nested(t, lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":null}}`)), "optimizer", "max_trials") != nil { @@ -291,7 +291,7 @@ func TestStrictSLADerivation(t *testing.T) { }) t.Run("an override bound alone does not derive it", func(t *testing.T) { input := lower(t, withInputOverrides(defaultedSpec(), `{"evaluation":{"sla":{"ttft_ms":500}}}`)) - if nested(t, input, "evaluation", "sla", "ttft_ms") != 500.0 || nested(t, input, "optimization", "strict_sla") != nil { + if nested(t, input, "evaluation", "sla", "ttft_ms") != json.Number("500") || nested(t, input, "optimization", "strict_sla") != nil { t.Fatalf("unexpected optimization input: %s", encode(t, input)) } }) @@ -317,13 +317,13 @@ func TestInputOverridesApplyOnceAfterDerivedDefaults(t *testing.T) { }) t.Run("max_trials null removes only max_trials", func(t *testing.T) { input := lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":null,"seed":3}}`)) - if nested(t, input, "optimizer", "max_trials") != nil || nested(t, input, "optimizer", "seed") != 3.0 { + if nested(t, input, "optimizer", "max_trials") != nil || nested(t, input, "optimizer", "seed") != json.Number("3") { t.Fatalf("unexpected optimizer: %s", encode(t, input)) } }) t.Run("an override max_trials replaces the derived value", func(t *testing.T) { input := lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":7}}`)) - if nested(t, input, "optimizer", "max_trials") != 7.0 { + if nested(t, input, "optimizer", "max_trials") != json.Number("7") { t.Fatalf("unexpected optimizer: %s", encode(t, input)) } }) @@ -428,6 +428,19 @@ func TestInputOverridesMustBeAnObject(t *testing.T) { } } +func TestInputOverridesPreserveLargeIntegers(t *testing.T) { + const seed = "9007199254740993" + + t.Log("Lowering the opaque override without passing its integer through float64") + input := lower(t, withInputOverrides(defaultedSpec(), `{"optimizer":{"seed":`+seed+`}}`)) + if got := nested(t, input, "optimizer", "seed"); got != json.Number(seed) { + t.Fatalf("optimizer.seed = %#v, want exact JSON number %s", got, seed) + } + if got := encode(t, input); !strings.Contains(got, `"seed":`+seed) { + t.Fatalf("encoded input lost integer precision: %s", got) + } +} + func TestHasPath(t *testing.T) { overrides := map[string]any{ "optimization": map[string]any{"target": nil}, @@ -464,8 +477,11 @@ func TestPublicMetricNames(t *testing.T) { "Goodput": "goodput", "GoodputPerGPU": "goodput_per_gpu", } { - if got := publicMetric(metric); got != want { - t.Errorf("publicMetric(%s) = %q, want %q", metric, got, want) + if got, err := publicMetric(metric); err != nil || got != want { + t.Errorf("publicMetric(%s) = %q, %v; want %q", metric, got, err, want) } } + if got, err := publicMetric("FutureMetric"); err == nil || got != "" { + t.Errorf("publicMetric(FutureMetric) = %q, %v; want an explicit rejection", got, err) + } } From 9af8728f83170e81ab95e87e4190c1a6fea16d0b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 26 Oct 2026 12:29:18 +0100 Subject: [PATCH 010/100] pr-16661: pr-3: squash aggregate v1beta2 GPU budget Squashed from https://github.com/devivasudevan/dynamo/pull/3. Signed-off-by: Dr. Stefan Schimanski --- .../internal/controller/suite_envtest_test.go | 2 +- ...deploymentrequest_v1beta2_hardware_test.go | 9 +++++--- ...ographdeploymentrequest_v1beta2_helpers.go | 3 +-- ...namographdeploymentrequest_v1beta2_test.go | 12 +++++------ ...ploymentrequest_validation_envtest_test.go | 21 +++++++++++++------ 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/deploy/operator/internal/controller/suite_envtest_test.go b/deploy/operator/internal/controller/suite_envtest_test.go index 3d2f9edd032c..ce13f3d349e3 100644 --- a/deploy/operator/internal/controller/suite_envtest_test.go +++ b/deploy/operator/internal/controller/suite_envtest_test.go @@ -31,7 +31,7 @@ const admissionBypassUsername = "operatorenv-controller-admission-bypass" type fixedHardwareDiscoverer struct{} func (fixedHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { - return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, nil + return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, nil } var ( diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index c797d4b6ec23..b440959f68ac 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -35,8 +35,8 @@ func hardwareOfV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.HardwareSpe func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { str := func(s string) *string { return &s } num := func(n int32) *int32 { return &n } - h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} - h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8} + h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} + h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64} tests := []struct { name string hardware *nvidiacomv1beta2.HardwareSpec @@ -50,6 +50,9 @@ func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { }{ {"omitted hardware discovers the SKU and the budget", nil, h200x8, nil, "h200_sxm", 8, 1, "", ""}, {"an empty hardware object is automatic discovery", &nvidiacomv1beta2.HardwareSpec{}, h200x8, nil, "h200_sxm", 8, 1, "", ""}, + {"same-SKU nodes with different GPU counts use the aggregate", nil, + gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12}, + nil, "h200_sxm", 12, 1, "", ""}, {"an empty gpu object is automatic discovery", &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{}}, h200x8, nil, "h200_sxm", 8, 1, "", ""}, {"the discovered budget is capped", nil, h200x64, nil, "h200_sxm", 32, 1, "", ""}, @@ -81,7 +84,7 @@ func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { nil, "", 0, 1, "", "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100)"}, {"a pinned SKU is not ambiguous in a heterogeneous cluster", hardwareOfV1beta2(str("h200_sxm"), nil), - gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, nil, "", 8, 1, "h200_sxm", ""}, + gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, nil, "", 8, 1, "h200_sxm", ""}, {"no GPUs at all reject the missing budget", nil, gpu.GPUInfo{System: "h200_sxm"}, nil, "", 0, 1, "", `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h200_sxm" were found in the cluster`}, } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 39e1e37226e9..2d33bc113aa7 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -157,8 +157,7 @@ func (o DGDRV1beta2Options) resolveHardware( // A filtered discovery with no GPUs means the caller pinned a SKU the cluster cannot run. var gpus int if found != nil { - // TODO(#17192): Use the discovered aggregate GPU count so mixed node sizes of one SKU are counted correctly. - gpus = found.GPUsPerNode * found.NodesWithGPUs + gpus = found.TotalGPUs } if spec.Hardware != nil && spec.Hardware.GPU != nil && spec.Hardware.GPU.SKU != nil && gpus <= 0 { return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Invalid( diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index 75164cc1cf19..9aeb10cb6396 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -75,7 +75,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("everything missing", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 16, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 16, NodesWithGPUs: 1, TotalGPUs: 16}} got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) if err != nil { t.Fatal(err) @@ -88,7 +88,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1, TotalGPUs: 4}} got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), nil)) if err != nil { t.Fatal(err) @@ -108,7 +108,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("the budget is capped", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 1024, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 1024, NodesWithGPUs: 1, TotalGPUs: 1024}} if got, err := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || got.budget != 32 { t.Fatalf("default cap: %+v", got) } @@ -126,7 +126,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { } func TestResolveDefaultsV1beta2(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}} got, err := DGDRV1beta2Options{Hardware: d}. resolveDefaults(context.Background(), admissionv1.Create, nil, modelOnlySpecV1beta2()) if err != nil { @@ -150,7 +150,7 @@ func v1beta2Envelope(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentR func TestDefaultEnvelopeV1beta2(t *testing.T) { options := DGDRV1beta2Options{ - Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}}, + Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}}, } dgdr := v1beta2Envelope(t, modelOnlySpecV1beta2()) @@ -246,7 +246,7 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}} got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), tt.op, tt.old, tt.spec) if err != nil { t.Fatal(err) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index bbed17e865ac..628eca4d031b 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -59,8 +59,8 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec":{"components":[{"name":"VllmPrefillWorker"}]} }` - discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} - discoveredH100 := gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1} + discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} + discoveredH100 := gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} tests := []struct { name string @@ -836,18 +836,27 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 hardware omitted is discovered and the budget capped", request: v1beta2DGDRForAdmission(nil, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8}, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h200_sxm", 32) }, }, + { + name: "v1beta2 same-SKU nodes with different GPU counts use the aggregate", + request: v1beta2DGDRForAdmission(nil, nil), + gpuDiscovery: true, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12}, + check: func(t *testing.T, result *unstructured.Unstructured) { + expectV1beta2GPUValues(t, result, "h200_sxm", 12) + }, + }, { name: "v1beta2 an empty hardware object is automatic discovery", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["hardware"] = map[string]any{} }, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h200_sxm", 8) }, @@ -858,7 +867,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} }, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 8}, + discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h100_sxm", 32) }, @@ -934,7 +943,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h200_sxm"}} }, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h200_sxm", 8) }, From 5a38e76b69506543480e172cc7309eb58d9c266a Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 09:49:36 +0100 Subject: [PATCH 011/100] pr-16661: pr-3: cover aggregate GPU verification Signed-off-by: Dr. Stefan Schimanski --- .../dynamographdeploymentrequest_v1beta2_hardware_test.go | 2 +- .../defaulting/dynamographdeploymentrequest_v1beta2_test.go | 2 +- .../dynamographdeploymentrequest_validation_envtest_test.go | 2 +- .../operator/internal/webhook/validation/suite_envtest_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index b440959f68ac..03dfda5912c7 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -62,7 +62,7 @@ func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { {"an explicit budget above the cluster size is kept", hardwareOfV1beta2(nil, num(64)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, {"an explicit budget above the cap is kept", hardwareOfV1beta2(str("h200_sxm"), num(64)), h200x8, nil, "", 0, 1, "h200_sxm", ""}, {"a SKU and a budget verify the SKU", hardwareOfV1beta2(str("h100_sxm"), num(4)), - gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, nil, "", 0, 1, "h100_sxm", ""}, {"complete hardware rejects failed verification", hardwareOfV1beta2(str("h100_sxm"), num(4)), gpu.GPUInfo{}, errors.New("down"), "", 0, 1, "h100_sxm", diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index 9aeb10cb6396..48c6d1bb15bb 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -65,7 +65,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { } }) t.Run("complete hardware verifies its SKU against the cluster", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1}} + d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1, TotalGPUs: 4}} got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) if err != nil { t.Fatal(err) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 628eca4d031b..414f2c9c7122 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -889,7 +889,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} }, nil), gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, + discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, check: func(t *testing.T, result *unstructured.Unstructured) { expectV1beta2GPUValues(t, result, "h100_sxm", 4) }, diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index fd9b2f56792f..67ee9f5e6f6b 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -125,7 +125,7 @@ func (f *fakeHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUS func setAdmissionH100Topology() { admissionHardware.set(gpu.GPUInfo{ - System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, + System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, }, nil) } From 215c47b38f75a0eca25b3984e10a703427e8839b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 15:03:30 +0100 Subject: [PATCH 012/100] pr-16661: pr-3: consolidate hardware discovery tests Signed-off-by: Dr. Stefan Schimanski --- ...deploymentrequest_v1beta2_hardware_test.go | 210 ++++++++++++------ 1 file changed, 147 insertions(+), 63 deletions(-) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index 03dfda5912c7..766ae6f55189 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -22,78 +22,166 @@ import ( "errors" "strings" "testing" + "time" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" admissionv1 "k8s.io/api/admission/v1" ) +type fakeDiscoverer struct { + found gpu.GPUInfo + err error + calls int + filter string + timeout time.Duration +} + +func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.calls++ + f.filter = string(filterSKU) + if deadline, ok := ctx.Deadline(); ok { + f.timeout = time.Until(deadline) + } + found := f.found + return &found, f.err +} + func hardwareOfV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.HardwareSpec { return &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} } -func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { +func hardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { + return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + DynamoGraphDeploymentRequestNativeSpec: nvidiacomv1beta2.DynamoGraphDeploymentRequestNativeSpec{ + NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{Hardware: hardware}, + }, + } +} + +func TestResolveHardwareV1beta2(t *testing.T) { str := func(s string) *string { return &s } num := func(n int32) *int32 { return &n } h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64} + completeH100 := hardwareOfV1beta2(str("h100_sxm"), num(4)) tests := []struct { - name string - hardware *nvidiacomv1beta2.HardwareSpec - found gpu.GPUInfo - err error - wantSKU string - wantBudget int32 - wantCalls int - wantFilter string - wantErr string + name string + op admissionv1.Operation + oldHardware *nvidiacomv1beta2.HardwareSpec + hardware *nvidiacomv1beta2.HardwareSpec + found gpu.GPUInfo + discoveryErr error + withoutDiscoverer bool + maxBudget int32 + timeout time.Duration + wantSKU string + wantBudget int32 + wantCalls int + wantFilter string + wantErr string + wantMinTimeout time.Duration + wantMaxTimeout time.Duration }{ - {"omitted hardware discovers the SKU and the budget", nil, h200x8, nil, "h200_sxm", 8, 1, "", ""}, - {"an empty hardware object is automatic discovery", &nvidiacomv1beta2.HardwareSpec{}, h200x8, nil, "h200_sxm", 8, 1, "", ""}, - {"same-SKU nodes with different GPU counts use the aggregate", nil, - gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12}, - nil, "h200_sxm", 12, 1, "", ""}, - {"an empty gpu object is automatic discovery", - &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{}}, h200x8, nil, "h200_sxm", 8, 1, "", ""}, - {"the discovered budget is capped", nil, h200x64, nil, "h200_sxm", 32, 1, "", ""}, - {"a SKU alone keeps the SKU, filters the cluster and persists the capped budget", - hardwareOfV1beta2(str("h200_sxm"), nil), h200x64, nil, "", 32, 1, "h200_sxm", ""}, - {"a budget alone keeps the budget and discovers the SKU", hardwareOfV1beta2(nil, num(4)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, - {"an explicit budget above the cluster size is kept", hardwareOfV1beta2(nil, num(64)), h200x8, nil, "h200_sxm", 0, 1, "", ""}, - {"an explicit budget above the cap is kept", hardwareOfV1beta2(str("h200_sxm"), num(64)), h200x8, nil, "", 0, 1, "h200_sxm", ""}, - {"a SKU and a budget verify the SKU", hardwareOfV1beta2(str("h100_sxm"), num(4)), - gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, - nil, "", 0, 1, "h100_sxm", ""}, - {"complete hardware rejects failed verification", hardwareOfV1beta2(str("h100_sxm"), num(4)), - gpu.GPUInfo{}, errors.New("down"), "", 0, 1, "h100_sxm", - `spec.hardware.gpu.sku: Invalid value: "h100_sxm": could not be verified against the cluster's GPUs (down)`}, - {"a failed discovery rejects", nil, gpu.GPUInfo{}, errors.New("no GPUs found"), "", 0, 1, "", - "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found)"}, - {"a GPU model the catalog does not know rejects", nil, - gpu.GPUInfo{Model: "Mystery GPU", GPUsPerNode: 8, NodesWithGPUs: 1}, nil, "", 0, 1, "", - `spec.hardware.gpu.sku: Required value: the cluster's GPU model "Mystery GPU" maps to no AISimulate SKU`}, - {"no GPUs of the SKU reject the SKU", hardwareOfV1beta2(str("h100_sxm"), nil), - gpu.GPUInfo{System: "h100_sxm"}, nil, "", 0, 1, "h100_sxm", - `spec.hardware.gpu.sku: Invalid value: "h100_sxm": no GPUs of this SKU were found in the cluster`}, - {"a heterogeneous cluster rejects an omitted SKU", nil, - gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, - nil, "", 0, 1, "", - "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100); set spec.hardware.gpu.sku"}, - {"a heterogeneous cluster rejects a budget alone too", hardwareOfV1beta2(nil, num(4)), - gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, - nil, "", 0, 1, "", - "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100)"}, - {"a pinned SKU is not ambiguous in a heterogeneous cluster", hardwareOfV1beta2(str("h200_sxm"), nil), - gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, nil, "", 8, 1, "h200_sxm", ""}, - {"no GPUs at all reject the missing budget", nil, gpu.GPUInfo{System: "h200_sxm"}, nil, "", 0, 1, "", - `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h200_sxm" were found in the cluster`}, + {name: "no discoverer leaves omitted hardware unresolved", op: admissionv1.Create, withoutDiscoverer: true}, + {name: "omitted hardware discovers the SKU and the budget", op: admissionv1.Create, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, wantMinTimeout: time.Millisecond, wantMaxTimeout: defaultHardwareDiscoveryTimeout}, + {name: "an empty hardware object is automatic discovery", op: admissionv1.Create, hardware: &nvidiacomv1beta2.HardwareSpec{}, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1}, + { + name: "same-SKU nodes with different GPU counts use the aggregate", op: admissionv1.Create, + found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12}, + wantSKU: "h200_sxm", wantBudget: 12, wantCalls: 1, + }, + { + name: "an empty gpu object is automatic discovery", op: admissionv1.Create, + hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{}}, found: h200x8, + wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, + }, + {name: "the discovered budget is capped", op: admissionv1.Create, found: h200x64, wantSKU: "h200_sxm", wantBudget: 32, wantCalls: 1}, + {name: "the discovered budget honors a configured cap", op: admissionv1.Create, found: h200x64, maxBudget: 8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1}, + { + name: "a SKU alone keeps the SKU, filters the cluster and persists the capped budget", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h200_sxm"), nil), found: h200x64, + wantBudget: 32, wantCalls: 1, wantFilter: "h200_sxm", + }, + {name: "a budget alone keeps the budget and discovers the SKU", op: admissionv1.Create, hardware: hardwareOfV1beta2(nil, num(4)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, + {name: "an explicit budget above the cluster size is kept", op: admissionv1.Create, hardware: hardwareOfV1beta2(nil, num(64)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, + {name: "an explicit budget above the cap is kept", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h200_sxm"), num(64)), found: h200x8, wantCalls: 1, wantFilter: "h200_sxm"}, + { + name: "a SKU and a budget verify the SKU", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h100_sxm"), num(4)), + found: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, + wantCalls: 1, wantFilter: "h100_sxm", + }, + { + name: "complete hardware rejects failed verification", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h100_sxm"), num(4)), discoveryErr: errors.New("down"), + wantCalls: 1, wantFilter: "h100_sxm", + wantErr: `spec.hardware.gpu.sku: Invalid value: "h100_sxm": could not be verified against the cluster's GPUs (down)`, + }, + {name: "a failed discovery rejects", op: admissionv1.Create, discoveryErr: errors.New("no GPUs found"), wantCalls: 1, + wantErr: "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found)"}, + { + name: "a GPU model the catalog does not know rejects", op: admissionv1.Create, + found: gpu.GPUInfo{Model: "Mystery GPU", GPUsPerNode: 8, NodesWithGPUs: 1}, wantCalls: 1, + wantErr: `spec.hardware.gpu.sku: Required value: the cluster's GPU model "Mystery GPU" maps to no AISimulate SKU`, + }, + { + name: "no GPUs of the SKU reject the SKU", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h100_sxm"), nil), found: gpu.GPUInfo{System: "h100_sxm"}, + wantCalls: 1, wantFilter: "h100_sxm", + wantErr: `spec.hardware.gpu.sku: Invalid value: "h100_sxm": no GPUs of this SKU were found in the cluster`, + }, + { + name: "a heterogeneous cluster rejects an omitted SKU", op: admissionv1.Create, + found: gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, + wantCalls: 1, + wantErr: "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100); set spec.hardware.gpu.sku", + }, + { + name: "a heterogeneous cluster rejects a budget alone too", op: admissionv1.Create, + hardware: hardwareOfV1beta2(nil, num(4)), + found: gpu.GPUInfo{System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}}, + wantCalls: 1, + wantErr: "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100)", + }, + { + name: "a pinned SKU is not ambiguous in a heterogeneous cluster", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h200_sxm"), nil), found: h200x8, + wantBudget: 8, wantCalls: 1, wantFilter: "h200_sxm", + }, + { + name: "no GPUs at all reject the missing budget", op: admissionv1.Create, + found: gpu.GPUInfo{System: "h200_sxm"}, wantCalls: 1, + wantErr: `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h200_sxm" were found in the cluster`, + }, + {name: "a configured timeout bounds discovery", op: admissionv1.Create, found: h200x8, timeout: time.Hour, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, wantMinTimeout: time.Minute, wantMaxTimeout: time.Hour}, + {name: "an unchanged incomplete update does not discover", op: admissionv1.Update, oldHardware: hardwareOfV1beta2(str("h100_sxm"), nil), hardware: hardwareOfV1beta2(str("h100_sxm"), nil), discoveryErr: errors.New("down")}, + {name: "an update without a stored spec does not discover", op: admissionv1.Update, found: h200x8}, + {name: "removing only the budget discovers only the budget", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("h100_sxm"), nil), found: h200x8, wantBudget: 8, wantCalls: 1, wantFilter: "h100_sxm"}, + {name: "removing only the SKU discovers only the SKU", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(nil, num(4)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, + {name: "removing all hardware discovers both fields", op: admissionv1.Update, oldHardware: completeH100, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1}, + {name: "changing complete hardware verifies the new SKU", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("b200_sxm"), num(2)), found: gpu.GPUInfo{System: "b200_sxm", TotalGPUs: 4}, wantCalls: 1, wantFilter: "b200_sxm"}, + {name: "delete never discovers", op: admissionv1.Delete, oldHardware: completeH100, found: h200x8}, + {name: "connect never discovers", op: admissionv1.Connect, oldHardware: completeH100, found: h200x8}, + {name: "an unknown operation never discovers", oldHardware: completeH100, found: h200x8}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - probe := &fakeDiscoverer{found: tt.found, err: tt.err} - spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} - spec.Hardware = tt.hardware - got, err := DGDRV1beta2Options{Hardware: probe}.resolveHardware(context.Background(), admissionv1.Create, nil, spec) + probe := &fakeDiscoverer{found: tt.found, err: tt.discoveryErr} + options := DGDRV1beta2Options{ + Hardware: probe, + MaxHardwareBudget: tt.maxBudget, + HardwareDiscoveryTimeout: tt.timeout, + } + if tt.withoutDiscoverer { + options.Hardware = nil + } + var old *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + if tt.oldHardware != nil { + old = hardwareSpecV1beta2(tt.oldHardware) + } + got, err := options.resolveHardware(context.Background(), tt.op, old, hardwareSpecV1beta2(tt.hardware)) if tt.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Fatalf("error = %v, want it to contain %q", err, tt.wantErr) @@ -110,16 +198,12 @@ func TestResolveHardwareCompletesTheTypedHardwareOrRejects(t *testing.T) { if tt.wantCalls > 0 && probe.filter != tt.wantFilter { t.Fatalf("discovery filter = %q, want %q", probe.filter, tt.wantFilter) } + if tt.wantMinTimeout > 0 && probe.timeout < tt.wantMinTimeout { + t.Fatalf("discovery timeout = %v, want at least %v", probe.timeout, tt.wantMinTimeout) + } + if tt.wantMaxTimeout > 0 && probe.timeout > tt.wantMaxTimeout { + t.Fatalf("discovery timeout = %v, want at most %v", probe.timeout, tt.wantMaxTimeout) + } }) } } - -func TestResolveHardwareLeavesAnUnchangedUpdateAlone(t *testing.T) { - probe := &fakeDiscoverer{err: errors.New("down")} - spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} - got, err := DGDRV1beta2Options{Hardware: probe}.resolveHardware(context.Background(), admissionv1.Update, spec.DeepCopy(), spec) - if err != nil || got.sku != "" || got.budget != 0 || probe.calls != 0 { - t.Fatalf("got %+v, %v after %d calls; an update that leaves spec.hardware unchanged must not read the cluster", - got, err, probe.calls) - } -} From aabbfa7c956af6a17d24b1e935c28cb6fda6565d Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 15:37:05 +0100 Subject: [PATCH 013/100] pr-16661: pr-3: modernize hardware test literals Signed-off-by: Dr. Stefan Schimanski --- .../dynamographdeploymentrequest_v1beta2_hardware_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index 766ae6f55189..3b666abecb90 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -53,11 +53,7 @@ func hardwareOfV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.HardwareSpe } func hardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ - DynamoGraphDeploymentRequestNativeSpec: nvidiacomv1beta2.DynamoGraphDeploymentRequestNativeSpec{ - NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{Hardware: hardware}, - }, - } + return &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{Hardware: hardware} } func TestResolveHardwareV1beta2(t *testing.T) { From d9364568f21f87b961ec2e35a870ccdb15de1cfb Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 26 Oct 2026 12:33:48 +0100 Subject: [PATCH 014/100] pr-17191: squash align and document v1beta2 defaults Squashed from https://github.com/ai-dynamo/dynamo/pull/17191. Signed-off-by: Dr. Stefan Schimanski --- .../dynamographdeploymentrequest_types.go | 5 ++-- .../v1beta2/dynamographdeploymentrun_types.go | 3 +- .../dynamographdeploymentrequest_v1beta2.go | 30 +++++++++++++------ ...ploymentrequest_validation_envtest_test.go | 11 +++++-- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 9b0303947b74..fe924f1aee4c 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -201,8 +201,9 @@ type ObjectiveSpec struct { // +listType=set Metrics []ObjectiveMetric `json:"metrics,omitempty"` - // SLA defines per-request latency bounds used to calculate goodput. When omitted, no - // bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. + // SLA defines per-request latency bounds used to calculate goodput. Admission defaults an + // omitted SLA to TTFT 2000 ms and ITL 30 ms. An explicit empty object opts out of those + // defaults. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. // +optional SLA *ObjectiveSLASpec `json:"sla,omitempty"` } diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go index 280a487756b4..f6643255e691 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go @@ -25,7 +25,8 @@ import ( ) // DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. -// Run creators resolve optional Request inputs before persisting the Run. +// Run creators resolve optional Request inputs and the final AISimulate parameters before +// persisting the Run. Request admission defaulting does not apply to Runs. // +kubebuilder:validation:XValidation:rule="has(self.model)",message="model is required" type DynamoGraphDeploymentRunSpec struct { NativeSearchInput `json:",inline"` diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index becc3fb11dff..5ff44a085de0 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -31,7 +31,7 @@ import ( ) // defaultMetricV1beta2 is the scalar target an Optimize objective receives when it names none. -const defaultMetricV1beta2 = nvidiacomv1beta2.ObjectiveMetric("Throughput") +const defaultMetricV1beta2 = nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") // Defaults of the objects admission creates. Admission is their only owner: the API server // defaults a mutated object in the webhook's version (v1beta1, whose v1beta2 carrier is @@ -40,6 +40,8 @@ const ( defaultStaticInputTokensV1beta2 int32 = 1024 defaultStaticOutputTokensV1beta2 int32 = 128 defaultStaticConcurrencyV1beta2 int32 = 10 + defaultSLATTFTMsV1beta2 = 2000.0 + defaultSLAITLMsV1beta2 = 30.0 defaultMaxCandidatesV1beta2 int32 = 1 ) @@ -50,10 +52,10 @@ const ( // API version because the canonical names belong to v1beta1. // // Admission owns every v1beta2 default, and each default has this one owner: backends, -// objective.mode, the scalar metric, recommendation.maxCandidates, workload.static and the -// cluster hardware. workload.static: {} opts out of the static workload; a nonempty static -// object receives any missing inputTokens, outputTokens and load defaults. objective.sla has -// no default: an omitted SLA stays absent. +// objective.mode, the scalar metric, objective.sla, recommendation.maxCandidates, +// workload.static and the cluster hardware. workload.static: {} opts out of the static +// workload; a nonempty static object receives any missing inputTokens, outputTokens and load +// defaults. objective.sla: {} opts out of the latency defaults. // spec.aisimulate.inputOverrides never changes a default. It overrides the defaulted typed // values later, when the input is lowered. @@ -228,10 +230,10 @@ func defaultStaticWorkloadSpecV1beta2(static *nvidiacomv1beta2.StaticWorkloadSpe } } -// defaultObjectiveSpecV1beta2 defaults the mode and gives an Optimize objective without a -// metric the default scalar target. It reads the typed objective alone: a goal.target in -// spec.aisimulate.inputOverrides overrides the completed typed value when the input is lowered, -// so it never suppresses this default. +// defaultObjectiveSpecV1beta2 defaults the mode, the Optimize metric and an absent SLA. +// Existing SLA objects are preserved, including the empty opt-out. It reads the typed objective +// alone: spec.aisimulate.inputOverrides overrides the completed typed values when the input is +// lowered, so it never suppresses these defaults. func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { if objective.Mode == "" { objective.Mode = nvidiacomv1beta2.ObjectiveModeOptimize @@ -239,6 +241,16 @@ func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { if objective.Mode == nvidiacomv1beta2.ObjectiveModeOptimize && objective.Metric == nil { objective.Metric = ptr.To(defaultMetricV1beta2) } + if objective.SLA == nil { + objective.SLA = &nvidiacomv1beta2.ObjectiveSLASpec{} + defaultObjectiveSLASpecV1beta2(objective.SLA) + } +} + +// defaultObjectiveSLASpecV1beta2 fills the whole-object SLA default admission creates. +func defaultObjectiveSLASpecV1beta2(sla *nvidiacomv1beta2.ObjectiveSLASpec) { + sla.TTFTMs = ptr.To(defaultSLATTFTMsV1beta2) + sla.ITLMs = ptr.To(defaultSLAITLMsV1beta2) } func defaultRecommendationSpecV1beta2(recommendation *nvidiacomv1beta2.RecommendationSpec) { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 414f2c9c7122..caee624849e7 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -1529,10 +1529,15 @@ func expectV1beta2StructuralDefaults(t *testing.T, obj *unstructured.Unstructure if got := v1beta2Number(t, obj, "spec", "workload", "static", "load", "concurrency"); got != 10 { t.Fatalf("workload.static.load.concurrency = %v, want 10", got) } - v1beta2Absent(t, obj, "spec", "objective", "sla") if got := v1beta2String(t, obj, "spec", "objective", "mode"); got != "Optimize" { t.Fatalf("objective.mode = %q, want Optimize", got) } + if got := v1beta2Number(t, obj, "spec", "objective", "sla", "ttftMs"); got != 2000 { + t.Fatalf("objective.sla.ttftMs = %v, want 2000", got) + } + if got := v1beta2Number(t, obj, "spec", "objective", "sla", "itlMs"); got != 30 { + t.Fatalf("objective.sla.itlMs = %v, want 30", got) + } if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm", "sglang", "trtllm"}) { t.Fatalf("backends = %#v, want vllm, sglang and trtllm", got) } @@ -1580,8 +1585,8 @@ func expectV1beta2ExplicitValues(t *testing.T, obj *unstructured.Unstructured) { // expectV1beta2AdmissionDefaults checks what admission supplies from the environment. func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured) { t.Helper() - if got := v1beta2String(t, obj, "spec", "objective", "metric"); got != "Throughput" { - t.Fatalf("objective.metric = %q, want Throughput", got) + if got := v1beta2String(t, obj, "spec", "objective", "metric"); got != "ThroughputPerGPU" { + t.Fatalf("objective.metric = %q, want ThroughputPerGPU", got) } if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != "h200_sxm" { t.Fatalf("hardware.gpu.sku = %q, want h200_sxm", got) From 35cb8ed379850c8cc9b6c3e07cc6ace706fa4920 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 26 Oct 2026 16:13:03 +0100 Subject: [PATCH 015/100] pr-17191: pr-4: align model-only objective defaults Upstreamed through https://github.com/devivasudevan/dynamo/pull/4. Signed-off-by: Dr. Stefan Schimanski --- .../dynamographdeploymentrequest_types.go | 2 +- ...dia.com_dynamographdeploymentrequests.yaml | 7 ++-- .../nvidia.com_dynamographdeploymentruns.yaml | 10 +++-- .../internal/aisimulate/input_test.go | 38 ++++++++++++------- ...ploymentrequest_validation_envtest_test.go | 1 - .../additional-resources/api-reference-k8s.md | 7 ++-- .../kubernetes-api/full-api-reference.mdx | 7 ++-- 7 files changed, 43 insertions(+), 29 deletions(-) diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index fe924f1aee4c..a504b6634391 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -190,7 +190,7 @@ type ObjectiveSpec struct { Mode ObjectiveMode `json:"mode,omitempty"` // Metric is the scalar target when mode is Optimize. - // When omitted in Optimize mode, admission defaults it to Throughput. + // When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. // +optional Metric *ObjectiveMetric `json:"metric,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index cb332d851982..6a57422d31b7 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9672,7 +9672,7 @@ spec: metric: description: |- Metric is the scalar target when mode is Optimize. - When omitted in Optimize mode, admission defaults it to Throughput. + When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. enum: - Throughput - ThroughputPerGPU @@ -9706,8 +9706,9 @@ spec: type: string sla: description: |- - SLA defines per-request latency bounds used to calculate goodput. When omitted, no - bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. + SLA defines per-request latency bounds used to calculate goodput. Admission defaults an + omitted SLA to TTFT 2000 ms and ITL 30 ms. An explicit empty object opts out of those + defaults. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 1192395c057e..3888695a8d9d 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -58,7 +58,8 @@ spec: spec: description: |- DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. - Run creators resolve optional Request inputs before persisting the Run. + Run creators resolve optional Request inputs and the final AISimulate parameters before + persisting the Run. Request admission defaulting does not apply to Runs. properties: aisimulate: description: AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. @@ -135,7 +136,7 @@ spec: metric: description: |- Metric is the scalar target when mode is Optimize. - When omitted in Optimize mode, admission defaults it to Throughput. + When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. enum: - Throughput - ThroughputPerGPU @@ -169,8 +170,9 @@ spec: type: string sla: description: |- - SLA defines per-request latency bounds used to calculate goodput. When omitted, no - bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. + SLA defines per-request latency bounds used to calculate goodput. Admission defaults an + omitted SLA to TTFT 2000 ms and ITL 30 ms. An explicit empty object opts out of those + defaults. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. properties: e2eMs: description: E2EMs is an alternative end-to-end bound in milliseconds. diff --git a/deploy/operator/internal/aisimulate/input_test.go b/deploy/operator/internal/aisimulate/input_test.go index 19011da7c30f..7152fdc6da08 100644 --- a/deploy/operator/internal/aisimulate/input_test.go +++ b/deploy/operator/internal/aisimulate/input_test.go @@ -29,11 +29,10 @@ import ( func ptrTo[T any](v T) *T { return &v } -// defaultedSpec is a model-only Request as it looks after structural and admission defaults: -// the fixed-concurrency workload, the scalar metric and discovered hardware. It -// has no SLA bounds and no KV-load range; withSLA and withKVRange add them. +// defaultedSpec is a model-only Request after structural and admission defaults: +// fixed-concurrency traffic, the scalar objective with latency bounds, and discovered hardware. func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { - metric := v1beta2.ObjectiveMetric("Throughput") + metric := v1beta2.ObjectiveMetric("ThroughputPerGPU") return &v1beta2.DynamoGraphDeploymentRequestSpec{ Model: &v1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, Backends: []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang, v1beta2.BackendTRTLLM}, @@ -43,17 +42,27 @@ func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { OutputTokens: ptrTo(int32(128)), Load: &v1beta2.StaticLoadSpec{Concurrency: ptrTo(int32(10))}, }}, - Objective: &v1beta2.ObjectiveSpec{Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric}, + Objective: &v1beta2.ObjectiveSpec{ + Mode: v1beta2.ObjectiveModeOptimize, + Metric: &metric, + SLA: &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)}, + }, Recommendation: &v1beta2.RecommendationSpec{MaxCandidates: 1}, } } -// withSLA adds the latency bounds the earlier defaults used to supply. +// withSLA restores the model-only latency defaults after a test changes them. func withSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)} return spec } +// withoutSLA models a caller that explicitly opted out with objective.sla: {}. +func withoutSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{} + return spec +} + // withKVRange replaces the fixed concurrency with a KV-load range search. func withKVRange(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ @@ -109,7 +118,8 @@ func nested(t *testing.T, input map[string]any, path ...string) any { func TestLowerModelOnlyRequest(t *testing.T) { want := `{"engine":{"backend":{"choices":["vllm","sglang","trtllm"]},"hardware":"h200_sxm","model":"meta-llama/Llama-3.1-8B-Instruct","workers":{"aggregated":{},"decode":{},"prefill":{}}},` + - `"optimization":{"constraints":{"max_candidate_gpus":8},"target":"throughput"},` + + `"evaluation":{"sla":{"itl_ms":30,"ttft_ms":2000}},` + + `"optimization":{"constraints":{"max_candidate_gpus":8},"strict_sla":true,"target":"throughput_per_gpu"},` + `"traffic":{"load":{"concurrency":10,"type":"concurrency"},"source":{"input_tokens":1024,"output_tokens":128,"type":"synthetic"},"stop":{"requests":1000}}}` if got := encode(t, lower(t, defaultedSpec())); got != want { t.Fatalf("effective input\n got: %s\nwant: %s", got, want) @@ -275,22 +285,22 @@ func TestStrictSLADerivation(t *testing.T) { t.Fatal("strict_sla not derived") } }) - t.Run("model-only derives nothing", func(t *testing.T) { - input := lower(t, defaultedSpec()) + t.Run("an absent SLA derives nothing", func(t *testing.T) { + spec := defaultedSpec() + spec.Objective.SLA = nil + input := lower(t, spec) if nested(t, input, "optimization", "strict_sla") != nil || nested(t, input, "evaluation", "sla") != nil { t.Fatalf("model-only input has SLA keys: %s", encode(t, input)) } }) t.Run("an empty SLA derives nothing", func(t *testing.T) { - spec := defaultedSpec() - spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{} - input := lower(t, spec) + input := lower(t, withoutSLA(defaultedSpec())) if nested(t, input, "optimization", "strict_sla") != nil || nested(t, input, "evaluation", "sla") != nil { t.Fatalf("an empty SLA produced SLA keys: %s", encode(t, input)) } }) t.Run("an override bound alone does not derive it", func(t *testing.T) { - input := lower(t, withInputOverrides(defaultedSpec(), `{"evaluation":{"sla":{"ttft_ms":500}}}`)) + input := lower(t, withInputOverrides(withoutSLA(defaultedSpec()), `{"evaluation":{"sla":{"ttft_ms":500}}}`)) if nested(t, input, "evaluation", "sla", "ttft_ms") != json.Number("500") || nested(t, input, "optimization", "strict_sla") != nil { t.Fatalf("unexpected optimization input: %s", encode(t, input)) } @@ -328,7 +338,7 @@ func TestInputOverridesApplyOnceAfterDerivedDefaults(t *testing.T) { } }) t.Run("nothing is derived from values only overrides introduce", func(t *testing.T) { - input := lower(t, withInputOverrides(defaultedSpec(), `{"traffic":{"load":{"fraction":{"range":{"min":0.1,"max":1}}}},"evaluation":{"sla":{"ttft_ms":500}}}`)) + input := lower(t, withInputOverrides(withoutSLA(defaultedSpec()), `{"traffic":{"load":{"fraction":{"range":{"min":0.1,"max":1}}}},"evaluation":{"sla":{"ttft_ms":500}}}`)) if nested(t, input, "optimizer") != nil || nested(t, input, "optimization", "strict_sla") != nil { t.Fatalf("a field was derived from override-only values: %s", encode(t, input)) } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index caee624849e7..3fcd93813f84 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -722,7 +722,6 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { discovered: discoveredH200, check: func(t *testing.T, result *unstructured.Unstructured) { v1beta2Empty(t, result, "spec", "workload", "static") - v1beta2Absent(t, result, "spec", "objective", "sla") expectV1beta2AdmissionDefaults(t, result) }, }, diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 76a174f6c2d1..20ef5210b5fe 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3546,7 +3546,8 @@ DynamoGraphDeploymentRun is one immutable accepted DGDR request and its executio DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. -Run creators resolve optional Request inputs before persisting the Run. +Run creators resolve optional Request inputs and the final AISimulate parameters before +persisting the Run. Request admission defaulting does not apply to Runs. @@ -3793,9 +3794,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `mode` _[ObjectiveMode](#objectivemode)_ | Mode selects scalar optimization. Only Optimize is currently supported. | | Enum: [Optimize]
Optional: \{\}
| -| `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to Throughput. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| +| `metric` _[ObjectiveMetric](#objectivemetric)_ | Metric is the scalar target when mode is Optimize.
When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
Optional: \{\}
| | `metrics` _[ObjectiveMetric](#objectivemetric) array_ | Metrics is reserved for the future Pareto implementation. | | Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU]
MaxItems: 2
MinItems: 2
Optional: \{\}
| -| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. When omitted, no
bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. | | Optional: \{\}
| +| `sla` _[ObjectiveSLASpec](#objectiveslaspec)_ | SLA defines per-request latency bounds used to calculate goodput. Admission defaults an
omitted SLA to TTFT 2000 ms and ITL 30 ms. An explicit empty object opts out of those
defaults. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. | | Optional: \{\}
| #### RecommendationSpec diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 97cf71ca53d4..923b12ffd8ef 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4990,7 +4990,8 @@ See [DynamoGraphDeploymentRunStatus](#dynamographdeploymentrunstatus). DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. -Run creators resolve optional Request inputs before persisting the Run. +Run creators resolve optional Request inputs and the final AISimulate parameters before +persisting the Run. Request admission defaulting does not apply to Runs. **Kind:** `type` @@ -5334,7 +5335,7 @@ See [ObjectiveMode](#objectivemode). -Metric is the scalar target when mode is Optimize. When omitted in Optimize mode, admission defaults it to Throughput. +Metric is the scalar target when mode is Optimize. When omitted in Optimize mode, admission defaults it to ThroughputPerGPU. See [ObjectiveMetric](#objectivemetric). **Validation:** Enum: [Throughput ThroughputPerGPU ThroughputPerUser TTFT E2ELatency Goodput GoodputPerGPU] Optional: \{\} @@ -5346,7 +5347,7 @@ See [ObjectiveMetric](#objectivemetric). -SLA defines per-request latency bounds used to calculate goodput. When omitted, no bounds apply. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. +SLA defines per-request latency bounds used to calculate goodput. Admission defaults an omitted SLA to TTFT 2000 ms and ITL 30 ms. An explicit empty object opts out of those defaults. spec.aisimulate.inputOverrides.evaluation.sla overrides the bounds it names. See [ObjectiveSLASpec](#objectiveslaspec). **Validation:** Optional: \{\} From de45db858598fec7d6b1367e7bfc7b81a6a3ec22 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 11:10:14 +0100 Subject: [PATCH 016/100] pr-17191: satisfy current Go lint Signed-off-by: Dr. Stefan Schimanski --- deploy/operator/api/roundtrip_fuzz_test.go | 3 +- .../internal/aisimulate/input_test.go | 30 +++++++++---------- deploy/operator/internal/gpu/heterogeneous.go | 2 +- ...namographdeploymentrequest_routing_test.go | 7 ++--- .../dynamographdeploymentrequest_v1beta2.go | 17 +++++------ ...ographdeploymentrequest_v1beta2_helpers.go | 3 +- ...namographdeploymentrequest_v1beta2_test.go | 22 +++++++------- .../internal/webhook/request_version_test.go | 6 ++-- ...ploymentrequest_validation_envtest_test.go | 15 ++-------- 9 files changed, 43 insertions(+), 62 deletions(-) diff --git a/deploy/operator/api/roundtrip_fuzz_test.go b/deploy/operator/api/roundtrip_fuzz_test.go index e3dbe001fd5f..6fc6158d09eb 100644 --- a/deploy/operator/api/roundtrip_fuzz_test.go +++ b/deploy/operator/api/roundtrip_fuzz_test.go @@ -60,7 +60,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/conversion" "sigs.k8s.io/randfill" "sigs.k8s.io/yaml" @@ -579,7 +578,7 @@ func fuzzBeta2Workload(workload *v1beta2.WorkloadSpec, c randfill.Continue) { static.Load = &v1beta2.StaticLoadSpec{Concurrency: oneOfPtr(c, int32(1), int32(32), int32(1024))} case 3: if c.Bool() { - static.Load = &v1beta2.StaticLoadSpec{KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{Value: ptr.To(0.5)}} + static.Load = &v1beta2.StaticLoadSpec{KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{Value: new(0.5)}} } else { static.Load = &v1beta2.StaticLoadSpec{KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{ Range: &v1beta2.KVCapacityFractionRange{Min: 0.1, Max: 1.0}, diff --git a/deploy/operator/internal/aisimulate/input_test.go b/deploy/operator/internal/aisimulate/input_test.go index 7152fdc6da08..5d187000ef69 100644 --- a/deploy/operator/internal/aisimulate/input_test.go +++ b/deploy/operator/internal/aisimulate/input_test.go @@ -27,8 +27,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -func ptrTo[T any](v T) *T { return &v } - // defaultedSpec is a model-only Request after structural and admission defaults: // fixed-concurrency traffic, the scalar objective with latency bounds, and discovered hardware. func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { @@ -36,16 +34,16 @@ func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { return &v1beta2.DynamoGraphDeploymentRequestSpec{ Model: &v1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"}, Backends: []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang, v1beta2.BackendTRTLLM}, - Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("h200_sxm"), Budget: ptrTo(int32(8))}}, + Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: new("h200_sxm"), Budget: new(int32(8))}}, Workload: &v1beta2.WorkloadSpec{Static: &v1beta2.StaticWorkloadSpec{ - InputTokens: ptrTo(int32(1024)), - OutputTokens: ptrTo(int32(128)), - Load: &v1beta2.StaticLoadSpec{Concurrency: ptrTo(int32(10))}, + InputTokens: new(int32(1024)), + OutputTokens: new(int32(128)), + Load: &v1beta2.StaticLoadSpec{Concurrency: new(int32(10))}, }}, Objective: &v1beta2.ObjectiveSpec{ Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric, - SLA: &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)}, + SLA: &v1beta2.ObjectiveSLASpec{TTFTMs: new(2000.0), ITLMs: new(30.0)}, }, Recommendation: &v1beta2.RecommendationSpec{MaxCandidates: 1}, } @@ -53,7 +51,7 @@ func defaultedSpec() *v1beta2.DynamoGraphDeploymentRequestSpec { // withSLA restores the model-only latency defaults after a test changes them. func withSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { - spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{TTFTMs: ptrTo(2000.0), ITLMs: ptrTo(30.0)} + spec.Objective.SLA = &v1beta2.ObjectiveSLASpec{TTFTMs: new(2000.0), ITLMs: new(30.0)} return spec } @@ -66,8 +64,8 @@ func withoutSLA(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoG // withKVRange replaces the fixed concurrency with a KV-load range search. func withKVRange(spec *v1beta2.DynamoGraphDeploymentRequestSpec) *v1beta2.DynamoGraphDeploymentRequestSpec { spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ - InputTokens: ptrTo(int32(4000)), - OutputTokens: ptrTo(int32(1000)), + InputTokens: new(int32(4000)), + OutputTokens: new(int32(1000)), Load: &v1beta2.StaticLoadSpec{KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{ Range: &v1beta2.KVCapacityFractionRange{Min: 0.1, Max: 1}, }}, @@ -143,9 +141,9 @@ func TestLowerExplicitRequest(t *testing.T) { spec := withSLA(defaultedSpec()) spec.Backends = []v1beta2.Backend{v1beta2.BackendVLLM, v1beta2.BackendSGLang} spec.Workload.Static = &v1beta2.StaticWorkloadSpec{ - InputTokens: ptrTo(int32(4000)), - OutputTokens: ptrTo(int32(1000)), - Load: &v1beta2.StaticLoadSpec{RequestsPerSecond: ptrTo(10.0)}, + InputTokens: new(int32(4000)), + OutputTokens: new(int32(1000)), + Load: &v1beta2.StaticLoadSpec{RequestsPerSecond: new(10.0)}, } spec.Objective.Metric = &metric spec = withInputOverrides(spec, `{"traffic":{"stop":{"requests":1000}},"optimizer":{"max_trials":100}}`) @@ -260,7 +258,7 @@ func TestMaxTrialsDefault(t *testing.T) { }) t.Run("not added for a pinned load", func(t *testing.T) { spec := withKVRange(defaultedSpec()) - spec.Workload.Static.Load.KVCapacityFraction = &v1beta2.KVCapacityFractionSpec{Value: ptrTo(0.5)} + spec.Workload.Static.Load.KVCapacityFraction = &v1beta2.KVCapacityFractionSpec{Value: new(0.5)} input := lower(t, spec) if nested(t, input, "optimizer") != nil { t.Fatalf("unexpected optimizer: %s", encode(t, input)) @@ -366,7 +364,7 @@ func TestUnderlayOmitsUnsetTypedFields(t *testing.T) { if got := encode(t, underlay(t, spec)); got != `{"engine":{"model":"m"}}` { t.Fatalf("got %s", got) } - spec.Hardware = &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: ptrTo("")}} + spec.Hardware = &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{SKU: new("")}} if got := encode(t, underlay(t, spec)); got != `{"engine":{"model":"m"}}` { t.Fatalf("an empty SKU must not lower: %s", got) } @@ -375,7 +373,7 @@ func TestUnderlayOmitsUnsetTypedFields(t *testing.T) { func TestUnderlayKVCapacityFractionShapes(t *testing.T) { spec := defaultedSpec() spec.Workload.Static.Load = &v1beta2.StaticLoadSpec{ - KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{Value: ptrTo(0.25)}, + KVCapacityFraction: &v1beta2.KVCapacityFractionSpec{Value: new(0.25)}, } if nested(t, underlay(t, spec), "traffic", "load", "fraction") != 0.25 { t.Fatal("a fixed fraction must lower to a scalar") diff --git a/deploy/operator/internal/gpu/heterogeneous.go b/deploy/operator/internal/gpu/heterogeneous.go index 312f9e362695..869e6115188f 100644 --- a/deploy/operator/internal/gpu/heterogeneous.go +++ b/deploy/operator/internal/gpu/heterogeneous.go @@ -34,7 +34,7 @@ func collectOtherModels( return nil } seen := map[string]struct{}{} - for i := 0; i < nodes; i++ { + for i := range nodes { if sku, model := at(i); sku != selected { seen[model] = struct{}{} } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go index bbf3ad192df0..d39b0093148f 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go @@ -33,10 +33,9 @@ import ( // v1beta1. requestVersion is the version the client used; empty means no conversion. func ctxForRequest(op admissionv1.Operation, requestVersion string) context.Context { gvk := nvidiacomv1beta1.DynamoGraphDeploymentRequestGVK - req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ + req := admission.Request{ Operation: op, - Kind: metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}, - }} + Kind: metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}} if requestVersion != "" { req.RequestKind = &metav1.GroupVersionKind{Group: gvk.Group, Version: requestVersion, Kind: gvk.Kind} } @@ -47,7 +46,7 @@ const modelOnlyCarrier = `{"model":{"name":"Qwen/Qwen3-32B"}}` func carrierRequest() *nvidiacomv1beta1.DynamoGraphDeploymentRequest { return &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ - ObjectMeta: metav1.ObjectMeta{Name: "r", Namespace: "default"}, + Name: "r", Namespace: "default", Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ V1Beta2: &apiextensionsv1.JSON{Raw: []byte(modelOnlyCarrier)}, }, diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index 5ff44a085de0..2906f7b53372 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -27,7 +27,6 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" admissionv1 "k8s.io/api/admission/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/utils/ptr" ) // defaultMetricV1beta2 is the scalar target an Optimize objective receives when it names none. @@ -198,10 +197,10 @@ func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, default // SKU and let admission size the budget. func defaultGPUHardwareSpecV1beta2(gpu *nvidiacomv1beta2.GPUHardwareSpec, defaults hardwareDefaultsV1beta2) { if gpu.SKU == nil && defaults.sku != "" { - gpu.SKU = ptr.To(defaults.sku) + gpu.SKU = new(defaults.sku) } if gpu.Budget == nil && defaults.budget > 0 { - gpu.Budget = ptr.To(defaults.budget) + gpu.Budget = new(defaults.budget) } } @@ -218,14 +217,14 @@ func defaultWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec) { // defaultStaticWorkloadSpecV1beta2 completes a static workload with every omitted default. func defaultStaticWorkloadSpecV1beta2(static *nvidiacomv1beta2.StaticWorkloadSpec) { if static.InputTokens == nil { - static.InputTokens = ptr.To(defaultStaticInputTokensV1beta2) + static.InputTokens = new(defaultStaticInputTokensV1beta2) } if static.OutputTokens == nil { - static.OutputTokens = ptr.To(defaultStaticOutputTokensV1beta2) + static.OutputTokens = new(defaultStaticOutputTokensV1beta2) } if static.Load == nil { static.Load = &nvidiacomv1beta2.StaticLoadSpec{ - Concurrency: ptr.To(defaultStaticConcurrencyV1beta2), + Concurrency: new(defaultStaticConcurrencyV1beta2), } } } @@ -239,7 +238,7 @@ func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { objective.Mode = nvidiacomv1beta2.ObjectiveModeOptimize } if objective.Mode == nvidiacomv1beta2.ObjectiveModeOptimize && objective.Metric == nil { - objective.Metric = ptr.To(defaultMetricV1beta2) + objective.Metric = new(defaultMetricV1beta2) } if objective.SLA == nil { objective.SLA = &nvidiacomv1beta2.ObjectiveSLASpec{} @@ -249,8 +248,8 @@ func defaultObjectiveSpecV1beta2(objective *nvidiacomv1beta2.ObjectiveSpec) { // defaultObjectiveSLASpecV1beta2 fills the whole-object SLA default admission creates. func defaultObjectiveSLASpecV1beta2(sla *nvidiacomv1beta2.ObjectiveSLASpec) { - sla.TTFTMs = ptr.To(defaultSLATTFTMsV1beta2) - sla.ITLMs = ptr.To(defaultSLAITLMsV1beta2) + sla.TTFTMs = new(defaultSLATTFTMsV1beta2) + sla.ITLMs = new(defaultSLAITLMsV1beta2) } func defaultRecommendationSpecV1beta2(recommendation *nvidiacomv1beta2.RecommendationSpec) { diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 2d33bc113aa7..7265d2c5a88e 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -190,8 +190,7 @@ func rejectHardwareV1beta2(err *field.Error) error { return &hardwareRejectionV1 // admissionVerdictV1beta2 turns a rejection into the invalid-object verdict validation uses, so a // client sees the field and its cause. Any other error is returned as it is. func admissionVerdictV1beta2(dgdr *nvidiacomv1beta1.DynamoGraphDeploymentRequest, err error) error { - var rejection *hardwareRejectionV1beta2 - if errors.As(err, &rejection) { + if rejection, ok := errors.AsType[*hardwareRejectionV1beta2](err); ok { return apierrors.NewInvalid(nvidiacomv1beta1.DynamoGraphDeploymentRequestGVK.GroupKind(), dgdr.Name, field.ErrorList{rejection.err}) } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go index 48c6d1bb15bb..3328c944b832 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go @@ -30,8 +30,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) -func v1beta2Ptr[T any](v T) *T { return &v } - type fakeDiscoverer struct { found gpu.GPUInfo err error @@ -66,7 +64,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { }) t.Run("complete hardware verifies its SKU against the cluster", func(t *testing.T) { d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1, TotalGPUs: 4}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2)))) + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(new("b200_sxm"), new(int32(2)))) if err != nil { t.Fatal(err) } @@ -89,7 +87,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { }) t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1, TotalGPUs: 4}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(v1beta2Ptr("b200_sxm"), nil)) + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(new("b200_sxm"), nil)) if err != nil { t.Fatal(err) } @@ -99,7 +97,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { }) t.Run("a pinned budget keeps its value", func(t *testing.T) { d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 64, NodesWithGPUs: 1}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, v1beta2Ptr(int32(3)))) + got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, new(int32(3)))) if err != nil { t.Fatal(err) } @@ -198,7 +196,7 @@ func TestHardwareIsNeverDiscoveredOnOtherOperations(t *testing.T) { for _, op := range []admissionv1.Operation{admissionv1.Delete, admissionv1.Connect, ""} { t.Run("operation "+string(op), func(t *testing.T) { d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} - old := hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + old := hardwareSpecV1beta2(new("h100_sxm"), new(int32(4))) got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), op, old, hardwareSpecV1beta2(nil, nil)) if err != nil { t.Fatal(err) @@ -212,7 +210,7 @@ func TestHardwareIsNeverDiscoveredOnOtherOperations(t *testing.T) { func TestResolveHardwareByOperationV1beta2(t *testing.T) { full := func() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - return hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), v1beta2Ptr(int32(4))) + return hardwareSpecV1beta2(new("h100_sxm"), new(int32(4))) } tests := []struct { name string @@ -227,21 +225,21 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { {"metadata-only update without hardware does not discover", admissionv1.Update, hardwareSpecV1beta2(nil, nil), hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, {"update with unchanged incomplete hardware does not discover", admissionv1.Update, - hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + hardwareSpecV1beta2(new("h100_sxm"), nil), hardwareSpecV1beta2(new("h100_sxm"), nil), hardwareDefaultsV1beta2{}, 0, ""}, {"update without a stored spec does not discover", admissionv1.Update, nil, hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, {"removing only the budget discovers only the budget", admissionv1.Update, - full(), hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil), + full(), hardwareSpecV1beta2(new("h100_sxm"), nil), hardwareDefaultsV1beta2{budget: 8}, 1, "h100_sxm"}, {"removing only the SKU discovers only the SKU", admissionv1.Update, - full(), hardwareSpecV1beta2(nil, v1beta2Ptr(int32(4))), + full(), hardwareSpecV1beta2(nil, new(int32(4))), hardwareDefaultsV1beta2{sku: "h200_sxm"}, 1, ""}, {"removing all typed hardware discovers both", admissionv1.Update, full(), hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, {"changing to complete hardware verifies the new SKU", admissionv1.Update, - full(), hardwareSpecV1beta2(v1beta2Ptr("b200_sxm"), v1beta2Ptr(int32(2))), + full(), hardwareSpecV1beta2(new("b200_sxm"), new(int32(2))), hardwareDefaultsV1beta2{}, 1, "b200_sxm"}, } for _, tt := range tests { @@ -260,7 +258,7 @@ func TestResolveHardwareByOperationV1beta2(t *testing.T) { } func TestDecodeOldSpecV1beta2(t *testing.T) { - old, err := json.Marshal(v1beta2Envelope(t, hardwareSpecV1beta2(v1beta2Ptr("h100_sxm"), nil))) + old, err := json.Marshal(v1beta2Envelope(t, hardwareSpecV1beta2(new("h100_sxm"), nil))) if err != nil { t.Fatal(err) } diff --git a/deploy/operator/internal/webhook/request_version_test.go b/deploy/operator/internal/webhook/request_version_test.go index 837bf346d0a4..312f9ce7f87c 100644 --- a/deploy/operator/internal/webhook/request_version_test.go +++ b/deploy/operator/internal/webhook/request_version_test.go @@ -20,7 +20,6 @@ package webhook import ( "testing" - admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) @@ -28,9 +27,8 @@ import ( // requestWithKinds returns a request whose stored kind is v1beta1, the version every webhook // receives after conversion, and whose original request kind is requestKind (empty for none). func requestWithKinds(requestKind string) admission.Request { - req := admission.Request{AdmissionRequest: admissionv1.AdmissionRequest{ - Kind: metav1.GroupVersionKind{Group: "nvidia.com", Version: "v1beta1", Kind: "DynamoGraphDeploymentRequest"}, - }} + req := admission.Request{ + Kind: metav1.GroupVersionKind{Group: "nvidia.com", Version: "v1beta1", Kind: "DynamoGraphDeploymentRequest"}} if requestKind != "" { req.RequestKind = &metav1.GroupVersionKind{Group: "nvidia.com", Version: requestKind, Kind: "DynamoGraphDeploymentRequest"} } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 3fcd93813f84..f72a479d1722 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -1407,11 +1407,9 @@ func beta2DGDRForAdmission( func v1beta2RunForAdmission(model string, labels map[string]string) *nvidiacomv1beta2.DynamoGraphDeploymentRun { return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ - TypeMeta: metav1.TypeMeta{ - APIVersion: nvidiacomv1beta2.GroupVersion.String(), - Kind: "DynamoGraphDeploymentRun", - }, - ObjectMeta: metav1.ObjectMeta{Name: "test-v1beta2-dgdr-run", Namespace: "default", Labels: labels}, + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRun", + Name: "test-v1beta2-dgdr-run", Namespace: "default", Labels: labels, Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ Model: &nvidiacomv1beta2.ModelSpec{Name: model}, }}, @@ -1502,13 +1500,6 @@ func v1beta2String(t *testing.T, obj *unstructured.Unstructured, path ...string) return text } -func v1beta2Absent(t *testing.T, obj *unstructured.Unstructured, path ...string) { - t.Helper() - if value, found := v1beta2Field(t, obj, path...); found { - t.Fatalf("%v = %#v, want it absent", path, value) - } -} - func v1beta2Empty(t *testing.T, obj *unstructured.Unstructured, path ...string) { t.Helper() value, found := v1beta2Field(t, obj, path...) From a152f0cf10728c09680249e5679bda43acd36417 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 14:06:50 +0100 Subject: [PATCH 017/100] pr-16661: pr-4: make trial budget explicit Signed-off-by: Dr. Stefan Schimanski --- .../operator/internal/aisimulate/defaults.go | 12 ++--- .../internal/aisimulate/input_test.go | 50 ++++++++----------- 2 files changed, 28 insertions(+), 34 deletions(-) diff --git a/deploy/operator/internal/aisimulate/defaults.go b/deploy/operator/internal/aisimulate/defaults.go index e6721282e186..d26bbd54fea5 100644 --- a/deploy/operator/internal/aisimulate/defaults.go +++ b/deploy/operator/internal/aisimulate/defaults.go @@ -27,8 +27,9 @@ const ( // this value when replacing it. DefaultRequestCount int64 = 1000 - // DefaultMaxTrials bounds a KV-load range search that has no explicit optimizer.max_trials. - // It matches the trial budget of AISimulate's unified CLI. + // DefaultMaxTrials bounds every search that has no explicit optimizer.max_trials. + // It matches the trial budget of AISimulate's unified CLI while making the + // effective input persisted in Run status self-contained. DefaultMaxTrials int64 = 320 ) @@ -59,10 +60,9 @@ func applyDefaults(underlay map[string]any) error { if !has(underlay, trafficStopRequests...) { b.set(float64(DefaultRequestCount), trafficStopRequests) } - fraction, _, _ := unstructured.NestedMap(underlay, trafficLoadFraction...) - if has(fraction, "range") && !has(underlay, optimizerMaxTrials...) { - b.set(float64(DefaultMaxTrials), optimizerMaxTrials) - } + } + if !has(underlay, optimizerMaxTrials...) { + b.set(float64(DefaultMaxTrials), optimizerMaxTrials) } if hasSLABound(underlay) && !has(underlay, optimizationStrictSLA...) { diff --git a/deploy/operator/internal/aisimulate/input_test.go b/deploy/operator/internal/aisimulate/input_test.go index 5d187000ef69..81ffb914bee2 100644 --- a/deploy/operator/internal/aisimulate/input_test.go +++ b/deploy/operator/internal/aisimulate/input_test.go @@ -118,6 +118,7 @@ func TestLowerModelOnlyRequest(t *testing.T) { want := `{"engine":{"backend":{"choices":["vllm","sglang","trtllm"]},"hardware":"h200_sxm","model":"meta-llama/Llama-3.1-8B-Instruct","workers":{"aggregated":{},"decode":{},"prefill":{}}},` + `"evaluation":{"sla":{"itl_ms":30,"ttft_ms":2000}},` + `"optimization":{"constraints":{"max_candidate_gpus":8},"strict_sla":true,"target":"throughput_per_gpu"},` + + `"optimizer":{"max_trials":320},` + `"traffic":{"load":{"concurrency":10,"type":"concurrency"},"source":{"input_tokens":1024,"output_tokens":128,"type":"synthetic"},"stop":{"requests":1000}}}` if got := encode(t, lower(t, defaultedSpec())); got != want { t.Fatalf("effective input\n got: %s\nwant: %s", got, want) @@ -246,32 +247,25 @@ func TestRequestCountDefault(t *testing.T) { } func TestMaxTrialsDefault(t *testing.T) { - t.Run("added for a KV-load range", func(t *testing.T) { - if nested(t, lower(t, withKVRange(defaultedSpec())), "optimizer", "max_trials") != 320.0 { - t.Fatal("no default max_trials for a range search") - } - }) - t.Run("not derived for the model-only fixed concurrency", func(t *testing.T) { - if input := lower(t, defaultedSpec()); nested(t, input, "optimizer") != nil { - t.Fatalf("unexpected optimizer: %s", encode(t, input)) - } - }) - t.Run("not added for a pinned load", func(t *testing.T) { - spec := withKVRange(defaultedSpec()) - spec.Workload.Static.Load.KVCapacityFraction = &v1beta2.KVCapacityFractionSpec{Value: new(0.5)} - input := lower(t, spec) - if nested(t, input, "optimizer") != nil { - t.Fatalf("unexpected optimizer: %s", encode(t, input)) - } - if nested(t, input, "traffic", "load", "fraction") != 0.5 { - t.Fatal("a one-value ratio should be a scalar") - } - }) + emptyWorkload := defaultedSpec() + emptyWorkload.Workload = &v1beta2.WorkloadSpec{} + + for name, spec := range map[string]*v1beta2.DynamoGraphDeploymentRequestSpec{ + "model-only fixed concurrency": defaultedSpec(), + "KV-load range": withKVRange(defaultedSpec()), + "empty workload": emptyWorkload, + } { + t.Run(name, func(t *testing.T) { + if nested(t, lower(t, spec), "optimizer", "max_trials") != 320.0 { + t.Fatal("optimizer.max_trials not derived") + } + }) + } t.Run("override value or null wins", func(t *testing.T) { - if nested(t, lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":7}}`)), "optimizer", "max_trials") != json.Number("7") { + if nested(t, lower(t, withInputOverrides(defaultedSpec(), `{"optimizer":{"max_trials":7}}`)), "optimizer", "max_trials") != json.Number("7") { t.Fatal("override max_trials ignored") } - if nested(t, lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":null}}`)), "optimizer", "max_trials") != nil { + if nested(t, lower(t, withInputOverrides(defaultedSpec(), `{"optimizer":{"max_trials":null}}`)), "optimizer", "max_trials") != nil { t.Fatal("max_trials re-added after an override null") } }) @@ -318,26 +312,26 @@ func TestStrictSLADerivation(t *testing.T) { func TestInputOverridesApplyOnceAfterDerivedDefaults(t *testing.T) { t.Run("optimizer null removes the complete derived optimizer", func(t *testing.T) { - input := lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":null}`)) + input := lower(t, withInputOverrides(defaultedSpec(), `{"optimizer":null}`)) if nested(t, input, "optimizer") != nil { t.Fatalf("optimizer survived an override null: %s", encode(t, input)) } }) t.Run("max_trials null removes only max_trials", func(t *testing.T) { - input := lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":null,"seed":3}}`)) + input := lower(t, withInputOverrides(defaultedSpec(), `{"optimizer":{"max_trials":null,"seed":3}}`)) if nested(t, input, "optimizer", "max_trials") != nil || nested(t, input, "optimizer", "seed") != json.Number("3") { t.Fatalf("unexpected optimizer: %s", encode(t, input)) } }) t.Run("an override max_trials replaces the derived value", func(t *testing.T) { - input := lower(t, withInputOverrides(withKVRange(defaultedSpec()), `{"optimizer":{"max_trials":7}}`)) + input := lower(t, withInputOverrides(defaultedSpec(), `{"optimizer":{"max_trials":7}}`)) if nested(t, input, "optimizer", "max_trials") != json.Number("7") { t.Fatalf("unexpected optimizer: %s", encode(t, input)) } }) - t.Run("nothing is derived from values only overrides introduce", func(t *testing.T) { + t.Run("cross-field defaults are not derived from values only overrides introduce", func(t *testing.T) { input := lower(t, withInputOverrides(withoutSLA(defaultedSpec()), `{"traffic":{"load":{"fraction":{"range":{"min":0.1,"max":1}}}},"evaluation":{"sla":{"ttft_ms":500}}}`)) - if nested(t, input, "optimizer") != nil || nested(t, input, "optimization", "strict_sla") != nil { + if nested(t, input, "optimizer", "max_trials") != 320.0 || nested(t, input, "optimization", "strict_sla") != nil { t.Fatalf("a field was derived from override-only values: %s", encode(t, input)) } }) From 2204938c2cdc597dbf99d8e8484d1b1f8fc066b0 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 15:00:11 +0100 Subject: [PATCH 018/100] pr-16661: pr-4: consolidate DGDR admission tests Signed-off-by: Dr. Stefan Schimanski --- ...namographdeploymentrequest_routing_test.go | 109 ---- ...namographdeploymentrequest_v1beta2_test.go | 306 ---------- .../webhook/defaulting/suite_envtest_test.go | 263 ++++++++- ...hdeploymentrequest_carrier_envtest_test.go | 3 + ...namographdeploymentrequest_v1beta2_test.go | 61 -- ...ploymentrequest_validation_envtest_test.go | 547 ------------------ 6 files changed, 265 insertions(+), 1024 deletions(-) delete mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go delete mode 100644 deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go delete mode 100644 deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go deleted file mode 100644 index d39b0093148f..000000000000 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_routing_test.go +++ /dev/null @@ -1,109 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package defaulting - -import ( - "bytes" - "context" - "testing" - - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - admissionv1 "k8s.io/api/admission/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" -) - -// ctxForRequest builds the admission context of a request the API server converted to -// v1beta1. requestVersion is the version the client used; empty means no conversion. -func ctxForRequest(op admissionv1.Operation, requestVersion string) context.Context { - gvk := nvidiacomv1beta1.DynamoGraphDeploymentRequestGVK - req := admission.Request{ - Operation: op, - Kind: metav1.GroupVersionKind{Group: gvk.Group, Version: gvk.Version, Kind: gvk.Kind}} - if requestVersion != "" { - req.RequestKind = &metav1.GroupVersionKind{Group: gvk.Group, Version: requestVersion, Kind: gvk.Kind} - } - return admission.NewContextWithRequest(context.Background(), req) -} - -const modelOnlyCarrier = `{"model":{"name":"Qwen/Qwen3-32B"}}` - -func carrierRequest() *nvidiacomv1beta1.DynamoGraphDeploymentRequest { - return &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ - Name: "r", Namespace: "default", - Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ - V1Beta2: &apiextensionsv1.JSON{Raw: []byte(modelOnlyCarrier)}, - }, - } -} - -func TestDGDRDefaulter_RoutesByRequestedVersion(t *testing.T) { - d := NewDGDRDefaulter("1.6.0", "").WithV1beta2(DGDRV1beta2Options{}) - - t.Run("RequestKind decides, not the converted Kind", func(t *testing.T) { - dgdr := carrierRequest() - if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { - t.Fatalf("Default() error = %v", err) - } - if dgdr.Spec.Image != "" { - t.Fatalf("v1beta1 image defaulting touched a v1beta2 request: %q", dgdr.Spec.Image) - } - if !bytes.Contains(dgdr.Spec.V1Beta2.Raw, []byte(`"backends"`)) { - t.Fatalf("native v1beta2 defaults were not applied: %s", dgdr.Spec.V1Beta2.Raw) - } - }) - - for _, version := range []string{"", "v1beta1", "v1alpha1"} { - t.Run("a foreign carrier is never mutated, requested as "+version, func(t *testing.T) { - for _, op := range []admissionv1.Operation{admissionv1.Create, admissionv1.Update} { - dgdr := carrierRequest() - before := append([]byte(nil), dgdr.Spec.V1Beta2.Raw...) - if err := d.Default(ctxForRequest(op, version), dgdr); err != nil { - t.Fatalf("Default() error = %v", err) - } - if !bytes.Equal(dgdr.Spec.V1Beta2.Raw, before) { - t.Fatalf("%s: carrier changed from %s to %s", op, before, dgdr.Spec.V1Beta2.Raw) - } - if dgdr.Spec.Image != "" { - t.Fatalf("%s: native v1beta1 defaults reached a foreign carrier", op) - } - } - }) - } - - t.Run("a v1beta1 payload written through v1beta2 is left for validation", func(t *testing.T) { - dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{Model: "m"}} - if err := d.Default(ctxForRequest(admissionv1.Create, "v1beta2"), dgdr); err != nil { - t.Fatalf("Default() error = %v", err) - } - if dgdr.Spec.Image != "" { - t.Fatalf("a foreign v1beta1 payload was defaulted: image %q", dgdr.Spec.Image) - } - }) - - t.Run("a native v1beta1 request keeps its defaults", func(t *testing.T) { - dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{Model: "m"}} - if err := d.Default(ctxForRequest(admissionv1.Create, ""), dgdr); err != nil { - t.Fatalf("Default() error = %v", err) - } - if dgdr.Spec.Image == "" { - t.Fatal("native v1beta1 image defaulting stopped working") - } - }) -} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go deleted file mode 100644 index 3328c944b832..000000000000 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_test.go +++ /dev/null @@ -1,306 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package defaulting - -import ( - "context" - "encoding/json" - "testing" - "time" - - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" - admissionv1 "k8s.io/api/admission/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" -) - -type fakeDiscoverer struct { - found gpu.GPUInfo - err error - calls int - filter string - timeout time.Duration -} - -func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { - f.calls++ - f.filter = string(filterSKU) - if deadline, ok := ctx.Deadline(); ok { - f.timeout = time.Until(deadline) - } - found := f.found - return &found, f.err -} - -func TestResolveHardwareV1beta2(t *testing.T) { - spec := func(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - s := modelOnlySpecV1beta2() - if sku != nil || budget != nil { - s.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} - } - return s - } - - t.Run("no discoverer", func(t *testing.T) { - if got, err := (DGDRV1beta2Options{}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || !(got == hardwareDefaultsV1beta2{}) { - t.Fatalf("got %+v", got) - } - }) - t.Run("complete hardware verifies its SKU against the cluster", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1, TotalGPUs: 4}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(new("b200_sxm"), new(int32(2)))) - if err != nil { - t.Fatal(err) - } - if d.calls != 1 || d.filter != "b200_sxm" || !(got == hardwareDefaultsV1beta2{}) { - t.Fatalf("calls=%d got=%+v", d.calls, got) - } - }) - t.Run("everything missing", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 16, NodesWithGPUs: 1, TotalGPUs: 16}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) - if err != nil { - t.Fatal(err) - } - if got.sku != "h200_sxm" || got.budget != 16 || d.filter != "" { - t.Fatalf("got %+v, filter %q", got, d.filter) - } - if d.timeout <= 0 || d.timeout > defaultHardwareDiscoveryTimeout { - t.Fatalf("discovery timeout = %v", d.timeout) - } - }) - t.Run("a pinned SKU filters discovery and keeps its value", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "b200_sxm", GPUsPerNode: 4, NodesWithGPUs: 1, TotalGPUs: 4}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(new("b200_sxm"), nil)) - if err != nil { - t.Fatal(err) - } - if d.filter != "b200_sxm" || got.sku != "" || got.budget != 4 { - t.Fatalf("got %+v, filter %q", got, d.filter) - } - }) - t.Run("a pinned budget keeps its value", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 64, NodesWithGPUs: 1}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, new(int32(3)))) - if err != nil { - t.Fatal(err) - } - if got.sku != "h200_sxm" || got.budget != 0 { - t.Fatalf("got %+v", got) - } - }) - t.Run("the budget is capped", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 1024, NodesWithGPUs: 1, TotalGPUs: 1024}} - if got, err := (DGDRV1beta2Options{Hardware: d}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || got.budget != 32 { - t.Fatalf("default cap: %+v", got) - } - if got, err := (DGDRV1beta2Options{Hardware: d, MaxHardwareBudget: 8}).resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)); err != nil || got.budget != 8 { - t.Fatalf("explicit cap: %+v", got) - } - }) - t.Run("the timeout is configurable", func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{GPUsPerNode: 1, NodesWithGPUs: 1}} - _, _ = DGDRV1beta2Options{Hardware: d, HardwareDiscoveryTimeout: time.Hour}.resolveHardware(context.Background(), admissionv1.Create, nil, spec(nil, nil)) - if d.timeout < time.Minute { - t.Fatalf("timeout = %v", d.timeout) - } - }) -} - -func TestResolveDefaultsV1beta2(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}} - got, err := DGDRV1beta2Options{Hardware: d}. - resolveDefaults(context.Background(), admissionv1.Create, nil, modelOnlySpecV1beta2()) - if err != nil { - t.Fatal(err) - } - if got.hardware.sku != "h200_sxm" || got.hardware.budget != 8 { - t.Fatalf("got %+v", got) - } -} - -func v1beta2Envelope(t *testing.T, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) *nvidiacomv1beta1.DynamoGraphDeploymentRequest { - t.Helper() - raw, err := json.Marshal(spec) - if err != nil { - t.Fatal(err) - } - dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} - dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: raw} - return dgdr -} - -func TestDefaultEnvelopeV1beta2(t *testing.T) { - options := DGDRV1beta2Options{ - Hardware: &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}}, - } - dgdr := v1beta2Envelope(t, modelOnlySpecV1beta2()) - - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { - t.Fatal(err) - } - var got nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec - if err := json.Unmarshal(dgdr.Spec.V1Beta2.Raw, &got); err != nil { - t.Fatal(err) - } - if got.Hardware == nil || got.Hardware.GPU == nil || *got.Hardware.GPU.SKU != "h200_sxm" || *got.Hardware.GPU.Budget != 8 { - t.Fatalf("hardware not defaulted: %+v", got.Hardware) - } - if got.Objective == nil || got.Objective.Metric == nil { - t.Fatalf("objective metric not defaulted: %+v", got.Objective) - } - - // A second admission of the defaulted object changes nothing, byte for byte. - first := string(dgdr.Spec.V1Beta2.Raw) - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, options); err != nil { - t.Fatal(err) - } - if string(dgdr.Spec.V1Beta2.Raw) != first { - t.Fatalf("not idempotent:\n%s\n%s", first, dgdr.Spec.V1Beta2.Raw) - } -} - -func TestDefaultEnvelopeV1beta2RejectsMalformedSpec(t *testing.T) { - dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{} - dgdr.Spec.V1Beta2 = &apiextensionsv1.JSON{Raw: []byte(`{"model": 3}`)} - if err := defaultDynamoGraphDeploymentRequestV1beta2(context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err == nil { - t.Fatal("want a decode error") - } -} - -func hardwareSpecV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - spec := modelOnlySpecV1beta2() - if sku != nil || budget != nil { - spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} - } - return spec -} - -func TestHardwareIsNeverDiscoveredOnOtherOperations(t *testing.T) { - for _, op := range []admissionv1.Operation{admissionv1.Delete, admissionv1.Connect, ""} { - t.Run("operation "+string(op), func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}} - old := hardwareSpecV1beta2(new("h100_sxm"), new(int32(4))) - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), op, old, hardwareSpecV1beta2(nil, nil)) - if err != nil { - t.Fatal(err) - } - if got != (hardwareDefaultsV1beta2{}) || d.calls != 0 { - t.Fatalf("hardware resolved on %q: %+v, calls %d", op, got, d.calls) - } - }) - } -} - -func TestResolveHardwareByOperationV1beta2(t *testing.T) { - full := func() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - return hardwareSpecV1beta2(new("h100_sxm"), new(int32(4))) - } - tests := []struct { - name string - op admissionv1.Operation - old, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec - want hardwareDefaultsV1beta2 - wantCalls int - wantFilter string - }{ - {"create discovers everything", admissionv1.Create, nil, hardwareSpecV1beta2(nil, nil), - hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, - {"metadata-only update without hardware does not discover", admissionv1.Update, - hardwareSpecV1beta2(nil, nil), hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, - {"update with unchanged incomplete hardware does not discover", admissionv1.Update, - hardwareSpecV1beta2(new("h100_sxm"), nil), hardwareSpecV1beta2(new("h100_sxm"), nil), - hardwareDefaultsV1beta2{}, 0, ""}, - {"update without a stored spec does not discover", admissionv1.Update, - nil, hardwareSpecV1beta2(nil, nil), hardwareDefaultsV1beta2{}, 0, ""}, - {"removing only the budget discovers only the budget", admissionv1.Update, - full(), hardwareSpecV1beta2(new("h100_sxm"), nil), - hardwareDefaultsV1beta2{budget: 8}, 1, "h100_sxm"}, - {"removing only the SKU discovers only the SKU", admissionv1.Update, - full(), hardwareSpecV1beta2(nil, new(int32(4))), - hardwareDefaultsV1beta2{sku: "h200_sxm"}, 1, ""}, - {"removing all typed hardware discovers both", admissionv1.Update, - full(), hardwareSpecV1beta2(nil, nil), - hardwareDefaultsV1beta2{sku: "h200_sxm", budget: 8}, 1, ""}, - {"changing to complete hardware verifies the new SKU", admissionv1.Update, - full(), hardwareSpecV1beta2(new("b200_sxm"), new(int32(2))), - hardwareDefaultsV1beta2{}, 1, "b200_sxm"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - d := &fakeDiscoverer{found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}} - got, err := DGDRV1beta2Options{Hardware: d}.resolveHardware(context.Background(), tt.op, tt.old, tt.spec) - if err != nil { - t.Fatal(err) - } - if got != tt.want || d.calls != tt.wantCalls || (tt.wantCalls > 0 && d.filter != tt.wantFilter) { - t.Fatalf("got %+v, calls %d, filter %q; want %+v, calls %d, filter %q", - got, d.calls, d.filter, tt.want, tt.wantCalls, tt.wantFilter) - } - }) - } -} - -func TestDecodeOldSpecV1beta2(t *testing.T) { - old, err := json.Marshal(v1beta2Envelope(t, hardwareSpecV1beta2(new("h100_sxm"), nil))) - if err != nil { - t.Fatal(err) - } - got, err := decodeOldSpecV1beta2(admissionv1.Update, old) - if err != nil || got == nil || got.Hardware == nil || *got.Hardware.GPU.SKU != "h100_sxm" { - t.Fatalf("got %+v, err %v", got, err) - } - if got, err := decodeOldSpecV1beta2(admissionv1.Create, old); got != nil || err != nil { - t.Fatalf("create has no old spec: %+v, %v", got, err) - } - if got, err := decodeOldSpecV1beta2(admissionv1.Update, []byte(`{}`)); got != nil || err != nil { - t.Fatalf("an old object without the envelope has no old spec: %+v, %v", got, err) - } - if _, err := decodeOldSpecV1beta2(admissionv1.Update, []byte(`{`)); err == nil { - t.Fatal("want a decode error") - } -} - -// modelOnlySpecV1beta2 is a Request as a client sends it: only the model is named. -func modelOnlySpecV1beta2() *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { - spec := &nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{} - spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "meta-llama/Llama-3.1-8B-Instruct"} - return spec -} - -// An envelope that admission has nothing to add to keeps its exact bytes, even when they are -// not in canonical form. -func TestDefaultEnvelopeV1beta2LeavesUnchangedEnvelopeUntouched(t *testing.T) { - spec := modelOnlySpecV1beta2() - defaultDynamoGraphDeploymentRequestSpecV1beta2(spec, dynamoGraphDeploymentRequestV1beta2DefaultOptions{}) - raw, err := json.Marshal(spec) - if err != nil { - t.Fatal(err) - } - dgdr := v1beta2Envelope(t, spec) - dgdr.Spec.V1Beta2.Raw = append([]byte(" "), raw...) - original := string(dgdr.Spec.V1Beta2.Raw) - if err := defaultDynamoGraphDeploymentRequestV1beta2( - context.Background(), dgdr, admissionv1.Create, nil, DGDRV1beta2Options{}); err != nil { - t.Fatal(err) - } - if string(dgdr.Spec.V1Beta2.Raw) != original { - t.Fatalf("envelope was rewritten without a change: %q", dgdr.Spec.V1Beta2.Raw) - } -} diff --git a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go index 87661ec6b97d..ea523300331d 100644 --- a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go @@ -6,14 +6,22 @@ package defaulting_test import ( + "context" + "encoding/json" + "errors" "os" "reflect" + "strings" + "sync" "testing" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -24,6 +32,8 @@ import ( const defaultingAdmissionOperatorVersion = "1.6.0" +var defaultingAdmissionHardware = &defaultingHardwareDiscoverer{} + var defaultingAdmissionEnv = operatorenv.New(operatorenv.Options{ Admission: operatorenv.AdmissionWebhooks{Mutating: true}, SetupWebhooks: func(mgr ctrl.Manager, opts operatorenv.WebhookSetupOptions) error { @@ -32,12 +42,34 @@ var defaultingAdmissionEnv = operatorenv.New(operatorenv.Options{ RuntimeConfig: opts.RuntimeConfig, OperatorVersion: opts.OperatorVersion, OperatorPrincipal: opts.OperatorPrincipal, - Gate: features.Gates{Grove: true}, + Gate: features.Gates{Grove: true, GPUDiscovery: true}, + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + Hardware: defaultingAdmissionHardware, + }, }) }, OperatorVersion: defaultingAdmissionOperatorVersion, }) +type defaultingHardwareDiscoverer struct { + mu sync.Mutex + found gpu.GPUInfo + err error +} + +func (f *defaultingHardwareDiscoverer) set(found gpu.GPUInfo, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.found, f.err = found, err +} + +func (f *defaultingHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + found := f.found + return &found, f.err +} + func TestMain(m *testing.M) { os.Exit(defaultingAdmissionEnv.RunM(m)) } @@ -269,6 +301,235 @@ func TestDynamoGraphDeploymentRequestDefaultingAdmission(t *testing.T) { } }) } + + discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} + v1beta2Tests := []struct { + name string + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest + discovered gpu.GPUInfo + discoveryErr error + wantCreateErr string + check func(*testing.T, *nvidiacomv1beta2.DynamoGraphDeploymentRequest) + update func(*nvidiacomv1beta2.DynamoGraphDeploymentRequest) + beforeUpdate func() + checkUpdate func(*testing.T, *nvidiacomv1beta2.DynamoGraphDeploymentRequest) + idempotent bool + }{ + { + name: "v1beta2 model-only request combines all admission defaults", + request: v1beta2DefaultingRequest(), + discovered: discoveredH200, + check: expectV1beta2ModelOnlyDefaults, + idempotent: true, + }, + { + name: "v1beta2 explicit empty static workload and SLA opt out", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}} + spec.Objective = &nvidiacomv1beta2.ObjectiveSpec{SLA: &nvidiacomv1beta2.ObjectiveSLASpec{}} + }), + discovered: discoveredH200, + check: func(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + if got := request.Spec.Workload.Static; got == nil || got.InputTokens != nil || got.OutputTokens != nil || got.Load != nil { + t.Fatalf("workload.static = %#v, want explicit empty opt-out", got) + } + if got := request.Spec.Objective.SLA; got == nil || got.TTFTMs != nil || got.ITLMs != nil || got.E2EMs != nil { + t.Fatalf("objective.sla = %#v, want explicit empty opt-out", got) + } + if request.Spec.Objective.Mode != nvidiacomv1beta2.ObjectiveModeOptimize || request.Spec.Objective.Metric == nil { + t.Fatalf("objective = %#v, want mode and metric defaults beside empty SLA", request.Spec.Objective) + } + }, + }, + { + name: "v1beta2 explicit empty parent objects receive their defaults", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{} + spec.Objective = &nvidiacomv1beta2.ObjectiveSpec{} + spec.Recommendation = &nvidiacomv1beta2.RecommendationSpec{} + }), + discovered: discoveredH200, + check: expectV1beta2ModelOnlyDefaults, + }, + { + name: "v1beta2 explicit values and public input overrides are preserved", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Backends = []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM} + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{InputTokens: new(int32(10))}} + metric := nvidiacomv1beta2.ObjectiveMetric("TTFT") + spec.Objective = &nvidiacomv1beta2.ObjectiveSpec{Metric: &metric} + spec.Recommendation = &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 3} + spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{InputOverrides: &runtime.RawExtension{ + Raw: []byte(`{"engine":{"hardware":"h100_sxm"},"custom":"kept"}`), + }} + }), + discovered: discoveredH200, + check: func(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + if want := []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM}; !reflect.DeepEqual(request.Spec.Backends, want) { + t.Fatalf("backends = %v, want %v", request.Spec.Backends, want) + } + if request.Spec.Workload == nil || request.Spec.Workload.Static == nil { + t.Fatalf("workload = %#v, want the partial static workload", request.Spec.Workload) + } + static := request.Spec.Workload.Static + if ptr.Deref(static.InputTokens, 0) != 10 || ptr.Deref(static.OutputTokens, 0) != 128 || static.Load == nil || ptr.Deref(static.Load.Concurrency, 0) != 10 { + t.Fatalf("workload.static = %#v, want explicit inputTokens and omitted-member defaults", static) + } + if request.Spec.Objective.Mode != nvidiacomv1beta2.ObjectiveModeOptimize || ptr.Deref(request.Spec.Objective.Metric, "") != nvidiacomv1beta2.ObjectiveMetric("TTFT") { + t.Fatalf("objective = %#v, want explicit TTFT metric with Optimize default", request.Spec.Objective) + } + if request.Spec.Recommendation.MaxCandidates != 3 { + t.Fatalf("maxCandidates = %d, want 3", request.Spec.Recommendation.MaxCandidates) + } + if request.Spec.AISimulate == nil || request.Spec.AISimulate.InputOverrides == nil { + t.Fatalf("aisimulate = %#v, want input overrides", request.Spec.AISimulate) + } + var overrides map[string]any + if err := json.Unmarshal(request.Spec.AISimulate.InputOverrides.Raw, &overrides); err != nil { + t.Fatalf("decode input overrides: %v", err) + } + if overrides["custom"] != "kept" { + t.Fatalf("inputOverrides = %#v, want custom value untouched", overrides) + } + engine, _ := overrides["engine"].(map[string]any) + if engine["hardware"] != "h100_sxm" { + t.Fatalf("inputOverrides = %#v, want engine.hardware untouched", overrides) + } + expectV1beta2Hardware(t, request, "h200_sxm", 8) + }, + }, + { + name: "v1beta2 unresolved explicit hardware reports its public field path", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: new("h100_sxm"), Budget: new(int32(4)), + }} + }), + discoveryErr: errors.New("no matching GPUs found"), + wantCreateErr: `spec.hardware.gpu.sku: Invalid value: "h100_sxm": could not be verified against the cluster's GPUs (no matching GPUs found)`, + }, + { + name: "v1beta2 metadata-only update is idempotent and does not rediscover hardware", + request: v1beta2DefaultingRequest(), + discovered: discoveredH200, + update: func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + request.Labels = map[string]string{"updated": "true"} + }, + beforeUpdate: func() { + defaultingAdmissionHardware.set(gpu.GPUInfo{}, errors.New("cluster unreadable")) + }, + checkUpdate: func(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + expectV1beta2Hardware(t, request, "h200_sxm", 8) + if request.Generation != 1 { + t.Fatalf("generation = %d, want 1 after metadata-only update", request.Generation) + } + }, + }, + { + name: "v1beta2 removing hardware on update rediscovers both fields", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: new("h200_sxm"), Budget: new(int32(4)), + }} + }), + discovered: discoveredH200, + update: func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + request.Spec.Hardware = nil + }, + checkUpdate: func(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + expectV1beta2Hardware(t, request, "h200_sxm", 8) + if request.Generation != 2 { + t.Fatalf("generation = %d, want 2 after hardware update", request.Generation) + } + }, + }, + } + + for _, tt := range v1beta2Tests { + t.Run(tt.name, func(t *testing.T) { + defaultingAdmissionHardware.set(tt.discovered, tt.discoveryErr) + t.Cleanup(func() { defaultingAdmissionHardware.set(gpu.GPUInfo{}, nil) }) + env := defaultingAdmissionEnv.ForTest(t) + request := tt.request.DeepCopy() + request.Name, request.Namespace = "request", env.Namespace() + + t.Log("Create the v1beta2 DGDR through the API server and production mutating handler") + err := env.Client().Create(t.Context(), request) + if tt.wantCreateErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantCreateErr) { + t.Fatalf("create v1beta2 DGDR error = %v, want it to contain %q", err, tt.wantCreateErr) + } + return + } + if err != nil { + t.Fatalf("create v1beta2 DGDR: %v", err) + } + if tt.check != nil { + tt.check(t, request) + } + if tt.idempotent { + assertCreateDefaultingIdempotent(t, env.Client(), request) + } + + if tt.update != nil { + tt.update(request) + if tt.beforeUpdate != nil { + tt.beforeUpdate() + } + t.Log("Update the v1beta2 DGDR through the same admission chain") + if err := env.Client().Update(t.Context(), request); err != nil { + t.Fatalf("update v1beta2 DGDR: %v", err) + } + tt.checkUpdate(t, request) + } + }) + } +} + +func v1beta2DefaultingRequest(mutate ...func(*nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec)) *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + DynamoGraphDeploymentRequestNativeSpec: nvidiacomv1beta2.DynamoGraphDeploymentRequestNativeSpec{ + NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-8B"}, + }, + }, + }, + } + for _, apply := range mutate { + apply(&request.Spec) + } + return request +} + +func expectV1beta2ModelOnlyDefaults(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + if want := []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendSGLang, nvidiacomv1beta2.BackendTRTLLM}; !reflect.DeepEqual(request.Spec.Backends, want) { + t.Fatalf("backends = %v, want %v", request.Spec.Backends, want) + } + expectV1beta2Hardware(t, request, "h200_sxm", 8) + if request.Spec.Workload == nil { + t.Fatal("workload is nil, want model-only defaults") + } + static := request.Spec.Workload.Static + if static == nil || ptr.Deref(static.InputTokens, 0) != 1024 || ptr.Deref(static.OutputTokens, 0) != 128 || static.Load == nil || ptr.Deref(static.Load.Concurrency, 0) != 10 { + t.Fatalf("workload = %#v, want 1024/128 tokens at concurrency 10", request.Spec.Workload) + } + objective := request.Spec.Objective + if objective == nil || objective.Mode != nvidiacomv1beta2.ObjectiveModeOptimize || ptr.Deref(objective.Metric, "") != nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") || objective.SLA == nil || ptr.Deref(objective.SLA.TTFTMs, 0) != 2000 || ptr.Deref(objective.SLA.ITLMs, 0) != 30 { + t.Fatalf("objective = %#v, want Optimize/ThroughputPerGPU with 2000/30 ms SLA", objective) + } + if request.Spec.Recommendation == nil || request.Spec.Recommendation.MaxCandidates != 1 { + t.Fatalf("recommendation = %#v, want maxCandidates 1", request.Spec.Recommendation) + } +} + +func expectV1beta2Hardware(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, sku string, budget int32) { + t.Helper() + hardware := request.Spec.Hardware + if hardware == nil || hardware.GPU == nil || ptr.Deref(hardware.GPU.SKU, "") != sku || ptr.Deref(hardware.GPU.Budget, 0) != budget { + t.Fatalf("hardware = %#v, want SKU %q and budget %d", hardware, sku, budget) + } } func assertCreateDefaultingIdempotent(t *testing.T, kubeClient client.Client, object client.Object) { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go index 2503a0da41e2..dd7123500a9e 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -25,6 +25,9 @@ func TestDynamoGraphDeploymentRequestCarrierObjectsStayWritable(t *testing.T) { t.Run("native v1beta2 written through v1beta1", func(t *testing.T) { carrierRoundTrip(t, beta2DGDRForAdmission(setBeta2AdmissionHardware), betaDGDRForAdmission(nil), "v1beta2") }) + t.Run("native v1beta2 written through v1alpha1", func(t *testing.T) { + carrierRoundTrip(t, beta2DGDRForAdmission(setBeta2AdmissionHardware), alphaDGDRForAdmission(nil), "v1beta2") + }) } func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, carrierKey string) { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go deleted file mode 100644 index 981129efb985..000000000000 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package validation - -import ( - "strings" - "testing" - - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -func v1beta2Details(errs field.ErrorList) string { - var out []string - for _, err := range errs { - out = append(out, err.Field+": "+err.Detail) - } - return strings.Join(out, "\n") -} - -func v1beta2RequireError(t *testing.T, errs field.ErrorList, substr string) { - t.Helper() - if !strings.Contains(v1beta2Details(errs), substr) { - t.Fatalf("want an error containing %q, got:\n%s", substr, v1beta2Details(errs)) - } -} - -func v1beta2RequireNoError(t *testing.T, errs field.ErrorList) { - t.Helper() - if len(errs) > 0 { - t.Fatalf("unexpected errors:\n%s", v1beta2Details(errs)) - } -} - -func v1beta2Envelope(raw string) *apiextensionsv1.JSON { - return &apiextensionsv1.JSON{Raw: []byte(raw)} -} - -func TestValidateEnvelopeV1beta2(t *testing.T) { - path := field.NewPath("spec", "v1beta2") - - v1beta2RequireNoError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2( - v1beta2Envelope(`{"model":{"name":"meta-llama/Llama-3.1-8B-Instruct"}}`), path)) - - v1beta2RequireError(t, validateDynamoGraphDeploymentRequestEnvelopeV1beta2(v1beta2Envelope(`{"model": 3}`), path), "spec.v1beta2: must be a valid v1beta2 spec") -} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index f72a479d1722..d22e7f800890 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -18,9 +18,6 @@ package validation_test import ( - "encoding/json" - "errors" - "reflect" "strings" "testing" @@ -549,212 +546,6 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, - // v1beta2: the CRD owns the unconditional defaults, admission the environment ones. - { - name: "v1beta2 model-only request gets schema and admission defaults", - request: v1beta2DGDRForAdmission(nil, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2StructuralDefaults(t, result) - expectV1beta2AdmissionDefaults(t, result) - }, - }, - { - name: "v1beta2 explicit hardware is kept after admission verifies the SKU", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != "h100_sxm" { - t.Fatalf("hardware.gpu.sku = %q, want h100_sxm", got) - } - if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != 4 { - t.Fatalf("hardware.gpu.budget = %v, want 4", got) - } - }, - }, - { - name: "v1beta2 explicit hardware is rejected when the SKU cannot be verified", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} - }, nil), - gpuDiscovery: true, - discoveryErr: errors.New("no matching GPUs found"), - wantWebhook: []string{ - `spec.hardware.gpu.sku: Invalid value: "h100_sxm": could not be verified against the cluster's GPUs (no matching GPUs found)`, - }, - }, - { - name: "v1beta2 input overrides are stored untouched and never decide hardware defaulting", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2InputOverrides(spec, map[string]any{ - "engine": map[string]any{"hardware": "h100_sxm"}, - "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": int64(4)}}, - }) - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h200_sxm", 8) - if got := v1beta2String(t, result, "spec", "aisimulate", "inputOverrides", "engine", "hardware"); got != "h100_sxm" { - t.Fatalf("inputOverrides.engine.hardware = %q, want h100_sxm", got) - } - }, - }, - { - name: "v1beta2 input overrides null does not suppress hardware discovery", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2InputOverrides(spec, map[string]any{"engine": map[string]any{"hardware": nil}}) - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got == "" { - t.Fatal("hardware.gpu.sku is empty: discovery must read the typed spec only") - } - }, - }, - { - name: "v1beta2 metadata-only update does not read the cluster again", - oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2InputOverrides(spec, map[string]any{ - "engine": map[string]any{"hardware": "h100_sxm"}, - "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": int64(8)}}, - }) - }, nil), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - setV1beta2InputOverrides(spec, map[string]any{ - "engine": map[string]any{"hardware": "h100_sxm"}, - "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": int64(8)}}, - }) - }, map[string]any{"updated": "true"}), - gpuDiscovery: true, - discovered: discoveredH200, - beforeUpdate: func(*testing.T) { - admissionHardware.set(gpu.GPUInfo{}, errors.New("the cluster is unreadable")) - }, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h200_sxm", 8) - if got := result.GetGeneration(); got != 1 { - t.Fatalf("generation = %d, want 1: the spec must not change", got) - } - }, - }, - { - name: "v1beta2 removing only the budget rediscovers only the budget", - oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} - }, nil), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPU(t, result, "h100_sxm", 8) - }, - }, - { - name: "v1beta2 removing only the SKU rediscovers only the SKU", - oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} - }, nil), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(4)}} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPU(t, result, "h200_sxm", 4) - }, - }, - { - name: "v1beta2 removing all typed hardware rediscovers the SKU and the budget", - oldRequest: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} - }, nil), - request: v1beta2DGDRForAdmission(nil, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPU(t, result, "h200_sxm", 8) - }, - }, - { - name: "v1beta2 empty workload receives the static model-only default", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - for key, want := range map[string]float64{"inputTokens": 1024, "outputTokens": 128} { - if got := v1beta2Number(t, result, "spec", "workload", "static", key); got != want { - t.Fatalf("workload.static.%s = %v, want %v", key, got, want) - } - } - if got := v1beta2Number(t, result, "spec", "workload", "static", "load", "concurrency"); got != 10 { - t.Fatalf("workload.static.load.concurrency = %v, want 10", got) - } - }, - }, - { - name: "v1beta2 empty workload static is an opt-out and stays empty", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"static": map[string]any{}} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - v1beta2Empty(t, result, "spec", "workload", "static") - }, - }, - { - name: "v1beta2 empty workload static stays empty beside public input overrides", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"static": map[string]any{}} - spec["aisimulate"] = map[string]any{"inputOverrides": mustJSONObject(t, `{"traffic":{"source":{"type":"synthetic","input_tokens":512,"output_tokens":128},"load":{"type":"concurrency","concurrency":8},"stop":{"requests":100}}}`)} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - v1beta2Empty(t, result, "spec", "workload", "static") - expectV1beta2AdmissionDefaults(t, result) - }, - }, - { - name: "v1beta2 empty sla is an opt-out and is not refilled", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["objective"] = map[string]any{"sla": map[string]any{}} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - v1beta2Empty(t, result, "spec", "objective", "sla") - if got := v1beta2Number(t, result, "spec", "workload", "static", "inputTokens"); got != 1024 { - t.Fatalf("workload.static.inputTokens = %v, want the admission default 1024", got) - } - }, - }, - { - name: "v1beta2 public input overrides win without rewriting the typed spec", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{"inputOverrides": mustJSONObject(t, `{"traffic":{"source":{"input_tokens":256}}}`)} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - if got := v1beta2Number(t, result, "spec", "workload", "static", "inputTokens"); got != 1024 { - t.Fatalf("typed workload.static.inputTokens = %v, want 1024", got) - } - if got := v1beta2Number(t, result, "spec", "aisimulate", "inputOverrides", "traffic", "source", "input_tokens"); got != 256 { - t.Fatalf("inputOverrides.traffic.source.input_tokens = %v, want 256", got) - } - }, - }, { name: "v1beta2 public input overrides may configure non-reserved dgdr run fields", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -831,148 +622,6 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec.aisimulate.inputOverrides.dgdr_run: Forbidden: must be an object because the operator owns required execution fields within it", }, }, - { - name: "v1beta2 hardware omitted is discovered and the budget capped", - request: v1beta2DGDRForAdmission(nil, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64}, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h200_sxm", 32) - }, - }, - { - name: "v1beta2 same-SKU nodes with different GPU counts use the aggregate", - request: v1beta2DGDRForAdmission(nil, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12}, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h200_sxm", 12) - }, - }, - { - name: "v1beta2 an empty hardware object is automatic discovery", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{} - }, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h200_sxm", 8) - }, - }, - { - name: "v1beta2 a SKU alone keeps the SKU and persists the capped budget", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} - }, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64}, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h100_sxm", 32) - }, - }, - { - name: "v1beta2 a budget alone keeps the budget and discovers the SKU", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(4)}} - }, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h200_sxm", 4) - }, - }, - { - name: "v1beta2 a SKU and a budget are kept after SKU verification", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} - }, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h100_sxm", 4) - }, - }, - { - name: "v1beta2 an explicit budget above the cap and the cluster size is kept", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"budget": int64(64)}} - }, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1}, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h200_sxm", 64) - }, - }, - { - name: "v1beta2 a failed discovery is rejected", - request: v1beta2DGDRForAdmission(nil, nil), - gpuDiscovery: true, - discoveryErr: errors.New("no GPUs found"), - wantWebhook: []string{ - "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found); " + - "set spec.hardware.gpu.sku and spec.hardware.gpu.budget", - }, - }, - { - name: "v1beta2 a GPU model the catalog does not know is rejected", - request: v1beta2DGDRForAdmission(nil, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{Model: "Mystery GPU", GPUsPerNode: 8, NodesWithGPUs: 1}, - wantWebhook: []string{ - `spec.hardware.gpu.sku: Required value: the cluster's GPU model "Mystery GPU" maps to no AISimulate SKU; ` + - "set spec.hardware.gpu.sku", - }, - }, - { - name: "v1beta2 a heterogeneous cluster does not pick a SKU", - request: v1beta2DGDRForAdmission(nil, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{ - System: "h200_sxm", Model: "H200", GPUsPerNode: 8, NodesWithGPUs: 1, OtherModels: []string{"H100"}, - }, - wantWebhook: []string{ - "spec.hardware.gpu.sku: Required value: the cluster has several GPU types (H200, H100); " + - "set spec.hardware.gpu.sku", - }, - }, - { - name: "v1beta2 a pinned SKU is accepted in a heterogeneous cluster", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h200_sxm"}} - }, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2GPUValues(t, result, "h200_sxm", 8) - }, - }, - - { - name: "v1beta2 a SKU without any matching GPU is rejected", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm"}} - }, nil), - gpuDiscovery: true, - discovered: gpu.GPUInfo{System: "h100_sxm"}, - wantWebhook: []string{ - `spec.hardware.gpu.sku: Invalid value: "h100_sxm": no GPUs of this SKU were found in the cluster`, - }, - }, - - { - name: "v1beta2 empty objective and recommendation receive their defaults", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["objective"] = map[string]any{} - spec["recommendation"] = map[string]any{} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - expectV1beta2StructuralDefaults(t, result) - expectV1beta2AdmissionDefaults(t, result) - }, - }, { name: "v1beta2 pareto objective is reserved until Candidate publication supports it", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -991,20 +640,6 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { discovered: discoveredH200, wantCELErr: `spec.objective: Invalid value: metrics is reserved for the future Pareto implementation`, }, - { - name: "v1beta2 explicit values are all kept", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["backends"] = []any{"vllm"} - spec["hardware"] = map[string]any{"gpu": map[string]any{"sku": "h100_sxm", "budget": int64(4)}} - spec["workload"] = map[string]any{"static": map[string]any{"inputTokens": int64(10)}} - spec["objective"] = map[string]any{"mode": "Optimize", "metric": "TTFT"} - spec["aisimulate"] = map[string]any{"inputOverrides": map[string]any{"custom": "kept"}} - spec["recommendation"] = map[string]any{"maxCandidates": int64(3)} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: expectV1beta2ExplicitValues, - }, { name: "v1beta2 duplicate backends are rejected on create", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -1072,58 +707,6 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { discovered: discoveredH200, wantCELErr: `spec.workload.static.load: Invalid value: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified`, }, - // v1beta2 UPDATE: defaults stay idempotent, hardware is never rediscovered, and any - // spec change is fully validated. - { - name: "v1beta2 update without a spec change keeps the spec and generation", - oldRequest: v1beta2DGDRForAdmission(nil, nil), - request: v1beta2DGDRForAdmission(nil, map[string]any{"updated": "true"}), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - if result.GetGeneration() != 1 { - t.Fatalf("generation = %d, want 1: defaulting must not change an unchanged spec", result.GetGeneration()) - } - expectV1beta2StructuralDefaults(t, result) - expectV1beta2AdmissionDefaults(t, result) - }, - }, - { - name: "v1beta2 native hardware survives a metadata-only update when the cluster is unreadable", - oldRequest: v1beta2NativeHardwareDGDR(nil), - request: v1beta2NativeHardwareDGDR(map[string]any{"updated": "true"}), - gpuDiscovery: true, - discovered: discoveredH200, - beforeUpdate: func(*testing.T) { - admissionHardware.set(gpu.GPUInfo{}, errors.New("the cluster is unreadable")) - }, - check: func(t *testing.T, result *unstructured.Unstructured) { - if result.GetGeneration() != 1 { - t.Fatalf("generation = %d, want 1", result.GetGeneration()) - } - expectV1beta2GPUValues(t, result, "h200_sxm", 8) - if got := v1beta2String(t, result, "spec", "aisimulate", "inputOverrides", "engine", "hardware"); got != "h100_sxm" { - t.Fatalf("inputOverrides.engine.hardware = %q, want h100_sxm", got) - } - }, - }, - { - name: "v1beta2 spec change is accepted and bumps the generation", - oldRequest: v1beta2DGDRForAdmission(nil, nil), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["model"] = map[string]any{"name": alternateAdmissionModel} - }, nil), - gpuDiscovery: true, - discovered: discoveredH200, - check: func(t *testing.T, result *unstructured.Unstructured) { - if result.GetGeneration() != 2 { - t.Fatalf("generation = %d, want 2", result.GetGeneration()) - } - if got := v1beta2String(t, result, "spec", "model", "name"); got != alternateAdmissionModel { - t.Fatalf("model.name = %q, want %q", got, alternateAdmissionModel) - } - }, - }, { name: "v1beta2 update to duplicate backends is rejected", oldRequest: v1beta2DGDRForAdmission(nil, nil), @@ -1418,18 +1001,6 @@ func v1beta2RunForAdmission(model string, labels map[string]string) *nvidiacomv1 const v1beta2AdmissionModel = "Qwen/Qwen3-0.6B" -// v1beta2NativeHardwareDGDR supplies hardware only through public AISimulate input overrides. -func v1beta2NativeHardwareDGDR(labels map[string]any) *unstructured.Unstructured { - return v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["aisimulate"] = map[string]any{ - "inputOverrides": map[string]any{ - "engine": map[string]any{"hardware": "h100_sxm"}, - "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": int64(8)}}, - }, - } - }, labels) -} - // v1beta2DGDRForAdmission builds a v1beta2 Request as a client sends it, before the CRD // defaults. It is unstructured on purpose: an omitted field, // an explicit empty object and a null are different requests, and typed Go cannot tell them @@ -1451,16 +1022,6 @@ func v1beta2DGDRForAdmission(mutate func(spec map[string]any), labels map[string }} } -// mustJSONObject decodes a native AISimulate input override document. -func mustJSONObject(t *testing.T, document string) map[string]any { - t.Helper() - var value map[string]any - if err := json.Unmarshal([]byte(document), &value); err != nil { - t.Fatalf("decode %q: %v", document, err) - } - return value -} - func v1beta2Field(t *testing.T, obj *unstructured.Unstructured, path ...string) (any, bool) { t.Helper() value, found, err := unstructured.NestedFieldNoCopy(obj.Object, path...) @@ -1500,92 +1061,6 @@ func v1beta2String(t *testing.T, obj *unstructured.Unstructured, path ...string) return text } -func v1beta2Empty(t *testing.T, obj *unstructured.Unstructured, path ...string) { - t.Helper() - value, found := v1beta2Field(t, obj, path...) - if !found || !reflect.DeepEqual(value, map[string]any{}) { - t.Fatalf("%v = %#v (found=%v), want an empty object", path, value, found) - } -} - -// expectV1beta2StructuralDefaults checks the defaults admission supplies to a model-only Request. -func expectV1beta2StructuralDefaults(t *testing.T, obj *unstructured.Unstructured) { - t.Helper() - for key, want := range map[string]float64{"inputTokens": 1024, "outputTokens": 128} { - if got := v1beta2Number(t, obj, "spec", "workload", "static", key); got != want { - t.Fatalf("workload.static.%s = %v, want %v", key, got, want) - } - } - if got := v1beta2Number(t, obj, "spec", "workload", "static", "load", "concurrency"); got != 10 { - t.Fatalf("workload.static.load.concurrency = %v, want 10", got) - } - if got := v1beta2String(t, obj, "spec", "objective", "mode"); got != "Optimize" { - t.Fatalf("objective.mode = %q, want Optimize", got) - } - if got := v1beta2Number(t, obj, "spec", "objective", "sla", "ttftMs"); got != 2000 { - t.Fatalf("objective.sla.ttftMs = %v, want 2000", got) - } - if got := v1beta2Number(t, obj, "spec", "objective", "sla", "itlMs"); got != 30 { - t.Fatalf("objective.sla.itlMs = %v, want 30", got) - } - if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm", "sglang", "trtllm"}) { - t.Fatalf("backends = %#v, want vllm, sglang and trtllm", got) - } - if got := v1beta2Number(t, obj, "spec", "recommendation", "maxCandidates"); got != 1 { - t.Fatalf("recommendation.maxCandidates = %v, want 1", got) - } -} - -// expectV1beta2ExplicitValues checks that every value the explicit-values row sent survived. -func expectV1beta2ExplicitValues(t *testing.T, obj *unstructured.Unstructured) { - t.Helper() - if got, _ := v1beta2Field(t, obj, "spec", "backends"); !reflect.DeepEqual(got, []any{"vllm"}) { - t.Fatalf("backends = %#v, want only vllm", got) - } - for _, want := range []struct { - path []string - value float64 - }{ - {[]string{"spec", "hardware", "gpu", "budget"}, 4}, - {[]string{"spec", "workload", "static", "inputTokens"}, 10}, - {[]string{"spec", "workload", "static", "outputTokens"}, 128}, - {[]string{"spec", "workload", "static", "load", "concurrency"}, 10}, - {[]string{"spec", "recommendation", "maxCandidates"}, 3}, - } { - if got := v1beta2Number(t, obj, want.path...); got != want.value { - t.Fatalf("%v = %v, want %v", want.path, got, want.value) - } - } - for _, want := range []struct { - path []string - value string - }{ - {[]string{"spec", "hardware", "gpu", "sku"}, "h100_sxm"}, - {[]string{"spec", "objective", "mode"}, "Optimize"}, - {[]string{"spec", "objective", "metric"}, "TTFT"}, - {[]string{"spec", "aisimulate", "inputOverrides", "custom"}, "kept"}, - } { - if got := v1beta2String(t, obj, want.path...); got != want.value { - t.Fatalf("%v = %q, want %q", want.path, got, want.value) - } - } - // A nonempty static workload receives every omitted member of the default. -} - -// expectV1beta2AdmissionDefaults checks what admission supplies from the environment. -func expectV1beta2AdmissionDefaults(t *testing.T, obj *unstructured.Unstructured) { - t.Helper() - if got := v1beta2String(t, obj, "spec", "objective", "metric"); got != "ThroughputPerGPU" { - t.Fatalf("objective.metric = %q, want ThroughputPerGPU", got) - } - if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != "h200_sxm" { - t.Fatalf("hardware.gpu.sku = %q, want h200_sxm", got) - } - if got := v1beta2Number(t, obj, "spec", "hardware", "gpu", "budget"); got != 8 { - t.Fatalf("hardware.gpu.budget = %v, want 8", got) - } -} - // setV1beta2InputOverrides replaces any input overrides the fixture carried. func setV1beta2InputOverrides(spec map[string]any, overrides map[string]any) { aisimulate, _ := spec["aisimulate"].(map[string]any) @@ -1603,25 +1078,3 @@ func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRe GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, } } - -// expectV1beta2GPU asserts the typed GPU hardware the API server stored. -func expectV1beta2GPU(t *testing.T, result *unstructured.Unstructured, sku string, budget float64) { - t.Helper() - if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != sku { - t.Fatalf("hardware.gpu.sku = %q, want %q", got, sku) - } - if got := v1beta2Number(t, result, "spec", "hardware", "gpu", "budget"); got != budget { - t.Fatalf("hardware.gpu.budget = %v, want %v", got, budget) - } -} - -// expectV1beta2GPUValues checks the typed hardware a Request was stored with. -func expectV1beta2GPUValues(t *testing.T, obj *unstructured.Unstructured, sku string, budget float64) { - t.Helper() - if got := v1beta2String(t, obj, "spec", "hardware", "gpu", "sku"); got != sku { - t.Fatalf("hardware.gpu.sku = %q, want %q", got, sku) - } - if got := v1beta2Number(t, obj, "spec", "hardware", "gpu", "budget"); got != budget { - t.Fatalf("hardware.gpu.budget = %v, want %v", got, budget) - } -} From f7af2f75ca1e28be8e21820b8f1ae1a21d88cf6c Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 15:37:28 +0100 Subject: [PATCH 019/100] pr-16661: pr-4: satisfy lint after admission test consolidation Signed-off-by: Dr. Stefan Schimanski --- .../webhook/defaulting/suite_envtest_test.go | 136 ++++++++++-------- ...ploymentrequest_validation_envtest_test.go | 54 +++---- 2 files changed, 92 insertions(+), 98 deletions(-) diff --git a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go index ea523300331d..ad6480cac09c 100644 --- a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go @@ -219,7 +219,7 @@ func TestDynamoComponentDeploymentDefaultingAdmission(t *testing.T) { } } -func TestDynamoGraphDeploymentRequestDefaultingAdmission(t *testing.T) { +func TestDynamoGraphDeploymentRequestV1beta1DefaultingAdmission(t *testing.T) { wantDefaultSpec := nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ Model: "Qwen/Qwen3-8B", Backend: nvidiacomv1beta1.BackendTypeAuto, @@ -301,7 +301,9 @@ func TestDynamoGraphDeploymentRequestDefaultingAdmission(t *testing.T) { } }) } +} +func TestDynamoGraphDeploymentRequestV1beta2DefaultingAdmission(t *testing.T) { discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} v1beta2Tests := []struct { name string @@ -329,17 +331,7 @@ func TestDynamoGraphDeploymentRequestDefaultingAdmission(t *testing.T) { spec.Objective = &nvidiacomv1beta2.ObjectiveSpec{SLA: &nvidiacomv1beta2.ObjectiveSLASpec{}} }), discovered: discoveredH200, - check: func(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { - if got := request.Spec.Workload.Static; got == nil || got.InputTokens != nil || got.OutputTokens != nil || got.Load != nil { - t.Fatalf("workload.static = %#v, want explicit empty opt-out", got) - } - if got := request.Spec.Objective.SLA; got == nil || got.TTFTMs != nil || got.ITLMs != nil || got.E2EMs != nil { - t.Fatalf("objective.sla = %#v, want explicit empty opt-out", got) - } - if request.Spec.Objective.Mode != nvidiacomv1beta2.ObjectiveModeOptimize || request.Spec.Objective.Metric == nil { - t.Fatalf("objective = %#v, want mode and metric defaults beside empty SLA", request.Spec.Objective) - } - }, + check: expectV1beta2ExplicitEmptyOptOuts, }, { name: "v1beta2 explicit empty parent objects receive their defaults", @@ -364,39 +356,7 @@ func TestDynamoGraphDeploymentRequestDefaultingAdmission(t *testing.T) { }} }), discovered: discoveredH200, - check: func(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { - if want := []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM}; !reflect.DeepEqual(request.Spec.Backends, want) { - t.Fatalf("backends = %v, want %v", request.Spec.Backends, want) - } - if request.Spec.Workload == nil || request.Spec.Workload.Static == nil { - t.Fatalf("workload = %#v, want the partial static workload", request.Spec.Workload) - } - static := request.Spec.Workload.Static - if ptr.Deref(static.InputTokens, 0) != 10 || ptr.Deref(static.OutputTokens, 0) != 128 || static.Load == nil || ptr.Deref(static.Load.Concurrency, 0) != 10 { - t.Fatalf("workload.static = %#v, want explicit inputTokens and omitted-member defaults", static) - } - if request.Spec.Objective.Mode != nvidiacomv1beta2.ObjectiveModeOptimize || ptr.Deref(request.Spec.Objective.Metric, "") != nvidiacomv1beta2.ObjectiveMetric("TTFT") { - t.Fatalf("objective = %#v, want explicit TTFT metric with Optimize default", request.Spec.Objective) - } - if request.Spec.Recommendation.MaxCandidates != 3 { - t.Fatalf("maxCandidates = %d, want 3", request.Spec.Recommendation.MaxCandidates) - } - if request.Spec.AISimulate == nil || request.Spec.AISimulate.InputOverrides == nil { - t.Fatalf("aisimulate = %#v, want input overrides", request.Spec.AISimulate) - } - var overrides map[string]any - if err := json.Unmarshal(request.Spec.AISimulate.InputOverrides.Raw, &overrides); err != nil { - t.Fatalf("decode input overrides: %v", err) - } - if overrides["custom"] != "kept" { - t.Fatalf("inputOverrides = %#v, want custom value untouched", overrides) - } - engine, _ := overrides["engine"].(map[string]any) - if engine["hardware"] != "h100_sxm" { - t.Fatalf("inputOverrides = %#v, want engine.hardware untouched", overrides) - } - expectV1beta2Hardware(t, request, "h200_sxm", 8) - }, + check: expectV1beta2ExplicitValues, }, { name: "v1beta2 unresolved explicit hardware reports its public field path", @@ -418,12 +378,7 @@ func TestDynamoGraphDeploymentRequestDefaultingAdmission(t *testing.T) { beforeUpdate: func() { defaultingAdmissionHardware.set(gpu.GPUInfo{}, errors.New("cluster unreadable")) }, - checkUpdate: func(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { - expectV1beta2Hardware(t, request, "h200_sxm", 8) - if request.Generation != 1 { - t.Fatalf("generation = %d, want 1 after metadata-only update", request.Generation) - } - }, + checkUpdate: expectV1beta2MetadataOnlyUpdate, }, { name: "v1beta2 removing hardware on update rediscovers both fields", @@ -436,12 +391,7 @@ func TestDynamoGraphDeploymentRequestDefaultingAdmission(t *testing.T) { update: func(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { request.Spec.Hardware = nil }, - checkUpdate: func(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { - expectV1beta2Hardware(t, request, "h200_sxm", 8) - if request.Generation != 2 { - t.Fatalf("generation = %d, want 2 after hardware update", request.Generation) - } - }, + checkUpdate: expectV1beta2HardwareUpdate, }, } @@ -507,7 +457,7 @@ func expectV1beta2ModelOnlyDefaults(t *testing.T, request *nvidiacomv1beta2.Dyna if want := []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM, nvidiacomv1beta2.BackendSGLang, nvidiacomv1beta2.BackendTRTLLM}; !reflect.DeepEqual(request.Spec.Backends, want) { t.Fatalf("backends = %v, want %v", request.Spec.Backends, want) } - expectV1beta2Hardware(t, request, "h200_sxm", 8) + expectV1beta2H200x8Hardware(t, request) if request.Spec.Workload == nil { t.Fatal("workload is nil, want model-only defaults") } @@ -524,11 +474,75 @@ func expectV1beta2ModelOnlyDefaults(t *testing.T, request *nvidiacomv1beta2.Dyna } } -func expectV1beta2Hardware(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, sku string, budget int32) { +func expectV1beta2ExplicitEmptyOptOuts(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + if got := request.Spec.Workload.Static; got == nil || got.InputTokens != nil || got.OutputTokens != nil || got.Load != nil { + t.Fatalf("workload.static = %#v, want explicit empty opt-out", got) + } + if got := request.Spec.Objective.SLA; got == nil || got.TTFTMs != nil || got.ITLMs != nil || got.E2EMs != nil { + t.Fatalf("objective.sla = %#v, want explicit empty opt-out", got) + } + if request.Spec.Objective.Mode != nvidiacomv1beta2.ObjectiveModeOptimize || request.Spec.Objective.Metric == nil { + t.Fatalf("objective = %#v, want mode and metric defaults beside empty SLA", request.Spec.Objective) + } +} + +func expectV1beta2ExplicitValues(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + if want := []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM}; !reflect.DeepEqual(request.Spec.Backends, want) { + t.Fatalf("backends = %v, want %v", request.Spec.Backends, want) + } + if request.Spec.Workload == nil || request.Spec.Workload.Static == nil { + t.Fatalf("workload = %#v, want the partial static workload", request.Spec.Workload) + } + static := request.Spec.Workload.Static + if ptr.Deref(static.InputTokens, 0) != 10 || ptr.Deref(static.OutputTokens, 0) != 128 || static.Load == nil || ptr.Deref(static.Load.Concurrency, 0) != 10 { + t.Fatalf("workload.static = %#v, want explicit inputTokens and omitted-member defaults", static) + } + if request.Spec.Objective.Mode != nvidiacomv1beta2.ObjectiveModeOptimize || ptr.Deref(request.Spec.Objective.Metric, "") != nvidiacomv1beta2.ObjectiveMetric("TTFT") { + t.Fatalf("objective = %#v, want explicit TTFT metric with Optimize default", request.Spec.Objective) + } + if request.Spec.Recommendation.MaxCandidates != 3 { + t.Fatalf("maxCandidates = %d, want 3", request.Spec.Recommendation.MaxCandidates) + } + if request.Spec.AISimulate == nil || request.Spec.AISimulate.InputOverrides == nil { + t.Fatalf("aisimulate = %#v, want input overrides", request.Spec.AISimulate) + } + var overrides map[string]any + if err := json.Unmarshal(request.Spec.AISimulate.InputOverrides.Raw, &overrides); err != nil { + t.Fatalf("decode input overrides: %v", err) + } + if overrides["custom"] != "kept" { + t.Fatalf("inputOverrides = %#v, want custom value untouched", overrides) + } + engine, _ := overrides["engine"].(map[string]any) + if engine["hardware"] != "h100_sxm" { + t.Fatalf("inputOverrides = %#v, want engine.hardware untouched", overrides) + } + expectV1beta2H200x8Hardware(t, request) +} + +func expectV1beta2MetadataOnlyUpdate(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + expectV1beta2H200x8Hardware(t, request) + if request.Generation != 1 { + t.Fatalf("generation = %d, want 1 after metadata-only update", request.Generation) + } +} + +func expectV1beta2HardwareUpdate(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + expectV1beta2H200x8Hardware(t, request) + if request.Generation != 2 { + t.Fatalf("generation = %d, want 2 after hardware update", request.Generation) + } +} + +func expectV1beta2H200x8Hardware(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { t.Helper() hardware := request.Spec.Hardware - if hardware == nil || hardware.GPU == nil || ptr.Deref(hardware.GPU.SKU, "") != sku || ptr.Deref(hardware.GPU.Budget, 0) != budget { - t.Fatalf("hardware = %#v, want SKU %q and budget %d", hardware, sku, budget) + if hardware == nil || hardware.GPU == nil || ptr.Deref(hardware.GPU.SKU, "") != "h200_sxm" || ptr.Deref(hardware.GPU.Budget, 0) != 8 { + t.Fatalf("hardware = %#v, want SKU h200_sxm and budget 8", hardware) } } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index d22e7f800890..e07f8693ad81 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -553,7 +553,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "generator": "aic", "snapshot_interval_seconds": int64(30), }}) - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, check: func(t *testing.T, result *unstructured.Unstructured) { @@ -575,7 +575,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "runtime_version_override": "1.2.3", "snapshot_dir": "/tmp/snapshots", }}) - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ @@ -593,7 +593,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 dgdr run input cannot be deleted", request: v1beta2DGDRForAdmission(func(spec map[string]any) { setV1beta2InputOverrides(spec, map[string]any{"dgdr_run": nil}) - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ @@ -604,7 +604,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 dgdr run input cannot be replaced by a scalar", request: v1beta2DGDRForAdmission(func(spec map[string]any) { setV1beta2InputOverrides(spec, map[string]any{"dgdr_run": "wrong"}) - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ @@ -615,7 +615,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 dgdr run input cannot be replaced by a list", request: v1beta2DGDRForAdmission(func(spec map[string]any) { setV1beta2InputOverrides(spec, map[string]any{"dgdr_run": []any{"wrong"}}) - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ @@ -626,7 +626,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 pareto objective is reserved until Candidate publication supports it", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["objective"] = map[string]any{"mode": "Pareto", "metrics": []any{"ThroughputPerGPU", "ThroughputPerUser"}} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantSchemaErr: `spec.objective.mode: Unsupported value: "Pareto": supported values: "Optimize"`, @@ -635,7 +635,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 optimize rejects the reserved Pareto metrics field", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["objective"] = map[string]any{"mode": "Optimize", "metrics": []any{"ThroughputPerGPU", "ThroughputPerUser"}} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantCELErr: `spec.objective: Invalid value: metrics is reserved for the future Pareto implementation`, @@ -644,14 +644,14 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 duplicate backends are rejected on create", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["backends"] = []any{"vllm", "vllm"} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantSchemaErr: `spec.backends[1]: Duplicate value: "vllm"`, }, { name: "v1beta2 missing model is rejected by the representation CEL", - request: v1beta2DGDRForAdmission(func(spec map[string]any) { delete(spec, "model") }, nil), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { delete(spec, "model") }), gpuDiscovery: true, discovered: discoveredH200, wantCELErr: `spec: Invalid value: exactly one of native v1beta2 fields or v1beta1 must be specified`, @@ -660,7 +660,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 empty model name is rejected by the schema", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["model"] = map[string]any{"name": ""} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantSchemaErr: `spec.model.name: Invalid value: "": spec.model.name in body should be at least 1 chars long`, @@ -671,7 +671,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { spec["workload"] = map[string]any{"static": map[string]any{"load": map[string]any{ "kvCapacityFraction": map[string]any{"range": map[string]any{"min": 0.9, "max": 0.1}}, }}} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantCELErr: `spec.workload.static.load.kvCapacityFraction.range: Invalid value: min must be less than max`, @@ -682,7 +682,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { spec["workload"] = map[string]any{"static": map[string]any{"load": map[string]any{ "kvCapacityFraction": map[string]any{"value": 0.0}, }}} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantSchemaErr: `spec.workload.static.load.kvCapacityFraction.value: Invalid value: 0: spec.workload.static.load.kvCapacityFraction.value in body should be greater than 0`, @@ -693,7 +693,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { spec["workload"] = map[string]any{"static": map[string]any{"load": map[string]any{ "concurrency": int64(10), "requestsPerSecond": 10.0, }}} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantCELErr: `spec.workload.static.load: Invalid value: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified`, @@ -702,17 +702,17 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { name: "v1beta2 empty load is not a second default opt-out", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"static": map[string]any{"load": map[string]any{}}} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantCELErr: `spec.workload.static.load: Invalid value: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified`, }, { name: "v1beta2 update to duplicate backends is rejected", - oldRequest: v1beta2DGDRForAdmission(nil, nil), + oldRequest: v1beta2DGDRForAdmission(nil), request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["backends"] = []any{"sglang", "sglang"} - }, nil), + }), gpuDiscovery: true, discovered: discoveredH200, wantSchemaErr: `spec.backends[1]: Duplicate value: "sglang"`, @@ -1005,15 +1005,12 @@ const v1beta2AdmissionModel = "Qwen/Qwen3-0.6B" // defaults. It is unstructured on purpose: an omitted field, // an explicit empty object and a null are different requests, and typed Go cannot tell them // apart. -func v1beta2DGDRForAdmission(mutate func(spec map[string]any), labels map[string]any) *unstructured.Unstructured { +func v1beta2DGDRForAdmission(mutate func(spec map[string]any)) *unstructured.Unstructured { spec := map[string]any{"model": map[string]any{"name": v1beta2AdmissionModel}} if mutate != nil { mutate(spec) } metadata := map[string]any{"name": "test-v1beta2-dgdr", "namespace": "default"} - if labels != nil { - metadata["labels"] = labels - } return &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "nvidia.com/v1beta2", "kind": "DynamoGraphDeploymentRequest", @@ -1031,23 +1028,6 @@ func v1beta2Field(t *testing.T, obj *unstructured.Unstructured, path ...string) return value, found } -func v1beta2Number(t *testing.T, obj *unstructured.Unstructured, path ...string) float64 { - t.Helper() - value, found := v1beta2Field(t, obj, path...) - if !found { - t.Fatalf("%v is not set", path) - } - switch n := value.(type) { - case int64: - return float64(n) - case float64: - return n - default: - t.Fatalf("%v = %#v, want a number", path, value) - return 0 - } -} - func v1beta2String(t *testing.T, obj *unstructured.Unstructured, path ...string) string { t.Helper() value, found := v1beta2Field(t, obj, path...) From 4bd078215c4bab5ce4945264773a6f8ad60ee372 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 15:43:40 +0100 Subject: [PATCH 020/100] pr-16661: pr-4: document independent trial budget [upstream: devivasudevan/dynamo#4] Signed-off-by: Dr. Stefan Schimanski --- .../api/v1beta2/dynamographdeploymentrequest_types.go | 4 ++-- .../bases/nvidia.com_dynamographdeploymentrequests.yaml | 4 ++-- .../crd/bases/nvidia.com_dynamographdeploymentruns.yaml | 4 ++-- .../additional-resources/api-reference-k8s.md | 8 ++++---- .../pages/reference/kubernetes-api/full-api-reference.mdx | 8 ++++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index a504b6634391..b0dc3aad1683 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -263,8 +263,8 @@ type NativeSearchInput struct { // traffic defaults. Any other static object is completed with its missing defaults. // spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over // every lowered traffic field it names. - // The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials - // nor a KV-capacity-fraction search. + // The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction + // search. Lowering independently defaults optimizer.max_trials to 320. // +optional Workload *WorkloadSpec `json:"workload,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 6a57422d31b7..1466165e1939 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9764,8 +9764,8 @@ spec: traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. - The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials - nor a KV-capacity-fraction search. + The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction + search. Lowering independently defaults optimizer.max_trials to 320. properties: static: description: |- diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 3888695a8d9d..68606522cc32 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -220,8 +220,8 @@ spec: traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. - The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials - nor a KV-capacity-fraction search. + The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction + search. Lowering independently defaults optimizer.max_trials to 320. properties: static: description: |- diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 20ef5210b5fe..39c1c4b61fd9 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3453,7 +3453,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3495,7 +3495,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3559,7 +3559,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3719,7 +3719,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials
nor a KV-capacity-fraction search. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 923b12ffd8ef..9df5d9746b76 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4811,7 +4811,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials nor a KV-capacity-fraction search. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -4893,7 +4893,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials nor a KV-capacity-fraction search. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5016,7 +5016,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials nor a KV-capacity-fraction search. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5247,7 +5247,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it creates neither optimizer.max_trials nor a KV-capacity-fraction search. +Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} From 0b8033962bb871fbbfc1511af89442f14c3a3a36 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 17:43:42 +0100 Subject: [PATCH 021/100] pr-16661: pr-3: preserve external AISimulate hardware SKUs [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- .../tests/test_hardware_catalog_sync.py | 74 +++++++++++++++++++ container/Dockerfile.test | 4 + .../dynamographdeploymentrequest_types.go | 16 +++- ...dia.com_dynamographdeploymentrequests.yaml | 17 ++++- .../operator/internal/aisimulate/hardware.go | 47 ++++++++++++ .../internal/aisimulate/hardware_skus.txt | 14 ++++ .../internal/aisimulate/input_test.go | 8 ++ ...deploymentrequest_v1beta2_hardware_test.go | 26 ++++--- ...ographdeploymentrequest_v1beta2_helpers.go | 42 ++++++----- .../webhook/defaulting/suite_envtest_test.go | 7 +- .../dynamographdeploymentrequest.go | 8 +- .../dynamographdeploymentrequest_v1beta2.go | 37 +++++++++- .../additional-resources/api-reference-k8s.md | 10 +-- .../dynamo-graph-deployment-request.mdx | 29 ++++++++ .../kubernetes-api/full-api-reference.mdx | 10 +-- 15 files changed, 290 insertions(+), 59 deletions(-) create mode 100644 components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py create mode 100644 deploy/operator/internal/aisimulate/hardware.go create mode 100644 deploy/operator/internal/aisimulate/hardware_skus.txt diff --git a/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py b/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py new file mode 100644 index 000000000000..fb3652b73003 --- /dev/null +++ b/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep DGDR's hardware hints synchronized with the pinned AISimulate wheel.""" + +from __future__ import annotations + +import re +from importlib import metadata +from pathlib import Path + +import pytest +import yaml + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + +REPO_ROOT = Path(__file__).resolve().parents[5] +CATALOG_PATH = REPO_ROOT / "deploy/operator/internal/aisimulate/hardware_skus.txt" +API_PATH = ( + REPO_ROOT / "deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go" +) +PIN_PATH = REPO_ROOT / "container/deps/requirements.aisimulate.txt" + + +def _pinned_version() -> str: + pins = [ + line.split("==", 1)[1].strip() + for line in PIN_PATH.read_text().splitlines() + if line.startswith("aisimulate==") + ] + assert len(pins) == 1, "requirements.aisimulate.txt must contain one exact pin" + return pins[0] + + +def _packaged_hardware_skus() -> list[str]: + distribution = metadata.distribution("aisimulate") + system_files = [ + path + for path in distribution.files or [] + if path.parent.as_posix() == "aisimulate_core/systems" + and path.suffix == ".yaml" + ] + result = [] + for packaged_path in system_files: + with distribution.locate_file(packaged_path).open(encoding="utf-8") as stream: + document = yaml.safe_load(stream) + if isinstance(document, dict) and "gpu" in document and "node" in document: + result.append(packaged_path.stem) + return sorted(result) + + +def _documented_hardware_skus() -> list[str]: + source = API_PATH.read_text() + match = re.search( + r"Known values in the bundled AISimulate release are (.*?)\n\s*// The field remains open", + source, + re.DOTALL, + ) + assert match is not None, "GPUHardwareSpec.SKU must document the bundled catalog" + return sorted(re.findall(r"`([^`]+)`", match.group(1))) + + +def test_dgdr_hardware_catalog_matches_pinned_aisimulate() -> None: + assert metadata.version("aisimulate") == _pinned_version() + + expected = _packaged_hardware_skus() + assert CATALOG_PATH.read_text().split() == expected + assert _documented_hardware_skus() == expected diff --git a/container/Dockerfile.test b/container/Dockerfile.test index 920a4732d535..245ce09d8135 100644 --- a/container/Dockerfile.test +++ b/container/Dockerfile.test @@ -129,6 +129,10 @@ COPY --chmod=664 --chown=dynamo:0 components/src/dynamo/mocker/config.py /worksp # Give the isolated copy a unique basename so pytest cannot alias it with # another non-package `test_config.py` during whole-workspace collection. COPY --chmod=664 --chown=dynamo:0 components/src/dynamo/mocker/tests/unit/test_config.py /workspace/components/src/dynamo/mocker/tests/unit/test_mocker_config.py +COPY --chmod=664 --chown=dynamo:0 components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py /workspace/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py +COPY --chmod=664 --chown=dynamo:0 deploy/operator/internal/aisimulate/hardware_skus.txt /workspace/deploy/operator/internal/aisimulate/hardware_skus.txt +COPY --chmod=664 --chown=dynamo:0 deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go /workspace/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +COPY --chmod=664 --chown=dynamo:0 container/deps/requirements.aisimulate.txt /workspace/container/deps/requirements.aisimulate.txt # BuildKit applies the file mode to parent directories it creates for these # single-file COPY instructions. Restore directory traversal/write access for diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index b0dc3aad1683..4170620bc01d 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -44,13 +44,20 @@ const ( // GPUHardwareSpec describes accelerator identity and the maximum allocation per candidate. type GPUHardwareSpec struct { // SKU identifies the accelerator using the AISimulate hardware catalog name. - // When omitted, admission discovers the accelerator from the cluster. + // Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, + // `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, + // `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. + // The field remains open so an external AISimulate systems catalog can add identifiers. + // Admission passes an unknown identifier through unchanged and warns when `budget` is also + // explicit. Without an explicit `budget`, it rejects the identifier because cluster discovery + // cannot associate capacity with it. When omitted, admission discovers the accelerator. // +optional // +kubebuilder:validation:MinLength=1 SKU *string `json:"sku,omitempty"` // Budget is the maximum total number of GPUs one candidate may use. - // When omitted, admission derives the available GPU budget from the cluster. + // When omitted for a known SKU, admission derives the available GPU budget from the cluster. + // An SKU unknown to the bundled AISimulate catalog requires an explicit `budget`. // +optional // +kubebuilder:validation:Minimum=1 Budget *int32 `json:"budget,omitempty"` @@ -59,7 +66,7 @@ type GPUHardwareSpec struct { // HardwareSpec bounds the accelerator configurations evaluated by the search. type HardwareSpec struct { // GPU describes accelerator identity and the maximum allocation per candidate. - // Omitted fields are resolved independently by admission. + // Admission resolves omitted fields from the cluster when the SKU is known. // +optional GPU *GPUHardwareSpec `json:"gpu,omitempty"` } @@ -312,7 +319,8 @@ type DynamoGraphDeploymentRequestNativeStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // ActiveRunRef identifies the one non-terminal run. + // ActiveRunRef identifies the reserved or active Run. During a spec change it continues to + // identify the superseded Run until foreground deletion completes, before a replacement is reserved. // +optional ActiveRunRef *corev1.LocalObjectReference `json:"activeRunRef,omitempty"` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 1466165e1939..2d79c1f9e782 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9636,19 +9636,26 @@ spec: gpu: description: |- GPU describes accelerator identity and the maximum allocation per candidate. - Omitted fields are resolved independently by admission. + Admission resolves omitted fields from the cluster when the SKU is known. properties: budget: description: |- Budget is the maximum total number of GPUs one candidate may use. - When omitted, admission derives the available GPU budget from the cluster. + When omitted for a known SKU, admission derives the available GPU budget from the cluster. + An SKU unknown to the bundled AISimulate catalog requires an explicit `budget`. format: int32 minimum: 1 type: integer sku: description: |- SKU identifies the accelerator using the AISimulate hardware catalog name. - When omitted, admission discovers the accelerator from the cluster. + Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, + `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, + `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. + The field remains open so an external AISimulate systems catalog can add identifiers. + Admission passes an unknown identifier through unchanged and warns when `budget` is also + explicit. Without an explicit `budget`, it rejects the identifier because cluster discovery + cannot associate capacity with it. When omitted, admission discovers the accelerator. minLength: 1 type: string type: object @@ -9853,7 +9860,9 @@ spec: description: DynamoGraphDeploymentRequestStatus represents reconciliation of persistent search intent. properties: activeRunRef: - description: ActiveRunRef identifies the one non-terminal run. + description: |- + ActiveRunRef identifies the reserved or active Run. During a spec change it continues to + identify the superseded Run until foreground deletion completes, before a replacement is reserved. properties: name: default: "" diff --git a/deploy/operator/internal/aisimulate/hardware.go b/deploy/operator/internal/aisimulate/hardware.go new file mode 100644 index 000000000000..004386117650 --- /dev/null +++ b/deploy/operator/internal/aisimulate/hardware.go @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + _ "embed" + "strings" +) + +// hardwareSKUs mirrors the system specifications packaged in the pinned AISimulate wheel. +// It is a compatibility hint for warnings and discovery, not an API enum: explicit +// identifiers outside this set remain valid and are passed through unchanged. +// +//go:embed hardware_skus.txt +var hardwareSKUsText string + +var hardwareSKUs = func() map[string]struct{} { + values := strings.Fields(hardwareSKUsText) + result := make(map[string]struct{}, len(values)) + for _, value := range values { + result[value] = struct{}{} + } + return result +}() + +// IsKnownHardwareSKU reports whether sku is packaged by the AISimulate release pinned by +// this Dynamo checkout. False does not make a value invalid: an external system catalog may +// support additional identifiers. +func IsKnownHardwareSKU(sku string) bool { + _, found := hardwareSKUs[sku] + return found +} diff --git a/deploy/operator/internal/aisimulate/hardware_skus.txt b/deploy/operator/internal/aisimulate/hardware_skus.txt new file mode 100644 index 000000000000..43053fd0c105 --- /dev/null +++ b/deploy/operator/internal/aisimulate/hardware_skus.txt @@ -0,0 +1,14 @@ +a100_pcie +a100_sxm +a30 +b200_sxm +b300_sxm +b60 +gb200 +gb300 +h100_pcie +h100_sxm +h200_sxm +l4 +l40s +rtx_pro_6000_server diff --git a/deploy/operator/internal/aisimulate/input_test.go b/deploy/operator/internal/aisimulate/input_test.go index 81ffb914bee2..a0271a431409 100644 --- a/deploy/operator/internal/aisimulate/input_test.go +++ b/deploy/operator/internal/aisimulate/input_test.go @@ -125,6 +125,14 @@ func TestLowerModelOnlyRequest(t *testing.T) { } } +func TestUnderlayPreservesOpenHardwareSKU(t *testing.T) { + spec := defaultedSpec() + *spec.Hardware.GPU.SKU = "future_accelerator" + if got := nested(t, underlay(t, spec), "engine", "hardware"); got != "future_accelerator" { + t.Fatalf("engine.hardware = %q, want the API value unchanged", got) + } +} + func TestLowerIsDeterministicAndDoesNotMutateTheSpec(t *testing.T) { const overrides = `{"optimizer":{"max_trials":100},"traffic":{"stop":{"requests":5}}}` spec := withInputOverrides(defaultedSpec(), overrides) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index 3b666abecb90..e5b11b39bf7c 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -102,18 +102,15 @@ func TestResolveHardwareV1beta2(t *testing.T) { }, {name: "a budget alone keeps the budget and discovers the SKU", op: admissionv1.Create, hardware: hardwareOfV1beta2(nil, num(4)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, {name: "an explicit budget above the cluster size is kept", op: admissionv1.Create, hardware: hardwareOfV1beta2(nil, num(64)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, - {name: "an explicit budget above the cap is kept", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h200_sxm"), num(64)), found: h200x8, wantCalls: 1, wantFilter: "h200_sxm"}, + {name: "an explicit SKU and budget bypass discovery", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h200_sxm"), num(64)), discoveryErr: errors.New("down")}, { - name: "a SKU and a budget verify the SKU", op: admissionv1.Create, - hardware: hardwareOfV1beta2(str("h100_sxm"), num(4)), - found: gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, - wantCalls: 1, wantFilter: "h100_sxm", + name: "an unknown explicit SKU and budget are authoritative", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("future_accelerator"), num(4)), discoveryErr: errors.New("down"), }, { - name: "complete hardware rejects failed verification", op: admissionv1.Create, - hardware: hardwareOfV1beta2(str("h100_sxm"), num(4)), discoveryErr: errors.New("down"), - wantCalls: 1, wantFilter: "h100_sxm", - wantErr: `spec.hardware.gpu.sku: Invalid value: "h100_sxm": could not be verified against the cluster's GPUs (down)`, + name: "an unknown SKU alone requires an explicit budget", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("future_accelerator"), nil), found: h200x8, + wantErr: `spec.hardware.gpu.budget: Required value: cannot discover a GPU budget for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget explicitly`, }, {name: "a failed discovery rejects", op: admissionv1.Create, discoveryErr: errors.New("no GPUs found"), wantCalls: 1, wantErr: "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found)"}, @@ -123,10 +120,15 @@ func TestResolveHardwareV1beta2(t *testing.T) { wantErr: `spec.hardware.gpu.sku: Required value: the cluster's GPU model "Mystery GPU" maps to no AISimulate SKU`, }, { - name: "no GPUs of the SKU reject the SKU", op: admissionv1.Create, + name: "a legacy discovery mapping outside the bundled catalog rejects", op: admissionv1.Create, + found: gpu.GPUInfo{System: "gb10", Model: "GB10", GPUsPerNode: 1, NodesWithGPUs: 1}, wantCalls: 1, + wantErr: `spec.hardware.gpu.sku: Required value: the cluster's GPU model "GB10" maps to no AISimulate SKU`, + }, + { + name: "no GPUs of a known SKU require an explicit budget", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h100_sxm"), nil), found: gpu.GPUInfo{System: "h100_sxm"}, wantCalls: 1, wantFilter: "h100_sxm", - wantErr: `spec.hardware.gpu.sku: Invalid value: "h100_sxm": no GPUs of this SKU were found in the cluster`, + wantErr: `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h100_sxm" were found in the cluster`, }, { name: "a heterogeneous cluster rejects an omitted SKU", op: admissionv1.Create, @@ -157,7 +159,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { {name: "removing only the budget discovers only the budget", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("h100_sxm"), nil), found: h200x8, wantBudget: 8, wantCalls: 1, wantFilter: "h100_sxm"}, {name: "removing only the SKU discovers only the SKU", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(nil, num(4)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, {name: "removing all hardware discovers both fields", op: admissionv1.Update, oldHardware: completeH100, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1}, - {name: "changing complete hardware verifies the new SKU", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("b200_sxm"), num(2)), found: gpu.GPUInfo{System: "b200_sxm", TotalGPUs: 4}, wantCalls: 1, wantFilter: "b200_sxm"}, + {name: "changing complete hardware keeps the explicit override", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("b200_sxm"), num(2)), discoveryErr: errors.New("down")}, {name: "delete never discovers", op: admissionv1.Delete, oldHardware: completeH100, found: h200x8}, {name: "connect never discovers", op: admissionv1.Connect, oldHardware: completeH100, found: h200x8}, {name: "an unknown operation never discovers", oldHardware: completeH100, found: h200x8}, diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 7265d2c5a88e..7ac29c3457b9 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -29,6 +29,7 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/aisimulate" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -79,8 +80,10 @@ func (o DGDRV1beta2Options) resolveDefaults( return dynamoGraphDeploymentRequestV1beta2DefaultOptions{hardware: hardware}, nil } -// resolveHardware returns the SKU and budget the typed spec omits and verifies an explicit SKU -// against the cluster. Discovery is restricted to the typed SKU when there is one. +// resolveHardware returns the SKU and budget the typed spec omits. A complete explicit pair is +// authoritative and bypasses discovery. When only a known SKU is present, discovery is restricted +// to it. An SKU outside the bundled AISimulate catalog remains an authoritative override only +// when its budget is explicit, because discovery cannot associate cluster capacity with it. // // Discovery runs on CREATE and on an UPDATE that changes spec.hardware. A Request stays // mutable and every Run snapshots it, so a caller may change or remove typed hardware before @@ -91,10 +94,9 @@ func (o DGDRV1beta2Options) resolveDefaults( // the typed hardware alone, never spec.aisimulate.inputOverrides. // // After a successful return the typed hardware is complete. An omitted SKU becomes the -// cluster's AISimulate SKU, and an omitted budget becomes min(matching GPUs, cap). An explicit -// budget is authoritative: it is never capped, whatever the cluster holds. What the cluster -// cannot supply is rejected on the field that fixes it: a failed discovery, a GPU model the -// AISimulate catalog does not know, and an explicit SKU without any matching GPU. +// cluster's AISimulate SKU, and an omitted budget becomes min(matching GPUs, cap). Explicit +// values are authoritative: they are neither rewritten nor capped. What the cluster cannot +// supply is rejected on the omitted field that fixes it. func (o DGDRV1beta2Options) resolveHardware( ctx context.Context, operation admissionv1.Operation, @@ -113,6 +115,16 @@ func (o DGDRV1beta2Options) resolveHardware( } hasBudget = spec.Hardware.GPU.Budget != nil } + if sku != "" && hasBudget { + return hardwareDefaultsV1beta2{}, nil + } + + if sku != "" && !aisimulate.IsKnownHardwareSKU(sku) { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required( + field.NewPath("spec", "hardware", "gpu", "budget"), + fmt.Sprintf("cannot discover a GPU budget for SKU %q because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget explicitly", sku), + )) + } timeout := o.HardwareDiscoveryTimeout if timeout <= 0 { timeout = defaultHardwareDiscoveryTimeout @@ -124,10 +136,9 @@ func (o DGDRV1beta2Options) resolveHardware( found, err := o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) if err != nil { if sku != "" { - return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Invalid( - gpuPath.Child("sku"), - sku, - fmt.Sprintf("could not be verified against the cluster's GPUs (%v)", err), + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required( + gpuPath.Child("budget"), + fmt.Sprintf("could not be discovered for SKU %q (%v); set spec.hardware.gpu.budget", sku, err), )) } return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath, fmt.Sprintf( @@ -144,7 +155,7 @@ func (o DGDRV1beta2Options) resolveHardware( strings.Join(append([]string{found.Model}, found.OtherModels...), ", ")))) } sku = string(gpu.KnownSKU(found)) - if sku == "" { + if sku == "" || !aisimulate.IsKnownHardwareSKU(sku) { detail := "the cluster's GPUs map to no AISimulate SKU; set spec.hardware.gpu.sku" if found != nil && found.Model != "" { detail = fmt.Sprintf("the cluster's GPU model %q maps to no AISimulate SKU; set spec.hardware.gpu.sku", found.Model) @@ -154,18 +165,11 @@ func (o DGDRV1beta2Options) resolveHardware( defaults.sku = sku } - // A filtered discovery with no GPUs means the caller pinned a SKU the cluster cannot run. + // Discovery is needed only for an omitted budget at this point. var gpus int if found != nil { gpus = found.TotalGPUs } - if spec.Hardware != nil && spec.Hardware.GPU != nil && spec.Hardware.GPU.SKU != nil && gpus <= 0 { - return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Invalid( - gpuPath.Child("sku"), - sku, - "no GPUs of this SKU were found in the cluster", - )) - } if !hasBudget { if gpus <= 0 { return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("budget"), fmt.Sprintf( diff --git a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go index ad6480cac09c..5b02c72681e7 100644 --- a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go @@ -359,14 +359,13 @@ func TestDynamoGraphDeploymentRequestV1beta2DefaultingAdmission(t *testing.T) { check: expectV1beta2ExplicitValues, }, { - name: "v1beta2 unresolved explicit hardware reports its public field path", + name: "v1beta2 unknown SKU without budget reports the manual override", request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ - SKU: new("h100_sxm"), Budget: new(int32(4)), + SKU: new("future_accelerator"), }} }), - discoveryErr: errors.New("no matching GPUs found"), - wantCreateErr: `spec.hardware.gpu.sku: Invalid value: "h100_sxm": could not be verified against the cluster's GPUs (no matching GPUs found)`, + wantCreateErr: `spec.hardware.gpu.budget: Required value: cannot discover a GPU budget for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget explicitly`, }, { name: "v1beta2 metadata-only update is idempotent and does not rediscover hardware", diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go index 3e2cd01c8509..72e3ce73caae 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest.go @@ -59,6 +59,10 @@ func (v *dynamoGraphDeploymentRequestValidation) warn(message string) { v.warnings = append(v.warnings, message) } +func (v *dynamoGraphDeploymentRequestValidation) warnf(format string, args ...any) { + v.warn(fmt.Sprintf(format, args...)) +} + // Validate performs stateless validation on request. ctx must carry the admission request and // request must not be nil. func (v *DynamoGraphDeploymentRequestValidator) Validate( @@ -123,7 +127,7 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe // A v1beta2 Request reaches this webhook as a v1beta1 object carrying spec.v1beta2; // the v1beta1 rules below do not apply to it. if spec.V1Beta2 != nil { - return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(spec.V1Beta2, fldPath.Child("v1beta2")) + return v.validateDynamoGraphDeploymentRequestEnvelopeV1beta2(spec.V1Beta2, fldPath.Child("v1beta2")) } allErrs := field.ErrorList{} @@ -290,7 +294,7 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe // spec runs on every CREATE and UPDATE: an unchanged spec is no reason to skip it. Ratcheting // belongs to the generated AISimulate schema validation, which will ratchet its own result. if newSpec.V1Beta2 != nil { - return validateDynamoGraphDeploymentRequestEnvelopeV1beta2(newSpec.V1Beta2, fldPath.Child("v1beta2")) + return v.validateDynamoGraphDeploymentRequestEnvelopeV1beta2(newSpec.V1Beta2, fldPath.Child("v1beta2")) } allErrs := field.ErrorList{} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index ca5093f39251..8f8c0e14eab9 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -31,7 +31,7 @@ import ( // does not decode is reported there. Errors about the decoded spec use the v1beta2 spec's own // field paths (spec.model, spec.aisimulate, ...), which are what the user wrote, so they are // rooted at spec rather than at fldPath. -func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( +func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRequestEnvelopeV1beta2( raw *apiextensionsv1.JSON, fldPath *field.Path, ) field.ErrorList { @@ -40,7 +40,7 @@ func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( return field.ErrorList{field.Invalid(fldPath, "", "must be a valid v1beta2 spec: "+err.Error())} } - allErrs := validateDynamoGraphDeploymentRequestSpecV1beta2(&spec, field.NewPath("spec")) + allErrs := v.validateDynamoGraphDeploymentRequestSpecV1beta2(&spec, field.NewPath("spec")) if _, err := aisimulate.Lower(&spec.NativeSearchInput); err != nil { allErrs = append(allErrs, field.Invalid( field.NewPath("spec"), @@ -52,12 +52,41 @@ func validateDynamoGraphDeploymentRequestEnvelopeV1beta2( } // validateDynamoGraphDeploymentRequestSpecV1beta2 validates spec. spec and fldPath must not be nil. -func validateDynamoGraphDeploymentRequestSpecV1beta2( +func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRequestSpecV1beta2( spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, fldPath *field.Path, ) field.ErrorList { + allErrs := field.ErrorList{} + if spec.Hardware != nil { + allErrs = append(allErrs, v.validateHardwareSpecV1beta2(spec.Hardware, fldPath.Child("hardware"))...) + } if spec.AISimulate == nil { + return allErrs + } + return append(allErrs, validateAISimulateSpec(spec.AISimulate, fldPath.Child("aisimulate"))...) +} + +// validateHardwareSpecV1beta2 validates hardware. spec and fldPath must not be nil. +func (v *dynamoGraphDeploymentRequestValidation) validateHardwareSpecV1beta2( + spec *nvidiacomv1beta2.HardwareSpec, + fldPath *field.Path, +) field.ErrorList { + if spec.GPU == nil { return nil } - return validateAISimulateSpec(spec.AISimulate, fldPath.Child("aisimulate")) + return v.validateGPUHardwareSpecV1beta2(spec.GPU, fldPath.Child("gpu")) +} + +// validateGPUHardwareSpecV1beta2 validates gpu. spec and fldPath must not be nil. +func (v *dynamoGraphDeploymentRequestValidation) validateGPUHardwareSpecV1beta2( + spec *nvidiacomv1beta2.GPUHardwareSpec, + fldPath *field.Path, +) field.ErrorList { + if spec.SKU != nil && !aisimulate.IsKnownHardwareSKU(*spec.SKU) { + v.warnf( + "%s: %q is not known to the AISimulate release bundled with this operator; the value is passed through unchanged and may work with an external AISimulate systems catalog", + fldPath.Child("sku"), *spec.SKU, + ) + } + return nil } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 39c1c4b61fd9..7629066d9f35 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3473,7 +3473,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the most recent DGDR generation accepted by the controller. | | Optional: \{\}
| -| `activeRunRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core)_ | ActiveRunRef identifies the one non-terminal run. | | Optional: \{\}
| +| `activeRunRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core)_ | ActiveRunRef identifies the reserved or active Run. During a spec change it continues to
identify the superseded Run until foreground deletion completes, before a replacement is reserved. | | Optional: \{\}
| | `recentRunRefs` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core) array_ | RecentRunRefs identifies retained terminal runs in newest-first order. | | Optional: \{\}
| | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#condition-v1-meta) array_ | Conditions reports reconciliation of the persistent request, not run execution outcomes. | | Optional: \{\}
| @@ -3516,7 +3516,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `observedGeneration` _integer_ | ObservedGeneration is the most recent DGDR generation accepted by the controller. | | Optional: \{\}
| -| `activeRunRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core)_ | ActiveRunRef identifies the one non-terminal run. | | Optional: \{\}
| +| `activeRunRef` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core)_ | ActiveRunRef identifies the reserved or active Run. During a spec change it continues to
identify the superseded Run until foreground deletion completes, before a replacement is reserved. | | Optional: \{\}
| | `recentRunRefs` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core) array_ | RecentRunRefs identifies retained terminal runs in newest-first order. | | Optional: \{\}
| | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#condition-v1-meta) array_ | Conditions reports reconciliation of the persistent request, not run execution outcomes. | | Optional: \{\}
| | `v1beta1` _[JSON](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#json-v1-apiextensions-k8s-io)_ | V1Beta1 losslessly preserves a native v1beta1 status while this object is
represented through v1beta2. | | Type: object
Optional: \{\}
| @@ -3624,8 +3624,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `sku` _string_ | SKU identifies the accelerator using the AISimulate hardware catalog name.
When omitted, admission discovers the accelerator from the cluster. | | MinLength: 1
Optional: \{\}
| -| `budget` _integer_ | Budget is the maximum total number of GPUs one candidate may use.
When omitted, admission derives the available GPU budget from the cluster. | | Minimum: 1
Optional: \{\}
| +| `sku` _string_ | SKU identifies the accelerator using the AISimulate hardware catalog name.
Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`,
`a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`,
`h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`.
The field remains open so an external AISimulate systems catalog can add identifiers.
Admission passes an unknown identifier through unchanged and warns when `budget` is also
explicit. Without an explicit `budget`, it rejects the identifier because cluster discovery
cannot associate capacity with it. When omitted, admission discovers the accelerator. | | MinLength: 1
Optional: \{\}
| +| `budget` _integer_ | Budget is the maximum total number of GPUs one candidate may use.
When omitted for a known SKU, admission derives the available GPU budget from the cluster.
An SKU unknown to the bundled AISimulate catalog requires an explicit `budget`. | | Minimum: 1
Optional: \{\}
| #### v1beta2 HardwareSpec @@ -3644,7 +3644,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `gpu` _[GPUHardwareSpec](#gpuhardwarespec)_ | GPU describes accelerator identity and the maximum allocation per candidate.
Omitted fields are resolved independently by admission. | | Optional: \{\}
| +| `gpu` _[GPUHardwareSpec](#gpuhardwarespec)_ | GPU describes accelerator identity and the maximum allocation per candidate.
Admission resolves omitted fields from the cluster when the SKU is known. | | Optional: \{\}
| #### KVCapacityFractionRange diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index b45b6b214a0d..6005a7e0e477 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -21,6 +21,35 @@ and modify a request through the API version that owns its native fields; do not set or edit the conversion field directly. An unchanged conversion view can be written back without changing the stored request. +### v1beta2 hardware SKU + +`spec.hardware.gpu.sku` in `nvidia.com/v1beta2` is an open string containing an +AISimulate system-catalog identifier. The AISimulate release bundled with this +Dynamo release includes `a100_pcie`, `a100_sxm`, `a30`, `b200_sxm`, `b300_sxm`, +`b60`, `gb200`, `gb300`, `h100_pcie`, `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and +`rtx_pro_6000_server`. + +Admission passes an explicit SKU through unchanged. It warns when the bundled +catalog does not contain that identifier, but does not reject it: an external +AISimulate systems catalog may provide additional identifiers. Set both `sku` +and `budget` to override cluster hardware discovery completely. With only an +unknown explicit SKU, admission cannot associate discovered capacity with that +identifier and rejects the request with instructions to set `budget` explicitly. + +```yaml +apiVersion: nvidia.com/v1beta2 +kind: DynamoGraphDeploymentRequest +metadata: + name: future-accelerator +spec: + model: + name: Qwen/Qwen3-0.6B + hardware: + gpu: + sku: future_accelerator # Passed to AISimulate unchanged; admission warns if unknown. + budget: 8 # Makes the explicit hardware independent of cluster discovery. +``` + ## Spec reference Only `model` is unconditionally required by the schema. Admission rejects a new DGDR whose supplied `spec.image` has no parseable semantic-version tag unless `spec.runtimeVersionOverride` is set. When `spec.image` is omitted, the operator supplies a versioned default as described below. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 9df5d9746b76..7f50ce436007 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4848,7 +4848,7 @@ ObservedGeneration is the most recent DGDR generation accepted by the controller -ActiveRunRef identifies the one non-terminal run. +ActiveRunRef identifies the reserved or active Run. During a spec change it continues to identify the superseded Run until foreground deletion completes, before a replacement is reserved. See [LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core). **Validation:** Optional: \{\} @@ -4936,7 +4936,7 @@ ObservedGeneration is the most recent DGDR generation accepted by the controller -ActiveRunRef identifies the one non-terminal run. +ActiveRunRef identifies the reserved or active Run. During a spec change it continues to identify the superseded Run until foreground deletion completes, before a replacement is reserved. See [LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#localobjectreference-v1-core). **Validation:** Optional: \{\} @@ -5146,12 +5146,12 @@ GPUHardwareSpec describes accelerator identity and the maximum allocation per ca **Appears in:** [HardwareSpec](#v1beta2-hardwarespec) -SKU identifies the accelerator using the AISimulate hardware catalog name. When omitted, admission discovers the accelerator from the cluster. +SKU identifies the accelerator using the AISimulate hardware catalog name. Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. The field remains open so an external AISimulate systems catalog can add identifiers. Admission passes an unknown identifier through unchanged and warns when `budget` is also explicit. Without an explicit `budget`, it rejects the identifier because cluster discovery cannot associate capacity with it. When omitted, admission discovers the accelerator. **Validation:** MinLength: 1 Optional: \{\} -Budget is the maximum total number of GPUs one candidate may use. When omitted, admission derives the available GPU budget from the cluster. +Budget is the maximum total number of GPUs one candidate may use. When omitted for a known SKU, admission derives the available GPU budget from the cluster. An SKU unknown to the bundled AISimulate catalog requires an explicit `budget`. **Validation:** Minimum: 1 Optional: \{\} @@ -5164,7 +5164,7 @@ HardwareSpec bounds the accelerator configurations evaluated by the search. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -GPU describes accelerator identity and the maximum allocation per candidate. Omitted fields are resolved independently by admission. +GPU describes accelerator identity and the maximum allocation per candidate. Admission resolves omitted fields from the cluster when the SKU is known. See [GPUHardwareSpec](#gpuhardwarespec). **Validation:** Optional: \{\} From 20d1abd3a8e154fc3375242c3a2662ec04376ec1 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 19:19:13 +0100 Subject: [PATCH 022/100] pr-16661: pr-3: make SKU catalog copyright-safe [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- .../tests/test_hardware_catalog_sync.py | 10 ++++++- .../operator/internal/aisimulate/hardware.go | 9 ++++-- .../internal/aisimulate/hardware_skus.txt | 3 ++ .../internal/aisimulate/hardware_test.go | 29 +++++++++++++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 deploy/operator/internal/aisimulate/hardware_test.go diff --git a/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py b/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py index fb3652b73003..00c052901ac7 100644 --- a/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py +++ b/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py @@ -66,9 +66,17 @@ def _documented_hardware_skus() -> list[str]: return sorted(re.findall(r"`([^`]+)`", match.group(1))) +def _operator_hardware_skus() -> list[str]: + return sorted( + line.strip() + for line in CATALOG_PATH.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + + def test_dgdr_hardware_catalog_matches_pinned_aisimulate() -> None: assert metadata.version("aisimulate") == _pinned_version() expected = _packaged_hardware_skus() - assert CATALOG_PATH.read_text().split() == expected + assert _operator_hardware_skus() == expected assert _documented_hardware_skus() == expected diff --git a/deploy/operator/internal/aisimulate/hardware.go b/deploy/operator/internal/aisimulate/hardware.go index 004386117650..667289217d2c 100644 --- a/deploy/operator/internal/aisimulate/hardware.go +++ b/deploy/operator/internal/aisimulate/hardware.go @@ -30,9 +30,12 @@ import ( var hardwareSKUsText string var hardwareSKUs = func() map[string]struct{} { - values := strings.Fields(hardwareSKUsText) - result := make(map[string]struct{}, len(values)) - for _, value := range values { + result := map[string]struct{}{} + for _, line := range strings.Split(hardwareSKUsText, "\n") { + value := strings.TrimSpace(line) + if value == "" || strings.HasPrefix(value, "#") { + continue + } result[value] = struct{}{} } return result diff --git a/deploy/operator/internal/aisimulate/hardware_skus.txt b/deploy/operator/internal/aisimulate/hardware_skus.txt index 43053fd0c105..e55bc7594a89 100644 --- a/deploy/operator/internal/aisimulate/hardware_skus.txt +++ b/deploy/operator/internal/aisimulate/hardware_skus.txt @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + a100_pcie a100_sxm a30 diff --git a/deploy/operator/internal/aisimulate/hardware_test.go b/deploy/operator/internal/aisimulate/hardware_test.go new file mode 100644 index 000000000000..3a50b6d33d30 --- /dev/null +++ b/deploy/operator/internal/aisimulate/hardware_test.go @@ -0,0 +1,29 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import "testing" + +func TestHardwareCatalogIgnoresComments(t *testing.T) { + if !IsKnownHardwareSKU("h200_sxm") { + t.Fatal("h200_sxm must be present in the bundled hardware catalog") + } + if IsKnownHardwareSKU("#") || IsKnownHardwareSKU("SPDX-License-Identifier:") { + t.Fatal("catalog comments must not be interpreted as hardware SKUs") + } +} From fe47d0611c9c36d810719cc5dee2ab27efa5c486 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 19:24:46 +0100 Subject: [PATCH 023/100] pr-16661: pr-3: modernize SKU catalog parser [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- deploy/operator/internal/aisimulate/hardware.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/operator/internal/aisimulate/hardware.go b/deploy/operator/internal/aisimulate/hardware.go index 667289217d2c..13c9af441feb 100644 --- a/deploy/operator/internal/aisimulate/hardware.go +++ b/deploy/operator/internal/aisimulate/hardware.go @@ -31,7 +31,7 @@ var hardwareSKUsText string var hardwareSKUs = func() map[string]struct{} { result := map[string]struct{}{} - for _, line := range strings.Split(hardwareSKUsText, "\n") { + for line := range strings.SplitSeq(hardwareSKUsText, "\n") { value := strings.TrimSpace(line) if value == "" || strings.HasPrefix(value, "#") { continue From c32369455f502287e59b1cb9357fda9d878dd3b4 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 20:09:41 +0100 Subject: [PATCH 024/100] pr-16661: pr-3: use selected-shape GPU capacity [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- .../internal/controller/suite_envtest_test.go | 2 +- ...amographdeploymentrequest_v1beta2_hardware_test.go | 10 +++++----- .../dynamographdeploymentrequest_v1beta2_helpers.go | 4 ++-- .../internal/webhook/defaulting/suite_envtest_test.go | 11 ++++++++++- ...ographdeploymentrequest_validation_envtest_test.go | 4 ++-- .../internal/webhook/validation/suite_envtest_test.go | 2 +- 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/deploy/operator/internal/controller/suite_envtest_test.go b/deploy/operator/internal/controller/suite_envtest_test.go index ce13f3d349e3..220da7d938e0 100644 --- a/deploy/operator/internal/controller/suite_envtest_test.go +++ b/deploy/operator/internal/controller/suite_envtest_test.go @@ -31,7 +31,7 @@ const admissionBypassUsername = "operatorenv-controller-admission-bypass" type fixedHardwareDiscoverer struct{} func (fixedHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { - return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8}, nil + return &gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8}, nil } var ( diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index e5b11b39bf7c..ea0dd1f7a69a 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -59,8 +59,8 @@ func hardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec) *nvidiacomv1be func TestResolveHardwareV1beta2(t *testing.T) { str := func(s string) *string { return &s } num := func(n int32) *int32 { return &n } - h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} - h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64} + h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8} + h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64, SelectedShapeGPUs: 64} completeH100 := hardwareOfV1beta2(str("h100_sxm"), num(4)) tests := []struct { name string @@ -84,9 +84,9 @@ func TestResolveHardwareV1beta2(t *testing.T) { {name: "omitted hardware discovers the SKU and the budget", op: admissionv1.Create, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, wantMinTimeout: time.Millisecond, wantMaxTimeout: defaultHardwareDiscoveryTimeout}, {name: "an empty hardware object is automatic discovery", op: admissionv1.Create, hardware: &nvidiacomv1beta2.HardwareSpec{}, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1}, { - name: "same-SKU nodes with different GPU counts use the aggregate", op: admissionv1.Create, - found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12}, - wantSKU: "h200_sxm", wantBudget: 12, wantCalls: 1, + name: "same-SKU nodes with different GPU counts use the selected node shape", op: admissionv1.Create, + found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12, SelectedShapeGPUs: 8}, + wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, }, { name: "an empty gpu object is automatic discovery", op: admissionv1.Create, diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 7ac29c3457b9..0fec7422f8a9 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -146,7 +146,7 @@ func (o DGDRV1beta2Options) resolveHardware( } // The cluster facts are mapped into the v1beta2 API here: the AISimulate catalog name of - // the GPU model (empty when the catalog does not know it) and the total GPU count. + // the GPU model (empty when the catalog does not know it) and the selected node-shape capacity. var defaults hardwareDefaultsV1beta2 if sku == "" { if found != nil && len(found.OtherModels) > 0 { @@ -168,7 +168,7 @@ func (o DGDRV1beta2Options) resolveHardware( // Discovery is needed only for an omitted budget at this point. var gpus int if found != nil { - gpus = found.TotalGPUs + gpus = found.SelectedShapeGPUs } if !hasBudget { if gpus <= 0 { diff --git a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go index 5b02c72681e7..4f0168dc4ef6 100644 --- a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go @@ -304,7 +304,7 @@ func TestDynamoGraphDeploymentRequestV1beta1DefaultingAdmission(t *testing.T) { } func TestDynamoGraphDeploymentRequestV1beta2DefaultingAdmission(t *testing.T) { - discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} + discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8} v1beta2Tests := []struct { name string request *nvidiacomv1beta2.DynamoGraphDeploymentRequest @@ -324,6 +324,15 @@ func TestDynamoGraphDeploymentRequestV1beta2DefaultingAdmission(t *testing.T) { check: expectV1beta2ModelOnlyDefaults, idempotent: true, }, + { + name: "v1beta2 heterogeneous nodes default the selected-shape budget", + request: v1beta2DefaultingRequest(), + discovered: gpu.GPUInfo{ + System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, + TotalGPUs: 12, SelectedShapeGPUs: 8, + }, + check: expectV1beta2H200x8Hardware, + }, { name: "v1beta2 explicit empty static workload and SLA opt out", request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index e07f8693ad81..5cb03fcb030b 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -56,8 +56,8 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec":{"components":[{"name":"VllmPrefillWorker"}]} }` - discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} - discoveredH100 := gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8} + discoveredH200 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8} + discoveredH100 := gpu.GPUInfo{System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8} tests := []struct { name string diff --git a/deploy/operator/internal/webhook/validation/suite_envtest_test.go b/deploy/operator/internal/webhook/validation/suite_envtest_test.go index 67ee9f5e6f6b..e90f023f197c 100644 --- a/deploy/operator/internal/webhook/validation/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/suite_envtest_test.go @@ -125,7 +125,7 @@ func (f *fakeHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUS func setAdmissionH100Topology() { admissionHardware.set(gpu.GPUInfo{ - System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, + System: "h100_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8, }, nil) } From 7bc8fb4ccc119fd118e1257a705c3f88bb146a58 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 12:08:02 +0100 Subject: [PATCH 025/100] pr-16661: pr-3: defer SKU catalog smoke test [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- .../tests/test_hardware_catalog_sync.py | 82 ------------------- container/Dockerfile.test | 4 - 2 files changed, 86 deletions(-) delete mode 100644 components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py diff --git a/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py b/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py deleted file mode 100644 index 00c052901ac7..000000000000 --- a/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Keep DGDR's hardware hints synchronized with the pinned AISimulate wheel.""" - -from __future__ import annotations - -import re -from importlib import metadata -from pathlib import Path - -import pytest -import yaml - -pytestmark = [ - pytest.mark.unit, - pytest.mark.gpu_0, - pytest.mark.pre_merge, - pytest.mark.planner, - pytest.mark.parallel, -] - -REPO_ROOT = Path(__file__).resolve().parents[5] -CATALOG_PATH = REPO_ROOT / "deploy/operator/internal/aisimulate/hardware_skus.txt" -API_PATH = ( - REPO_ROOT / "deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go" -) -PIN_PATH = REPO_ROOT / "container/deps/requirements.aisimulate.txt" - - -def _pinned_version() -> str: - pins = [ - line.split("==", 1)[1].strip() - for line in PIN_PATH.read_text().splitlines() - if line.startswith("aisimulate==") - ] - assert len(pins) == 1, "requirements.aisimulate.txt must contain one exact pin" - return pins[0] - - -def _packaged_hardware_skus() -> list[str]: - distribution = metadata.distribution("aisimulate") - system_files = [ - path - for path in distribution.files or [] - if path.parent.as_posix() == "aisimulate_core/systems" - and path.suffix == ".yaml" - ] - result = [] - for packaged_path in system_files: - with distribution.locate_file(packaged_path).open(encoding="utf-8") as stream: - document = yaml.safe_load(stream) - if isinstance(document, dict) and "gpu" in document and "node" in document: - result.append(packaged_path.stem) - return sorted(result) - - -def _documented_hardware_skus() -> list[str]: - source = API_PATH.read_text() - match = re.search( - r"Known values in the bundled AISimulate release are (.*?)\n\s*// The field remains open", - source, - re.DOTALL, - ) - assert match is not None, "GPUHardwareSpec.SKU must document the bundled catalog" - return sorted(re.findall(r"`([^`]+)`", match.group(1))) - - -def _operator_hardware_skus() -> list[str]: - return sorted( - line.strip() - for line in CATALOG_PATH.read_text().splitlines() - if line.strip() and not line.lstrip().startswith("#") - ) - - -def test_dgdr_hardware_catalog_matches_pinned_aisimulate() -> None: - assert metadata.version("aisimulate") == _pinned_version() - - expected = _packaged_hardware_skus() - assert _operator_hardware_skus() == expected - assert _documented_hardware_skus() == expected diff --git a/container/Dockerfile.test b/container/Dockerfile.test index 245ce09d8135..920a4732d535 100644 --- a/container/Dockerfile.test +++ b/container/Dockerfile.test @@ -129,10 +129,6 @@ COPY --chmod=664 --chown=dynamo:0 components/src/dynamo/mocker/config.py /worksp # Give the isolated copy a unique basename so pytest cannot alias it with # another non-package `test_config.py` during whole-workspace collection. COPY --chmod=664 --chown=dynamo:0 components/src/dynamo/mocker/tests/unit/test_config.py /workspace/components/src/dynamo/mocker/tests/unit/test_mocker_config.py -COPY --chmod=664 --chown=dynamo:0 components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py /workspace/components/src/dynamo/aisimulate/tests/test_hardware_catalog_sync.py -COPY --chmod=664 --chown=dynamo:0 deploy/operator/internal/aisimulate/hardware_skus.txt /workspace/deploy/operator/internal/aisimulate/hardware_skus.txt -COPY --chmod=664 --chown=dynamo:0 deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go /workspace/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go -COPY --chmod=664 --chown=dynamo:0 container/deps/requirements.aisimulate.txt /workspace/container/deps/requirements.aisimulate.txt # BuildKit applies the file mode to parent directories it creates for these # single-file COPY instructions. Restore directory traversal/write access for From 188a0755e19a7e7afe4c3ad56dc4a5fa8cade074 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 12:55:23 +0100 Subject: [PATCH 026/100] pr-16661: pr-3: synchronize operator SKU catalog [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- .github/filters.yaml | 4 + .../hack/test_aisimulate_hardware_catalog.py | 93 +++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 deploy/operator/hack/test_aisimulate_hardware_catalog.py diff --git a/.github/filters.yaml b/.github/filters.yaml index a0f8a7a005bc..fc6ea9298ff4 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -362,6 +362,10 @@ planner: - 'components/src/dynamo/profiler/**' - 'components/src/dynamo/aisimulate/**' - 'components/src/dynamo/global_router/**' + # Re-run the SKU contract when either compared operator input changes. + - 'deploy/operator/hack/test_aisimulate_hardware_catalog.py' + - 'deploy/operator/internal/aisimulate/hardware_skus.txt' + - 'deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go' # Keep this tombstone until the vendored-tree deletion has merged so the # deleted paths in that PR remain covered by the Planner CI filter. - 'aisimulate/**' diff --git a/deploy/operator/hack/test_aisimulate_hardware_catalog.py b/deploy/operator/hack/test_aisimulate_hardware_catalog.py new file mode 100644 index 000000000000..d4bd24083524 --- /dev/null +++ b/deploy/operator/hack/test_aisimulate_hardware_catalog.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Keep DGDR's hardware hints synchronized with the pinned AISimulate wheel.""" + +from __future__ import annotations + +import re +import tomllib +from importlib import metadata +from pathlib import Path + +import pytest +import yaml +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name +from packaging.version import Version + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + +REPO_ROOT = Path(__file__).resolve().parents[3] +CATALOG_PATH = REPO_ROOT / "deploy/operator/internal/aisimulate/hardware_skus.txt" +API_PATH = ( + REPO_ROOT / "deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go" +) +PYPROJECT_PATH = REPO_ROOT / "pyproject.toml" + + +def _pinned_version() -> Version: + with PYPROJECT_PATH.open("rb") as stream: + dependencies = tomllib.load(stream)["project"]["dependencies"] + requirements = [ + Requirement(dependency) + for dependency in dependencies + if canonicalize_name(Requirement(dependency).name) == "aisimulate" + ] + assert len(requirements) == 1, "pyproject.toml must contain one AISimulate pin" + + requirement = requirements[0] + specifiers = list(requirement.specifier) + assert requirement.url is None + assert len(specifiers) == 1 and specifiers[0].operator == "==" + return Version(specifiers[0].version) + + +def _packaged_hardware_skus() -> list[str]: + distribution = metadata.distribution("aisimulate") + system_files = [ + path + for path in distribution.files or [] + if path.parent.as_posix() == "aisimulate_core/systems" + and path.suffix == ".yaml" + ] + result = [] + for packaged_path in system_files: + with distribution.locate_file(packaged_path).open(encoding="utf-8") as stream: + document = yaml.safe_load(stream) + if isinstance(document, dict) and "gpu" in document and "node" in document: + result.append(packaged_path.stem) + return sorted(result) + + +def _documented_hardware_skus() -> list[str]: + source = API_PATH.read_text() + match = re.search( + r"Known values in the bundled AISimulate release are (.*?)\n\s*// The field remains open", + source, + re.DOTALL, + ) + assert match is not None, "GPUHardwareSpec.SKU must document the bundled catalog" + return sorted(re.findall(r"`([^`]+)`", match.group(1))) + + +def _operator_hardware_skus() -> list[str]: + return sorted( + line.strip() + for line in CATALOG_PATH.read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ) + + +def test_dgdr_hardware_catalog_matches_pinned_aisimulate() -> None: + assert Version(metadata.version("aisimulate")) == _pinned_version() + + expected = _packaged_hardware_skus() + assert _operator_hardware_skus() == expected + assert _documented_hardware_skus() == expected From 390c89ca917ec61f06dbbbbf60ff1768a3057b78 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 15:50:25 +0100 Subject: [PATCH 027/100] pr-16661: pr-3: persist selected node topology [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- .../components/operator/files/role.yaml | 9 +- .../dynamographdeploymentrequest_types.go | 29 ++- .../api/v1beta2/zz_generated.deepcopy.go | 5 + ...dia.com_dynamographdeploymentrequests.yaml | 27 ++- .../nvidia.com_dynamographdeploymentruns.yaml | 30 +++- deploy/operator/internal/gpu/discovery.go | 4 +- deploy/operator/internal/gpu/heterogeneous.go | 26 ++- .../internal/gpu/heterogeneous_test.go | 29 +-- .../dynamographdeploymentrequest_v1beta2.go | 18 +- ...deploymentrequest_v1beta2_hardware_test.go | 117 +++++++++--- ...ographdeploymentrequest_v1beta2_helpers.go | 166 ++++++++++++++---- ...ploymentrequest_validation_envtest_test.go | 58 +++++- .../additional-resources/api-reference-k8s.md | 18 +- .../dynamo-graph-deployment-request.mdx | 36 +++- .../kubernetes-api/full-api-reference.mdx | 22 ++- 15 files changed, 463 insertions(+), 131 deletions(-) diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 49e0d5309ea6..3fff0d64273d 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -24,9 +24,10 @@ rules: - "" resources: - nodes - - pods/log verbs: - get + - list + - watch - apiGroups: - "" resources: @@ -48,6 +49,12 @@ rules: - list - patch - watch +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get - apiGroups: - "" resources: diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 4170620bc01d..25536741ca97 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -48,27 +48,40 @@ type GPUHardwareSpec struct { // `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, // `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. // The field remains open so an external AISimulate systems catalog can add identifiers. - // Admission passes an unknown identifier through unchanged and warns when `budget` is also - // explicit. Without an explicit `budget`, it rejects the identifier because cluster discovery - // cannot associate capacity with it. When omitted, admission discovers the accelerator. + // Admission passes an unknown identifier through unchanged and warns when `budget` and the + // parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because + // cluster discovery cannot associate capacity or topology with it. When omitted from a + // nonempty hardware object, admission discovers the accelerator. // +optional // +kubebuilder:validation:MinLength=1 SKU *string `json:"sku,omitempty"` // Budget is the maximum total number of GPUs one candidate may use. // When omitted for a known SKU, admission derives the available GPU budget from the cluster. - // An SKU unknown to the bundled AISimulate catalog requires an explicit `budget`. + // An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an + // explicit `numGpusPerNode` in the parent hardware object. // +optional // +kubebuilder:validation:Minimum=1 Budget *int32 `json:"budget,omitempty"` } -// HardwareSpec bounds the accelerator configurations evaluated by the search. +// HardwareSpec bounds the accelerator configurations evaluated by the search. An exactly empty +// object opts out of operator cluster discovery and typed hardware defaults. type HardwareSpec struct { // GPU describes accelerator identity and the maximum allocation per candidate. - // Admission resolves omitted fields from the cluster when the SKU is known. + // In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. // +optional GPU *GPUHardwareSpec `json:"gpu,omitempty"` + + // NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers + // to AISimulate's num_gpus_per_node input. + // An explicit value is authoritative. In a nonempty hardware object, admission discovers + // an omitted value from the selected cluster node shape. The Run controller never discovers + // this value. An exactly empty hardware object opts out of all operator hardware discovery + // and typed hardware defaults, leaving AISimulate input and defaults authoritative. + // +optional + // +kubebuilder:validation:Minimum=1 + NumGPUsPerNode *int32 `json:"numGpusPerNode,omitempty"` } // KVCapacityFractionRange defines the searched fraction of a candidate's KV capacity. @@ -260,7 +273,9 @@ type NativeSearchInput struct { Backends []Backend `json:"backends,omitempty"` // Hardware bounds the accelerator configurations evaluated by the search. - // Admission discovers any omitted hardware values from the cluster. + // When omitted, admission discovers SKU, budget and node width. An exactly empty object opts + // out of operator hardware discovery and typed defaults. Admission completes every nonempty + // object with any missing hardware values; explicit values remain authoritative. // +optional Hardware *HardwareSpec `json:"hardware,omitempty"` diff --git a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go index fe68ab15bfa9..899c5121053e 100644 --- a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go @@ -567,6 +567,11 @@ func (in *HardwareSpec) DeepCopyInto(out *HardwareSpec) { *out = new(GPUHardwareSpec) (*in).DeepCopyInto(*out) } + if in.NumGPUsPerNode != nil { + in, out := &in.NumGPUsPerNode, &out.NumGPUsPerNode + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HardwareSpec. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 2d79c1f9e782..5a1d6bb9d9c9 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9631,18 +9631,21 @@ spec: hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. - Admission discovers any omitted hardware values from the cluster. + When omitted, admission discovers SKU, budget and node width. An exactly empty object opts + out of operator hardware discovery and typed defaults. Admission completes every nonempty + object with any missing hardware values; explicit values remain authoritative. properties: gpu: description: |- GPU describes accelerator identity and the maximum allocation per candidate. - Admission resolves omitted fields from the cluster when the SKU is known. + In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. properties: budget: description: |- Budget is the maximum total number of GPUs one candidate may use. When omitted for a known SKU, admission derives the available GPU budget from the cluster. - An SKU unknown to the bundled AISimulate catalog requires an explicit `budget`. + An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an + explicit `numGpusPerNode` in the parent hardware object. format: int32 minimum: 1 type: integer @@ -9653,12 +9656,24 @@ spec: `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. The field remains open so an external AISimulate systems catalog can add identifiers. - Admission passes an unknown identifier through unchanged and warns when `budget` is also - explicit. Without an explicit `budget`, it rejects the identifier because cluster discovery - cannot associate capacity with it. When omitted, admission discovers the accelerator. + Admission passes an unknown identifier through unchanged and warns when `budget` and the + parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because + cluster discovery cannot associate capacity or topology with it. When omitted from a + nonempty hardware object, admission discovers the accelerator. minLength: 1 type: string type: object + numGpusPerNode: + description: |- + NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers + to AISimulate's num_gpus_per_node input. + An explicit value is authoritative. In a nonempty hardware object, admission discovers + an omitted value from the selected cluster node shape. The Run controller never discovers + this value. An exactly empty hardware object opts out of all operator hardware discovery + and typed hardware defaults, leaving AISimulate input and defaults authoritative. + format: int32 + minimum: 1 + type: integer type: object model: description: Model identifies the model whose deployment configurations are evaluated. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 68606522cc32..bb2e6bc0ae58 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -95,27 +95,49 @@ spec: hardware: description: |- Hardware bounds the accelerator configurations evaluated by the search. - Admission discovers any omitted hardware values from the cluster. + When omitted, admission discovers SKU, budget and node width. An exactly empty object opts + out of operator hardware discovery and typed defaults. Admission completes every nonempty + object with any missing hardware values; explicit values remain authoritative. properties: gpu: description: |- GPU describes accelerator identity and the maximum allocation per candidate. - Omitted fields are resolved independently by admission. + In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. properties: budget: description: |- Budget is the maximum total number of GPUs one candidate may use. - When omitted, admission derives the available GPU budget from the cluster. + When omitted for a known SKU, admission derives the available GPU budget from the cluster. + An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an + explicit `numGpusPerNode` in the parent hardware object. format: int32 minimum: 1 type: integer sku: description: |- SKU identifies the accelerator using the AISimulate hardware catalog name. - When omitted, admission discovers the accelerator from the cluster. + Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, + `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, + `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. + The field remains open so an external AISimulate systems catalog can add identifiers. + Admission passes an unknown identifier through unchanged and warns when `budget` and the + parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because + cluster discovery cannot associate capacity or topology with it. When omitted from a + nonempty hardware object, admission discovers the accelerator. minLength: 1 type: string type: object + numGpusPerNode: + description: |- + NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers + to AISimulate's num_gpus_per_node input. + An explicit value is authoritative. In a nonempty hardware object, admission discovers + an omitted value from the selected cluster node shape. The Run controller never discovers + this value. An exactly empty hardware object opts out of all operator hardware discovery + and typed hardware defaults, leaving AISimulate input and defaults authoritative. + format: int32 + minimum: 1 + type: integer type: object model: description: Model identifies the model whose deployment configurations are evaluated. diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go index 97233ca7d81f..5a64f454fed2 100644 --- a/deploy/operator/internal/gpu/discovery.go +++ b/deploy/operator/internal/gpu/discovery.go @@ -458,7 +458,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, if err != nil { cloudProvider = CloudProviderUnknown } - bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, len(allNodes), + bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, bestNode.Model, len(allNodes), func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return allNodes[i].sku, allNodes[i].info.Model }) bestNode.System = bestSKU bestNode.CloudProvider = cloudProvider @@ -931,7 +931,7 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK rdmaDetected = true } - bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, len(allNodes), + bestNode.OtherModels = collectOtherModels(filterSKU, bestSKU, bestNode.Model, len(allNodes), func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return allNodes[i].sku, allNodes[i].info.Model }) bestNode.System = bestSKU bestNode.NodesWithGPUs = nodesWithGPUs diff --git a/deploy/operator/internal/gpu/heterogeneous.go b/deploy/operator/internal/gpu/heterogeneous.go index 869e6115188f..7bb4861c7e3e 100644 --- a/deploy/operator/internal/gpu/heterogeneous.go +++ b/deploy/operator/internal/gpu/heterogeneous.go @@ -22,28 +22,38 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" ) -// collectOtherModels returns, sorted and without repeats, the models of the nodes whose SKU -// differs from the selected one. A filtered discovery asked for one SKU and returns nothing: -// the nodes of other SKUs were never candidates. at yields the SKU and the model of node i. +// collectOtherModels returns, sorted and without repeats, the models of nodes whose GPU identity +// differs from the selected one. It uses the same identity rule as capacity aggregation: known +// GPUs are grouped by SKU and unknown GPUs by normalized model name. A filtered discovery asked +// for one SKU and returns nothing because nodes of other SKUs were never candidates. func collectOtherModels( - filterSKU, selected nvidiacomv1beta1.GPUSKUType, + filterSKU, selectedSKU nvidiacomv1beta1.GPUSKUType, + selectedModel string, nodes int, at func(i int) (nvidiacomv1beta1.GPUSKUType, string), ) []string { if filterSKU != "" { return nil } - seen := map[string]struct{}{} + seen := map[string]string{} for i := range nodes { - if sku, model := at(i); sku != selected { - seen[model] = struct{}{} + sku, model := at(i) + if matchesSelectedGPUType(sku, selectedSKU, model, selectedModel) { + continue + } + identity := "sku:" + string(sku) + if sku == "" { + identity = "model:" + normalize(model) + } + if existing, exists := seen[identity]; !exists || model < existing { + seen[identity] = model } } if len(seen) == 0 { return nil } models := make([]string, 0, len(seen)) - for model := range seen { + for _, model := range seen { models = append(models, model) } sort.Strings(models) diff --git a/deploy/operator/internal/gpu/heterogeneous_test.go b/deploy/operator/internal/gpu/heterogeneous_test.go index 71380d227077..c07dfc1451b9 100644 --- a/deploy/operator/internal/gpu/heterogeneous_test.go +++ b/deploy/operator/internal/gpu/heterogeneous_test.go @@ -29,24 +29,33 @@ func TestCollectOtherModels(t *testing.T) { model string } tests := []struct { - name string - filter nvidiacomv1beta1.GPUSKUType - selected nvidiacomv1beta1.GPUSKUType - nodes []node - want []string + name string + filter nvidiacomv1beta1.GPUSKUType + selectedSKU nvidiacomv1beta1.GPUSKUType + selectedModel string + nodes []node + want []string }{ - {"a cluster of one SKU", "", "h200_sxm", []node{{"h200_sxm", "H200"}, {"h200_sxm", "H200"}}, nil}, - {"a second SKU is reported once, sorted", "", "h200_sxm", + {"a cluster of one SKU", "", "h200_sxm", "H200", []node{{"h200_sxm", "H200"}, {"h200_sxm", "H200"}}, nil}, + {"a second SKU is reported once, sorted", "", "h200_sxm", "H200", []node{{"h200_sxm", "H200"}, {"h100_sxm", "H100"}, {"b200_sxm", "B200"}, {"h100_sxm", "H100"}}, []string{"B200", "H100"}}, - {"a model the catalog does not know still counts", "", "h200_sxm", + {"a model the catalog does not know still counts", "", "h200_sxm", "H200", []node{{"h200_sxm", "H200"}, {"", "Mystery GPU"}}, []string{"Mystery GPU"}}, - {"a filtered discovery never reports", "h200_sxm", "h200_sxm", + {"distinct models unknown to the catalog remain distinct", "", "", "Mystery Alpha", + []node{{"", "Mystery Alpha"}, {"", "Mystery Beta"}}, []string{"Mystery Beta"}}, + {"normalized spellings of an unknown selected model share its identity", "", "", "Mystery GPU", + []node{{"", "Mystery GPU"}, {"", "mystery-gpu"}}, nil}, + {"normalized spellings of another unknown model are reported once", "", "", "Mystery Alpha", + []node{{"", "Mystery Alpha"}, {"", "mystery-beta"}, {"", "Mystery Beta"}}, []string{"Mystery Beta"}}, + {"known models with the same SKU are reported once", "", "h200_sxm", "H200", + []node{{"h200_sxm", "H200"}, {"h100_sxm", "NVIDIA H100"}, {"h100_sxm", "H100 SXM"}}, []string{"H100 SXM"}}, + {"a filtered discovery never reports", "h200_sxm", "h200_sxm", "H200", []node{{"h200_sxm", "H200"}, {"h100_sxm", "H100"}}, nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := collectOtherModels(tt.filter, tt.selected, len(tt.nodes), + got := collectOtherModels(tt.filter, tt.selectedSKU, tt.selectedModel, len(tt.nodes), func(i int) (nvidiacomv1beta1.GPUSKUType, string) { return tt.nodes[i].sku, tt.nodes[i].model }) if !reflect.DeepEqual(got, tt.want) { t.Fatalf("got %v, want %v", got, tt.want) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index 2906f7b53372..612b4fb9eb71 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -52,9 +52,10 @@ const ( // // Admission owns every v1beta2 default, and each default has this one owner: backends, // objective.mode, the scalar metric, objective.sla, recommendation.maxCandidates, -// workload.static and the cluster hardware. workload.static: {} opts out of the static +// workload.static and discovered cluster hardware. workload.static: {} opts out of the static // workload; a nonempty static object receives any missing inputTokens, outputTokens and load -// defaults. objective.sla: {} opts out of the latency defaults. +// defaults. objective.sla: {} opts out of the latency defaults. hardware: {} opts out of +// cluster discovery and typed hardware defaults; a nonempty object is completed. // spec.aisimulate.inputOverrides never changes a default. It overrides the defaulted typed // values later, when the input is lowered. @@ -123,14 +124,15 @@ func decodeOldSpecV1beta2( // dynamoGraphDeploymentRequestV1beta2DefaultOptions carries the request-scoped values the // traversal needs. An empty field means the environment had nothing to offer. type dynamoGraphDeploymentRequestV1beta2DefaultOptions struct { - // hardware is the GPU identity and budget discovered from the cluster. + // hardware is the GPU identity, budget and node width discovered from the cluster. hardware hardwareDefaultsV1beta2 } // hardwareDefaultsV1beta2 is the hardware the cluster can offer for the values a Request omits. type hardwareDefaultsV1beta2 struct { - sku string - budget int32 + sku string + budget int32 + numGPUsPerNode int32 } // defaultDynamoGraphDeploymentRequestSpecV1beta2 defaults spec in place. It applies on @@ -160,7 +162,8 @@ func defaultNativeSearchInputV1beta2( input.Backends = []nvidiacomv1beta2.Backend{"vllm", "sglang", "trtllm"} } - // Hardware has no static default: it exists only when the cluster supplied some. + // Omitted hardware receives discovered values. An explicit empty object reaches this + // traversal with empty hardware options and remains the opt-out object. if input.Hardware == nil && opts.hardware != (hardwareDefaultsV1beta2{}) { input.Hardware = &nvidiacomv1beta2.HardwareSpec{} } @@ -191,6 +194,9 @@ func defaultHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec, default if hardware.GPU != nil { defaultGPUHardwareSpecV1beta2(hardware.GPU, defaults) } + if hardware.NumGPUsPerNode == nil && defaults.numGPUsPerNode > 0 { + hardware.NumGPUsPerNode = new(defaults.numGPUsPerNode) + } } // defaultGPUHardwareSpecV1beta2 fills each omitted value on its own, so a caller can pin the diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index ea0dd1f7a69a..0d612b5721eb 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -48,8 +48,12 @@ func (f *fakeDiscoverer) Discover(ctx context.Context, filterSKU nvidiacomv1beta return &found, f.err } -func hardwareOfV1beta2(sku *string, budget *int32) *nvidiacomv1beta2.HardwareSpec { - return &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} +func hardwareOfV1beta2(sku *string, budget *int32, numGPUsPerNode ...*int32) *nvidiacomv1beta2.HardwareSpec { + hardware := &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: sku, Budget: budget}} + if len(numGPUsPerNode) > 0 { + hardware.NumGPUsPerNode = numGPUsPerNode[0] + } + return hardware } func hardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec) *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec { @@ -61,7 +65,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { num := func(n int32) *int32 { return &n } h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8} h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64, SelectedShapeGPUs: 64} - completeH100 := hardwareOfV1beta2(str("h100_sxm"), num(4)) + completeH100 := hardwareOfV1beta2(str("h100_sxm"), num(4), num(8)) tests := []struct { name string op admissionv1.Operation @@ -74,6 +78,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { timeout time.Duration wantSKU string wantBudget int32 + wantGPUsPerNode int32 wantCalls int wantFilter string wantErr string @@ -81,36 +86,65 @@ func TestResolveHardwareV1beta2(t *testing.T) { wantMaxTimeout time.Duration }{ {name: "no discoverer leaves omitted hardware unresolved", op: admissionv1.Create, withoutDiscoverer: true}, - {name: "omitted hardware discovers the SKU and the budget", op: admissionv1.Create, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, wantMinTimeout: time.Millisecond, wantMaxTimeout: defaultHardwareDiscoveryTimeout}, - {name: "an empty hardware object is automatic discovery", op: admissionv1.Create, hardware: &nvidiacomv1beta2.HardwareSpec{}, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1}, + {name: "omitted hardware discovers the SKU budget and node width", op: admissionv1.Create, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantGPUsPerNode: 8, wantCalls: 1, wantMinTimeout: time.Millisecond, wantMaxTimeout: defaultHardwareDiscoveryTimeout}, + {name: "an empty hardware object opts out of discovery", op: admissionv1.Create, hardware: &nvidiacomv1beta2.HardwareSpec{}, discoveryErr: errors.New("discovery must not run")}, { name: "same-SKU nodes with different GPU counts use the selected node shape", op: admissionv1.Create, found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 2, TotalGPUs: 12, SelectedShapeGPUs: 8}, - wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, + wantSKU: "h200_sxm", wantBudget: 8, wantGPUsPerNode: 8, wantCalls: 1, }, { name: "an empty gpu object is automatic discovery", op: admissionv1.Create, hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{}}, found: h200x8, - wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, + wantSKU: "h200_sxm", wantBudget: 8, wantGPUsPerNode: 8, wantCalls: 1, }, - {name: "the discovered budget is capped", op: admissionv1.Create, found: h200x64, wantSKU: "h200_sxm", wantBudget: 32, wantCalls: 1}, - {name: "the discovered budget honors a configured cap", op: admissionv1.Create, found: h200x64, maxBudget: 8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1}, + {name: "the discovered budget is capped", op: admissionv1.Create, found: h200x64, wantSKU: "h200_sxm", wantBudget: 32, wantGPUsPerNode: 8, wantCalls: 1}, + {name: "the discovered budget honors a configured cap", op: admissionv1.Create, found: h200x64, maxBudget: 8, wantSKU: "h200_sxm", wantBudget: 8, wantGPUsPerNode: 8, wantCalls: 1}, { name: "a SKU alone keeps the SKU, filters the cluster and persists the capped budget", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h200_sxm"), nil), found: h200x64, - wantBudget: 32, wantCalls: 1, wantFilter: "h200_sxm", + wantBudget: 32, wantGPUsPerNode: 8, wantCalls: 1, wantFilter: "h200_sxm", + }, + { + name: "a failed discovery for a SKU alone asks for capacity and topology", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h200_sxm"), nil), discoveryErr: errors.New("cluster unreadable"), + wantCalls: 1, wantFilter: "h200_sxm", + wantErr: `spec.hardware.gpu.budget: Required value: could not be discovered for SKU "h200_sxm" (cluster unreadable); set spec.hardware.gpu.budget and spec.hardware.numGpusPerNode`, + }, + {name: "a budget alone keeps the budget and discovers the SKU", op: admissionv1.Create, hardware: hardwareOfV1beta2(nil, num(4)), found: h200x8, wantSKU: "h200_sxm", wantGPUsPerNode: 8, wantCalls: 1}, + {name: "an explicit budget above the cluster size is kept", op: admissionv1.Create, hardware: hardwareOfV1beta2(nil, num(64)), found: h200x8, wantSKU: "h200_sxm", wantGPUsPerNode: 8, wantCalls: 1}, + {name: "an explicit SKU and budget discover the missing node width", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h200_sxm"), num(64)), found: h200x8, wantGPUsPerNode: 8, wantCalls: 1, wantFilter: "h200_sxm"}, + { + name: "a failed node-width discovery asks for the missing field", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h200_sxm"), num(64)), discoveryErr: errors.New("cluster unreadable"), + wantCalls: 1, wantFilter: "h200_sxm", + wantErr: `spec.hardware.numGpusPerNode: Required value: could not be discovered for SKU "h200_sxm" (cluster unreadable); set spec.hardware.numGpusPerNode`, + }, + { + name: "a discovered shape without a node width asks for the missing field", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h200_sxm"), num(64)), + found: gpu.GPUInfo{System: "h200_sxm", SelectedShapeGPUs: 8}, wantCalls: 1, wantFilter: "h200_sxm", + wantErr: `spec.hardware.numGpusPerNode: Required value: could not be discovered for SKU "h200_sxm"; set spec.hardware.numGpusPerNode`, + }, + {name: "fully explicit hardware bypasses discovery", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h200_sxm"), num(64), num(4)), discoveryErr: errors.New("down")}, + { + name: "an explicit node width is authoritative while a missing budget triggers discovery", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("h200_sxm"), nil, num(4)), found: h200x8, + wantBudget: 8, wantCalls: 1, wantFilter: "h200_sxm", }, - {name: "a budget alone keeps the budget and discovers the SKU", op: admissionv1.Create, hardware: hardwareOfV1beta2(nil, num(4)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, - {name: "an explicit budget above the cluster size is kept", op: admissionv1.Create, hardware: hardwareOfV1beta2(nil, num(64)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, - {name: "an explicit SKU and budget bypass discovery", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h200_sxm"), num(64)), discoveryErr: errors.New("down")}, { - name: "an unknown explicit SKU and budget are authoritative", op: admissionv1.Create, - hardware: hardwareOfV1beta2(str("future_accelerator"), num(4)), discoveryErr: errors.New("down"), + name: "an unknown explicit SKU and budget require an explicit node width", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("future_accelerator"), num(4)), + wantErr: `spec.hardware.numGpusPerNode: Required value: cannot discover a node width for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.numGpusPerNode explicitly`, }, { - name: "an unknown SKU alone requires an explicit budget", op: admissionv1.Create, + name: "fully explicit unknown hardware is authoritative", op: admissionv1.Create, + hardware: hardwareOfV1beta2(str("future_accelerator"), num(4), num(2)), discoveryErr: errors.New("down"), + }, + { + name: "an unknown SKU alone requires explicit capacity and topology", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("future_accelerator"), nil), found: h200x8, - wantErr: `spec.hardware.gpu.budget: Required value: cannot discover a GPU budget for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget explicitly`, + wantErr: `spec.hardware.gpu.budget: Required value: cannot discover GPU capacity or topology for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget and spec.hardware.numGpusPerNode explicitly`, }, {name: "a failed discovery rejects", op: admissionv1.Create, discoveryErr: errors.New("no GPUs found"), wantCalls: 1, wantErr: "spec.hardware.gpu: Required value: the cluster's GPUs could not be discovered (no GPUs found)"}, @@ -146,20 +180,21 @@ func TestResolveHardwareV1beta2(t *testing.T) { { name: "a pinned SKU is not ambiguous in a heterogeneous cluster", op: admissionv1.Create, hardware: hardwareOfV1beta2(str("h200_sxm"), nil), found: h200x8, - wantBudget: 8, wantCalls: 1, wantFilter: "h200_sxm", + wantBudget: 8, wantGPUsPerNode: 8, wantCalls: 1, wantFilter: "h200_sxm", }, { name: "no GPUs at all reject the missing budget", op: admissionv1.Create, found: gpu.GPUInfo{System: "h200_sxm"}, wantCalls: 1, wantErr: `spec.hardware.gpu.budget: Required value: no GPUs of SKU "h200_sxm" were found in the cluster`, }, - {name: "a configured timeout bounds discovery", op: admissionv1.Create, found: h200x8, timeout: time.Hour, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1, wantMinTimeout: time.Minute, wantMaxTimeout: time.Hour}, + {name: "a configured timeout bounds discovery", op: admissionv1.Create, found: h200x8, timeout: time.Hour, wantSKU: "h200_sxm", wantBudget: 8, wantGPUsPerNode: 8, wantCalls: 1, wantMinTimeout: time.Minute, wantMaxTimeout: time.Hour}, {name: "an unchanged incomplete update does not discover", op: admissionv1.Update, oldHardware: hardwareOfV1beta2(str("h100_sxm"), nil), hardware: hardwareOfV1beta2(str("h100_sxm"), nil), discoveryErr: errors.New("down")}, {name: "an update without a stored spec does not discover", op: admissionv1.Update, found: h200x8}, - {name: "removing only the budget discovers only the budget", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("h100_sxm"), nil), found: h200x8, wantBudget: 8, wantCalls: 1, wantFilter: "h100_sxm"}, - {name: "removing only the SKU discovers only the SKU", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(nil, num(4)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, - {name: "removing all hardware discovers both fields", op: admissionv1.Update, oldHardware: completeH100, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantCalls: 1}, - {name: "changing complete hardware keeps the explicit override", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("b200_sxm"), num(2)), discoveryErr: errors.New("down")}, + {name: "removing only the budget discovers the budget", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("h100_sxm"), nil, num(8)), found: h200x8, wantBudget: 8, wantCalls: 1, wantFilter: "h100_sxm"}, + {name: "removing only the SKU discovers the SKU", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(nil, num(4), num(8)), found: h200x8, wantSKU: "h200_sxm", wantCalls: 1}, + {name: "removing all hardware discovers all fields", op: admissionv1.Update, oldHardware: completeH100, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantGPUsPerNode: 8, wantCalls: 1}, + {name: "replacing hardware with an empty object opts out", op: admissionv1.Update, oldHardware: completeH100, hardware: &nvidiacomv1beta2.HardwareSpec{}, discoveryErr: errors.New("discovery must not run")}, + {name: "changing complete hardware keeps the explicit override", op: admissionv1.Update, oldHardware: completeH100, hardware: hardwareOfV1beta2(str("b200_sxm"), num(2), num(8)), discoveryErr: errors.New("down")}, {name: "delete never discovers", op: admissionv1.Delete, oldHardware: completeH100, found: h200x8}, {name: "connect never discovers", op: admissionv1.Connect, oldHardware: completeH100, found: h200x8}, {name: "an unknown operation never discovers", oldHardware: completeH100, found: h200x8}, @@ -187,8 +222,8 @@ func TestResolveHardwareV1beta2(t *testing.T) { } else if err != nil { t.Fatalf("unexpected error: %v", err) } - if err == nil && (got.sku != tt.wantSKU || got.budget != tt.wantBudget) { - t.Fatalf("defaults = %+v, want sku %q budget %d", got, tt.wantSKU, tt.wantBudget) + if err == nil && (got.sku != tt.wantSKU || got.budget != tt.wantBudget || got.numGPUsPerNode != tt.wantGPUsPerNode) { + t.Fatalf("defaults = %+v, want sku %q budget %d numGpusPerNode %d", got, tt.wantSKU, tt.wantBudget, tt.wantGPUsPerNode) } if probe.calls != tt.wantCalls { t.Fatalf("discovery calls = %d, want %d", probe.calls, tt.wantCalls) @@ -205,3 +240,35 @@ func TestResolveHardwareV1beta2(t *testing.T) { }) } } + +func TestDefaultHardwareSpecV1beta2(t *testing.T) { + t.Run("preserves the explicit empty-object opt-out", func(t *testing.T) { + hardware := &nvidiacomv1beta2.HardwareSpec{} + defaultHardwareSpecV1beta2(hardware, hardwareDefaultsV1beta2{}) + if hardware.GPU != nil || hardware.NumGPUsPerNode != nil { + t.Fatalf("hardware = %#v, want the empty-object opt-out unchanged", hardware) + } + }) + + t.Run("persists every discovered field", func(t *testing.T) { + hardware := &nvidiacomv1beta2.HardwareSpec{} + defaultHardwareSpecV1beta2(hardware, hardwareDefaultsV1beta2{ + sku: "h200_sxm", budget: 8, numGPUsPerNode: 8, + }) + if hardware.GPU == nil || hardware.GPU.SKU == nil || *hardware.GPU.SKU != "h200_sxm" || + hardware.GPU.Budget == nil || *hardware.GPU.Budget != 8 || + hardware.NumGPUsPerNode == nil || *hardware.NumGPUsPerNode != 8 { + t.Fatalf("hardware = %#v, want every discovered field", hardware) + } + }) + + t.Run("preserves an explicit node width", func(t *testing.T) { + hardware := &nvidiacomv1beta2.HardwareSpec{NumGPUsPerNode: new(int32(4))} + defaultHardwareSpecV1beta2(hardware, hardwareDefaultsV1beta2{ + sku: "h200_sxm", budget: 8, numGPUsPerNode: 8, + }) + if hardware.NumGPUsPerNode == nil || *hardware.NumGPUsPerNode != 4 { + t.Fatalf("numGpusPerNode = %v, want explicit value 4", hardware.NumGPUsPerNode) + } + }) +} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 0fec7422f8a9..9c353f6c0588 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -64,10 +64,12 @@ type DGDRV1beta2Options struct { MaxHardwareBudget int32 } +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch + // resolveDefaults reads the environment for the values a v1beta2 spec omits. It does not -// mutate spec. It fails when the typed hardware cannot be completed: the error is the -// admission verdict, so a Request is never stored with hardware that depends on AISimulate's -// implicit defaults or on a cluster that could not be read. +// mutate spec. It fails when a nonempty typed hardware object cannot be completed: the error +// is the admission verdict, so a Request is stored with either exactly empty hardware or all +// three typed hardware values. func (o DGDRV1beta2Options) resolveDefaults( ctx context.Context, operation admissionv1.Operation, @@ -80,10 +82,13 @@ func (o DGDRV1beta2Options) resolveDefaults( return dynamoGraphDeploymentRequestV1beta2DefaultOptions{hardware: hardware}, nil } -// resolveHardware returns the SKU and budget the typed spec omits. A complete explicit pair is -// authoritative and bypasses discovery. When only a known SKU is present, discovery is restricted -// to it. An SKU outside the bundled AISimulate catalog remains an authoritative override only -// when its budget is explicit, because discovery cannot associate cluster capacity with it. +// resolveHardware returns the SKU, budget and node width that cluster discovery can supply. +// An exactly empty hardware object opts out of all operator discovery and typed hardware +// defaults. Omitted hardware discovers all three values. Every nonempty partial object discovers +// only its missing values; an explicit node width is always preserved by structural defaulting. +// When a known SKU is present, discovery is restricted to it. A SKU outside the bundled +// AISimulate catalog must supply both budget and node width because discovery cannot associate +// cluster capacity or topology with it. // // Discovery runs on CREATE and on an UPDATE that changes spec.hardware. A Request stays // mutable and every Run snapshots it, so a caller may change or remove typed hardware before @@ -93,61 +98,140 @@ func (o DGDRV1beta2Options) resolveDefaults( // the writer's back. oldSpec is the stored spec on UPDATE and nil otherwise. The decision reads // the typed hardware alone, never spec.aisimulate.inputOverrides. // -// After a successful return the typed hardware is complete. An omitted SKU becomes the -// cluster's AISimulate SKU, and an omitted budget becomes min(matching GPUs, cap). Explicit -// values are authoritative: they are neither rewritten nor capped. What the cluster cannot -// supply is rejected on the omitted field that fixes it. +// After discovery, an omitted SKU becomes the cluster's AISimulate SKU, an omitted budget +// becomes min(matching GPUs, cap), and an omitted node width becomes the selected node shape. +// Explicit values are authoritative: they are neither rewritten nor capped. What the cluster +// cannot supply is rejected on the omitted SKU, budget or node width that fixes it. func (o DGDRV1beta2Options) resolveHardware( ctx context.Context, operation admissionv1.Operation, oldSpec, spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec, ) (hardwareDefaultsV1beta2, error) { + // Presence distinguishes the explicit empty-object opt-out from omitted hardware. + if isEmptyHardwareSpecV1beta2(spec.Hardware) { + return hardwareDefaultsV1beta2{}, nil + } + changed := operation == admissionv1.Update && oldSpec != nil && !apiequality.Semantic.DeepEqual(oldSpec.Hardware, spec.Hardware) if o.Hardware == nil || (operation != admissionv1.Create && !changed) { return hardwareDefaultsV1beta2{}, nil } - var sku string - var hasBudget bool - if spec.Hardware != nil && spec.Hardware.GPU != nil { - if spec.Hardware.GPU.SKU != nil { - sku = *spec.Hardware.GPU.SKU - } - hasBudget = spec.Hardware.GPU.Budget != nil + intent := hardwareIntentFromSpecV1beta2(spec.Hardware) + if intent.complete() { + return hardwareDefaultsV1beta2{}, nil } - if sku != "" && hasBudget { + // Preserve an explicitly invalid empty SKU for structural validation instead of replacing it. + if intent.hasSKU && intent.sku == "" { return hardwareDefaultsV1beta2{}, nil } - if sku != "" && !aisimulate.IsKnownHardwareSKU(sku) { - return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required( - field.NewPath("spec", "hardware", "gpu", "budget"), - fmt.Sprintf("cannot discover a GPU budget for SKU %q because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget explicitly", sku), - )) + if err := validateDiscoverableHardwareV1beta2(intent); err != nil { + return hardwareDefaultsV1beta2{}, err + } + + found, err := o.discoverHardwareV1beta2(ctx, intent.sku) + if err != nil { + return hardwareDefaultsV1beta2{}, hardwareDiscoveryErrorV1beta2(intent, err) } + return o.hardwareDefaultsFromDiscoveryV1beta2(intent, found) +} + +type hardwareIntentV1beta2 struct { + sku string + hasSKU bool + hasBudget bool + hasNumGPUsPerNode bool +} + +func hardwareIntentFromSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec) hardwareIntentV1beta2 { + var intent hardwareIntentV1beta2 + if hardware == nil { + return intent + } + intent.hasNumGPUsPerNode = hardware.NumGPUsPerNode != nil + if hardware.GPU == nil { + return intent + } + intent.hasBudget = hardware.GPU.Budget != nil + if hardware.GPU.SKU != nil { + intent.hasSKU = true + intent.sku = *hardware.GPU.SKU + } + return intent +} + +func (i hardwareIntentV1beta2) complete() bool { + return i.hasSKU && i.hasBudget && i.hasNumGPUsPerNode +} + +func validateDiscoverableHardwareV1beta2(intent hardwareIntentV1beta2) error { + if intent.sku == "" || aisimulate.IsKnownHardwareSKU(intent.sku) { + return nil + } + hardwarePath := field.NewPath("spec", "hardware") + if !intent.hasBudget { + detail := fmt.Sprintf("cannot discover a GPU budget for SKU %q because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget explicitly", intent.sku) + if !intent.hasNumGPUsPerNode { + detail = fmt.Sprintf("cannot discover GPU capacity or topology for SKU %q because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget and spec.hardware.numGpusPerNode explicitly", intent.sku) + } + return rejectHardwareV1beta2(field.Required(hardwarePath.Child("gpu", "budget"), detail)) + } + return rejectHardwareV1beta2(field.Required( + hardwarePath.Child("numGpusPerNode"), + fmt.Sprintf("cannot discover a node width for SKU %q because it is not known to the bundled AISimulate catalog; set spec.hardware.numGpusPerNode explicitly", intent.sku), + )) +} + +func (o DGDRV1beta2Options) discoverHardwareV1beta2( + ctx context.Context, + sku string, +) (*gpu.GPUInfo, error) { timeout := o.HardwareDiscoveryTimeout if timeout <= 0 { timeout = defaultHardwareDiscoveryTimeout } ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() + return o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) +} - gpuPath := field.NewPath("spec", "hardware", "gpu") - found, err := o.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(sku)) - if err != nil { - if sku != "" { - return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required( - gpuPath.Child("budget"), - fmt.Sprintf("could not be discovered for SKU %q (%v); set spec.hardware.gpu.budget", sku, err), - )) +func hardwareDiscoveryErrorV1beta2(intent hardwareIntentV1beta2, err error) error { + hardwarePath := field.NewPath("spec", "hardware") + gpuPath := hardwarePath.Child("gpu") + if intent.sku != "" { + missingPath := gpuPath.Child("budget") + missingField := "spec.hardware.gpu.budget" + if !intent.hasBudget && !intent.hasNumGPUsPerNode { + missingField = "spec.hardware.gpu.budget and spec.hardware.numGpusPerNode" + } + if intent.hasBudget { + missingPath = hardwarePath.Child("numGpusPerNode") + missingField = "spec.hardware.numGpusPerNode" } - return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath, fmt.Sprintf( - "the cluster's GPUs could not be discovered (%v); set spec.hardware.gpu.sku and spec.hardware.gpu.budget", err))) + return rejectHardwareV1beta2(field.Required( + missingPath, + fmt.Sprintf("could not be discovered for SKU %q (%v); set %s", intent.sku, err, missingField), + )) } + return rejectHardwareV1beta2(field.Required(gpuPath, fmt.Sprintf( + "the cluster's GPUs could not be discovered (%v); set spec.hardware.gpu.sku, spec.hardware.gpu.budget, and spec.hardware.numGpusPerNode", err))) +} + +func (o DGDRV1beta2Options) hardwareDefaultsFromDiscoveryV1beta2( + intent hardwareIntentV1beta2, + found *gpu.GPUInfo, +) (hardwareDefaultsV1beta2, error) { + hardwarePath := field.NewPath("spec", "hardware") + gpuPath := hardwarePath.Child("gpu") + sku := intent.sku // The cluster facts are mapped into the v1beta2 API here: the AISimulate catalog name of // the GPU model (empty when the catalog does not know it) and the selected node-shape capacity. var defaults hardwareDefaultsV1beta2 + if !intent.hasNumGPUsPerNode && found != nil && found.GPUsPerNode > 0 { + defaults.numGPUsPerNode = int32(found.GPUsPerNode) + } if sku == "" { if found != nil && len(found.OtherModels) > 0 { return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("sku"), fmt.Sprintf( @@ -165,12 +249,12 @@ func (o DGDRV1beta2Options) resolveHardware( defaults.sku = sku } - // Discovery is needed only for an omitted budget at this point. + // Complete the omitted budget from matching nodes before checking the selected node width. var gpus int if found != nil { gpus = found.SelectedShapeGPUs } - if !hasBudget { + if !intent.hasBudget { if gpus <= 0 { return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required(gpuPath.Child("budget"), fmt.Sprintf( "no GPUs of SKU %q were found in the cluster; set spec.hardware.gpu.budget", sku))) @@ -181,9 +265,19 @@ func (o DGDRV1beta2Options) resolveHardware( } defaults.budget = int32(min(gpus, int(maxBudget))) } + if !intent.hasNumGPUsPerNode && defaults.numGPUsPerNode == 0 { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Required( + hardwarePath.Child("numGpusPerNode"), + fmt.Sprintf("could not be discovered for SKU %q; set spec.hardware.numGpusPerNode", sku), + )) + } return defaults, nil } +func isEmptyHardwareSpecV1beta2(hardware *nvidiacomv1beta2.HardwareSpec) bool { + return hardware != nil && hardware.GPU == nil && hardware.NumGPUsPerNode == nil +} + // hardwareRejectionV1beta2 is hardware the cluster cannot complete, with the field that fixes it. type hardwareRejectionV1beta2 struct{ err *field.Error } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 5cb03fcb030b..a05d02763135 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -18,6 +18,7 @@ package validation_test import ( + "errors" "strings" "testing" @@ -546,6 +547,49 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, }, + { + name: "v1beta2 unknown hardware without capacity or topology recommends the manual override", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{"gpu": map[string]any{ + "sku": "future_accelerator", + }} + }), + gpuDiscovery: true, + wantWebhook: []string{ + `spec.hardware.gpu.budget: Required value: cannot discover GPU capacity or topology for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget and spec.hardware.numGpusPerNode explicitly`, + }, + }, + { + name: "v1beta2 unknown explicit hardware is passed through with a warning", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{ + "gpu": map[string]any{"sku": "future_accelerator", "budget": int64(4)}, + "numGpusPerNode": int64(2), + } + }), + gpuDiscovery: true, + discoveryErr: errors.New("discovery must not run for complete explicit hardware"), + wantWarnings: []string{ + `spec.hardware.gpu.sku: "future_accelerator" is not known to the AISimulate release bundled with this operator; the value is passed through unchanged and may work with an external AISimulate systems catalog`, + }, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "hardware", "gpu", "sku"); got != "future_accelerator" { + t.Fatalf("spec.hardware.gpu.sku = %q, want unchanged override", got) + } + }, + }, + { + name: "v1beta2 bundled hardware does not warn", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["hardware"] = map[string]any{ + "gpu": map[string]any{"sku": "h200_sxm", "budget": int64(4)}, + "numGpusPerNode": int64(8), + } + }), + gpuDiscovery: true, + discoveryErr: errors.New("discovery must not run for complete explicit hardware"), + }, + { name: "v1beta2 public input overrides may configure non-reserved dgdr run fields", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -782,11 +826,14 @@ func TestDynamoGraphDeploymentRunSpecIsImmutable(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - actual := runAdmissionTest(t, admissionTestCase{ + test := admissionTestCase{ object: tt.run, - oldObject: tt.oldRun, wantCELError: tt.wantCELErr, - }) + } + if tt.oldRun != nil { + test.oldObject = tt.oldRun + } + actual := runAdmissionTest(t, test) if tt.wantCELErr == "" && actual.GetGeneration() != 1 { t.Fatalf("generation = %d, want 1 for a metadata-only update", actual.GetGeneration()) } @@ -1053,8 +1100,9 @@ func setV1beta2InputOverrides(spec map[string]any, overrides map[string]any) { // setBeta2AdmissionHardware names the hardware, so fixtures do not depend on GPU discovery. func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { - sku, budget := "h100_sxm", int32(4) + sku, budget, numGPUsPerNode := "h100_sxm", int32(4), int32(4) request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ - GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + NumGPUsPerNode: &numGPUsPerNode, } } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 7629066d9f35..24654c484f97 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3452,7 +3452,7 @@ _Appears in:_ | --- | --- | --- | --- | | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| -| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| +| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| @@ -3494,7 +3494,7 @@ _Appears in:_ | --- | --- | --- | --- | | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| -| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| +| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| @@ -3558,7 +3558,7 @@ _Appears in:_ | --- | --- | --- | --- | | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| -| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| +| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| @@ -3624,15 +3624,16 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `sku` _string_ | SKU identifies the accelerator using the AISimulate hardware catalog name.
Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`,
`a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`,
`h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`.
The field remains open so an external AISimulate systems catalog can add identifiers.
Admission passes an unknown identifier through unchanged and warns when `budget` is also
explicit. Without an explicit `budget`, it rejects the identifier because cluster discovery
cannot associate capacity with it. When omitted, admission discovers the accelerator. | | MinLength: 1
Optional: \{\}
| -| `budget` _integer_ | Budget is the maximum total number of GPUs one candidate may use.
When omitted for a known SKU, admission derives the available GPU budget from the cluster.
An SKU unknown to the bundled AISimulate catalog requires an explicit `budget`. | | Minimum: 1
Optional: \{\}
| +| `sku` _string_ | SKU identifies the accelerator using the AISimulate hardware catalog name.
Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`,
`a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`,
`h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`.
The field remains open so an external AISimulate systems catalog can add identifiers.
Admission passes an unknown identifier through unchanged and warns when `budget` and the
parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because
cluster discovery cannot associate capacity or topology with it. When omitted from a
nonempty hardware object, admission discovers the accelerator. | | MinLength: 1
Optional: \{\}
| +| `budget` _integer_ | Budget is the maximum total number of GPUs one candidate may use.
When omitted for a known SKU, admission derives the available GPU budget from the cluster.
An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an
explicit `numGpusPerNode` in the parent hardware object. | | Minimum: 1
Optional: \{\}
| #### v1beta2 HardwareSpec -HardwareSpec bounds the accelerator configurations evaluated by the search. +HardwareSpec bounds the accelerator configurations evaluated by the search. An exactly empty +object opts out of operator cluster discovery and typed hardware defaults. @@ -3644,7 +3645,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `gpu` _[GPUHardwareSpec](#gpuhardwarespec)_ | GPU describes accelerator identity and the maximum allocation per candidate.
Admission resolves omitted fields from the cluster when the SKU is known. | | Optional: \{\}
| +| `gpu` _[GPUHardwareSpec](#gpuhardwarespec)_ | GPU describes accelerator identity and the maximum allocation per candidate.
In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. | | Optional: \{\}
| +| `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers
to AISimulate's num_gpus_per_node input.
An explicit value is authoritative. In a nonempty hardware object, admission discovers
an omitted value from the selected cluster node shape. The Run controller never discovers
this value. An exactly empty hardware object opts out of all operator hardware discovery
and typed hardware defaults, leaving AISimulate input and defaults authoritative. | | Minimum: 1
Optional: \{\}
| #### KVCapacityFractionRange @@ -3718,7 +3720,7 @@ _Appears in:_ | --- | --- | --- | --- | | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| -| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
Admission discovers any omitted hardware values from the cluster. | | Optional: \{\}
| +| `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| | `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index 6005a7e0e477..530994cd155c 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -31,10 +31,18 @@ Dynamo release includes `a100_pcie`, `a100_sxm`, `a30`, `b200_sxm`, `b300_sxm`, Admission passes an explicit SKU through unchanged. It warns when the bundled catalog does not contain that identifier, but does not reject it: an external -AISimulate systems catalog may provide additional identifiers. Set both `sku` -and `budget` to override cluster hardware discovery completely. With only an -unknown explicit SKU, admission cannot associate discovered capacity with that -identifier and rejects the request with instructions to set `budget` explicitly. +AISimulate systems catalog may provide additional identifiers. + +Omit `hardware` to let admission discover and persist `sku`, `budget`, and +`numGpusPerNode`. An exactly empty `hardware: {}` object opts out of all operator +cluster and DCGM discovery and typed hardware defaults. Any nonempty object, +including `hardware: {gpu: {}}`, is partial hardware intent: admission discovers +and persists each missing field while preserving every explicit value. The Run +controller never performs hardware discovery. + +Set all three values to bypass discovery with fully explicit hardware. An unknown +SKU must provide both `budget` and `numGpusPerNode`, because admission cannot map +that identifier to cluster capacity or topology. ```yaml apiVersion: nvidia.com/v1beta2 @@ -47,7 +55,25 @@ spec: hardware: gpu: sku: future_accelerator # Passed to AISimulate unchanged; admission warns if unknown. - budget: 8 # Makes the explicit hardware independent of cluster discovery. + budget: 8 + numGpusPerNode: 8 # Completes explicit hardware; admission performs no discovery. +``` + +To leave the complete hardware input to AISimulate and `inputOverrides`, use the +empty-object opt-out and supply both the upstream hardware selector and candidate +GPU budget yourself. The operator does not validate this opaque input against the +AISimulate schema yet; invalid or incomplete overrides fail during the Run. + +```yaml +spec: + hardware: {} + aisimulate: + inputOverrides: + engine: + hardware: h200_sxm + optimization: + constraints: + max_candidate_gpus: 8 ``` ## Spec reference diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 7f50ce436007..6d449b70c09f 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4805,7 +4805,7 @@ See [Backend](#backend). -Hardware bounds the accelerator configurations evaluated by the search. Admission discovers any omitted hardware values from the cluster. +Hardware bounds the accelerator configurations evaluated by the search. When omitted, admission discovers SKU, budget and node width. An exactly empty object opts out of operator hardware discovery and typed defaults. Admission completes every nonempty object with any missing hardware values; explicit values remain authoritative. See [HardwareSpec](#v1beta2-hardwarespec). **Validation:** Optional: \{\} @@ -4887,7 +4887,7 @@ See [Backend](#backend). -Hardware bounds the accelerator configurations evaluated by the search. Admission discovers any omitted hardware values from the cluster. +Hardware bounds the accelerator configurations evaluated by the search. When omitted, admission discovers SKU, budget and node width. An exactly empty object opts out of operator hardware discovery and typed defaults. Admission completes every nonempty object with any missing hardware values; explicit values remain authoritative. See [HardwareSpec](#v1beta2-hardwarespec). **Validation:** Optional: \{\} @@ -5010,7 +5010,7 @@ See [Backend](#backend). -Hardware bounds the accelerator configurations evaluated by the search. Admission discovers any omitted hardware values from the cluster. +Hardware bounds the accelerator configurations evaluated by the search. When omitted, admission discovers SKU, budget and node width. An exactly empty object opts out of operator hardware discovery and typed defaults. Admission completes every nonempty object with any missing hardware values; explicit values remain authoritative. See [HardwareSpec](#v1beta2-hardwarespec). **Validation:** Optional: \{\} @@ -5146,28 +5146,34 @@ GPUHardwareSpec describes accelerator identity and the maximum allocation per ca **Appears in:** [HardwareSpec](#v1beta2-hardwarespec) -SKU identifies the accelerator using the AISimulate hardware catalog name. Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. The field remains open so an external AISimulate systems catalog can add identifiers. Admission passes an unknown identifier through unchanged and warns when `budget` is also explicit. Without an explicit `budget`, it rejects the identifier because cluster discovery cannot associate capacity with it. When omitted, admission discovers the accelerator. +SKU identifies the accelerator using the AISimulate hardware catalog name. Known values in the bundled AISimulate release are `a100_pcie`, `a100_sxm`, `a30`, `b200_sxm`, `b300_sxm`, `b60`, `gb200`, `gb300`, `h100_pcie`, `h100_sxm`, `h200_sxm`, `l4`, `l40s`, and `rtx_pro_6000_server`. The field remains open so an external AISimulate systems catalog can add identifiers. Admission passes an unknown identifier through unchanged and warns when `budget` and the parent `numGpusPerNode` are also explicit. Without both, it rejects the identifier because cluster discovery cannot associate capacity or topology with it. When omitted from a nonempty hardware object, admission discovers the accelerator. **Validation:** MinLength: 1 Optional: \{\} -Budget is the maximum total number of GPUs one candidate may use. When omitted for a known SKU, admission derives the available GPU budget from the cluster. An SKU unknown to the bundled AISimulate catalog requires an explicit `budget`. +Budget is the maximum total number of GPUs one candidate may use. When omitted for a known SKU, admission derives the available GPU budget from the cluster. An SKU unknown to the bundled AISimulate catalog requires an explicit `budget` and an explicit `numGpusPerNode` in the parent hardware object. **Validation:** Minimum: 1 Optional: \{\} -HardwareSpec bounds the accelerator configurations evaluated by the search. +HardwareSpec bounds the accelerator configurations evaluated by the search. An exactly empty +object opts out of operator cluster discovery and typed hardware defaults. **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -GPU describes accelerator identity and the maximum allocation per candidate. Admission resolves omitted fields from the cluster when the SKU is known. +GPU describes accelerator identity and the maximum allocation per candidate. In a nonempty hardware object, admission resolves an omitted SKU or budget from the cluster. See [GPUHardwareSpec](#gpuhardwarespec). **Validation:** Optional: \{\} + + +NumGPUsPerNode is the number of GPUs available to one candidate on each node. It lowers to AISimulate's num_gpus_per_node input. An explicit value is authoritative. In a nonempty hardware object, admission discovers an omitted value from the selected cluster node shape. The Run controller never discovers this value. An exactly empty hardware object opts out of all operator hardware discovery and typed hardware defaults, leaving AISimulate input and defaults authoritative. +**Validation:** Minimum: 1 Optional: \{\} + @@ -5241,7 +5247,7 @@ See [Backend](#backend). -Hardware bounds the accelerator configurations evaluated by the search. Admission discovers any omitted hardware values from the cluster. +Hardware bounds the accelerator configurations evaluated by the search. When omitted, admission discovers SKU, budget and node width. An exactly empty object opts out of operator hardware discovery and typed defaults. Admission completes every nonempty object with any missing hardware values; explicit values remain authoritative. See [HardwareSpec](#v1beta2-hardwarespec). **Validation:** Optional: \{\} From 093779d86489c4300269fb121dd3cea3157b0254 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 15:57:05 +0100 Subject: [PATCH 028/100] pr-16661: pr-3: verify hardware admission semantics [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- .../webhook/defaulting/suite_envtest_test.go | 82 +++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go index 4f0168dc4ef6..3001d912142d 100644 --- a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go @@ -368,13 +368,53 @@ func TestDynamoGraphDeploymentRequestV1beta2DefaultingAdmission(t *testing.T) { check: expectV1beta2ExplicitValues, }, { - name: "v1beta2 unknown SKU without budget reports the manual override", + name: "v1beta2 explicit node width wins while admission discovers a missing budget", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: new("h200_sxm")}, + NumGPUsPerNode: new(int32(4)), + } + }), + discovered: discoveredH200, + check: expectV1beta2H200Budget8Width4, + }, + { + name: "v1beta2 explicit empty hardware opts out of discovery and typed defaults", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{} + }), + discoveryErr: errors.New("discovery must not run"), + check: expectV1beta2EmptyHardwareOptOut, + }, + { + name: "v1beta2 explicit SKU and budget discover the missing node width", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: new("h200_sxm"), Budget: new(int32(4)), + }} + }), + discovered: discoveredH200, + check: expectV1beta2H200Budget4Width8, + }, + { + name: "v1beta2 fully explicit hardware bypasses discovery", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: new("h200_sxm"), Budget: new(int32(4))}, + NumGPUsPerNode: new(int32(4)), + } + }), + discoveryErr: errors.New("discovery must not run"), + check: expectV1beta2H200Budget4Width4, + }, + { + name: "v1beta2 unknown SKU without capacity or topology reports the manual override", request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ SKU: new("future_accelerator"), }} }), - wantCreateErr: `spec.hardware.gpu.budget: Required value: cannot discover a GPU budget for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget explicitly`, + wantCreateErr: `spec.hardware.gpu.budget: Required value: cannot discover GPU capacity or topology for SKU "future_accelerator" because it is not known to the bundled AISimulate catalog; set spec.hardware.gpu.budget and spec.hardware.numGpusPerNode explicitly`, }, { name: "v1beta2 metadata-only update is idempotent and does not rediscover hardware", @@ -389,7 +429,7 @@ func TestDynamoGraphDeploymentRequestV1beta2DefaultingAdmission(t *testing.T) { checkUpdate: expectV1beta2MetadataOnlyUpdate, }, { - name: "v1beta2 removing hardware on update rediscovers both fields", + name: "v1beta2 removing hardware on update rediscovers all hardware fields", request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ SKU: new("h200_sxm"), Budget: new(int32(4)), @@ -549,8 +589,40 @@ func expectV1beta2HardwareUpdate(t *testing.T, request *nvidiacomv1beta2.DynamoG func expectV1beta2H200x8Hardware(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { t.Helper() hardware := request.Spec.Hardware - if hardware == nil || hardware.GPU == nil || ptr.Deref(hardware.GPU.SKU, "") != "h200_sxm" || ptr.Deref(hardware.GPU.Budget, 0) != 8 { - t.Fatalf("hardware = %#v, want SKU h200_sxm and budget 8", hardware) + if hardware == nil || hardware.GPU == nil || ptr.Deref(hardware.GPU.SKU, "") != "h200_sxm" || ptr.Deref(hardware.GPU.Budget, 0) != 8 || ptr.Deref(hardware.NumGPUsPerNode, 0) != 8 { + t.Fatalf("hardware = %#v, want SKU h200_sxm, budget 8 and numGpusPerNode 8", hardware) + } +} + +func expectV1beta2H200Budget8Width4(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + hardware := request.Spec.Hardware + if hardware == nil || hardware.GPU == nil || ptr.Deref(hardware.GPU.SKU, "") != "h200_sxm" || ptr.Deref(hardware.GPU.Budget, 0) != 8 || ptr.Deref(hardware.NumGPUsPerNode, 0) != 4 { + t.Fatalf("hardware = %#v, want SKU h200_sxm, budget 8 and explicit numGpusPerNode 4", hardware) + } +} + +func expectV1beta2EmptyHardwareOptOut(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + hardware := request.Spec.Hardware + if hardware == nil || hardware.GPU != nil || hardware.NumGPUsPerNode != nil { + t.Fatalf("hardware = %#v, want the explicit empty-object opt-out", hardware) + } +} + +func expectV1beta2H200Budget4Width8(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + hardware := request.Spec.Hardware + if hardware == nil || hardware.GPU == nil || ptr.Deref(hardware.GPU.SKU, "") != "h200_sxm" || ptr.Deref(hardware.GPU.Budget, 0) != 4 || ptr.Deref(hardware.NumGPUsPerNode, 0) != 8 { + t.Fatalf("hardware = %#v, want SKU h200_sxm, budget 4 and discovered numGpusPerNode 8", hardware) + } +} + +func expectV1beta2H200Budget4Width4(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + hardware := request.Spec.Hardware + if hardware == nil || hardware.GPU == nil || ptr.Deref(hardware.GPU.SKU, "") != "h200_sxm" || ptr.Deref(hardware.GPU.Budget, 0) != 4 || ptr.Deref(hardware.NumGPUsPerNode, 0) != 4 { + t.Fatalf("hardware = %#v, want fully explicit SKU h200_sxm, budget 4 and numGpusPerNode 4", hardware) } } From 48ba217d5345a62e5daefe7ae1b10eef513b590e Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 16:25:17 +0100 Subject: [PATCH 029/100] pr-16661: pr-3: bound discovered node width [upstream: devivasudevan/dynamo#3] Signed-off-by: Dr. Stefan Schimanski --- ...phdeploymentrequest_v1beta2_hardware_test.go | 17 +++++++++++++++-- ...amographdeploymentrequest_v1beta2_helpers.go | 8 ++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go index 0d612b5721eb..a111c6ca1536 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_hardware_test.go @@ -20,6 +20,8 @@ package defaulting import ( "context" "errors" + "math" + "strconv" "strings" "testing" "time" @@ -66,7 +68,7 @@ func TestResolveHardwareV1beta2(t *testing.T) { h200x8 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 1, TotalGPUs: 8, SelectedShapeGPUs: 8} h200x64 := gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: 8, NodesWithGPUs: 8, TotalGPUs: 64, SelectedShapeGPUs: 64} completeH100 := hardwareOfV1beta2(str("h100_sxm"), num(4), num(8)) - tests := []struct { + type testCase struct { name string op admissionv1.Operation oldHardware *nvidiacomv1beta2.HardwareSpec @@ -84,7 +86,8 @@ func TestResolveHardwareV1beta2(t *testing.T) { wantErr string wantMinTimeout time.Duration wantMaxTimeout time.Duration - }{ + } + tests := []testCase{ {name: "no discoverer leaves omitted hardware unresolved", op: admissionv1.Create, withoutDiscoverer: true}, {name: "omitted hardware discovers the SKU budget and node width", op: admissionv1.Create, found: h200x8, wantSKU: "h200_sxm", wantBudget: 8, wantGPUsPerNode: 8, wantCalls: 1, wantMinTimeout: time.Millisecond, wantMaxTimeout: defaultHardwareDiscoveryTimeout}, {name: "an empty hardware object opts out of discovery", op: admissionv1.Create, hardware: &nvidiacomv1beta2.HardwareSpec{}, discoveryErr: errors.New("discovery must not run")}, @@ -199,6 +202,16 @@ func TestResolveHardwareV1beta2(t *testing.T) { {name: "connect never discovers", op: admissionv1.Connect, oldHardware: completeH100, found: h200x8}, {name: "an unknown operation never discovers", oldHardware: completeH100, found: h200x8}, } + if strconv.IntSize > 32 { + tooManyGPUs := int64(math.MaxInt32) + 1 + tests = append(tests, testCase{ + name: "a discovered node width must fit the API integer", + op: admissionv1.Create, + found: gpu.GPUInfo{System: "h200_sxm", GPUsPerNode: int(tooManyGPUs), SelectedShapeGPUs: 8}, + wantCalls: 1, + wantErr: "spec.hardware.numGpusPerNode: Invalid value: 2147483648: discovered node width exceeds the API maximum of 2147483647", + }) + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { probe := &fakeDiscoverer{found: tt.found, err: tt.discoveryErr} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go index 9c353f6c0588..0530200f6104 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2_helpers.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "math" "strings" "time" @@ -230,6 +231,13 @@ func (o DGDRV1beta2Options) hardwareDefaultsFromDiscoveryV1beta2( // the GPU model (empty when the catalog does not know it) and the selected node-shape capacity. var defaults hardwareDefaultsV1beta2 if !intent.hasNumGPUsPerNode && found != nil && found.GPUsPerNode > 0 { + if found.GPUsPerNode > math.MaxInt32 { + return hardwareDefaultsV1beta2{}, rejectHardwareV1beta2(field.Invalid( + hardwarePath.Child("numGpusPerNode"), + found.GPUsPerNode, + fmt.Sprintf("discovered node width exceeds the API maximum of %d", math.MaxInt32), + )) + } defaults.numGPUsPerNode = int32(found.GPUsPerNode) } if sku == "" { From bc6257ea71094ec20568eb289fdf12a6f6796c48 Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:34:20 +0100 Subject: [PATCH 030/100] pr-15895: squash DGDR live output adapter Squashed from https://github.com/ai-dynamo/dynamo/pull/15895. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- .github/codeowners/areas.yaml | 4 + CODEOWNERS | 1 + .../aisimulate/output/dgd/renderers/base.py | 4 +- .../aisimulate/output/dgdr_run/__init__.py | 34 + .../aisimulate/output/dgdr_run/adapter.py | 763 ++++++++++++ .../aisimulate/output/dgdr_run/snapshot.py | 148 +++ .../aisimulate/tests/dgdr_run/test_adapter.py | 1077 +++++++++++++++++ .../tests/dgdr_run/test_snapshot.py | 148 +++ pyproject.toml | 1 + 9 files changed, 2179 insertions(+), 1 deletion(-) create mode 100644 components/src/dynamo/aisimulate/output/dgdr_run/__init__.py create mode 100644 components/src/dynamo/aisimulate/output/dgdr_run/adapter.py create mode 100644 components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py create mode 100644 components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py create mode 100644 components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py diff --git a/.github/codeowners/areas.yaml b/.github/codeowners/areas.yaml index 1194e6730eec..46667349a22e 100644 --- a/.github/codeowners/areas.yaml +++ b/.github/codeowners/areas.yaml @@ -899,6 +899,10 @@ shared: owners: - planner - operator +- glob: components/src/dynamo/aisimulate/output/dgdr_run/ + owners: + - planner + - operator - glob: components/src/dynamo/planner/connectors/kubernetes.py owners: - planner diff --git a/CODEOWNERS b/CODEOWNERS index 50d852d90af4..548d8c27c151 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -789,6 +789,7 @@ requirements.txt /deploy/operator/internal/webhook/**/*checkpoint* @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners /.github/actions/build-deploy-component/action.yml @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners @ai-dynamo/dynamo-planner-codeowners /.github/actions/check-deploy-component/action.yml @ai-dynamo/dynamo-ops-codeowners @ai-dynamo/dynamo-operator-codeowners @ai-dynamo/dynamo-gms-codeowners +/components/src/dynamo/aisimulate/output/dgdr_run/ @ai-dynamo/dynamo-planner-codeowners @ai-dynamo/dynamo-operator-codeowners /components/src/dynamo/common/backend/publisher.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-kv-memory-codeowners /components/src/dynamo/common/backend/telemetry.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-observability-codeowners /components/src/dynamo/common/utils/image_utils.py @ai-dynamo/dynamo-runtime-codeowners @ai-dynamo/dynamo-diffusion-codeowners diff --git a/components/src/dynamo/aisimulate/output/dgd/renderers/base.py b/components/src/dynamo/aisimulate/output/dgd/renderers/base.py index 54154adab30c..b59981bb4ad4 100644 --- a/components/src/dynamo/aisimulate/output/dgd/renderers/base.py +++ b/components/src/dynamo/aisimulate/output/dgd/renderers/base.py @@ -42,7 +42,9 @@ def _runtime_version(runtime_image: str, override: str | None) -> str: class CandidateLike(Protocol): """Public Sweeper result fields used by DGD renderers.""" - config: dict[str, Any] + @property + def config(self) -> dict[str, Any]: + ... class CandidateMaterializationError(ValueError): diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py b/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py new file mode 100644 index 000000000000..290ce406d8f5 --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/__init__.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live DynamoGraphDeploymentRun output adapter for AISimulate (``dgdr_run``).""" + +from dynamo.aisimulate.output.dgdr_run.adapter import ( + DGDRRunOutputAdapter, + DGDRRunOutputConfig, + DGDRRunOutputPlugin, + candidate_id_for, + create_adapter, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) + +__all__ = [ + "SNAPSHOT_FILE_NAME", + "CandidateOutcome", + "DGDRRunOutputAdapter", + "DGDRRunOutputConfig", + "DGDRRunOutputPlugin", + "DGDRRunSnapshot", + "RunPhase", + "SnapshotCandidate", + "candidate_id_for", + "create_adapter", + "write_snapshot", +] diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py new file mode 100644 index 000000000000..eb310d5582ca --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -0,0 +1,763 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live DGDRRun output adapter (``--output dgdr_run``). + +Independent of the standalone ``dgd`` adapter, which is called once with the +final recommendation. This adapter subscribes to Sweeper's ``on_round`` and +``on_candidate`` callbacks for the whole search, keeps the *current bounded +projection* of the run in memory, and periodically publishes it as an +atomically replaced ``dgdr_run_snapshot.yaml`` (see ``snapshot.py``) that the +publisher sidecar in the same pod consumes. + +Both adapters share one Candidate-to-DGD implementation through +``dynamo.aisimulate.output.dgd.renderers`` (its public +CandidateMaterializationError / DGDGenerationOptions / render_dgd surface); +nothing else is shared. + +Design points (all adapter-specific, none of them shared with ``dgd``): + +* **Callbacks never block and never lose state.** They only update the latest + in-memory projection under a short lock and wake the writer. Nothing is + queued, so nothing can be dropped under pressure: only *file writes* are + coalesced, never logical state changes. +* **Bounded.** At most ``max_candidates`` points are retained and published, + no matter how many are evaluated. The bound comes from the run's + ``recommendation.maxCandidates``; it is not derived from any transport. +* **Rate limited.** The writer publishes the latest complete state at most once + per ``snapshot_interval_seconds``. The initial snapshot (``start()``) and the + terminal snapshot (``close()``) bypass the limit. +* **Materialize once.** A point's DGD is rendered the first time it enters the + projection and cached by its stable id, so progress or ordering changes never + re-render retained points. +* **Terminal snapshot is a completion barrier.** ``close()`` stops accepting + updates, folds in everything accepted so far, and synchronously writes the + terminal snapshot before returning. A failure to write it is raised, never + swallowed, so the Sweeper container cannot exit successfully without it. + +Scalar searches only for now (best-first ordering by score); a Pareto +selection policy is a separate piece of work and is rejected explicitly. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import logging +import math +import threading +import time +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, fields +from pathlib import Path +from typing import Any, Literal, NoReturn, Protocol + +import yaml +from aisimulate.output_adapter import ( + OUTPUT_ADAPTER_API_VERSION, + RecommendationOutputCallbacks, +) +from aisimulate.sweeper.config import SmartSearchConfig +from aisimulate.sweeper.result import SweepResult +from pydantic import BaseModel, ConfigDict, Field + +from dynamo.aisimulate.output.dgd.renderers import ( + CandidateMaterializationError, + DGDGenerationOptions, + render_dgd, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) + +logger = logging.getLogger(__name__) + +_ID_PREFIX = "evaluated-point-" +_MAX_RENDER_ATTEMPTS = 3 +_MAX_ERROR_MESSAGE = 500 +_WRITE_RETRY_BACKOFF = 1.0 +_ID_HEX_LENGTH = 12 +# Metric names the adapter derives itself. They are reserved: a measurement AISimulate resolves under +# one of these names is replaced by the adapter's value (and logged), never merged with it. +_RESERVED_METRICS = ("score", "usedGpus", "objectives") + + +class CandidateRecordLike(Protocol): + """The subset of aisimulate's ``CandidateRecord`` this adapter reads (read-only).""" + + @property + def candidate_id(self) -> str: + ... + + @property + def status(self) -> Any: + ... # aisimulate.sweeper.result.CandidateStatus + + @property + def score(self) -> float | None: + ... + + @property + def used_gpus(self) -> int | None: + ... + + @property + def config(self) -> Mapping[str, Any]: + ... + + @property + def metrics(self) -> Mapping[str, float | None]: + ... + + +def _plain(value: Any) -> Any: + """Copy ``value`` into plain, YAML-safe containers (detached from the caller).""" + return json.loads(json.dumps(value, default=str, sort_keys=True)) + + +def _bounded(text: str) -> str: + return text[:_MAX_ERROR_MESSAGE] + + +def _finite(value: Any) -> Any: + """Replace non-finite floats with ``None`` so the snapshot stays valid JSON/YAML.""" + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, dict): + return {key: _finite(item) for key, item in value.items()} + if isinstance(value, list): + return [_finite(item) for item in value] + return value + + +class _RendererUnavailable(Exception): + """Rendering failed for a reason that is not specific to one candidate.""" + + +def candidate_id_for(parameters: Mapping[str, Any]) -> str: + """Stable identity of an evaluated point. + + Depends only on the point's resolved parameters: never on rank, list + position, or the order in which points were observed. + """ + canonical = json.dumps(parameters, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return f"{_ID_PREFIX}{digest[:_ID_HEX_LENGTH]}" + + +def _is_feasible(record: CandidateRecordLike) -> bool: + # By value, not identity: callers may pass aisimulate's CandidateStatus + # enum, a plain string, or a test double. + return str(getattr(record.status, "value", record.status)) == "feasible" + + +@dataclass(frozen=True) +class _Retained: + """One retained point: immutable evaluation facts, no rendering.""" + + id: str + score: float + used_gpus: int + parameters: dict[str, Any] + metrics: dict[str, Any] + + @property + def rank_key(self) -> tuple[float, int, str]: + """Ascending sort key: best first. + + Mirrors Sweeper's scalar ``rank()``: highest score, then fewer GPUs, + then the canonical config JSON. It is only used to bound the live + retained set; the final snapshot keeps Sweeper's own selection order. + """ + return ( + -self.score, + self.used_gpus, + json.dumps(self.parameters, sort_keys=True, separators=(",", ":")), + ) + + +@dataclass(frozen=True) +class _Terminal: + phase: RunPhase + message: str + error: str + + +class DGDRRunOutputConfig(BaseModel): + """Configuration accepted for ``--output dgdr_run``.""" + + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1, pattern=r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") + snapshot_dir: str | None = Field(default=None, min_length=1) + namespace: str | None = None + generator: Literal["aic", "direct"] = "aic" + runtime_image: str = Field(min_length=1) + runtime_version_override: str | None = None + num_gpus_per_node: int = Field(gt=0) + # The already-defaulted spec.recommendation.maxCandidates (API range 1..100). The adapter owns + # no default of its own: a missing value must fail, not silently change the cardinality. + max_candidates: int = Field(ge=1, le=100) + snapshot_interval_seconds: float = Field(default=5.0, ge=0) + + def generation_options(self) -> DGDGenerationOptions: + return DGDGenerationOptions( + runtime_image=self.runtime_image, + runtime_version_override=self.runtime_version_override, + namespace=self.namespace, + num_gpus_per_node=self.num_gpus_per_node, + ) + + +class DGDRRunOutputAdapter: + """Publishes the current bounded projection of one Sweeper run. + + One instance per run. Pass the bound methods to ``Sweeper.run`` and use the + adapter as a context manager (or call ``start()``/``close()``) so the + terminal snapshot is always written, including when the run raises:: + + with DGDRRunOutputAdapter(config, workload, snapshot_dir=path) as adapter: + sweeper.run(on_round=adapter.on_round, on_candidate=adapter.on_candidate) + """ + + def __init__( + self, + config: DGDRRunOutputConfig, + workload: Any, + *, + snapshot_dir: Path | None = None, + is_pareto: bool = False, + keep_arrival_order: bool = False, + ) -> None: + if is_pareto: + raise NotImplementedError( + "dgdr_run only supports scalar objectives so far; Pareto " + "retention needs its own selection policy rather than an " + "approximation of top-N-by-score" + ) + directory = snapshot_dir if snapshot_dir is not None else config.snapshot_dir + if directory is None: + raise ValueError("a snapshot directory is required") + self._directory = Path(directory) + self._config = config + self._options = config.generation_options() + self._workload = workload + self._max_candidates = config.max_candidates + self._interval = config.snapshot_interval_seconds + + # Latest in-memory desired state. Callbacks mutate it under _lock. + self._lock = threading.Lock() + self._retained: dict[str, _Retained] = {} + self._round_no = 0 + self._evaluated = 0 + self._terminal: _Terminal | None = None + self._accepting = True + + # Writer coordination. + self._wake = threading.Event() + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._next_allowed = float("-inf") + + # Materialization cache, keyed by stable id. Touched only while + # holding _publish_lock. A failure is retried on later writes up to + # _MAX_RENDER_ATTEMPTS in total, because a first render can fail + # transiently (cold start); after that it is final. + # The final result is already ranked by Sweeper (goal and SLA aware); + # publishing it must not re-rank it. + self._keep_arrival_order = keep_arrival_order + self._publish_lock = threading.Lock() + self._materialized: dict[str, SnapshotCandidate] = {} + self._render_attempts: dict[str, int] = {} + self._writes = 0 + + # -- observability --------------------------------------------------- + + @property + def snapshot_writes(self) -> int: + """Number of snapshots written so far (initial and terminal included).""" + return self._writes + + @property + def snapshot_path(self) -> Path: + return self._directory / SNAPSHOT_FILE_NAME + + # -- Sweeper callbacks: update latest state, never block, never raise -- + + def on_candidate(self, record: CandidateRecordLike) -> None: + try: + retained = self._retain(record) if _is_feasible(record) else None + except Exception: # a callback must not break the search + logger.exception("dgdr_run: ignoring candidate that could not be read") + retained = None + with self._lock: + if not self._accepting: + return + self._evaluated += 1 + # First observation wins: an evaluated point is immutable, so a + # re-evaluation of the same point never changes what was published. + if retained is not None and retained.id not in self._retained: + self._retained[retained.id] = retained + if len(self._retained) > self._max_candidates: + worst = max(self._retained.values(), key=lambda r: r.rank_key) + del self._retained[worst.id] + self._wake.set() + + def on_round(self, round_number: int, candidates: list[Any]) -> None: + del candidates # cumulative feasible list: the adapter tracks its own set + with self._lock: + if not self._accepting: + return + self._round_no = max(self._round_no, round_number) + self._wake.set() + + def report_progress(self, *, evaluated: int, round_no: int = 0) -> None: + """Raise the evaluated count and round to authoritative values (never lowers them).""" + with self._lock: + if not self._accepting: + return + self._evaluated = max(self._evaluated, evaluated) + self._round_no = max(self._round_no, round_no) + self._wake.set() + + @staticmethod + def _retain(record: CandidateRecordLike) -> _Retained | None: + if record.score is None or record.used_gpus is None: + logger.warning( + "dgdr_run: feasible candidate %s has no score/used_gpus; skipped", + record.candidate_id, + ) + return None + parameters = _plain(dict(record.config)) + score = float(record.score) + # Every measurement AISimulate resolved for the point (throughput, TTFT, ITL, E2E latency, + # goodput, power, ...) is kept under its own name, with non-finite values as null. The + # adapter's rank metadata is added under the reserved names. + metrics: dict[str, Any] = _finite( + _plain(dict(getattr(record, "metrics", None) or {})) + ) + for reserved in _RESERVED_METRICS: + if reserved in metrics: + logger.warning( + "dgdr_run: candidate %s reports a metric named %r, which the adapter reserves; replaced", + record.candidate_id, + reserved, + ) + del metrics[reserved] + metrics["score"] = score if math.isfinite(score) else None + metrics["usedGpus"] = int(record.used_gpus) + objectives = getattr(record, "objectives", None) + if objectives: + metrics["objectives"] = _finite(_plain(dict(objectives))) + return _Retained( + id=candidate_id_for(parameters), + # A NaN score has no place in an ordering; it ranks as the worst. + score=-math.inf if math.isnan(score) else score, + used_gpus=int(record.used_gpus), + parameters=parameters, + metrics=metrics, + ) + + # -- lifecycle --------------------------------------------------------- + + def start(self) -> None: + """Write the initial snapshot immediately, then start the writer.""" + if self._thread is not None: + raise RuntimeError("DGDRRunOutputAdapter already started") + self._publish() + self._next_allowed = time.monotonic() + self._interval + self._thread = threading.Thread( + target=self._writer_loop, name="dgdr-run-snapshot-writer", daemon=True + ) + self._thread.start() + + def close( + self, + *, + phase: RunPhase = RunPhase.SUCCEEDED, + message: str = "", + error: str = "", + ) -> None: + """Write the terminal snapshot; returns only once it is on disk. + + Stops accepting updates, folds in every update accepted before this + call, bypasses the rate limit, and replaces the file synchronously. + Raises if the terminal snapshot cannot be written. Safe to call without + ``start()`` and safe to call twice (the second call is a no-op). + """ + if not phase.terminal: + raise ValueError("close() requires a terminal phase") + with self._lock: + if self._terminal is not None: + return + self._accepting = False + self._terminal = _Terminal(phase=phase, message=message, error=error) + self._stop.set() + self._wake.set() + if self._thread is not None: + self._thread.join() + self._publish_or_record_failure() + + def _publish_or_record_failure(self) -> None: + """Publish; if rendering itself is broken, record that as a terminal failure. + + A broken renderer is a run-level problem, not a candidate-specific one, so it + must not become ``MATERIALIZATION_FAILED``. The terminal ``Failed`` snapshot + carries the candidates that were already rendered, and the original exception + is re-raised. Failures to write the file keep propagating unchanged. + """ + try: + self._publish() + except _RendererUnavailable as unavailable: + self._fail_renderer(unavailable) + + def _fail_renderer(self, unavailable: _RendererUnavailable) -> NoReturn: + cause = unavailable.__cause__ or unavailable + logger.error("dgdr_run: renderer unavailable", exc_info=cause) + with self._lock: + self._accepting = False + self._terminal = _Terminal( + phase=RunPhase.FAILED, + message=_bounded(str(cause)), + error=type(cause).__name__, + ) + self._publish(render=False) + raise cause + + def _materialize_retained(self) -> list[SnapshotCandidate]: + with self._publish_lock: + with self._lock: + items = list(self._retained.values()) + return [self._materialize(item) for item in items] + + def close_selection(self) -> None: + """Write the terminal snapshot of a finished search that selected candidates. + + ``Succeeded`` means at least one selected candidate has a manifest. Candidates are + materialized (with the same bounded retries as the live path) before the phase is + chosen; when none materialized the run is ``Failed`` with a bounded aggregate reason, + and the same error is raised so the recommendation does not exit successfully. + """ + entries: list[SnapshotCandidate] = [] + for attempt in range(_MAX_RENDER_ATTEMPTS): + try: + entries = self._materialize_retained() + except _RendererUnavailable as unavailable: + self._fail_renderer(unavailable) + if all(entry.outcome is CandidateOutcome.MATERIALIZED for entry in entries): + break + if attempt + 1 < _MAX_RENDER_ATTEMPTS: + time.sleep(_WRITE_RETRY_BACKOFF) + if any(entry.outcome is CandidateOutcome.MATERIALIZED for entry in entries): + self.close(phase=RunPhase.SUCCEEDED) + return + first = next((entry.error for entry in entries if entry.error), "") + message = _bounded( + f"no selected candidate could be materialized ({len(entries)} failed): {first}" + ) + self.close( + phase=RunPhase.FAILED, + message=message, + error="CandidateMaterializationError", + ) + raise CandidateMaterializationError(message) + + def finish_search(self) -> None: + """The search ended normally: flush the last live state and stop the writer. + + Not a terminal state by itself: the terminal ``Succeeded`` snapshot is written + afterwards by ``write()``, which sees the final result. A search that found no + feasible candidate never reaches ``write()``, so that case is recorded here as + ``Failed`` instead of being left looking in progress. + """ + with self._lock: + if self._terminal is not None: + return + nothing_found = not self._retained + if nothing_found: + self.close(phase=RunPhase.FAILED, message="no feasible candidate found") + return + self._stop.set() + self._wake.set() + if self._thread is not None: + self._thread.join() + self._publish_or_record_failure() + + def fail(self, error: BaseException) -> None: + """The search raised: write the terminal ``Failed`` snapshot.""" + self.close( + phase=RunPhase.FAILED, + error=type(error).__name__, + message=_bounded(str(error)), + ) + + def __enter__(self) -> DGDRRunOutputAdapter: + self.start() + return self + + def __exit__(self, exc_type: type[BaseException] | None, *_: Any) -> None: + if exc_type is None: + self.close(phase=RunPhase.SUCCEEDED) + return + try: + self.close(phase=RunPhase.FAILED, error=exc_type.__name__) + except Exception: # never mask the run's own exception + logger.exception("dgdr_run: could not write the terminal snapshot") + + # -- writer ---------------------------------------------------------- + + def _writer_loop(self) -> None: + while True: + self._wake.wait() + if self._stop.is_set(): + return + delay = self._next_allowed - time.monotonic() + if delay > 0 and self._stop.wait(delay): + return + wait = self._interval + try: + self._publish() + except Exception: # keep the writer alive; retry + logger.exception("dgdr_run: snapshot write failed; will retry") + self._wake.set() + # Even with a zero interval, a persistent failure must not spin. + wait = max(wait, _WRITE_RETRY_BACKOFF) + self._next_allowed = time.monotonic() + wait + + def _publish(self, *, render: bool = True) -> None: + with self._publish_lock: + with self._lock: + # Cleared together with the copy so no update is ever lost: an + # update after this point sets the event again. + self._wake.clear() + ordered = ( + list(self._retained.values()) + if self._keep_arrival_order + else sorted(self._retained.values(), key=lambda r: r.rank_key) + ) + round_no = self._round_no + evaluated = self._evaluated + terminal = self._terminal + if render: + candidates = tuple(self._materialize(item) for item in ordered) + else: + candidates = tuple( + self._materialized[item.id] + for item in ordered + if item.id in self._materialized + ) + live_ids = {item.id for item in ordered} + for stale in set(self._materialized) - live_ids: + del self._materialized[stale] + self._render_attempts.pop(stale, None) + write_snapshot( + self._directory, + DGDRRunSnapshot( + phase=terminal.phase if terminal else RunPhase.RUNNING, + round_no=round_no, + evaluated=evaluated, + candidates=candidates, + message=terminal.message if terminal else "", + error=terminal.error if terminal else "", + ), + ) + self._writes += 1 + + def _materialize(self, item: _Retained) -> SnapshotCandidate: + cached = self._materialized.get(item.id) + if cached is not None and ( + cached.outcome is CandidateOutcome.MATERIALIZED + or self._render_attempts.get(item.id, 0) >= _MAX_RENDER_ATTEMPTS + ): + return cached + self._render_attempts[item.id] = self._render_attempts.get(item.id, 0) + 1 + try: + manifest = render_dgd( + _RenderInput(config=item.parameters, used_gpus=item.used_gpus), + self._workload, + self._options, + dgd_name=f"{self._config.name}-{item.id.removeprefix(_ID_PREFIX)}", + renderer=self._config.generator, + ) + entry = SnapshotCandidate( + id=item.id, + outcome=CandidateOutcome.MATERIALIZED, + parameters=item.parameters, + metrics=item.metrics, + manifest=manifest, + ) + except CandidateMaterializationError as exc: + entry = SnapshotCandidate( + id=item.id, + outcome=CandidateOutcome.MATERIALIZATION_FAILED, + parameters=item.parameters, + metrics=item.metrics, + error=_bounded(str(exc)), + ) + except Exception as exc: + raise _RendererUnavailable(str(exc)) from exc + self._materialized[item.id] = entry + return entry + + +@dataclass(frozen=True) +class _RenderInput: + """The evaluated candidate as the renderer sees it: its config and GPU count.""" + + config: dict[str, Any] + used_gpus: int + + +class DGDRRunOutputPlugin: + """The object AISimulate discovers for ``--output dgdr_run``. + + AISimulate calls ``subscribe(config, context=...)`` once, in the process that + runs the search, and wires the returned callbacks into the Sweeper: that is + the live path (initial snapshot immediately, then coalesced snapshots). + ``write()`` runs afterwards in the CLI process, only when the search + succeeded with a selection, and publishes the authoritative terminal + snapshot from the final result. A search that fails or finds nothing never + reaches ``write()``, so it leaves no terminal snapshot; the publisher + detects that from the Sweeper container's exit. + """ + + name = "dgdr_run" + api_version = OUTPUT_ADAPTER_API_VERSION + + def subscribe( + self, config: Mapping[str, Any], context: Any = None + ) -> RecommendationOutputCallbacks | None: + """Validate the configuration and, when possible, start live publishing. + + ``context`` (``RecommendationOutputContext``) carries the Sweeper workload, + which rendering needs. Without a context or without ``snapshot_dir`` there + is nothing to publish live, so only the config is validated and the final + snapshot is still produced by ``write()``. + """ + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + resolved.generation_options() + workload = getattr(context, "workload", None) + if workload is None or resolved.snapshot_dir is None: + return None + # Live publishing needs both lifecycle hooks: without on_complete nothing stops + # the live writer before write() publishes the terminal snapshot, which a pending + # live write could then overwrite. Older AISimulate releases lack them; the + # terminal snapshot then comes from write() alone. + supported = {field.name for field in fields(RecommendationOutputCallbacks)} + if not {"on_complete", "on_failure"} <= supported: + logger.info( + "dgdr_run: AISimulate has no lifecycle callbacks; not publishing live" + ) + return None + adapter = DGDRRunOutputAdapter(resolved, workload) + adapter.start() + return RecommendationOutputCallbacks( + on_candidate=adapter.on_candidate, + on_round=adapter.on_round, + on_complete=adapter.finish_search, + on_failure=adapter.fail, + ) + + def live( + self, + config: Mapping[str, Any], + *, + workload: Any, + is_pareto: bool = False, + ) -> DGDRRunOutputAdapter: + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + return DGDRRunOutputAdapter(resolved, workload, is_pareto=is_pareto) + + def write( + self, + config: Mapping[str, Any], + *, + result: SweepResult, + output_dir: Path, + ) -> Sequence[str | Path]: + resolved = DGDRRunOutputConfig.model_validate(dict(config)) + if result.views.pareto_front: + raise NotImplementedError("dgdr_run does not support Pareto results yet") + selected = list(result.selected_candidates)[: resolved.max_candidates] + if not selected: + raise CandidateMaterializationError("no feasible candidate found") + workload = SmartSearchConfig.model_validate(result.provenance.config).workload + directory = Path(resolved.snapshot_dir or output_dir) + adapter = DGDRRunOutputAdapter( + resolved, workload, snapshot_dir=directory, keep_arrival_order=True + ) + for candidate in selected: + adapter.on_candidate(_as_feasible(candidate)) + counts = getattr(result, "counts", None) + evaluated = getattr(counts, "evaluated", None) + adapter.report_progress( + evaluated=evaluated if isinstance(evaluated, int) else 0, + # The final result does not carry the round, but the live snapshot + # written by the search process does; progress never goes backwards. + round_no=_last_published_round(directory), + ) + adapter.close_selection() + # AISimulate requires reported artifacts to live under output_dir; a snapshot + # on the shared volume does not, and is consumed by the publisher instead. + if directory.resolve() == Path(output_dir).resolve(): + return [Path(SNAPSHOT_FILE_NAME)] + return [] + + +def _last_published_round(directory: Path) -> int: + """The round recorded by the live snapshot already in ``directory``, else 0.""" + try: + loaded = yaml.safe_load((directory / SNAPSHOT_FILE_NAME).read_text()) + round_no = loaded["progress"]["round"] + except (OSError, yaml.YAMLError, KeyError, TypeError): + return 0 + return round_no if isinstance(round_no, int) and round_no > 0 else 0 + + +@dataclass(frozen=True) +class _FinalCandidate: + candidate_id: str + config: dict[str, Any] + score: float | None + used_gpus: int | None + objectives: dict[str, float] | None = None + status: str = "feasible" + metrics: dict[str, float | None] = dataclasses.field(default_factory=dict) + + +def _as_feasible(candidate: Any) -> _FinalCandidate: + score = getattr(candidate, "score", None) + used_gpus = getattr(candidate, "used_gpus", None) + if score is None or used_gpus is None: + raise CandidateMaterializationError( + "selected candidate has no score/used_gpus, so it cannot be ranked" + ) + return _FinalCandidate( + candidate_id=str(getattr(candidate, "candidate_id", "")), + config=dict(candidate.config), + score=score, + used_gpus=used_gpus, + objectives=getattr(candidate, "objectives", None), + metrics=dict(getattr(candidate, "metrics", None) or {}), + ) + + +def create_adapter() -> DGDRRunOutputPlugin: + """Create the Dynamo plugin discovered by AISimulate.""" + return DGDRRunOutputPlugin() + + +__all__ = [ + "CandidateRecordLike", + "DGDRRunOutputAdapter", + "DGDRRunOutputConfig", + "DGDRRunOutputPlugin", + "candidate_id_for", + "create_adapter", +] diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py b/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py new file mode 100644 index 000000000000..f14ff16264ed --- /dev/null +++ b/components/src/dynamo/aisimulate/output/dgdr_run/snapshot.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Desired-state snapshot of one DGDRRun, written to the pod-local volume. + +The Sweeper container atomically replaces a single YAML file with the complete +*current* projection of one ``DynamoGraphDeploymentRun``; the publisher +sidecar in the same pod consumes it directly (shared ``emptyDir``). Nothing +here involves a ConfigMap, the Dynamo event plane, or any cross-pod RPC. + +The snapshot is desired state, not a command list and not history: + +* ``run`` carries progress and, once ``terminal`` is true, the final outcome; +* ``candidates`` is the bounded set of evaluated points that should exist as + ``DynamoGraphDeploymentCandidate`` objects, best-first for scalar searches. + +Each candidate has a stable ``id`` that depends only on the evaluated point +(its resolved parameters), never on its list position or rank. Reordering +the list must therefore never change an identity or a rendered manifest; it +only changes the ordered references the publisher writes to the run status. + +Each published snapshot is complete. The publisher only needs the latest one, +so intermediate states may be coalesced away, but a terminal snapshot must +always be written (see ``DGDRRunOutputAdapter.close``). +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any + +import yaml + +from dynamo.aisimulate.output.dgd.writers.atomic import replace_text + +SNAPSHOT_FILE_NAME = "dgdr_run_snapshot.yaml" +SCHEMA_VERSION = 1 + + +class RunPhase(str, Enum): + """Run lifecycle as reported by the producer. Anything but RUNNING is terminal.""" + + RUNNING = "Running" + SUCCEEDED = "Succeeded" + FAILED = "Failed" + + @property + def terminal(self) -> bool: + return self is not RunPhase.RUNNING + + +class CandidateOutcome(str, Enum): + MATERIALIZED = "materialized" + MATERIALIZATION_FAILED = "materialization_failed" + + +@dataclass(frozen=True) +class SnapshotCandidate: + """One evaluated point in the bounded projection. + + ``parameters`` and ``metrics`` are the immutable evaluation facts the + publisher needs to build the candidate's resource: a DGD manifest alone + does not identify an evaluated point, because two evaluations can render + the same DGD under different parameters and still have different metrics. + """ + + id: str + outcome: CandidateOutcome + parameters: dict[str, Any] = field(default_factory=dict) + metrics: dict[str, Any] = field(default_factory=dict) + manifest: str | None = None + error: str | None = None + + def __post_init__(self) -> None: + if not self.id: + raise ValueError("candidate id must not be empty") + if self.outcome is CandidateOutcome.MATERIALIZED: + if not self.manifest: + raise ValueError("a materialized candidate must carry its manifest") + if self.error is not None: + raise ValueError("a materialized candidate must not carry an error") + elif not self.error: + raise ValueError("a failed candidate must carry its error") + elif self.manifest is not None: + raise ValueError("a failed candidate must not carry a manifest") + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "id": self.id, + "outcome": self.outcome.value, + "parameters": self.parameters, + "metrics": self.metrics, + } + if self.manifest is not None: + payload["manifest"] = self.manifest + if self.error is not None: + payload["error"] = self.error + return payload + + +@dataclass(frozen=True) +class DGDRRunSnapshot: + """The complete current projection of one run.""" + + phase: RunPhase + round_no: int + evaluated: int + candidates: tuple[SnapshotCandidate, ...] = () + message: str = "" + error: str = "" + + def to_dict(self) -> dict[str, Any]: + run: dict[str, Any] = { + "phase": self.phase.value, + "terminal": self.phase.terminal, + } + if self.message: + run["message"] = self.message + if self.error: + run["error"] = self.error + return { + "schemaVersion": SCHEMA_VERSION, + "timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "run": run, + "progress": {"round": self.round_no, "evaluated": self.evaluated}, + "candidates": [candidate.to_dict() for candidate in self.candidates], + } + + +def write_snapshot(directory: Path, snapshot: DGDRRunSnapshot) -> Path: + """Atomically replace the snapshot file; readers never see a partial file.""" + path = directory / SNAPSHOT_FILE_NAME + replace_text(path, yaml.safe_dump(snapshot.to_dict(), sort_keys=False)) + return path + + +__all__ = [ + "SCHEMA_VERSION", + "SNAPSHOT_FILE_NAME", + "CandidateOutcome", + "DGDRRunSnapshot", + "RunPhase", + "SnapshotCandidate", + "write_snapshot", +] diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py new file mode 100644 index 000000000000..deccceeb250b --- /dev/null +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -0,0 +1,1077 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +import yaml + +from dynamo.aisimulate.output.dgd.renderers import CandidateMaterializationError +from dynamo.aisimulate.output.dgdr_run import adapter as adapter_module +from dynamo.aisimulate.output.dgdr_run.adapter import ( + DGDRRunOutputAdapter, + DGDRRunOutputConfig, + candidate_id_for, + create_adapter, +) +from dynamo.aisimulate.output.dgdr_run.snapshot import SNAPSHOT_FILE_NAME, RunPhase + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + + +@dataclass +class Record: + candidate_id: str + score: float | None + used_gpus: int | None + config: dict[str, Any] + status: str = "feasible" + objectives: dict[str, float] | None = None + metrics: dict[str, float | None] = field(default_factory=dict) + + +def record(tag: str, score: float, gpus: int = 8, **kwargs: Any) -> Record: + return Record( + candidate_id=f"sweeper-{tag}", + score=score, + used_gpus=gpus, + config={"backend": "vllm", "backend_version": "0.1", "tag": tag}, + **kwargs, + ) + + +def make_config(tmp_path: Path, **overrides: Any) -> DGDRRunOutputConfig: + values: dict[str, Any] = { + "name": "sweep", + "snapshot_dir": str(tmp_path), + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "max_candidates": 3, + "snapshot_interval_seconds": 0, + } + values.update(overrides) + return DGDRRunOutputConfig(**values) + + +def load(tmp_path: Path) -> dict[str, Any]: + return yaml.safe_load((tmp_path / SNAPSHOT_FILE_NAME).read_text()) + + +def manifests_by_id(snapshot: dict[str, Any]) -> dict[str, str]: + return {c["id"]: c["manifest"] for c in snapshot["candidates"]} + + +def tags(snapshot: dict[str, Any]) -> list[str]: + return [candidate["parameters"]["tag"] for candidate in snapshot["candidates"]] + + +def wait_for(predicate: Any, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError("condition not reached before the timeout") + + +@pytest.fixture +def renders(monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Replace the shared materializer with a counting fake.""" + calls: list[str] = [] + + def fake_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: + tag = candidate.config["tag"] + calls.append(tag) + if tag.startswith("bad"): + raise CandidateMaterializationError(f"cannot render {tag}") + return ( + "apiVersion: nvidia.com/v1beta1\n" + "kind: DynamoGraphDeployment\n" + f"metadata:\n name: {kwargs['dgd_name']}\n" + f"spec:\n marker: {tag}\n" + ) + + monkeypatch.setattr(adapter_module, "render_dgd", fake_render) + return calls + + +# -- projection --------------------------------------------------------------- + + +def test_start_writes_the_initial_snapshot_immediately( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=60) + with DGDRRunOutputAdapter(config, workload=None): + snapshot = load(tmp_path) + assert snapshot["run"] == {"phase": "Running", "terminal": False} + assert snapshot["candidates"] == [] + assert snapshot["progress"] == {"round": 0, "evaluated": 0} + + +def test_projection_is_bounded_and_best_first_whatever_is_evaluated( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + for index in range(500): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + adapter.close() + + snapshot = load(tmp_path) + assert tags(snapshot) == ["p499", "p498", "p497"] + assert snapshot["progress"]["evaluated"] == 500 + # Only retained points are ever materialized, never all 500. + assert len(renders) == 3 + + +def test_ties_break_on_fewer_gpus_then_id_deterministically( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path, max_candidates=4), None) + adapter.on_candidate(record("big", score=5.0, gpus=16)) + adapter.on_candidate(record("small", score=5.0, gpus=8)) + adapter.on_candidate(record("best", score=9.0, gpus=64)) + adapter.close() + + assert tags(load(tmp_path)) == ["best", "small", "big"] + + +def test_infeasible_candidates_are_counted_but_never_retained( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("ok", 1.0)) + adapter.on_candidate(record("nope", 99.0, status="infeasible")) + adapter.close() + + snapshot = load(tmp_path) + assert tags(snapshot) == ["ok"] + assert snapshot["progress"]["evaluated"] == 2 + + +def test_snapshot_preserves_the_evaluated_point_not_just_its_manifest( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 87.7, gpus=4, objectives={"goodput": 12.5})) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialized" + assert candidate["parameters"] == { + "backend": "vllm", + "backend_version": "0.1", + "tag": "a", + } + assert candidate["metrics"] == { + "score": 87.7, + "usedGpus": 4, + "objectives": {"goodput": 12.5}, + } + assert "marker: a" in candidate["manifest"] + + +def test_round_progress_is_monotonic(tmp_path: Path, renders: list[str]) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_round(3, []) + adapter.on_round(2, []) + adapter.close() + assert load(tmp_path)["progress"]["round"] == 3 + + +# -- identity, immutability, materialization ---------------------------------- + + +def test_identity_and_manifests_do_not_depend_on_rank_or_observation_order( + tmp_path: Path, renders: list[str] +) -> None: + first = tmp_path / "first" + second = tmp_path / "second" + one = DGDRRunOutputAdapter(make_config(first), workload=None) + one.on_candidate(record("a", 1.0)) + one.on_candidate(record("b", 2.0)) + one.close() + two = DGDRRunOutputAdapter(make_config(second), workload=None) + two.on_candidate(record("b", 1.0)) + two.on_candidate(record("a", 2.0)) + two.close() + + one_snapshot, two_snapshot = load(first), load(second) + assert tags(one_snapshot) == ["b", "a"] + assert tags(two_snapshot) == ["a", "b"] + assert manifests_by_id(one_snapshot) == manifests_by_id(two_snapshot) + + +def test_identity_depends_only_on_the_resolved_parameters() -> None: + assert candidate_id_for({"a": 1, "b": 2}) == candidate_id_for({"b": 2, "a": 1}) + assert candidate_id_for({"a": 1}) != candidate_id_for({"a": 2}) + assert candidate_id_for({"a": 1}).startswith("evaluated-point-") + + +def test_a_new_better_point_reorders_but_does_not_rematerialize_retained_ones( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + adapter.on_candidate(record("b", 2.0)) + wait_for(lambda: tags(load(tmp_path)) == ["b", "a"]) + before = {c["id"]: c["manifest"] for c in load(tmp_path)["candidates"]} + assert sorted(renders) == ["a", "b"] + + adapter.on_candidate(record("c", 3.0)) + wait_for(lambda: tags(load(tmp_path)) == ["c", "b", "a"]) + adapter.close() + + after = {c["id"]: c["manifest"] for c in load(tmp_path)["candidates"]} + assert {key: after[key] for key in before} == before + assert sorted(renders) == ["a", "b", "c"] + + +def test_progress_updates_do_not_rematerialize( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + for round_no in range(1, 20): + adapter.on_round(round_no, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 19) + adapter.close() + assert renders == ["a"] + + +def test_a_point_is_immutable_once_observed(tmp_path: Path, renders: list[str]) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 1.0)) + adapter.on_candidate(record("a", 50.0)) # same parameters, noisy re-evaluation + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["metrics"]["score"] == 1.0 + assert load(tmp_path)["progress"]["evaluated"] == 2 + + +def test_a_persistent_materialization_failure_is_reported_and_retried_a_bounded_number_of_times( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("bad1", 5.0)) + wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) + for round_no in range(1, 10): + adapter.on_round(round_no, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 9) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialization_failed" + assert "cannot render bad1" in candidate["error"] + assert "manifest" not in candidate + assert renders == ["bad1"] * adapter_module._MAX_RENDER_ATTEMPTS + + +# -- rate limiting and coalescing --------------------------------------------- + + +def test_many_callbacks_in_one_interval_produce_one_periodic_write( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=0.5) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + assert adapter.snapshot_writes == 1 + + for index in range(300): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + time.sleep(0.15) + assert adapter.snapshot_writes == 1 + + wait_for(lambda: adapter.snapshot_writes == 2, timeout=5) + snapshot = load(tmp_path) + # The one periodic write carries the latest state, not an arbitrary + # intermediate one. + assert snapshot["progress"]["evaluated"] == 300 + assert tags(snapshot) == ["p299", "p298", "p297"] + + time.sleep(0.2) + assert adapter.snapshot_writes == 2 + adapter.close() + + +def test_terminal_state_bypasses_the_rate_limit_and_is_always_flushed( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=3600) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + for index in range(100): + adapter.on_candidate(record(f"p{index:03d}", score=float(index))) + + started = time.monotonic() + adapter.close(phase=RunPhase.FAILED, message="boom", error="OOM") + assert time.monotonic() - started < 5 + + snapshot = load(tmp_path) + assert snapshot["run"] == { + "phase": "Failed", + "terminal": True, + "message": "boom", + "error": "OOM", + } + assert tags(snapshot) == ["p099", "p098", "p097"] + assert snapshot["progress"]["evaluated"] == 100 + assert adapter.snapshot_writes == 2 # initial + terminal, nothing in between + + +def test_the_terminal_snapshot_contains_every_update_accepted_before_close( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=3600) + with DGDRRunOutputAdapter(config, workload=None) as adapter: + for index in range(50): + adapter.on_candidate(record(f"w{index}", score=1.0 + index)) + adapter.on_candidate(record("winner", score=1000.0)) + + snapshot = load(tmp_path) + assert snapshot["run"]["terminal"] is True + assert tags(snapshot)[0] == "winner" + + +def test_callbacks_never_wait_for_a_slow_writer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + release = threading.Event() + + def slow_render(candidate: Any, *args: Any, **kwargs: Any) -> str: + release.wait(5) + return "kind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", slow_render) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 1.0)) + time.sleep(0.1) # the writer is now stuck rendering + + started = time.monotonic() + for index in range(1000): + adapter.on_candidate(record(f"q{index}", score=float(index))) + assert time.monotonic() - started < 2 + release.set() + adapter.close() + # Nothing was dropped while the writer was stuck. + assert load(tmp_path)["progress"]["evaluated"] == 1001 + + +def test_updates_after_close_are_ignored_and_close_is_idempotent( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 1.0)) + adapter.close(phase=RunPhase.SUCCEEDED) + written = (tmp_path / SNAPSHOT_FILE_NAME).read_text() + writes = adapter.snapshot_writes + + adapter.on_candidate(record("late", 99.0)) + adapter.on_round(9, []) + adapter.close(phase=RunPhase.FAILED, error="ignored") + + assert (tmp_path / SNAPSHOT_FILE_NAME).read_text() == written + assert adapter.snapshot_writes == writes + + +def test_close_rejects_a_non_terminal_phase(tmp_path: Path) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + with pytest.raises(ValueError, match="terminal"): + adapter.close(phase=RunPhase.RUNNING) + + +# -- failure handling --------------------------------------------------------- + + +def test_context_manager_records_success_and_failure( + tmp_path: Path, renders: list[str] +) -> None: + ok = tmp_path / "ok" + with DGDRRunOutputAdapter(make_config(ok), workload=None): + pass + assert load(ok)["run"]["phase"] == "Succeeded" + + failed = tmp_path / "failed" + with pytest.raises(RuntimeError, match="search crashed"): + with DGDRRunOutputAdapter(make_config(failed), workload=None): + raise RuntimeError("search crashed") + snapshot = load(failed) + assert snapshot["run"]["phase"] == "Failed" + assert snapshot["run"]["error"] == "RuntimeError" + + +def test_a_terminal_write_failure_is_raised_not_swallowed( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + + def broken(*_: Any, **__: Any) -> None: + raise OSError("disk full") + + monkeypatch.setattr(adapter_module, "write_snapshot", broken) + with pytest.raises(OSError, match="disk full"): + adapter.close() + + +def test_a_failed_periodic_write_is_retried_with_the_latest_state( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0.02) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + real = adapter_module.write_snapshot + attempts = {"count": 0} + + def flaky(directory: Path, snapshot: Any) -> Path: + attempts["count"] += 1 + if attempts["count"] == 1: + raise OSError("transient") + return real(directory, snapshot) + + monkeypatch.setattr(adapter_module, "write_snapshot", flaky) + adapter.on_candidate(record("a", 1.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + adapter.close() + assert attempts["count"] >= 2 + + +def test_a_candidate_that_cannot_be_read_does_not_break_the_search( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(SimpleNamespace(status="feasible")) # type: ignore[arg-type] + adapter.on_candidate(record("ok", 1.0)) + adapter.close() + assert tags(load(tmp_path)) == ["ok"] + + +# -- configuration and plugin ------------------------------------------------- + + +def test_pareto_objectives_are_rejected_explicitly(tmp_path: Path) -> None: + with pytest.raises(NotImplementedError, match="scalar"): + DGDRRunOutputAdapter(make_config(tmp_path), workload=None, is_pareto=True) + + +@pytest.mark.parametrize( + "overrides", + [ + {"max_candidates": 0}, + {"max_candidates": 101}, + {"snapshot_interval_seconds": -1}, + {"name": "Not_DNS"}, + {"num_gpus_per_node": 0}, + {"unknown_field": 1}, + ], +) +def test_invalid_configuration_is_rejected( + tmp_path: Path, overrides: dict[str, Any] +) -> None: + with pytest.raises(ValueError): + make_config(tmp_path, **overrides) + + +def test_max_candidates_follows_the_api_range(tmp_path: Path) -> None: + # spec.recommendation.maxCandidates is 1..100; the adapter accepts exactly that range. + assert make_config(tmp_path, max_candidates=1).max_candidates == 1 + assert make_config(tmp_path, max_candidates=100).max_candidates == 100 + for invalid in (0, 101, 200): + with pytest.raises(ValueError): + make_config(tmp_path, max_candidates=invalid) + + +def test_max_candidates_has_no_default_of_its_own() -> None: + # The operator supplies the already-defaulted API value; a missing one must fail. + with pytest.raises(ValueError, match="max_candidates"): + DGDRRunOutputConfig( + name="sweep", + snapshot_dir="/tmp/x", + runtime_image="nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + num_gpus_per_node=8, + ) + + +def test_a_snapshot_directory_is_required() -> None: + config = DGDRRunOutputConfig( + name="sweep", + runtime_image="nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + num_gpus_per_node=8, + max_candidates=1, + ) + with pytest.raises(ValueError, match="snapshot directory"): + DGDRRunOutputAdapter(config, workload=None) + + +def test_the_plugin_is_discoverable_and_builds_a_live_adapter(tmp_path: Path) -> None: + plugin = create_adapter() + assert plugin.name == "dgdr_run" + live = plugin.live( + { + "name": "sweep", + "snapshot_dir": str(tmp_path), + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "max_candidates": 5, + }, + workload=None, + ) + assert isinstance(live, DGDRRunOutputAdapter) + + +def test_plugin_subscribe_validates_config_before_search() -> None: + plugin = adapter_module.DGDRRunOutputPlugin() + good = { + "name": "run", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "max_candidates": 5, + } + + assert plugin.subscribe(good) is None + with pytest.raises(ValueError, match="num_gpus_per_node"): + plugin.subscribe({**good, "num_gpus_per_node": 0}) + with pytest.raises(ValueError, match="canonical"): + plugin.subscribe( + {**good, "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest"} + ) + + +def test_plugin_write_publishes_the_final_selection_as_a_terminal_snapshot( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + class FakeSearchConfig: + @staticmethod + def model_validate(_: Any) -> Any: + return SimpleNamespace(workload="workload") + + monkeypatch.setattr(adapter_module, "SmartSearchConfig", FakeSearchConfig) + result = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[ + record("a", 9.0), + record("b", 5.0), + ], + ) + written = create_adapter().write( + { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "max_candidates": 5, + }, + result=result, # type: ignore[arg-type] + output_dir=tmp_path, + ) + + assert written == [Path(SNAPSHOT_FILE_NAME)] + snapshot = load(tmp_path) + assert snapshot["run"]["terminal"] is True + assert tags(snapshot) == ["a", "b"] + + +def test_plugin_write_keeps_sweepers_selection_order( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + # Sweeper's selection can be goal-dependent (e.g. fewest GPUs first), so the + # adapter must not re-rank it by score. + result = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[record("low", 1.0), record("high", 9.0)], + counts=SimpleNamespace(evaluated=2), + ) + create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) # type: ignore[arg-type] + assert tags(load(tmp_path)) == ["low", "high"] + + +def test_plugin_write_refuses_pareto_and_empty_results(tmp_path: Path) -> None: + config = { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "max_candidates": 5, + } + pareto = SimpleNamespace(views=SimpleNamespace(pareto_front=[object()])) + with pytest.raises(NotImplementedError): + create_adapter().write(config, result=pareto, output_dir=tmp_path) # type: ignore[arg-type] + empty = SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), selected_candidates=[] + ) + with pytest.raises(CandidateMaterializationError): + create_adapter().write(config, result=empty, output_dir=tmp_path) # type: ignore[arg-type] + + +PLUGIN_CONFIG = { + "name": "sweep", + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "num_gpus_per_node": 8, + "max_candidates": 5, + "snapshot_interval_seconds": 0, +} + + +def test_plugin_subscribe_with_a_context_publishes_live( + tmp_path: Path, renders: list[str] +) -> None: + config = {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)} + callbacks = create_adapter().subscribe( + config, context=SimpleNamespace(workload="workload") + ) + + assert callbacks is not None + assert callbacks.on_candidate is not None + assert callbacks.on_round is not None + assert load(tmp_path)["run"] == {"phase": "Running", "terminal": False} + + callbacks.on_candidate(record("a", 9.0)) + wait_for(lambda: tags(load(tmp_path)) == ["a"]) + assert callbacks.on_complete is not None + callbacks.on_complete() + + +def test_plugin_subscribe_without_context_or_snapshot_dir_only_validates( + tmp_path: Path, +) -> None: + plugin = create_adapter() + with_dir = {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)} + assert plugin.subscribe(with_dir) is None + assert plugin.subscribe(with_dir, context=SimpleNamespace(workload=None)) is None + assert ( + plugin.subscribe(PLUGIN_CONFIG, context=SimpleNamespace(workload="workload")) + is None + ) + assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() + + +def _final_result(evaluated: int = 42) -> Any: + return SimpleNamespace( + views=SimpleNamespace(pareto_front=[]), + provenance=SimpleNamespace(config={}), + selected_candidates=[record("a", 9.0)], + counts=SimpleNamespace(evaluated=evaluated), + ) + + +@pytest.fixture +def fake_search_config(monkeypatch: pytest.MonkeyPatch) -> None: + class FakeSearchConfig: + @staticmethod + def model_validate(_: Any) -> Any: + return SimpleNamespace(workload="workload") + + monkeypatch.setattr(adapter_module, "SmartSearchConfig", FakeSearchConfig) + + +def test_plugin_write_reports_no_artifact_for_a_snapshot_outside_output_dir( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + shared = tmp_path / "shared" + shared.mkdir() + out = tmp_path / "out" + out.mkdir() + + written = create_adapter().write( + {**PLUGIN_CONFIG, "snapshot_dir": str(shared)}, + result=_final_result(), + output_dir=out, + ) + + assert written == [] + assert load(shared)["run"]["terminal"] is True + assert not (out / SNAPSHOT_FILE_NAME).exists() + + +def test_a_transient_materialization_failure_is_retried_and_recovers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + attempts: list[int] = [] + + def flaky_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: + attempts.append(1) + if len(attempts) == 1: + raise CandidateMaterializationError("cold start") + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", flaky_render) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + adapter.on_candidate(record("a", 5.0)) + wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) + assert load(tmp_path)["candidates"][0]["outcome"] == "materialization_failed" + + adapter.on_round(1, []) # any later write retries the failed render + wait_for(lambda: load(tmp_path)["candidates"][0]["outcome"] == "materialized") + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert candidate["outcome"] == "materialized" + assert "error" not in candidate + assert len(attempts) == 2 + + +def test_plugin_write_without_a_live_snapshot_starts_at_round_zero( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + create_adapter().write(PLUGIN_CONFIG, result=_final_result(2), output_dir=tmp_path) + assert load(tmp_path)["progress"] == {"round": 0, "evaluated": 2} + + +# -- lifecycle: how the search ended -------------------------------------------- + + +def _subscribed(tmp_path: Path) -> Any: + callbacks = create_adapter().subscribe( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + context=SimpleNamespace(workload="workload"), + ) + assert callbacks is not None + return callbacks + + +def test_a_failed_search_leaves_a_terminal_failed_snapshot( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_round(2, []) + + callbacks.on_failure(RuntimeError("replay crashed")) + + snapshot = load(tmp_path) + assert snapshot["run"] == { + "phase": "Failed", + "terminal": True, + "message": "replay crashed", + "error": "RuntimeError", + } + assert tags(snapshot) == ["a"] + assert snapshot["progress"]["round"] == 2 + + +def test_a_long_failure_message_is_truncated( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_failure(RuntimeError("x" * 5000)) + assert len(load(tmp_path)["run"]["message"]) == adapter_module._MAX_ERROR_MESSAGE + + +def test_a_search_that_finds_nothing_is_recorded_as_failed( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("infeasible", 1.0, status="infeasible")) + + callbacks.on_complete() + + run = load(tmp_path)["run"] + assert run["phase"] == "Failed" + assert run["terminal"] is True + assert run["message"] == "no feasible candidate found" + + +def test_a_completed_search_flushes_stops_the_writer_and_stays_non_terminal( + tmp_path: Path, renders: list[str] +) -> None: + config = make_config(tmp_path, snapshot_interval_seconds=60) + adapter = DGDRRunOutputAdapter(config, workload=None) + adapter.start() + adapter.on_candidate(record("a", 9.0)) + adapter.on_round(3, []) + + adapter.finish_search() + + snapshot = load(tmp_path) + assert snapshot["run"] == {"phase": "Running", "terminal": False} + assert tags(snapshot) == ["a"] + assert snapshot["progress"] == {"round": 3, "evaluated": 1} + # The writer is gone: nothing can overwrite the terminal snapshot written next. + assert adapter._thread is not None and not adapter._thread.is_alive() + writes = adapter.snapshot_writes + adapter.on_round(4, []) + time.sleep(0.1) + assert adapter.snapshot_writes == writes + + +def test_the_terminal_snapshot_after_a_completed_search_keeps_the_live_progress( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_round(3, []) + callbacks.on_complete() + + create_adapter().write( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + result=_final_result(1), + output_dir=tmp_path, + ) + + snapshot = load(tmp_path) + assert snapshot["run"]["phase"] == "Succeeded" + assert snapshot["progress"] == {"round": 3, "evaluated": 1} + + +def test_failure_after_completion_is_a_noop_and_completion_after_failure_too( + tmp_path: Path, renders: list[str] +) -> None: + callbacks = _subscribed(tmp_path) + callbacks.on_candidate(record("a", 9.0)) + callbacks.on_failure(RuntimeError("first")) + callbacks.on_complete() + callbacks.on_failure(RuntimeError("second")) + assert load(tmp_path)["run"]["message"] == "first" + + +# -- robustness ----------------------------------------------------------------- + + +def test_a_broken_renderer_leaves_a_terminal_failed_snapshot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def broken(*_: Any, **__: Any) -> str: + raise RuntimeError("renderer not installed") + + monkeypatch.setattr(adapter_module, "render_dgd", broken) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0)) + + with pytest.raises(RuntimeError, match="renderer not installed"): + adapter.close() + + snapshot = load(tmp_path) + assert snapshot["run"]["phase"] == "Failed" + assert snapshot["run"]["terminal"] is True + assert snapshot["run"]["error"] == "RuntimeError" + assert all(c["outcome"] != "materialization_failed" for c in snapshot["candidates"]) + + +def test_non_finite_scores_and_objectives_stay_valid_for_the_publisher( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", float("-inf"), objectives={"e2e": float("nan")})) + adapter.on_candidate(record("b", float("nan"))) + adapter.close() + + text = (tmp_path / SNAPSHOT_FILE_NAME).read_text() + assert ( + "inf" not in text.lower().replace("information", "") + and "nan" not in text.lower() + ) + metrics = { + c["parameters"]["tag"]: c["metrics"] for c in load(tmp_path)["candidates"] + } + assert metrics["a"]["score"] is None and metrics["a"]["objectives"] == {"e2e": None} + assert metrics["b"]["score"] is None + + +def test_a_candidate_error_message_is_bounded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def failing(*_: Any, **__: Any) -> str: + raise CandidateMaterializationError("x" * 5000) + + monkeypatch.setattr(adapter_module, "render_dgd", failing) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0)) + adapter.close() + + (candidate,) = load(tmp_path)["candidates"] + assert 0 < len(candidate["error"]) <= 500 + + +def test_subscribe_does_not_publish_live_without_lifecycle_callbacks( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + @dataclass + class LegacyCallbacks: + on_candidate: Any = None + on_round: Any = None + + monkeypatch.setattr( + adapter_module, "RecommendationOutputCallbacks", LegacyCallbacks + ) + callbacks = create_adapter().subscribe( + {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, + context=SimpleNamespace(workload="workload"), + ) + assert callbacks is None + assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() + + +def test_a_persistent_write_failure_does_not_spin_the_writer( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0.2) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.start() + attempts = {"count": 0} + + def failing(directory: Path, snapshot: Any) -> Path: + attempts["count"] += 1 + raise OSError("disk full") + + monkeypatch.setattr(adapter_module, "write_snapshot", failing) + adapter.on_candidate(record("a", 1.0)) + time.sleep(0.6) + seen = attempts["count"] + monkeypatch.undo() + adapter.close() + assert 1 <= seen <= 5 + + +def test_plugin_write_keeps_the_objectives_of_the_selected_candidates( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + result = _final_result(1) + result.selected_candidates = [record("a", 9.0, objectives={"throughput": 12.5})] + create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) + (candidate,) = load(tmp_path)["candidates"] + assert candidate["metrics"]["objectives"] == {"throughput": 12.5} + + +def test_the_renderer_receives_the_candidates_gpu_count( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + seen: list[Any] = [] + + def capture(candidate: Any, *_: Any, **kwargs: Any) -> str: + seen.append(candidate) + return "kind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate(record("a", 9.0, gpus=4)) + adapter.close() + + (candidate,) = seen + assert candidate.used_gpus == 4 + assert candidate.config["tag"] == "a" + + +# -- evaluated metrics survive, and a run without a deployable candidate fails ---- + + +SOURCE_METRICS: dict[str, float | None] = { + "throughput": 70.4, + "ttft": 18.2, + "itl": 7.1, + "e2e_latency": 300.5, + "goodput": 66.0, + "power_w": 410.0, + "normalized_power": 0.8, + "broken": float("nan"), +} + + +def test_every_finite_source_metric_survives_in_live_and_terminal_snapshots( + tmp_path: Path, renders: list[str], fake_search_config: None +) -> None: + expected = { + **{key: value for key, value in SOURCE_METRICS.items() if key != "broken"}, + "broken": None, + "score": 87.7, + "usedGpus": 4, + } + + live = tmp_path / "live" + live.mkdir() + adapter = DGDRRunOutputAdapter(make_config(live), workload=None) + adapter.start() + adapter.on_candidate(record("a", 87.7, gpus=4, metrics=dict(SOURCE_METRICS))) + wait_for(lambda: tags(load(live)) == ["a"]) + assert load(live)["candidates"][0]["metrics"] == expected + adapter.close() + + final = tmp_path / "final" + final.mkdir() + result = _final_result(1) + result.selected_candidates = [ + record("a", 87.7, gpus=4, metrics=dict(SOURCE_METRICS)) + ] + create_adapter().write( + {**PLUGIN_CONFIG, "snapshot_dir": str(final)}, result=result, output_dir=final + ) + assert load(final)["candidates"][0]["metrics"] == expected + + +def test_reserved_metric_names_are_replaced_by_the_adapters_values( + tmp_path: Path, renders: list[str] +) -> None: + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + adapter.on_candidate( + record("a", 5.0, gpus=2, metrics={"score": 999.0, "usedGpus": 99, "ttft": 1.0}) + ) + adapter.close() + assert load(tmp_path)["candidates"][0]["metrics"] == { + "ttft": 1.0, + "score": 5.0, + "usedGpus": 2, + } + + +def test_a_selection_where_every_candidate_fails_to_materialize_is_a_failed_run( + tmp_path: Path, + renders: list[str], + fake_search_config: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0) + result = _final_result(2) + result.selected_candidates = [record("bad1", 9.0), record("bad2", 5.0)] + + with pytest.raises(CandidateMaterializationError, match="no selected candidate"): + create_adapter().write( + {**PLUGIN_CONFIG, "max_candidates": 2}, result=result, output_dir=tmp_path + ) + + snapshot = load(tmp_path) + assert snapshot["run"]["phase"] == "Failed" + assert snapshot["run"]["terminal"] is True + assert snapshot["run"]["error"] == "CandidateMaterializationError" + assert "cannot render bad1" in snapshot["run"]["message"] + assert len(snapshot["run"]["message"]) <= adapter_module._MAX_ERROR_MESSAGE + assert all(c["outcome"] == "materialization_failed" for c in snapshot["candidates"]) + + +def test_a_selection_with_one_materialized_candidate_still_succeeds( + tmp_path: Path, + renders: list[str], + fake_search_config: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0) + result = _final_result(2) + result.selected_candidates = [record("bad1", 9.0), record("good", 5.0)] + + create_adapter().write( + {**PLUGIN_CONFIG, "max_candidates": 2}, result=result, output_dir=tmp_path + ) + + snapshot = load(tmp_path) + assert snapshot["run"]["phase"] == "Succeeded" + assert {c["parameters"]["tag"]: c["outcome"] for c in snapshot["candidates"]} == { + "bad1": "materialization_failed", + "good": "materialized", + } diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py new file mode 100644 index 000000000000..6ea05b6536e2 --- /dev/null +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_snapshot.py @@ -0,0 +1,148 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +import yaml + +from dynamo.aisimulate.output.dgdr_run.snapshot import ( + SCHEMA_VERSION, + SNAPSHOT_FILE_NAME, + CandidateOutcome, + DGDRRunSnapshot, + RunPhase, + SnapshotCandidate, + write_snapshot, +) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.planner, + pytest.mark.parallel, +] + + +def materialized(candidate_id: str = "evaluated-point-a") -> SnapshotCandidate: + return SnapshotCandidate( + id=candidate_id, + outcome=CandidateOutcome.MATERIALIZED, + parameters={"tp": 4}, + metrics={"score": 1.5, "usedGpus": 4}, + manifest="kind: DynamoGraphDeployment\n", + ) + + +def test_wire_format_of_a_running_snapshot() -> None: + payload = DGDRRunSnapshot( + phase=RunPhase.RUNNING, + round_no=7, + evaluated=48, + candidates=(materialized(),), + ).to_dict() + + assert payload["schemaVersion"] == SCHEMA_VERSION + assert payload["run"] == {"phase": "Running", "terminal": False} + assert payload["progress"] == {"round": 7, "evaluated": 48} + assert payload["candidates"] == [ + { + "id": "evaluated-point-a", + "outcome": "materialized", + "parameters": {"tp": 4}, + "metrics": {"score": 1.5, "usedGpus": 4}, + "manifest": "kind: DynamoGraphDeployment\n", + } + ] + assert payload["timestamp"].endswith("Z") + + +def test_terminal_snapshot_carries_the_outcome() -> None: + payload = DGDRRunSnapshot( + phase=RunPhase.FAILED, + round_no=2, + evaluated=9, + message="stopped", + error="OOM", + ).to_dict() + assert payload["run"] == { + "phase": "Failed", + "terminal": True, + "message": "stopped", + "error": "OOM", + } + assert payload["candidates"] == [] + + +def test_a_failed_candidate_carries_an_error_and_no_manifest() -> None: + failed = SnapshotCandidate( + id="evaluated-point-b", + outcome=CandidateOutcome.MATERIALIZATION_FAILED, + error="cannot render", + ) + assert failed.to_dict() == { + "id": "evaluated-point-b", + "outcome": "materialization_failed", + "parameters": {}, + "metrics": {}, + "error": "cannot render", + } + + +@pytest.mark.parametrize( + "kwargs", + [ + {"id": "", "outcome": CandidateOutcome.MATERIALIZED, "manifest": "x"}, + {"id": "a", "outcome": CandidateOutcome.MATERIALIZED}, + { + "id": "a", + "outcome": CandidateOutcome.MATERIALIZED, + "manifest": "x", + "error": "y", + }, + {"id": "a", "outcome": CandidateOutcome.MATERIALIZATION_FAILED}, + { + "id": "a", + "outcome": CandidateOutcome.MATERIALIZATION_FAILED, + "error": "y", + "manifest": "x", + }, + ], +) +def test_inconsistent_candidates_are_rejected(kwargs: dict) -> None: + with pytest.raises(ValueError): + SnapshotCandidate(**kwargs) + + +def test_write_replaces_the_file_atomically_and_leaves_no_temporary_files( + tmp_path: Path, +) -> None: + for round_no in range(2): + path = write_snapshot( + tmp_path, + DGDRRunSnapshot(phase=RunPhase.RUNNING, round_no=round_no, evaluated=0), + ) + assert path == tmp_path / SNAPSHOT_FILE_NAME + assert yaml.safe_load(path.read_text())["progress"]["round"] == round_no + + assert [p.name for p in tmp_path.iterdir()] == [SNAPSHOT_FILE_NAME] + + +def test_the_manifest_survives_a_yaml_round_trip_byte_for_byte( + tmp_path: Path, +) -> None: + manifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec:\n a: 'x: y'\n" + candidate = SnapshotCandidate( + id="evaluated-point-c", + outcome=CandidateOutcome.MATERIALIZED, + manifest=manifest, + ) + path = write_snapshot( + tmp_path, + DGDRRunSnapshot( + phase=RunPhase.RUNNING, round_no=0, evaluated=1, candidates=(candidate,) + ), + ) + (loaded,) = yaml.safe_load(path.read_text())["candidates"] + assert loaded["manifest"] == manifest diff --git a/pyproject.toml b/pyproject.toml index 9e0583619b80..e49e4df036d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -133,6 +133,7 @@ dynamo = "dynamo.replay.simulation:DynamoReplayRunnerFactory" [project.entry-points."aisimulate.output_adapters"] dgd = "dynamo.aisimulate.output.dgd:create_adapter" +dgdr_run = "dynamo.aisimulate.output.dgdr_run:create_adapter" [dependency-groups] docs = [ From 048bffc93fe529af6eec3e67db9bdb1ca0f6d2ba Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 09:50:29 +0100 Subject: [PATCH 031/100] pr-15895: pr-7: bound AIC resource names [upstream: devivasudevan/dynamo#7] Signed-off-by: Dr. Stefan Schimanski --- .../aisimulate/output/dgdr_run/adapter.py | 23 +++++++- .../aisimulate/tests/dgdr_run/test_adapter.py | 53 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index eb310d5582ca..2c3f9b9e155a 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -83,6 +83,9 @@ _MAX_ERROR_MESSAGE = 500 _WRITE_RETRY_BACKOFF = 1.0 _ID_HEX_LENGTH = 12 +# AIC appends component-specific suffixes to the prefix. Keep the same bound as +# the AIC comparison harness so every generated Kubernetes name remains valid. +_MAX_DGD_NAME_LENGTH = 28 # Metric names the adapter derives itself. They are reserved: a measurement AISimulate resolves under # one of these names is replaced by the adapter's value (and logged), never merged with it. _RESERVED_METRICS = ("score", "usedGpus", "objectives") @@ -151,6 +154,18 @@ def candidate_id_for(parameters: Mapping[str, Any]) -> str: return f"{_ID_PREFIX}{digest[:_ID_HEX_LENGTH]}" +def _dgd_name(run_name: str, candidate_id: str) -> str: + """Return one stable DNS-safe AIC prefix with room for component suffixes.""" + candidate_suffix = candidate_id.removeprefix(_ID_PREFIX) + requested = f"{run_name}-{candidate_suffix}" + normalized = requested.replace(".", "-") + if normalized == requested and len(normalized) <= _MAX_DGD_NAME_LENGTH: + return normalized + digest = hashlib.sha256(requested.encode("utf-8")).hexdigest()[:8] + prefix = normalized[: _MAX_DGD_NAME_LENGTH - len(digest) - 1].rstrip("-") + return f"{prefix}-{digest}" + + def _is_feasible(record: CandidateRecordLike) -> bool: # By value, not identity: callers may pass aisimulate's CandidateStatus # enum, a plain string, or a test double. @@ -194,7 +209,11 @@ class DGDRRunOutputConfig(BaseModel): model_config = ConfigDict(extra="forbid") - name: str = Field(min_length=1, pattern=r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") + name: str = Field( + min_length=1, + max_length=253, + pattern=r"^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\.[a-z0-9](?:[-a-z0-9]*[a-z0-9])?)*$", + ) snapshot_dir: str | None = Field(default=None, min_length=1) namespace: str | None = None generator: Literal["aic", "direct"] = "aic" @@ -582,7 +601,7 @@ def _materialize(self, item: _Retained) -> SnapshotCandidate: _RenderInput(config=item.parameters, used_gpus=item.used_gpus), self._workload, self._options, - dgd_name=f"{self._config.name}-{item.id.removeprefix(_ID_PREFIX)}", + dgd_name=_dgd_name(self._config.name, item.id), renderer=self._config.generator, ) entry = SnapshotCandidate( diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index deccceeb250b..90e0c4d7dff2 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -219,6 +219,59 @@ def test_identity_depends_only_on_the_resolved_parameters() -> None: assert candidate_id_for({"a": 1}).startswith("evaluated-point-") +def test_aic_name_prefix_is_stable_bounded_and_candidate_specific( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + names: list[str] = [] + + def capture_name(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: + del candidate, workload, options + assert kwargs["renderer"] == "aic" + names.append(kwargs["dgd_name"]) + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_name) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, name="a" * 63, max_candidates=2), workload=None + ) + adapter.on_candidate(record("first", 2.0)) + adapter.on_candidate(record("second", 1.0)) + adapter.close() + + assert len(names) == 2 + assert len(set(names)) == 2 + assert all(len(name) <= 28 for name in names) + assert all(name[0].isalnum() and name[-1].isalnum() for name in names) + assert names[0] == adapter_module._dgd_name( + "a" * 63, candidate_id_for(record("first", 2.0).config) + ) + + +def test_aic_name_prefix_accepts_and_normalizes_a_dotted_run_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + names: list[str] = [] + + def capture_name(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: + del candidate, workload, options + names.append(kwargs["dgd_name"]) + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_name) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, name="team.model", max_candidates=1), workload=None + ) + adapter.on_candidate(record("first", 2.0)) + adapter.close() + + assert len(names) == 1 + assert "." not in names[0] + assert len(names[0]) <= 28 + assert names[0] == adapter_module._dgd_name( + "team.model", candidate_id_for(record("first", 2.0).config) + ) + + def test_a_new_better_point_reorders_but_does_not_rematerialize_retained_ones( tmp_path: Path, renders: list[str] ) -> None: From a9b5a91724b1efd2ea9b41603d1e3ce9dafcd296 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 10:18:32 +0100 Subject: [PATCH 032/100] pr-15895: pr-7: require AISimulate lifecycle callbacks [upstream: devivasudevan/dynamo#7] Signed-off-by: Dr. Stefan Schimanski --- .../aisimulate/output/dgdr_run/adapter.py | 12 +----------- .../aisimulate/tests/dgdr_run/test_adapter.py | 19 ------------------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index 2c3f9b9e155a..5965741614c7 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -49,7 +49,7 @@ import threading import time from collections.abc import Mapping, Sequence -from dataclasses import dataclass, fields +from dataclasses import dataclass from pathlib import Path from typing import Any, Literal, NoReturn, Protocol @@ -664,16 +664,6 @@ def subscribe( workload = getattr(context, "workload", None) if workload is None or resolved.snapshot_dir is None: return None - # Live publishing needs both lifecycle hooks: without on_complete nothing stops - # the live writer before write() publishes the terminal snapshot, which a pending - # live write could then overwrite. Older AISimulate releases lack them; the - # terminal snapshot then comes from write() alone. - supported = {field.name for field in fields(RecommendationOutputCallbacks)} - if not {"on_complete", "on_failure"} <= supported: - logger.info( - "dgdr_run: AISimulate has no lifecycle callbacks; not publishing live" - ) - return None adapter = DGDRRunOutputAdapter(resolved, workload) adapter.start() return RecommendationOutputCallbacks( diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index 90e0c4d7dff2..4f5e3b76a0c6 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -954,25 +954,6 @@ def failing(*_: Any, **__: Any) -> str: assert 0 < len(candidate["error"]) <= 500 -def test_subscribe_does_not_publish_live_without_lifecycle_callbacks( - tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch -) -> None: - @dataclass - class LegacyCallbacks: - on_candidate: Any = None - on_round: Any = None - - monkeypatch.setattr( - adapter_module, "RecommendationOutputCallbacks", LegacyCallbacks - ) - callbacks = create_adapter().subscribe( - {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, - context=SimpleNamespace(workload="workload"), - ) - assert callbacks is None - assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() - - def test_a_persistent_write_failure_does_not_spin_the_writer( tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch ) -> None: From 5e12670e3837418c10c7554e211da24e08affabd Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:36:38 +0100 Subject: [PATCH 033/100] pr-15894: squash DGDR Run publisher sidecar Squashed from https://github.com/ai-dynamo/dynamo/pull/15894. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- deploy/operator/Makefile | 2 +- .../operator/cmd/dgdr-run-publisher/main.go | 81 ++ .../operator/internal/dgdcreconcile/diff.go | 120 +++ .../internal/dgdcreconcile/diff_test.go | 123 +++ .../operator/internal/dgdrrunpublisher/doc.go | 66 ++ .../internal/dgdrrunpublisher/input.go | 194 ++++ .../internal/dgdrrunpublisher/input_test.go | 179 ++++ .../internal/dgdrrunpublisher/manager.go | 221 ++++ .../internal/dgdrrunpublisher/outcome.go | 131 +++ .../internal/dgdrrunpublisher/outcome_test.go | 87 ++ .../internal/dgdrrunpublisher/plan.go | 167 +++ .../internal/dgdrrunpublisher/plan_test.go | 141 +++ .../internal/dgdrrunpublisher/reconciler.go | 368 +++++++ .../dgdrrunpublisher/reconciler_test.go | 990 ++++++++++++++++++ .../internal/dgdrrunpublisher/snapshot.go | 201 ++++ .../dgdrrunpublisher/snapshot_test.go | 174 +++ .../internal/dgdrrunpublisher/terminal.go | 115 ++ .../dgdrrunpublisher/terminal_test.go | 142 +++ .../internal/dgdrrunpublisher/writer.go | 268 +++++ .../dgdrrunpublisher/writer_envtest_test.go | 559 ++++++++++ 20 files changed, 4328 insertions(+), 1 deletion(-) create mode 100644 deploy/operator/cmd/dgdr-run-publisher/main.go create mode 100644 deploy/operator/internal/dgdcreconcile/diff.go create mode 100644 deploy/operator/internal/dgdcreconcile/diff_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/doc.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/input.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/input_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/manager.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/outcome.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/outcome_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/plan.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/plan_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/reconciler.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/reconciler_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/snapshot.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/snapshot_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/terminal.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/terminal_test.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/writer.go create mode 100644 deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 610ec5b55412..372b6479b263 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -4,7 +4,7 @@ IMG ?= controller:latest # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.30.0 -ENVTEST_PACKAGES ?= ./internal/controller/... ./internal/webhook/defaulting ./internal/webhook/validation ./cmd/crd-apply +ENVTEST_PACKAGES ?= ./internal/controller/... ./internal/webhook/defaulting ./internal/webhook/validation ./cmd/crd-apply ./internal/dgdrrunpublisher ENVTEST_ASSETS_CMD = $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path CLUSTERTEST_PACKAGES ?= ./internal/testing/clusterenv ./internal/controller/... diff --git a/deploy/operator/cmd/dgdr-run-publisher/main.go b/deploy/operator/cmd/dgdr-run-publisher/main.go new file mode 100644 index 000000000000..6fbdaf03a34d --- /dev/null +++ b/deploy/operator/cmd/dgdr-run-publisher/main.go @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Command dgdr-run-publisher is the sidecar of a DGDR(v2) run Job. It is a +// controller-runtime controller for one DynamoGraphDeploymentRun: it observes the +// Sweeper's pod-local snapshot file and reconciles it into DynamoGraphDeploymentCandidates +// and DynamoGraphDeploymentRun.status. See internal/dgdrrunpublisher. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func main() { + os.Exit(run()) +} + +func run() int { + var cfg dgdrrunpublisher.Config + var terminationLog string + logOptions := zap.Options{} + logOptions.BindFlags(flag.CommandLine) + flag.StringVar(&cfg.SnapshotDir, "snapshot-dir", "", "directory shared with the Sweeper container") + flag.StringVar(&cfg.RunName, "run-name", "", "name of the DynamoGraphDeploymentRun") + flag.StringVar(&cfg.RunUID, "run-uid", "", "UID of the DynamoGraphDeploymentRun this pod serves") + flag.StringVar(&cfg.SweeperContainer, "sweeper-container", "sweeper", "name of the Sweeper container in this pod") + flag.DurationVar(&cfg.MinPollInterval, "poll-interval", time.Second, "snapshot check interval after a change") + flag.DurationVar( + &cfg.MaxPollInterval, "max-poll-interval", 5*time.Second, + "longest snapshot check interval while the snapshot is unchanged", + ) + flag.StringVar( + &terminationLog, "termination-log", dgdrrunpublisher.DefaultTerminationLog, + "file the failure outcome is written to as the container termination message", + ) + flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&logOptions))) + + cfg.Namespace, cfg.PodName = os.Getenv("POD_NAMESPACE"), os.Getenv("POD_NAME") + if cfg.SnapshotDir == "" || cfg.RunName == "" || cfg.RunUID == "" || cfg.Namespace == "" || cfg.PodName == "" { + fmt.Fprintln(os.Stderr, "--snapshot-dir, --run-name, --run-uid and the POD_NAMESPACE/POD_NAME "+ + "environment variables are required") + return dgdrrunpublisher.ExitReconcileFailed + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + err := dgdrrunpublisher.Run(ctx, ctrl.GetConfigOrDie(), cfg) + if err != nil { + ctrl.Log.WithName("dgdr-run-publisher").Error(err, "publisher stopped") + if writeErr := dgdrrunpublisher.WriteTerminationMessage(terminationLog, err); writeErr != nil { + ctrl.Log.WithName("dgdr-run-publisher").Error(writeErr, "writing termination message") + } + } + return dgdrrunpublisher.ExitCode(err) +} diff --git a/deploy/operator/internal/dgdcreconcile/diff.go b/deploy/operator/internal/dgdcreconcile/diff.go new file mode 100644 index 000000000000..b9de85216190 --- /dev/null +++ b/deploy/operator/internal/dgdcreconcile/diff.go @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package dgdcreconcile computes the create/delete diff between the candidates a run +// snapshot wants to exist and the DynamoGraphDeploymentCandidates (DGDCs) observed in +// the cluster. +// +// Identity is the stable candidate id carried by the snapshot, which depends only on +// the evaluated point -- never on rank or list position. A DGDC is immutable once its +// initial status is populated, so an identity present on both sides needs no action: +// there is deliberately no "update" action. Reordering candidates is expressed only +// through the ordered candidate references on the run status, which this package does +// not touch. +// +// The package is pure (no Kubernetes types, no I/O) so the diff can be tested on its +// own; callers fetch current state and apply the returned Actions. +package dgdcreconcile + +import "fmt" + +// DiffInputError means the desired or current set was malformed (e.g. a duplicate or +// empty identity). Retrying will not fix it, so callers treat it as terminal. +type DiffInputError struct { + msg string +} + +func (e *DiffInputError) Error() string { return e.msg } + +func newDiffInputError(format string, args ...any) *DiffInputError { + return &DiffInputError{msg: fmt.Sprintf(format, args...)} +} + +type DesiredCandidate struct { + // ID is the stable candidate identity from the snapshot. + ID string + // Spec is the DGD manifest (YAML) to materialize. + Spec string + // Parameters and Metrics are the immutable evaluation facts recorded in the + // DGDC status at creation. + Parameters map[string]any + Metrics map[string]any +} + +// CurrentDGDC is one DGDC observed in the cluster. ID is read back from the label +// recorded at creation, never recomputed. +type CurrentDGDC struct { + Name string + ID string + // Incomplete means the DGDC exists but its initial status was never populated (for + // example the process stopped between creating it and writing its status). Its spec + // is immutable, so only the status is completed. + Incomplete bool +} + +// Actions is what the caller must apply. There are no updates: DGDCs are immutable. +type Actions struct { + // Creates are desired candidates with no DGDC yet. + Creates []DesiredCandidate + // Incomplete are desired candidates whose DGDC exists but whose initial status was + // never populated. The caller finishes the status (never the spec) before the + // candidate is referenced from the run. + Incomplete []DesiredCandidate + Deletes []string // DGDC names +} + +// ComputeActions diffs desired against current. Output order follows the input order +// so results are deterministic. +func ComputeActions(desired []DesiredCandidate, current []CurrentDGDC) (Actions, error) { + desiredIDs := make(map[string]struct{}, len(desired)) + for _, candidate := range desired { + if candidate.ID == "" { + return Actions{}, newDiffInputError("desired candidate has an empty id") + } + if _, dup := desiredIDs[candidate.ID]; dup { + return Actions{}, newDiffInputError("duplicate id in desired set: %s", candidate.ID) + } + desiredIDs[candidate.ID] = struct{}{} + } + currentIDs := make(map[string]struct{}, len(current)) + incomplete := make(map[string]bool, len(current)) + for _, existing := range current { + if existing.ID == "" { + return Actions{}, newDiffInputError("DGDC %s has no candidate id", existing.Name) + } + if _, dup := currentIDs[existing.ID]; dup { + return Actions{}, newDiffInputError("duplicate id in current set: %s", existing.ID) + } + currentIDs[existing.ID] = struct{}{} + incomplete[existing.ID] = existing.Incomplete + } + + var actions Actions + for _, candidate := range desired { + if _, exists := currentIDs[candidate.ID]; !exists { + actions.Creates = append(actions.Creates, candidate) + } else if incomplete[candidate.ID] { + actions.Incomplete = append(actions.Incomplete, candidate) + } + } + for _, existing := range current { + if _, wanted := desiredIDs[existing.ID]; !wanted { + actions.Deletes = append(actions.Deletes, existing.Name) + } + } + return actions, nil +} diff --git a/deploy/operator/internal/dgdcreconcile/diff_test.go b/deploy/operator/internal/dgdcreconcile/diff_test.go new file mode 100644 index 000000000000..3d0f155e2173 --- /dev/null +++ b/deploy/operator/internal/dgdcreconcile/diff_test.go @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdcreconcile + +import ( + "errors" + "reflect" + "testing" +) + +func d(ids ...string) []DesiredCandidate { + out := make([]DesiredCandidate, 0, len(ids)) + for _, id := range ids { + out = append(out, DesiredCandidate{ID: id, Spec: "spec-" + id}) + } + return out +} + +func c(pairs ...string) []CurrentDGDC { + var out []CurrentDGDC + for i := 0; i+1 < len(pairs); i += 2 { + out = append(out, CurrentDGDC{Name: pairs[i], ID: pairs[i+1]}) + } + return out +} + +func createIDs(a Actions) []string { + var ids []string + for _, x := range a.Creates { + ids = append(ids, x.ID) + } + return ids +} + +func TestComputeActions(t *testing.T) { + tests := []struct { + name string + desired []DesiredCandidate + current []CurrentDGDC + wantCreates []string + wantIncomplete []string + wantDeletes []string + }{ + {"empty both", nil, nil, nil, nil, nil}, + {"all new", d("a", "b"), nil, []string{"a", "b"}, nil, nil}, + {"all gone", nil, c("n-a", "a", "n-b", "b"), nil, nil, []string{"n-a", "n-b"}}, + {"steady state needs no action", d("a", "b"), c("n-a", "a", "n-b", "b"), nil, nil, nil}, + {"reordering desired needs no action", d("b", "a"), c("n-a", "a", "n-b", "b"), nil, nil, nil}, + {"incomplete DGDC gets its status finished, not a new spec", d("a", "b"), []CurrentDGDC{{Name: "n-a", ID: "a"}, {Name: "n-b", ID: "b", Incomplete: true}}, nil, []string{"b"}, nil}, + {"mixed", d("a", "c"), c("n-a", "a", "n-b", "b"), []string{"c"}, nil, []string{"n-b"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := ComputeActions(tc.desired, tc.current) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(createIDs(got), tc.wantCreates) { + t.Errorf("creates = %v, want %v", createIDs(got), tc.wantCreates) + } + var gotIncomplete []string + for _, x := range got.Incomplete { + gotIncomplete = append(gotIncomplete, x.ID) + } + if !reflect.DeepEqual(gotIncomplete, tc.wantIncomplete) { + t.Errorf("incomplete = %v, want %v", gotIncomplete, tc.wantIncomplete) + } + if !reflect.DeepEqual(got.Deletes, tc.wantDeletes) { + t.Errorf("deletes = %v, want %v", got.Deletes, tc.wantDeletes) + } + }) + } +} + +func TestComputeActionsInputErrors(t *testing.T) { + tests := []struct { + name string + desired []DesiredCandidate + current []CurrentDGDC + }{ + {"duplicate desired", d("a", "a"), nil}, + {"empty desired id", d(""), nil}, + {"duplicate current", nil, c("n1", "a", "n2", "a")}, + {"empty current id", nil, c("n1", "")}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := ComputeActions(tc.desired, tc.current) + var inputErr *DiffInputError + if !errors.As(err, &inputErr) { + t.Fatalf("want *DiffInputError, got %v", err) + } + }) + } +} + +func TestComputeActionsDeterministicOrder(t *testing.T) { + got, err := ComputeActions(d("z", "m", "a"), c("n-q", "q", "n-b", "b")) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(createIDs(got), []string{"z", "m", "a"}) { + t.Errorf("creates order = %v", createIDs(got)) + } + if !reflect.DeepEqual(got.Deletes, []string{"n-q", "n-b"}) { + t.Errorf("deletes order = %v", got.Deletes) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/doc.go b/deploy/operator/internal/dgdrrunpublisher/doc.go new file mode 100644 index 000000000000..76a8b1affe08 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/doc.go @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package dgdrrunpublisher is the sidecar that turns the Sweeper's pod-local desired +// state snapshot into Kubernetes state: it creates/deletes DynamoGraphDeploymentCandidates +// and updates the ordered candidate references and progress on +// DynamoGraphDeploymentRun.status. +// +// It is a controller-runtime controller for exactly one DynamoGraphDeploymentRun, with +// two loops that meet at one workqueue key (the Run's namespace/name): +// +// 1. The input observer (Observer) checks the snapshot file the Sweeper writes into a +// shared emptyDir. On a semantic change it atomically stores the new desired state +// (InputStore) and enqueues the key. Unchanged input enqueues nothing and backs the +// check interval off; a change resets it. +// 2. The Reconciler consumes the key. Informer events for the Run, its candidates and +// this Pod enqueue the same key. It reads the latest stored input and the cached +// cluster state, then creates missing candidates with server-side apply, completes +// candidate status (its metrics), updates the Run's published-candidate count, progress +// time and ordered references only where they differ, and finally deletes obsolete candidates. Failed writes are +// retried by the workqueue with backoff. +// +// The publisher's own writes come back as watch events and the cache can trail them. The +// Reconciler keeps no record of what it wrote: it decides from the current observations, +// constrains its status writes to the resourceVersion it observed (a stale decision +// conflicts and is retried through the workqueue); see Reconciler. +// +// The manager's cache covers this Run and Pod by name and Candidates in their namespace. +// Candidate membership is determined from controller ownership rather than mutable +// labels, so there is no API-server polling and no spec comparison: candidate specs are +// immutable once created. +// +// The sidecar is a regular container of the same Job pod as the Sweeper. The Job only +// completes once the publisher has exited, so a successful exit acknowledges that the +// final snapshot was reconciled. The run controller derives the terminal Completed +// condition from the Job result; the publisher never writes a phase. On a failure exit it +// also writes the bounded terminal phase, message and error as JSON to the container +// termination log (/dev/termination-log), which the run controller reads from the +// publisher container's terminated state; see WriteTerminationMessage. +// +// Termination (see Reconciler): +// +// terminal Succeeded snapshot reconciled -> exit 0 (it must hold a materialized +// candidate, otherwise ExitRunFailed) +// terminal Failed snapshot reconciled -> exit ExitRunFailed +// Sweeper exited non-zero, no terminal snapshot -> reconcile last snapshot, exit 0 +// (the Sweeper's exit fails the Job) +// Sweeper exited zero, no terminal snapshot -> reconcile last snapshot, +// exit ExitProtocolViolation +// invalid or regressing snapshot, Sweeper exited -> exit ExitProtocolViolation +// final/last snapshot cannot be reconciled -> exit ExitReconcileFailed +package dgdrrunpublisher diff --git a/deploy/operator/internal/dgdrrunpublisher/input.go b/deploy/operator/internal/dgdrrunpublisher/input.go new file mode 100644 index 000000000000..9d3183638a8a --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/input.go @@ -0,0 +1,194 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "os" + "reflect" + "sync" + "time" +) + +// Input is the latest accepted desired state plus the most recent problem reading the +// input. Snapshot is never mutated after it is stored, so readers need no copy. +type Input struct { + // Snapshot is the last valid snapshot, or nil if none has been accepted yet. + Snapshot *Snapshot + // Err is why the most recent read failed (invalid or regressing snapshot, or an I/O + // error). It is cleared by the next valid read. + Err error +} + +// InputStore atomically holds the latest desired state for the reconciler. +type InputStore struct { + mu sync.RWMutex + input Input +} + +// Load returns the current input. +func (s *InputStore) Load() Input { + s.mu.RLock() + defer s.mu.RUnlock() + return s.input +} + +// accept stores snap as the latest desired state if it differs semantically from the +// current one, and reports whether it did. A snapshot that moves progress backwards is +// rejected and recorded as the input error; the last valid snapshot stays in place. +func (s *InputStore) accept(snap *Snapshot) (changed bool) { + s.mu.Lock() + defer s.mu.Unlock() + if err := checkProgress(s.input.Snapshot, snap); err != nil { + s.input.Err = err + return false + } + s.input.Err = nil + if s.input.Snapshot != nil && sameContent(s.input.Snapshot, snap) { + return false + } + s.input.Snapshot = snap + return true +} + +// fail records why the input could not be read, keeping the last valid snapshot. +func (s *InputStore) fail(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.input.Err = err +} + +// sameContent compares two snapshots ignoring the write timestamp: rewriting identical +// content with a new timestamp is not a change the cluster needs to hear about. +func sameContent(a, b *Snapshot) bool { + left, right := *a, *b + left.Timestamp, right.Timestamp = "", "" + return reflect.DeepEqual(left, right) +} + +// checkProgress rejects a snapshot that moves progress backwards relative to the +// previously accepted one: an older round, or fewer evaluated points within the same +// round. A terminal snapshot is exempt from the evaluated check because its count is the +// search's final total, not the live counter. +func checkProgress(previous, next *Snapshot) error { + if previous == nil { + return nil + } + if next.Progress.Round < previous.Progress.Round { + return violation("snapshot round regressed from %d to %d", previous.Progress.Round, next.Progress.Round) + } + if !next.Run.Terminal && next.Progress.Round == previous.Progress.Round && next.Progress.Evaluated < previous.Progress.Evaluated { + return violation("snapshot evaluated count regressed from %d to %d in round %d", previous.Progress.Evaluated, next.Progress.Evaluated, previous.Progress.Round) + } + return nil +} + +// Observer is the input-observation loop: it checks the Sweeper's snapshot file and, on a +// semantic change, stores it and enqueues the run key. It never talks to the API server. +type Observer struct { + Store *InputStore + Path string + // OnError reports a problem reading the snapshot, once per distinct message. It is + // optional; the manager wires it to the controller-runtime logger. + OnError func(err error) + // Enqueue asks the controller to reconcile the run. It must not block. + Enqueue func() + // MinInterval is the check interval after a change; it doubles while the input is + // unchanged, up to MaxInterval. + MinInterval time.Duration + MaxInterval time.Duration + + mu sync.Mutex // serializes Check between the loop and the reconciler + lastErr string +} + +// Check reads the snapshot once and reports whether it was a semantic change (in which +// case it has been stored and the run key enqueued). A missing file is not an error. The +// reconciler also calls it directly once the Sweeper has exited, to be certain the +// Sweeper's final write has been seen. +func (o *Observer) Check() bool { + o.mu.Lock() + defer o.mu.Unlock() + + data, err := os.ReadFile(o.Path) + if errors.Is(err, os.ErrNotExist) { + return false + } + if err != nil { + o.noteError(err) + o.Store.fail(err) + return false + } + snap, err := ParseSnapshot(data) + if err != nil { + o.noteError(err) + o.Store.fail(err) + return false + } + changed := o.Store.accept(snap) + if err := o.Store.Load().Err; err != nil { + o.noteError(err) + return false + } + o.lastErr = "" + if changed && o.Enqueue != nil { + o.Enqueue() + } + return changed +} + +// noteError logs a problem once per distinct message rather than on every check. +func (o *Observer) noteError(err error) { + if err.Error() == o.lastErr { + return + } + o.lastErr = err.Error() + if o.OnError != nil { + o.OnError(err) + } +} + +// NextInterval is the delay before the next check: it resets to min after a change and +// otherwise doubles up to max. +func NextInterval(current, minimum, maximum time.Duration, changed bool) time.Duration { + if changed || current < minimum { + return minimum + } + next := current * 2 + if next > maximum { + return maximum + } + return next +} + +// Run checks the snapshot until ctx is done. +func (o *Observer) Run(ctx context.Context) error { + interval := o.MinInterval + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-timer.C: + } + interval = NextInterval(interval, o.MinInterval, o.MaxInterval, o.Check()) + timer.Reset(interval) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/input_test.go b/deploy/operator/internal/dgdrrunpublisher/input_test.go new file mode 100644 index 000000000000..8b13734c92fa --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/input_test.go @@ -0,0 +1,179 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +func newObserver(t *testing.T) (*Observer, *InputStore, string, *int) { + t.Helper() + dir := t.TempDir() + enqueued := 0 + store := &InputStore{} + return &Observer{ + Store: store, + Path: filepath.Join(dir, SnapshotFileName), + Enqueue: func() { enqueued++ }, + MinInterval: time.Millisecond, + MaxInterval: 4 * time.Millisecond, + }, store, dir, &enqueued +} + +func TestMissingSnapshotIsNotAChange(t *testing.T) { + o, store, _, enqueued := newObserver(t) + if o.Check() || *enqueued != 0 || store.Load().Snapshot != nil || store.Load().Err != nil { + t.Fatalf("missing file: input=%+v enqueued=%d", store.Load(), *enqueued) + } +} + +func TestChangedSnapshotIsStoredAndEnqueuedOnce(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if !o.Check() || *enqueued != 1 { + t.Fatalf("first snapshot: enqueued=%d", *enqueued) + } + if store.Load().Snapshot == nil || store.Load().Snapshot.Candidates[0].ID != "a" { + t.Fatalf("snapshot not stored: %+v", store.Load()) + } + if o.Check() || *enqueued != 1 { + t.Fatalf("unchanged input must not enqueue again: enqueued=%d", *enqueued) + } + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if !o.Check() || *enqueued != 2 { + t.Fatalf("changed input must enqueue: enqueued=%d", *enqueued) + } +} + +func TestRewriteWithOnlyANewTimestampIsNotAChange(t *testing.T) { + o, _, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) + o.Check() + writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:14Z")) + if o.Check() || *enqueued != 1 { + t.Fatalf("a new timestamp alone is not a semantic change: enqueued=%d", *enqueued) + } +} + +func TestInvalidSnapshotKeepsTheLastValidOneAndIsRecorded(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + o.Check() + writeSnapshot(t, dir, []byte(`{"schemaVersion":2}`)) + if o.Check() || *enqueued != 1 { + t.Fatalf("an invalid snapshot must not enqueue: enqueued=%d", *enqueued) + } + in := store.Load() + if !errors.Is(in.Err, ErrProtocolViolation) || in.Snapshot == nil || in.Snapshot.Candidates[0].ID != "a" { + t.Fatalf("input = %+v", in) + } + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if !o.Check() || store.Load().Err != nil { + t.Fatalf("a valid snapshot must clear the error: %+v", store.Load()) + } +} + +func TestRegressingSnapshotIsRejected(t *testing.T) { + for name, regressed := range map[string][2]int{"older round": {2, 99}, "fewer evaluated in the same round": {3, 4}} { + t.Run(name, func(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + writeProgress(t, dir, PhaseRunning, 3, 5) + o.Check() + writeProgress(t, dir, PhaseRunning, regressed[0], regressed[1]) + if o.Check() || *enqueued != 1 { + t.Fatalf("a regressing snapshot must not be stored: enqueued=%d", *enqueued) + } + in := store.Load() + if !errors.Is(in.Err, ErrProtocolViolation) || ExitCode(in.Err) != ExitProtocolViolation { + t.Fatalf("err = %v", in.Err) + } + if in.Snapshot.Progress.Round != 3 || in.Snapshot.Progress.Evaluated != 5 { + t.Fatalf("the last valid snapshot must stay: %+v", in.Snapshot.Progress) + } + }) + } +} + +func TestTerminalSnapshotMayReportItsFinalEvaluatedTotal(t *testing.T) { + o, store, dir, _ := newObserver(t) + writeProgress(t, dir, PhaseRunning, 3, 9) + o.Check() + writeProgress(t, dir, PhaseSucceeded, 3, 7) + if !o.Check() || store.Load().Err != nil || !store.Load().Snapshot.Run.Terminal { + t.Fatalf("terminal snapshot rejected: %+v", store.Load()) + } +} + +func TestUnreadableSnapshotIsRecordedAsANonViolation(t *testing.T) { + o, store, dir, _ := newObserver(t) + if err := os.Mkdir(filepath.Join(dir, SnapshotFileName), 0o700); err != nil { // reading a directory fails + t.Fatal(err) + } + o.Check() + if err := store.Load().Err; err == nil || errors.Is(err, ErrProtocolViolation) { + t.Fatalf("err = %v, want a plain read error", err) + } +} + +func TestNextIntervalBacksOffWhileUnchangedAndResetsOnChange(t *testing.T) { + minimum, maximum := time.Second, 5*time.Second + got := []time.Duration{} + interval := minimum + for i := 0; i < 5; i++ { + interval = NextInterval(interval, minimum, maximum, false) + got = append(got, interval) + } + want := []time.Duration{2 * time.Second, 4 * time.Second, 5 * time.Second, 5 * time.Second, 5 * time.Second} + for i := range want { + if got[i] != want[i] { + t.Fatalf("intervals = %v, want %v", got, want) + } + } + if NextInterval(5*time.Second, minimum, maximum, true) != minimum { + t.Fatal("a change must reset the interval") + } +} + +func TestObserverRunEnqueuesAChangeAndStopsWithItsContext(t *testing.T) { + o, _, dir, _ := newObserver(t) + events := make(chan struct{}, 4) + o.Enqueue = func() { events <- struct{}{} } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- o.Run(ctx) }() + writeSnapshot(t, dir, snapshotJSON(t, PhaseRunning, cand{id: "a"})) + select { + case <-events: + case <-time.After(5 * time.Second): + t.Fatal("the observer never reported the snapshot") + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Run did not stop") + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/manager.go b/deploy/operator/internal/dgdrrunpublisher/manager.go new file mode 100644 index 000000000000..e7f33b047ceb --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/manager.go @@ -0,0 +1,221 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" + + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// Config is everything one publisher needs to know about its run and pod. +type Config struct { + Namespace string + RunName string + // RunUID is the UID of the run this pod serves. The run controller knows it when it + // builds the Job; the publisher uses it to establish Candidate ownership. + RunUID string + PodName string + SweeperContainer string + SnapshotDir string + // MinPollInterval and MaxPollInterval bound the snapshot check interval, which backs + // off from the minimum to the maximum while the input is unchanged. + MinPollInterval time.Duration + MaxPollInterval time.Duration +} + +func (c Config) validate() error { + switch { + case c.Namespace == "", c.RunName == "", c.RunUID == "", c.PodName == "", c.SnapshotDir == "": + return errors.New("namespace, run name, run uid, pod name and snapshot dir are required") + case c.MinPollInterval <= 0 || c.MaxPollInterval < c.MinPollInterval: + return errors.New("poll intervals must be positive and the maximum at least the minimum") + } + return nil +} + +// NewScheme registers every type the publisher reads or writes. +func NewScheme() (*runtime.Scheme, error) { + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{clientgoscheme.AddToScheme, v1beta1.AddToScheme, v1beta2.AddToScheme} { + if err := add(scheme); err != nil { + return nil, err + } + } + return scheme, nil +} + +// cacheOptions limits the manager's informers to one namespace and selects this Run and +// Pod by name. Candidates remain namespace-scoped because their labels are mutable; +// reconciliation establishes membership from their controller owner reference. +func cacheOptions(c Config) cache.Options { + return cache.Options{ + DefaultNamespaces: map[string]cache.Config{c.Namespace: {}}, + ByObject: map[client.Object]cache.ByObject{ + &v1beta2.DynamoGraphDeploymentRun{}: { + Field: fields.OneTermEqualSelector("metadata.name", c.RunName), + }, + &v1beta2.DynamoGraphDeploymentCandidate{}: {}, + &corev1.Pod{}: { + Field: fields.OneTermEqualSelector("metadata.name", c.PodName), + }, + }, + } +} + +// Run starts the controller manager and returns the publisher's outcome (see the package +// comment for its meaning and ExitCode for the exit code), or ctx's error if the process +// is told to stop first. +func Run(ctx context.Context, restConfig *rest.Config, cfg Config) error { + if err := cfg.validate(); err != nil { + return err + } + scheme, err := NewScheme() + if err != nil { + return fmt.Errorf("building scheme: %w", err) + } + shutdownTimeout := 5 * time.Second + mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ + Scheme: scheme, + Cache: cacheOptions(cfg), + Metrics: metricsserver.Options{BindAddress: "0"}, + LeaderElection: false, + GracefulShutdownTimeout: &shutdownTimeout, + }) + if err != nil { + return fmt.Errorf("creating manager: %w", err) + } + + key := types.NamespacedName{Namespace: cfg.Namespace, Name: cfg.RunName} + events := make(chan event.GenericEvent, 1) + store := &InputStore{} + observer := &Observer{ + Store: store, + Path: SnapshotPath(cfg.SnapshotDir), + MinInterval: cfg.MinPollInterval, + MaxInterval: cfg.MaxPollInterval, + OnError: func(err error) { + ctrl.Log.WithName("input-observer").Error(err, "reading snapshot") + }, + Enqueue: func() { + // One pending event is enough: the workqueue deduplicates by key anyway. + select { + case events <- event.GenericEvent{Object: runKeyObject(key)}: + default: + } + }, + } + completion := NewCompletion() + reconciler := &Reconciler{ + Client: mgr.GetClient(), + Writer: &clientWriter{ + Client: mgr.GetClient(), + Reader: mgr.GetAPIReader(), + Namespace: cfg.Namespace, + RunName: cfg.RunName, + RunUID: cfg.RunUID, + }, + Namespace: cfg.Namespace, + RunName: cfg.RunName, + RunUID: cfg.RunUID, + PodName: cfg.PodName, + SweeperContainer: cfg.SweeperContainer, + Observer: observer, + Store: store, + Completion: completion, + } + + if err := setupController(mgr, reconciler, key, events); err != nil { + return fmt.Errorf("building controller: %w", err) + } + if err := mgr.Add(manager.RunnableFunc(observer.Run)); err != nil { + return fmt.Errorf("adding input observer: %w", err) + } + + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + select { + case <-completion.Done(): + cancel() // the outcome is known: stop the manager and exit + case <-runCtx.Done(): + } + }() + startErr := mgr.Start(runCtx) + + if completion.Finished() { + return completion.Err() + } + if ctx.Err() != nil { + return ctx.Err() + } + if startErr != nil { + return fmt.Errorf("manager stopped: %w", startErr) + } + return errors.New("manager stopped before the run finished") +} + +// setupController centralizes the production event wiring: every Kubernetes and +// local-input event reaches the one Run key. +func setupController(mgr manager.Manager, reconciler reconcile.Reconciler, key types.NamespacedName, events <-chan event.GenericEvent) error { + podToRun := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { + return []reconcile.Request{{NamespacedName: key}} + }) + return ctrl.NewControllerManagedBy(mgr). + Named("dgdr-run-publisher"). + For(&v1beta2.DynamoGraphDeploymentRun{}). + Owns(&v1beta2.DynamoGraphDeploymentCandidate{}). + Watches(&corev1.Pod{}, podToRun). + WatchesRawSource(source.Channel(events, &handler.EnqueueRequestForObject{})). + WithOptions(controller.Options{MaxConcurrentReconciles: 1}). + Complete(reconciler) +} + +// SnapshotPath is where the Sweeper writes the snapshot inside the shared directory. +func SnapshotPath(dir string) string { + return filepath.Join(dir, SnapshotFileName) +} + +// runKeyObject is the object carried by enqueue events; only its name and namespace +// matter to the handler. +func runKeyObject(key types.NamespacedName) client.Object { + run := &v1beta2.DynamoGraphDeploymentRun{} + run.Namespace, run.Name = key.Namespace, key.Name + return run +} diff --git a/deploy/operator/internal/dgdrrunpublisher/outcome.go b/deploy/operator/internal/dgdrrunpublisher/outcome.go new file mode 100644 index 000000000000..408f5aa221d6 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/outcome.go @@ -0,0 +1,131 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "errors" + "fmt" + "sync" +) + +// Process exit codes of the publisher binary. +const ( + ExitAcknowledged = 0 + ExitReconcileFailed = 1 + ExitProtocolViolation = 3 + ExitRunFailed = 4 +) + +// ErrMissingTerminalSnapshot means the Sweeper exited successfully without publishing a +// terminal snapshot, which violates the producer protocol. +var ErrMissingTerminalSnapshot = fmt.Errorf("%w: sweeper exited without publishing a terminal snapshot", ErrProtocolViolation) + +// ErrRunFailed means the Sweeper reported failure or produced no usable recommendation. +// The snapshot is reconciled first; the error keeps that outcome from letting the Job +// complete successfully. The concrete error is a *RunFailedError carrying the bounded +// diagnostics. +var ErrRunFailed = errors.New("sweeper reported a failed run") + +// ExitCode maps the result of Run to the process exit code. +func ExitCode(err error) int { + switch { + case err == nil: + return ExitAcknowledged + case errors.Is(err, ErrProtocolViolation): + return ExitProtocolViolation + case errors.Is(err, ErrRunFailed): + return ExitRunFailed + default: + return ExitReconcileFailed + } +} + +// SweeperState is the observed state of the Sweeper container in this pod. +type SweeperState struct { + Exited bool + ExitCode int32 +} + +// terminalResult is the outcome once a terminal snapshot has been reconciled. A Failed +// snapshot fails the run with the producer's own (bounded) diagnostics. A Succeeded one +// must also contain a materialized candidate: success with nothing to recommend is a +// failed run, not a result. The error is a *RunFailedError, so errors.Is(err, +// ErrRunFailed) and ExitCode are unchanged. +func terminalResult(desired DesiredState) error { + if desired.Phase == PhaseSucceeded && len(desired.Candidates) > 0 { + return nil + } + outcome := TerminalOutcome{Phase: PhaseFailed, Message: desired.Message, Error: desired.Error} + if desired.Phase == PhaseSucceeded { + outcome.Message = "no candidate was materialized" + if desired.Failed > 0 { + noun := "failures" + if desired.Failed == 1 { + noun = "failure" + } + outcome.Message += fmt.Sprintf(": %d candidate materialization %s", desired.Failed, noun) + } + outcome.Error = desired.FirstFailure + } + return &RunFailedError{Outcome: outcome} +} + +// exitResult is the outcome when the Sweeper exited and the last snapshot it wrote (if +// any) has been reconciled, but that snapshot is not terminal. +func exitResult(state SweeperState) error { + if state.ExitCode != 0 { + // Crash or caught failure without a terminal snapshot: the last-known projection + // is reconciled; the Sweeper's own exit code fails the Job. + return nil + } + return ErrMissingTerminalSnapshot +} + +// Completion records the publisher's outcome exactly once and wakes whoever waits on it. +type Completion struct { + once sync.Once + done chan struct{} + err error +} + +// NewCompletion returns an unfinished Completion. +func NewCompletion() *Completion { return &Completion{done: make(chan struct{})} } + +// Finish records err as the outcome. Only the first call has an effect. +func (c *Completion) Finish(err error) { + c.once.Do(func() { + c.err = err + close(c.done) + }) +} + +// Done is closed once the outcome is known. +func (c *Completion) Done() <-chan struct{} { return c.done } + +// Err is the outcome. It is only meaningful after Done is closed. +func (c *Completion) Err() error { return c.err } + +// Finished reports whether the outcome is known. +func (c *Completion) Finished() bool { + select { + case <-c.done: + return true + default: + return false + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/outcome_test.go b/deploy/operator/internal/dgdrrunpublisher/outcome_test.go new file mode 100644 index 000000000000..9462ea9fd49a --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/outcome_test.go @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" +) + +func TestExitCodes(t *testing.T) { + cases := []struct { + err error + want int + }{ + {nil, ExitAcknowledged}, + {ErrRunFailed, ExitRunFailed}, + {fmt.Errorf("%w: details", ErrRunFailed), ExitRunFailed}, + {ErrMissingTerminalSnapshot, ExitProtocolViolation}, + {violation("bad"), ExitProtocolViolation}, + {fmt.Errorf("reconciling last snapshot: %w", violation("bad")), ExitProtocolViolation}, + {errors.New("api down"), ExitReconcileFailed}, + } + for _, tc := range cases { + if got := ExitCode(tc.err); got != tc.want { + t.Errorf("ExitCode(%v) = %d, want %d", tc.err, got, tc.want) + } + } +} + +func TestTerminalResult(t *testing.T) { + if err := terminalResult(DesiredState{Phase: PhaseSucceeded, Candidates: []dgdcreconcile.DesiredCandidate{{ID: "a"}}}); err != nil { + t.Fatalf("succeeded: %v", err) + } + failed := terminalResult(DesiredState{Phase: PhaseFailed, Message: "search failed", Error: "backend unavailable"}) + if !errors.Is(failed, ErrRunFailed) || !strings.Contains(failed.Error(), "search failed") || !strings.Contains(failed.Error(), "backend unavailable") { + t.Fatalf("failed: %v", failed) + } + empty := terminalResult(DesiredState{Phase: PhaseSucceeded, Failed: 2}) + if !errors.Is(empty, ErrRunFailed) || !strings.Contains(empty.Error(), "2 candidate materialization failures") || !strings.Contains(empty.Error(), "no candidate was materialized") { + t.Fatalf("empty success: %v", empty) + } +} + +func TestExitResultWithoutATerminalSnapshot(t *testing.T) { + if err := exitResult(SweeperState{Exited: true, ExitCode: 137}); err != nil { + t.Fatalf("a crash is reported by the Sweeper's own exit code, not the publisher: %v", err) + } + if err := exitResult(SweeperState{Exited: true}); !errors.Is(err, ErrMissingTerminalSnapshot) { + t.Fatalf("a clean exit without a terminal snapshot is a protocol violation: %v", err) + } +} + +func TestCompletionKeepsTheFirstOutcome(t *testing.T) { + c := NewCompletion() + if c.Finished() { + t.Fatal("finished before Finish") + } + c.Finish(ErrRunFailed) + c.Finish(nil) + select { + case <-c.Done(): + default: + t.Fatal("Done not closed") + } + if !c.Finished() || !errors.Is(c.Err(), ErrRunFailed) { + t.Fatalf("err = %v", c.Err()) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/plan.go b/deploy/operator/internal/dgdrrunpublisher/plan.go new file mode 100644 index 000000000000..10559e5845fc --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/plan.go @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "crypto/sha256" + "encoding/hex" + "slices" + "strings" + "time" + + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" +) + +// SnapshotFileName is the file the Sweeper container writes. +const SnapshotFileName = "dgdr_run_snapshot.yaml" + +// maxObjectName is the Kubernetes limit on an object name (DNS subdomain). +const maxObjectName = 253 + +// RunStatus is the part of DynamoGraphDeploymentRun.status the publisher owns: the +// ordered candidate references, the number of published candidates +// (status.progress.candidates.published) and the coarse progress time. CandidateNames is +// in rank order (best first for scalar searches). Published always equals +// len(CandidateNames) in a desired status; it is kept separately so a status whose +// progress block was removed or edited by someone else is noticed and repaired. +// LastProgress is the zero time when the snapshot carries no usable timestamp, in which +// case the existing value is left untouched. +// +// The snapshot's round and evaluated counters have no counterpart in the Run status: it +// reports per-branch rounds and outcome counters that the snapshot does not carry yet, +// so they are not published rather than guessed. +type RunStatus struct { + Published int32 + LastProgress time.Time + CandidateNames []string +} + +// DesiredState is what one snapshot says should exist in the cluster. +type DesiredState struct { + // Candidates are the materialized candidates, in snapshot (rank) order. + Candidates []dgdcreconcile.DesiredCandidate + // Names maps a candidate id to its DGDC name. + Names map[string]string + Status RunStatus + Terminal bool + Phase string + // Message and Error are the producer's terminal diagnostics. They pass through the + // process result into the container termination message. + Message string + Error string + // FirstFailure is the bounded "id: error" of the first candidate whose + // materialization failed, for the run-failed diagnostics. + FirstFailure string + // Failed counts the candidates whose materialization failed. They are not created; + // the count is logged and prevents an empty terminal projection from succeeding. + Failed int +} + +// CandidateName is the DGDC name for a candidate id. It depends only on the run +// incarnation (name and UID) and the stable id, never on rank. The UID component keeps a +// recreated run of the same name from colliding with candidates of the previous one that +// are still waiting for garbage collection. A run name too long to leave room for the +// rest is truncated and a hash of the full run name is added, so names stay unique and +// within the limit. +func CandidateName(runName, runUID, id string) string { + uidSum := sha256.Sum256([]byte(runUID)) + suffix := "-" + hex.EncodeToString(uidSum[:4]) + "-" + id + if len(runName)+len(suffix) <= maxObjectName { + return runName + suffix + } + nameSum := sha256.Sum256([]byte(runName)) + hash := hex.EncodeToString(nameSum[:6]) + room := maxObjectName - len(suffix) - len(hash) - 1 + return strings.TrimRight(runName[:room], "-.") + "-" + hash + suffix +} + +// Desire projects a snapshot onto the desired cluster state of one run incarnation. +// Candidates whose materialization failed are not created; they are only counted in +// DesiredState.Failed. +func Desire(runName, runUID string, snap *Snapshot) DesiredState { + state := DesiredState{ + Names: make(map[string]string, len(snap.Candidates)), + Terminal: snap.Run.Terminal, + Phase: snap.Run.Phase, + Message: boundDiagnostic(snap.Run.Message), + Error: boundDiagnostic(snap.Run.Error), + } + failed := 0 + var names []string + for _, candidate := range snap.Candidates { + if candidate.Outcome != OutcomeMaterialized { + failed++ + if state.FirstFailure == "" { + state.FirstFailure = boundDiagnostic(candidate.ID + ": " + candidate.Error) + } + continue + } + name := CandidateName(runName, runUID, candidate.ID) + state.Names[candidate.ID] = name + names = append(names, name) + state.Candidates = append(state.Candidates, dgdcreconcile.DesiredCandidate{ + ID: candidate.ID, + Spec: candidate.Manifest, + Parameters: candidate.Parameters, + Metrics: candidate.Metrics, + }) + } + state.Failed = failed + state.Status = RunStatus{ + Published: int32(len(names)), + LastProgress: progressTime(snap), + CandidateNames: names, + } + return state +} + +// progressTime is the snapshot's own timestamp (written by the Sweeper container in the +// same pod) truncated to the second the API stores, or the zero time when it is missing +// or malformed. Using the snapshot's time instead of the clock keeps reconciliation +// deterministic. +func progressTime(snap *Snapshot) time.Time { + parsed, err := time.Parse(time.RFC3339, snap.Timestamp) + if err != nil { + return time.Time{} + } + return parsed.UTC().Truncate(time.Second) +} + +// WithPreservedProgress returns the status to write. When the snapshot carries no usable +// timestamp, the progress time already known (observed or written earlier) is kept: +// omitting a field from a server-side-apply payload removes it once this manager owns +// it, so "leave it alone" has to be spelled out as "write the same value again". +func (desired RunStatus) WithPreservedProgress(known time.Time) RunStatus { + if desired.LastProgress.IsZero() { + desired.LastProgress = known + } + return desired +} + +// NeedsUpdate reports whether the observed run status differs from the desired one, so +// the status is only written when something changed. A zero desired LastProgress means +// "leave the observed value alone". +func (desired RunStatus) NeedsUpdate(observed RunStatus) bool { + if desired.Published != observed.Published { + return true + } + if !desired.LastProgress.IsZero() && !desired.LastProgress.Equal(observed.LastProgress) { + return true + } + return !slices.Equal(desired.CandidateNames, observed.CandidateNames) +} diff --git a/deploy/operator/internal/dgdrrunpublisher/plan_test.go b/deploy/operator/internal/dgdrrunpublisher/plan_test.go new file mode 100644 index 000000000000..12268168a6a9 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/plan_test.go @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "reflect" + "strings" + "testing" + "time" +) + +func TestCandidateNameIsBoundedAndUniquePerRunIncarnation(t *testing.T) { + id := "evaluated-point-0123456789ab" + if cn("a") != CandidateName("run", testRunUID, "a") || cn("a") == CandidateName("run", "uid-2", "a") { + t.Fatalf("a recreated run must get different names: %q", cn("a")) + } + long := strings.Repeat("r", 253) + other := strings.Repeat("r", 252) + "s" + name := CandidateName(long, testRunUID, id) + if len(name) > 253 || !strings.HasSuffix(name, "-"+id) || name == CandidateName(other, testRunUID, id) || name != CandidateName(long, testRunUID, id) { + t.Fatalf("name = %q (%d)", name, len(name)) + } +} + +func desire(t *testing.T, data []byte) DesiredState { + t.Helper() + return Desire("run", testRunUID, mustParse(t, data)) +} + +func TestSwapKeepsCandidateNamesAndOnlyReordersReferences(t *testing.T) { + first := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + second := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"})) + if !reflect.DeepEqual(first.Names, second.Names) { + t.Fatalf("names depend on rank: %v vs %v", first.Names, second.Names) + } + if want := []string{cn("b"), cn("a")}; !reflect.DeepEqual(second.Status.CandidateNames, want) { + t.Fatalf("ordered references = %v, want %v", second.Status.CandidateNames, want) + } + if !second.Status.NeedsUpdate(first.Status) { + t.Fatal("a reorder must update the run's references") + } +} + +func TestFailedMaterializationIsNotDesiredButCounted(t *testing.T) { + state := desire(t, withRunDiagnostics(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true}), "search warning", "render error")) + if len(state.Candidates) != 1 || state.Candidates[0].ID != "a" { + t.Fatalf("candidates = %+v", state.Candidates) + } + if !reflect.DeepEqual(state.Status.CandidateNames, []string{cn("a")}) { + t.Fatalf("refs must not include the failed candidate: %v", state.Status.CandidateNames) + } + if state.Failed != 1 { + t.Fatalf("failed = %d, want 1", state.Failed) + } + if state.Status.Published != 1 { + t.Fatalf("published = %d, want 1: failed candidates are not published", state.Status.Published) + } + if state.Message != "search warning" || state.Error != "render error" { + t.Fatalf("terminal diagnostics = %q / %q", state.Message, state.Error) + } +} + +func TestPublishedCountsTheReferencedCandidates(t *testing.T) { + state := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + if got, want := state.Status.Published, int32(len(state.Status.CandidateNames)); got != want || got != 2 { + t.Fatalf("published = %d, refs = %d, want 2", got, want) + } +} + +func TestLastProgressComesFromTheSnapshotTimestamp(t *testing.T) { + data := withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z") + want := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + if got := desire(t, data).Status.LastProgress; !got.Equal(want) { + t.Fatalf("LastProgress = %v, want %v", got, want) + } +} + +func TestMissingOrMalformedTimestampLeavesProgressTimeAlone(t *testing.T) { + for _, timestamp := range []string{"", "yesterday"} { + state := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), timestamp)) + if !state.Status.LastProgress.IsZero() { + t.Fatalf("timestamp %q: LastProgress = %v, want zero", timestamp, state.Status.LastProgress) + } + observed := state.Status + observed.LastProgress = time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if state.Status.NeedsUpdate(observed) { + t.Fatalf("timestamp %q: an unknown progress time must not force an update", timestamp) + } + } +} + +func TestNeedsUpdateOnlyWhenSomethingDiffers(t *testing.T) { + base := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")).Status + if base.NeedsUpdate(base) { + t.Fatal("identical status needs no write") + } + changes := map[string]func(*RunStatus){ + "published": func(s *RunStatus) { s.Published++ }, + "progress time": func(s *RunStatus) { s.LastProgress = s.LastProgress.Add(time.Second) }, + "references": func(s *RunStatus) { s.CandidateNames = nil }, + } + for name, mutate := range changes { + observed := base + observed.CandidateNames = append([]string(nil), base.CandidateNames...) + mutate(&observed) + if !base.NeedsUpdate(observed) { + t.Errorf("%s: a difference must trigger an update", name) + } + } +} + +func TestWithPreservedProgressKeepsTheKnownTimeOnlyWhenTheSnapshotHasNone(t *testing.T) { + known := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + without := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "")).Status + if got := without.WithPreservedProgress(known).LastProgress; !got.Equal(known) { + t.Fatalf("a snapshot without a timestamp must keep the known time, got %v", got) + } + newer := time.Date(2026, 10, 5, 21, 40, 0, 0, time.UTC) + with := desire(t, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:40:00Z")).Status + if got := with.WithPreservedProgress(known).LastProgress; !got.Equal(newer) { + t.Fatalf("a snapshot with a timestamp must win, got %v", got) + } + if got := without.WithPreservedProgress(time.Time{}).LastProgress; !got.IsZero() { + t.Fatalf("nothing known, nothing to preserve, got %v", got) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go new file mode 100644 index 000000000000..9e51272b60d9 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -0,0 +1,368 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "errors" + "fmt" + "strings" + + dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // LabelRunUID ties a candidate to one incarnation of its run. The run name can be + // reused after a delete and recreate; the UID cannot, so candidates left over from a + // previous run of the same name are never mistaken for current ones. It is a repairable + // index; the controller owner UID remains the authoritative ownership relation. + LabelRunUID = "nvidia.com/dgdr-run-uid" + // LabelCandidateID records the stable candidate id (the identity read back on list). + LabelCandidateID = "nvidia.com/dgdr-candidate-id" + + // AnnotationAdditionalResources carries the companion resources (for example + // generated ConfigMaps) rendered next to the DGD, as multi-document YAML. It is the + // same key and format the DGDR controller uses. + AnnotationAdditionalResources = "dgdr.nvidia.com/additional-resources" + + // fieldOwner is the server-side-apply field manager of the publisher. + fieldOwner = "dgdr-run-publisher" + + // finalReconcileAttempts bounds the retries once the Sweeper has exited: nothing will + // rewrite the snapshot, so a persistent failure must end the process. + finalReconcileAttempts = 3 +) + +// Reconciler reconciles the one DynamoGraphDeploymentRun this publisher serves. +// +// Client is the manager's client: reads come from the scoped informer cache (this Run, +// its candidates, this Pod). Writes go through Writer to the API server. It needs RBAC +// (namespaced Role bound to the run's ServiceAccount): runs get/list/watch and +// runs/status patch (server-side apply); candidates get/list/watch/create/patch/delete +// and candidates/status patch; pods get/list/watch (own pod, for Sweeper crash +// detection; the cache narrows list/watch to it with a field selector because generated +// Job pod names cannot be listed in resourceNames). +// +// Write echoes and cache lag: the publisher's own writes come back as watch events, and +// the cache can trail them. The reconciler keeps no record of its writes. Every pass +// decides from the current observations (the cached run and candidates, the latest input +// snapshot), so a repeated pass while the cache catches up may repeat an idempotent +// write, which is harmless, and an echo of a write finds nothing left to do. Anything +// that changes behind the publisher's back (a deleted candidate, a modified run status) +// is repaired on the next pass. +// +// Stale reads: every status write is constrained to the resourceVersion the decision was +// based on (the cached run, the cached candidate, or the version read back from the API +// server for a candidate applied a moment ago). A decision made from an older cache +// observation fails with a conflict instead of overwriting newer state, and is retried +// through the workqueue once the informer has delivered the newer version. A candidate's +// status is only populated when the API server's own copy, read right after the apply, +// has none yet, so a stale cache can never make the publisher rewrite populated status. +type Reconciler struct { + Client client.Client + Writer writer + Namespace string + RunName string + RunUID string + PodName string + SweeperContainer string + + Observer *Observer + Store *InputStore + Completion *Completion + + // The fields below are touched only by Reconcile; the controller runs one worker, so + // they need no locking. + + // finalFailures counts failed reconciles after the Sweeper exited. + finalFailures int +} + +func (r *Reconciler) key() types.NamespacedName { + return types.NamespacedName{Namespace: r.Namespace, Name: r.RunName} +} + +// Reconcile handles the single run key. Both the input observer and informer events +// enqueue it; the returned error makes the workqueue retry with backoff. +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if req.NamespacedName != r.key() || r.Completion.Finished() { + return ctrl.Result{}, nil + } + return ctrl.Result{}, r.reconcile(ctx) +} + +func (r *Reconciler) reconcile(ctx context.Context) error { + var run v1beta2.DynamoGraphDeploymentRun + if err := r.Client.Get(ctx, r.key(), &run); err != nil { + return fmt.Errorf("getting run: %w", err) + } + if string(run.UID) != r.RunUID { + // The run was deleted and recreated under the same name; this pod belongs to the + // previous incarnation and must not touch the new one. + r.Completion.Finish(fmt.Errorf("run %s has uid %s, this publisher serves uid %s", r.RunName, run.UID, r.RunUID)) + return nil + } + + // Read the container state BEFORE the snapshot: the Sweeper's last write precedes its + // exit, so an exit observed here means that write is visible to the check below. + state, stateErr := r.sweeperState(ctx) + if stateErr != nil { + ctrl.LoggerFrom(ctx).Error(stateErr, "reading sweeper container state") + } + exited := stateErr == nil && state.Exited + if exited { + r.Observer.Check() + } + + input := r.Store.Load() + if input.Err != nil && exited { + if errors.Is(input.Err, ErrProtocolViolation) { + r.Completion.Finish(input.Err) + return nil + } + return r.retryOrFail(exited, input.Err) + } + if input.Snapshot == nil { + if exited { + r.Completion.Finish(exitResult(state)) + } + return nil // the observer enqueues the key when the first snapshot arrives + } + + desired := Desire(r.RunName, r.RunUID, input.Snapshot) + if err := r.apply(ctx, &run, desired); err != nil { + return r.retryOrFail(exited, err) + } + switch { + case desired.Terminal: + r.Completion.Finish(terminalResult(desired)) + case exited: + r.Completion.Finish(exitResult(state)) + } + return nil +} + +// retryOrFail hands a failure to the workqueue's backoff, except where retrying cannot +// help (malformed projection) or the Sweeper is gone and the attempts are used up. +func (r *Reconciler) retryOrFail(exited bool, err error) error { + var inputErr *dgdcreconcile.DiffInputError + if errors.As(err, &inputErr) { + r.Completion.Finish(err) // retrying cannot fix it + return nil + } + if apierrors.IsConflict(err) { + // A write was based on a stale cached read. The informer delivers the newer + // version and the retry decides again, so this is not a failed attempt. + return err + } + if exited { + r.finalFailures++ + if r.finalFailures >= finalReconcileAttempts { + r.Completion.Finish(fmt.Errorf("reconciling last snapshot: %w", err)) + return nil + } + } + return err +} + +// apply drives the cluster toward the desired state: create missing candidates, finish +// the status of candidates that lack it, update the run, then delete obsolete candidates. +// It is idempotent and safe to retry; existing candidate specs are never touched. +func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired DesiredState) error { + if desired.Failed > 0 { + ctrl.LoggerFrom(ctx).V(1).Info("candidates failed materialization and are not published", "count", desired.Failed) + } + var list v1beta2.DynamoGraphDeploymentCandidateList + if err := r.Client.List(ctx, &list, client.InNamespace(r.Namespace)); err != nil { + return fmt.Errorf("listing candidates: %w", err) + } + current := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) + ownedObsolete := make([]string, 0, len(list.Items)) + uids := make(map[string]types.UID, len(list.Items)) + versions := make(map[string]string, len(list.Items)) // cached resourceVersion per candidate + expectedIDs := make(map[string]string, len(desired.Names)) + for id, name := range desired.Names { + expectedIDs[name] = id + } + for i := range list.Items { + item := &list.Items[i] + + // A label match scopes the cache but does not establish ownership. Foreign + // candidates must never satisfy desired state or become deletion targets. + owner := metav1.GetControllerOf(item) + if owner == nil || owner.UID != run.UID { + continue + } + + // The deterministic name is the authoritative identity. An owned object under + // another name is obsolete; an expected object with a stale identity label is + // reapplied in place instead of deleted. + expectedID, isExpected := expectedIDs[item.Name] + if !isExpected { + ownedObsolete = append(ownedObsolete, item.Name) + uids[item.Name] = item.UID + continue + } + if item.Labels[LabelRunUID] != r.RunUID || item.Labels[LabelCandidateID] != expectedID { + continue + } + current = append(current, dgdcreconcile.CurrentDGDC{ + Name: item.Name, + ID: expectedID, + Incomplete: item.Status.Metrics == nil, + }) + uids[item.Name] = item.UID + versions[item.Name] = item.ResourceVersion + } + actions, err := dgdcreconcile.ComputeActions(desired.Candidates, current) + if err != nil { + return err + } + actions.Deletes = append(actions.Deletes, ownedObsolete...) + // populated names the candidates whose status the API server already has; they are + // known from the live read that follows each apply, not from the cache. + populated := map[string]struct{}{} + for _, candidate := range actions.Creates { + name := desired.Names[candidate.ID] + stored, err := r.Writer.CreateCandidate(ctx, run, name, candidate) + if err != nil { + return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) + } + versions[name] = stored.ResourceVersion // a new candidate is not in the cache; use the version the API server returned + if stored.Populated { + populated[name] = struct{}{} + } + } + unpopulated := make([]dgdcreconcile.DesiredCandidate, 0, len(actions.Creates)+len(actions.Incomplete)) + unpopulated = append(unpopulated, actions.Creates...) + unpopulated = append(unpopulated, actions.Incomplete...) + for _, candidate := range unpopulated { + name := desired.Names[candidate.ID] + if _, done := populated[name]; done { + continue + } + if err := r.Writer.PopulateStatus(ctx, name, versions[name], candidate); err != nil { + return fmt.Errorf("populating status of candidate %s: %w", candidate.ID, err) + } + } + + if err := r.updateRunStatus(ctx, run, desired.Status); err != nil { + return err + } + + for _, name := range actions.Deletes { + if err := r.Writer.DeleteCandidate(ctx, name, uids[name]); err != nil { + return fmt.Errorf("deleting candidate %s: %w", name, err) + } + } + return nil +} + +// updateRunStatus writes the run status when it differs from what the cache shows. A +// snapshot without a usable timestamp keeps the progress time the cached run already has, +// because a server-side-apply payload that omits the field would remove it. The write is +// constrained to the cached run's resourceVersion, so a decision made from a stale cache +// fails with a conflict instead of overwriting newer status. +func (r *Reconciler) updateRunStatus(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired RunStatus) error { + observed := observedStatus(run) + if !desired.NeedsUpdate(observed) { + return nil + } + write := desired.WithPreservedProgress(observed.LastProgress) + if err := r.Writer.UpdateRunStatus(ctx, run.ResourceVersion, write); err != nil { + return fmt.Errorf("updating run status: %w", err) + } + return nil +} + +// observedStatus is the publisher-owned part of the run's current (cached) status. +func observedStatus(run *v1beta2.DynamoGraphDeploymentRun) RunStatus { + var observed RunStatus + if progress := run.Status.Progress; progress != nil && progress.Candidates != nil { + observed.Published = progress.Candidates.Published + } + if run.Status.LastProgressTime != nil { + observed.LastProgress = run.Status.LastProgressTime.UTC() + } + for _, ref := range run.Status.CandidateRefs { + observed.CandidateNames = append(observed.CandidateNames, ref.Name) + } + return observed +} + +// sweeperState reads the Sweeper container's status from this pod, via the cache. +func (r *Reconciler) sweeperState(ctx context.Context) (SweeperState, error) { + var pod corev1.Pod + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: r.Namespace, Name: r.PodName}, &pod); err != nil { + return SweeperState{}, err + } + return sweeperStateOf(&pod, r.SweeperContainer), nil +} + +func sweeperStateOf(pod *corev1.Pod, container string) SweeperState { + for _, status := range pod.Status.ContainerStatuses { + if status.Name != container { + continue + } + if terminated := status.State.Terminated; terminated != nil { + return SweeperState{Exited: true, ExitCode: terminated.ExitCode} + } + return SweeperState{} + } + return SweeperState{} +} + +// decodeManifest extracts the DGD spec and the companion resources from a rendered +// manifest. A DGD in either supported apiVersion is converted to v1beta1, the same way +// the DGDR controller does it. +func decodeManifest(candidate dgdcreconcile.DesiredCandidate) (*v1beta1.DynamoGraphDeploymentSpec, string, error) { + manifest, err := ParseManifest(candidate.Spec) + if err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + obj := &unstructured.Unstructured{Object: manifest.DGD} + var beta v1beta1.DynamoGraphDeployment + switch obj.GetAPIVersion() { + case dgdv1alpha1.GroupVersion.String(): + var alpha dgdv1alpha1.DynamoGraphDeployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &alpha); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + if err := alpha.ConvertTo(&beta); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + case v1beta1.GroupVersion.String(): + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &beta); err != nil { + return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) + } + default: + return nil, "", fmt.Errorf("candidate %s: unsupported DynamoGraphDeployment apiVersion %q", candidate.ID, obj.GetAPIVersion()) + } + return &beta.Spec, strings.Join(manifest.Companions, "\n---\n"), nil +} diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go new file mode 100644 index 000000000000..58a35b02e880 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -0,0 +1,990 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const ( + testNamespace = "ns" + testRunName = "run" + testPodName = "pod-1" +) + +// memWriter is the reconciler's writer for tests. It logs every write attempt (a failed +// attempt is logged with a trailing "!") and, unless lag is set, applies the write to +// the fake client the reconciler reads from, which stands in for the informer cache. +// With lag set, writes are logged but never become visible to the reconciler, like a +// cache that has not caught up; the server map still records them. +type memWriter struct { + t *testing.T + c client.Client + ops []string + status []RunStatus + // statusVersions and populateVersions record the resourceVersion each write was + // constrained to. + statusVersions []string + populateVersions map[string]string + lag bool + // server is what the API server holds, whatever the cache shows: the candidates that + // exist, and whether each has its status. + server map[string]bool + // failOn makes the named operation ("create", "populate", "status", "delete") fail; + // once, unless sticky. + failOn map[string]error + sticky bool +} + +func (w *memWriter) fail(op string) error { + err, ok := w.failOn[op] + if !ok { + return nil + } + if !w.sticky { + delete(w.failOn, op) + } + return err +} + +func (w *memWriter) record(op string, err error) error { + if err != nil { + op += "!" + } + w.ops = append(w.ops, op) + return err +} + +func (w *memWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) { + if err := w.record("create:"+name, w.fail("create")); err != nil { + return StoredCandidate{}, err + } + // The apply is idempotent; the read-back reports what the server holds. + populated := w.server[name] + w.server[name] = populated + stored := StoredCandidate{ResourceVersion: createdVersion(name), Populated: populated} + if w.lag { + return stored, nil + } + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: name, + UID: types.UID("uid-" + name), + Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: candidate.ID}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(run, v1beta2.DynamoGraphDeploymentRunGVK)}, + }} + return stored, client.IgnoreAlreadyExists(w.c.Create(ctx, object)) +} + +// createdVersion is the resourceVersion the fake API server "returns" for a new candidate. +func createdVersion(name string) string { return "created-" + name } + +func (w *memWriter) PopulateStatus(ctx context.Context, name, resourceVersion string, _ dgdcreconcile.DesiredCandidate) error { + if err := w.record("populate:"+name, w.fail("populate")); err != nil { + return err + } + w.populateVersions[name] = resourceVersion + w.server[name] = true + if w.lag { + return nil + } + var object v1beta2.DynamoGraphDeploymentCandidate + if err := w.c.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: name}, &object); err != nil { + return err + } + object.Status.Metrics = &runtime.RawExtension{Raw: []byte("{}")} + return w.c.Update(ctx, &object) +} + +func (w *memWriter) UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error { + if err := w.record("status", w.fail("status")); err != nil { + return err + } + w.status = append(w.status, status) + w.statusVersions = append(w.statusVersions, resourceVersion) + if w.lag { + return nil + } + var run v1beta2.DynamoGraphDeploymentRun + if err := w.c.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + return err + } + run.Status.Progress = &v1beta2.DGDRRunProgress{Candidates: &v1beta2.CandidateProgress{Published: status.Published}} + if !status.LastProgress.IsZero() { + progressed := metav1.NewTime(status.LastProgress) + run.Status.LastProgressTime = &progressed + } + run.Status.CandidateRefs = nil + for _, name := range status.CandidateNames { + run.Status.CandidateRefs = append(run.Status.CandidateRefs, corev1.LocalObjectReference{Name: name}) + } + return w.c.Update(ctx, &run) +} + +func (w *memWriter) DeleteCandidate(ctx context.Context, name string, _ types.UID) error { + if err := w.record("delete:"+name, w.fail("delete")); err != nil { + return err + } + if w.lag { + return nil + } + return client.IgnoreNotFound(w.c.Delete(ctx, &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}})) +} + +type fixture struct { + t *testing.T + r *Reconciler + w *memWriter + c client.Client +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + scheme, err := NewScheme() + if err != nil { + t.Fatal(err) + } + run := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testRunName, UID: testRunUID}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testPodName}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(run, pod).Build() + store := &InputStore{} + observer := &Observer{ + Store: store, + Path: filepath.Join(t.TempDir(), SnapshotFileName), + MinInterval: time.Millisecond, + MaxInterval: time.Millisecond, + } + w := &memWriter{t: t, c: c, failOn: map[string]error{}, populateVersions: map[string]string{}, server: map[string]bool{}} + return &fixture{t: t, c: c, w: w, r: &Reconciler{ + Client: c, + Writer: w, + Namespace: testNamespace, + RunName: testRunName, + RunUID: testRunUID, + PodName: testPodName, + SweeperContainer: "sweeper", + Observer: observer, + Store: store, + Completion: NewCompletion(), + }} +} + +// publish writes a snapshot the way the Sweeper does and lets the observer see it. +func (f *fixture) publish(data []byte) { + f.t.Helper() + writeSnapshot(f.t, filepath.Dir(f.r.Observer.Path), data) + f.r.Observer.Check() +} + +func (f *fixture) reconcile() error { + f.t.Helper() + _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: testRunName}}) + return err +} + +func (f *fixture) mustReconcile() { + f.t.Helper() + if err := f.reconcile(); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +// exitSweeper reports the Sweeper container as terminated, as the pod status does. +func (f *fixture) exitSweeper(code int32) { + f.t.Helper() + var pod corev1.Pod + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testPodName}, &pod); err != nil { + f.t.Fatal(err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "sweeper", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: code}}}} + if err := f.c.Status().Update(context.Background(), &pod); err != nil { + f.t.Fatal(err) + } +} + +// deleteCandidate removes a candidate behind the publisher's back, from the API server +// and from the cache. +func (f *fixture) deleteCandidate(name string) { + f.t.Helper() + delete(f.w.server, name) + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}} + if err := f.c.Delete(context.Background(), object); err != nil { + f.t.Fatal(err) + } +} + +func (f *fixture) takeOps() []string { + ops := f.w.ops + f.w.ops = nil + return ops +} + +func (f *fixture) lastStatus() RunStatus { + f.t.Helper() + if len(f.w.status) == 0 { + f.t.Fatal("no run status was written") + } + return f.w.status[len(f.w.status)-1] +} + +func (f *fixture) currentRun() *v1beta2.DynamoGraphDeploymentRun { + f.t.Helper() + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + f.t.Fatal(err) + } + return &run +} + +func candidateForRun(name, id string, run *v1beta2.DynamoGraphDeploymentRun) *v1beta2.DynamoGraphDeploymentCandidate { + return &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + Namespace: run.Namespace, + Name: name, + Labels: map[string]string{LabelRunUID: string(run.UID), LabelCandidateID: id}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(run, v1beta2.DynamoGraphDeploymentRunGVK)}, + }} +} + +// withProgress rewrites a snapshot's progress counters. +func withProgress(t *testing.T, data []byte, round, evaluated int) []byte { + t.Helper() + doc := map[string]any{} + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + doc["progress"] = map[string]any{"round": round, "evaluated": evaluated} + out, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestReconcileCreatesPopulatesAndReferencesCandidatesInOrder(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + want := []string{"create:" + cn("a"), "create:" + cn("b"), "populate:" + cn("a"), "populate:" + cn("b"), "status"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } + if got, want := f.lastStatus().CandidateNames, []string{cn("a"), cn("b")}; !reflect.DeepEqual(got, want) { + t.Fatalf("refs = %v, want %v", got, want) + } +} + +func TestSwapOnlyReordersTheReferences(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + f.takeOps() + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "b"}, cand{id: "a"})) + f.mustReconcile() + if got := f.takeOps(); !reflect.DeepEqual(got, []string{"status"}) { + t.Fatalf("a swap must not touch any candidate, ops = %v", got) + } + if got, want := f.lastStatus().CandidateNames, []string{cn("b"), cn("a")}; !reflect.DeepEqual(got, want) { + t.Fatalf("refs = %v, want %v", got, want) + } +} + +func TestReconcilingUnchangedStateWritesNothing(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + f.mustReconcile() + f.mustReconcile() + if got := f.takeOps(); len(got) != 0 { + t.Fatalf("unchanged state must cause no writes, got %v", got) + } +} + +func TestCacheLagRepeatsOnlyIdempotentWrites(t *testing.T) { + f := newFixture(t) + f.w.lag = true // the cache never shows what was written + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.takeOps(), []string{"create:" + cn("a"), "populate:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("first reconcile ops = %v, want %v", got, want) + } + f.mustReconcile() + // The candidate apply repeats harmlessly and the server reports the candidate populated, + // so its status is not written a second time. The run status is written again, still + // constrained to the cached version: against a real API server the first write has + // moved the run on, so this one is rejected as stale and retried through the workqueue. + if got, want := f.takeOps(), []string{"create:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("second reconcile ops = %v, want %v", got, want) + } + if len(f.w.statusVersions) != 2 || f.w.statusVersions[0] != f.w.statusVersions[1] { + t.Fatalf("both status writes must carry the cached run version, got %v", f.w.statusVersions) + } +} + +func TestExternallyDeletedCandidateIsRecreated(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + f.deleteCandidate(cn("a")) + f.mustReconcile() + if got, want := f.takeOps(), []string{"create:" + cn("a"), "populate:" + cn("a")}; !reflect.DeepEqual(got, want) { + t.Fatalf("a candidate deleted behind the publisher's back must be repaired, ops = %v, want %v", got, want) + } +} + +func TestExternallyModifiedRunStatusIsRepaired(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + run.Status.CandidateRefs = []corev1.LocalObjectReference{{Name: "changed-by-someone-else"}} + if err := f.c.Update(context.Background(), &run); err != nil { + t.Fatal(err) + } + f.mustReconcile() + if got, want := f.takeOps(), []string{"status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("a run status modified behind the publisher's back must be repaired, ops = %v, want %v", got, want) + } +} + +func TestPartialSuccessRetryRepeatsOnlyTheUnfinishedWrites(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = errors.New("api unavailable") + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if err := f.reconcile(); err == nil { + t.Fatal("a failed run status write must be returned for the workqueue to retry with backoff") + } + f.mustReconcile() // the retry, possibly triggered early by the echo of the candidate create + want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status!", "status"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestCreateFailureSkipsTheRunStatusAndIsRetried(t *testing.T) { + f := newFixture(t) + f.w.failOn["create"] = errors.New("boom") + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + if err := f.reconcile(); err == nil { + t.Fatal("want the create failure returned") + } + for _, op := range f.takeOps() { + if op == "status" { + t.Fatal("the run must not reference a candidate that was not created") + } + } + f.mustReconcile() + if got := f.takeOps(); got[len(got)-1] != "status" { + t.Fatalf("ops after retry = %v", got) + } +} + +func TestDroppedCandidateIsDeletedAfterTheRunStatusUpdate(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) + f.mustReconcile() + f.takeOps() + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), 3, 9)) + f.mustReconcile() + if got, want := f.takeOps(), []string{"status", "delete:" + cn("b")}; !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true})) + f.mustReconcile() + for _, op := range f.takeOps() { + if strings.Contains(op, "x") && strings.HasPrefix(op, "create") { + t.Fatalf("a failed candidate must not be created: %s", op) + } + } + status := f.lastStatus() + if !reflect.DeepEqual(status.CandidateNames, []string{cn("a")}) || status.Published != 1 { + t.Fatalf("status = %+v", status) + } +} + +func TestCandidateWithoutStatusIsCompletedNotRecreated(t *testing.T) { + f := newFixture(t) + existing := candidateForRun(cn("a"), "a", f.currentRun()) + if err := f.c.Create(context.Background(), existing); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.takeOps(), []string{"populate:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestProgressTimeSurvivesASnapshotWithoutATimestamp(t *testing.T) { + f := newFixture(t) + first := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) + f.mustReconcile() + f.publish(withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}), "")) + f.mustReconcile() + if len(f.w.status) != 2 { + t.Fatalf("want two status writes, got %d", len(f.w.status)) + } + // The preserved time comes from the cached run, the current observed state; a stale + // cache would lose the version check instead of the timestamp. + if got := f.lastStatus().LastProgress; !got.Equal(first) { + t.Fatalf("the second write must repeat the observed progress time %v, got %v", first, got) + } +} + +func TestTerminalSucceededSnapshotIsReconciledThenAcknowledged(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if len(f.w.status) != 1 { + t.Fatal("the terminal snapshot must be reconciled before the publisher exits") + } +} + +func TestTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { + f := newFixture(t) + f.publish(withRunDiagnostics(t, snapshotJSON(t, PhaseFailed, cand{id: "a"}), "search failed", "backend unavailable")) + f.mustReconcile() + if err := f.r.Completion.Err(); !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed || + !strings.Contains(err.Error(), "search failed") || !strings.Contains(err.Error(), "backend unavailable") { + t.Fatalf("err = %v", err) + } + if len(f.w.status) != 1 { + t.Fatal("the failed snapshot must still be reconciled") + } +} + +func TestTerminalSucceededSnapshotWithoutAMaterializedCandidateFails(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "x", failed: true})) + f.mustReconcile() + + err := f.r.Completion.Err() + if !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed || !strings.Contains(err.Error(), "1 candidate materialization failure") { + t.Fatalf("err = %v", err) + } + if ops := f.takeOps(); len(ops) != 0 { + t.Fatalf("a failed materialization must not produce cluster writes, got %v", ops) + } +} + +func TestTerminalSnapshotReconcileFailureIsRetriedUntilItSucceeds(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = errors.New("api unavailable") + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + if err := f.reconcile(); err == nil || f.r.Completion.Finished() { + t.Fatalf("a terminal snapshot that could not be reconciled must be retried, err=%v", err) + } + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } +} + +func TestSweeperCrashWithoutTerminalSnapshotReconcilesTheLastSnapshot(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(137) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("the Sweeper's own exit code fails the Job, not the publisher: finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if len(f.w.status) != 1 { + t.Fatal("the last snapshot must be reconciled before exit") + } +} + +func TestSweeperCleanExitWithoutTerminalSnapshotIsAProtocolViolation(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(0) + f.mustReconcile() + if err := f.r.Completion.Err(); !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitProtocolViolation { + t.Fatalf("err = %v", err) + } + if len(f.w.status) != 1 { + t.Fatal("the last snapshot must be reconciled before exit") + } +} + +func TestSweeperExitBeforeAnySnapshot(t *testing.T) { + crash := newFixture(t) + crash.exitSweeper(1) + crash.mustReconcile() + if !crash.r.Completion.Finished() || crash.r.Completion.Err() != nil { + t.Fatalf("crash: finished=%v err=%v", crash.r.Completion.Finished(), crash.r.Completion.Err()) + } + clean := newFixture(t) + clean.exitSweeper(0) + clean.mustReconcile() + if err := clean.r.Completion.Err(); !errors.Is(err, ErrMissingTerminalSnapshot) { + t.Fatalf("clean exit with no snapshot at all: err = %v", err) + } +} + +func TestSnapshotWrittenJustBeforeTheSweeperExitIsReadAfterTheExitIsSeen(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + // The Sweeper writes its terminal snapshot and exits; the observer has not looked yet. + writeSnapshot(t, filepath.Dir(f.r.Observer.Path), snapshotJSON(t, PhaseSucceeded, cand{id: "a"}, cand{id: "b"})) + f.exitSweeper(0) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("the terminal snapshot must be picked up on exit: finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } + if got := f.lastStatus().CandidateNames; len(got) != 2 { + t.Fatalf("refs = %v", got) + } +} + +func TestInvalidFinalSnapshotIsAProtocolViolationExit(t *testing.T) { + f := newFixture(t) + writeSnapshot(t, filepath.Dir(f.r.Observer.Path), []byte(`{"schemaVersion":2}`)) + f.exitSweeper(0) + f.mustReconcile() + if err := f.r.Completion.Err(); ExitCode(err) != ExitProtocolViolation { + t.Fatalf("err = %v, exit = %d", err, ExitCode(err)) + } +} + +func TestRegressingSnapshotIsNotReconciled(t *testing.T) { + f := newFixture(t) + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), 3, 5)) + f.mustReconcile() + f.takeOps() + f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"}), 2, 99)) + f.mustReconcile() + if got := f.takeOps(); len(got) != 0 { + t.Fatalf("a regressing snapshot must not be reconciled, got %v", got) + } +} + +func TestFinalReconcileFailureEndsTheProcessAfterBoundedRetries(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = errors.New("api unavailable") + f.w.sticky = true + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(1) + for attempt := 1; attempt < finalReconcileAttempts; attempt++ { + if err := f.reconcile(); err == nil || f.r.Completion.Finished() { + t.Fatalf("attempt %d: want a retry, err=%v", attempt, err) + } + } + f.mustReconcile() + if err := f.r.Completion.Err(); err == nil || ExitCode(err) != ExitReconcileFailed { + t.Fatalf("err = %v", err) + } +} + +func TestForeignLabeledCandidateDoesNotSatisfyOrGetDeleted(t *testing.T) { + f := newFixture(t) + foreignRun := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: "foreign-run", + UID: "foreign-run-uid", + }} + foreign := candidateForRun("foreign-candidate", "a", foreignRun) + foreign.Labels[LabelRunUID] = testRunUID + if err := f.c.Create(context.Background(), foreign); err != nil { + t.Fatal(err) + } + + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.takeOps(), []string{"create:" + cn("a"), "populate:" + cn("a"), "status"}; !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } + if err := f.c.Get(context.Background(), client.ObjectKeyFromObject(foreign), &v1beta2.DynamoGraphDeploymentCandidate{}); err != nil { + t.Fatalf("foreign candidate must remain untouched: %v", err) + } +} + +func TestOwnedCandidateWithWrongNameIsReplacedBeforeDeletion(t *testing.T) { + f := newFixture(t) + wrong := candidateForRun("wrong-name", "a", f.currentRun()) + if err := f.c.Create(context.Background(), wrong); err != nil { + t.Fatal(err) + } + + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status", "delete:wrong-name"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestOwnedObsoleteCandidateWithoutRunLabelIsDeleted(t *testing.T) { + f := newFixture(t) + obsolete := candidateForRun(cn("obsolete"), "obsolete", f.currentRun()) + delete(obsolete.Labels, LabelRunUID) + if err := f.c.Create(context.Background(), obsolete); err != nil { + t.Fatal(err) + } + + f.publish(snapshotJSON(t, PhaseRunning)) + f.mustReconcile() + if got, want := f.takeOps(), []string{"delete:" + obsolete.Name}; !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestOwnedCandidateWithWrongIdentityLabelIsReappliedNotDeleted(t *testing.T) { + f := newFixture(t) + wrong := candidateForRun(cn("a"), "wrong-id", f.currentRun()) + if err := f.c.Create(context.Background(), wrong); err != nil { + t.Fatal(err) + } + + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestOwnedCandidateWithMissingRunLabelIsReappliedNotDeleted(t *testing.T) { + f := newFixture(t) + drifted := candidateForRun(cn("a"), "a", f.currentRun()) + delete(drifted.Labels, LabelRunUID) + if err := f.c.Create(context.Background(), drifted); err != nil { + t.Fatal(err) + } + + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status"} + if got := f.takeOps(); !reflect.DeepEqual(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func TestRunRecreatedUnderTheSameNameIsNotTouched(t *testing.T) { + f := newFixture(t) + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + run.UID = "another-uid" + if err := f.c.Update(context.Background(), &run); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() == nil { + t.Fatal("a publisher serving a previous incarnation must stop") + } + if len(f.w.ops) != 0 { + t.Fatalf("it must not write anything: %v", f.w.ops) + } +} + +func TestReconcileIgnoresOtherKeysAndAFinishedRun(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + if _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: "other"}}); err != nil || len(f.w.ops) != 0 { + t.Fatalf("another key must be ignored: err=%v ops=%v", err, f.w.ops) + } + f.mustReconcile() + ops := len(f.w.ops) + f.mustReconcile() + if len(f.w.ops) != ops { + t.Fatal("a finished run must not be reconciled again") + } +} + +func podWith(statuses ...corev1.ContainerStatus) *corev1.Pod { + return &corev1.Pod{Status: corev1.PodStatus{ContainerStatuses: statuses}} +} + +func TestSweeperStateOfReadsOnlyTheSweeperContainer(t *testing.T) { + terminated := func(name string, code int32) corev1.ContainerStatus { + return corev1.ContainerStatus{Name: name, State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: code}}} + } + running := corev1.ContainerStatus{Name: "sweeper", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}} + + cases := map[string]struct { + pod *corev1.Pod + want SweeperState + }{ + "no statuses yet": {podWith(), SweeperState{}}, + "running": {podWith(running), SweeperState{}}, + "exited non-zero": {podWith(terminated("sweeper", 3)), SweeperState{Exited: true, ExitCode: 3}}, + "exited zero": {podWith(terminated("sweeper", 0)), SweeperState{Exited: true}}, + "other container only": {podWith(terminated("publisher", 1)), SweeperState{}}, + } + for name, tc := range cases { + if got := sweeperStateOf(tc.pod, "sweeper"); got != tc.want { + t.Errorf("%s: got %+v, want %+v", name, got, tc.want) + } + } +} + +func TestObservedStatusMirrorsTheRunStatus(t *testing.T) { + when := metav1.NewTime(time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC)) + run := &v1beta2.DynamoGraphDeploymentRun{Status: v1beta2.DynamoGraphDeploymentRunStatus{ + Progress: &v1beta2.DGDRRunProgress{Candidates: &v1beta2.CandidateProgress{Published: 2}}, + LastProgressTime: &when, + CandidateRefs: []corev1.LocalObjectReference{{Name: "x"}, {Name: "y"}}, + }} + want := RunStatus{Published: 2, LastProgress: when.Time, CandidateNames: []string{"x", "y"}} + if got := observedStatus(run); !reflect.DeepEqual(got, want) { + t.Fatalf("got %+v, want %+v", got, want) + } + if got := observedStatus(&v1beta2.DynamoGraphDeploymentRun{}); got.Published != 0 || !got.LastProgress.IsZero() { + t.Fatalf("an empty status must not look like reported progress: %+v", got) + } +} + +func TestCacheOptionsCoverThisRunPodAndNamespaceCandidates(t *testing.T) { + cfg := Config{Namespace: "ns", RunName: "run", RunUID: "uid-1", PodName: "pod-1"} + opts := cacheOptions(cfg) + if _, ok := opts.DefaultNamespaces["ns"]; !ok || len(opts.DefaultNamespaces) != 1 { + t.Fatalf("cache must be limited to the run's namespace: %v", opts.DefaultNamespaces) + } + if len(opts.ByObject) != 3 { + t.Fatalf("want selectors for the run, its candidates and the pod, got %d", len(opts.ByObject)) + } + for object, selector := range opts.ByObject { + switch object.(type) { + case *v1beta2.DynamoGraphDeploymentRun: + if selector.Field.String() != "metadata.name=run" { + t.Errorf("run selector = %q", selector.Field) + } + case *v1beta2.DynamoGraphDeploymentCandidate: + if selector.Field != nil || selector.Label != nil { + t.Errorf("candidates must remain visible after mutable label drift: %+v", selector) + } + case *corev1.Pod: + if selector.Field.String() != "metadata.name=pod-1" { + t.Errorf("pod selector = %q", selector.Field) + } + default: + t.Errorf("unexpected cached type %T", object) + } + } +} + +func nested(t *testing.T, object *unstructured.Unstructured, fields ...string) (any, bool) { + t.Helper() + value, found, err := unstructured.NestedFieldNoCopy(object.Object, fields...) + if err != nil { + t.Fatal(err) + } + return value, found +} + +func TestCandidateApplyPayload(t *testing.T) { + scheme, err := NewScheme() + if err != nil { + t.Fatal(err) + } + run := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testRunName, UID: testRunUID}} + candidate := dgdcreconcile.DesiredCandidate{ID: "a", Spec: validManifest, Parameters: map[string]any{"tp": 2}} + apply, err := buildCandidateApply(scheme, testNamespace, testRunUID, run, cn("a"), candidate) + if err != nil { + t.Fatal(err) + } + if apply.GetAPIVersion() != "nvidia.com/v1beta2" || apply.GetKind() != "DynamoGraphDeploymentCandidate" || apply.GetName() != cn("a") || apply.GetNamespace() != testNamespace { + t.Fatalf("identity = %s %s %s/%s", apply.GetAPIVersion(), apply.GetKind(), apply.GetNamespace(), apply.GetName()) + } + if _, found := nested(t, apply, "status"); found { + t.Fatal("status goes through the status subresource") + } + if _, found := nested(t, apply, "metadata", "creationTimestamp"); found { + t.Fatal("creationTimestamp must not be part of an apply payload") + } + if got := apply.GetLabels(); got[LabelRunUID] != testRunUID || got[LabelCandidateID] != "a" { + t.Fatalf("labels = %v", got) + } + if _, found := nested(t, apply, "spec", "parameters"); !found { + t.Fatal("the resolved parameters belong in the spec") + } + owners := apply.GetOwnerReferences() + if len(owners) != 1 || owners[0].UID != testRunUID || owners[0].Controller == nil || !*owners[0].Controller { + t.Fatalf("owner references = %+v", owners) + } +} + +func TestCandidateStatusApplyPayloadIsDeterministic(t *testing.T) { + candidate := dgdcreconcile.DesiredCandidate{ID: "a", Metrics: map[string]any{"score": 2}} + first, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate) + if err != nil { + t.Fatal(err) + } + second, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(first.Object, second.Object) { + t.Fatal("the same input must produce the same payload") + } + if got, _ := nested(t, first, "metadata", "resourceVersion"); got != "17" { + t.Fatalf("the status write must be constrained to the version it was decided on, got %v", got) + } + if _, found := nested(t, first, "status", "metrics"); !found { + t.Fatal("the metrics are the candidate's status") + } + if _, found := nested(t, first, "status", "conditions"); found { + t.Fatal("candidate status is metrics only") + } +} + +func TestCandidateStatusApplyAlwaysWritesMetricsEvenWhenEmpty(t *testing.T) { + apply, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", dgdcreconcile.DesiredCandidate{ID: "a"}) + if err != nil { + t.Fatal(err) + } + metrics, found := nested(t, apply, "status", "metrics") + if !found { + t.Fatal("an evaluated candidate without metrics must still carry an empty metrics object: its presence marks it populated") + } + if object, ok := metrics.(map[string]any); !ok || len(object) != 0 { + t.Fatalf("metrics = %#v, want an empty object", metrics) + } +} + +func TestRunStatusApplyPayload(t *testing.T) { + when := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) + status := RunStatus{Published: 2, LastProgress: when, CandidateNames: []string{"x", "y"}} + apply := buildRunStatusApply(testNamespace, testRunName, "42", status) + if apply.GetKind() != "DynamoGraphDeploymentRun" || apply.GetName() != testRunName { + t.Fatalf("identity = %s %s", apply.GetKind(), apply.GetName()) + } + if got, _ := nested(t, apply, "metadata", "resourceVersion"); got != "42" { + t.Fatalf("the status write must be constrained to the observed run version, got %v", got) + } + if _, found := nested(t, buildRunStatusApply(testNamespace, testRunName, "", status), "metadata", "resourceVersion"); found { + t.Fatal("no version known, no constraint") + } + if got, _ := nested(t, apply, "status", "lastProgressTime"); got != "2026-10-05T21:35:12Z" { + t.Fatalf("lastProgressTime = %v", got) + } + if got, _ := nested(t, apply, "status", "progress", "candidates", "published"); got != int64(2) { + t.Fatalf("published = %#v", got) + } + for _, field := range []string{"message", "provenance"} { + if _, found := nested(t, apply, "status", field); found { + t.Fatalf("status.%s does not exist in the API and must not be written", field) + } + } + refs, _ := nested(t, apply, "status", "candidateRefs") + if list, ok := refs.([]any); !ok || len(list) != 2 { + t.Fatalf("candidateRefs = %v", refs) + } + // A status without a progress time leaves the field out; the reconciler is what + // carries the known time into the status it writes. + status.LastProgress = time.Time{} + if _, found := nested(t, buildRunStatusApply(testNamespace, testRunName, "42", status), "status", "lastProgressTime"); found { + t.Fatal("no time known, so none is written") + } +} + +func TestRunStatusWriteIsConstrainedToTheObservedRunVersion(t *testing.T) { + f := newFixture(t) + var run v1beta2.DynamoGraphDeploymentRun + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: testRunName}, &run); err != nil { + t.Fatal(err) + } + if run.ResourceVersion == "" { + t.Fatal("the fake client should assign a resourceVersion") + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if len(f.w.statusVersions) != 1 || f.w.statusVersions[0] != run.ResourceVersion { + t.Fatalf("status versions = %v, want [%s]", f.w.statusVersions, run.ResourceVersion) + } +} + +func TestNewCandidateStatusUsesTheVersionReturnedByTheCreate(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got, want := f.w.populateVersions[cn("a")], createdVersion(cn("a")); got != want { + t.Fatalf("populate version = %q, want the version from the create (%q)", got, want) + } +} + +func TestExistingCandidateStatusUsesItsCachedVersion(t *testing.T) { + f := newFixture(t) + existing := candidateForRun(cn("a"), "a", f.currentRun()) + if err := f.c.Create(context.Background(), existing); err != nil { + t.Fatal(err) + } + var cached v1beta2.DynamoGraphDeploymentCandidate + if err := f.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: cn("a")}, &cached); err != nil { + t.Fatal(err) + } + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + if got := f.w.populateVersions[cn("a")]; got == "" || got != cached.ResourceVersion { + t.Fatalf("populate version = %q, want the cached version %q", got, cached.ResourceVersion) + } +} + +func TestConflictIsRetriedWithoutUsingUpTheFinalAttempts(t *testing.T) { + f := newFixture(t) + f.w.failOn["status"] = apierrors.NewConflict(schema.GroupResource{Group: "nvidia.com", Resource: "dynamographdeploymentruns"}, testRunName, errors.New("stale")) + f.w.sticky = true + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.exitSweeper(1) + for attempt := 0; attempt < 2*finalReconcileAttempts; attempt++ { + if err := f.reconcile(); !apierrors.IsConflict(err) || f.r.Completion.Finished() { + t.Fatalf("attempt %d: a conflict from a stale read must be retried, err=%v finished=%v", attempt, err, f.r.Completion.Finished()) + } + } + delete(f.w.failOn, "status") // the cache caught up + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go new file mode 100644 index 000000000000..5bbe05125bbd --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -0,0 +1,201 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "sigs.k8s.io/yaml" +) + +// ErrProtocolViolation means the Sweeper broke the snapshot contract: a snapshot that +// cannot be decoded or validated, one that moves progress backwards, or a missing +// terminal snapshot. It maps to ExitProtocolViolation. +var ErrProtocolViolation = errors.New("snapshot protocol violation") + +func violation(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrProtocolViolation, fmt.Sprintf(format, args...)) +} + +// maxCandidateIDLength keeps ids well inside the 63-character Kubernetes label-value +// limit, since the id is recorded verbatim as a label on the candidate. +const maxCandidateIDLength = 48 + +// candidateIDPattern is a DNS-1123 label, valid both as a label value and inside an +// object name. +var candidateIDPattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +const dgdKind = "DynamoGraphDeployment" + +// supportedDGDAPIVersions are the DGD versions the candidate path can convert to v1beta1. +var supportedDGDAPIVersions = map[string]bool{ + "nvidia.com/v1alpha1": true, + "nvidia.com/v1beta1": true, +} + +// Manifest is one rendered candidate: exactly one DynamoGraphDeployment plus any +// companion resources (for example generated ConfigMaps) the renderer emitted with it. +type Manifest struct { + DGD map[string]any + Companions []string // raw YAML documents, in order +} + +var documentSeparator = regexp.MustCompile(`(?m)^---[ \t]*$`) + +// ParseManifest splits a multi-document manifest and locates its single +// DynamoGraphDeployment, which must carry a spec. +func ParseManifest(manifest string) (*Manifest, error) { + var out Manifest + for _, doc := range documentSeparator.Split(manifest, -1) { + if strings.TrimSpace(doc) == "" { + continue + } + var obj map[string]any + if err := yaml.Unmarshal([]byte(doc), &obj); err != nil { + return nil, fmt.Errorf("manifest document is not a YAML mapping: %w", err) + } + if len(obj) == 0 { + continue // comment-only document + } + if obj["kind"] != dgdKind { + out.Companions = append(out.Companions, strings.TrimSpace(doc)) + continue + } + if out.DGD != nil { + return nil, errors.New("manifest has more than one DynamoGraphDeployment") + } + if version, _ := obj["apiVersion"].(string); !supportedDGDAPIVersions[version] { + return nil, fmt.Errorf("DynamoGraphDeployment has unsupported apiVersion %q", version) + } + if _, ok := obj["spec"].(map[string]any); !ok { + return nil, errors.New("DynamoGraphDeployment has no spec") + } + out.DGD = obj + } + if out.DGD == nil { + return nil, errors.New("manifest has no DynamoGraphDeployment") + } + return &out, nil +} + +func validateManifest(id, manifest string) error { + if _, err := ParseManifest(manifest); err != nil { + return violation("manifest of candidate %q: %v", id, err) + } + return nil +} + +// SnapshotSchemaVersion is the only wire version this publisher understands. +const SnapshotSchemaVersion = 1 + +// Candidate outcomes carried by the snapshot. +const ( + OutcomeMaterialized = "materialized" + OutcomeMaterializationFailed = "materialization_failed" +) + +// Run phases carried by the snapshot. +const ( + PhaseRunning = "Running" + PhaseSucceeded = "Succeeded" + PhaseFailed = "Failed" +) + +// Snapshot is the complete current projection of one run, written atomically by the +// Sweeper container (see dynamo.aisimulate.output.dgdr_run.snapshot). It is desired +// state, not a command list: each snapshot supersedes the previous one entirely. +type Snapshot struct { + SchemaVersion int `json:"schemaVersion"` + Timestamp string `json:"timestamp,omitempty"` + Run SnapshotRun `json:"run"` + Progress SnapshotProgress `json:"progress"` + Candidates []SnapshotCandidate `json:"candidates"` +} + +type SnapshotRun struct { + Phase string `json:"phase"` + Terminal bool `json:"terminal"` + Message string `json:"message,omitempty"` + Error string `json:"error,omitempty"` +} + +type SnapshotProgress struct { + Round int32 `json:"round"` + Evaluated int32 `json:"evaluated"` +} + +// SnapshotCandidate is one evaluated point. Candidates are ordered best-first for +// scalar searches; the id never depends on that order. +type SnapshotCandidate struct { + ID string `json:"id"` + Outcome string `json:"outcome"` + Parameters map[string]any `json:"parameters,omitempty"` + Metrics map[string]any `json:"metrics,omitempty"` + Manifest string `json:"manifest,omitempty"` + Error string `json:"error,omitempty"` +} + +func ParseSnapshot(data []byte) (*Snapshot, error) { + var snap Snapshot + if err := yaml.Unmarshal(data, &snap); err != nil { + return nil, violation("decoding snapshot: %v", err) + } + if snap.SchemaVersion != SnapshotSchemaVersion { + return nil, violation("unsupported snapshot schemaVersion %d (want %d)", snap.SchemaVersion, SnapshotSchemaVersion) + } + switch snap.Run.Phase { + case PhaseRunning, PhaseSucceeded, PhaseFailed: + default: + return nil, violation("unknown run phase %q", snap.Run.Phase) + } + if snap.Run.Terminal != (snap.Run.Phase != PhaseRunning) { + return nil, violation("run phase %q inconsistent with terminal=%t", snap.Run.Phase, snap.Run.Terminal) + } + seen := make(map[string]struct{}, len(snap.Candidates)) + for i, candidate := range snap.Candidates { + if candidate.ID == "" { + return nil, violation("candidate %d has an empty id", i) + } + if len(candidate.ID) > maxCandidateIDLength || !candidateIDPattern.MatchString(candidate.ID) { + return nil, violation("candidate id %q is not a lowercase DNS label of at most %d characters", candidate.ID, maxCandidateIDLength) + } + if _, dup := seen[candidate.ID]; dup { + return nil, violation("duplicate candidate id %q", candidate.ID) + } + seen[candidate.ID] = struct{}{} + switch candidate.Outcome { + case OutcomeMaterialized: + if candidate.Manifest == "" { + return nil, violation("materialized candidate %q has no manifest", candidate.ID) + } + if err := validateManifest(candidate.ID, candidate.Manifest); err != nil { + return nil, err + } + case OutcomeMaterializationFailed: + if candidate.Error == "" { + return nil, violation("failed candidate %q has no error", candidate.ID) + } + default: + return nil, violation("candidate %q has unknown outcome %q", candidate.ID, candidate.Outcome) + } + } + return &snap, nil +} diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go new file mode 100644 index 000000000000..e0287a5c36d3 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go @@ -0,0 +1,174 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +const validManifest = "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" + +const testRunUID = "uid-1" + +func cn(id string) string { return CandidateName("run", testRunUID, id) } + +type cand struct { + id string + failed bool + metrics map[string]any +} + +// snapshotJSON renders a snapshot (JSON is valid YAML, which the real parser accepts). +func snapshotJSON(t *testing.T, phase string, cands ...cand) []byte { + t.Helper() + var list []map[string]any + for _, c := range cands { + entry := map[string]any{"id": c.id, "parameters": map[string]any{"tp": 2}, "metrics": c.metrics} + if c.failed { + entry["outcome"] = OutcomeMaterializationFailed + entry["error"] = "boom" + } else { + entry["outcome"] = OutcomeMaterialized + entry["manifest"] = validManifest + } + list = append(list, entry) + } + data, err := json.Marshal(map[string]any{ + "schemaVersion": 1, + "run": map[string]any{"phase": phase, "terminal": phase != PhaseRunning}, + "progress": map[string]any{"round": 3, "evaluated": len(cands)}, + "candidates": list, + }) + if err != nil { + t.Fatal(err) + } + return data +} + +func writeSnapshot(t *testing.T, dir string, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, SnapshotFileName), data, 0o600); err != nil { + t.Fatal(err) + } +} + +func mustParse(t *testing.T, data []byte) *Snapshot { + t.Helper() + snap, err := ParseSnapshot(data) + if err != nil { + t.Fatal(err) + } + return snap +} + +func withTimestamp(t *testing.T, data []byte, timestamp string) []byte { + t.Helper() + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + doc["timestamp"] = timestamp + out, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return out +} + +func withRunDiagnostics(t *testing.T, data []byte, message, runError string) []byte { + t.Helper() + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + run := doc["run"].(map[string]any) + run["message"] = message + run["error"] = runError + out, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestParseSnapshotValidation(t *testing.T) { + bad := map[string]string{ + "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, + "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, + "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, + "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, + "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, + "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, + "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + } + bad["long id"] = strings.Replace(bad["long id"], "%s", strings.Repeat("a", 64), 1) + for name, doc := range bad { + if _, err := ParseSnapshot([]byte(doc)); !errors.Is(err, ErrProtocolViolation) { + t.Errorf("%s: want protocol violation, got %v", name, err) + } + } +} + +func TestParseManifestHandlesCompanionResources(t *testing.T) { + dgd := "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\nspec: {}\n" + cm := "apiVersion: v1\nkind: ConfigMap\nmetadata: {}\n" + for name, manifest := range map[string]string{ + "dgd first": dgd + "---\n" + cm, + "configmap first": "# generated\n---\n" + cm + "---\n" + dgd, + } { + got, err := ParseManifest(manifest) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if got.DGD["kind"] != "DynamoGraphDeployment" || len(got.Companions) != 1 || !strings.Contains(got.Companions[0], "ConfigMap") { + t.Fatalf("%s: %+v", name, got) + } + } + for name, manifest := range map[string]string{ + "two dgds": dgd + "---\n" + dgd, + "no dgd": cm, + "no spec": "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n", + "no apiVersion": "kind: DynamoGraphDeployment\nspec: {}\n", + "unsupported apiVersion": "apiVersion: nvidia.com/v9\nkind: DynamoGraphDeployment\nspec: {}\n", + } { + if _, err := ParseManifest(manifest); err == nil { + t.Errorf("%s: want error", name) + } + } +} +func writeProgress(t *testing.T, dir, phase string, round, evaluated int) { + t.Helper() + doc := map[string]any{} + if err := json.Unmarshal(snapshotJSON(t, phase, cand{id: "a"}), &doc); err != nil { + t.Fatal(err) + } + doc["progress"] = map[string]any{"round": round, "evaluated": evaluated} + data, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + writeSnapshot(t, dir, data) +} diff --git a/deploy/operator/internal/dgdrrunpublisher/terminal.go b/deploy/operator/internal/dgdrrunpublisher/terminal.go new file mode 100644 index 000000000000..a9ddda4e0c59 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/terminal.go @@ -0,0 +1,115 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "strings" + "unicode" + "unicode/utf8" +) + +// DefaultTerminationLog is where the kubelet reads a container's termination message +// from unless the pod spec says otherwise. The message ends up in the container's +// terminated state, which is how the run controller learns why the publisher failed a +// run without parsing logs. +const DefaultTerminationLog = "/dev/termination-log" + +// maxDiagnosticBytes bounds each free-text field carried out of the publisher. The kubelet +// keeps at most 4096 bytes of a termination message, and the whole message is JSON, so +// two fields of this size (every byte possibly escaped) plus the phase stay inside it. +const maxDiagnosticBytes = 768 + +// TerminalOutcome is the producer's account of how the run ended: the terminal phase and +// the bounded free-text diagnostics from the snapshot. It is the payload of the handoff to +// the run controller, so a Completed condition can say why, not only that the Job failed. +type TerminalOutcome struct { + Phase string `json:"phase,omitempty"` + Message string `json:"message,omitempty"` + Error string `json:"error,omitempty"` +} + +// RunFailedError is the publisher's outcome when the run ended without a usable result. +// It wraps ErrRunFailed, so errors.Is(err, ErrRunFailed) and ExitCode keep working, and +// carries the bounded diagnostics for the handoff. +type RunFailedError struct { + Outcome TerminalOutcome +} + +func (e *RunFailedError) Error() string { + parts := []string{ErrRunFailed.Error()} + for _, detail := range []string{e.Outcome.Message, e.Outcome.Error} { + if detail != "" { + parts = append(parts, detail) + } + } + return strings.Join(parts, ": ") +} + +// Unwrap keeps errors.Is(err, ErrRunFailed) true. +func (e *RunFailedError) Unwrap() error { return ErrRunFailed } + +// boundDiagnostic makes producer-supplied text safe to carry: valid UTF-8, no control +// characters (a multi-line trace becomes one line), at most maxDiagnosticBytes, cut on a +// character boundary. +func boundDiagnostic(text string) string { + text = strings.ToValidUTF8(strings.TrimSpace(text), "?") + text = strings.Map(func(r rune) rune { + if unicode.IsControl(r) { + return ' ' + } + return r + }, text) + if len(text) <= maxDiagnosticBytes { + return text + } + const ellipsis = "…" + cut := maxDiagnosticBytes - len(ellipsis) + for cut > 0 && !utf8.RuneStart(text[cut]) { + cut-- + } + return text[:cut] + ellipsis +} + +// TerminationOutcome is what the publisher reports for err: the run's own outcome for a +// run failure, otherwise the bounded error text. +func TerminationOutcome(err error) TerminalOutcome { + var failed *RunFailedError + if errors.As(err, &failed) { + return failed.Outcome + } + return TerminalOutcome{Error: boundDiagnostic(err.Error())} +} + +// WriteTerminationMessage records the outcome of a failed publisher as JSON at path (the +// container termination log). It writes nothing for a nil error. +func WriteTerminationMessage(path string, err error) error { + if err == nil { + return nil + } + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if encodeErr := encoder.Encode(TerminationOutcome(err)); encodeErr != nil { + return encodeErr + } + return os.WriteFile(path, buf.Bytes(), 0o644) +} diff --git a/deploy/operator/internal/dgdrrunpublisher/terminal_test.go b/deploy/operator/internal/dgdrrunpublisher/terminal_test.go new file mode 100644 index 000000000000..2ca586817368 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/terminal_test.go @@ -0,0 +1,142 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf8" +) + +func TestTerminalResultIsAStructuredRunFailure(t *testing.T) { + var failure *RunFailedError + err := terminalResult(DesiredState{Phase: PhaseFailed, Message: "m", Error: "e"}) + if !errors.Is(err, ErrRunFailed) || !errors.As(err, &failure) || failure.Outcome != (TerminalOutcome{Phase: PhaseFailed, Message: "m", Error: "e"}) { + t.Fatalf("failed: %v", err) + } + if got, want := err.Error(), ErrRunFailed.Error()+": m: e"; got != want { + t.Fatalf("error text = %q, want %q", got, want) + } + err = terminalResult(DesiredState{Phase: PhaseSucceeded, Failed: 1, FirstFailure: "x: render failed"}) + if !errors.As(err, &failure) || failure.Outcome.Error != "x: render failed" || !strings.Contains(failure.Outcome.Message, "1 candidate materialization failure") { + t.Fatalf("empty success: %v", err) + } +} + +func TestDesireCarriesBoundedDiagnostics(t *testing.T) { + long := strings.Repeat("e", 3*maxDiagnosticBytes) + state := desire(t, withRunDiagnostics(t, snapshotJSON(t, PhaseFailed, cand{id: "a"}), "no recommendation\ncould be materialized", long)) + if state.Phase != PhaseFailed || state.Message != "no recommendation could be materialized" { + t.Fatalf("state = %+v", state) + } + if len(state.Error) > maxDiagnosticBytes || !strings.HasPrefix(state.Error, "eee") { + t.Fatalf("error must be kept but bounded, got %d bytes", len(state.Error)) + } +} + +func TestBoundDiagnostic(t *testing.T) { + if got := boundDiagnostic(" a\nb\x00c \t"); got != "a b c" { + t.Fatalf("got %q", got) + } + multibyte := strings.Repeat("é", maxDiagnosticBytes) // two bytes each, so the cut must not split one + got := boundDiagnostic(multibyte) + if len(got) > maxDiagnosticBytes || !utf8.ValidString(got) || !strings.HasSuffix(got, "…") { + t.Fatalf("got %d bytes, valid=%t", len(got), utf8.ValidString(got)) + } + if got := boundDiagnostic("bad\xffbyte"); !utf8.ValidString(got) { + t.Fatalf("invalid UTF-8 must be replaced: %q", got) + } +} + +func TestTerminalSucceededWithoutAMaterializedCandidateCarriesTheCandidateFailure(t *testing.T) { + f := newFixture(t) + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "x", failed: true})) + f.mustReconcile() + var failure *RunFailedError + err := f.r.Completion.Err() + if !errors.Is(err, ErrRunFailed) || !errors.As(err, &failure) || ExitCode(err) != ExitRunFailed { + t.Fatalf("err = %v", err) + } + if !strings.HasPrefix(failure.Outcome.Error, "x: ") { + t.Fatalf("the candidate failure must be part of the diagnostics: %+v", failure.Outcome) + } +} + +func TestTerminalFailureDiagnosticsSurviveTheCompletionHandoff(t *testing.T) { + f := newFixture(t) + message := "no recommendation could be materialized" + long := "backend vllm rejected the generated worker configuration " + strings.Repeat("x", 3*maxDiagnosticBytes) + f.publish(withRunDiagnostics(t, snapshotJSON(t, PhaseFailed, cand{id: "a"}), message, long)) + f.mustReconcile() + + err := f.r.Completion.Err() + var failure *RunFailedError + if !errors.Is(err, ErrRunFailed) || !errors.As(err, &failure) || ExitCode(err) != ExitRunFailed { + t.Fatalf("err = %v", err) + } + if failure.Outcome.Message != message || !strings.HasPrefix(failure.Outcome.Error, "backend vllm rejected") || len(failure.Outcome.Error) > maxDiagnosticBytes { + t.Fatalf("outcome = %+v", failure.Outcome) + } + + // The handoff the run controller reads: the publisher container's termination message. + path := filepath.Join(t.TempDir(), "termination-log") + if err := WriteTerminationMessage(path, err); err != nil { + t.Fatal(err) + } + data, readErr := os.ReadFile(path) + if readErr != nil { + t.Fatal(readErr) + } + if len(data) > 4096 { + t.Fatalf("termination message is %d bytes; the kubelet keeps 4096", len(data)) + } + var got TerminalOutcome + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + if got != failure.Outcome { + t.Fatalf("termination message = %+v, want %+v", got, failure.Outcome) + } +} + +func TestTerminationMessageForOtherFailuresAndSuccess(t *testing.T) { + dir := t.TempDir() + untouched := filepath.Join(dir, "success") + if err := WriteTerminationMessage(untouched, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(untouched); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("success writes no termination message: %v", err) + } + other := filepath.Join(dir, "other") + if err := WriteTerminationMessage(other, errors.New("api down")); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(other) + if err != nil { + t.Fatal(err) + } + var got TerminalOutcome + if err := json.Unmarshal(data, &got); err != nil || got.Error != "api down" || got.Phase != "" { + t.Fatalf("got %+v, err %v", got, err) + } +} diff --git a/deploy/operator/internal/dgdrrunpublisher/writer.go b/deploy/operator/internal/dgdrrunpublisher/writer.go new file mode 100644 index 000000000000..e3b1030f59c6 --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/writer.go @@ -0,0 +1,268 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "fmt" + "time" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// StoredCandidate is the API server's view of a candidate right after it was applied. +type StoredCandidate struct { + // ResourceVersion is the version a status write for this candidate is constrained to. + ResourceVersion string + // Populated reports that the candidate already has its evaluation status (a metrics + // object, possibly empty), for example when a stale cache led the reconciler to apply a + // candidate that exists. + Populated bool +} + +// writer is every write the reconciler makes to the API server. It is a seam so the +// reconciler's behavior can be tested without server-side apply support in a fake client. +type writer interface { + // CreateCandidate atomically creates a missing candidate or reapplies one already + // controlled by the Run, then returns the API server's own copy. It must never adopt + // an object that is not controlled by the Run. + CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) + // PopulateStatus records the candidate's evaluation metrics. The + // write is constrained to resourceVersion, so a stale view of the candidate fails + // with a conflict instead of rewriting status that was already populated. + PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error + // UpdateRunStatus writes the publisher-owned run status fields, constrained to the + // resourceVersion of the run the decision to write was based on. + UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error + // DeleteCandidate deletes the candidate with the given UID precondition. + DeleteCandidate(ctx context.Context, name string, uid types.UID) error +} + +// clientWriter writes with server-side apply through a controller-runtime client. +type clientWriter struct { + Client client.Client + // Reader reads straight from the API server. A candidate's resourceVersion right + // after its creation cannot come from the informer cache, which has not seen it yet. + Reader client.Reader + Namespace string + RunName string + RunUID string +} + +var _ writer = (*clientWriter)(nil) + +func (w *clientWriter) apply(ctx context.Context, object *unstructured.Unstructured, status bool) error { + configuration := client.ApplyConfigurationFromUnstructured(object) + if status { + return w.Client.Status().Apply(ctx, configuration, client.FieldOwner(fieldOwner), client.ForceOwnership) + } + return w.Client.Apply(ctx, configuration, client.FieldOwner(fieldOwner), client.ForceOwnership) +} + +// CreateCandidate creates a candidate atomically, or reapplies it after a live ownership +// check when the cache missed an existing object. The apply is constrained to the UID and +// resourceVersion returned by Create or Get, so a concurrent delete/recreate or ownership +// change conflicts instead of being adopted. The final live read tells the caller whether +// status is populated already: the cache cannot, because it is exactly what may be stale. +func (w *clientWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (StoredCandidate, error) { + object, err := buildCandidateApply(w.Client.Scheme(), w.Namespace, w.RunUID, run, name, candidate) + if err != nil { + return StoredCandidate{}, err + } + + // POST provides the missing-object precondition that server-side apply lacks. If the + // name is already occupied, classify the live object before applying anything to it. + var observedUID types.UID + created := object.DeepCopy() + if err := w.Client.Create(ctx, created, client.FieldOwner(fieldOwner)); err == nil { + object.SetResourceVersion(created.GetResourceVersion()) + observedUID = created.GetUID() + } else if apierrors.IsAlreadyExists(err) { + var existing v1beta2.DynamoGraphDeploymentCandidate + if err := w.Reader.Get(ctx, types.NamespacedName{Namespace: w.Namespace, Name: name}, &existing); err != nil { + return StoredCandidate{}, fmt.Errorf("reading existing candidate %s: %w", name, err) + } + owner := metav1.GetControllerOf(&existing) + if owner == nil || owner.UID != run.UID { + return StoredCandidate{}, fmt.Errorf("candidate %s already exists without controller ownership by run %s", name, run.Name) + } + object.SetResourceVersion(existing.ResourceVersion) + observedUID = existing.UID + } else { + return StoredCandidate{}, fmt.Errorf("creating candidate %s: %w", name, err) + } + object.SetUID(observedUID) + + if err := w.apply(ctx, object, false); err != nil { + return StoredCandidate{}, err + } + var stored v1beta2.DynamoGraphDeploymentCandidate + if err := w.Reader.Get(ctx, types.NamespacedName{Namespace: w.Namespace, Name: name}, &stored); err != nil { + return StoredCandidate{}, err + } + owner := metav1.GetControllerOf(&stored) + if stored.UID != observedUID { + return StoredCandidate{}, fmt.Errorf("candidate %s changed uid from %s to %s during apply", name, observedUID, stored.UID) + } + if owner == nil || owner.UID != run.UID { + return StoredCandidate{}, fmt.Errorf("candidate %s is no longer controlled by run %s after apply", name, run.Name) + } + return StoredCandidate{ + ResourceVersion: stored.ResourceVersion, + Populated: stored.Status.Metrics != nil, + }, nil +} + +func (w *clientWriter) PopulateStatus(ctx context.Context, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) error { + object, err := buildCandidateStatusApply(w.Namespace, name, resourceVersion, candidate) + if err != nil { + return err + } + return w.apply(ctx, object, true) +} + +func (w *clientWriter) UpdateRunStatus(ctx context.Context, resourceVersion string, status RunStatus) error { + return w.apply(ctx, buildRunStatusApply(w.Namespace, w.RunName, resourceVersion, status), true) +} + +func (w *clientWriter) DeleteCandidate(ctx context.Context, name string, uid types.UID) error { + object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: w.Namespace, Name: name}} + var options []client.DeleteOption + if uid != "" { + options = append(options, client.Preconditions{UID: &uid}) + } + return client.IgnoreNotFound(w.Client.Delete(ctx, object, options...)) +} + +// buildCandidateApply is the server-side-apply payload that creates a candidate: labels +// and controller owner reference, the DGD spec with the resolved parameters, and the +// companion resources as an annotation. The status goes through the status subresource. +func buildCandidateApply(scheme *runtime.Scheme, namespace, runUID string, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (*unstructured.Unstructured, error) { + spec, companions, err := decodeManifest(candidate) + if err != nil { + return nil, err + } + parameters, err := normalizeJSON(candidate.Parameters) + if err != nil { + return nil, err + } + object := &v1beta2.DynamoGraphDeploymentCandidate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Labels: map[string]string{LabelRunUID: runUID, LabelCandidateID: candidate.ID}, + }, + Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{DynamoGraphDeploymentSpec: *spec}, + } + if companions != "" { + object.Annotations = map[string]string{AnnotationAdditionalResources: companions} + } + if err := controllerutil.SetControllerReference(run, object, scheme); err != nil { + return nil, err + } + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object) + if err != nil { + return nil, err + } + delete(content, "status") + unstructured.RemoveNestedField(content, "metadata", "creationTimestamp") + if err := unstructured.SetNestedField(content, parameters, "spec", "parameters"); err != nil { + return nil, err + } + apply := &unstructured.Unstructured{Object: content} + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentCandidateGVK) + return apply, nil +} + +// withResourceVersion constrains a server-side apply to one version of the object: the +// API server rejects it with a conflict if the object has changed since. +func withResourceVersion(metadata map[string]any, resourceVersion string) map[string]any { + if resourceVersion != "" { + metadata["resourceVersion"] = resourceVersion + } + return metadata +} + +// buildCandidateStatusApply is the status payload of a candidate: the metrics. Metrics are +// always written, even when empty, because their presence is what marks the candidate as +// populated. Candidate status is written once and then stays immutable, so the payload +// carries the resourceVersion the "not populated yet" decision was based on. +func buildCandidateStatusApply(namespace, name, resourceVersion string, candidate dgdcreconcile.DesiredCandidate) (*unstructured.Unstructured, error) { + metrics, err := normalizeJSON(candidate.Metrics) + if err != nil { + return nil, err + } + apply := &unstructured.Unstructured{Object: map[string]any{ + "metadata": withResourceVersion(map[string]any{"namespace": namespace, "name": name}, resourceVersion), + "status": map[string]any{"metrics": metrics}, + }} + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentCandidateGVK) + return apply, nil +} + +// buildRunStatusApply is the status payload of the run: the candidate references, the +// published-candidate count and the progress time. It lists every publisher-owned +// field it wants to keep: under server-side apply a field this manager owns and omits +// from the payload is removed. It carries the resourceVersion of the run the decision to +// write was based on, so a decision made from a stale cached run conflicts instead of +// overwriting a newer status. +func buildRunStatusApply(namespace, runName, resourceVersion string, status RunStatus) *unstructured.Unstructured { + refs := make([]any, 0, len(status.CandidateNames)) + for _, name := range status.CandidateNames { + refs = append(refs, map[string]any{"name": name}) + } + fields := map[string]any{ + "candidateRefs": refs, + "progress": map[string]any{"candidates": map[string]any{"published": int64(status.Published)}}, + } + if !status.LastProgress.IsZero() { + fields["lastProgressTime"] = status.LastProgress.UTC().Format(time.RFC3339) + } + apply := &unstructured.Unstructured{Object: map[string]any{ + "metadata": withResourceVersion(map[string]any{"namespace": namespace, "name": runName}, resourceVersion), + "status": fields, + }} + apply.SetGroupVersionKind(v1beta2.DynamoGraphDeploymentRunGVK) + return apply +} + +// normalizeJSON round-trips a value through JSON so it only holds the types unstructured +// objects allow (int64, float64, string, bool, maps, slices). +func normalizeJSON(value map[string]any) (map[string]any, error) { + out := map[string]any{} + if value == nil { + return out, nil + } + raw, err := json.Marshal(value) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &out); err != nil { + return nil, err + } + return out, nil +} diff --git a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go new file mode 100644 index 000000000000..6437e8c363cb --- /dev/null +++ b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go @@ -0,0 +1,559 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dgdrrunpublisher + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorenv" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +// These tests send the publisher's real server-side-apply payloads to an API server that +// has the v1beta2 CRDs installed. The reconciler tests use a fake writer; this is the +// check that the payloads are accepted by the real schema, that the publisher owns what +// it writes, and that the resourceVersion constraints behave as the reconciler assumes. + +type writerFixture struct { + t *testing.T + ctx context.Context + client client.Client + config *rest.Config + run *v1beta2.DynamoGraphDeploymentRun + writer *clientWriter +} + +func newWriterFixture(t *testing.T) *writerFixture { + t.Helper() + env := operatorenv.New(operatorenv.Options{ + // The publisher has no webhooks; the environment only requires the hook to exist. + SetupWebhooks: func(ctrl.Manager, operatorenv.WebhookSetupOptions) error { return nil }, + }).RunT(t) + ctx := context.Background() + c := env.Client() + + run := &v1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{Namespace: env.Namespace(), Name: "publisher-run"}, + Spec: v1beta2.DynamoGraphDeploymentRunSpec{ + NativeSearchInput: v1beta2.NativeSearchInput{Model: &v1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}}, + }, + } + if err := c.Create(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + return &writerFixture{ + t: t, + ctx: ctx, + client: c, + config: env.RESTConfig(), + run: run, + writer: &clientWriter{ + Client: c, + Reader: c, + Namespace: run.Namespace, + RunName: run.Name, + RunUID: string(run.UID), + }, + } +} + +func (f *writerFixture) name(id string) string { + return CandidateName(f.run.Name, string(f.run.UID), id) +} + +func (f *writerFixture) candidate(name string) *v1beta2.DynamoGraphDeploymentCandidate { + f.t.Helper() + var got v1beta2.DynamoGraphDeploymentCandidate + if err := f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: name}, &got); err != nil { + f.t.Fatalf("get candidate %s: %v", name, err) + } + return &got +} + +func (f *writerFixture) currentRun() *v1beta2.DynamoGraphDeploymentRun { + f.t.Helper() + var got v1beta2.DynamoGraphDeploymentRun + if err := f.client.Get(f.ctx, client.ObjectKeyFromObject(f.run), &got); err != nil { + f.t.Fatalf("get run: %v", err) + } + return &got +} + +func envtestCandidate() dgdcreconcile.DesiredCandidate { + return dgdcreconcile.DesiredCandidate{ + ID: "a", + Spec: validManifest + "---\napiVersion: v1\nkind: ConfigMap\nmetadata:\n name: companion\n", + Parameters: map[string]any{"tp": 2}, + Metrics: map[string]any{"throughput": 12.5}, + } +} + +func foreignCandidate(t *testing.T, namespace, name string, candidate dgdcreconcile.DesiredCandidate) *v1beta2.DynamoGraphDeploymentCandidate { + t.Helper() + spec, _, err := decodeManifest(candidate) + if err != nil { + t.Fatal(err) + } + parameters, err := json.Marshal(candidate.Parameters) + if err != nil { + t.Fatal(err) + } + return &v1beta2.DynamoGraphDeploymentCandidate{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: name, + Labels: map[string]string{"owner": "foreign"}, + }, + Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{ + DynamoGraphDeploymentSpec: *spec, + Parameters: runtime.RawExtension{Raw: parameters}, + }, + } +} + +type writeInterceptClient struct { + client.Client + beforeCreate func() + beforeApply func() +} + +func (c *writeInterceptClient) Create(ctx context.Context, object client.Object, options ...client.CreateOption) error { + if c.beforeCreate != nil { + hook := c.beforeCreate + c.beforeCreate = nil + hook() + } + return c.Client.Create(ctx, object, options...) +} + +func (c *writeInterceptClient) Apply(ctx context.Context, configuration runtime.ApplyConfiguration, options ...client.ApplyOption) error { + if c.beforeApply != nil { + hook := c.beforeApply + c.beforeApply = nil + hook() + } + return c.Client.Apply(ctx, configuration, options...) +} + +type reconcileRecorder struct { + delegate *Reconciler + count atomic.Int64 +} + +func (r *reconcileRecorder) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + result, err := r.delegate.Reconcile(ctx, request) + r.count.Add(1) + return result, err +} + +func waitForReconcileQuiet(t *testing.T, recorder *reconcileRecorder, quiet, timeout time.Duration) { + t.Helper() + var last int64 = -1 + var stableSince time.Time + require.Eventually(t, func() bool { + current := recorder.count.Load() + if current == 0 { + return false + } + if current != last { + last = current + stableSince = time.Now() + return false + } + return time.Since(stableSince) >= quiet + }, timeout, 10*time.Millisecond, "reconciliation count did not become stable") +} + +func hasApplyManager(fields []metav1.ManagedFieldsEntry, subresource string) bool { + for _, entry := range fields { + if entry.Manager == fieldOwner && entry.Operation == metav1.ManagedFieldsOperationApply && entry.Subresource == subresource { + return true + } + } + return false +} + +func TestEnvtestCandidateApplyIsAcceptedAndIdempotent(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + + stored, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()) + if err != nil { + t.Fatalf("create candidate: %v", err) + } + if stored.ResourceVersion == "" || stored.Populated { + t.Fatalf("stored = %+v, want a resourceVersion and no status yet", stored) + } + + got := f.candidate(name) + if got.Labels[LabelRunUID] != string(f.run.UID) || got.Labels[LabelCandidateID] != "a" { + t.Errorf("labels = %v", got.Labels) + } + if owner := metav1.GetControllerOf(got); owner == nil || owner.UID != f.run.UID || owner.Kind != "DynamoGraphDeploymentRun" { + t.Errorf("controller owner = %+v, want the run", owner) + } + if !strings.Contains(got.Annotations[AnnotationAdditionalResources], "kind: ConfigMap") { + t.Errorf("companion annotation = %q", got.Annotations[AnnotationAdditionalResources]) + } + if !hasApplyManager(got.ManagedFields, "") { + t.Errorf("no apply manager %q on the candidate: %+v", fieldOwner, got.ManagedFields) + } + + again, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()) + if err != nil { + t.Fatalf("re-apply candidate: %v", err) + } + if again.ResourceVersion != stored.ResourceVersion { + t.Errorf("re-applying the same candidate changed resourceVersion %s -> %s", stored.ResourceVersion, again.ResourceVersion) + } + + drifted := f.candidate(name) + delete(drifted.Labels, LabelRunUID) + if err := f.client.Update(f.ctx, drifted); err != nil { + t.Fatalf("remove run label: %v", err) + } + if _, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()); err != nil { + t.Fatalf("repair candidate labels: %v", err) + } + if got := f.candidate(name); got.Labels[LabelRunUID] != string(f.run.UID) { + t.Fatalf("run label was not repaired: %v", got.Labels) + } +} + +func TestEnvtestCandidateApplyDoesNotAdoptAForeignObject(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + candidate := envtestCandidate() + foreign := foreignCandidate(t, f.run.Namespace, name, candidate) + if err := f.client.Create(f.ctx, foreign); err != nil { + t.Fatalf("create foreign candidate: %v", err) + } + + if _, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate); err == nil { + t.Fatal("applying a candidate over a foreign object succeeded") + } + got := f.candidate(name) + if owner := metav1.GetControllerOf(got); owner != nil { + t.Fatalf("foreign candidate was adopted by %+v", owner) + } + if got.Labels["owner"] != "foreign" || got.Labels[LabelRunUID] != "" { + t.Fatalf("foreign candidate was modified: labels = %v", got.Labels) + } +} + +func TestEnvtestCandidateCreateRaceDoesNotAdoptForeignObject(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + candidate := envtestCandidate() + foreign := foreignCandidate(t, f.run.Namespace, name, candidate) + intercept := &writeInterceptClient{Client: f.client} + intercept.beforeCreate = func() { + if err := f.client.Create(f.ctx, foreign); err != nil { + t.Fatalf("create racing foreign candidate: %v", err) + } + } + writer := *f.writer + writer.Client = intercept + + if _, err := writer.CreateCandidate(f.ctx, f.run, name, candidate); err == nil { + t.Fatal("candidate creation adopted the foreign object that won the name race") + } + got := f.candidate(name) + if owner := metav1.GetControllerOf(got); owner != nil { + t.Fatalf("racing foreign candidate was adopted by %+v", owner) + } + if got.Labels["owner"] != "foreign" || got.Labels[LabelRunUID] != "" { + t.Fatalf("racing foreign candidate was modified: labels = %v", got.Labels) + } +} + +func TestEnvtestCandidateApplyRejectsDeleteRecreateRace(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + candidate := envtestCandidate() + if _, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate); err != nil { + t.Fatalf("create owned candidate: %v", err) + } + ownedUID := f.candidate(name).UID + foreign := foreignCandidate(t, f.run.Namespace, name, candidate) + intercept := &writeInterceptClient{Client: f.client} + intercept.beforeApply = func() { + current := f.candidate(name) + if err := f.client.Delete(f.ctx, current); err != nil { + t.Fatalf("delete owned candidate during apply race: %v", err) + } + if err := f.client.Create(f.ctx, foreign); err != nil { + t.Fatalf("recreate foreign candidate during apply race: %v", err) + } + } + writer := *f.writer + writer.Client = intercept + + if _, err := writer.CreateCandidate(f.ctx, f.run, name, candidate); err == nil { + t.Fatal("UID- and resourceVersion-constrained apply accepted a replacement object") + } + got := f.candidate(name) + if got.UID == ownedUID { + t.Fatalf("candidate UID did not change across delete/recreate: %s", got.UID) + } + if owner := metav1.GetControllerOf(got); owner != nil { + t.Fatalf("replacement foreign candidate was adopted by %+v", owner) + } + if got.Labels["owner"] != "foreign" || got.Labels[LabelRunUID] != "" { + t.Fatalf("replacement foreign candidate was modified: labels = %v", got.Labels) + } +} + +func TestEnvtestCandidateStatusIsWrittenOnceAndGuardedByResourceVersion(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + candidate := envtestCandidate() + + stored, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate) + if err != nil { + t.Fatalf("create candidate: %v", err) + } + if err := f.writer.PopulateStatus(f.ctx, name, stored.ResourceVersion, candidate); err != nil { + t.Fatalf("populate status: %v", err) + } + got := f.candidate(name) + if got.Status.Metrics == nil { + t.Fatal("status.metrics is not set after PopulateStatus") + } + if !hasApplyManager(got.ManagedFields, "status") { + t.Errorf("no status apply manager %q: %+v", fieldOwner, got.ManagedFields) + } + + // A decision made from the version before the status was written must conflict. + err = f.writer.PopulateStatus(f.ctx, name, stored.ResourceVersion, candidate) + if !apierrors.IsConflict(err) { + t.Fatalf("populate with a stale resourceVersion: err = %v, want a conflict", err) + } + + // Re-applying the candidate after its status exists reports it as populated. + after, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate) + if err != nil { + t.Fatalf("re-apply candidate: %v", err) + } + if !after.Populated { + t.Error("a candidate with status was reported as not populated") + } +} + +func TestEnvtestRunStatusIsOwnedByThePublisher(t *testing.T) { + f := newWriterFixture(t) + first := time.Date(2026, 10, 22, 12, 0, 0, 0, time.UTC) + + staleVersion := f.currentRun().ResourceVersion + if err := f.writer.UpdateRunStatus(f.ctx, staleVersion, RunStatus{ + Published: 2, + LastProgress: first, + CandidateNames: []string{f.name("a"), f.name("b")}, + }); err != nil { + t.Fatalf("update run status: %v", err) + } + run := f.currentRun() + if len(run.Status.CandidateRefs) != 2 || run.Status.CandidateRefs[0].Name != f.name("a") { + t.Fatalf("candidateRefs = %+v", run.Status.CandidateRefs) + } + if run.Status.Progress == nil || run.Status.Progress.Candidates == nil || run.Status.Progress.Candidates.Published != 2 { + t.Fatalf("progress = %+v, want 2 published", run.Status.Progress) + } + if run.Status.LastProgressTime == nil || !run.Status.LastProgressTime.UTC().Equal(first) { + t.Fatalf("lastProgressTime = %v, want %v", run.Status.LastProgressTime, first) + } + if !hasApplyManager(run.ManagedFields, "status") { + t.Errorf("no status apply manager %q: %+v", fieldOwner, run.ManagedFields) + } + + // A write based on the version read before the first write conflicts. + err := f.writer.UpdateRunStatus(f.ctx, staleVersion, RunStatus{Published: 1, CandidateNames: []string{f.name("a")}}) + if !apierrors.IsConflict(err) { + t.Fatalf("update with a stale resourceVersion: err = %v, want a conflict", err) + } + + // Fields the publisher owns and omits are removed: this is why the reconciler carries + // the existing progress time into every payload. + if err := f.writer.UpdateRunStatus(f.ctx, run.ResourceVersion, RunStatus{Published: 1, CandidateNames: []string{f.name("a")}}); err != nil { + t.Fatalf("shrinking update: %v", err) + } + run = f.currentRun() + if len(run.Status.CandidateRefs) != 1 || run.Status.Progress.Candidates.Published != 1 { + t.Errorf("status after shrinking = %+v", run.Status) + } + if run.Status.LastProgressTime != nil { + t.Errorf("lastProgressTime = %v, want it removed when the payload omits it", run.Status.LastProgressTime) + } +} + +func TestEnvtestDeleteCandidateHonorsTheUIDPrecondition(t *testing.T) { + f := newWriterFixture(t) + name := f.name("a") + if _, err := f.writer.CreateCandidate(f.ctx, f.run, name, envtestCandidate()); err != nil { + t.Fatalf("create candidate: %v", err) + } + uid := f.candidate(name).UID + + if err := f.writer.DeleteCandidate(f.ctx, name, types.UID("not-the-uid")); err == nil { + t.Fatal("deleting with the wrong UID succeeded; a recreated candidate would be lost") + } + f.candidate(name) // still there + + if err := f.writer.DeleteCandidate(f.ctx, name, uid); err != nil { + t.Fatalf("delete with the right UID: %v", err) + } + var gone v1beta2.DynamoGraphDeploymentCandidate + err := f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: name}, &gone) + if !apierrors.IsNotFound(err) { + t.Fatalf("candidate still exists after delete: %v", err) + } + if err := f.writer.DeleteCandidate(f.ctx, name, uid); err != nil { + t.Fatalf("deleting a missing candidate must not fail: %v", err) + } +} + +func TestEnvtestCandidateUpdateTriggersStatusRepair(t *testing.T) { + f := newWriterFixture(t) + podName := "publisher-pod" + candidate := envtestCandidate() + snapshot, err := ParseSnapshot(snapshotJSON(t, PhaseRunning, cand{id: "a", metrics: candidate.Metrics})) + if err != nil { + t.Fatal(err) + } + desired := Desire(f.run.Name, string(f.run.UID), snapshot) + + t.Log("Arrange a fully converged Run and Candidate before starting the manager") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: f.run.Namespace, Name: podName}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "sweeper", Image: "example.invalid/sweeper"}}}, + } + if err := f.client.Create(f.ctx, pod); err != nil { + t.Fatalf("create pod: %v", err) + } + name := f.name("a") + stored, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate) + if err != nil { + t.Fatalf("create candidate: %v", err) + } + if err := f.writer.PopulateStatus(f.ctx, name, stored.ResourceVersion, candidate); err != nil { + t.Fatalf("populate candidate: %v", err) + } + if err := f.writer.UpdateRunStatus(f.ctx, f.currentRun().ResourceVersion, desired.Status); err != nil { + t.Fatalf("converge run status: %v", err) + } + + t.Log("Start the production controller watches around an instrumented Reconciler") + scheme, err := NewScheme() + if err != nil { + t.Fatal(err) + } + manager, err := ctrl.NewManager(f.config, ctrl.Options{ + Scheme: scheme, + Cache: cacheOptions(Config{Namespace: f.run.Namespace, RunName: f.run.Name, RunUID: string(f.run.UID), PodName: podName}), + Metrics: metricsserver.Options{BindAddress: "0"}, + LeaderElection: false, + }) + if err != nil { + t.Fatalf("create manager: %v", err) + } + store := &InputStore{} + store.accept(snapshot) + completion := NewCompletion() + reconciler := &Reconciler{ + Client: manager.GetClient(), + Writer: &clientWriter{ + Client: manager.GetClient(), + Reader: manager.GetAPIReader(), + Namespace: f.run.Namespace, + RunName: f.run.Name, + RunUID: string(f.run.UID), + }, + Namespace: f.run.Namespace, + RunName: f.run.Name, + RunUID: string(f.run.UID), + PodName: podName, + SweeperContainer: "sweeper", + Store: store, + Completion: completion, + } + recorder := &reconcileRecorder{delegate: reconciler} + events := make(chan event.GenericEvent, 1) + key := types.NamespacedName{Namespace: f.run.Namespace, Name: f.run.Name} + if err := setupController(manager, recorder, key, events); err != nil { + t.Fatalf("setup controller: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- manager.Start(ctx) + }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("publisher stopped: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("publisher did not stop") + } + }) + + waitForReconcileQuiet(t, recorder, 250*time.Millisecond, 10*time.Second) + beforeUpdate := recorder.count.Load() + + t.Log("Clear only Candidate status after the controller is quiet") + var current v1beta2.DynamoGraphDeploymentCandidate + if err := f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: name}, ¤t); err != nil { + t.Fatalf("get candidate: %v", err) + } + current.Status.Metrics = nil + if err := f.client.Status().Update(f.ctx, ¤t); err != nil { + t.Fatalf("clear candidate status: %v", err) + } + require.Eventually(t, func() bool { return recorder.count.Load() > beforeUpdate }, 10*time.Second, 25*time.Millisecond, + "Candidate update did not trigger a new reconciliation") + require.Eventually(t, func() bool { + var repaired v1beta2.DynamoGraphDeploymentCandidate + if err := f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: name}, &repaired); err != nil { + return false + } + return repaired.Status.Metrics != nil + }, 10*time.Second, 25*time.Millisecond, "candidate status update did not trigger repair") +} From eceeec6145cd57aebec675f8050a727f86147263 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 26 Oct 2026 15:16:00 +0100 Subject: [PATCH 034/100] pr-15894: pr-5: squash publisher terminal snapshot contract Squashed from https://github.com/devivasudevan/dynamo/pull/5. Signed-off-by: Dr. Stefan Schimanski --- .../internal/dgdrrunpublisher/input.go | 14 +++-- .../internal/dgdrrunpublisher/input_test.go | 55 +++++++++++++++++-- .../internal/dgdrrunpublisher/snapshot.go | 6 ++ .../dgdrrunpublisher/snapshot_test.go | 22 ++++---- 4 files changed, 75 insertions(+), 22 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/input.go b/deploy/operator/internal/dgdrrunpublisher/input.go index 9d3183638a8a..1045375d9f38 100644 --- a/deploy/operator/internal/dgdrrunpublisher/input.go +++ b/deploy/operator/internal/dgdrrunpublisher/input.go @@ -55,7 +55,7 @@ func (s *InputStore) Load() Input { func (s *InputStore) accept(snap *Snapshot) (changed bool) { s.mu.Lock() defer s.mu.Unlock() - if err := checkProgress(s.input.Snapshot, snap); err != nil { + if err := checkTransition(s.input.Snapshot, snap); err != nil { s.input.Err = err return false } @@ -82,14 +82,16 @@ func sameContent(a, b *Snapshot) bool { return reflect.DeepEqual(left, right) } -// checkProgress rejects a snapshot that moves progress backwards relative to the -// previously accepted one: an older round, or fewer evaluated points within the same -// round. A terminal snapshot is exempt from the evaluated check because its count is the -// search's final total, not the live counter. -func checkProgress(previous, next *Snapshot) error { +// checkTransition rejects changes after a terminal snapshot and progress that moves +// backwards. A terminal snapshot is exempt from the evaluated check because its count is +// the search's final total, not the live counter. +func checkTransition(previous, next *Snapshot) error { if previous == nil { return nil } + if previous.Run.Terminal && !sameContent(previous, next) { + return violation("snapshot changed after terminal phase %q", previous.Run.Phase) + } if next.Progress.Round < previous.Progress.Round { return violation("snapshot round regressed from %d to %d", previous.Progress.Round, next.Progress.Round) } diff --git a/deploy/operator/internal/dgdrrunpublisher/input_test.go b/deploy/operator/internal/dgdrrunpublisher/input_test.go index 8b13734c92fa..f8d66197b298 100644 --- a/deploy/operator/internal/dgdrrunpublisher/input_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/input_test.go @@ -66,12 +66,20 @@ func TestChangedSnapshotIsStoredAndEnqueuedOnce(t *testing.T) { } func TestRewriteWithOnlyANewTimestampIsNotAChange(t *testing.T) { - o, _, dir, enqueued := newObserver(t) - writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:12Z")) - o.Check() - writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), "2026-10-05T21:35:14Z")) - if o.Check() || *enqueued != 1 { - t.Fatalf("a new timestamp alone is not a semantic change: enqueued=%d", *enqueued) + for _, phase := range []string{PhaseRunning, PhaseSucceeded} { + t.Run(phase, func(t *testing.T) { + o, _, dir, enqueued := newObserver(t) + + t.Log("Accept the first snapshot") + writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, phase, cand{id: "a"}), "2026-10-05T21:35:12Z")) + o.Check() + + t.Log("Rewrite the same snapshot with only a new timestamp") + writeSnapshot(t, dir, withTimestamp(t, snapshotJSON(t, phase, cand{id: "a"}), "2026-10-05T21:35:14Z")) + if o.Check() || *enqueued != 1 { + t.Fatalf("a new timestamp alone is not a semantic change: enqueued=%d", *enqueued) + } + }) } } @@ -124,6 +132,41 @@ func TestTerminalSnapshotMayReportItsFinalEvaluatedTotal(t *testing.T) { } } +func TestTerminalSnapshotCannotBeReplaced(t *testing.T) { + tests := []struct { + name string + replacement []byte + }{ + {name: "return to running", replacement: snapshotJSON(t, PhaseRunning, cand{id: "a"})}, + {name: "change final candidates", replacement: snapshotJSON(t, PhaseSucceeded, cand{id: "a"}, cand{id: "b"})}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + o, store, dir, enqueued := newObserver(t) + + t.Log("Accept the terminal snapshot") + writeSnapshot(t, dir, snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) + if !o.Check() { + t.Fatal("terminal snapshot was not accepted") + } + + t.Log("Attempt to replace the latched terminal state") + writeSnapshot(t, dir, test.replacement) + if o.Check() || *enqueued != 1 { + t.Fatalf("replacement must not be stored: enqueued=%d", *enqueued) + } + + input := store.Load() + if !errors.Is(input.Err, ErrProtocolViolation) { + t.Fatalf("err = %v, want protocol violation", input.Err) + } + if input.Snapshot.Run.Phase != PhaseSucceeded || len(input.Snapshot.Candidates) != 1 { + t.Fatalf("terminal snapshot was replaced: %+v", input.Snapshot) + } + }) + } +} + func TestUnreadableSnapshotIsRecordedAsANonViolation(t *testing.T) { o, store, dir, _ := newObserver(t) if err := os.Mkdir(filepath.Join(dir, SnapshotFileName), 0o700); err != nil { // reading a directory fails diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go index 5bbe05125bbd..df9f2d3e9598 100644 --- a/deploy/operator/internal/dgdrrunpublisher/snapshot.go +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -183,6 +183,9 @@ func ParseSnapshot(data []byte) (*Snapshot, error) { seen[candidate.ID] = struct{}{} switch candidate.Outcome { case OutcomeMaterialized: + if candidate.Error != "" { + return nil, violation("materialized candidate %q has an error", candidate.ID) + } if candidate.Manifest == "" { return nil, violation("materialized candidate %q has no manifest", candidate.ID) } @@ -190,6 +193,9 @@ func ParseSnapshot(data []byte) (*Snapshot, error) { return nil, err } case OutcomeMaterializationFailed: + if candidate.Manifest != "" { + return nil, violation("failed candidate %q has a manifest", candidate.ID) + } if candidate.Error == "" { return nil, violation("failed candidate %q has no error", candidate.ID) } diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go index e0287a5c36d3..cbd6cfd5579a 100644 --- a/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go @@ -113,16 +113,18 @@ func withRunDiagnostics(t *testing.T, data []byte, message, runError string) []b func TestParseSnapshotValidation(t *testing.T) { bad := map[string]string{ - "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, - "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, - "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, - "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, - "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, - "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, - "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, - "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, - "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, - "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "schema": `{"schemaVersion":2,"run":{"phase":"Running","terminal":false}}`, + "phase": `{"schemaVersion":1,"run":{"phase":"Weird","terminal":false}}`, + "terminal flag": `{"schemaVersion":1,"run":{"phase":"Succeeded","terminal":false}}`, + "duplicate": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"},{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "no manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized"}]}`, + "materialized with error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n","error":"boom"}]}`, + "no error": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed"}]}`, + "failed with manifest": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialization_failed","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n","error":"boom"}]}`, + "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, + "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, + "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, } bad["long id"] = strings.Replace(bad["long id"], "%s", strings.Repeat("a", 64), 1) for name, doc := range bad { From 396fcddb0b2529bd73fde614ee194f8fbf429b4e Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 11:10:44 +0100 Subject: [PATCH 035/100] pr-15894: satisfy current Go lint Signed-off-by: Dr. Stefan Schimanski --- .../internal/dgdcreconcile/diff_test.go | 3 +- .../internal/dgdrrunpublisher/input_test.go | 2 +- .../internal/dgdrrunpublisher/reconciler.go | 3 +- .../dgdrrunpublisher/reconciler_test.go | 31 ++++++++++--------- .../internal/dgdrrunpublisher/terminal.go | 3 +- .../internal/dgdrrunpublisher/writer.go | 12 +++---- .../dgdrrunpublisher/writer_envtest_test.go | 14 ++++----- 7 files changed, 32 insertions(+), 36 deletions(-) diff --git a/deploy/operator/internal/dgdcreconcile/diff_test.go b/deploy/operator/internal/dgdcreconcile/diff_test.go index 3d0f155e2173..3f4bf191bb5c 100644 --- a/deploy/operator/internal/dgdcreconcile/diff_test.go +++ b/deploy/operator/internal/dgdcreconcile/diff_test.go @@ -101,8 +101,7 @@ func TestComputeActionsInputErrors(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { _, err := ComputeActions(tc.desired, tc.current) - var inputErr *DiffInputError - if !errors.As(err, &inputErr) { + if _, ok := errors.AsType[*DiffInputError](err); !ok { t.Fatalf("want *DiffInputError, got %v", err) } }) diff --git a/deploy/operator/internal/dgdrrunpublisher/input_test.go b/deploy/operator/internal/dgdrrunpublisher/input_test.go index f8d66197b298..470ae7ab14ae 100644 --- a/deploy/operator/internal/dgdrrunpublisher/input_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/input_test.go @@ -182,7 +182,7 @@ func TestNextIntervalBacksOffWhileUnchangedAndResetsOnChange(t *testing.T) { minimum, maximum := time.Second, 5*time.Second got := []time.Duration{} interval := minimum - for i := 0; i < 5; i++ { + for range 5 { interval = NextInterval(interval, minimum, maximum, false) got = append(got, interval) } diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go index 9e51272b60d9..c65837fcdd83 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -171,8 +171,7 @@ func (r *Reconciler) reconcile(ctx context.Context) error { // retryOrFail hands a failure to the workqueue's backoff, except where retrying cannot // help (malformed projection) or the Sweeper is gone and the attempts are used up. func (r *Reconciler) retryOrFail(exited bool, err error) error { - var inputErr *dgdcreconcile.DiffInputError - if errors.As(err, &inputErr) { + if _, ok := errors.AsType[*dgdcreconcile.DiffInputError](err); ok { r.Completion.Finish(err) // retrying cannot fix it return nil } diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go index 58a35b02e880..0022b75bdfb9 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -101,13 +101,13 @@ func (w *memWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoGrap if w.lag { return stored, nil } - object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + object := &v1beta2.DynamoGraphDeploymentCandidate{ Namespace: testNamespace, Name: name, UID: types.UID("uid-" + name), Labels: map[string]string{LabelRunUID: testRunUID, LabelCandidateID: candidate.ID}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(run, v1beta2.DynamoGraphDeploymentRunGVK)}, - }} + } return stored, client.IgnoreAlreadyExists(w.c.Create(ctx, object)) } @@ -163,7 +163,10 @@ func (w *memWriter) DeleteCandidate(ctx context.Context, name string, _ types.UI if w.lag { return nil } - return client.IgnoreNotFound(w.c.Delete(ctx, &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}})) + return client.IgnoreNotFound(w.c.Delete(ctx, &v1beta2.DynamoGraphDeploymentCandidate{ + Namespace: testNamespace, + Name: name, + })) } type fixture struct { @@ -179,8 +182,8 @@ func newFixture(t *testing.T) *fixture { if err != nil { t.Fatal(err) } - run := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testRunName, UID: testRunUID}} - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testPodName}} + run := &v1beta2.DynamoGraphDeploymentRun{Namespace: testNamespace, Name: testRunName, UID: testRunUID} + pod := &corev1.Pod{Namespace: testNamespace, Name: testPodName} c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(run, pod).Build() store := &InputStore{} observer := &Observer{ @@ -213,7 +216,7 @@ func (f *fixture) publish(data []byte) { func (f *fixture) reconcile() error { f.t.Helper() - _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: testRunName}}) + _, err := f.r.Reconcile(context.Background(), ctrl.Request{Namespace: testNamespace, Name: testRunName}) return err } @@ -242,7 +245,7 @@ func (f *fixture) exitSweeper(code int32) { func (f *fixture) deleteCandidate(name string) { f.t.Helper() delete(f.w.server, name) - object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: name}} + object := &v1beta2.DynamoGraphDeploymentCandidate{Namespace: testNamespace, Name: name} if err := f.c.Delete(context.Background(), object); err != nil { f.t.Fatal(err) } @@ -272,12 +275,12 @@ func (f *fixture) currentRun() *v1beta2.DynamoGraphDeploymentRun { } func candidateForRun(name, id string, run *v1beta2.DynamoGraphDeploymentRun) *v1beta2.DynamoGraphDeploymentCandidate { - return &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{ + return &v1beta2.DynamoGraphDeploymentCandidate{ Namespace: run.Namespace, Name: name, Labels: map[string]string{LabelRunUID: string(run.UID), LabelCandidateID: id}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(run, v1beta2.DynamoGraphDeploymentRunGVK)}, - }} + } } // withProgress rewrites a snapshot's progress counters. @@ -626,11 +629,11 @@ func TestFinalReconcileFailureEndsTheProcessAfterBoundedRetries(t *testing.T) { func TestForeignLabeledCandidateDoesNotSatisfyOrGetDeleted(t *testing.T) { f := newFixture(t) - foreignRun := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{ + foreignRun := &v1beta2.DynamoGraphDeploymentRun{ Namespace: testNamespace, Name: "foreign-run", UID: "foreign-run-uid", - }} + } foreign := candidateForRun("foreign-candidate", "a", foreignRun) foreign.Labels[LabelRunUID] = testRunUID if err := f.c.Create(context.Background(), foreign); err != nil { @@ -731,7 +734,7 @@ func TestRunRecreatedUnderTheSameNameIsNotTouched(t *testing.T) { func TestReconcileIgnoresOtherKeysAndAFinishedRun(t *testing.T) { f := newFixture(t) f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) - if _, err := f.r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: testNamespace, Name: "other"}}); err != nil || len(f.w.ops) != 0 { + if _, err := f.r.Reconcile(context.Background(), ctrl.Request{Namespace: testNamespace, Name: "other"}); err != nil || len(f.w.ops) != 0 { t.Fatalf("another key must be ignored: err=%v ops=%v", err, f.w.ops) } f.mustReconcile() @@ -828,7 +831,7 @@ func TestCandidateApplyPayload(t *testing.T) { if err != nil { t.Fatal(err) } - run := &v1beta2.DynamoGraphDeploymentRun{ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace, Name: testRunName, UID: testRunUID}} + run := &v1beta2.DynamoGraphDeploymentRun{Namespace: testNamespace, Name: testRunName, UID: testRunUID} candidate := dgdcreconcile.DesiredCandidate{ID: "a", Spec: validManifest, Parameters: map[string]any{"tp": 2}} apply, err := buildCandidateApply(scheme, testNamespace, testRunUID, run, cn("a"), candidate) if err != nil { @@ -977,7 +980,7 @@ func TestConflictIsRetriedWithoutUsingUpTheFinalAttempts(t *testing.T) { f.w.sticky = true f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) f.exitSweeper(1) - for attempt := 0; attempt < 2*finalReconcileAttempts; attempt++ { + for attempt := range 2 * finalReconcileAttempts { if err := f.reconcile(); !apierrors.IsConflict(err) || f.r.Completion.Finished() { t.Fatalf("attempt %d: a conflict from a stale read must be retried, err=%v finished=%v", attempt, err, f.r.Completion.Finished()) } diff --git a/deploy/operator/internal/dgdrrunpublisher/terminal.go b/deploy/operator/internal/dgdrrunpublisher/terminal.go index a9ddda4e0c59..904d9b7861e9 100644 --- a/deploy/operator/internal/dgdrrunpublisher/terminal.go +++ b/deploy/operator/internal/dgdrrunpublisher/terminal.go @@ -92,8 +92,7 @@ func boundDiagnostic(text string) string { // TerminationOutcome is what the publisher reports for err: the run's own outcome for a // run failure, otherwise the bounded error text. func TerminationOutcome(err error) TerminalOutcome { - var failed *RunFailedError - if errors.As(err, &failed) { + if failed, ok := errors.AsType[*RunFailedError](err); ok { return failed.Outcome } return TerminalOutcome{Error: boundDiagnostic(err.Error())} diff --git a/deploy/operator/internal/dgdrrunpublisher/writer.go b/deploy/operator/internal/dgdrrunpublisher/writer.go index e3b1030f59c6..5ef51d861f16 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer.go @@ -150,7 +150,7 @@ func (w *clientWriter) UpdateRunStatus(ctx context.Context, resourceVersion stri } func (w *clientWriter) DeleteCandidate(ctx context.Context, name string, uid types.UID) error { - object := &v1beta2.DynamoGraphDeploymentCandidate{ObjectMeta: metav1.ObjectMeta{Namespace: w.Namespace, Name: name}} + object := &v1beta2.DynamoGraphDeploymentCandidate{Namespace: w.Namespace, Name: name} var options []client.DeleteOption if uid != "" { options = append(options, client.Preconditions{UID: &uid}) @@ -171,12 +171,10 @@ func buildCandidateApply(scheme *runtime.Scheme, namespace, runUID string, run * return nil, err } object := &v1beta2.DynamoGraphDeploymentCandidate{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - Labels: map[string]string{LabelRunUID: runUID, LabelCandidateID: candidate.ID}, - }, - Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{DynamoGraphDeploymentSpec: *spec}, + Namespace: namespace, + Name: name, + Labels: map[string]string{LabelRunUID: runUID, LabelCandidateID: candidate.ID}, + Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{DynamoGraphDeploymentSpec: *spec}, } if companions != "" { object.Annotations = map[string]string{AnnotationAdditionalResources: companions} diff --git a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go index 6437e8c363cb..2faaeab9e161 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go @@ -68,7 +68,7 @@ func newWriterFixture(t *testing.T) *writerFixture { c := env.Client() run := &v1beta2.DynamoGraphDeploymentRun{ - ObjectMeta: metav1.ObjectMeta{Namespace: env.Namespace(), Name: "publisher-run"}, + Namespace: env.Namespace(), Name: "publisher-run", Spec: v1beta2.DynamoGraphDeploymentRunSpec{ NativeSearchInput: v1beta2.NativeSearchInput{Model: &v1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}}, }, @@ -134,11 +134,9 @@ func foreignCandidate(t *testing.T, namespace, name string, candidate dgdcreconc t.Fatal(err) } return &v1beta2.DynamoGraphDeploymentCandidate{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - Labels: map[string]string{"owner": "foreign"}, - }, + Namespace: namespace, + Name: name, + Labels: map[string]string{"owner": "foreign"}, Spec: v1beta2.DynamoGraphDeploymentCandidateSpec{ DynamoGraphDeploymentSpec: *spec, Parameters: runtime.RawExtension{Raw: parameters}, @@ -459,8 +457,8 @@ func TestEnvtestCandidateUpdateTriggersStatusRepair(t *testing.T) { t.Log("Arrange a fully converged Run and Candidate before starting the manager") pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Namespace: f.run.Namespace, Name: podName}, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "sweeper", Image: "example.invalid/sweeper"}}}, + Namespace: f.run.Namespace, Name: podName, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "sweeper", Image: "example.invalid/sweeper"}}}, } if err := f.client.Create(f.ctx, pod); err != nil { t.Fatalf("create pod: %v", err) From f44ee5bec2496ea257aed748d73fc9f78af4196c Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 15:00:47 +0100 Subject: [PATCH 036/100] pr-15894: pr-5: consolidate Run publisher tests Signed-off-by: Dr. Stefan Schimanski --- .../internal/dgdrrunpublisher/outcome_test.go | 49 +++- .../internal/dgdrrunpublisher/plan_test.go | 11 + .../dgdrrunpublisher/reconciler_test.go | 247 ++++-------------- .../dgdrrunpublisher/terminal_test.go | 40 --- .../dgdrrunpublisher/writer_envtest_test.go | 33 +++ 5 files changed, 141 insertions(+), 239 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/outcome_test.go b/deploy/operator/internal/dgdrrunpublisher/outcome_test.go index 9462ea9fd49a..61aff5fd7630 100644 --- a/deploy/operator/internal/dgdrrunpublisher/outcome_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/outcome_test.go @@ -20,7 +20,6 @@ package dgdrrunpublisher import ( "errors" "fmt" - "strings" "testing" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" @@ -47,16 +46,46 @@ func TestExitCodes(t *testing.T) { } func TestTerminalResult(t *testing.T) { - if err := terminalResult(DesiredState{Phase: PhaseSucceeded, Candidates: []dgdcreconcile.DesiredCandidate{{ID: "a"}}}); err != nil { - t.Fatalf("succeeded: %v", err) - } - failed := terminalResult(DesiredState{Phase: PhaseFailed, Message: "search failed", Error: "backend unavailable"}) - if !errors.Is(failed, ErrRunFailed) || !strings.Contains(failed.Error(), "search failed") || !strings.Contains(failed.Error(), "backend unavailable") { - t.Fatalf("failed: %v", failed) + tests := []struct { + name string + state DesiredState + outcome *TerminalOutcome + errorText string + }{ + { + name: "success with a materialized candidate", + state: DesiredState{Phase: PhaseSucceeded, Candidates: []dgdcreconcile.DesiredCandidate{{ID: "a"}}}, + }, + { + name: "producer failure keeps its diagnostics", + state: DesiredState{Phase: PhaseFailed, Message: "search failed", Error: "backend unavailable"}, + outcome: &TerminalOutcome{Phase: PhaseFailed, Message: "search failed", Error: "backend unavailable"}, + errorText: ErrRunFailed.Error() + ": search failed: backend unavailable", + }, + { + name: "success without a materialized candidate is a failure", + state: DesiredState{Phase: PhaseSucceeded, Failed: 1, FirstFailure: "x: render failed"}, + outcome: &TerminalOutcome{Phase: PhaseFailed, Message: "no candidate was materialized: 1 candidate materialization failure", Error: "x: render failed"}, + errorText: ErrRunFailed.Error() + ": no candidate was materialized: 1 candidate materialization failure: x: render failed", + }, } - empty := terminalResult(DesiredState{Phase: PhaseSucceeded, Failed: 2}) - if !errors.Is(empty, ErrRunFailed) || !strings.Contains(empty.Error(), "2 candidate materialization failures") || !strings.Contains(empty.Error(), "no candidate was materialized") { - t.Fatalf("empty success: %v", empty) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := terminalResult(test.state) + if test.outcome == nil { + if err != nil { + t.Fatalf("terminalResult() = %v, want success", err) + } + return + } + var failure *RunFailedError + if !errors.Is(err, ErrRunFailed) || !errors.As(err, &failure) || failure.Outcome != *test.outcome { + t.Fatalf("terminalResult() = %#v, want outcome %+v", err, *test.outcome) + } + if got := err.Error(); got != test.errorText { + t.Fatalf("error text = %q, want %q", got, test.errorText) + } + }) } } diff --git a/deploy/operator/internal/dgdrrunpublisher/plan_test.go b/deploy/operator/internal/dgdrrunpublisher/plan_test.go index 12268168a6a9..103b79d37d1f 100644 --- a/deploy/operator/internal/dgdrrunpublisher/plan_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/plan_test.go @@ -75,6 +75,17 @@ func TestFailedMaterializationIsNotDesiredButCounted(t *testing.T) { } } +func TestDesireCarriesBoundedDiagnostics(t *testing.T) { + long := strings.Repeat("e", 3*maxDiagnosticBytes) + state := desire(t, withRunDiagnostics(t, snapshotJSON(t, PhaseFailed, cand{id: "a"}), "no recommendation\ncould be materialized", long)) + if state.Phase != PhaseFailed || state.Message != "no recommendation could be materialized" { + t.Fatalf("state = %+v", state) + } + if len(state.Error) > maxDiagnosticBytes || !strings.HasPrefix(state.Error, "eee") { + t.Fatalf("error must be kept but bounded, got %d bytes", len(state.Error)) + } +} + func TestPublishedCountsTheReferencedCandidates(t *testing.T) { state := desire(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "b"})) if got, want := state.Status.Published, int32(len(state.Status.CandidateNames)); got != want || got != 2 { diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go index 0022b75bdfb9..06c0e8fdb9b9 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -32,7 +32,6 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -434,21 +433,6 @@ func TestDroppedCandidateIsDeletedAfterTheRunStatusUpdate(t *testing.T) { } } -func TestFailedMaterializationIsNotCreatedButCounted(t *testing.T) { - f := newFixture(t) - f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"}, cand{id: "x", failed: true})) - f.mustReconcile() - for _, op := range f.takeOps() { - if strings.Contains(op, "x") && strings.HasPrefix(op, "create") { - t.Fatalf("a failed candidate must not be created: %s", op) - } - } - status := f.lastStatus() - if !reflect.DeepEqual(status.CandidateNames, []string{cn("a")}) || status.Published != 1 { - t.Fatalf("status = %+v", status) - } -} - func TestCandidateWithoutStatusIsCompletedNotRecreated(t *testing.T) { f := newFixture(t) existing := candidateForRun(cn("a"), "a", f.currentRun()) @@ -510,7 +494,9 @@ func TestTerminalSucceededSnapshotWithoutAMaterializedCandidateFails(t *testing. f.mustReconcile() err := f.r.Completion.Err() - if !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed || !strings.Contains(err.Error(), "1 candidate materialization failure") { + var failure *RunFailedError + if !errors.Is(err, ErrRunFailed) || !errors.As(err, &failure) || ExitCode(err) != ExitRunFailed || + !strings.Contains(err.Error(), "1 candidate materialization failure") || !strings.HasPrefix(failure.Outcome.Error, "x: ") { t.Fatalf("err = %v", err) } if ops := f.takeOps(); len(ops) != 0 { @@ -650,64 +636,62 @@ func TestForeignLabeledCandidateDoesNotSatisfyOrGetDeleted(t *testing.T) { } } -func TestOwnedCandidateWithWrongNameIsReplacedBeforeDeletion(t *testing.T) { - f := newFixture(t) - wrong := candidateForRun("wrong-name", "a", f.currentRun()) - if err := f.c.Create(context.Background(), wrong); err != nil { - t.Fatal(err) - } - - f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) - f.mustReconcile() - want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status", "delete:wrong-name"} - if got := f.takeOps(); !reflect.DeepEqual(got, want) { - t.Fatalf("ops = %v, want %v", got, want) - } -} - -func TestOwnedObsoleteCandidateWithoutRunLabelIsDeleted(t *testing.T) { - f := newFixture(t) - obsolete := candidateForRun(cn("obsolete"), "obsolete", f.currentRun()) - delete(obsolete.Labels, LabelRunUID) - if err := f.c.Create(context.Background(), obsolete); err != nil { - t.Fatal(err) - } - - f.publish(snapshotJSON(t, PhaseRunning)) - f.mustReconcile() - if got, want := f.takeOps(), []string{"delete:" + obsolete.Name}; !reflect.DeepEqual(got, want) { - t.Fatalf("ops = %v, want %v", got, want) - } -} - -func TestOwnedCandidateWithWrongIdentityLabelIsReappliedNotDeleted(t *testing.T) { - f := newFixture(t) - wrong := candidateForRun(cn("a"), "wrong-id", f.currentRun()) - if err := f.c.Create(context.Background(), wrong); err != nil { - t.Fatal(err) - } - - f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) - f.mustReconcile() - want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status"} - if got := f.takeOps(); !reflect.DeepEqual(got, want) { - t.Fatalf("ops = %v, want %v", got, want) - } -} - -func TestOwnedCandidateWithMissingRunLabelIsReappliedNotDeleted(t *testing.T) { - f := newFixture(t) - drifted := candidateForRun(cn("a"), "a", f.currentRun()) - delete(drifted.Labels, LabelRunUID) - if err := f.c.Create(context.Background(), drifted); err != nil { - t.Fatal(err) - } +func TestOwnedCandidateDriftIsReconciledSafely(t *testing.T) { + tests := []struct { + name string + existingName string + existingID string + removeRunLabel bool + desired []cand + want []string + }{ + { + name: "wrong name is replaced before deletion", + existingName: "wrong-name", + existingID: "a", + desired: []cand{{id: "a"}}, + want: []string{"create:" + cn("a"), "populate:" + cn("a"), "status", "delete:wrong-name"}, + }, + { + name: "obsolete candidate without run label is deleted", + existingName: cn("obsolete"), + existingID: "obsolete", + removeRunLabel: true, + want: []string{"delete:" + cn("obsolete")}, + }, + { + name: "wrong identity label is reapplied", + existingName: cn("a"), + existingID: "wrong-id", + desired: []cand{{id: "a"}}, + want: []string{"create:" + cn("a"), "populate:" + cn("a"), "status"}, + }, + { + name: "missing run label is reapplied", + existingName: cn("a"), + existingID: "a", + removeRunLabel: true, + desired: []cand{{id: "a"}}, + want: []string{"create:" + cn("a"), "populate:" + cn("a"), "status"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f := newFixture(t) + existing := candidateForRun(test.existingName, test.existingID, f.currentRun()) + if test.removeRunLabel { + delete(existing.Labels, LabelRunUID) + } + if err := f.c.Create(context.Background(), existing); err != nil { + t.Fatal(err) + } - f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) - f.mustReconcile() - want := []string{"create:" + cn("a"), "populate:" + cn("a"), "status"} - if got := f.takeOps(); !reflect.DeepEqual(got, want) { - t.Fatalf("ops = %v, want %v", got, want) + f.publish(snapshotJSON(t, PhaseRunning, test.desired...)) + f.mustReconcile() + if got := f.takeOps(); !reflect.DeepEqual(got, test.want) { + t.Fatalf("ops = %v, want %v", got, test.want) + } + }) } } @@ -817,121 +801,6 @@ func TestCacheOptionsCoverThisRunPodAndNamespaceCandidates(t *testing.T) { } } -func nested(t *testing.T, object *unstructured.Unstructured, fields ...string) (any, bool) { - t.Helper() - value, found, err := unstructured.NestedFieldNoCopy(object.Object, fields...) - if err != nil { - t.Fatal(err) - } - return value, found -} - -func TestCandidateApplyPayload(t *testing.T) { - scheme, err := NewScheme() - if err != nil { - t.Fatal(err) - } - run := &v1beta2.DynamoGraphDeploymentRun{Namespace: testNamespace, Name: testRunName, UID: testRunUID} - candidate := dgdcreconcile.DesiredCandidate{ID: "a", Spec: validManifest, Parameters: map[string]any{"tp": 2}} - apply, err := buildCandidateApply(scheme, testNamespace, testRunUID, run, cn("a"), candidate) - if err != nil { - t.Fatal(err) - } - if apply.GetAPIVersion() != "nvidia.com/v1beta2" || apply.GetKind() != "DynamoGraphDeploymentCandidate" || apply.GetName() != cn("a") || apply.GetNamespace() != testNamespace { - t.Fatalf("identity = %s %s %s/%s", apply.GetAPIVersion(), apply.GetKind(), apply.GetNamespace(), apply.GetName()) - } - if _, found := nested(t, apply, "status"); found { - t.Fatal("status goes through the status subresource") - } - if _, found := nested(t, apply, "metadata", "creationTimestamp"); found { - t.Fatal("creationTimestamp must not be part of an apply payload") - } - if got := apply.GetLabels(); got[LabelRunUID] != testRunUID || got[LabelCandidateID] != "a" { - t.Fatalf("labels = %v", got) - } - if _, found := nested(t, apply, "spec", "parameters"); !found { - t.Fatal("the resolved parameters belong in the spec") - } - owners := apply.GetOwnerReferences() - if len(owners) != 1 || owners[0].UID != testRunUID || owners[0].Controller == nil || !*owners[0].Controller { - t.Fatalf("owner references = %+v", owners) - } -} - -func TestCandidateStatusApplyPayloadIsDeterministic(t *testing.T) { - candidate := dgdcreconcile.DesiredCandidate{ID: "a", Metrics: map[string]any{"score": 2}} - first, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate) - if err != nil { - t.Fatal(err) - } - second, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", candidate) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(first.Object, second.Object) { - t.Fatal("the same input must produce the same payload") - } - if got, _ := nested(t, first, "metadata", "resourceVersion"); got != "17" { - t.Fatalf("the status write must be constrained to the version it was decided on, got %v", got) - } - if _, found := nested(t, first, "status", "metrics"); !found { - t.Fatal("the metrics are the candidate's status") - } - if _, found := nested(t, first, "status", "conditions"); found { - t.Fatal("candidate status is metrics only") - } -} - -func TestCandidateStatusApplyAlwaysWritesMetricsEvenWhenEmpty(t *testing.T) { - apply, err := buildCandidateStatusApply(testNamespace, cn("a"), "17", dgdcreconcile.DesiredCandidate{ID: "a"}) - if err != nil { - t.Fatal(err) - } - metrics, found := nested(t, apply, "status", "metrics") - if !found { - t.Fatal("an evaluated candidate without metrics must still carry an empty metrics object: its presence marks it populated") - } - if object, ok := metrics.(map[string]any); !ok || len(object) != 0 { - t.Fatalf("metrics = %#v, want an empty object", metrics) - } -} - -func TestRunStatusApplyPayload(t *testing.T) { - when := time.Date(2026, 10, 5, 21, 35, 12, 0, time.UTC) - status := RunStatus{Published: 2, LastProgress: when, CandidateNames: []string{"x", "y"}} - apply := buildRunStatusApply(testNamespace, testRunName, "42", status) - if apply.GetKind() != "DynamoGraphDeploymentRun" || apply.GetName() != testRunName { - t.Fatalf("identity = %s %s", apply.GetKind(), apply.GetName()) - } - if got, _ := nested(t, apply, "metadata", "resourceVersion"); got != "42" { - t.Fatalf("the status write must be constrained to the observed run version, got %v", got) - } - if _, found := nested(t, buildRunStatusApply(testNamespace, testRunName, "", status), "metadata", "resourceVersion"); found { - t.Fatal("no version known, no constraint") - } - if got, _ := nested(t, apply, "status", "lastProgressTime"); got != "2026-10-05T21:35:12Z" { - t.Fatalf("lastProgressTime = %v", got) - } - if got, _ := nested(t, apply, "status", "progress", "candidates", "published"); got != int64(2) { - t.Fatalf("published = %#v", got) - } - for _, field := range []string{"message", "provenance"} { - if _, found := nested(t, apply, "status", field); found { - t.Fatalf("status.%s does not exist in the API and must not be written", field) - } - } - refs, _ := nested(t, apply, "status", "candidateRefs") - if list, ok := refs.([]any); !ok || len(list) != 2 { - t.Fatalf("candidateRefs = %v", refs) - } - // A status without a progress time leaves the field out; the reconciler is what - // carries the known time into the status it writes. - status.LastProgress = time.Time{} - if _, found := nested(t, buildRunStatusApply(testNamespace, testRunName, "42", status), "status", "lastProgressTime"); found { - t.Fatal("no time known, so none is written") - } -} - func TestRunStatusWriteIsConstrainedToTheObservedRunVersion(t *testing.T) { f := newFixture(t) var run v1beta2.DynamoGraphDeploymentRun diff --git a/deploy/operator/internal/dgdrrunpublisher/terminal_test.go b/deploy/operator/internal/dgdrrunpublisher/terminal_test.go index 2ca586817368..0f30276c2040 100644 --- a/deploy/operator/internal/dgdrrunpublisher/terminal_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/terminal_test.go @@ -27,32 +27,6 @@ import ( "unicode/utf8" ) -func TestTerminalResultIsAStructuredRunFailure(t *testing.T) { - var failure *RunFailedError - err := terminalResult(DesiredState{Phase: PhaseFailed, Message: "m", Error: "e"}) - if !errors.Is(err, ErrRunFailed) || !errors.As(err, &failure) || failure.Outcome != (TerminalOutcome{Phase: PhaseFailed, Message: "m", Error: "e"}) { - t.Fatalf("failed: %v", err) - } - if got, want := err.Error(), ErrRunFailed.Error()+": m: e"; got != want { - t.Fatalf("error text = %q, want %q", got, want) - } - err = terminalResult(DesiredState{Phase: PhaseSucceeded, Failed: 1, FirstFailure: "x: render failed"}) - if !errors.As(err, &failure) || failure.Outcome.Error != "x: render failed" || !strings.Contains(failure.Outcome.Message, "1 candidate materialization failure") { - t.Fatalf("empty success: %v", err) - } -} - -func TestDesireCarriesBoundedDiagnostics(t *testing.T) { - long := strings.Repeat("e", 3*maxDiagnosticBytes) - state := desire(t, withRunDiagnostics(t, snapshotJSON(t, PhaseFailed, cand{id: "a"}), "no recommendation\ncould be materialized", long)) - if state.Phase != PhaseFailed || state.Message != "no recommendation could be materialized" { - t.Fatalf("state = %+v", state) - } - if len(state.Error) > maxDiagnosticBytes || !strings.HasPrefix(state.Error, "eee") { - t.Fatalf("error must be kept but bounded, got %d bytes", len(state.Error)) - } -} - func TestBoundDiagnostic(t *testing.T) { if got := boundDiagnostic(" a\nb\x00c \t"); got != "a b c" { t.Fatalf("got %q", got) @@ -67,20 +41,6 @@ func TestBoundDiagnostic(t *testing.T) { } } -func TestTerminalSucceededWithoutAMaterializedCandidateCarriesTheCandidateFailure(t *testing.T) { - f := newFixture(t) - f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "x", failed: true})) - f.mustReconcile() - var failure *RunFailedError - err := f.r.Completion.Err() - if !errors.Is(err, ErrRunFailed) || !errors.As(err, &failure) || ExitCode(err) != ExitRunFailed { - t.Fatalf("err = %v", err) - } - if !strings.HasPrefix(failure.Outcome.Error, "x: ") { - t.Fatalf("the candidate failure must be part of the diagnostics: %+v", failure.Outcome) - } -} - func TestTerminalFailureDiagnosticsSurviveTheCompletionHandoff(t *testing.T) { f := newFixture(t) message := "no recommendation could be materialized" diff --git a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go index 2faaeab9e161..c89becf3bdc4 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go @@ -228,6 +228,13 @@ func TestEnvtestCandidateApplyIsAcceptedAndIdempotent(t *testing.T) { if !strings.Contains(got.Annotations[AnnotationAdditionalResources], "kind: ConfigMap") { t.Errorf("companion annotation = %q", got.Annotations[AnnotationAdditionalResources]) } + var parameters map[string]any + if err := json.Unmarshal(got.Spec.Parameters.Raw, ¶meters); err != nil || parameters["tp"] != float64(2) { + t.Errorf("parameters = %s, err = %v", got.Spec.Parameters.Raw, err) + } + if got.Status.Metrics != nil { + t.Errorf("candidate apply wrote through the status subresource: %+v", got.Status) + } if !hasApplyManager(got.ManagedFields, "") { t.Errorf("no apply manager %q on the candidate: %+v", fieldOwner, got.ManagedFields) } @@ -353,6 +360,10 @@ func TestEnvtestCandidateStatusIsWrittenOnceAndGuardedByResourceVersion(t *testi if got.Status.Metrics == nil { t.Fatal("status.metrics is not set after PopulateStatus") } + var metrics map[string]any + if err := json.Unmarshal(got.Status.Metrics.Raw, &metrics); err != nil || metrics["throughput"] != 12.5 { + t.Errorf("metrics = %s, err = %v", got.Status.Metrics.Raw, err) + } if !hasApplyManager(got.ManagedFields, "status") { t.Errorf("no status apply manager %q: %+v", fieldOwner, got.ManagedFields) } @@ -371,6 +382,28 @@ func TestEnvtestCandidateStatusIsWrittenOnceAndGuardedByResourceVersion(t *testi if !after.Populated { t.Error("a candidate with status was reported as not populated") } + + // Even an evaluated candidate without metrics gets an empty object. Presence, not + // contents, is the persisted marker that its status has been populated. + empty := envtestCandidate() + empty.ID = "empty" + empty.Metrics = nil + emptyName := f.name(empty.ID) + emptyStored, err := f.writer.CreateCandidate(f.ctx, f.run, emptyName, empty) + if err != nil { + t.Fatalf("create candidate without metrics: %v", err) + } + if err := f.writer.PopulateStatus(f.ctx, emptyName, emptyStored.ResourceVersion, empty); err != nil { + t.Fatalf("populate candidate without metrics: %v", err) + } + var emptyMetrics map[string]any + got = f.candidate(emptyName) + if got.Status.Metrics == nil { + t.Fatal("candidate without metrics was not marked populated") + } + if err := json.Unmarshal(got.Status.Metrics.Raw, &emptyMetrics); err != nil || len(emptyMetrics) != 0 { + t.Fatalf("metrics = %s, err = %v; want an empty object", got.Status.Metrics.Raw, err) + } } func TestEnvtestRunStatusIsOwnedByThePublisher(t *testing.T) { From 44c519f3e788dcb240d5e87a40bb4dd5cc645518 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 17:44:31 +0100 Subject: [PATCH 037/100] pr-15894: pr-5: retry terminal publication [upstream: devivasudevan/dynamo#5] Signed-off-by: Dr. Stefan Schimanski --- .../operator/internal/dgdrrunpublisher/doc.go | 4 ++- .../internal/dgdrrunpublisher/reconciler.go | 36 +++++-------------- .../dgdrrunpublisher/reconciler_test.go | 26 ++++++++++---- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/doc.go b/deploy/operator/internal/dgdrrunpublisher/doc.go index 76a8b1affe08..ca31483033a7 100644 --- a/deploy/operator/internal/dgdrrunpublisher/doc.go +++ b/deploy/operator/internal/dgdrrunpublisher/doc.go @@ -62,5 +62,7 @@ // Sweeper exited zero, no terminal snapshot -> reconcile last snapshot, // exit ExitProtocolViolation // invalid or regressing snapshot, Sweeper exited -> exit ExitProtocolViolation -// final/last snapshot cannot be reconciled -> exit ExitReconcileFailed +// final/last snapshot cannot be read -> exit ExitReconcileFailed +// final/last snapshot has a permanent projection error -> exit ExitReconcileFailed +// transient API/network failure after Sweeper exit -> retry until success or Pod cancellation package dgdrrunpublisher diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go index c65837fcdd83..5b895403e3a6 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -28,7 +28,6 @@ import ( v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -53,10 +52,6 @@ const ( // fieldOwner is the server-side-apply field manager of the publisher. fieldOwner = "dgdr-run-publisher" - - // finalReconcileAttempts bounds the retries once the Sweeper has exited: nothing will - // rewrite the snapshot, so a persistent failure must end the process. - finalReconcileAttempts = 3 ) // Reconciler reconciles the one DynamoGraphDeploymentRun this publisher serves. @@ -96,12 +91,6 @@ type Reconciler struct { Observer *Observer Store *InputStore Completion *Completion - - // The fields below are touched only by Reconcile; the controller runs one worker, so - // they need no locking. - - // finalFailures counts failed reconciles after the Sweeper exited. - finalFailures int } func (r *Reconciler) key() types.NamespacedName { @@ -146,7 +135,8 @@ func (r *Reconciler) reconcile(ctx context.Context) error { r.Completion.Finish(input.Err) return nil } - return r.retryOrFail(exited, input.Err) + r.Completion.Finish(fmt.Errorf("reading last snapshot: %w", input.Err)) + return nil } if input.Snapshot == nil { if exited { @@ -157,7 +147,7 @@ func (r *Reconciler) reconcile(ctx context.Context) error { desired := Desire(r.RunName, r.RunUID, input.Snapshot) if err := r.apply(ctx, &run, desired); err != nil { - return r.retryOrFail(exited, err) + return r.retryOrFail(err) } switch { case desired.Terminal: @@ -168,25 +158,15 @@ func (r *Reconciler) reconcile(ctx context.Context) error { return nil } -// retryOrFail hands a failure to the workqueue's backoff, except where retrying cannot -// help (malformed projection) or the Sweeper is gone and the attempts are used up. -func (r *Reconciler) retryOrFail(exited bool, err error) error { +// retryOrFail hands a transient failure to the workqueue's backoff. A malformed +// projection is permanent and completes the publisher immediately. Kubernetes API and +// network failures remain retryable after the Sweeper exits because the terminal snapshot +// is durable and the publisher Pod's lifecycle is the cancellation boundary. +func (r *Reconciler) retryOrFail(err error) error { if _, ok := errors.AsType[*dgdcreconcile.DiffInputError](err); ok { r.Completion.Finish(err) // retrying cannot fix it return nil } - if apierrors.IsConflict(err) { - // A write was based on a stale cached read. The informer delivers the newer - // version and the retry decides again, so this is not a failed attempt. - return err - } - if exited { - r.finalFailures++ - if r.finalFailures >= finalReconcileAttempts { - r.Completion.Finish(fmt.Errorf("reconciling last snapshot: %w", err)) - return nil - } - } return err } diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go index 06c0e8fdb9b9..92ced15065ab 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -584,6 +584,19 @@ func TestInvalidFinalSnapshotIsAProtocolViolationExit(t *testing.T) { } } +func TestFinalSnapshotReadFailureEndsThePublisher(t *testing.T) { + f := newFixture(t) + f.r.Store.fail(errors.New("permission denied")) + f.exitSweeper(1) + + f.mustReconcile() + + err := f.r.Completion.Err() + if err == nil || ExitCode(err) != ExitReconcileFailed || !strings.Contains(err.Error(), "reading last snapshot") { + t.Fatalf("err = %v, want terminal snapshot read failure", err) + } +} + func TestRegressingSnapshotIsNotReconciled(t *testing.T) { f := newFixture(t) f.publish(withProgress(t, snapshotJSON(t, PhaseRunning, cand{id: "a"}), 3, 5)) @@ -596,20 +609,21 @@ func TestRegressingSnapshotIsNotReconciled(t *testing.T) { } } -func TestFinalReconcileFailureEndsTheProcessAfterBoundedRetries(t *testing.T) { +func TestFinalReconcileRetriesUntilTheAPIRecovers(t *testing.T) { f := newFixture(t) f.w.failOn["status"] = errors.New("api unavailable") f.w.sticky = true f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) f.exitSweeper(1) - for attempt := 1; attempt < finalReconcileAttempts; attempt++ { + for attempt := 1; attempt <= 5; attempt++ { if err := f.reconcile(); err == nil || f.r.Completion.Finished() { t.Fatalf("attempt %d: want a retry, err=%v", attempt, err) } } + delete(f.w.failOn, "status") f.mustReconcile() - if err := f.r.Completion.Err(); err == nil || ExitCode(err) != ExitReconcileFailed { - t.Fatalf("err = %v", err) + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) } } @@ -843,13 +857,13 @@ func TestExistingCandidateStatusUsesItsCachedVersion(t *testing.T) { } } -func TestConflictIsRetriedWithoutUsingUpTheFinalAttempts(t *testing.T) { +func TestConflictRemainsRetryableUntilTheCacheCatchesUp(t *testing.T) { f := newFixture(t) f.w.failOn["status"] = apierrors.NewConflict(schema.GroupResource{Group: "nvidia.com", Resource: "dynamographdeploymentruns"}, testRunName, errors.New("stale")) f.w.sticky = true f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) f.exitSweeper(1) - for attempt := range 2 * finalReconcileAttempts { + for attempt := range 6 { if err := f.reconcile(); !apierrors.IsConflict(err) || f.r.Completion.Finished() { t.Fatalf("attempt %d: a conflict from a stale read must be retried, err=%v finished=%v", attempt, err, f.r.Completion.Finished()) } From b540fb09f7220c45df550c0862e2ab7ba7d9dcdd Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 19:02:36 +0100 Subject: [PATCH 038/100] pr-15894: pr-5: wait for publication cache echoes [upstream: devivasudevan/dynamo#5] Signed-off-by: Dr. Stefan Schimanski --- .../internal/dgdrrunpublisher/reconciler.go | 152 +++++++++------ .../dgdrrunpublisher/reconciler_test.go | 57 +++++- .../internal/dgdrrunpublisher/snapshot.go | 37 +++- .../dgdrrunpublisher/snapshot_test.go | 1 + .../dgdrrunpublisher/terminal_test.go | 1 + .../internal/dgdrrunpublisher/writer.go | 6 +- .../dgdrrunpublisher/writer_envtest_test.go | 177 +++++++++++------- 7 files changed, 306 insertions(+), 125 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler.go b/deploy/operator/internal/dgdrrunpublisher/reconciler.go index 5b895403e3a6..190680f0b863 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler.go @@ -21,16 +21,12 @@ import ( "context" "errors" "fmt" - "strings" + "slices" - dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" - v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdcreconcile" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -65,12 +61,11 @@ const ( // Job pod names cannot be listed in resourceNames). // // Write echoes and cache lag: the publisher's own writes come back as watch events, and -// the cache can trail them. The reconciler keeps no record of its writes. Every pass -// decides from the current observations (the cached run and candidates, the latest input -// snapshot), so a repeated pass while the cache catches up may repeat an idempotent -// write, which is harmless, and an echo of a write finds nothing left to do. Anything -// that changes behind the publisher's back (a deleted candidate, a modified run status) -// is repaired on the next pass. +// the cache can trail them. Every pass still decides from current cached observations, +// so repeated idempotent writes are harmless. The reconciler remembers only writes that +// have not yet appeared in the cache; this is a completion barrier, not desired state. +// A terminal result is acknowledged only after those echoes arrive and a final pass +// observes that the desired state needs no more writes. // // Stale reads: every status write is constrained to the resourceVersion the decision was // based on (the cached run, the cached candidate, or the version read back from the API @@ -91,6 +86,24 @@ type Reconciler struct { Observer *Observer Store *InputStore Completion *Completion + + pendingCandidates map[string]candidateObservation + pendingRun *runObservation +} + +// candidateObservation describes the cache echo that proves a candidate write has +// become visible. A deletion is observed when the written UID is absent; a create or +// status write is observed only on the exact candidate incarnation. +type candidateObservation struct { + UID types.UID + ID string + Present bool + Populated bool +} + +type runObservation struct { + BaseResourceVersion string + Status RunStatus } func (r *Reconciler) key() types.NamespacedName { @@ -146,9 +159,13 @@ func (r *Reconciler) reconcile(ctx context.Context) error { } desired := Desire(r.RunName, r.RunUID, input.Snapshot) - if err := r.apply(ctx, &run, desired); err != nil { + quiescent, err := r.apply(ctx, &run, desired) + if err != nil { return r.retryOrFail(err) } + if !quiescent { + return nil + } switch { case desired.Terminal: r.Completion.Finish(terminalResult(desired)) @@ -173,14 +190,16 @@ func (r *Reconciler) retryOrFail(err error) error { // apply drives the cluster toward the desired state: create missing candidates, finish // the status of candidates that lack it, update the run, then delete obsolete candidates. // It is idempotent and safe to retry; existing candidate specs are never touched. -func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired DesiredState) error { +func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired DesiredState) (bool, error) { if desired.Failed > 0 { ctrl.LoggerFrom(ctx).V(1).Info("candidates failed materialization and are not published", "count", desired.Failed) } var list v1beta2.DynamoGraphDeploymentCandidateList if err := r.Client.List(ctx, &list, client.InNamespace(r.Namespace)); err != nil { - return fmt.Errorf("listing candidates: %w", err) + return false, fmt.Errorf("listing candidates: %w", err) } + r.observeWrites(run, list.Items) + wrote := false current := make([]dgdcreconcile.CurrentDGDC, 0, len(list.Items)) ownedObsolete := make([]string, 0, len(list.Items)) uids := make(map[string]types.UID, len(list.Items)) @@ -221,7 +240,7 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme } actions, err := dgdcreconcile.ComputeActions(desired.Candidates, current) if err != nil { - return err + return false, err } actions.Deletes = append(actions.Deletes, ownedObsolete...) // populated names the candidates whose status the API server already has; they are @@ -231,8 +250,11 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme name := desired.Names[candidate.ID] stored, err := r.Writer.CreateCandidate(ctx, run, name, candidate) if err != nil { - return fmt.Errorf("creating candidate %s: %w", candidate.ID, err) + return false, fmt.Errorf("creating candidate %s: %w", candidate.ID, err) } + wrote = true + r.expectCandidate(name, candidateObservation{UID: stored.UID, ID: candidate.ID, Present: true, Populated: stored.Populated}) + uids[name] = stored.UID versions[name] = stored.ResourceVersion // a new candidate is not in the cache; use the version the API server returned if stored.Populated { populated[name] = struct{}{} @@ -247,20 +269,26 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme continue } if err := r.Writer.PopulateStatus(ctx, name, versions[name], candidate); err != nil { - return fmt.Errorf("populating status of candidate %s: %w", candidate.ID, err) + return false, fmt.Errorf("populating status of candidate %s: %w", candidate.ID, err) } + wrote = true + r.expectCandidate(name, candidateObservation{UID: uids[name], ID: candidate.ID, Present: true, Populated: true}) } - if err := r.updateRunStatus(ctx, run, desired.Status); err != nil { - return err + updated, err := r.updateRunStatus(ctx, run, desired.Status) + if err != nil { + return false, err } + wrote = wrote || updated for _, name := range actions.Deletes { if err := r.Writer.DeleteCandidate(ctx, name, uids[name]); err != nil { - return fmt.Errorf("deleting candidate %s: %w", name, err) + return false, fmt.Errorf("deleting candidate %s: %w", name, err) } + wrote = true + r.expectCandidate(name, candidateObservation{UID: uids[name]}) } - return nil + return !wrote && len(r.pendingCandidates) == 0 && r.pendingRun == nil, nil } // updateRunStatus writes the run status when it differs from what the cache shows. A @@ -268,16 +296,61 @@ func (r *Reconciler) apply(ctx context.Context, run *v1beta2.DynamoGraphDeployme // because a server-side-apply payload that omits the field would remove it. The write is // constrained to the cached run's resourceVersion, so a decision made from a stale cache // fails with a conflict instead of overwriting newer status. -func (r *Reconciler) updateRunStatus(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired RunStatus) error { +func (r *Reconciler) updateRunStatus(ctx context.Context, run *v1beta2.DynamoGraphDeploymentRun, desired RunStatus) (bool, error) { observed := observedStatus(run) if !desired.NeedsUpdate(observed) { - return nil + return false, nil } write := desired.WithPreservedProgress(observed.LastProgress) if err := r.Writer.UpdateRunStatus(ctx, run.ResourceVersion, write); err != nil { - return fmt.Errorf("updating run status: %w", err) + return false, fmt.Errorf("updating run status: %w", err) } - return nil + r.pendingRun = &runObservation{BaseResourceVersion: run.ResourceVersion, Status: write} + return true, nil +} + +func (r *Reconciler) expectCandidate(name string, observation candidateObservation) { + if r.pendingCandidates == nil { + r.pendingCandidates = map[string]candidateObservation{} + } + r.pendingCandidates[name] = observation +} + +// observeWrites removes completion-barrier entries only when the corresponding write is +// visible through the informer cache. API-server read-backs are deliberately insufficient: +// completion stops the process, and therefore must not race the watch echoes that trigger +// repairs for a later terminal snapshot. +func (r *Reconciler) observeWrites(run *v1beta2.DynamoGraphDeploymentRun, candidates []v1beta2.DynamoGraphDeploymentCandidate) { + byName := make(map[string]*v1beta2.DynamoGraphDeploymentCandidate, len(candidates)) + for i := range candidates { + byName[candidates[i].Name] = &candidates[i] + } + for name, expected := range r.pendingCandidates { + candidate := byName[name] + if !expected.Present { + if candidate == nil || candidate.UID != expected.UID { + delete(r.pendingCandidates, name) + } + continue + } + if candidate == nil || candidate.UID != expected.UID { + continue + } + owner := metav1.GetControllerOf(candidate) + if owner == nil || owner.UID != run.UID || candidate.Labels[LabelRunUID] != r.RunUID || candidate.Labels[LabelCandidateID] != expected.ID { + continue + } + if !expected.Populated || candidate.Status.Metrics != nil { + delete(r.pendingCandidates, name) + } + } + if r.pendingRun != nil && run.ResourceVersion != r.pendingRun.BaseResourceVersion && runStatusEqual(observedStatus(run), r.pendingRun.Status) { + r.pendingRun = nil + } +} + +func runStatusEqual(left, right RunStatus) bool { + return left.Published == right.Published && left.LastProgress.Equal(right.LastProgress) && slices.Equal(left.CandidateNames, right.CandidateNames) } // observedStatus is the publisher-owned part of the run's current (cached) status. @@ -316,32 +389,3 @@ func sweeperStateOf(pod *corev1.Pod, container string) SweeperState { } return SweeperState{} } - -// decodeManifest extracts the DGD spec and the companion resources from a rendered -// manifest. A DGD in either supported apiVersion is converted to v1beta1, the same way -// the DGDR controller does it. -func decodeManifest(candidate dgdcreconcile.DesiredCandidate) (*v1beta1.DynamoGraphDeploymentSpec, string, error) { - manifest, err := ParseManifest(candidate.Spec) - if err != nil { - return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) - } - obj := &unstructured.Unstructured{Object: manifest.DGD} - var beta v1beta1.DynamoGraphDeployment - switch obj.GetAPIVersion() { - case dgdv1alpha1.GroupVersion.String(): - var alpha dgdv1alpha1.DynamoGraphDeployment - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &alpha); err != nil { - return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) - } - if err := alpha.ConvertTo(&beta); err != nil { - return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) - } - case v1beta1.GroupVersion.String(): - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &beta); err != nil { - return nil, "", fmt.Errorf("candidate %s: %w", candidate.ID, err) - } - default: - return nil, "", fmt.Errorf("candidate %s: unsupported DynamoGraphDeployment apiVersion %q", candidate.ID, obj.GetAPIVersion()) - } - return &beta.Spec, strings.Join(manifest.Companions, "\n---\n"), nil -} diff --git a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go index 92ced15065ab..3be526dac0fe 100644 --- a/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/reconciler_test.go @@ -96,7 +96,7 @@ func (w *memWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoGrap // The apply is idempotent; the read-back reports what the server holds. populated := w.server[name] w.server[name] = populated - stored := StoredCandidate{ResourceVersion: createdVersion(name), Populated: populated} + stored := StoredCandidate{UID: types.UID("uid-" + name), ResourceVersion: createdVersion(name), Populated: populated} if w.lag { return stored, nil } @@ -467,6 +467,10 @@ func TestTerminalSucceededSnapshotIsReconciledThenAcknowledged(t *testing.T) { f := newFixture(t) f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "a"})) f.mustReconcile() + if f.r.Completion.Finished() { + t.Fatal("the publisher must wait for its candidate and run-status writes to reach the cache") + } + f.mustReconcile() if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) } @@ -475,10 +479,54 @@ func TestTerminalSucceededSnapshotIsReconciledThenAcknowledged(t *testing.T) { } } +func TestTerminalCompletionWaitsForWritesFromThePreviousSnapshot(t *testing.T) { + f := newFixture(t) + f.w.lag = true + f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) + f.mustReconcile() + f.takeOps() + + // The terminal snapshot arrives before the cache sees the candidate and Run status + // created for the previous snapshot. Their absence is not convergence: the API writes + // have succeeded and their watch echoes are still in flight. + f.publish(snapshotJSON(t, PhaseSucceeded, cand{id: "b"})) + f.mustReconcile() + if f.r.Completion.Finished() { + t.Fatal("terminal completion raced writes that were not observed through the cache") + } + + // Deliver those echoes. The terminal projection then removes the now-visible + // candidate and status reference, and waits for those writes in turn. + run := f.currentRun() + run.Status.Progress = &v1beta2.DGDRRunProgress{Candidates: &v1beta2.CandidateProgress{Published: 1}} + run.Status.CandidateRefs = []corev1.LocalObjectReference{{Name: cn("a")}} + if err := f.c.Update(context.Background(), run); err != nil { + t.Fatal(err) + } + candidate := candidateForRun(cn("a"), "a", run) + candidate.UID = types.UID("uid-" + cn("a")) + candidate.Status.Metrics = &runtime.RawExtension{Raw: []byte("{}")} + if err := f.c.Create(context.Background(), candidate); err != nil { + t.Fatal(err) + } + f.w.server[cn("b")] = false + f.w.lag = false + f.mustReconcile() + if f.r.Completion.Finished() { + t.Fatal("terminal completion raced the candidate deletion and final Run status") + } + + f.mustReconcile() + if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { + t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) + } +} + func TestTerminalFailedSnapshotIsReconciledThenFails(t *testing.T) { f := newFixture(t) f.publish(withRunDiagnostics(t, snapshotJSON(t, PhaseFailed, cand{id: "a"}), "search failed", "backend unavailable")) f.mustReconcile() + f.mustReconcile() if err := f.r.Completion.Err(); !errors.Is(err, ErrRunFailed) || ExitCode(err) != ExitRunFailed || !strings.Contains(err.Error(), "search failed") || !strings.Contains(err.Error(), "backend unavailable") { t.Fatalf("err = %v", err) @@ -512,6 +560,7 @@ func TestTerminalSnapshotReconcileFailureIsRetriedUntilItSucceeds(t *testing.T) t.Fatalf("a terminal snapshot that could not be reconciled must be retried, err=%v", err) } f.mustReconcile() + f.mustReconcile() if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) } @@ -522,6 +571,7 @@ func TestSweeperCrashWithoutTerminalSnapshotReconcilesTheLastSnapshot(t *testing f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) f.exitSweeper(137) f.mustReconcile() + f.mustReconcile() if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { t.Fatalf("the Sweeper's own exit code fails the Job, not the publisher: finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) } @@ -535,6 +585,7 @@ func TestSweeperCleanExitWithoutTerminalSnapshotIsAProtocolViolation(t *testing. f.publish(snapshotJSON(t, PhaseRunning, cand{id: "a"})) f.exitSweeper(0) f.mustReconcile() + f.mustReconcile() if err := f.r.Completion.Err(); !errors.Is(err, ErrMissingTerminalSnapshot) || ExitCode(err) != ExitProtocolViolation { t.Fatalf("err = %v", err) } @@ -566,6 +617,7 @@ func TestSnapshotWrittenJustBeforeTheSweeperExitIsReadAfterTheExitIsSeen(t *test writeSnapshot(t, filepath.Dir(f.r.Observer.Path), snapshotJSON(t, PhaseSucceeded, cand{id: "a"}, cand{id: "b"})) f.exitSweeper(0) f.mustReconcile() + f.mustReconcile() if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { t.Fatalf("the terminal snapshot must be picked up on exit: finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) } @@ -622,6 +674,7 @@ func TestFinalReconcileRetriesUntilTheAPIRecovers(t *testing.T) { } delete(f.w.failOn, "status") f.mustReconcile() + f.mustReconcile() if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) } @@ -736,6 +789,7 @@ func TestReconcileIgnoresOtherKeysAndAFinishedRun(t *testing.T) { t.Fatalf("another key must be ignored: err=%v ops=%v", err, f.w.ops) } f.mustReconcile() + f.mustReconcile() ops := len(f.w.ops) f.mustReconcile() if len(f.w.ops) != ops { @@ -870,6 +924,7 @@ func TestConflictRemainsRetryableUntilTheCacheCatchesUp(t *testing.T) { } delete(f.w.failOn, "status") // the cache caught up f.mustReconcile() + f.mustReconcile() if !f.r.Completion.Finished() || f.r.Completion.Err() != nil { t.Fatalf("finished=%v err=%v", f.r.Completion.Finished(), f.r.Completion.Err()) } diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot.go b/deploy/operator/internal/dgdrrunpublisher/snapshot.go index df9f2d3e9598..5c9b5a7fd2e1 100644 --- a/deploy/operator/internal/dgdrrunpublisher/snapshot.go +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot.go @@ -23,6 +23,10 @@ import ( "regexp" "strings" + dgdv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/yaml" ) @@ -97,12 +101,41 @@ func ParseManifest(manifest string) (*Manifest, error) { } func validateManifest(id, manifest string) error { - if _, err := ParseManifest(manifest); err != nil { - return violation("manifest of candidate %q: %v", id, err) + if _, _, err := decodeManifest(id, manifest); err != nil { + return violation("%v", err) } return nil } +// decodeManifest extracts the DGD spec and companion resources from a rendered +// manifest. A DGD in either supported apiVersion crosses the same typed conversion +// boundary before the snapshot is accepted and before a Candidate is built. +func decodeManifest(id, source string) (*v1beta1.DynamoGraphDeploymentSpec, string, error) { + manifest, err := ParseManifest(source) + if err != nil { + return nil, "", fmt.Errorf("manifest of candidate %q: %w", id, err) + } + obj := &unstructured.Unstructured{Object: manifest.DGD} + var beta v1beta1.DynamoGraphDeployment + switch obj.GetAPIVersion() { + case dgdv1alpha1.GroupVersion.String(): + var alpha dgdv1alpha1.DynamoGraphDeployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &alpha); err != nil { + return nil, "", fmt.Errorf("manifest of candidate %q: decoding %s DynamoGraphDeployment: %w", id, obj.GetAPIVersion(), err) + } + if err := alpha.ConvertTo(&beta); err != nil { + return nil, "", fmt.Errorf("manifest of candidate %q: converting %s DynamoGraphDeployment: %w", id, obj.GetAPIVersion(), err) + } + case v1beta1.GroupVersion.String(): + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &beta); err != nil { + return nil, "", fmt.Errorf("manifest of candidate %q: decoding %s DynamoGraphDeployment: %w", id, obj.GetAPIVersion(), err) + } + default: + return nil, "", fmt.Errorf("manifest of candidate %q: unsupported DynamoGraphDeployment apiVersion %q", id, obj.GetAPIVersion()) + } + return &beta.Spec, strings.Join(manifest.Companions, "\n---\n"), nil +} + // SnapshotSchemaVersion is the only wire version this publisher understands. const SnapshotSchemaVersion = 1 diff --git a/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go index cbd6cfd5579a..aef40b37a211 100644 --- a/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/snapshot_test.go @@ -124,6 +124,7 @@ func TestParseSnapshotValidation(t *testing.T) { "unsafe id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"Not_Valid","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, "long id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"%s","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, "wrong kind": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"kind: ConfigMap\\nspec: {}\\n"}]}`, + "malformed typed spec": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"a","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec:\\n components: not-a-list\\n"}]}`, "empty id": `{"schemaVersion":1,"run":{"phase":"Running","terminal":false},"candidates":[{"id":"","outcome":"materialized","manifest":"apiVersion: nvidia.com/v1beta1\\nkind: DynamoGraphDeployment\\nspec: {}\\n"}]}`, } bad["long id"] = strings.Replace(bad["long id"], "%s", strings.Repeat("a", 64), 1) diff --git a/deploy/operator/internal/dgdrrunpublisher/terminal_test.go b/deploy/operator/internal/dgdrrunpublisher/terminal_test.go index 0f30276c2040..0c3129350b4e 100644 --- a/deploy/operator/internal/dgdrrunpublisher/terminal_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/terminal_test.go @@ -47,6 +47,7 @@ func TestTerminalFailureDiagnosticsSurviveTheCompletionHandoff(t *testing.T) { long := "backend vllm rejected the generated worker configuration " + strings.Repeat("x", 3*maxDiagnosticBytes) f.publish(withRunDiagnostics(t, snapshotJSON(t, PhaseFailed, cand{id: "a"}), message, long)) f.mustReconcile() + f.mustReconcile() err := f.r.Completion.Err() var failure *RunFailedError diff --git a/deploy/operator/internal/dgdrrunpublisher/writer.go b/deploy/operator/internal/dgdrrunpublisher/writer.go index 5ef51d861f16..f0ff7dbf0fc4 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer.go @@ -36,6 +36,9 @@ import ( // StoredCandidate is the API server's view of a candidate right after it was applied. type StoredCandidate struct { + // UID identifies the incarnation returned by the API server. The publisher waits + // until that incarnation is visible through its cache before acknowledging completion. + UID types.UID // ResourceVersion is the version a status write for this candidate is constrained to. ResourceVersion string // Populated reports that the candidate already has its evaluation status (a metrics @@ -132,6 +135,7 @@ func (w *clientWriter) CreateCandidate(ctx context.Context, run *v1beta2.DynamoG return StoredCandidate{}, fmt.Errorf("candidate %s is no longer controlled by run %s after apply", name, run.Name) } return StoredCandidate{ + UID: stored.UID, ResourceVersion: stored.ResourceVersion, Populated: stored.Status.Metrics != nil, }, nil @@ -162,7 +166,7 @@ func (w *clientWriter) DeleteCandidate(ctx context.Context, name string, uid typ // and controller owner reference, the DGD spec with the resolved parameters, and the // companion resources as an annotation. The status goes through the status subresource. func buildCandidateApply(scheme *runtime.Scheme, namespace, runUID string, run *v1beta2.DynamoGraphDeploymentRun, name string, candidate dgdcreconcile.DesiredCandidate) (*unstructured.Unstructured, error) { - spec, companions, err := decodeManifest(candidate) + spec, companions, err := decodeManifest(candidate.ID, candidate.Spec) if err != nil { return nil, err } diff --git a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go index c89becf3bdc4..4343732d9842 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go @@ -125,7 +125,7 @@ func envtestCandidate() dgdcreconcile.DesiredCandidate { func foreignCandidate(t *testing.T, namespace, name string, candidate dgdcreconcile.DesiredCandidate) *v1beta2.DynamoGraphDeploymentCandidate { t.Helper() - spec, _, err := decodeManifest(candidate) + spec, _, err := decodeManifest(candidate.ID, candidate.Spec) if err != nil { t.Fatal(err) } @@ -197,6 +197,76 @@ func waitForReconcileQuiet(t *testing.T, recorder *reconcileRecorder, quiet, tim }, timeout, 10*time.Millisecond, "reconciliation count did not become stable") } +func startPublisherController(t *testing.T, f *writerFixture, snapshot *Snapshot) (*reconcileRecorder, *Completion, *InputStore, chan event.GenericEvent) { + t.Helper() + podName := "publisher-pod" + pod := &corev1.Pod{ + Namespace: f.run.Namespace, Name: podName, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "sweeper", Image: "example.invalid/sweeper"}}}, + } + if err := f.client.Create(f.ctx, pod); err != nil { + t.Fatalf("create pod: %v", err) + } + + scheme, err := NewScheme() + if err != nil { + t.Fatal(err) + } + manager, err := ctrl.NewManager(f.config, ctrl.Options{ + Scheme: scheme, + Cache: cacheOptions(Config{Namespace: f.run.Namespace, RunName: f.run.Name, RunUID: string(f.run.UID), PodName: podName}), + Metrics: metricsserver.Options{BindAddress: "0"}, + LeaderElection: false, + }) + if err != nil { + t.Fatalf("create manager: %v", err) + } + store := &InputStore{} + store.accept(snapshot) + completion := NewCompletion() + reconciler := &Reconciler{ + Client: manager.GetClient(), + Writer: &clientWriter{ + Client: manager.GetClient(), + Reader: manager.GetAPIReader(), + Namespace: f.run.Namespace, + RunName: f.run.Name, + RunUID: string(f.run.UID), + }, + Namespace: f.run.Namespace, + RunName: f.run.Name, + RunUID: string(f.run.UID), + PodName: podName, + SweeperContainer: "sweeper", + Store: store, + Completion: completion, + } + recorder := &reconcileRecorder{delegate: reconciler} + events := make(chan event.GenericEvent, 1) + key := types.NamespacedName{Namespace: f.run.Namespace, Name: f.run.Name} + if err := setupController(manager, recorder, key, events); err != nil { + t.Fatalf("setup controller: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + done <- manager.Start(ctx) + }() + t.Cleanup(func() { + cancel() + select { + case err := <-done: + if err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("publisher stopped: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("publisher did not stop") + } + }) + return recorder, completion, store, events +} + func hasApplyManager(fields []metav1.ManagedFieldsEntry, subresource string) bool { for _, entry := range fields { if entry.Manager == fieldOwner && entry.Operation == metav1.ManagedFieldsOperationApply && entry.Subresource == subresource { @@ -478,9 +548,8 @@ func TestEnvtestDeleteCandidateHonorsTheUIDPrecondition(t *testing.T) { } } -func TestEnvtestCandidateUpdateTriggersStatusRepair(t *testing.T) { +func TestEnvtestProductionWatchesDriveRepairAndTerminalCompletion(t *testing.T) { f := newWriterFixture(t) - podName := "publisher-pod" candidate := envtestCandidate() snapshot, err := ParseSnapshot(snapshotJSON(t, PhaseRunning, cand{id: "a", metrics: candidate.Metrics})) if err != nil { @@ -489,13 +558,6 @@ func TestEnvtestCandidateUpdateTriggersStatusRepair(t *testing.T) { desired := Desire(f.run.Name, string(f.run.UID), snapshot) t.Log("Arrange a fully converged Run and Candidate before starting the manager") - pod := &corev1.Pod{ - Namespace: f.run.Namespace, Name: podName, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "sweeper", Image: "example.invalid/sweeper"}}}, - } - if err := f.client.Create(f.ctx, pod); err != nil { - t.Fatalf("create pod: %v", err) - } name := f.name("a") stored, err := f.writer.CreateCandidate(f.ctx, f.run, name, candidate) if err != nil { @@ -509,63 +571,7 @@ func TestEnvtestCandidateUpdateTriggersStatusRepair(t *testing.T) { } t.Log("Start the production controller watches around an instrumented Reconciler") - scheme, err := NewScheme() - if err != nil { - t.Fatal(err) - } - manager, err := ctrl.NewManager(f.config, ctrl.Options{ - Scheme: scheme, - Cache: cacheOptions(Config{Namespace: f.run.Namespace, RunName: f.run.Name, RunUID: string(f.run.UID), PodName: podName}), - Metrics: metricsserver.Options{BindAddress: "0"}, - LeaderElection: false, - }) - if err != nil { - t.Fatalf("create manager: %v", err) - } - store := &InputStore{} - store.accept(snapshot) - completion := NewCompletion() - reconciler := &Reconciler{ - Client: manager.GetClient(), - Writer: &clientWriter{ - Client: manager.GetClient(), - Reader: manager.GetAPIReader(), - Namespace: f.run.Namespace, - RunName: f.run.Name, - RunUID: string(f.run.UID), - }, - Namespace: f.run.Namespace, - RunName: f.run.Name, - RunUID: string(f.run.UID), - PodName: podName, - SweeperContainer: "sweeper", - Store: store, - Completion: completion, - } - recorder := &reconcileRecorder{delegate: reconciler} - events := make(chan event.GenericEvent, 1) - key := types.NamespacedName{Namespace: f.run.Namespace, Name: f.run.Name} - if err := setupController(manager, recorder, key, events); err != nil { - t.Fatalf("setup controller: %v", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { - done <- manager.Start(ctx) - }() - t.Cleanup(func() { - cancel() - select { - case err := <-done: - if err != nil && !errors.Is(err, context.Canceled) { - t.Errorf("publisher stopped: %v", err) - } - case <-time.After(5 * time.Second): - t.Error("publisher did not stop") - } - }) - + recorder, completion, store, events := startPublisherController(t, f, snapshot) waitForReconcileQuiet(t, recorder, 250*time.Millisecond, 10*time.Second) beforeUpdate := recorder.count.Load() @@ -587,4 +593,41 @@ func TestEnvtestCandidateUpdateTriggersStatusRepair(t *testing.T) { } return repaired.Status.Metrics != nil }, 10*time.Second, 25*time.Millisecond, "candidate status update did not trigger repair") + waitForReconcileQuiet(t, recorder, 250*time.Millisecond, 10*time.Second) + + t.Log("Publish a terminal replacement and let write echoes drive completion") + terminal, err := ParseSnapshot(snapshotJSON(t, PhaseSucceeded, cand{id: "b", metrics: candidate.Metrics})) + if err != nil { + t.Fatal(err) + } + if !store.accept(terminal) { + t.Fatal("terminal snapshot was not accepted") + } + beforeTerminal := recorder.count.Load() + events <- event.GenericEvent{Object: runKeyObject(types.NamespacedName{Namespace: f.run.Namespace, Name: f.run.Name})} + select { + case <-completion.Done(): + case <-time.After(10 * time.Second): + t.Fatal("terminal completion did not observe its replacement Candidate, Run status, and deletion through the cache") + } + if err := completion.Err(); err != nil { + t.Fatalf("terminal completion: %v", err) + } + if got := recorder.count.Load() - beforeTerminal; got < 2 { + t.Fatalf("terminal reconciliations = %d, want a write pass and a cache-observation pass", got) + } + + t.Log("Verify the state acknowledged by the publisher is present in the API server") + if got := f.candidate(f.name("b")); got.Status.Metrics == nil { + t.Fatal("candidate metrics were not populated before completion") + } + run := f.currentRun() + if len(run.Status.CandidateRefs) != 1 || run.Status.CandidateRefs[0].Name != f.name("b") { + t.Fatalf("candidateRefs = %+v, want the terminal candidate", run.Status.CandidateRefs) + } + var removed v1beta2.DynamoGraphDeploymentCandidate + err = f.client.Get(f.ctx, types.NamespacedName{Namespace: f.run.Namespace, Name: f.name("a")}, &removed) + if !apierrors.IsNotFound(err) { + t.Fatalf("obsolete candidate still exists after completion: %v", err) + } } From d3af812d9607b1261e221e14218d0f6b0ff9059f Mon Sep 17 00:00:00 2001 From: devivasudevan <49675305+devivasudevan@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:41:08 +0100 Subject: [PATCH 039/100] pr-17097: squash v1beta2 DGDR Request controller Squashed from https://github.com/ai-dynamo/dynamo/pull/17097. Signed-off-by: devivasudevan <49675305+devivasudevan@users.noreply.github.com> --- deploy/helm/charts/platform/README.md | 2 +- .../components/operator/files/role.yaml | 10 + .../operator/templates/deployment.yaml | 1 + .../platform/components/operator/values.yaml | 4 +- ...namespace_restriction_deployment_test.yaml | 22 + deploy/helm/charts/platform/values.yaml | 5 +- ...dynamographdeploymentrequest_conditions.go | 23 + .../dynamographdeploymentrun_conditions.go | 23 + .../v1beta2/dynamographdeploymentrun_types.go | 9 +- deploy/operator/cmd/main.go | 10 + ...aphdeploymentrequest_v1beta2_controller.go | 399 +++++++++++ ...request_v1beta2_controller_envtest_test.go | 153 +++++ ...ploymentrequest_v1beta2_controller_test.go | 624 ++++++++++++++++++ deploy/operator/internal/controller/setup.go | 16 + .../additional-resources/api-reference-k8s.md | 5 + .../kubernetes-api/full-api-reference.mdx | 5 + 16 files changed, 1304 insertions(+), 7 deletions(-) create mode 100644 deploy/operator/api/v1beta2/dynamographdeploymentrequest_conditions.go create mode 100644 deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index aebd71ec9fbb..9bc82dd97b8f 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -546,7 +546,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.grovePreflight.deploymentName | string | `""` | External Grove Deployment name. Requires namespace; empty discovers by container name. | | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether the cluster-wide operator applies CRDs from its image through the crd-apply init container. When false, the chart installs no CRDs; apply them separately before starting a cluster-wide operator. Namespace-restricted installations must set this to false and use the CRDs managed by the cluster-wide operator. | -| dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. The operator does not reconcile v1beta2 requests. | +| dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. When true, the operator also reconciles v1beta2 Requests into Runs; the controller that executes a Run is not part of the operator yet. | | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 3fff0d64273d..f2391bcaf6de 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -311,6 +311,16 @@ rules: - get - patch - update +- apiGroups: + - nvidia.com + resources: + - dynamographdeploymentruns + verbs: + - create + - delete + - get + - list + - watch - apiGroups: - nvidia.com resources: diff --git a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml index 8a4ea7dd9080..83e5c4acfc41 100644 --- a/deploy/helm/charts/platform/components/operator/templates/deployment.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/deployment.yaml @@ -92,6 +92,7 @@ spec: {{- with .Values.controllerManager.manager.dgdrDefaultImage }} - {{ printf "--dgdr-default-image=%s" . | quote }} {{- end }} + - --dgdr-v1beta2-enabled={{ .Values.dgdrV1Beta2.enabled }} command: - /manager env: diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index c526a93bebde..53ccc596eba2 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -36,7 +36,9 @@ dgdrV1Beta2: # Candidate objects are persisted in etcd; incompatible schema changes may make # them unreadable or block future CRD upgrades. When false, the CRD init container # installs the legacy Request schemas without v1beta2 conversion envelopes and - # omits Run and Candidate. The operator does not reconcile v1beta2 requests. + # omits Run and Candidate. When true, the operator also reconciles v1beta2 + # Requests into Runs; the controller that executes a Run is not part of the + # operator yet. enabled: false # Environment variables to pass to operator Deployment. diff --git a/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml b/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml index 41789e477c6b..d9855f666593 100644 --- a/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml +++ b/deploy/helm/charts/platform/tests/namespace_restriction_deployment_test.yaml @@ -93,3 +93,25 @@ tests: asserts: - failedTemplate: errorPattern: "webhook.namespaceSelector must be empty" + + - it: passes the DGDR v1beta2 switch to the manager + template: charts/dynamo-operator/templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --dgdr-v1beta2-enabled=false + documentSelector: + path: kind + value: Deployment + + - it: enables the DGDR v1beta2 controller in the manager + template: charts/dynamo-operator/templates/deployment.yaml + set: + dynamo-operator.dgdrV1Beta2.enabled: true + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --dgdr-v1beta2-enabled=true + documentSelector: + path: kind + value: Deployment diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 13c14c9c058f..fe6daeec961b 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -97,8 +97,9 @@ dynamo-operator: # -- EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 # Request API with conversion and install the Run and Candidate APIs. Run and # Candidate objects are persisted in etcd; incompatible schema changes may make - # them unreadable or block future CRD upgrades. The operator does not reconcile - # v1beta2 requests. + # them unreadable or block future CRD upgrades. When true, the operator also + # reconciles v1beta2 Requests into Runs; the controller that executes a Run is + # not part of the operator yet. enabled: false # Environment variables to pass to operator Deployment. diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_conditions.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_conditions.go new file mode 100644 index 000000000000..0e7ac1a4d9b8 --- /dev/null +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_conditions.go @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1beta2 + +// DGDRRequestConditionAccepted reports whether the current Request generation has been +// materialized as an immutable DynamoGraphDeploymentRun. Execution state belongs to the Run, not to +// this condition: a Run is finished when its DGDRRunConditionCompleted condition is True. +const DGDRRequestConditionAccepted = "Accepted" diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go new file mode 100644 index 000000000000..fdc0f0dd1d50 --- /dev/null +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_conditions.go @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1beta2 + +// DGDRRunConditionCompleted is the condition that makes a Run's outcome authoritative: it is True once +// the Run reached a terminal result. status.phase only describes how a completed Run ended, and +// consumers (such as the Request controller) decide on this condition, not on the phase. +const DGDRRunConditionCompleted = "Completed" diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go index f6643255e691..7a07c15eecae 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go @@ -24,10 +24,13 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. -// Run creators resolve optional Request inputs and the final AISimulate parameters before -// persisting the Run. Request admission defaulting does not apply to Runs. +// DynamoGraphDeploymentRunSpec is an immutable snapshot of an accepted DGDR spec. +// Request admission resolves its structural defaults before the Request controller copies it; +// Request admission defaulting does not apply to Runs. // +kubebuilder:validation:XValidation:rule="has(self.model)",message="model is required" +// +// The Run reconciler lowers this immutable typed snapshot plus inputOverrides exactly once and +// records the resulting public recommendation input in status.aisimulate.input. type DynamoGraphDeploymentRunSpec struct { NativeSearchInput `json:",inline"` } diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index e51be09f02d3..7c9c8acbe3b5 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -157,6 +157,7 @@ func main() { var operatorImage string var operatorImagePullPolicy string var dgdrDefaultImage string + var dgdrV1beta2Enabled bool flag.StringVar(&configFile, "config", "", "Path to operator configuration file (required)") flag.StringVar(&operatorVersion, "operator-version", "unknown", "Version of the operator (used in lease holder identity)") @@ -170,6 +171,8 @@ func main() { "Image pull policy for operator helper init containers") flag.StringVar(&dgdrDefaultImage, "dgdr-default-image", "", "Default DGDR profiler image, put into DGDR spec.image when unset; empty derives dynamo-planner:") + flag.BoolVar(&dgdrV1beta2Enabled, "dgdr-v1beta2-enabled", false, + "Reconcile native v1beta2 DGDRs; requires the v1beta2 Request and Run CRDs to be installed") opts := zap.Options{ Development: true, } @@ -496,6 +499,7 @@ func main() { mgr, operatorCfg, runtimeConfig, dockerSecretRetriever, sshKeyManager, operatorImage, pullPolicy, + dgdrV1beta2Enabled, ); err != nil { setupLog.Error(err, "failed to register controllers") os.Exit(1) @@ -545,6 +549,7 @@ func registerControllers( sshKeyManager *secret.SSHKeyManager, operatorImage string, operatorPullPolicy corev1.PullPolicy, + dgdrV1beta2Enabled bool, ) error { if operatorCfg.Namespace.Restricted == "" { // The cluster-wide manager cache covers all namespaces. ExcludedNamespaces only filters @@ -605,6 +610,11 @@ func registerControllers( }); err != nil { return err } + if dgdrV1beta2Enabled { + if err := controller.SetupDynamoGraphDeploymentRequestV1beta2(mgr, setupOptions); err != nil { + return err + } + } if err := controller.SetupDynamoModel(mgr, controller.DynamoModelSetupOptions{ SetupOptions: setupOptions, }); err != nil { diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go new file mode 100644 index 000000000000..fa3e9fe4521e --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -0,0 +1,399 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strconv" + "strings" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + // dgdrV1beta2GenerationAnnotation records the Request generation a Run was created for. It is + // the identity of a Run within its Request. metadata.name is only the convention used when a Run is + // created, so the controller never reconstructs or validates it for an existing Run. + dgdrV1beta2GenerationAnnotation = "nvidia.com/dgdr-request-generation" + + // dgdrRunControllerUIDIndex indexes Runs by the UID of their controller OwnerReference. The + // OwnerReference is the only definition of which Runs belong to a Request. + dgdrRunControllerUIDIndex = "dynamo.nvidia.com/dgdr-run-controller-uid" + + // dgdrV1beta2RetainedRuns is how many terminal Runs a Request keeps. Older ones are deleted. + dgdrV1beta2RetainedRuns = 5 + + dgdrReasonRunCreated = "RunCreated" + dgdrReasonInvalidSpec = "InvalidSpec" + dgdrReasonWaitingForActiveRun = "WaitingForActiveRun" + dgdrReasonRunNameConflict = "RunNameConflict" + + // dgdrV1beta2MaxRunNameLength is the longest object name Kubernetes accepts. + dgdrV1beta2MaxRunNameLength = 253 + + // dgdrV1beta2ConflictRetry is how long to wait before looking again at a Run name held by another owner. + dgdrV1beta2ConflictRetry = 5 * time.Second +) + +// DynamoGraphDeploymentRequestV1beta2Reconciler turns an accepted native v1beta2 Request into an +// immutable Run. It never reads the cluster for hardware: admission has already supplied it, and a +// Request that still lacks it is reported instead of searched for. The Run controller executes the Run. +type DynamoGraphDeploymentRequestV1beta2Reconciler struct { + client.Client + Config *configv1alpha1.OperatorConfiguration + RuntimeConfig *commonController.RuntimeConfig +} + +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests,verbs=get;list;watch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch;create;delete + +// Reconcile is level based: once no older Run is active it creates the Run of the latest generation +// only, so generations edited in between are coalesced and never get a Run of their own. It records +// the active and retained Runs in the Request status before it deletes any retired Run. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( + ctx context.Context, + req ctrl.Request, +) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := r.Get(ctx, req.NamespacedName, request); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + // A native v1beta1 Request is a read-only carrier here; the v1beta1 reconciler owns it. + if request.DeletionTimestamp != nil || request.Spec.V1Beta1 != nil { + return ctrl.Result{}, nil + } + original := request.DeepCopy() + result := ctrl.Result{} + + runs, err := r.ownedRuns(ctx, request) + if err != nil { + return ctrl.Result{}, err + } + + wantName := dgdrV1beta2RunName(request) + entries, err := dgdrV1beta2RunEntries(runs) + if err != nil { + return ctrl.Result{}, err + } + // The generation annotation identifies the Run of the current generation, whatever its name. + var current *nvidiacomv1beta2.DynamoGraphDeploymentRun + for i := range entries { + if entries[i].generation == request.Generation { + current = entries[i].run.DeepCopy() + } + } + active, _ := dgdrV1beta2SplitRuns(entries) + + accepted := current != nil + runName := wantName + if current != nil { + runName = current.Name + } + condition := dgdrV1beta2Condition(request, metav1.ConditionTrue, dgdrReasonRunCreated, + fmt.Sprintf("Run %q exists for generation %d", runName, request.Generation)) + if current == nil { + if active != nil { + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonWaitingForActiveRun, + fmt.Sprintf("Run %q is still active; the Run for generation %d starts when it finishes", + active.Name, request.Generation)) + } else if run, buildErr := r.buildRun(request); buildErr != nil { + logger.Info("Request cannot be turned into a Run", "reason", buildErr.Error()) + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, buildErr.Error()) + } else { + err := r.Create(ctx, run) + switch { + case err == nil: + runs = append(runs, *run) + accepted = true + case apierrors.IsAlreadyExists(err): + // A stale cache, or a Run left by an earlier Request of the same name that is still being + // garbage collected. Only a Run this Request controls counts as ours. + existing := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if getErr := r.Get(ctx, client.ObjectKeyFromObject(run), existing); getErr != nil { + return ctrl.Result{}, fmt.Errorf("get existing Run %q: %w", wantName, getErr) + } + if metav1.IsControlledBy(existing, request) && dgdrV1beta2RunIsGeneration(existing, request.Generation) { + runs = append(runs, *existing) + accepted = true + } else { + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonRunNameConflict, + fmt.Sprintf("Run %q exists and is not the Run of generation %d of this Request; trying again once it is gone", wantName, request.Generation)) + result.RequeueAfter = dgdrV1beta2ConflictRetry + } + default: + return ctrl.Result{}, fmt.Errorf("create Run %q: %w", wantName, err) + } + } + } + + entries, err = dgdrV1beta2RunEntries(runs) + if err != nil { + return ctrl.Result{}, err + } + active, terminal := dgdrV1beta2SplitRuns(entries) + keep := min(len(terminal), dgdrV1beta2RetainedRuns) + retired := terminal[keep:] + + if accepted { + request.Status.ObservedGeneration = request.Generation + } + request.Status.ActiveRunRef = nil + if active != nil { + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: active.Name} + } + request.Status.RecentRunRefs = nil + for i := 0; i < keep; i++ { + request.Status.RecentRunRefs = append(request.Status.RecentRunRefs, corev1.LocalObjectReference{Name: terminal[i].Name}) + } + meta.SetStatusCondition(&request.Status.Conditions, condition) + + if !apiequality.Semantic.DeepEqual(original.Status, request.Status) { + if err := r.Status().Patch(ctx, request, client.MergeFrom(original)); err != nil { + return ctrl.Result{}, fmt.Errorf("update Request status: %w", err) + } + } + + // The status no longer references these Runs, so they may go. If a deletion fails, an unreferenced + // child stays for the next pass; the status never points at a Run that was already deleted. + for i := range retired { + if err := r.Delete(ctx, &retired[i]); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete retired Run %q: %w", retired[i].Name, err) + } + } + return result, nil +} + +// buildRun freezes the accepted Request input into an immutable Run spec. The Run reconciler lowers +// this snapshot once, records the effective public recommendation input in status.aisimulate.input, +// and passes that same document to the Job. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + frozen := request.Spec.NativeSearchInput.DeepCopy() + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: dgdrV1beta2RunName(request), + Namespace: request.Namespace, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: strconv.FormatInt(request.Generation, 10)}, + }, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: *frozen}, + } + if err := controllerutil.SetControllerReference(request, run, r.Scheme()); err != nil { + return nil, fmt.Errorf("set Run owner: %w", err) + } + return run, nil +} + +// ownedRuns returns the Runs this Request controls. The controller OwnerReference is the only +// source of ownership: the cache index selects candidates and IsControlledBy is the exact check. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) ownedRuns( + ctx context.Context, + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) ([]nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { + list := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + if err := r.List(ctx, list, client.InNamespace(request.Namespace), + client.MatchingFields{dgdrRunControllerUIDIndex: string(request.UID)}); err != nil { + return nil, fmt.Errorf("list Runs: %w", err) + } + owned := make([]nvidiacomv1beta2.DynamoGraphDeploymentRun, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], request) { + owned = append(owned, list.Items[i]) + } + } + return owned, nil +} + +// SetupWithManager registers the controller. It watches the Runs it owns, so a finished Run wakes +// the Request that was waiting for it. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &nvidiacomv1beta2.DynamoGraphDeploymentRun{}, + dgdrRunControllerUIDIndex, + dgdrRunControllerUIDs, + ); err != nil { + return fmt.Errorf("index Runs by controller UID: %w", err) + } + return ctrl.NewControllerManagedBy(mgr). + Named("dynamographdeploymentrequest-v1beta2"). + For(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}). + Owns(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + WithEventFilter(commonController.EphemeralDeploymentEventFilter(r.Config, r.RuntimeConfig)). + Complete(r) +} + +// dgdrRunControllerUIDs is the index function behind dgdrRunControllerUIDIndex. +func dgdrRunControllerUIDs(obj client.Object) []string { + owner := metav1.GetControllerOf(obj) + if owner == nil { + return nil + } + return []string{string(owner.UID)} +} + +// dgdrV1beta2RunName names the Run of one Request generation. A metadata-only update keeps the +// generation, so it never creates a second Run. +func dgdrV1beta2RunName(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) string { + return dgdrV1beta2RunNameFor(request.Name, request.Generation) +} + +// dgdrV1beta2RunNameFor is - whenever that fits in an object name. A longer +// name keeps the generation suffix and a stable hash of the full Request name, and truncates only the +// Request-name part, so every valid Request name has a valid Run name and two long names that share a +// prefix still get different Run names. +func dgdrV1beta2RunNameFor(requestName string, generation int64) string { + suffix := "-" + strconv.FormatInt(generation, 10) + if len(requestName)+len(suffix) <= dgdrV1beta2MaxRunNameLength { + return requestName + suffix + } + sum := sha256.Sum256([]byte(requestName)) + hash := hex.EncodeToString(sum[:6]) + maxPrefix := dgdrV1beta2MaxRunNameLength - len(suffix) - 1 - len(hash) + prefix := strings.TrimRight(requestName[:maxPrefix], "-.") + return prefix + "-" + hash + suffix +} + +func dgdrV1beta2Condition( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + status metav1.ConditionStatus, + reason, message string, +) metav1.Condition { + return metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRequestConditionAccepted, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: request.Generation, + } +} + +// dgdrV1beta2IsTerminal reports whether the Run has published its Completed condition. The conditions +// are authoritative; the phase only describes how a completed Run ended. +func dgdrV1beta2IsTerminal(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { + return meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) +} + +// dgdrV1beta2RunGeneration reads the Request generation a Run was created for from its generation +// annotation. The annotation, not metadata.name, identifies a Run within its Request: the name is only +// the convention used when a Run is created, so a later change of the naming scheme never orphans a +// Run that already exists. A missing or invalid annotation is an error, never generation zero. +func dgdrV1beta2RunGeneration(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) (int64, error) { + value, ok := run.Annotations[dgdrV1beta2GenerationAnnotation] + if !ok { + return 0, fmt.Errorf("run %q has no Request generation annotation", run.Name) + } + generation, err := strconv.ParseInt(value, 10, 64) + if err != nil || generation <= 0 { + return 0, fmt.Errorf("run %q has invalid Request generation %q", run.Name, value) + } + return generation, nil +} + +// dgdrV1beta2RunIsGeneration reports whether the Run's annotation names the given generation. +func dgdrV1beta2RunIsGeneration(run *nvidiacomv1beta2.DynamoGraphDeploymentRun, generation int64) bool { + got, err := dgdrV1beta2RunGeneration(run) + return err == nil && got == generation +} + +type dgdrV1beta2RunEntry struct { + run nvidiacomv1beta2.DynamoGraphDeploymentRun + generation int64 +} + +// dgdrV1beta2Newer orders Runs newest first: by Request generation, then creation time, then name. +func dgdrV1beta2Newer(a, b *dgdrV1beta2RunEntry) bool { + if a.generation != b.generation { + return a.generation > b.generation + } + if !a.run.CreationTimestamp.Equal(&b.run.CreationTimestamp) { + return b.run.CreationTimestamp.Before(&a.run.CreationTimestamp) + } + return a.run.Name > b.run.Name +} + +// dgdrV1beta2RunEntries pairs every owned Run with the Request generation its annotation names. +func dgdrV1beta2RunEntries(runs []nvidiacomv1beta2.DynamoGraphDeploymentRun) ([]dgdrV1beta2RunEntry, error) { + entries := make([]dgdrV1beta2RunEntry, 0, len(runs)) + for i := range runs { + generation, err := dgdrV1beta2RunGeneration(&runs[i]) + if err != nil { + return nil, err + } + entries = append(entries, dgdrV1beta2RunEntry{run: runs[i], generation: generation}) + } + return entries, nil +} + +// dgdrV1beta2SplitRuns returns the newest non-terminal Run, if any, and the terminal Runs newest first. +func dgdrV1beta2SplitRuns( + entries []dgdrV1beta2RunEntry, +) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, []nvidiacomv1beta2.DynamoGraphDeploymentRun) { + sorted := append([]dgdrV1beta2RunEntry(nil), entries...) + sort.SliceStable(sorted, func(i, j int) bool { return dgdrV1beta2Newer(&sorted[i], &sorted[j]) }) + var active *nvidiacomv1beta2.DynamoGraphDeploymentRun + var terminal []nvidiacomv1beta2.DynamoGraphDeploymentRun + for i := range sorted { + switch { + case !dgdrV1beta2IsTerminal(&sorted[i].run): + if active == nil { + active = &sorted[i].run + } + default: + terminal = append(terminal, sorted[i].run) + } + } + return active, terminal +} + +// requireDGDRV1beta2API fails when the cluster does not serve the v1beta2 Request and Run APIs. +// Without them the controller's caches never sync and the manager would hang at startup. +func requireDGDRV1beta2API(mgr ctrl.Manager) error { + group, version := nvidiacomv1beta2.GroupVersion.Group, nvidiacomv1beta2.GroupVersion.Version + for _, kind := range []string{"DynamoGraphDeploymentRequest", "DynamoGraphDeploymentRun"} { + _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: group, Kind: kind}, version) + if meta.IsNoMatchError(err) { + return fmt.Errorf("--dgdr-v1beta2-enabled is set but the cluster does not serve %s/%s %s; "+ + "install the v1beta2 CRDs (dgdrV1Beta2.enabled)", group, version, kind) + } + if err != nil { + return fmt.Errorf("check that %s/%s %s is served: %w", group, version, kind, err) + } + } + return nil +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go new file mode 100644 index 000000000000..e5cc1a163519 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -0,0 +1,153 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "testing" + "time" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// This test runs the controller under a real manager, so only the watch events (a Request edit, a Run +// status update, a deleted Run) can make it reconcile. It fails if SetupWithManager loses the Owns +// watch, an event predicate drops Run status updates, or the owner mapping breaks. It also covers the +// API server and admission side: the Request is created model-only and admission supplies the image +// and the hardware. +func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { + g := NewWithT(t) + env := sharedEnv.ForTest(t) + c := env.Client() + ctx := context.Background() + + env.StartManager(func(mgr ctrl.Manager) error { + return (&DynamoGraphDeploymentRequestV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: env.OperatorConfig(), + RuntimeConfig: env.RuntimeConfig(), + }).SetupWithManager(mgr) + }) + + const timeout = 30 * time.Second + const poll = 200 * time.Millisecond + + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "watched", Namespace: env.Namespace()}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-32B"}, + }, + }, + } + key := client.ObjectKeyFromObject(request) + getRequest := func(g Gomega) *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + live := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + g.Expect(c.Get(ctx, key, live)).To(Succeed()) + return live + } + accepted := func(live *nvidiacomv1beta2.DynamoGraphDeploymentRequest) *metav1.Condition { + return meta.FindStatusCondition(live.Status.Conditions, nvidiacomv1beta2.DGDRRequestConditionAccepted) + } + getRun := func(g Gomega, name string) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + g.Expect(c.Get(ctx, types.NamespacedName{Namespace: env.Namespace(), Name: name}, run)).To(Succeed()) + return run + } + + t.Log("Creating the Request produces its first Run without a direct Reconcile call") + g.Expect(c.Create(ctx, request)).To(Succeed()) + var firstName string + g.Eventually(func(g Gomega) { + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(accepted(live)).NotTo(BeNil()) + g.Expect(accepted(live).Status).To(Equal(metav1.ConditionTrue)) + g.Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) + firstName = live.Status.ActiveRunRef.Name + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + first := getRun(g, firstName) + g.Expect(metav1.IsControlledBy(first, getRequest(g))).To(BeTrue()) + g.Expect(first.Spec.Hardware).NotTo(BeNil()) + g.Expect(first.Spec.AISimulate.Image).To(Equal(controllerAISimulateImage)) + g.Expect(first.Spec.AISimulate.VersionOverride).To(Equal("1.2.0")) + input, err := frozenInputForTest(first) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(input).To(HaveKey("search_space")) + + t.Log("Editing the Request while its Run is active is reported and starts nothing") + g.Eventually(func(g Gomega) { + live := getRequest(g) + live.Spec.Model.Name = "Qwen/Qwen3-30B-A3B" + g.Expect(c.Update(ctx, live)).To(Succeed()) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + g.Eventually(func(g Gomega) { + live := getRequest(g) + condition := accepted(live) + g.Expect(condition).NotTo(BeNil()) + g.Expect(condition.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(condition.Reason).To(Equal(dgdrReasonWaitingForActiveRun)) + g.Expect(condition.ObservedGeneration).To(Equal(live.Generation)) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + t.Log("Only the Run status update wakes the Request: Completed releases the next generation") + g.Eventually(func(g Gomega) { + run := getRun(g, firstName) + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase("Succeeded") + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionTrue, + Reason: "Finished", + }) + g.Expect(c.Status().Update(ctx, run)).To(Succeed()) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + var secondName string + var secondUID types.UID + g.Eventually(func(g Gomega) { + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(live.Status.ActiveRunRef.Name).NotTo(Equal(firstName)) + g.Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) + g.Expect(accepted(live).Status).To(Equal(metav1.ConditionTrue)) + g.Expect(live.Status.RecentRunRefs).To(HaveLen(1)) + g.Expect(live.Status.RecentRunRefs[0].Name).To(Equal(firstName)) + second := getRun(g, live.Status.ActiveRunRef.Name) + g.Expect(second.Spec.Model.Name).To(Equal("Qwen/Qwen3-30B-A3B")) + secondName, secondUID = second.Name, second.UID + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + + t.Log("Deleting the active Run wakes the Request through the owner mapping and it is recreated") + g.Expect(c.Delete(ctx, getRun(g, secondName))).To(Succeed()) + g.Eventually(func(g Gomega) { + recreated := getRun(g, secondName) + g.Expect(recreated.UID).NotTo(Equal(secondUID)) + live := getRequest(g) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(live.Status.ActiveRunRef.Name).To(Equal(secondName)) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go new file mode 100644 index 000000000000..bd65285fa766 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -0,0 +1,624 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +type dgdrV1beta2Fixture struct { + t *testing.T + client client.Client + reconciler *DynamoGraphDeploymentRequestV1beta2Reconciler +} + +func newDGDRV1beta2Fixture(t *testing.T, objects ...client.Object) *dgdrV1beta2Fixture { + t.Helper() + return newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{}, objects...) +} + +func newDGDRV1beta2FixtureWithInterceptor( + t *testing.T, + funcs interceptor.Funcs, + objects ...client.Object, +) *dgdrV1beta2Fixture { + t.Helper() + scheme := runtime.NewScheme() + if err := nvidiacomv1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add v1beta2 to scheme: %v", err) + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithIndex(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, dgdrRunControllerUIDIndex, dgdrRunControllerUIDs). + WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + WithInterceptorFuncs(funcs). + Build() + return &dgdrV1beta2Fixture{t: t, client: c, reconciler: &DynamoGraphDeploymentRequestV1beta2Reconciler{Client: c}} +} + +func (f *dgdrV1beta2Fixture) reconcile() { + f.t.Helper() + if err := f.tryReconcile(); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +func (f *dgdrV1beta2Fixture) tryReconcile() error { + f.t.Helper() + _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + }) + return err +} + +func (f *dgdrV1beta2Fixture) request() *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + f.t.Helper() + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: "search"}, request); err != nil { + f.t.Fatalf("get request: %v", err) + } + return request +} + +func (f *dgdrV1beta2Fixture) runNames() []string { + f.t.Helper() + list := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + if err := f.client.List(f.t.Context(), list, client.InNamespace("default")); err != nil { + f.t.Fatalf("list runs: %v", err) + } + names := make([]string, 0, len(list.Items)) + for _, run := range list.Items { + names = append(names, run.Name) + } + return names +} + +func (f *dgdrV1beta2Fixture) updateRunStatus(name string, mutate func(*nvidiacomv1beta2.DynamoGraphDeploymentRunStatus)) { + f.t.Helper() + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: name}, run); err != nil { + f.t.Fatalf("get run %s: %v", name, err) + } + mutate(&run.Status) + if err := f.client.Status().Update(f.t.Context(), run); err != nil { + f.t.Fatalf("update status of run %s: %v", name, err) + } +} + +// finish ends a Run the way the Run controller does: it publishes Completed and the outcome phase. +func (f *dgdrV1beta2Fixture) finish(name, phase string) { + f.t.Helper() + f.updateRunStatus(name, func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) + setCompletedForTest(status) + }) +} + +// setPhaseOnly writes a terminal phase without the Completed condition. +func (f *dgdrV1beta2Fixture) setPhaseOnly(name, phase string) { + f.t.Helper() + f.updateRunStatus(name, func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) + }) +} + +// setCompletedOnly publishes Completed without a terminal phase. +func (f *dgdrV1beta2Fixture) setCompletedOnly(name string) { + f.t.Helper() + f.updateRunStatus(name, setCompletedForTest) +} + +func setCompletedForTest(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionTrue, + Reason: "Finished", + }) +} + +func (f *dgdrV1beta2Fixture) bumpGeneration(generation int64) { + f.t.Helper() + request := f.request() + request.Generation = generation + if err := f.client.Update(f.t.Context(), request); err != nil { + f.t.Fatalf("bump generation: %v", err) + } +} + +func dgdrV1beta2Request(withHardware bool) *nvidiacomv1beta2.DynamoGraphDeploymentRequest { + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "search", Namespace: "default", UID: "request-uid", Generation: 1}, + } + request.Spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"} + if withHardware { + sku, budget := "h100_sxm", int32(4) + request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + } + } + return request +} + +func dgdrV1beta2AcceptedCondition(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) metav1.Condition { + t.Helper() + condition := meta.FindStatusCondition(request.Status.Conditions, nvidiacomv1beta2.DGDRRequestConditionAccepted) + if condition == nil { + t.Fatal("no Accepted condition") + } + return *condition +} + +func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + + t.Log("One Run exists, owned by the Request, with the accepted input snapshot") + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: "search-1"}, run); err != nil { + t.Fatalf("get run: %v", err) + } + if !metav1.IsControlledBy(run, f.request()) { + t.Fatal("run is not controlled by the request") + } + if !apiequality.Semantic.DeepEqual(run.Spec.NativeSearchInput, f.request().Spec.NativeSearchInput) { + t.Fatalf("run input = %#v, want the accepted Request snapshot %#v", run.Spec.NativeSearchInput, f.request().Spec.NativeSearchInput) + } + + t.Log("The status records the accepted generation and the active Run") + request := f.request() + if request.Status.ObservedGeneration != 1 { + t.Fatalf("observedGeneration = %d, want 1", request.Status.ObservedGeneration) + } + if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-1" { + t.Fatalf("activeRunRef = %v, want search-1", request.Status.ActiveRunRef) + } + if condition := dgdrV1beta2AcceptedCondition(t, request); condition.Status != metav1.ConditionTrue { + t.Fatalf("Accepted = %v, want True", condition) + } +} + +func TestV1beta2RequestControllerIsIdempotent(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.reconcile() + f.reconcile() + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want only search-1", names) + } +} + +func TestV1beta2RequestControllerWaitsForTheActiveRunBeforeStartingTheNext(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + + t.Log("A spec change while search-1 runs creates nothing and says why") + f.bumpGeneration(2) + f.reconcile() + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want only the active run", names) + } + request := f.request() + if request.Status.ObservedGeneration != 1 { + t.Fatalf("observedGeneration = %d, want 1 until generation 2 is accepted", request.Status.ObservedGeneration) + } + condition := dgdrV1beta2AcceptedCondition(t, request) + if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonWaitingForActiveRun { + t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonWaitingForActiveRun) + } + + t.Log("Once search-1 is completed, search-2 is created and search-1 moves to the history") + f.finish("search-1", "Succeeded") + f.reconcile() + request = f.request() + if request.Status.ObservedGeneration != 2 { + t.Fatalf("observedGeneration = %d, want 2", request.Status.ObservedGeneration) + } + if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-2" { + t.Fatalf("activeRunRef = %v, want search-2", request.Status.ActiveRunRef) + } + if len(request.Status.RecentRunRefs) != 1 || request.Status.RecentRunRefs[0].Name != "search-1" { + t.Fatalf("recentRunRefs = %v, want [search-1]", request.Status.RecentRunRefs) + } +} + +func TestV1beta2RequestControllerCoalescesIntermediateGenerations(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() // search-1 is active + + f.bumpGeneration(2) + f.reconcile() + f.bumpGeneration(3) + f.reconcile() + + // Completing search-1 admits the current level, generation 3. + f.finish("search-1", "Succeeded") + f.reconcile() + + // Generation 2 was an intermediate desired state and is never materialized. + names := f.runNames() + if len(names) != 2 { + t.Fatalf("runs = %v, want search-1 and search-3", names) + } + for _, name := range names { + if name != "search-1" && name != "search-3" { + t.Fatalf("runs = %v, want search-1 and search-3 only", names) + } + } +} + +func TestV1beta2RequestControllerIgnoresAPhaseWithoutTheCompletedCondition(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.bumpGeneration(2) + + f.setPhaseOnly("search-1", "Succeeded") + f.reconcile() + + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want the next Run held back until Completed is published", names) + } + if condition := dgdrV1beta2AcceptedCondition(t, f.request()); condition.Reason != dgdrReasonWaitingForActiveRun { + t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonWaitingForActiveRun) + } +} + +func TestV1beta2RequestControllerReleasesOnCompletedWithoutATerminalPhase(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) + f.reconcile() + f.bumpGeneration(2) + + f.setCompletedOnly("search-1") // the phase is still empty + f.reconcile() + + if names := f.runNames(); len(names) != 2 { + t.Fatalf("runs = %v, want search-2 started once Completed is True", names) + } +} + +func TestV1beta2RequestControllerCountsAnUnlabeledOwnedRunAsActive(t *testing.T) { + request := dgdrV1beta2Request(true) + active := dgdrV1beta2OwnedRun(request, 1, false) // correct controller OwnerReference, no labels + f := newDGDRV1beta2Fixture(t, request, active) + f.bumpGeneration(2) + + f.reconcile() + + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want no second active Run next to search-1", names) + } + if ref := f.request().Status.ActiveRunRef; ref == nil || ref.Name != "search-1" { + t.Fatalf("activeRunRef = %v, want search-1", ref) + } +} + +func TestV1beta2RequestControllerCreatesNoRunWithoutHardware(t *testing.T) { + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(false)) + f.reconcile() + if names := f.runNames(); len(names) != 0 { + t.Fatalf("runs = %v, want none", names) + } + request := f.request() + condition := dgdrV1beta2AcceptedCondition(t, request) + if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonInvalidSpec { + t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonInvalidSpec) + } + if request.Status.ObservedGeneration != 0 { + t.Fatalf("observedGeneration = %d, want 0 for a rejected generation", request.Status.ObservedGeneration) + } +} + +func TestV1beta2RequestControllerLeavesForeignCarriersAlone(t *testing.T) { + request := dgdrV1beta2Request(true) + carrier := request.DeepCopy() + carrier.Spec.V1Beta1 = &apiextensionsJSONForTest + f := newDGDRV1beta2Fixture(t, carrier) + f.reconcile() + if names := f.runNames(); len(names) != 0 { + t.Fatalf("runs = %v, want none for a v1beta1 carrier", names) + } + if got := f.request().Status.ObservedGeneration; got != 0 { + t.Fatalf("observedGeneration = %d, want the carrier's status untouched", got) + } +} + +func dgdrV1beta2RunFixtures(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, count int) []client.Object { + objects := []client.Object{request} + for generation := 1; generation <= count; generation++ { + objects = append(objects, dgdrV1beta2OwnedRun(request, generation, true)) + } + return objects +} + +func TestV1beta2RequestControllerRetainsFiveTerminalRuns(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + f := newDGDRV1beta2Fixture(t, dgdrV1beta2RunFixtures(request, 7)...) + f.reconcile() + + t.Log("The two oldest terminal runs are deleted and the newest five are listed newest first") + assertRetained(t, f) +} + +func assertRetained(t *testing.T, f *dgdrV1beta2Fixture) { + t.Helper() + for _, name := range []string{"search-1", "search-2"} { + err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: name}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("run %s: err = %v, want it deleted", name, err) + } + } + refs := f.request().Status.RecentRunRefs + want := []string{"search-7", "search-6", "search-5", "search-4", "search-3"} + if len(refs) != len(want) { + t.Fatalf("recentRunRefs = %v, want %v", refs, want) + } + for i := range want { + if refs[i].Name != want[i] { + t.Fatalf("recentRunRefs = %v, want %v", refs, want) + } + } +} + +func TestV1beta2RequestControllerOrdersRetainedRunsByTheirGenerationAnnotation(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + objects := []client.Object{request} + for generation := 1; generation <= 7; generation++ { + run := dgdrV1beta2OwnedRun(request, generation, true) + run.Name = fmt.Sprintf("arbitrary-%02d", 100-generation) // sorts opposite to the generation + objects = append(objects, run) + } + f := newDGDRV1beta2Fixture(t, objects...) + f.reconcile() + + refs := f.request().Status.RecentRunRefs + want := []string{"arbitrary-93", "arbitrary-94", "arbitrary-95", "arbitrary-96", "arbitrary-97"} + if len(refs) != len(want) { + t.Fatalf("recentRunRefs = %v, want %v", refs, want) + } + for i := range want { + if refs[i].Name != want[i] { + t.Fatalf("recentRunRefs = %v, want %v", refs, want) + } + } + for _, name := range f.runNames() { + if name == "arbitrary-98" || name == "arbitrary-99" { + t.Fatalf("run %q of generation %s was not retired", name, map[string]string{"arbitrary-98": "2", "arbitrary-99": "1"}[name]) + } + } +} + +func TestV1beta2RequestControllerRejectsAControlledRunWithoutAGenerationAnnotation(t *testing.T) { + request := dgdrV1beta2Request(true) + odd := dgdrV1beta2OwnedRun(request, 1, true) + odd.Name = "unrelated" + delete(odd.Annotations, dgdrV1beta2GenerationAnnotation) + f := newDGDRV1beta2Fixture(t, request, odd) + + if err := f.tryReconcile(); err == nil || !strings.Contains(err.Error(), "unrelated") { + t.Fatalf("err = %v, want an error naming the run", err) + } + if names := f.runNames(); len(names) != 1 { + t.Fatalf("runs = %v, want the odd run left alone and none created", names) + } +} + +func TestV1beta2RequestControllerReusesAnOwnedRunWhateverItsName(t *testing.T) { + request := dgdrV1beta2Request(true) + existing := dgdrV1beta2OwnedRun(request, int(request.Generation), false) + existing.Name = "an-arbitrary-valid-name" + f := newDGDRV1beta2Fixture(t, request, existing) + + f.reconcile() + + if names := f.runNames(); len(names) != 1 || names[0] != existing.Name { + t.Fatalf("runs = %v, want only %q: an owned Run of the current generation is reused", names, existing.Name) + } + live := f.request() + if live.Status.ActiveRunRef == nil || live.Status.ActiveRunRef.Name != existing.Name || + live.Status.ObservedGeneration != request.Generation { + t.Fatalf("status = %+v, want %q active for generation %d", live.Status, existing.Name, request.Generation) + } + if condition := dgdrV1beta2AcceptedCondition(t, live); condition.Status != metav1.ConditionTrue { + t.Fatalf("Accepted = %v, want True", condition) + } +} + +func TestV1beta2RequestControllerPublishesStatusBeforeDeletingRetiredRuns(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Generation = 8 + failPatch := true + f := newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{ + SubResourcePatch: func( + ctx context.Context, c client.Client, subResource string, obj client.Object, + patch client.Patch, opts ...client.SubResourcePatchOption, + ) error { + if failPatch { + return errors.New("injected status patch failure") + } + return c.SubResource(subResource).Patch(ctx, obj, patch, opts...) + }, + }, dgdrV1beta2RunFixtures(request, 7)...) + + if err := f.tryReconcile(); err == nil { + t.Fatal("reconcile succeeded although the status patch failed") + } + // search-8 was created; the two oldest terminal Runs must still exist because the status did not + // move on. + if names := f.runNames(); len(names) != 8 { + t.Fatalf("runs = %v, want all eight: nothing may be deleted before the status is published", names) + } + + failPatch = false + f.reconcile() + assertRetained(t, f) +} + +func dgdrV1beta2OwnedRun( + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + generation int, + terminal bool, +) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", request.Name, generation), + Namespace: request.Namespace, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: fmt.Sprint(generation)}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRequest", + Name: request.Name, + UID: request.UID, + Controller: boolPointerForTest(true), + }}, + }, + } + if terminal { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase("Succeeded") + setCompletedForTest(&run.Status) + } + return run +} + +func boolPointerForTest(value bool) *bool { return &value } + +// apiextensionsJSONForTest is a non-empty v1beta1 carrier payload. +var apiextensionsJSONForTest = apiextensionsv1.JSON{Raw: []byte(`{"model":"carrier"}`)} + +func TestV1beta2RequestControllerKeepsTheReadableRunNameWhenItFits(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Name = strings.Repeat("a", 250) + request.Generation = 3 + if got, want := dgdrV1beta2RunName(request), request.Name+"-3"; got != want { + t.Fatalf("run name = %q, want %q", got, want) + } +} + +func TestV1beta2RequestControllerDerivesDistinctValidNamesForOverlengthRequests(t *testing.T) { + a := dgdrV1beta2Request(true) + a.Name = strings.Repeat("a", 252) + b := dgdrV1beta2Request(true) + b.Name = strings.Repeat("a", 251) + "b" // the same retained prefix as a + nameA, nameB := dgdrV1beta2RunName(a), dgdrV1beta2RunName(b) + for _, name := range []string{nameA, nameB} { + if len(name) > dgdrV1beta2MaxRunNameLength || !strings.HasSuffix(name, "-1") { + t.Fatalf("run name %q (%d characters) is not a valid name with the generation suffix", name, len(name)) + } + } + if nameA == nameB { + t.Fatalf("two different Request names produced the same Run name %q", nameA) + } + +} + +func TestV1beta2RequestControllerReconcilesAnOverlengthRequestToOneRun(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Name = strings.Repeat("a", 252) + f := newDGDRV1beta2Fixture(t, request) + key := types.NamespacedName{Namespace: request.Namespace, Name: request.Name} + for range 3 { + if _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: key}); err != nil { + t.Fatalf("reconcile: %v", err) + } + } + want := dgdrV1beta2RunName(request) + if names := f.runNames(); len(names) != 1 || names[0] != want { + t.Fatalf("runs = %v, want only %q", names, want) + } + live := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := f.client.Get(t.Context(), key, live); err != nil { + t.Fatalf("get request: %v", err) + } + if live.Status.ActiveRunRef == nil || live.Status.ActiveRunRef.Name != want || live.Status.ObservedGeneration != 1 { + t.Fatalf("status = %+v, want the overlength Run active for generation 1", live.Status) + } +} + +func TestV1beta2RequestControllerDoesNotAdoptARunOfAnotherOwner(t *testing.T) { + stale := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "search-1", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRequest", + Name: "search", + UID: "old-uid", + Controller: boolPointerForTest(true), + }}, + }, + } + f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true), stale) + + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + }) + if err != nil { + t.Fatalf("reconcile: %v", err) + } + if result.RequeueAfter <= 0 { + t.Fatal("no retry was scheduled for the Run name held by another owner") + } + request := f.request() + if condition := dgdrV1beta2AcceptedCondition(t, request); condition.Reason != dgdrReasonRunNameConflict { + t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonRunNameConflict) + } + if request.Status.ActiveRunRef != nil { + t.Fatalf("activeRunRef = %v, want none: the old Run is not ours", request.Status.ActiveRunRef) + } + if request.Status.ObservedGeneration != 0 { + t.Fatalf("observedGeneration = %d, want 0", request.Status.ObservedGeneration) + } +} + +func TestV1beta2RequestControllerPreservesInputOverridesInTheRunSnapshot(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{InputOverrides: &runtime.RawExtension{ + Raw: []byte(`{"traffic":{"source":{"input_tokens":null}},"optimizer":{"max_trials":null}}`), + }} + f := newDGDRV1beta2Fixture(t, request) + + run, err := f.reconciler.buildRun(request) + if err != nil { + t.Fatalf("build run: %v", err) + } + if run.Spec.AISimulate == nil || run.Spec.AISimulate.InputOverrides == nil { + t.Fatal("Run did not preserve spec.aisimulate.inputOverrides") + } + if got, want := string(run.Spec.AISimulate.InputOverrides.Raw), string(request.Spec.AISimulate.InputOverrides.Raw); got != want { + t.Fatalf("run inputOverrides = %s, want %s", got, want) + } +} diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index 327a89b72df2..f90319683a71 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -129,6 +129,22 @@ func SetupDynamoGraphDeploymentRequest(mgr ctrl.Manager, opts DynamoGraphDeploym return nil } +// SetupDynamoGraphDeploymentRequestV1beta2 registers the controller that turns native v1beta2 Requests +// into Runs. It fails at startup, with a clear message, when the cluster does not serve the v1beta2 APIs. +func SetupDynamoGraphDeploymentRequestV1beta2(mgr ctrl.Manager, opts SetupOptions) error { + if err := requireDGDRV1beta2API(mgr); err != nil { + return err + } + if err := (&DynamoGraphDeploymentRequestV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: opts.Config, + RuntimeConfig: opts.RuntimeConfig, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create DynamoGraphDeploymentRequest v1beta2 controller: %w", err) + } + return nil +} + func SetupDynamoModel(mgr ctrl.Manager, opts DynamoModelSetupOptions) error { if err := (&DynamoModelReconciler{ Client: mgr.GetClient(), diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 24654c484f97..065eeba5bb3c 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3549,6 +3549,11 @@ DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR Run creators resolve optional Request inputs and the final AISimulate parameters before persisting the Run. Request admission defaulting does not apply to Runs. +On a Run, spec.aisimulate.parameters is the complete resolved native AISimulate input, frozen when +the Request controller created the Run. The Run executes that object as it is. It is not lowered +from the typed fields again: they are copies of the Request, and lowering them would bring back +keys that a JSON merge patch null had deleted from the stored input. + _Appears in:_ diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 6d449b70c09f..0650a197f4bc 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4993,6 +4993,11 @@ DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR Run creators resolve optional Request inputs and the final AISimulate parameters before persisting the Run. Request admission defaulting does not apply to Runs. +On a Run, spec.aisimulate.parameters is the complete resolved native AISimulate input, frozen when +the Request controller created the Run. The Run executes that object as it is. It is not lowered +from the typed fields again: they are copies of the Request, and lowering them would bring back +keys that a JSON merge patch null had deleted from the stored input. + **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRun](#dynamographdeploymentrun) From 9a0c7766edcb9ac03bad4dbbb6ac79ff140ce51b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 11:11:13 +0100 Subject: [PATCH 040/100] pr-17097: satisfy current Go lint Signed-off-by: Dr. Stefan Schimanski --- ...aphdeploymentrequest_v1beta2_controller.go | 12 ++--- ...request_v1beta2_controller_envtest_test.go | 2 +- ...ploymentrequest_v1beta2_controller_test.go | 50 ++++++++----------- 3 files changed, 28 insertions(+), 36 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index fa3e9fe4521e..02b8dfa62b0f 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -177,7 +177,7 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: active.Name} } request.Status.RecentRunRefs = nil - for i := 0; i < keep; i++ { + for i := range keep { request.Status.RecentRunRefs = append(request.Status.RecentRunRefs, corev1.LocalObjectReference{Name: terminal[i].Name}) } meta.SetStatusCondition(&request.Status.Conditions, condition) @@ -206,12 +206,10 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) buildRun( ) (*nvidiacomv1beta2.DynamoGraphDeploymentRun, error) { frozen := request.Spec.NativeSearchInput.DeepCopy() run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: dgdrV1beta2RunName(request), - Namespace: request.Namespace, - Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: strconv.FormatInt(request.Generation, 10)}, - }, - Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: *frozen}, + Name: dgdrV1beta2RunName(request), + Namespace: request.Namespace, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: strconv.FormatInt(request.Generation, 10)}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: *frozen}, } if err := controllerutil.SetControllerReference(request, run, r.Scheme()); err != nil { return nil, fmt.Errorf("set Run owner: %w", err) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go index e5cc1a163519..e75b7bd1e390 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -56,7 +56,7 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { const poll = 200 * time.Millisecond request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ - ObjectMeta: metav1.ObjectMeta{Name: "watched", Namespace: env.Namespace()}, + Name: "watched", Namespace: env.Namespace(), Spec: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-32B"}, diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index bd65285fa766..51ff097ff34d 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -79,7 +79,7 @@ func (f *dgdrV1beta2Fixture) reconcile() { func (f *dgdrV1beta2Fixture) tryReconcile() error { f.t.Helper() _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{ - NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + Namespace: "default", Name: "search", }) return err } @@ -160,7 +160,7 @@ func (f *dgdrV1beta2Fixture) bumpGeneration(generation int64) { func dgdrV1beta2Request(withHardware bool) *nvidiacomv1beta2.DynamoGraphDeploymentRequest { request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ - ObjectMeta: metav1.ObjectMeta{Name: "search", Namespace: "default", UID: "request-uid", Generation: 1}, + Name: "search", Namespace: "default", UID: "request-uid", Generation: 1, } request.Spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"} if withHardware { @@ -493,18 +493,16 @@ func dgdrV1beta2OwnedRun( terminal bool, ) *nvidiacomv1beta2.DynamoGraphDeploymentRun { run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%d", request.Name, generation), - Namespace: request.Namespace, - Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: fmt.Sprint(generation)}, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: nvidiacomv1beta2.GroupVersion.String(), - Kind: "DynamoGraphDeploymentRequest", - Name: request.Name, - UID: request.UID, - Controller: boolPointerForTest(true), - }}, - }, + Name: fmt.Sprintf("%s-%d", request.Name, generation), + Namespace: request.Namespace, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: fmt.Sprint(generation)}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRequest", + Name: request.Name, + UID: request.UID, + Controller: new(true), + }}, } if terminal { run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase("Succeeded") @@ -513,8 +511,6 @@ func dgdrV1beta2OwnedRun( return run } -func boolPointerForTest(value bool) *bool { return &value } - // apiextensionsJSONForTest is a non-empty v1beta1 carrier payload. var apiextensionsJSONForTest = apiextensionsv1.JSON{Raw: []byte(`{"model":"carrier"}`)} @@ -569,22 +565,20 @@ func TestV1beta2RequestControllerReconcilesAnOverlengthRequestToOneRun(t *testin func TestV1beta2RequestControllerDoesNotAdoptARunOfAnotherOwner(t *testing.T) { stale := &nvidiacomv1beta2.DynamoGraphDeploymentRun{ - ObjectMeta: metav1.ObjectMeta{ - Name: "search-1", - Namespace: "default", - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: nvidiacomv1beta2.GroupVersion.String(), - Kind: "DynamoGraphDeploymentRequest", - Name: "search", - UID: "old-uid", - Controller: boolPointerForTest(true), - }}, - }, + Name: "search-1", + Namespace: "default", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), + Kind: "DynamoGraphDeploymentRequest", + Name: "search", + UID: "old-uid", + Controller: new(true), + }}, } f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true), stale) result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{ - NamespacedName: types.NamespacedName{Namespace: "default", Name: "search"}, + Namespace: "default", Name: "search", }) if err != nil { t.Fatalf("reconcile: %v", err) From e9aac605befe776f75252787f9949177f590a6e0 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 15:01:39 +0100 Subject: [PATCH 041/100] pr-17097: pr-8: remove duplicate Request controller tests Signed-off-by: Dr. Stefan Schimanski --- ...ploymentrequest_v1beta2_controller_test.go | 40 ++----------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index 51ff097ff34d..e322cdd09636 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -127,20 +127,6 @@ func (f *dgdrV1beta2Fixture) finish(name, phase string) { }) } -// setPhaseOnly writes a terminal phase without the Completed condition. -func (f *dgdrV1beta2Fixture) setPhaseOnly(name, phase string) { - f.t.Helper() - f.updateRunStatus(name, func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { - status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) - }) -} - -// setCompletedOnly publishes Completed without a terminal phase. -func (f *dgdrV1beta2Fixture) setCompletedOnly(name string) { - f.t.Helper() - f.updateRunStatus(name, setCompletedForTest) -} - func setCompletedForTest(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: nvidiacomv1beta2.DGDRRunConditionCompleted, @@ -210,16 +196,6 @@ func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing. } } -func TestV1beta2RequestControllerIsIdempotent(t *testing.T) { - f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) - f.reconcile() - f.reconcile() - f.reconcile() - if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { - t.Fatalf("runs = %v, want only search-1", names) - } -} - func TestV1beta2RequestControllerWaitsForTheActiveRunBeforeStartingTheNext(t *testing.T) { f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) f.reconcile() @@ -284,7 +260,9 @@ func TestV1beta2RequestControllerIgnoresAPhaseWithoutTheCompletedCondition(t *te f.reconcile() f.bumpGeneration(2) - f.setPhaseOnly("search-1", "Succeeded") + f.updateRunStatus("search-1", func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + status.Phase = nvidiacomv1beta2.DGDRRunPhaseSucceeded + }) f.reconcile() if names := f.runNames(); len(names) != 1 { @@ -300,7 +278,7 @@ func TestV1beta2RequestControllerReleasesOnCompletedWithoutATerminalPhase(t *tes f.reconcile() f.bumpGeneration(2) - f.setCompletedOnly("search-1") // the phase is still empty + f.updateRunStatus("search-1", setCompletedForTest) // the phase is still empty f.reconcile() if names := f.runNames(); len(names) != 2 { @@ -362,16 +340,6 @@ func dgdrV1beta2RunFixtures(request *nvidiacomv1beta2.DynamoGraphDeploymentReque return objects } -func TestV1beta2RequestControllerRetainsFiveTerminalRuns(t *testing.T) { - request := dgdrV1beta2Request(true) - request.Generation = 8 - f := newDGDRV1beta2Fixture(t, dgdrV1beta2RunFixtures(request, 7)...) - f.reconcile() - - t.Log("The two oldest terminal runs are deleted and the newest five are listed newest first") - assertRetained(t, f) -} - func assertRetained(t *testing.T, f *dgdrV1beta2Fixture) { t.Helper() for _, name := range []string{"search-1", "search-2"} { From 3a5cc8566a626ebf8dca407b1873aa976be963cb Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 17:44:01 +0100 Subject: [PATCH 042/100] pr-17097: pr-8: cancel superseded Request runs [upstream: devivasudevan/dynamo#8] Signed-off-by: Dr. Stefan Schimanski --- ...aphdeploymentrequest_v1beta2_controller.go | 157 +++++++++++++++--- ...request_v1beta2_controller_envtest_test.go | 32 ++-- ...ploymentrequest_v1beta2_controller_test.go | 157 +++++++++++++----- deploy/operator/internal/controller/setup.go | 1 + 4 files changed, 269 insertions(+), 78 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index 02b8dfa62b0f..dbf356ad0025 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -21,6 +21,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "sort" "strconv" @@ -55,10 +56,11 @@ const ( // dgdrV1beta2RetainedRuns is how many terminal Runs a Request keeps. Older ones are deleted. dgdrV1beta2RetainedRuns = 5 - dgdrReasonRunCreated = "RunCreated" - dgdrReasonInvalidSpec = "InvalidSpec" - dgdrReasonWaitingForActiveRun = "WaitingForActiveRun" - dgdrReasonRunNameConflict = "RunNameConflict" + dgdrReasonRunCreated = "RunCreated" + dgdrReasonRunCreating = "RunCreating" + dgdrReasonCancelingStaleRun = "CancelingSupersededRun" + dgdrReasonInvalidSpec = "InvalidSpec" + dgdrReasonRunNameConflict = "RunNameConflict" // dgdrV1beta2MaxRunNameLength is the longest object name Kubernetes accepts. dgdrV1beta2MaxRunNameLength = 253 @@ -72,6 +74,7 @@ const ( // Request that still lacks it is reported instead of searched for. The Run controller executes the Run. type DynamoGraphDeploymentRequestV1beta2Reconciler struct { client.Client + APIReader client.Reader Config *configv1alpha1.OperatorConfiguration RuntimeConfig *commonController.RuntimeConfig } @@ -128,35 +131,52 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( fmt.Sprintf("Run %q exists for generation %d", runName, request.Generation)) if current == nil { if active != nil { - condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonWaitingForActiveRun, - fmt.Sprintf("Run %q is still active; the Run for generation %d starts when it finishes", - active.Name, request.Generation)) + if active.DeletionTimestamp.IsZero() { + if err := r.Delete(ctx, active, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("cancel superseded Run %q: %w", active.Name, err) + } + } + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonCancelingStaleRun, + fmt.Sprintf("Canceling superseded Run %q before generation %d starts", active.Name, request.Generation)) + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: active.Name} + meta.SetStatusCondition(&request.Status.Conditions, condition) + if err := r.patchStatus(ctx, request, original); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil } else if run, buildErr := r.buildRun(request); buildErr != nil { logger.Info("Request cannot be turned into a Run", "reason", buildErr.Error()) condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, buildErr.Error()) } else { - err := r.Create(ctx, run) + marker := meta.FindStatusCondition(request.Status.Conditions, nvidiacomv1beta2.DGDRRequestConditionAccepted) + markerMatches := request.Status.ActiveRunRef != nil && request.Status.ActiveRunRef.Name == wantName && + marker != nil && marker.ObservedGeneration == request.Generation + switch { - case err == nil: - runs = append(runs, *run) - accepted = true - case apierrors.IsAlreadyExists(err): - // A stale cache, or a Run left by an earlier Request of the same name that is still being - // garbage collected. Only a Run this Request controls counts as ours. - existing := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} - if getErr := r.Get(ctx, client.ObjectKeyFromObject(run), existing); getErr != nil { - return ctrl.Result{}, fmt.Errorf("get existing Run %q: %w", wantName, getErr) + case markerMatches && (marker.Reason == dgdrReasonRunCreating || marker.Reason == dgdrReasonRunCreated): + return r.createReservedRun(ctx, request, run, original) + case request.Status.ActiveRunRef != nil && marker != nil && marker.Reason == dgdrReasonRunCreating: + // The Request informer has observed a newer spec before the Run informer observed + // the reserved create. Resolve its unknown outcome directly: an owned Run waits for + // the cache before cancellation, while authoritative absence releases the stale + // reservation. This is the only uncached read in the Request transaction. + pending, err := r.reservedRunPending(ctx, request, marker.ObservedGeneration) + if err != nil { + return ctrl.Result{}, err } - if metav1.IsControlledBy(existing, request) && dgdrV1beta2RunIsGeneration(existing, request.Generation) { - runs = append(runs, *existing) - accepted = true - } else { - condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonRunNameConflict, - fmt.Sprintf("Run %q exists and is not the Run of generation %d of this Request; trying again once it is gone", wantName, request.Generation)) - result.RequeueAfter = dgdrV1beta2ConflictRetry + if pending { + return ctrl.Result{RequeueAfter: dgdrV1beta2ConflictRetry}, nil } + fallthrough default: - return ctrl.Result{}, fmt.Errorf("create Run %q: %w", wantName, err) + condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonRunCreating, + fmt.Sprintf("Creating Run %q for generation %d", wantName, request.Generation)) + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: wantName} + meta.SetStatusCondition(&request.Status.Conditions, condition) + if err := r.patchStatus(ctx, request, original); err != nil { + return ctrl.Result{}, err + } + return r.createReservedRun(ctx, request, run, request.DeepCopy()) } } } @@ -183,8 +203,8 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( meta.SetStatusCondition(&request.Status.Conditions, condition) if !apiequality.Semantic.DeepEqual(original.Status, request.Status) { - if err := r.Status().Patch(ctx, request, client.MergeFrom(original)); err != nil { - return ctrl.Result{}, fmt.Errorf("update Request status: %w", err) + if err := r.patchStatus(ctx, request, original); err != nil { + return ctrl.Result{}, err } } @@ -198,6 +218,86 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( return result, nil } +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) patchStatus( + ctx context.Context, + request, original *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) error { + patch := client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{}) + if err := r.Status().Patch(ctx, request, patch); err != nil { + return fmt.Errorf("update Request status: %w", err) + } + return nil +} + +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) reservedRunPending( + ctx context.Context, + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + generation int64, +) (bool, error) { + if r.APIReader == nil { + return false, errors.New("resolve reserved Run create: APIReader is not configured") + } + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + key := client.ObjectKey{Namespace: request.Namespace, Name: request.Status.ActiveRunRef.Name} + if err := r.APIReader.Get(ctx, key, run); apierrors.IsNotFound(err) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("resolve reserved Run %q: %w", key.Name, err) + } + return metav1.IsControlledBy(run, request) && dgdrV1beta2RunIsGeneration(run, generation), nil +} + +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) createReservedRun( + ctx context.Context, + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + original *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) (ctrl.Result, error) { + err := r.Create(ctx, run) + if err != nil && !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, fmt.Errorf("create Run %q: %w", run.Name, err) + } + + // A successful create reaches the API server before its watch event necessarily reaches this + // controller. The fixed Run name makes retry safe; only publish RunCreated once the cached Get + // proves that this informer has observed the owned Run. + existing := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := r.Get(ctx, client.ObjectKeyFromObject(run), existing); apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: dgdrV1beta2ConflictRetry}, nil + } else if err != nil { + return ctrl.Result{}, fmt.Errorf("get existing Run %q: %w", run.Name, err) + } + if metav1.IsControlledBy(existing, request) && dgdrV1beta2RunIsGeneration(existing, request.Generation) { + return r.markRunCreated(ctx, request, existing.Name, original) + } + + condition := dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonRunNameConflict, + fmt.Sprintf("Run %q exists and is not the Run of generation %d of this Request; trying again once it is gone", run.Name, request.Generation)) + request.Status.ActiveRunRef = nil + meta.SetStatusCondition(&request.Status.Conditions, condition) + if err := r.patchStatus(ctx, request, original); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: dgdrV1beta2ConflictRetry}, nil +} + +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) markRunCreated( + ctx context.Context, + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + runName string, + original *nvidiacomv1beta2.DynamoGraphDeploymentRequest, +) (ctrl.Result, error) { + request.Status.ObservedGeneration = request.Generation + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: runName} + condition := dgdrV1beta2Condition(request, metav1.ConditionTrue, dgdrReasonRunCreated, + fmt.Sprintf("Run %q exists for generation %d", runName, request.Generation)) + meta.SetStatusCondition(&request.Status.Conditions, condition) + if err := r.patchStatus(ctx, request, original); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, nil +} + // buildRun freezes the accepted Request input into an immutable Run spec. The Run reconciler lowers // this snapshot once, records the effective public recommendation input in status.aisimulate.input, // and passes that same document to the Job. @@ -240,6 +340,9 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) ownedRuns( // SetupWithManager registers the controller. It watches the Runs it owns, so a finished Run wakes // the Request that was waiting for it. func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } if err := mgr.GetFieldIndexer().IndexField( context.Background(), &nvidiacomv1beta2.DynamoGraphDeploymentRun{}, diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go index e75b7bd1e390..bf776b185b55 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -26,6 +26,7 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -47,6 +48,7 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { env.StartManager(func(mgr ctrl.Manager) error { return (&DynamoGraphDeploymentRequestV1beta2Reconciler{ Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), Config: env.OperatorConfig(), RuntimeConfig: env.RuntimeConfig(), }).SetupWithManager(mgr) @@ -99,33 +101,36 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(input).To(HaveKey("search_space")) - t.Log("Editing the Request while its Run is active is reported and starts nothing") + t.Log("Editing the Request foreground-cancels the active Run") g.Eventually(func(g Gomega) { live := getRequest(g) live.Spec.Model.Name = "Qwen/Qwen3-30B-A3B" g.Expect(c.Update(ctx, live)).To(Succeed()) }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) g.Eventually(func(g Gomega) { + terminating := getRun(g, firstName) + g.Expect(terminating.DeletionTimestamp).NotTo(BeNil()) live := getRequest(g) condition := accepted(live) g.Expect(condition).NotTo(BeNil()) g.Expect(condition.Status).To(Equal(metav1.ConditionFalse)) - g.Expect(condition.Reason).To(Equal(dgdrReasonWaitingForActiveRun)) - g.Expect(condition.ObservedGeneration).To(Equal(live.Generation)) + g.Expect(condition.Reason).To(Equal(dgdrReasonCancelingStaleRun)) + g.Expect(live.Status.ActiveRunRef).NotTo(BeNil()) + g.Expect(live.Status.ActiveRunRef.Name).To(Equal(firstName)) }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) - t.Log("Only the Run status update wakes the Request: Completed releases the next generation") + t.Log("Complete foreground deletion; envtest has no garbage-collector controller") + g.Eventually(func(g Gomega) { + terminating := getRun(g, firstName) + terminating.Finalizers = nil + g.Expect(c.Update(ctx, terminating)).To(Succeed()) + }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) g.Eventually(func(g Gomega) { - run := getRun(g, firstName) - run.Status.Phase = nvidiacomv1beta2.DGDRRunPhase("Succeeded") - meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ - Type: nvidiacomv1beta2.DGDRRunConditionCompleted, - Status: metav1.ConditionTrue, - Reason: "Finished", - }) - g.Expect(c.Status().Update(ctx, run)).To(Succeed()) + err := c.Get(ctx, types.NamespacedName{Namespace: env.Namespace(), Name: firstName}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), "superseded Run lookup returned %v", err) }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + t.Log("The completed deletion event wakes the Request and starts only the newest generation") var secondName string var secondUID types.UID g.Eventually(func(g Gomega) { @@ -134,8 +139,7 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { g.Expect(live.Status.ActiveRunRef.Name).NotTo(Equal(firstName)) g.Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) g.Expect(accepted(live).Status).To(Equal(metav1.ConditionTrue)) - g.Expect(live.Status.RecentRunRefs).To(HaveLen(1)) - g.Expect(live.Status.RecentRunRefs[0].Name).To(Equal(firstName)) + g.Expect(live.Status.RecentRunRefs).To(BeEmpty()) second := getRun(g, live.Status.ActiveRunRef.Name) g.Expect(second.Spec.Model.Name).To(Equal("Qwen/Qwen3-30B-A3B")) secondName, secondUID = second.Name, second.UID diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index e322cdd09636..8e9f6663cec3 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -25,6 +25,7 @@ import ( "testing" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -66,7 +67,9 @@ func newDGDRV1beta2FixtureWithInterceptor( WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRequest{}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}). WithInterceptorFuncs(funcs). Build() - return &dgdrV1beta2Fixture{t: t, client: c, reconciler: &DynamoGraphDeploymentRequestV1beta2Reconciler{Client: c}} + return &dgdrV1beta2Fixture{t: t, client: c, reconciler: &DynamoGraphDeploymentRequestV1beta2Reconciler{ + Client: c, APIReader: c, + }} } func (f *dgdrV1beta2Fixture) reconcile() { @@ -196,27 +199,26 @@ func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing. } } -func TestV1beta2RequestControllerWaitsForTheActiveRunBeforeStartingTheNext(t *testing.T) { +func TestV1beta2RequestControllerCancelsTheActiveRunBeforeStartingTheNext(t *testing.T) { f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) f.reconcile() - t.Log("A spec change while search-1 runs creates nothing and says why") + t.Log("A spec change foreground-cancels search-1 before creating its replacement") f.bumpGeneration(2) f.reconcile() - if names := f.runNames(); len(names) != 1 { - t.Fatalf("runs = %v, want only the active run", names) + if names := f.runNames(); len(names) != 0 { + t.Fatalf("runs = %v, want the superseded Run deleted before its replacement", names) } request := f.request() if request.Status.ObservedGeneration != 1 { t.Fatalf("observedGeneration = %d, want 1 until generation 2 is accepted", request.Status.ObservedGeneration) } condition := dgdrV1beta2AcceptedCondition(t, request) - if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonWaitingForActiveRun { - t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonWaitingForActiveRun) + if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonCancelingStaleRun { + t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonCancelingStaleRun) } - t.Log("Once search-1 is completed, search-2 is created and search-1 moves to the history") - f.finish("search-1", "Succeeded") + t.Log("The next pass observes deletion and creates only the current generation") f.reconcile() request = f.request() if request.Status.ObservedGeneration != 2 { @@ -225,8 +227,96 @@ func TestV1beta2RequestControllerWaitsForTheActiveRunBeforeStartingTheNext(t *te if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-2" { t.Fatalf("activeRunRef = %v, want search-2", request.Status.ActiveRunRef) } - if len(request.Status.RecentRunRefs) != 1 || request.Status.RecentRunRefs[0].Name != "search-1" { - t.Fatalf("recentRunRefs = %v, want [search-1]", request.Status.RecentRunRefs) + if len(request.Status.RecentRunRefs) != 0 { + t.Fatalf("recentRunRefs = %v, want no canceled Run in terminal history", request.Status.RecentRunRefs) + } +} + +func TestV1beta2RequestControllerDoesNotStartANewGenerationWhileTheRunCacheLags(t *testing.T) { + request := dgdrV1beta2Request(true) + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: "search-1"} + meta.SetStatusCondition(&request.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRequestConditionAccepted, + Status: metav1.ConditionFalse, + ObservedGeneration: 1, + Reason: dgdrReasonRunCreating, + Message: "Creating Run search-1 for generation 1", + }) + hideRuns := true + f := newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, isRunList := list.(*nvidiacomv1beta2.DynamoGraphDeploymentRunList); hideRuns && isRunList { + return nil + } + return c.List(ctx, list, opts...) + }, + }, request, dgdrV1beta2OwnedRun(request, 1, false)) + + t.Log("Observe generation 2 before the Run cache has observed generation 1's reserved create") + f.bumpGeneration(2) + + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "default", + Name: "search", + }}) + if err != nil { + t.Fatalf("reconcile with lagging Run cache: %v", err) + } + if result.RequeueAfter <= 0 { + t.Fatal("lagging Run cache did not schedule a bounded retry") + } + + t.Log("The durable RunCreating reservation prevents generation 2 from being created beside generation 1") + hideRuns = false + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want only search-1 while its cache observation lags", names) + } + live := f.request() + if live.Status.ActiveRunRef == nil || live.Status.ActiveRunRef.Name != "search-1" { + t.Fatalf("activeRunRef = %v, want the durable search-1 reservation", live.Status.ActiveRunRef) + } + if live.Status.ObservedGeneration != 0 { + t.Fatalf("observedGeneration = %d, want 0 until the reserved Run is observed", live.Status.ObservedGeneration) + } + + t.Log("Once the cache catches up, the old Run is canceled before generation 2 starts") + f.reconcile() + f.reconcile() + if names := f.runNames(); len(names) != 1 || names[0] != "search-2" { + t.Fatalf("runs = %v, want only search-2", names) + } +} + +func TestV1beta2RequestControllerReleasesAReservationAfterAFailedCreate(t *testing.T) { + failCreate := true + f := newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, object client.Object, opts ...client.CreateOption) error { + if _, isRun := object.(*nvidiacomv1beta2.DynamoGraphDeploymentRun); isRun && failCreate { + failCreate = false + return errors.New("unknown create outcome") + } + return c.Create(ctx, object, opts...) + }, + }, dgdrV1beta2Request(true)) + + t.Log("Persist generation 1's reservation before its Run create fails") + if err := f.tryReconcile(); err == nil || !strings.Contains(err.Error(), "unknown create outcome") { + t.Fatalf("reconcile error = %v, want failed Run create", err) + } + reserved := f.request() + if reserved.Status.ActiveRunRef == nil || reserved.Status.ActiveRunRef.Name != "search-1" { + t.Fatalf("activeRunRef = %v, want durable search-1 reservation", reserved.Status.ActiveRunRef) + } + + t.Log("A newer generation replaces the stale reservation after the API confirms no Run exists") + f.bumpGeneration(2) + f.reconcile() + if names := f.runNames(); len(names) != 1 || names[0] != "search-2" { + t.Fatalf("runs = %v, want only search-2", names) + } + accepted := dgdrV1beta2AcceptedCondition(t, f.request()) + if accepted.Status != metav1.ConditionTrue || accepted.ObservedGeneration != 2 { + t.Fatalf("condition = %v, want generation 2 accepted", accepted) } } @@ -239,37 +329,28 @@ func TestV1beta2RequestControllerCoalescesIntermediateGenerations(t *testing.T) f.bumpGeneration(3) f.reconcile() - // Completing search-1 admits the current level, generation 3. - f.finish("search-1", "Succeeded") - f.reconcile() - - // Generation 2 was an intermediate desired state and is never materialized. + // Generation 2 was an intermediate desired state and is never materialized; generation 1 + // was canceled instead of retained as a completed search. names := f.runNames() - if len(names) != 2 { - t.Fatalf("runs = %v, want search-1 and search-3", names) - } - for _, name := range names { - if name != "search-1" && name != "search-3" { - t.Fatalf("runs = %v, want search-1 and search-3 only", names) - } + if len(names) != 1 || names[0] != "search-3" { + t.Fatalf("runs = %v, want only search-3", names) } } -func TestV1beta2RequestControllerIgnoresAPhaseWithoutTheCompletedCondition(t *testing.T) { +func TestV1beta2RequestControllerAllowsTheCurrentRunToRemainActiveIndefinitely(t *testing.T) { f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) f.reconcile() - f.bumpGeneration(2) f.updateRunStatus("search-1", func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { status.Phase = nvidiacomv1beta2.DGDRRunPhaseSucceeded }) f.reconcile() - if names := f.runNames(); len(names) != 1 { - t.Fatalf("runs = %v, want the next Run held back until Completed is published", names) + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want the current Run left alone without a new generation", names) } - if condition := dgdrV1beta2AcceptedCondition(t, f.request()); condition.Reason != dgdrReasonWaitingForActiveRun { - t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonWaitingForActiveRun) + if condition := dgdrV1beta2AcceptedCondition(t, f.request()); condition.Reason != dgdrReasonRunCreated { + t.Fatalf("condition = %v, want reason %s", condition, dgdrReasonRunCreated) } } @@ -286,7 +367,7 @@ func TestV1beta2RequestControllerReleasesOnCompletedWithoutATerminalPhase(t *tes } } -func TestV1beta2RequestControllerCountsAnUnlabeledOwnedRunAsActive(t *testing.T) { +func TestV1beta2RequestControllerCancelsAnUnlabeledOwnedRun(t *testing.T) { request := dgdrV1beta2Request(true) active := dgdrV1beta2OwnedRun(request, 1, false) // correct controller OwnerReference, no labels f := newDGDRV1beta2Fixture(t, request, active) @@ -294,11 +375,11 @@ func TestV1beta2RequestControllerCountsAnUnlabeledOwnedRunAsActive(t *testing.T) f.reconcile() - if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { - t.Fatalf("runs = %v, want no second active Run next to search-1", names) + if names := f.runNames(); len(names) != 0 { + t.Fatalf("runs = %v, want the superseded owned Run canceled", names) } if ref := f.request().Status.ActiveRunRef; ref == nil || ref.Name != "search-1" { - t.Fatalf("activeRunRef = %v, want search-1", ref) + t.Fatalf("activeRunRef = %v, want search-1 retained until cancellation is observed", ref) } } @@ -371,6 +452,7 @@ func TestV1beta2RequestControllerOrdersRetainedRunsByTheirGenerationAnnotation(t } f := newDGDRV1beta2Fixture(t, objects...) f.reconcile() + f.reconcile() refs := f.request().Status.RecentRunRefs want := []string{"arbitrary-93", "arbitrary-94", "arbitrary-95", "arbitrary-96", "arbitrary-97"} @@ -444,14 +526,15 @@ func TestV1beta2RequestControllerPublishesStatusBeforeDeletingRetiredRuns(t *tes if err := f.tryReconcile(); err == nil { t.Fatal("reconcile succeeded although the status patch failed") } - // search-8 was created; the two oldest terminal Runs must still exist because the status did not - // move on. - if names := f.runNames(); len(names) != 8 { - t.Fatalf("runs = %v, want all eight: nothing may be deleted before the status is published", names) + // The durable reservation is published before search-8 is created. A failed status write must + // leave every existing Run untouched and must not perform the create side effect. + if names := f.runNames(); len(names) != 7 { + t.Fatalf("runs = %v, want the seven existing Runs and no unreserved search-8", names) } failPatch = false f.reconcile() + f.reconcile() assertRetained(t, f) } diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index f90319683a71..ed71c6aa4a9f 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -137,6 +137,7 @@ func SetupDynamoGraphDeploymentRequestV1beta2(mgr ctrl.Manager, opts SetupOption } if err := (&DynamoGraphDeploymentRequestV1beta2Reconciler{ Client: mgr.GetClient(), + APIReader: mgr.GetAPIReader(), Config: opts.Config, RuntimeConfig: opts.RuntimeConfig, }).SetupWithManager(mgr); err != nil { From 455ec685b8e0917878f060ab9b1cf5c9e30e287b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 19:02:54 +0100 Subject: [PATCH 043/100] pr-17097: pr-8: retain superseded Request runs [upstream: devivasudevan/dynamo#8] Signed-off-by: Dr. Stefan Schimanski --- .../components/operator/files/role.yaml | 1 + ...aphdeploymentrequest_v1beta2_controller.go | 36 +++++- ...request_v1beta2_controller_envtest_test.go | 35 +++--- ...ploymentrequest_v1beta2_controller_test.go | 104 +++++++++++++++--- 4 files changed, 141 insertions(+), 35 deletions(-) diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index f2391bcaf6de..b1c343d5d57d 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -320,6 +320,7 @@ rules: - delete - get - list + - patch - watch - apiGroups: - nvidia.com diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index dbf356ad0025..2dba0a46b767 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -49,6 +49,11 @@ const ( // created, so the controller never reconstructs or validates it for an existing Run. dgdrV1beta2GenerationAnnotation = "nvidia.com/dgdr-request-generation" + // dgdrV1beta2CancelRequestedByGenerationAnnotation records the newer Request generation that + // superseded a non-terminal Run. The Request controller owns this intent; the Run controller + // stops the execution and publishes the terminal outcome without deleting the Run. + dgdrV1beta2CancelRequestedByGenerationAnnotation = "nvidia.com/dgdr-cancel-requested-by-generation" + // dgdrRunControllerUIDIndex indexes Runs by the UID of their controller OwnerReference. The // OwnerReference is the only definition of which Runs belong to a Request. dgdrRunControllerUIDIndex = "dynamo.nvidia.com/dgdr-run-controller-uid" @@ -81,7 +86,7 @@ type DynamoGraphDeploymentRequestV1beta2Reconciler struct { // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests,verbs=get;list;watch // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentrequests/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch;create;delete +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch;create;patch;delete // Reconcile is level based: once no older Run is active it creates the Run of the latest generation // only, so generations edited in between are coalesced and never get a Run of their own. It records @@ -132,12 +137,12 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( if current == nil { if active != nil { if active.DeletionTimestamp.IsZero() { - if err := r.Delete(ctx, active, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !apierrors.IsNotFound(err) { - return ctrl.Result{}, fmt.Errorf("cancel superseded Run %q: %w", active.Name, err) + if err := r.requestRunCancellation(ctx, active, request.Generation); err != nil { + return ctrl.Result{}, err } } condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonCancelingStaleRun, - fmt.Sprintf("Canceling superseded Run %q before generation %d starts", active.Name, request.Generation)) + fmt.Sprintf("Waiting for superseded Run %q to stop before generation %d starts", active.Name, request.Generation)) request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: active.Name} meta.SetStatusCondition(&request.Status.Conditions, condition) if err := r.patchStatus(ctx, request, original); err != nil { @@ -218,6 +223,29 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( return result, nil } +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) requestRunCancellation( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + supersedingGeneration int64, +) error { + if existing := run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]; existing != "" { + generation, err := strconv.ParseInt(existing, 10, 64) + if err == nil && generation >= supersedingGeneration { + return nil + } + } + want := strconv.FormatInt(supersedingGeneration, 10) + original := run.DeepCopy() + if run.Annotations == nil { + run.Annotations = map[string]string{} + } + run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] = want + if err := r.Patch(ctx, run, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("request cancellation of superseded Run %q: %w", run.Name, err) + } + return nil +} + func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) patchStatus( ctx context.Context, request, original *nvidiacomv1beta2.DynamoGraphDeploymentRequest, diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go index bf776b185b55..b1cb8175ec5e 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -21,12 +21,13 @@ package controller import ( "context" + "strconv" "testing" "time" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" . "github.com/onsi/gomega" - apierrors "k8s.io/apimachinery/pkg/api/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -35,7 +36,7 @@ import ( ) // This test runs the controller under a real manager, so only the watch events (a Request edit, a Run -// status update, a deleted Run) can make it reconcile. It fails if SetupWithManager loses the Owns +// annotation/status update, a deleted Run) can make it reconcile. It fails if SetupWithManager loses the Owns // watch, an event predicate drops Run status updates, or the owner mapping breaks. It also covers the // API server and admission side: the Request is created model-only and admission supplies the image // and the hardware. @@ -101,16 +102,18 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(input).To(HaveKey("search_space")) - t.Log("Editing the Request foreground-cancels the active Run") + t.Log("Editing the Request causally requests cancellation without deleting the active Run") g.Eventually(func(g Gomega) { live := getRequest(g) live.Spec.Model.Name = "Qwen/Qwen3-30B-A3B" g.Expect(c.Update(ctx, live)).To(Succeed()) }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) g.Eventually(func(g Gomega) { - terminating := getRun(g, firstName) - g.Expect(terminating.DeletionTimestamp).NotTo(BeNil()) live := getRequest(g) + terminating := getRun(g, firstName) + g.Expect(terminating.DeletionTimestamp).To(BeNil()) + g.Expect(terminating.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]). + To(Equal(strconv.FormatInt(live.Generation, 10))) condition := accepted(live) g.Expect(condition).NotTo(BeNil()) g.Expect(condition.Status).To(Equal(metav1.ConditionFalse)) @@ -119,18 +122,20 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { g.Expect(live.Status.ActiveRunRef.Name).To(Equal(firstName)) }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) - t.Log("Complete foreground deletion; envtest has no garbage-collector controller") - g.Eventually(func(g Gomega) { - terminating := getRun(g, firstName) - terminating.Finalizers = nil - g.Expect(c.Update(ctx, terminating)).To(Succeed()) - }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) + t.Log("Publish the Run controller's terminal superseded outcome") g.Eventually(func(g Gomega) { - err := c.Get(ctx, types.NamespacedName{Namespace: env.Namespace(), Name: firstName}, &nvidiacomv1beta2.DynamoGraphDeploymentRun{}) - g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), "superseded Run lookup returned %v", err) + terminal := getRun(g, firstName) + terminal.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseFailed + meta.SetStatusCondition(&terminal.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionTrue, + Reason: "Superseded", + ObservedGeneration: terminal.Generation, + }) + g.Expect(c.Status().Update(ctx, terminal)).To(Succeed()) }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) - t.Log("The completed deletion event wakes the Request and starts only the newest generation") + t.Log("The Completed status event wakes the Request and starts only the newest generation") var secondName string var secondUID types.UID g.Eventually(func(g Gomega) { @@ -139,7 +144,7 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { g.Expect(live.Status.ActiveRunRef.Name).NotTo(Equal(firstName)) g.Expect(live.Status.ObservedGeneration).To(Equal(live.Generation)) g.Expect(accepted(live).Status).To(Equal(metav1.ConditionTrue)) - g.Expect(live.Status.RecentRunRefs).To(BeEmpty()) + g.Expect(live.Status.RecentRunRefs).To(ConsistOf(corev1.LocalObjectReference{Name: firstName})) second := getRun(g, live.Status.ActiveRunRef.Name) g.Expect(second.Spec.Model.Name).To(Equal("Qwen/Qwen3-30B-A3B")) secondName, secondUID = second.Name, second.UID diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index 8e9f6663cec3..9d4e57b9dbc2 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -109,6 +109,15 @@ func (f *dgdrV1beta2Fixture) runNames() []string { return names } +func (f *dgdrV1beta2Fixture) run(name string) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + f.t.Helper() + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: name}, run); err != nil { + f.t.Fatalf("get run %s: %v", name, err) + } + return run +} + func (f *dgdrV1beta2Fixture) updateRunStatus(name string, mutate func(*nvidiacomv1beta2.DynamoGraphDeploymentRunStatus)) { f.t.Helper() run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} @@ -203,11 +212,14 @@ func TestV1beta2RequestControllerCancelsTheActiveRunBeforeStartingTheNext(t *tes f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) f.reconcile() - t.Log("A spec change foreground-cancels search-1 before creating its replacement") + t.Log("A spec change requests cancellation without deleting search-1") f.bumpGeneration(2) f.reconcile() - if names := f.runNames(); len(names) != 0 { - t.Fatalf("runs = %v, want the superseded Run deleted before its replacement", names) + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want the superseded Run retained", names) + } + if got := f.run("search-1").Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]; got != "2" { + t.Fatalf("cancellation generation = %q, want 2", got) } request := f.request() if request.Status.ObservedGeneration != 1 { @@ -218,7 +230,15 @@ func TestV1beta2RequestControllerCancelsTheActiveRunBeforeStartingTheNext(t *tes t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonCancelingStaleRun) } - t.Log("The next pass observes deletion and creates only the current generation") + t.Log("Retries leave the retained Run active until its controller publishes Completed") + f.reconcile() + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want no replacement before cancellation completes", names) + } + + t.Log("The terminal cancellation outcome releases the latest generation") + f.updateRunStatus("search-1", setCompletedForTest) + f.reconcile() f.reconcile() request = f.request() if request.Status.ObservedGeneration != 2 { @@ -227,8 +247,8 @@ func TestV1beta2RequestControllerCancelsTheActiveRunBeforeStartingTheNext(t *tes if request.Status.ActiveRunRef == nil || request.Status.ActiveRunRef.Name != "search-2" { t.Fatalf("activeRunRef = %v, want search-2", request.Status.ActiveRunRef) } - if len(request.Status.RecentRunRefs) != 0 { - t.Fatalf("recentRunRefs = %v, want no canceled Run in terminal history", request.Status.RecentRunRefs) + if len(request.Status.RecentRunRefs) != 1 || request.Status.RecentRunRefs[0].Name != "search-1" { + t.Fatalf("recentRunRefs = %v, want the canceled Run retained in terminal history", request.Status.RecentRunRefs) } } @@ -279,11 +299,15 @@ func TestV1beta2RequestControllerDoesNotStartANewGenerationWhileTheRunCacheLags( t.Fatalf("observedGeneration = %d, want 0 until the reserved Run is observed", live.Status.ObservedGeneration) } - t.Log("Once the cache catches up, the old Run is canceled before generation 2 starts") + t.Log("Once the cache catches up, cancellation is requested and the old Run remains retained") f.reconcile() + if got := f.run("search-1").Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]; got != "2" { + t.Fatalf("cancellation generation = %q, want 2", got) + } + f.updateRunStatus("search-1", setCompletedForTest) f.reconcile() - if names := f.runNames(); len(names) != 1 || names[0] != "search-2" { - t.Fatalf("runs = %v, want only search-2", names) + if names := f.runNames(); len(names) != 2 { + t.Fatalf("runs = %v, want retained search-1 and active search-2", names) } } @@ -329,11 +353,56 @@ func TestV1beta2RequestControllerCoalescesIntermediateGenerations(t *testing.T) f.bumpGeneration(3) f.reconcile() - // Generation 2 was an intermediate desired state and is never materialized; generation 1 - // was canceled instead of retained as a completed search. - names := f.runNames() - if len(names) != 1 || names[0] != "search-3" { - t.Fatalf("runs = %v, want only search-3", names) + t.Log("The latest generation replaces the causal cancellation marker without creating generation 2") + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want only the retained active search-1", names) + } + if got := f.run("search-1").Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]; got != "3" { + t.Fatalf("cancellation generation = %q, want the latest generation 3", got) + } + + t.Log("A stale retry cannot move the causal cancellation generation backwards") + if err := f.reconciler.requestRunCancellation(t.Context(), f.run("search-1"), 2); err != nil { + t.Fatalf("retry stale cancellation: %v", err) + } + if got := f.run("search-1").Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]; got != "3" { + t.Fatalf("cancellation generation = %q after stale retry, want 3", got) + } + + t.Log("Once cancellation completes, generation 3 starts and generation 2 remains coalesced") + f.updateRunStatus("search-1", setCompletedForTest) + f.reconcile() + if names := f.runNames(); len(names) != 2 || names[0] != "search-1" || names[1] != "search-3" { + t.Fatalf("runs = %v, want retained search-1 and active search-3", names) + } +} + +func TestV1beta2RequestControllerRepairsInvalidOrStaleCancellationGeneration(t *testing.T) { + tests := []struct { + name string + existing string + }{ + {name: "malformed", existing: "malformed"}, + {name: "zero", existing: "0"}, + {name: "negative", existing: "-1"}, + {name: "stale", existing: "2"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := dgdrV1beta2Request(true) + run := dgdrV1beta2OwnedRun(request, 1, false) + run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] = tt.existing + f := newDGDRV1beta2Fixture(t, request, run) + f.bumpGeneration(3) + + t.Log("Reconcile the newer Request generation") + f.reconcile() + + if got := f.run(run.Name).Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]; got != "3" { + t.Fatalf("cancellation generation = %q, want repaired generation 3", got) + } + }) } } @@ -375,8 +444,11 @@ func TestV1beta2RequestControllerCancelsAnUnlabeledOwnedRun(t *testing.T) { f.reconcile() - if names := f.runNames(); len(names) != 0 { - t.Fatalf("runs = %v, want the superseded owned Run canceled", names) + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want the superseded owned Run retained", names) + } + if got := f.run("search-1").Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]; got != "2" { + t.Fatalf("cancellation generation = %q, want 2", got) } if ref := f.request().Status.ActiveRunRef; ref == nil || ref.Name != "search-1" { t.Fatalf("activeRunRef = %v, want search-1 retained until cancellation is observed", ref) From ac6ca9b78634f1d17239955dd05733100fcfcb9c Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 19:24:22 +0100 Subject: [PATCH 044/100] pr-17097: pr-8: satisfy current operator lint [upstream: devivasudevan/dynamo#8] Signed-off-by: Dr. Stefan Schimanski --- ...aphdeploymentrequest_v1beta2_controller.go | 120 ++++++++++-------- ...ploymentrequest_v1beta2_controller_test.go | 27 ++-- 2 files changed, 77 insertions(+), 70 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index 2dba0a46b767..2d6842edea9e 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -95,8 +95,6 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( ctx context.Context, req ctrl.Request, ) (ctrl.Result, error) { - logger := log.FromContext(ctx) - request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} if err := r.Get(ctx, req.NamespacedName, request); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) @@ -106,7 +104,6 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( return ctrl.Result{}, nil } original := request.DeepCopy() - result := ctrl.Result{} runs, err := r.ownedRuns(ctx, request) if err != nil { @@ -135,55 +132,11 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( condition := dgdrV1beta2Condition(request, metav1.ConditionTrue, dgdrReasonRunCreated, fmt.Sprintf("Run %q exists for generation %d", runName, request.Generation)) if current == nil { - if active != nil { - if active.DeletionTimestamp.IsZero() { - if err := r.requestRunCancellation(ctx, active, request.Generation); err != nil { - return ctrl.Result{}, err - } - } - condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonCancelingStaleRun, - fmt.Sprintf("Waiting for superseded Run %q to stop before generation %d starts", active.Name, request.Generation)) - request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: active.Name} - meta.SetStatusCondition(&request.Status.Conditions, condition) - if err := r.patchStatus(ctx, request, original); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{}, nil - } else if run, buildErr := r.buildRun(request); buildErr != nil { - logger.Info("Request cannot be turned into a Run", "reason", buildErr.Error()) - condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, buildErr.Error()) - } else { - marker := meta.FindStatusCondition(request.Status.Conditions, nvidiacomv1beta2.DGDRRequestConditionAccepted) - markerMatches := request.Status.ActiveRunRef != nil && request.Status.ActiveRunRef.Name == wantName && - marker != nil && marker.ObservedGeneration == request.Generation - - switch { - case markerMatches && (marker.Reason == dgdrReasonRunCreating || marker.Reason == dgdrReasonRunCreated): - return r.createReservedRun(ctx, request, run, original) - case request.Status.ActiveRunRef != nil && marker != nil && marker.Reason == dgdrReasonRunCreating: - // The Request informer has observed a newer spec before the Run informer observed - // the reserved create. Resolve its unknown outcome directly: an owned Run waits for - // the cache before cancellation, while authoritative absence releases the stale - // reservation. This is the only uncached read in the Request transaction. - pending, err := r.reservedRunPending(ctx, request, marker.ObservedGeneration) - if err != nil { - return ctrl.Result{}, err - } - if pending { - return ctrl.Result{RequeueAfter: dgdrV1beta2ConflictRetry}, nil - } - fallthrough - default: - condition = dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonRunCreating, - fmt.Sprintf("Creating Run %q for generation %d", wantName, request.Generation)) - request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: wantName} - meta.SetStatusCondition(&request.Status.Conditions, condition) - if err := r.patchStatus(ctx, request, original); err != nil { - return ctrl.Result{}, err - } - return r.createReservedRun(ctx, request, run, request.DeepCopy()) - } + invalidCondition, result, err := r.reconcileMissingRun(ctx, request, active, original, wantName) + if err != nil || invalidCondition == nil { + return result, err } + condition = *invalidCondition } entries, err = dgdrV1beta2RunEntries(runs) @@ -220,7 +173,70 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) Reconcile( return ctrl.Result{}, fmt.Errorf("delete retired Run %q: %w", retired[i].Name, err) } } - return result, nil + return ctrl.Result{}, nil +} + +// reconcileMissingRun serializes Run creation behind the previous active Run and a durable +// RunCreating reservation. A non-nil condition is the only non-terminal return: buildRun rejected +// the accepted Request snapshot, so the caller still has to publish InvalidSpec and retained history. +func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) reconcileMissingRun( + ctx context.Context, + request *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + active *nvidiacomv1beta2.DynamoGraphDeploymentRun, + original *nvidiacomv1beta2.DynamoGraphDeploymentRequest, + wantName string, +) (*metav1.Condition, ctrl.Result, error) { + if active != nil { + if active.DeletionTimestamp.IsZero() { + if err := r.requestRunCancellation(ctx, active, request.Generation); err != nil { + return nil, ctrl.Result{}, err + } + } + condition := dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonCancelingStaleRun, + fmt.Sprintf("Waiting for superseded Run %q to stop before generation %d starts", active.Name, request.Generation)) + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: active.Name} + meta.SetStatusCondition(&request.Status.Conditions, condition) + return nil, ctrl.Result{}, r.patchStatus(ctx, request, original) + } + + run, err := r.buildRun(request) + if err != nil { + log.FromContext(ctx).Info("Request cannot be turned into a Run", "reason", err.Error()) + condition := dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, err.Error()) + return &condition, ctrl.Result{}, nil + } + + marker := meta.FindStatusCondition(request.Status.Conditions, nvidiacomv1beta2.DGDRRequestConditionAccepted) + markerMatches := request.Status.ActiveRunRef != nil && request.Status.ActiveRunRef.Name == wantName && + marker != nil && marker.ObservedGeneration == request.Generation + if markerMatches && (marker.Reason == dgdrReasonRunCreating || marker.Reason == dgdrReasonRunCreated) { + result, err := r.createReservedRun(ctx, request, run, original) + return nil, result, err + } + + if request.Status.ActiveRunRef != nil && marker != nil && marker.Reason == dgdrReasonRunCreating { + // The Request informer has observed a newer spec before the Run informer observed + // the reserved create. Resolve its unknown outcome directly: an owned Run waits for + // the cache before cancellation, while authoritative absence releases the stale + // reservation. This is the only uncached read in the Request transaction. + pending, err := r.reservedRunPending(ctx, request, marker.ObservedGeneration) + if err != nil { + return nil, ctrl.Result{}, err + } + if pending { + return nil, ctrl.Result{RequeueAfter: dgdrV1beta2ConflictRetry}, nil + } + } + + condition := dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonRunCreating, + fmt.Sprintf("Creating Run %q for generation %d", wantName, request.Generation)) + request.Status.ActiveRunRef = &corev1.LocalObjectReference{Name: wantName} + meta.SetStatusCondition(&request.Status.Conditions, condition) + if err := r.patchStatus(ctx, request, original); err != nil { + return nil, ctrl.Result{}, err + } + result, err := r.createReservedRun(ctx, request, run, request.DeepCopy()) + return nil, result, err } func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) requestRunCancellation( diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index 9d4e57b9dbc2..eafafd44c773 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -118,8 +118,9 @@ func (f *dgdrV1beta2Fixture) run(name string) *nvidiacomv1beta2.DynamoGraphDeplo return run } -func (f *dgdrV1beta2Fixture) updateRunStatus(name string, mutate func(*nvidiacomv1beta2.DynamoGraphDeploymentRunStatus)) { +func (f *dgdrV1beta2Fixture) updateRunStatus(mutate func(*nvidiacomv1beta2.DynamoGraphDeploymentRunStatus)) { f.t.Helper() + const name = "search-1" run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: name}, run); err != nil { f.t.Fatalf("get run %s: %v", name, err) @@ -130,15 +131,6 @@ func (f *dgdrV1beta2Fixture) updateRunStatus(name string, mutate func(*nvidiacom } } -// finish ends a Run the way the Run controller does: it publishes Completed and the outcome phase. -func (f *dgdrV1beta2Fixture) finish(name, phase string) { - f.t.Helper() - f.updateRunStatus(name, func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { - status.Phase = nvidiacomv1beta2.DGDRRunPhase(phase) - setCompletedForTest(status) - }) -} - func setCompletedForTest(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: nvidiacomv1beta2.DGDRRunConditionCompleted, @@ -237,7 +229,7 @@ func TestV1beta2RequestControllerCancelsTheActiveRunBeforeStartingTheNext(t *tes } t.Log("The terminal cancellation outcome releases the latest generation") - f.updateRunStatus("search-1", setCompletedForTest) + f.updateRunStatus(setCompletedForTest) f.reconcile() f.reconcile() request = f.request() @@ -275,10 +267,9 @@ func TestV1beta2RequestControllerDoesNotStartANewGenerationWhileTheRunCacheLags( t.Log("Observe generation 2 before the Run cache has observed generation 1's reserved create") f.bumpGeneration(2) - result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{ Namespace: "default", - Name: "search", - }}) + Name: "search"}) if err != nil { t.Fatalf("reconcile with lagging Run cache: %v", err) } @@ -304,7 +295,7 @@ func TestV1beta2RequestControllerDoesNotStartANewGenerationWhileTheRunCacheLags( if got := f.run("search-1").Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation]; got != "2" { t.Fatalf("cancellation generation = %q, want 2", got) } - f.updateRunStatus("search-1", setCompletedForTest) + f.updateRunStatus(setCompletedForTest) f.reconcile() if names := f.runNames(); len(names) != 2 { t.Fatalf("runs = %v, want retained search-1 and active search-2", names) @@ -370,7 +361,7 @@ func TestV1beta2RequestControllerCoalescesIntermediateGenerations(t *testing.T) } t.Log("Once cancellation completes, generation 3 starts and generation 2 remains coalesced") - f.updateRunStatus("search-1", setCompletedForTest) + f.updateRunStatus(setCompletedForTest) f.reconcile() if names := f.runNames(); len(names) != 2 || names[0] != "search-1" || names[1] != "search-3" { t.Fatalf("runs = %v, want retained search-1 and active search-3", names) @@ -410,7 +401,7 @@ func TestV1beta2RequestControllerAllowsTheCurrentRunToRemainActiveIndefinitely(t f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) f.reconcile() - f.updateRunStatus("search-1", func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { + f.updateRunStatus(func(status *nvidiacomv1beta2.DynamoGraphDeploymentRunStatus) { status.Phase = nvidiacomv1beta2.DGDRRunPhaseSucceeded }) f.reconcile() @@ -428,7 +419,7 @@ func TestV1beta2RequestControllerReleasesOnCompletedWithoutATerminalPhase(t *tes f.reconcile() f.bumpGeneration(2) - f.updateRunStatus("search-1", setCompletedForTest) // the phase is still empty + f.updateRunStatus(setCompletedForTest) // the phase is still empty f.reconcile() if names := f.runNames(); len(names) != 2 { From cee8fce6272712ce14b20128ff03f5575df72d94 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 17:27:57 +0100 Subject: [PATCH 045/100] pr-17407: squash derive sibling images from release tags Squashed from https://github.com/ai-dynamo/dynamo/pull/17407. Signed-off-by: Dr. Stefan Schimanski --- .../src/dynamo/common/container_images.py | 48 ++++++++++++ .../unit/test_planner_image_selection.py | 36 +++++---- .../dynamo/profiler/utils/profile_common.py | 78 +------------------ 3 files changed, 71 insertions(+), 91 deletions(-) create mode 100644 components/src/dynamo/common/container_images.py diff --git a/components/src/dynamo/common/container_images.py b/components/src/dynamo/common/container_images.py new file mode 100644 index 000000000000..4fb519450210 --- /dev/null +++ b/components/src/dynamo/common/container_images.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Published Dynamo container-image naming conventions.""" + +BACKEND_IMAGE_NAMES: dict[str, str] = { + "vllm": "vllm-runtime", + "sglang": "sglang-runtime", + "trtllm": "tensorrtllm-runtime", +} + +PLANNER_IMAGE_NAME = "dynamo-planner" + + +def _replace_image_name(image_ref: str, new_name: str) -> str: + """Replace the image name using the source image's shared release tag. + + OCI digests identify a manifest within one repository. They cannot be + copied from a Dynamo image to a sibling planner or backend repository. + """ + slash_idx = image_ref.rfind("/") + prefix = image_ref[: slash_idx + 1] if slash_idx >= 0 else "" + suffix = image_ref[slash_idx + 1 :] + name_and_tag = suffix.partition("@")[0] + colon_idx = name_and_tag.rfind(":") + if colon_idx < 0 or colon_idx == len(name_and_tag) - 1: + raise ValueError( + f"Cannot derive {new_name!r} from image {image_ref!r}: " + "a shared release tag is required; repository-specific digests " + "cannot be transferred to sibling images" + ) + return f"{prefix}{new_name}{name_and_tag[colon_idx:]}" + + +def derive_planner_image(source_image: str) -> str: + """Derive the planner image that belongs to a published Dynamo image.""" + return _replace_image_name(source_image, PLANNER_IMAGE_NAME) + + +def derive_backend_image(source_image: str, backend: str) -> str: + """Derive one backend runtime image from a published Dynamo image.""" + backend_image_name = BACKEND_IMAGE_NAMES.get(backend) + if backend_image_name is None: + raise ValueError( + f"Cannot derive backend image for unknown backend '{backend}'. " + f"Supported backends: {list(BACKEND_IMAGE_NAMES.keys())}" + ) + return _replace_image_name(source_image, backend_image_name) diff --git a/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py b/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py index cc41f23a8ccb..fc205deb3750 100644 --- a/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py +++ b/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py @@ -85,19 +85,15 @@ def _main_container(component: dict) -> dict: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.2.3", "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", ), - ( - "nvcr.io/nvidia/ai-dynamo/dynamo-frontend@sha256:deadbeef", - "nvcr.io/nvidia/ai-dynamo/dynamo-planner@sha256:deadbeef", - ), ( "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.2.3@sha256:deadbeef", - "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3@sha256:deadbeef", + "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", ), ], ) -def test_derive_planner_image_preserves_registry_tag_and_digest( +def test_derive_planner_image_uses_the_shared_release_tag( image: str, expected: str -): +) -> None: assert derive_planner_image(image) == expected @@ -109,24 +105,34 @@ def test_derive_planner_image_preserves_registry_tag_and_digest( "vllm", "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", ), - ( - "nvcr.io/nvidia/ai-dynamo/dynamo-frontend@sha256:deadbeef", - "sglang", - "nvcr.io/nvidia/ai-dynamo/sglang-runtime@sha256:deadbeef", - ), ( "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.2.3@sha256:deadbeef", "trtllm", - "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.3@sha256:deadbeef", + "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.3", ), ], ) -def test_derive_backend_image_preserves_registry_tag_and_digest( +def test_derive_backend_image_uses_the_shared_release_tag( image: str, backend: str, expected: str -): +) -> None: assert derive_backend_image(image, backend) == expected +@pytest.mark.parametrize( + "target", + ["planner", "backend"], +) +def test_sibling_image_derivation_rejects_digest_only_references( + target: str, +) -> None: + image = "nvcr.io/nvidia/ai-dynamo/dynamo-frontend@sha256:deadbeef" + with pytest.raises(ValueError, match="shared release tag"): + if target == "planner": + derive_planner_image(image) + else: + derive_backend_image(image, "vllm") + + def test_add_planner_to_config_uses_dynamo_planner_image(): image = "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.2.3" dgdr = _make_dgdr(image) diff --git a/components/src/dynamo/profiler/utils/profile_common.py b/components/src/dynamo/profiler/utils/profile_common.py index 4fc42e335c8d..c01b06c312fc 100644 --- a/components/src/dynamo/profiler/utils/profile_common.py +++ b/components/src/dynamo/profiler/utils/profile_common.py @@ -23,6 +23,8 @@ import pandas as pd +from dynamo.common.container_images import derive_backend_image as derive_backend_image +from dynamo.common.container_images import derive_planner_image as derive_planner_image from dynamo.planner.config.planner_config import PlannerPreDeploymentSweepMode from dynamo.profiler.utils.config_modifiers.parallelization_mapping import ( PickedParallelConfig, @@ -36,82 +38,6 @@ logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# Published container image naming conventions -# --------------------------------------------------------------------------- - -# Mapping from backend name to the image-name component of the published -# backend runtime image. -# e.g. vllm → nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.1.1 -BACKEND_IMAGE_NAMES: dict[str, str] = { - "vllm": "vllm-runtime", - "sglang": "sglang-runtime", - "trtllm": "tensorrtllm-runtime", -} - -PLANNER_IMAGE_NAME = "dynamo-planner" - - -def _replace_image_name(image_ref: str, new_name: str) -> str: - """Replace the image name component in a Docker image reference. - - Preserves the registry path prefix and tag suffix, only replacing the - last ``/``-delimited component (before any ``:tag``). - """ - slash_idx = image_ref.rfind("/") - prefix = image_ref[: slash_idx + 1] if slash_idx >= 0 else "" - suffix = image_ref[slash_idx + 1 :] - name_and_tag, has_digest, digest = suffix.partition("@") - colon_idx = name_and_tag.rfind(":") - tag = name_and_tag[colon_idx:] if colon_idx >= 0 else "" - digest_suffix = f"@{digest}" if has_digest else "" - return f"{prefix}{new_name}{tag}{digest_suffix}" - - -def derive_planner_image(profiler_image: str) -> str: - """Derive the planner service image from the profiler image reference.""" - return _replace_image_name(profiler_image, PLANNER_IMAGE_NAME) - - -def derive_backend_image(profiler_image: str, backend: str) -> str: - """Derive the backend worker image from the profiler image. - - Replaces the image name (the last ``/``-delimited component, before any - ``:tag``) with the backend-specific runtime image name, preserving the - registry path and tag unchanged. - - Examples:: - - derive_backend_image( - "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.1.1", "vllm" - ) - # → "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.1.1" - - derive_backend_image("myregistry.io/sglang-runtime:1.1.1", "sglang") - # → "myregistry.io/sglang-runtime:1.1.1" - - Args: - profiler_image: Any Docker image reference of the form - ``[REGISTRY/]NAME[:TAG]``. - backend: The resolved backend type (``'vllm'``, ``'sglang'``, or - ``'trtllm'``). - - Returns: - The backend container image string. - - Raises: - ValueError: If *backend* is not a recognised backend. - """ - backend_image_name = BACKEND_IMAGE_NAMES.get(backend) - if backend_image_name is None: - raise ValueError( - f"Cannot derive backend image for unknown backend '{backend}'. " - f"Supported backends: {list(BACKEND_IMAGE_NAMES.keys())}" - ) - - return _replace_image_name(profiler_image, backend_image_name) - - # --------------------------------------------------------------------------- # Operational defaults not part of DynamoGraphDeploymentRequestSpec # --------------------------------------------------------------------------- From e00a9eaef965e33fccedf2c75aef6c1cea201774 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 16:11:05 +0100 Subject: [PATCH 046/100] pr-17195: align shared image helper with pr-17407 Signed-off-by: Dr. Stefan Schimanski --- .../src/dynamo/common/container_images.py | 24 ++++---- .../unit/test_planner_image_selection.py | 59 ++++++++++++++++++- .../dynamo/profiler/utils/profile_common.py | 2 + 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/components/src/dynamo/common/container_images.py b/components/src/dynamo/common/container_images.py index 4fb519450210..e7466ffb0d00 100644 --- a/components/src/dynamo/common/container_images.py +++ b/components/src/dynamo/common/container_images.py @@ -13,31 +13,31 @@ def _replace_image_name(image_ref: str, new_name: str) -> str: - """Replace the image name using the source image's shared release tag. - - OCI digests identify a manifest within one repository. They cannot be - copied from a Dynamo image to a sibling planner or backend repository. - """ + """Replace an image name while preserving its shared release identity.""" slash_idx = image_ref.rfind("/") prefix = image_ref[: slash_idx + 1] if slash_idx >= 0 else "" suffix = image_ref[slash_idx + 1 :] - name_and_tag = suffix.partition("@")[0] + name_and_tag, has_digest, _ = suffix.partition("@") colon_idx = name_and_tag.rfind(":") - if colon_idx < 0 or colon_idx == len(name_and_tag) - 1: + if colon_idx == len(name_and_tag) - 1 or (has_digest and colon_idx < 0): raise ValueError( f"Cannot derive {new_name!r} from image {image_ref!r}: " "a shared release tag is required; repository-specific digests " "cannot be transferred to sibling images" ) - return f"{prefix}{new_name}{name_and_tag[colon_idx:]}" + tag = name_and_tag[colon_idx:] if colon_idx >= 0 else "" + source_name = name_and_tag[:colon_idx] if colon_idx >= 0 else name_and_tag + if source_name == new_name: + return image_ref + return f"{prefix}{new_name}{tag}" -def derive_planner_image(source_image: str) -> str: +def derive_planner_image(profiler_image: str) -> str: """Derive the planner image that belongs to a published Dynamo image.""" - return _replace_image_name(source_image, PLANNER_IMAGE_NAME) + return _replace_image_name(profiler_image, PLANNER_IMAGE_NAME) -def derive_backend_image(source_image: str, backend: str) -> str: +def derive_backend_image(profiler_image: str, backend: str) -> str: """Derive one backend runtime image from a published Dynamo image.""" backend_image_name = BACKEND_IMAGE_NAMES.get(backend) if backend_image_name is None: @@ -45,4 +45,4 @@ def derive_backend_image(source_image: str, backend: str) -> str: f"Cannot derive backend image for unknown backend '{backend}'. " f"Supported backends: {list(BACKEND_IMAGE_NAMES.keys())}" ) - return _replace_image_name(source_image, backend_image_name) + return _replace_image_name(profiler_image, backend_image_name) diff --git a/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py b/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py index fc205deb3750..5cc9747f7a68 100644 --- a/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py +++ b/components/src/dynamo/profiler/tests/unit/test_planner_image_selection.py @@ -89,6 +89,26 @@ def _main_container(component: dict) -> dict: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.2.3@sha256:deadbeef", "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", ), + ( + "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3@sha256:deadbeef", + "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3@sha256:deadbeef", + ), + ( + "nvcr.io/nvidia/ai-dynamo/dynamo-frontend", + "nvcr.io/nvidia/ai-dynamo/dynamo-planner", + ), + ( + "dynamo-frontend", + "dynamo-planner", + ), + ( + "localhost:5000/dynamo-frontend", + "localhost:5000/dynamo-planner", + ), + ( + "localhost:5000/dynamo-frontend:1.2.3@sha256:deadbeef", + "localhost:5000/dynamo-planner:1.2.3", + ), ], ) def test_derive_planner_image_uses_the_shared_release_tag( @@ -110,6 +130,31 @@ def test_derive_planner_image_uses_the_shared_release_tag( "trtllm", "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.3", ), + ( + "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.3@sha256:deadbeef", + "sglang", + "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.3@sha256:deadbeef", + ), + ( + "nvcr.io/nvidia/ai-dynamo/dynamo-frontend", + "vllm", + "nvcr.io/nvidia/ai-dynamo/vllm-runtime", + ), + ( + "dynamo-frontend", + "vllm", + "vllm-runtime", + ), + ( + "localhost:5000/dynamo-frontend", + "vllm", + "localhost:5000/vllm-runtime", + ), + ( + "localhost:5000/dynamo-frontend:1.2.3@sha256:deadbeef", + "vllm", + "localhost:5000/vllm-runtime:1.2.3", + ), ], ) def test_derive_backend_image_uses_the_shared_release_tag( @@ -122,10 +167,18 @@ def test_derive_backend_image_uses_the_shared_release_tag( "target", ["planner", "backend"], ) -def test_sibling_image_derivation_rejects_digest_only_references( - target: str, +@pytest.mark.parametrize( + "image", + [ + "nvcr.io/nvidia/ai-dynamo/dynamo-frontend@sha256:deadbeef", + "localhost:5000/dynamo-frontend@sha256:deadbeef", + "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:", + "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:@sha256:deadbeef", + ], +) +def test_sibling_image_derivation_rejects_digest_only_and_empty_tag_references( + target: str, image: str ) -> None: - image = "nvcr.io/nvidia/ai-dynamo/dynamo-frontend@sha256:deadbeef" with pytest.raises(ValueError, match="shared release tag"): if target == "planner": derive_planner_image(image) diff --git a/components/src/dynamo/profiler/utils/profile_common.py b/components/src/dynamo/profiler/utils/profile_common.py index c01b06c312fc..9d967e0a35d3 100644 --- a/components/src/dynamo/profiler/utils/profile_common.py +++ b/components/src/dynamo/profiler/utils/profile_common.py @@ -23,6 +23,8 @@ import pandas as pd +from dynamo.common.container_images import BACKEND_IMAGE_NAMES as BACKEND_IMAGE_NAMES +from dynamo.common.container_images import PLANNER_IMAGE_NAME as PLANNER_IMAGE_NAME from dynamo.common.container_images import derive_backend_image as derive_backend_image from dynamo.common.container_images import derive_planner_image as derive_planner_image from dynamo.planner.config.planner_config import PlannerPreDeploymentSweepMode From 711a4ae5216e819f834b28c232c6eb3bc8b91c6d Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 26 Oct 2026 16:08:56 +0100 Subject: [PATCH 047/100] pr-17195: squash execute v1beta2 DGDR runs Squashed from https://github.com/ai-dynamo/dynamo/pull/17195. Signed-off-by: Dr. Stefan Schimanski --- .../aisimulate/output/dgdr_run/adapter.py | 22 +- .../aisimulate/tests/dgdr_run/test_adapter.py | 59 ++ .../components/operator/files/role.yaml | 13 + .../platform/components/operator/values.yaml | 6 +- deploy/operator/Dockerfile | 4 +- deploy/operator/cmd/main.go | 20 +- ...request_v1beta2_controller_envtest_test.go | 15 +- ...ploymentrequest_v1beta2_controller_test.go | 44 +- ...mographdeploymentrun_v1beta2_controller.go | 683 ++++++++++++++++++ ...phdeploymentrun_v1beta2_controller_test.go | 474 ++++++++++++ deploy/operator/internal/controller/setup.go | 50 ++ .../dynamographdeploymentrequest_handler.go | 8 +- ...namographdeploymentrequest_handler_test.go | 4 +- .../dynamographdeploymentrequest_helpers.go | 11 +- 14 files changed, 1363 insertions(+), 50 deletions(-) create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index 5965741614c7..492b8c5030c7 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -75,6 +75,7 @@ SnapshotCandidate, write_snapshot, ) +from dynamo.common.container_images import derive_backend_image logger = logging.getLogger(__name__) @@ -218,6 +219,9 @@ class DGDRRunOutputConfig(BaseModel): namespace: str | None = None generator: Literal["aic", "direct"] = "aic" runtime_image: str = Field(min_length=1) + # A Run searches multiple backends from one planner image. Standalone callers keep + # the historical fixed-image behavior unless they opt into per-candidate derivation. + derive_runtime_images: bool = False runtime_version_override: str | None = None num_gpus_per_node: int = Field(gt=0) # The already-defaulted spec.recommendation.maxCandidates (API range 1..100). The adapter owns @@ -225,9 +229,14 @@ class DGDRRunOutputConfig(BaseModel): max_candidates: int = Field(ge=1, le=100) snapshot_interval_seconds: float = Field(default=5.0, ge=0) - def generation_options(self) -> DGDGenerationOptions: + def generation_options(self, backend: str) -> DGDGenerationOptions: + runtime_image = ( + derive_backend_image(self.runtime_image, backend) + if self.derive_runtime_images + else self.runtime_image + ) return DGDGenerationOptions( - runtime_image=self.runtime_image, + runtime_image=runtime_image, runtime_version_override=self.runtime_version_override, namespace=self.namespace, num_gpus_per_node=self.num_gpus_per_node, @@ -265,7 +274,8 @@ def __init__( raise ValueError("a snapshot directory is required") self._directory = Path(directory) self._config = config - self._options = config.generation_options() + # Validate the source image and runtime version before the first callback. + config.generation_options("vllm") self._workload = workload self._max_candidates = config.max_candidates self._interval = config.snapshot_interval_seconds @@ -600,7 +610,9 @@ def _materialize(self, item: _Retained) -> SnapshotCandidate: manifest = render_dgd( _RenderInput(config=item.parameters, used_gpus=item.used_gpus), self._workload, - self._options, + self._config.generation_options( + str(item.parameters.get("backend", "")) + ), dgd_name=_dgd_name(self._config.name, item.id), renderer=self._config.generator, ) @@ -660,7 +672,7 @@ def subscribe( snapshot is still produced by ``write()``. """ resolved = DGDRRunOutputConfig.model_validate(dict(config)) - resolved.generation_options() + resolved.generation_options("vllm") workload = getattr(context, "workload", None) if workload is None or resolved.snapshot_dir is None: return None diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index 4f5e3b76a0c6..d0723258f09d 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -182,6 +182,65 @@ def test_snapshot_preserves_the_evaluated_point_not_just_its_manifest( assert "marker: a" in candidate["manifest"] +def test_each_candidate_uses_the_runtime_image_for_its_backend( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + images: dict[str, str] = {} + + def capture_image( + candidate: Any, workload: Any, options: Any, **kwargs: Any + ) -> str: + del workload, kwargs + images[candidate.config["backend"]] = options.runtime_image + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_image) + adapter = DGDRRunOutputAdapter( + make_config( + tmp_path, + runtime_image="nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", + derive_runtime_images=True, + ), + workload=None, + ) + for index, backend in enumerate(("vllm", "sglang", "trtllm")): + candidate = record(backend, float(index)) + candidate.config["backend"] = backend + adapter.on_candidate(candidate) + adapter.close() + + assert images == { + "vllm": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "sglang": "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.3", + "trtllm": "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.3", + } + + +def test_explicit_runtime_image_stays_fixed_by_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + images: list[str] = [] + + def capture_image( + candidate: Any, workload: Any, options: Any, **kwargs: Any + ) -> str: + del candidate, workload, kwargs + images.append(options.runtime_image) + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_image) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, runtime_image="example.com/my-runtime:1.2.3"), + workload=None, + ) + candidate = record("sglang", 1.0) + candidate.config["backend"] = "sglang" + adapter.on_candidate(candidate) + adapter.close() + + assert images == ["example.com/my-runtime:1.2.3"] + + def test_round_progress_is_monotonic(tmp_path: Path, renders: list[str]) -> None: adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) adapter.on_round(3, []) diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index b1c343d5d57d..f4420c3b53e8 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -59,11 +59,21 @@ rules: - "" resources: - secrets + verbs: + - create + - get + - list + - update + - watch +- apiGroups: + - "" + resources: - serviceaccounts verbs: - create - get - list + - patch - update - watch - apiGroups: @@ -303,6 +313,7 @@ rules: resources: - dynamocomponentdeployments/status - dynamographdeploymentrequests/status + - dynamographdeploymentruns/status - dynamographdeployments/status - dynamographdeploymentscalingadapters/status - dynamomodels/status @@ -360,6 +371,7 @@ rules: - delete - get - list + - patch - update - watch - apiGroups: @@ -370,6 +382,7 @@ rules: - create - get - list + - patch - update - watch - apiGroups: diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 53ccc596eba2..b6eab5503f81 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -113,9 +113,9 @@ controllerManager: repository: nvcr.io/nvidia/ai-dynamo/kubernetes-operator # tag is optional - if not set, will use Chart.AppVersion tag: "" - # Default DGDR profiler image, put into DGDR spec.image when unset. Empty - # derives dynamo-planner:, which only exists for GA - # releases. + # Planner image used to execute v1beta2 DGDR Runs and default v1beta1 + # spec.image. Empty derives dynamo-planner:, which only + # exists for GA releases. dgdrDefaultImage: "" resources: limits: diff --git a/deploy/operator/Dockerfile b/deploy/operator/Dockerfile index aeb137587691..b903b4c20838 100644 --- a/deploy/operator/Dockerfile +++ b/deploy/operator/Dockerfile @@ -53,6 +53,7 @@ FROM base AS builder RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o manager ./cmd/main.go && \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o crd-apply ./cmd/crd-apply/main.go && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o dgdr-run-publisher ./cmd/dgdr-run-publisher && \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -trimpath -ldflags="-s -w" -o dgd-apply-overrides ./cmd/dgd-apply-overrides # Emit a CycloneDX SBOM for the operator's Go module set so the licenses @@ -60,7 +61,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o manager ./cm # v1.7.0 to keep the SBOM tool itself reproducible. # # `app -main ./cmd` covers the manager binary (./cmd/main.go = package -# main in ./cmd). crd-apply and dgd-apply-overrides are separate mains in +# main in ./cmd). crd-apply, dgdr-run-publisher and dgd-apply-overrides are separate mains in # the same module, so their deps are a subset of manager's — a single SBOM # is sufficient. Tried `mod` first but its license detection is weaker than # `app`'s — everything came back UNKNOWN. @@ -208,6 +209,7 @@ FROM nvcr.io/nvidia/distroless/go:v4.1.7 WORKDIR / COPY --from=builder /workspace/manager . COPY --from=builder /workspace/crd-apply . +COPY --from=builder /workspace/dgdr-run-publisher . COPY --from=builder /workspace/dgd-apply-overrides . COPY --from=builder /workspace/config/crd/bases/ /opt/dynamo-operator/crds/ COPY --from=licenses /legal /legal diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index 7c9c8acbe3b5..b91f0a940519 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -170,7 +170,7 @@ func main() { flag.StringVar(&operatorImagePullPolicy, "operator-image-pull-policy", string(corev1.PullIfNotPresent), "Image pull policy for operator helper init containers") flag.StringVar(&dgdrDefaultImage, "dgdr-default-image", "", - "Default DGDR profiler image, put into DGDR spec.image when unset; empty derives dynamo-planner:") + "Planner image used by v1beta2 Runs and as the v1beta1 spec.image default; empty derives dynamo-planner:") flag.BoolVar(&dgdrV1beta2Enabled, "dgdr-v1beta2-enabled", false, "Reconcile native v1beta2 DGDRs; requires the v1beta2 Request and Run CRDs to be installed") opts := zap.Options{ @@ -193,11 +193,13 @@ func main() { setupLog.Info("Operator configuration loaded successfully", "configFile", configFile) // Validate and normalize operator version to semver - if _, err := semver.NewVersion(operatorVersion); err != nil { + parsedOperatorVersion, err := semver.NewVersion(operatorVersion) + if err != nil { setupLog.Error(err, "operator-version is not valid semver", "provided", operatorVersion, "error", err.Error()) os.Exit(1) } + dynamoVersion := fmt.Sprintf("%d.%d.%d", parsedOperatorVersion.Major(), parsedOperatorVersion.Minor(), parsedOperatorVersion.Patch()) setupLog.Info("Operator version configured", "version", operatorVersion) pullPolicy := corev1.PullPolicy(operatorImagePullPolicy) @@ -495,10 +497,11 @@ func main() { // Register controllers synchronously before mgr.Start(). // Controllers don't depend on TLS certificates. + plannerImage := webhookdefaulting.PlannerImageFor(operatorVersion, dgdrDefaultImage) if err := registerControllers( mgr, operatorCfg, runtimeConfig, dockerSecretRetriever, sshKeyManager, - operatorImage, pullPolicy, + operatorImage, pullPolicy, plannerImage, dynamoVersion, dgdrV1beta2Enabled, ); err != nil { setupLog.Error(err, "failed to register controllers") @@ -549,6 +552,8 @@ func registerControllers( sshKeyManager *secret.SSHKeyManager, operatorImage string, operatorPullPolicy corev1.PullPolicy, + plannerImage string, + dynamoVersion string, dgdrV1beta2Enabled bool, ) error { if operatorCfg.Namespace.Restricted == "" { @@ -614,6 +619,15 @@ func registerControllers( if err := controller.SetupDynamoGraphDeploymentRequestV1beta2(mgr, setupOptions); err != nil { return err } + if err := controller.SetupDynamoGraphDeploymentRunV1beta2(mgr, controller.DynamoGraphDeploymentRunSetupOptions{ + SetupOptions: setupOptions, + PlannerImage: plannerImage, + DynamoVersion: dynamoVersion, + OperatorImage: operatorImage, + OperatorImagePullPolicy: operatorPullPolicy, + }); err != nil { + return err + } } if err := controller.SetupDynamoModel(mgr, controller.DynamoModelSetupOptions{ SetupOptions: setupOptions, diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go index b1cb8175ec5e..3571934eac26 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_envtest_test.go @@ -28,6 +28,7 @@ import ( nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -38,8 +39,8 @@ import ( // This test runs the controller under a real manager, so only the watch events (a Request edit, a Run // annotation/status update, a deleted Run) can make it reconcile. It fails if SetupWithManager loses the Owns // watch, an event predicate drops Run status updates, or the owner mapping breaks. It also covers the -// API server and admission side: the Request is created model-only and admission supplies the image -// and the hardware. +// API server and admission side: the Request is created model-only and admission persists the +// structural defaults and discovered hardware before the controller snapshots it. func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { g := NewWithT(t) env := sharedEnv.ForTest(t) @@ -94,13 +95,11 @@ func TestDGDRV1beta2ControllerReactsToRequestAndRunEvents(t *testing.T) { }).WithTimeout(timeout).WithPolling(poll).Should(Succeed()) first := getRun(g, firstName) - g.Expect(metav1.IsControlledBy(first, getRequest(g))).To(BeTrue()) + live := getRequest(g) + g.Expect(metav1.IsControlledBy(first, live)).To(BeTrue()) g.Expect(first.Spec.Hardware).NotTo(BeNil()) - g.Expect(first.Spec.AISimulate.Image).To(Equal(controllerAISimulateImage)) - g.Expect(first.Spec.AISimulate.VersionOverride).To(Equal("1.2.0")) - input, err := frozenInputForTest(first) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(input).To(HaveKey("search_space")) + g.Expect(apiequality.Semantic.DeepEqual(first.Spec.NativeSearchInput, live.Spec.NativeSearchInput)).To(BeTrue()) + g.Expect(first.Status.AISimulate).To(BeNil()) t.Log("Editing the Request causally requests cancellation without deleting the active Run") g.Eventually(func(g Gomega) { diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index eafafd44c773..22c4b313ac45 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -153,6 +153,9 @@ func dgdrV1beta2Request(withHardware bool) *nvidiacomv1beta2.DynamoGraphDeployme Name: "search", Namespace: "default", UID: "request-uid", Generation: 1, } request.Spec.Model = &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"} + request.Spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{ + InputOverrides: &runtime.RawExtension{Raw: []byte(`{"optimizer":{"max_trials":7}}`)}, + } if withHardware { sku, budget := "h100_sxm", int32(4) request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ @@ -172,10 +175,11 @@ func dgdrV1beta2AcceptedCondition(t *testing.T, request *nvidiacomv1beta2.Dynamo } func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing.T) { + requestInput := dgdrV1beta2Request(true).Spec.NativeSearchInput.DeepCopy() f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(true)) f.reconcile() - t.Log("One Run exists, owned by the Request, with the accepted input snapshot") + t.Log("One Run exists, owned by the Request, with the admitted typed input snapshot") run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} if err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: "search-1"}, run); err != nil { t.Fatalf("get run: %v", err) @@ -183,8 +187,11 @@ func TestV1beta2RequestControllerCreatesTheRunOfTheCurrentGeneration(t *testing. if !metav1.IsControlledBy(run, f.request()) { t.Fatal("run is not controlled by the request") } - if !apiequality.Semantic.DeepEqual(run.Spec.NativeSearchInput, f.request().Spec.NativeSearchInput) { - t.Fatalf("run input = %#v, want the accepted Request snapshot %#v", run.Spec.NativeSearchInput, f.request().Spec.NativeSearchInput) + if !apiequality.Semantic.DeepEqual(run.Spec.NativeSearchInput, *requestInput) { + t.Fatalf("run input = %#v, want the admitted Request snapshot %#v", run.Spec.NativeSearchInput, *requestInput) + } + if run.Status.AISimulate != nil { + t.Fatalf("status.aisimulate = %#v, want the Run controller to lower and record it", run.Status.AISimulate) } t.Log("The status records the accepted generation and the active Run") @@ -446,19 +453,19 @@ func TestV1beta2RequestControllerCancelsAnUnlabeledOwnedRun(t *testing.T) { } } -func TestV1beta2RequestControllerCreatesNoRunWithoutHardware(t *testing.T) { +func TestV1beta2RequestControllerDoesNotRepeatAdmissionValidation(t *testing.T) { f := newDGDRV1beta2Fixture(t, dgdrV1beta2Request(false)) f.reconcile() - if names := f.runNames(); len(names) != 0 { - t.Fatalf("runs = %v, want none", names) + if names := f.runNames(); len(names) != 1 || names[0] != "search-1" { + t.Fatalf("runs = %v, want the admitted Request snapshotted without controller-side validation", names) } request := f.request() condition := dgdrV1beta2AcceptedCondition(t, request) - if condition.Status != metav1.ConditionFalse || condition.Reason != dgdrReasonInvalidSpec { - t.Fatalf("condition = %v, want False/%s", condition, dgdrReasonInvalidSpec) + if condition.Status != metav1.ConditionTrue || condition.Reason != dgdrReasonRunCreated { + t.Fatalf("condition = %v, want True/%s", condition, dgdrReasonRunCreated) } - if request.Status.ObservedGeneration != 0 { - t.Fatalf("observedGeneration = %d, want 0 for a rejected generation", request.Status.ObservedGeneration) + if request.Status.ObservedGeneration != 1 { + t.Fatalf("observedGeneration = %d, want 1", request.Status.ObservedGeneration) } } @@ -712,11 +719,10 @@ func TestV1beta2RequestControllerDoesNotAdoptARunOfAnotherOwner(t *testing.T) { } } -func TestV1beta2RequestControllerPreservesInputOverridesInTheRunSnapshot(t *testing.T) { +func TestV1beta2RequestControllerPreservesInputOverridesForRunLowering(t *testing.T) { request := dgdrV1beta2Request(true) - request.Spec.AISimulate = &nvidiacomv1beta2.AISimulateSpec{InputOverrides: &runtime.RawExtension{ - Raw: []byte(`{"traffic":{"source":{"input_tokens":null}},"optimizer":{"max_trials":null}}`), - }} + const overrides = `{"traffic":{"source":{"input_tokens":null}},"optimizer":{"max_trials":null}}` + request.Spec.AISimulate.InputOverrides = &runtime.RawExtension{Raw: []byte(overrides)} f := newDGDRV1beta2Fixture(t, request) run, err := f.reconciler.buildRun(request) @@ -724,9 +730,13 @@ func TestV1beta2RequestControllerPreservesInputOverridesInTheRunSnapshot(t *test t.Fatalf("build run: %v", err) } if run.Spec.AISimulate == nil || run.Spec.AISimulate.InputOverrides == nil { - t.Fatal("Run did not preserve spec.aisimulate.inputOverrides") + t.Fatal("Run lost spec.aisimulate.inputOverrides") + } + if got := string(run.Spec.AISimulate.InputOverrides.Raw); got != overrides { + t.Fatalf("Run inputOverrides = %s, want %s", got, overrides) } - if got, want := string(run.Spec.AISimulate.InputOverrides.Raw), string(request.Spec.AISimulate.InputOverrides.Raw); got != want { - t.Fatalf("run inputOverrides = %s, want %s", got, want) + request.Spec.AISimulate.InputOverrides.Raw[0] = '[' + if got := string(run.Spec.AISimulate.InputOverrides.Raw); got != overrides { + t.Fatalf("Run inputOverrides changed with the Request: %s", got) } } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go new file mode 100644 index 000000000000..c55fc06bcd86 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -0,0 +1,683 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path" + "strconv" + "strings" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/aisimulate" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/yaml" +) + +const ( + dgdrRunJobControllerUIDIndex = ".metadata.dgdrRunControllerUID" + dgdrRunMaxJobNameLength = 63 + + dgdrRunInputKey = "input.yaml" + dgdrRunInputMountPath = "/etc/aisimulate" + dgdrRunInputPath = dgdrRunInputMountPath + "/" + dgdrRunInputKey + dgdrRunSnapshotPath = "/var/run/aisimulate/output" + dgdrRunServiceAccountMountPath = "/var/run/secrets/kubernetes.io/serviceaccount" + dgdrRunPublisherBinary = "/dgdr-run-publisher" + dgdrRunSweeperContainer = "sweeper" + dgdrRunPublisherContainer = "publisher" + dgdrRunInputVolume = "input" + dgdrRunSnapshotVolume = "snapshot" + dgdrRunServiceAccountVolume = "publisher-kube-api" + dgdrRunGPUsPerNodeAnnotation = "nvidia.com/dgdr-gpus-per-node" + dgdrRunSnapshotPathAnnotation = "nvidia.com/dgdr-snapshot-path" + dgdrRunResourceComponent = "aisimulate" + dgdrRunServiceAccountTokenTTL = int64(3600) + dgdrRunReasonPending = "JobPending" + dgdrRunReasonRunning = "JobRunning" + dgdrRunReasonSucceeded = "SearchSucceeded" + dgdrRunReasonSearchFailed = "SearchFailed" + dgdrRunReasonProtocolViolation = "ProtocolViolation" + dgdrRunReasonPublicationFailed = "PublicationFailed" + dgdrRunReasonJobFailed = "JobFailed" + dgdrRunMessagePending = "AISimulate Job is waiting to start" + dgdrRunMessageRunning = "AISimulate Job is running" + dgdrRunMessageSucceeded = "AISimulate search completed successfully" +) + +type dgdrRunHardwareDiscoverer interface { + Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) +} + +// DynamoGraphDeploymentRunV1beta2Reconciler executes one immutable Run as an +// AISimulate Job. The publisher sidecar owns progress and candidate publication; +// this controller owns the Job reference, execution phase, timestamps and Completed. +type DynamoGraphDeploymentRunV1beta2Reconciler struct { + client.Client + Config *configv1alpha1.OperatorConfiguration + RuntimeConfig *commoncontroller.RuntimeConfig + Hardware dgdrRunHardwareDiscoverer + PlannerImage string + DynamoVersion string + OperatorImage string + OperatorImagePullPolicy corev1.PullPolicy +} + +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( + ctx context.Context, + req ctrl.Request, +) (ctrl.Result, error) { + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := r.Get(ctx, req.NamespacedName, run); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + if run.DeletionTimestamp != nil || meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { + return ctrl.Result{}, nil + } + jobs, err := r.ownedJobs(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + if len(jobs) > 1 { + return ctrl.Result{}, fmt.Errorf("Run %q controls %d Jobs; expected at most one", run.Name, len(jobs)) + } + + var job *batchv1.Job + if len(jobs) == 1 { + job = &jobs[0] + } + resourceName := dgdrRunResourceName(run.Name) + if err := r.reconcilePublisherAccess(ctx, run, resourceName); err != nil { + return ctrl.Result{}, err + } + execution, err := r.reconcileInput(ctx, run, job, resourceName) + if err != nil { + return ctrl.Result{}, err + } + + if job == nil { + job = r.buildJob(run, resourceName, execution) + if err := controllerutil.SetControllerReference(run, job, r.Scheme()); err != nil { + return ctrl.Result{}, fmt.Errorf("set Job owner: %w", err) + } + if err := r.Create(ctx, job); err != nil { + // A cache miss followed by AlreadyExists is expected to heal on the next + // reconciliation once the informer observes the Job. Never bypass the cache. + return ctrl.Result{}, fmt.Errorf("create Job %q: %w", job.Name, err) + } + } + + return ctrl.Result{}, r.reconcileRunStatus(ctx, run, job, execution) +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) ownedJobs( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) ([]batchv1.Job, error) { + list := &batchv1.JobList{} + if err := r.List(ctx, list, client.InNamespace(run.Namespace), + client.MatchingFields{dgdrRunJobControllerUIDIndex: string(run.UID)}); err != nil { + return nil, fmt.Errorf("list Jobs of Run %q: %w", run.Name, err) + } + owned := make([]batchv1.Job, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], run) { + owned = append(owned, list.Items[i]) + } + } + return owned, nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + name string, +) error { + sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { + return controllerutil.SetControllerReference(run, sa, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile ServiceAccount %q: %w", name, err) + } + + role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { + role.Rules = dgdrRunPublisherRules(run.Name) + return controllerutil.SetControllerReference(run, role, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile Role %q: %w", name, err) + } + + binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, binding, func() error { + binding.RoleRef = rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: name} + binding.Subjects = []rbacv1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: run.Namespace}} + return controllerutil.SetControllerReference(run, binding, r.Scheme()) + }); err != nil { + return fmt.Errorf("reconcile RoleBinding %q: %w", name, err) + } + return nil +} + +func dgdrRunPublisherRules(runName string) []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentruns"}, + ResourceNames: []string{runName}, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentruns/status"}, + ResourceNames: []string{runName}, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentcandidates"}, + Verbs: []string{"get", "list", "watch", "create", "patch", "delete"}, + }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentcandidates/status"}, + Verbs: []string{"get", "update", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }, + } +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, + name string, +) (dgdrRunExecution, error) { + configMap := &corev1.ConfigMap{} + key := types.NamespacedName{Namespace: run.Namespace, Name: name} + err := r.Get(ctx, key, configMap) + if err == nil { + if !metav1.IsControlledBy(configMap, run) { + return dgdrRunExecution{}, fmt.Errorf("ConfigMap %q is not controlled by Run %q", name, run.Name) + } + gpusPerNode, parseErr := positiveAnnotation(configMap.Annotations, dgdrRunGPUsPerNodeAnnotation) + if parseErr != nil { + return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q: %w", name, parseErr) + } + input, inputErr := dgdrRunStatusInput(configMap.Data[dgdrRunInputKey]) + if inputErr != nil { + return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q input: %w", name, inputErr) + } + execution := dgdrRunExecution{ + GPUsPerNode: gpusPerNode, + SnapshotPath: annotationOrDefault(configMap.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath), + Input: input, + } + if job != nil { + jobGPUs, jobErr := positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) + if jobErr != nil { + return dgdrRunExecution{}, fmt.Errorf("read Job %q: %w", job.Name, jobErr) + } + if jobGPUs != gpusPerNode { + return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree about GPUs per node", job.Name, name) + } + jobSnapshotPath := annotationOrDefault(job.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath) + if jobSnapshotPath != execution.SnapshotPath { + return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree about the snapshot path", job.Name, name) + } + } + return execution, nil + } + if !apierrors.IsNotFound(err) { + return dgdrRunExecution{}, fmt.Errorf("get ConfigMap %q: %w", name, err) + } + + execution := dgdrRunExecution{SnapshotPath: dgdrRunSnapshotPath} + if job != nil { + execution.GPUsPerNode, err = positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) + if err != nil { + return dgdrRunExecution{}, fmt.Errorf("read Job %q: %w", job.Name, err) + } + execution.SnapshotPath = annotationOrDefault(job.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath) + } + if execution.GPUsPerNode == 0 { + execution.GPUsPerNode, err = r.discoverGPUsPerNode(ctx, run) + if err != nil { + return dgdrRunExecution{}, err + } + } + input, snapshotPath, err := dgdrRunInput(run, r.PlannerImage, r.DynamoVersion, execution.GPUsPerNode) + if err != nil { + return dgdrRunExecution{}, err + } + contents, err := yaml.Marshal(input) + if err != nil { + return dgdrRunExecution{}, fmt.Errorf("encode AISimulate input: %w", err) + } + execution.Input, err = dgdrRunStatusInput(string(contents)) + if err != nil { + return dgdrRunExecution{}, err + } + if job == nil { + execution.SnapshotPath = snapshotPath + } + + configMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: run.Namespace, + Annotations: map[string]string{ + dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, + }, + }, + Immutable: ptr.To(true), + Data: map[string]string{dgdrRunInputKey: string(contents)}, + } + if err := controllerutil.SetControllerReference(run, configMap, r.Scheme()); err != nil { + return dgdrRunExecution{}, fmt.Errorf("set ConfigMap owner: %w", err) + } + if err := r.Create(ctx, configMap); err != nil { + return dgdrRunExecution{}, fmt.Errorf("create ConfigMap %q: %w", name, err) + } + return execution, nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) (int, error) { + if r.Hardware == nil { + return 0, errors.New("discover GPUs per node: hardware discoverer is not configured") + } + if run.Spec.Hardware == nil || run.Spec.Hardware.GPU == nil || run.Spec.Hardware.GPU.SKU == nil { + return 0, errors.New("discover GPUs per node: spec.hardware.gpu.sku was not resolved by Request admission") + } + info, err := r.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(*run.Spec.Hardware.GPU.SKU)) + if err != nil { + return 0, fmt.Errorf("discover GPUs per node: %w", err) + } + if info == nil || info.GPUsPerNode < 1 { + return 0, errors.New("discover GPUs per node: discovery returned no positive GPU count") + } + return info.GPUsPerNode, nil +} + +func dgdrRunInput( + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + plannerImage string, + dynamoVersion string, + gpusPerNode int, +) (map[string]any, string, error) { + if run.Spec.Recommendation == nil || run.Spec.Recommendation.MaxCandidates < 1 { + return nil, "", errors.New("spec.recommendation.maxCandidates must be defaulted before the Run is created") + } + output := map[string]any{ + "name": run.Name, + "snapshot_dir": dgdrRunSnapshotPath, + "namespace": run.Namespace, + "runtime_image": plannerImage, + "derive_runtime_images": true, + "num_gpus_per_node": int64(gpusPerNode), + "max_candidates": int64(run.Spec.Recommendation.MaxCandidates), + } + if _, err := runtimeversion.ParseImageVersion(plannerImage); err != nil { + if _, versionErr := runtimeversion.Parse(dynamoVersion); versionErr != nil { + return nil, "", fmt.Errorf("resolve Dynamo version for planner image %q: %w", plannerImage, versionErr) + } + output["runtime_version_override"] = dynamoVersion + } + underlay := map[string]any{"dgdr_run": output} + input, err := aisimulate.LowerWithUnderlay(&run.Spec.NativeSearchInput, underlay) + if err != nil { + return nil, "", err + } + snapshotPath := dgdrRunSnapshotPath + if configured, found, err := unstructured.NestedString(input, "dgdr_run", "snapshot_dir"); err == nil && found && path.IsAbs(configured) { + snapshotPath = path.Clean(configured) + } + return input, snapshotPath, nil +} + +type dgdrRunExecution struct { + GPUsPerNode int + SnapshotPath string + Input *runtime.RawExtension +} + +func dgdrRunStatusInput(contents string) (*runtime.RawExtension, error) { + var input map[string]any + if err := yaml.Unmarshal([]byte(contents), &input); err != nil { + return nil, fmt.Errorf("decode AISimulate input: %w", err) + } + if input == nil { + return nil, errors.New("decode AISimulate input: expected an object") + } + raw, err := json.Marshal(input) + if err != nil { + return nil, fmt.Errorf("encode AISimulate status input: %w", err) + } + return &runtime.RawExtension{Raw: raw}, nil +} + +func positiveAnnotation(annotations map[string]string, key string) (int, error) { + value := annotations[key] + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 1 { + return 0, fmt.Errorf("annotation %s=%q is not a positive integer", key, value) + } + return parsed, nil +} + +func annotationOrDefault(annotations map[string]string, key, defaultValue string) string { + if value := annotations[key]; value != "" { + return value + } + return defaultValue +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + name string, + execution dgdrRunExecution, +) *batchv1.Job { + pullPolicy := r.OperatorImagePullPolicy + if pullPolicy == "" { + pullPolicy = corev1.PullIfNotPresent + } + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: run.Namespace, + Annotations: map[string]string{ + dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, + }, + }, + Spec: batchv1.JobSpec{ + BackoffLimit: ptr.To[int32](0), + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ServiceAccountName: name, + AutomountServiceAccountToken: ptr.To(false), + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{FSGroup: ptr.To[int64](65532)}, + Containers: []corev1.Container{ + { + Name: dgdrRunSweeperContainer, + Image: r.PlannerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"aisimulate"}, + Args: []string{ + "recommend", "--stack", "dynamo", + "--config", dgdrRunInputPath, + "--output", "dgdr_run", + "--output-dir", execution.SnapshotPath, + "--overwrite", + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true}, + {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, + }, + }, + { + Name: dgdrRunPublisherContainer, + Image: r.OperatorImage, + ImagePullPolicy: pullPolicy, + Command: []string{dgdrRunPublisherBinary}, + Args: []string{ + "--snapshot-dir", execution.SnapshotPath, + "--run-name", run.Name, + "--run-uid", string(run.UID), + "--sweeper-container", dgdrRunSweeperContainer, + }, + Env: []corev1.EnvVar{ + {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, + {Name: dgdrRunServiceAccountVolume, MountPath: dgdrRunServiceAccountMountPath, ReadOnly: true}, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: dgdrRunInputVolume, + VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + }}, + }, + {Name: dgdrRunSnapshotVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + dgdrRunPublisherKubeAPIVolume(), + }, + }, + }, + }, + } + return job +} + +func dgdrRunPublisherKubeAPIVolume() corev1.Volume { + return corev1.Volume{ + Name: dgdrRunServiceAccountVolume, + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: "token", ExpirationSeconds: ptr.To(dgdrRunServiceAccountTokenTTL), + }}, + {ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{Name: ConfigMapNameKubeRootCA}, + Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}, + }}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{ + Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}, + }}}}, + }, + }}, + } +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRunStatus( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, + execution dgdrRunExecution, +) error { + if execution.Input == nil { + return errors.New("update Run status: effective AISimulate input is missing") + } + original := run.DeepCopy() + run.Status.AISimulate = &nvidiacomv1beta2.AISimulateRunStatus{Input: execution.Input.DeepCopy()} + run.Status.JobRef = &nvidiacomv1beta2.ResourceReference{Name: job.Name, UID: job.UID} + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhasePending + reason, message := dgdrRunReasonPending, dgdrRunMessagePending + completed := metav1.ConditionFalse + + if job.Status.StartTime != nil { + run.Status.StartTime = job.Status.StartTime.DeepCopy() + } + if condition := dgdrRunJobCondition(job, batchv1.JobComplete); condition != nil && condition.Status == corev1.ConditionTrue { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseSucceeded + reason, message = dgdrRunReasonSucceeded, dgdrRunMessageSucceeded + completed = metav1.ConditionTrue + run.Status.CompletionTime = dgdrRunCompletionTime(job) + } else if condition := dgdrRunJobCondition(job, batchv1.JobFailed); condition != nil && condition.Status == corev1.ConditionTrue { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseFailed + reason, message = r.dgdrRunFailure(ctx, job, condition) + completed = metav1.ConditionTrue + run.Status.CompletionTime = dgdrRunCompletionTime(job) + } else if job.Status.Active > 0 || job.Status.StartTime != nil { + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseRunning + reason, message = dgdrRunReasonRunning, dgdrRunMessageRunning + } + + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: completed, + Reason: reason, + Message: message, + ObservedGeneration: run.Generation, + }) + if apiequality.Semantic.DeepEqual(original.Status, run.Status) { + return nil + } + if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { + return fmt.Errorf("update Run %q status: %w", run.Name, err) + } + return nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) dgdrRunFailure( + ctx context.Context, + job *batchv1.Job, + condition *batchv1.JobCondition, +) (string, string) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(job.Namespace), client.MatchingLabels{batchv1.JobNameLabel: job.Name}); err != nil { + log.FromContext(ctx).Error(err, "Could not inspect failed AISimulate Job pods", "job", job.Name) + return dgdrRunReasonJobFailed, condition.Message + } + for i := range pods.Items { + pod := &pods.Items[i] + if !metav1.IsControlledBy(pod, job) { + continue + } + for _, status := range pod.Status.ContainerStatuses { + terminated := status.State.Terminated + if terminated == nil || terminated.ExitCode == 0 { + continue + } + message := terminated.Message + if message == "" { + message = terminated.Reason + } + if message == "" { + message = fmt.Sprintf("container %s exited with code %d", status.Name, terminated.ExitCode) + } + if status.Name == dgdrRunPublisherContainer { + switch int(terminated.ExitCode) { + case dgdrrunpublisher.ExitRunFailed: + return dgdrRunReasonSearchFailed, message + case dgdrrunpublisher.ExitProtocolViolation: + return dgdrRunReasonProtocolViolation, message + case dgdrrunpublisher.ExitReconcileFailed: + return dgdrRunReasonPublicationFailed, message + } + } + if status.Name == dgdrRunSweeperContainer { + return dgdrRunReasonSearchFailed, message + } + } + } + message := condition.Message + if message == "" { + message = "AISimulate Job failed" + } + return dgdrRunReasonJobFailed, message +} + +func dgdrRunCompletionTime(job *batchv1.Job) *metav1.Time { + if job.Status.CompletionTime != nil { + return job.Status.CompletionTime.DeepCopy() + } + for _, condition := range job.Status.Conditions { + if (condition.Type == batchv1.JobComplete || condition.Type == batchv1.JobFailed) && condition.Status == corev1.ConditionTrue { + return condition.LastTransitionTime.DeepCopy() + } + } + return nil +} + +func dgdrRunJobCondition(job *batchv1.Job, conditionType batchv1.JobConditionType) *batchv1.JobCondition { + for i := range job.Status.Conditions { + if job.Status.Conditions[i].Type == conditionType { + return &job.Status.Conditions[i] + } + } + return nil +} + +func dgdrRunResourceName(runName string) string { + suffix := "-" + dgdrRunResourceComponent + if len(runName)+len(suffix) <= dgdrRunMaxJobNameLength { + return runName + suffix + } + sum := sha256.Sum256([]byte(runName)) + hash := hex.EncodeToString(sum[:6]) + room := dgdrRunMaxJobNameLength - len(suffix) - len(hash) - 1 + prefix := strings.TrimRight(runName[:room], "-.") + return prefix + "-" + hash + suffix +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.PlannerImage == "" { + return errors.New("planner image is required for the DGDR Run sweeper") + } + if r.OperatorImage == "" { + return errors.New("operator image is required for the DGDR Run publisher") + } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), &batchv1.Job{}, dgdrRunJobControllerUIDIndex, dgdrRunJobControllerUIDs, + ); err != nil { + return fmt.Errorf("index Jobs by Run controller UID: %w", err) + } + return ctrl.NewControllerManagedBy(mgr). + Named("dynamographdeploymentrun-v1beta2"). + For(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}). + Owns(&batchv1.Job{}). + Owns(&corev1.ConfigMap{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.Config, r.RuntimeConfig)). + Complete(r) +} + +func dgdrRunJobControllerUIDs(obj client.Object) []string { + owner := metav1.GetControllerOf(obj) + if owner == nil || owner.Kind != "DynamoGraphDeploymentRun" || owner.APIVersion != nvidiacomv1beta2.GroupVersion.String() { + return nil + } + return []string{string(owner.UID)} +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go new file mode 100644 index 000000000000..d400a1b7868e --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -0,0 +1,474 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/yaml" +) + +type fixedRunHardware struct { + info gpu.GPUInfo + err error + calls []nvidiacomv1beta1.GPUSKUType +} + +func (f *fixedRunHardware) Discover(_ context.Context, sku nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + f.calls = append(f.calls, sku) + if f.err != nil { + return nil, f.err + } + return &f.info, nil +} + +type dgdrRunFixture struct { + t *testing.T + client client.Client + reconciler *DynamoGraphDeploymentRunV1beta2Reconciler +} + +func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects ...client.Object) *dgdrRunFixture { + t.Helper() + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("add Kubernetes types: %v", err) + } + if err := nvidiacomv1beta2.AddToScheme(scheme); err != nil { + t.Fatalf("add v1beta2 types: %v", err) + } + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + WithIndex(&batchv1.Job{}, dgdrRunJobControllerUIDIndex, dgdrRunJobControllerUIDs). + WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, &batchv1.Job{}, &corev1.Pod{}). + Build() + return &dgdrRunFixture{ + t: t, + client: c, + reconciler: &DynamoGraphDeploymentRunV1beta2Reconciler{ + Client: c, + Hardware: hardware, + PlannerImage: "example.com/dynamo-planner:1.2.3", + OperatorImage: "example.com/dynamo-operator:1.2.3", + OperatorImagePullPolicy: corev1.PullAlways, + }, + } +} + +func (f *dgdrRunFixture) reconcile() { + f.t.Helper() + if _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ + Namespace: "default", Name: "search-1", + }}); err != nil { + f.t.Fatalf("reconcile: %v", err) + } +} + +func (f *dgdrRunFixture) run() *nvidiacomv1beta2.DynamoGraphDeploymentRun { + f.t.Helper() + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + if err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: "search-1"}, run); err != nil { + f.t.Fatalf("get Run: %v", err) + } + return run +} + +func (f *dgdrRunFixture) job() *batchv1.Job { + f.t.Helper() + jobs := &batchv1.JobList{} + if err := f.client.List(f.t.Context(), jobs, client.InNamespace("default")); err != nil { + f.t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 1 { + f.t.Fatalf("Jobs = %d, want 1", len(jobs.Items)) + } + return &jobs.Items[0] +} + +func dgdrRunForTest(t *testing.T, inputOverrides map[string]any) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + t.Helper() + raw, err := json.Marshal(inputOverrides) + if err != nil { + t.Fatalf("marshal parameters: %v", err) + } + return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ + ObjectMeta: metav1.ObjectMeta{Name: "search-1", Namespace: "default", UID: "run-uid", Generation: 1}, + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, + Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: ptr.To("h200_sxm"), Budget: ptr.To[int32](16), + }}, + AISimulate: &nvidiacomv1beta2.AISimulateSpec{InputOverrides: &runtime.RawExtension{Raw: raw}}, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 2}, + }}, + } +} + +func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + run := dgdrRunForTest(t, map[string]any{ + "engine": map[string]any{"hardware": "h100_sxm"}, + "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": 12}}, + "dgdr_run": map[string]any{ + "max_candidates": 7, + "runtime_image": "example.com/custom-vllm-runtime:1.2.3", + "name": "native-output-name", + "snapshot_dir": "/work/results", + }, + }) + f := newDGDRRunFixture(t, hardware, run) + + f.reconcile() + f.reconcile() + + job := f.job() + assertDGDRRunJobIsolation(t, job, run, f.reconciler.PlannerImage, f.reconciler.OperatorImage) + assertDGDRRunInput(t, f.client, job, hardware) + assertDGDRRunSupportResources(t, f.client, job, run) + stored := f.run() + assertDGDRRunPendingStatus(t, stored, job.Name) + assertDGDRRunStatusInputMatchesConfigMap(t, f.client, stored, job) +} + +func assertDGDRRunJobIsolation( + t *testing.T, + job *batchv1.Job, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + plannerImage string, + operatorImage string, +) { + t.Helper() + if !metav1.IsControlledBy(job, run) { + t.Fatal("Job is not controlled by its Run") + } + if got := job.Annotations[dgdrRunGPUsPerNodeAnnotation]; got != "8" { + t.Fatalf("GPUs-per-node annotation = %q, want 8", got) + } + if job.Spec.BackoffLimit == nil || *job.Spec.BackoffLimit != 0 { + t.Fatalf("backoffLimit = %v, want 0", job.Spec.BackoffLimit) + } + if job.Spec.Template.Spec.AutomountServiceAccountToken == nil || *job.Spec.Template.Spec.AutomountServiceAccountToken { + t.Fatal("the pod must disable automatic service-account token mounting") + } + sweeper := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + publisher := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunPublisherContainer) + if sweeper.Image != plannerImage { + t.Fatalf("sweeper image = %q, want operator-configured planner image %q", sweeper.Image, plannerImage) + } + if want := []string{"aisimulate"}; !reflect.DeepEqual(sweeper.Command, want) { + t.Fatalf("sweeper command = %q, want %q", sweeper.Command, want) + } + if want := []string{ + "recommend", "--stack", "dynamo", + "--config", dgdrRunInputPath, + "--output", "dgdr_run", + "--output-dir", "/work/results", + "--overwrite", + }; !reflect.DeepEqual(sweeper.Args, want) { + t.Fatalf("sweeper args = %q, want %q", sweeper.Args, want) + } + if publisher.Image != operatorImage || publisher.ImagePullPolicy != corev1.PullAlways { + t.Fatalf("publisher image = %q/%s, want configured operator image/Always", publisher.Image, publisher.ImagePullPolicy) + } + if hasMount(sweeper.VolumeMounts, dgdrRunServiceAccountVolume) { + t.Fatal("sweeper unexpectedly receives the Kubernetes service-account token") + } + if !hasMount(publisher.VolumeMounts, dgdrRunServiceAccountVolume) { + t.Fatal("publisher does not receive its projected Kubernetes credentials") + } + if !hasMountAt(sweeper.VolumeMounts, dgdrRunSnapshotVolume, "/work/results") || + !hasMountAt(publisher.VolumeMounts, dgdrRunSnapshotVolume, "/work/results") { + t.Fatal("sweeper and publisher do not share the parameter-overridden snapshot directory") + } +} + +func assertDGDRRunStatusInputMatchesConfigMap( + t *testing.T, + c client.Client, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, +) { + t.Helper() + if run.Status.AISimulate == nil || run.Status.AISimulate.Input == nil { + t.Fatal("status.aisimulate.input is empty") + } + var statusInput map[string]any + if err := json.Unmarshal(run.Status.AISimulate.Input.Raw, &statusInput); err != nil { + t.Fatalf("decode status.aisimulate.input: %v", err) + } + configMap := &corev1.ConfigMap{} + if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get input ConfigMap: %v", err) + } + var configInput map[string]any + if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &configInput); err != nil { + t.Fatalf("decode input ConfigMap: %v", err) + } + if !reflect.DeepEqual(statusInput, configInput) { + t.Fatalf("status input = %#v, want exact ConfigMap input %#v", statusInput, configInput) + } +} + +func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardware *fixedRunHardware) { + t.Helper() + configMap := &corev1.ConfigMap{} + if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get input ConfigMap: %v", err) + } + if configMap.Immutable == nil || !*configMap.Immutable { + t.Fatal("input ConfigMap is not immutable") + } + var input map[string]any + if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &input); err != nil { + t.Fatalf("decode input ConfigMap: %v", err) + } + output := input["dgdr_run"].(map[string]any) + if output["name"] != "native-output-name" || output["snapshot_dir"] != "/work/results" { + t.Fatalf("native parameter overrides were not preserved: %v", output) + } + if output["max_candidates"] != float64(7) || output["runtime_image"] != "example.com/custom-vllm-runtime:1.2.3" { + t.Fatalf("native parameter overrides were not preserved: %v", output) + } + if output["derive_runtime_images"] != true { + t.Fatalf("derive_runtime_images = %v, want true for a backend search", output["derive_runtime_images"]) + } + if got := input["engine"].(map[string]any)["hardware"]; got != "h100_sxm" { + t.Fatalf("engine.hardware = %v, want authoritative input override h100_sxm", got) + } + constraints := input["optimization"].(map[string]any)["constraints"].(map[string]any) + if got := constraints["max_candidate_gpus"]; got != float64(12) { + t.Fatalf("optimization.constraints.max_candidate_gpus = %v, want authoritative input override 12", got) + } + if !reflect.DeepEqual(hardware.calls, []nvidiacomv1beta1.GPUSKUType{"h200_sxm"}) { + t.Fatalf("hardware discovery calls = %v, want the admitted typed SKU h200_sxm", hardware.calls) + } +} + +func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { + tests := []struct { + name string + plannerImage string + dynamoVersion string + wantOverride string + wantErr bool + }{ + {name: "canonical planner tag", plannerImage: "example.com/dynamo-planner:1.7.0"}, + {name: "opaque CI tag", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7.0", wantOverride: "1.7.0"}, + {name: "opaque tag without canonical version", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := dgdrRunForTest(t, map[string]any{}) + + input, _, err := dgdrRunInput(run, tt.plannerImage, tt.dynamoVersion, 8) + if (err != nil) != tt.wantErr { + t.Fatalf("lower input error = %v, wantErr %t", err, tt.wantErr) + } + if tt.wantErr { + return + } + output := input["dgdr_run"].(map[string]any) + got, found := output["runtime_version_override"] + if tt.wantOverride == "" && found { + t.Fatalf("dgdr_run.runtime_version_override = %v, want omitted", got) + } + if tt.wantOverride != "" && got != tt.wantOverride { + t.Fatalf("dgdr_run.runtime_version_override = %v, want %s", got, tt.wantOverride) + } + }) + } +} + +func assertDGDRRunSupportResources( + t *testing.T, + c client.Client, + job *batchv1.Job, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) { + t.Helper() + for _, object := range []client.Object{ + &corev1.ServiceAccount{}, &rbacv1.Role{}, &rbacv1.RoleBinding{}, + } { + if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), object); err != nil { + t.Fatalf("get %T: %v", object, err) + } + if !metav1.IsControlledBy(object, run) { + t.Fatalf("%T is not controlled by the Run", object) + } + } +} + +func assertDGDRRunPendingStatus(t *testing.T, stored *nvidiacomv1beta2.DynamoGraphDeploymentRun, jobName string) { + t.Helper() + if stored.Status.Phase != nvidiacomv1beta2.DGDRRunPhasePending || stored.Status.JobRef == nil || stored.Status.JobRef.Name != jobName { + t.Fatalf("Run status = %+v, want Pending and JobRef %q", stored.Status, jobName) + } + condition := meta.FindStatusCondition(stored.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != dgdrRunReasonPending { + t.Fatalf("Completed condition = %+v, want False/%s", condition, dgdrRunReasonPending) + } +} + +func TestV1beta2RunControllerOwnsOnlyLifecycleStatus(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + f := newDGDRRunFixture(t, hardware, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + job := f.job() + + run := f.run() + progressTime := metav1.NewTime(time.Date(2026, 10, 23, 10, 0, 0, 0, time.UTC)) + run.Status.Progress = &nvidiacomv1beta2.DGDRRunProgress{Candidates: &nvidiacomv1beta2.CandidateProgress{Published: 1}} + run.Status.CandidateRefs = []corev1.LocalObjectReference{{Name: "candidate-a"}} + run.Status.LastProgressTime = &progressTime + if err := f.client.Status().Update(t.Context(), run); err != nil { + t.Fatalf("publish progress status: %v", err) + } + + start := metav1.NewTime(progressTime.Add(time.Minute)) + job.Status.StartTime = &start + job.Status.Active = 1 + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("mark Job running: %v", err) + } + f.reconcile() + run = f.run() + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseRunning || run.Status.StartTime == nil || !run.Status.StartTime.Equal(&start) { + t.Fatalf("running status = %+v", run.Status) + } + if run.Status.Progress == nil || len(run.Status.CandidateRefs) != 1 || run.Status.LastProgressTime == nil { + t.Fatalf("publisher-owned status was lost: %+v", run.Status) + } + + completed := metav1.NewTime(start.Add(5 * time.Minute)) + job = f.job() + job.Status.Active = 0 + job.Status.CompletionTime = &completed + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobComplete, Status: corev1.ConditionTrue, LastTransitionTime: completed, + }} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("complete Job: %v", err) + } + f.reconcile() + run = f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseSucceeded || condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("completed status = %+v", run.Status) + } + if run.Status.CompletionTime == nil || !run.Status.CompletionTime.Equal(&completed) { + t.Fatalf("completionTime = %v, want %v", run.Status.CompletionTime, completed) + } +} + +func TestV1beta2RunControllerClassifiesPublisherProtocolFailure(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + job := f.job() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: job.Name + "-pod", + Namespace: job.Namespace, + Labels: map[string]string{batchv1.JobNameLabel: job.Name}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", Name: job.Name, UID: job.UID, + Controller: ptr.To(true), + }}, + }, + } + if err := f.client.Create(t.Context(), pod); err != nil { + t.Fatalf("create Job pod: %v", err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: dgdrRunPublisherContainer, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: int32(dgdrrunpublisher.ExitProtocolViolation), Message: "terminal snapshot missing", + }}, + }} + if err := f.client.Status().Update(t.Context(), pod); err != nil { + t.Fatalf("update Job pod status: %v", err) + } + + failed := metav1.Now() + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobFailed, Status: corev1.ConditionTrue, LastTransitionTime: failed, + }} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("fail Job: %v", err) + } + f.reconcile() + + run := f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed || condition == nil || condition.Reason != dgdrRunReasonProtocolViolation { + t.Fatalf("failed status = %+v", run.Status) + } + if !strings.Contains(condition.Message, "terminal snapshot missing") { + t.Fatalf("failure message = %q, want publisher termination message", condition.Message) + } +} + +func TestDGDRRunResourceNameIsStableAndBounded(t *testing.T) { + short := dgdrRunResourceName("search-1") + if short != "search-1-aisimulate" { + t.Fatalf("short name = %q", short) + } + long := strings.Repeat("a", 200) + one, two := dgdrRunResourceName(long), dgdrRunResourceName(long) + if one != two || len(one) > dgdrRunMaxJobNameLength || !strings.HasSuffix(one, "-aisimulate") { + t.Fatalf("bounded name = %q", one) + } +} + +func containerNamed(t *testing.T, containers []corev1.Container, name string) corev1.Container { + t.Helper() + for _, container := range containers { + if container.Name == name { + return container + } + } + t.Fatalf("container %q not found", name) + return corev1.Container{} +} + +func hasMount(mounts []corev1.VolumeMount, name string) bool { + for _, mount := range mounts { + if mount.Name == name { + return true + } + } + return false +} + +func hasMountAt(mounts []corev1.VolumeMount, name, path string) bool { + for _, mount := range mounts { + if mount.Name == name && mount.MountPath == path { + return true + } + } + return false +} diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index ed71c6aa4a9f..839e6893273e 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -11,6 +11,7 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" lpxcontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/lpx" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/modelendpoint" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" @@ -42,6 +43,16 @@ type DynamoGraphDeploymentRequestSetupOptions struct { OperatorImagePullPolicy corev1.PullPolicy } +type DynamoGraphDeploymentRunSetupOptions struct { + SetupOptions + GPUDiscoveryCache *gpu.GPUDiscoveryCache + GPUDiscovery *gpu.GPUDiscovery + PlannerImage string + DynamoVersion string + OperatorImage string + OperatorImagePullPolicy corev1.PullPolicy +} + type DynamoModelSetupOptions struct { SetupOptions ModelEndpointClient *modelendpoint.Client @@ -61,6 +72,20 @@ func (o DynamoGraphDeploymentRequestSetupOptions) gpuDiscovery() *gpu.GPUDiscove return gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint) } +func (o DynamoGraphDeploymentRunSetupOptions) gpuDiscoveryCache() *gpu.GPUDiscoveryCache { + if o.GPUDiscoveryCache != nil { + return o.GPUDiscoveryCache + } + return gpu.NewGPUDiscoveryCache() +} + +func (o DynamoGraphDeploymentRunSetupOptions) gpuDiscovery() *gpu.GPUDiscovery { + if o.GPUDiscovery != nil { + return o.GPUDiscovery + } + return gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint) +} + func (o DynamoModelSetupOptions) modelEndpointClient() *modelendpoint.Client { if o.ModelEndpointClient != nil { return o.ModelEndpointClient @@ -146,6 +171,31 @@ func SetupDynamoGraphDeploymentRequestV1beta2(mgr ctrl.Manager, opts SetupOption return nil } +// SetupDynamoGraphDeploymentRunV1beta2 registers the controller that executes immutable +// v1beta2 Runs as AISimulate Jobs and observes their lifecycle. +func SetupDynamoGraphDeploymentRunV1beta2(mgr ctrl.Manager, opts DynamoGraphDeploymentRunSetupOptions) error { + if err := (&DynamoGraphDeploymentRunV1beta2Reconciler{ + Client: mgr.GetClient(), + Config: opts.Config, + RuntimeConfig: opts.RuntimeConfig, + Hardware: &gpu.Discoverer{ + Reader: mgr.GetClient(), + DCGM: opts.gpuDiscovery(), + Cache: opts.gpuDiscoveryCache(), + NodeLabels: func() bool { + return opts.RuntimeConfig != nil && opts.RuntimeConfig.Gate.Enabled(features.GPUDiscovery) + }, + }, + PlannerImage: opts.PlannerImage, + DynamoVersion: opts.DynamoVersion, + OperatorImage: opts.OperatorImage, + OperatorImagePullPolicy: opts.OperatorImagePullPolicy, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create DynamoGraphDeploymentRun v1beta2 controller: %w", err) + } + return nil +} + func SetupDynamoModel(mgr ctrl.Manager, opts DynamoModelSetupOptions) error { if err := (&DynamoModelReconciler{ Client: mgr.GetClient(), diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go index 01601c6cd602..a9a13943f008 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler.go @@ -100,17 +100,11 @@ func (d *DGDRDefaulter) Default(ctx context.Context, obj runtime.Object) error { defaultDynamoGraphDeploymentRequest(dgdr, dynamoGraphDeploymentRequestDefaultOptions{ operation: req.Operation, - image: d.defaultImageFor(), + image: PlannerImageFor(d.OperatorVersion, d.DefaultImage), }) return nil } -// defaultPlannerImageFor returns explicitImage when set, else the planner image with a -// canonical operator-version tag, or an empty string when the version cannot be parsed. -func defaultPlannerImageFor(operatorVersion, explicitImage string) string { - return NewDGDRDefaulter(operatorVersion, explicitImage).defaultImageFor() -} - // RegisterWithManager registers the DGDR defaulting webhook with the manager. func (d *DGDRDefaulter) RegisterWithManager(mgr manager.Manager) error { defaulter := internalwebhook.NewLeaseAwareDefaulter(d, internalwebhook.GetExcludedNamespaces()) diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go index 9d45a2543a8f..61cadc3f2131 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_handler_test.go @@ -77,9 +77,9 @@ func TestDefaultPlannerImageFor(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := defaultPlannerImageFor(tt.operatorVersion, tt.defaultImage) + got := PlannerImageFor(tt.operatorVersion, tt.defaultImage) if got != tt.expectedImage { - t.Errorf("defaultPlannerImageFor() = %q, want %q", got, tt.expectedImage) + t.Errorf("PlannerImageFor() = %q, want %q", got, tt.expectedImage) } }) } diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_helpers.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_helpers.go index bdcd1fa3d19b..5cb894cc3139 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_helpers.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_helpers.go @@ -16,11 +16,14 @@ import ( // the dedicated dynamo-planner image. const defaultImage = "nvcr.io/nvidia/ai-dynamo/dynamo-planner" -func (d *DGDRDefaulter) defaultImageFor() string { - if d.DefaultImage != "" { - return d.DefaultImage +// PlannerImageFor returns explicitImage when set, else the planner image with a canonical +// operator-version tag, or an empty string when the version cannot be parsed. The same resolved +// image defaults v1beta1 Requests and executes v1beta2 Runs. +func PlannerImageFor(operatorVersion, explicitImage string) string { + if explicitImage != "" { + return explicitImage } - version, err := semver.NewVersion(d.OperatorVersion) + version, err := semver.NewVersion(operatorVersion) if err != nil { return "" } From 96d3ddadd50f8296e3edb7071c652f2263f9bf1f Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 10:04:06 +0100 Subject: [PATCH 048/100] pr-17195: freeze v1beta2 Run execution contract Signed-off-by: Dr. Stefan Schimanski --- deploy/helm/charts/platform/README.md | 2 +- .../platform/components/operator/values.yaml | 6 +- deploy/helm/charts/platform/values.yaml | 6 +- deploy/operator/cmd/main.go | 1 + .../nvidia.com_dynamographdeploymentruns.yaml | 9 +- ...aphdeploymentrequest_v1beta2_controller.go | 14 +- ...mographdeploymentrun_v1beta2_controller.go | 346 ++++++++++++++---- ...phdeploymentrun_v1beta2_controller_test.go | 257 ++++++++++++- deploy/operator/internal/controller/setup.go | 2 + .../additional-resources/api-reference-k8s.md | 14 +- .../kubernetes-api/full-api-reference.mdx | 14 +- .../scripts/tests/test_gen_kubernetes_api.py | 2 +- 12 files changed, 547 insertions(+), 126 deletions(-) diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index 9bc82dd97b8f..7ef6cdb861e6 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -546,7 +546,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.grovePreflight.deploymentName | string | `""` | External Grove Deployment name. Requires namespace; empty discovers by container name. | | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether the cluster-wide operator applies CRDs from its image through the crd-apply init container. When false, the chart installs no CRDs; apply them separately before starting a cluster-wide operator. Namespace-restricted installations must set this to false and use the CRDs managed by the cluster-wide operator. | -| dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. When true, the operator also reconciles v1beta2 Requests into Runs; the controller that executes a Run is not part of the operator yet. | +| dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. When true, the operator reconciles v1beta2 Requests into Runs and executes each Run as an AISimulate planner Job with a publisher sidecar that persists deployment Candidates. | | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index b6eab5503f81..269173f55baa 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -36,9 +36,9 @@ dgdrV1Beta2: # Candidate objects are persisted in etcd; incompatible schema changes may make # them unreadable or block future CRD upgrades. When false, the CRD init container # installs the legacy Request schemas without v1beta2 conversion envelopes and - # omits Run and Candidate. When true, the operator also reconciles v1beta2 - # Requests into Runs; the controller that executes a Run is not part of the - # operator yet. + # omits Run and Candidate. When true, the operator reconciles v1beta2 + # Requests into Runs and executes each Run as an AISimulate planner Job with + # a publisher sidecar that persists deployment Candidates. enabled: false # Environment variables to pass to operator Deployment. diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index fe6daeec961b..a68de01cca42 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -97,9 +97,9 @@ dynamo-operator: # -- EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 # Request API with conversion and install the Run and Candidate APIs. Run and # Candidate objects are persisted in etcd; incompatible schema changes may make - # them unreadable or block future CRD upgrades. When true, the operator also - # reconciles v1beta2 Requests into Runs; the controller that executes a Run is - # not part of the operator yet. + # them unreadable or block future CRD upgrades. When true, the operator + # reconciles v1beta2 Requests into Runs and executes each Run as an AISimulate + # planner Job with a publisher sidecar that persists deployment Candidates. enabled: false # Environment variables to pass to operator Deployment. diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index b91f0a940519..d7b8d4e98eea 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -621,6 +621,7 @@ func registerControllers( } if err := controller.SetupDynamoGraphDeploymentRunV1beta2(mgr, controller.DynamoGraphDeploymentRunSetupOptions{ SetupOptions: setupOptions, + DockerSecretRetriever: dockerSecretRetriever, PlannerImage: plannerImage, DynamoVersion: dynamoVersion, OperatorImage: operatorImage, diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index bb2e6bc0ae58..c057b554069a 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -57,9 +57,12 @@ spec: type: object spec: description: |- - DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. - Run creators resolve optional Request inputs and the final AISimulate parameters before - persisting the Run. Request admission defaulting does not apply to Runs. + DynamoGraphDeploymentRunSpec is an immutable snapshot of an accepted DGDR spec. + Request admission resolves its structural defaults before the Request controller copies it; + Request admission defaulting does not apply to Runs. + + The Run reconciler lowers this immutable typed snapshot plus inputOverrides exactly once and + records the resulting public recommendation input in status.aisimulate.input. properties: aisimulate: description: AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index 2d6842edea9e..4cd79565f8e1 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -526,12 +526,20 @@ func dgdrV1beta2SplitRuns( return active, terminal } -// requireDGDRV1beta2API fails when the cluster does not serve the v1beta2 Request and Run APIs. +// requireDGDRV1beta2API fails when the cluster does not serve the complete v1beta2 execution API. // Without them the controller's caches never sync and the manager would hang at startup. func requireDGDRV1beta2API(mgr ctrl.Manager) error { + return requireDGDRV1beta2RESTMappings(mgr.GetRESTMapper()) +} + +func requireDGDRV1beta2RESTMappings(mapper meta.RESTMapper) error { group, version := nvidiacomv1beta2.GroupVersion.Group, nvidiacomv1beta2.GroupVersion.Version - for _, kind := range []string{"DynamoGraphDeploymentRequest", "DynamoGraphDeploymentRun"} { - _, err := mgr.GetRESTMapper().RESTMapping(schema.GroupKind{Group: group, Kind: kind}, version) + for _, kind := range []string{ + "DynamoGraphDeploymentRequest", + "DynamoGraphDeploymentRun", + "DynamoGraphDeploymentCandidate", + } { + _, err := mapper.RESTMapping(schema.GroupKind{Group: group, Kind: kind}, version) if meta.IsNoMatchError(err) { return fmt.Errorf("--dgdr-v1beta2-enabled is set but the cluster does not serve %s/%s %s; "+ "install the v1beta2 CRDs (dgdrV1Beta2.enabled)", group, version, kind) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index c55fc06bcd86..2d684532851b 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -12,9 +12,9 @@ import ( "encoding/json" "errors" "fmt" - "path" "strconv" "strings" + "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" @@ -47,6 +47,15 @@ const ( dgdrRunMaxJobNameLength = 63 dgdrRunInputKey = "input.yaml" + dgdrRunPlannerImageKey = "planner-image" + dgdrRunPlannerPullPolicyKey = "planner-image-pull-policy" + dgdrRunRuntimeImageSourceKey = "runtime-image-source" + dgdrRunPublisherImageKey = "publisher-image" + dgdrRunPublisherPullPolicyKey = "publisher-image-pull-policy" + dgdrRunDynamoVersionKey = "dynamo-version" + dgdrRunGPUsPerNodeKey = "gpus-per-node" + dgdrRunSnapshotPathKey = "snapshot-path" + dgdrRunImagePullSecretsKey = "image-pull-secrets.json" dgdrRunInputMountPath = "/etc/aisimulate" dgdrRunInputPath = dgdrRunInputMountPath + "/" + dgdrRunInputKey dgdrRunSnapshotPath = "/var/run/aisimulate/output" @@ -71,8 +80,11 @@ const ( dgdrRunMessagePending = "AISimulate Job is waiting to start" dgdrRunMessageRunning = "AISimulate Job is running" dgdrRunMessageSucceeded = "AISimulate search completed successfully" + dgdrRunExecutionCacheRetry = time.Second ) +var errDGDRRunExecutionPending = errors.New("immutable execution ConfigMap is not visible") + type dgdrRunHardwareDiscoverer interface { Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) } @@ -82,6 +94,7 @@ type dgdrRunHardwareDiscoverer interface { // this controller owns the Job reference, execution phase, timestamps and Completed. type DynamoGraphDeploymentRunV1beta2Reconciler struct { client.Client + APIReader client.Reader Config *configv1alpha1.OperatorConfiguration RuntimeConfig *commoncontroller.RuntimeConfig Hardware dgdrRunHardwareDiscoverer @@ -89,6 +102,7 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { DynamoVersion string OperatorImage string OperatorImagePullPolicy corev1.PullPolicy + DockerSecretRetriever DockerSecretRetriever } // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch @@ -128,6 +142,9 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( } execution, err := r.reconcileInput(ctx, run, job, resourceName) if err != nil { + if errors.Is(err, errDGDRRunExecutionPending) { + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil + } return ctrl.Result{}, err } @@ -240,30 +257,13 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( if !metav1.IsControlledBy(configMap, run) { return dgdrRunExecution{}, fmt.Errorf("ConfigMap %q is not controlled by Run %q", name, run.Name) } - gpusPerNode, parseErr := positiveAnnotation(configMap.Annotations, dgdrRunGPUsPerNodeAnnotation) + execution, parseErr := dgdrRunExecutionFromConfigMap(configMap) if parseErr != nil { return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q: %w", name, parseErr) } - input, inputErr := dgdrRunStatusInput(configMap.Data[dgdrRunInputKey]) - if inputErr != nil { - return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q input: %w", name, inputErr) - } - execution := dgdrRunExecution{ - GPUsPerNode: gpusPerNode, - SnapshotPath: annotationOrDefault(configMap.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath), - Input: input, - } if job != nil { - jobGPUs, jobErr := positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) - if jobErr != nil { - return dgdrRunExecution{}, fmt.Errorf("read Job %q: %w", job.Name, jobErr) - } - if jobGPUs != gpusPerNode { - return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree about GPUs per node", job.Name, name) - } - jobSnapshotPath := annotationOrDefault(job.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath) - if jobSnapshotPath != execution.SnapshotPath { - return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree about the snapshot path", job.Name, name) + if validateErr := validateDGDRRunJobExecution(job, execution); validateErr != nil { + return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree: %w", job.Name, name, validateErr) } } return execution, nil @@ -272,21 +272,55 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( return dgdrRunExecution{}, fmt.Errorf("get ConfigMap %q: %w", name, err) } - execution := dgdrRunExecution{SnapshotPath: dgdrRunSnapshotPath} if job != nil { - execution.GPUsPerNode, err = positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) - if err != nil { - return dgdrRunExecution{}, fmt.Errorf("read Job %q: %w", job.Name, err) + if r.APIReader != nil { + persisted := &corev1.ConfigMap{} + apiErr := r.APIReader.Get(ctx, key, persisted) + if apiErr == nil { + return dgdrRunExecution{}, fmt.Errorf( + "%w: ConfigMap %q exists in the API but is not visible in the controller cache", + errDGDRRunExecutionPending, name, + ) + } + if !apierrors.IsNotFound(apiErr) { + return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q directly from the API: %w", name, apiErr) + } } - execution.SnapshotPath = annotationOrDefault(job.Annotations, dgdrRunSnapshotPathAnnotation, dgdrRunSnapshotPath) + return dgdrRunExecution{}, fmt.Errorf( + "immutable execution ConfigMap %q is missing while Job %q exists; refusing to reconstruct it from current operator configuration", + name, job.Name, + ) } - if execution.GPUsPerNode == 0 { - execution.GPUsPerNode, err = r.discoverGPUsPerNode(ctx, run) - if err != nil { - return dgdrRunExecution{}, err - } + + // Resolve every execution value before creating the immutable artifact. + execution := dgdrRunExecution{ + SnapshotPath: dgdrRunSnapshotPath, + PlannerImage: r.PlannerImage, + PlannerPullPolicy: corev1.PullIfNotPresent, + PublisherImage: r.OperatorImage, + PublisherPullPolicy: r.OperatorImagePullPolicy, + } + if execution.PublisherPullPolicy == "" { + execution.PublisherPullPolicy = corev1.PullIfNotPresent + } + execution.GPUsPerNode, err = r.discoverGPUsPerNode(ctx, run) + if err != nil { + return dgdrRunExecution{}, err + } + execution.DynamoVersion, err = dgdrRunDynamoVersion(execution.PlannerImage, r.DynamoVersion) + if err != nil { + return dgdrRunExecution{}, err + } + execution.RuntimeImageSource, err = dgdrRunRuntimeImageSource(execution.PlannerImage, execution.DynamoVersion) + if err != nil { + return dgdrRunExecution{}, err } - input, snapshotPath, err := dgdrRunInput(run, r.PlannerImage, r.DynamoVersion, execution.GPUsPerNode) + execution.ImagePullSecrets, err = r.dgdrRunImagePullSecrets(run.Namespace, execution.PlannerImage, execution.PublisherImage) + if err != nil { + return dgdrRunExecution{}, err + } + + input, err := dgdrRunInput(run, execution.RuntimeImageSource, execution.DynamoVersion, execution.GPUsPerNode) if err != nil { return dgdrRunExecution{}, err } @@ -298,8 +332,9 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( if err != nil { return dgdrRunExecution{}, err } - if job == nil { - execution.SnapshotPath = snapshotPath + pullSecrets, err := json.Marshal(execution.ImagePullSecrets) + if err != nil { + return dgdrRunExecution{}, fmt.Errorf("encode image pull secrets: %w", err) } configMap = &corev1.ConfigMap{ @@ -312,7 +347,18 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( }, }, Immutable: ptr.To(true), - Data: map[string]string{dgdrRunInputKey: string(contents)}, + Data: map[string]string{ + dgdrRunInputKey: string(contents), + dgdrRunPlannerImageKey: execution.PlannerImage, + dgdrRunPlannerPullPolicyKey: string(execution.PlannerPullPolicy), + dgdrRunRuntimeImageSourceKey: execution.RuntimeImageSource, + dgdrRunPublisherImageKey: execution.PublisherImage, + dgdrRunPublisherPullPolicyKey: string(execution.PublisherPullPolicy), + dgdrRunDynamoVersionKey: execution.DynamoVersion, + dgdrRunGPUsPerNodeKey: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathKey: execution.SnapshotPath, + dgdrRunImagePullSecretsKey: string(pullSecrets), + }, } if err := controllerutil.SetControllerReference(run, configMap, r.Scheme()); err != nil { return dgdrRunExecution{}, fmt.Errorf("set ConfigMap owner: %w", err) @@ -345,44 +391,194 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( func dgdrRunInput( run *nvidiacomv1beta2.DynamoGraphDeploymentRun, - plannerImage string, + runtimeImageSource string, dynamoVersion string, gpusPerNode int, -) (map[string]any, string, error) { +) (map[string]any, error) { if run.Spec.Recommendation == nil || run.Spec.Recommendation.MaxCandidates < 1 { - return nil, "", errors.New("spec.recommendation.maxCandidates must be defaulted before the Run is created") + return nil, errors.New("spec.recommendation.maxCandidates must be defaulted before the Run is created") } output := map[string]any{ - "name": run.Name, - "snapshot_dir": dgdrRunSnapshotPath, - "namespace": run.Namespace, - "runtime_image": plannerImage, - "derive_runtime_images": true, - "num_gpus_per_node": int64(gpusPerNode), - "max_candidates": int64(run.Spec.Recommendation.MaxCandidates), - } - if _, err := runtimeversion.ParseImageVersion(plannerImage); err != nil { - if _, versionErr := runtimeversion.Parse(dynamoVersion); versionErr != nil { - return nil, "", fmt.Errorf("resolve Dynamo version for planner image %q: %w", plannerImage, versionErr) - } - output["runtime_version_override"] = dynamoVersion + "name": run.Name, + "max_candidates": int64(run.Spec.Recommendation.MaxCandidates), } underlay := map[string]any{"dgdr_run": output} input, err := aisimulate.LowerWithUnderlay(&run.Spec.NativeSearchInput, underlay) if err != nil { - return nil, "", err - } - snapshotPath := dgdrRunSnapshotPath - if configured, found, err := unstructured.NestedString(input, "dgdr_run", "snapshot_dir"); err == nil && found && path.IsAbs(configured) { - snapshotPath = path.Clean(configured) + return nil, err + } + + // Apply operator-owned execution values after user overrides have won over public search fields. + reserved := map[string]any{ + "name": run.Name, + "max_candidates": int64(run.Spec.Recommendation.MaxCandidates), + "snapshot_dir": dgdrRunSnapshotPath, + "namespace": run.Namespace, + "runtime_image": runtimeImageSource, + "runtime_version_override": dynamoVersion, + "derive_runtime_images": true, + "num_gpus_per_node": int64(gpusPerNode), + } + for key, value := range reserved { + if err := unstructured.SetNestedField(input, value, "dgdr_run", key); err != nil { + return nil, fmt.Errorf("set operator-owned dgdr_run.%s: %w", key, err) + } } - return input, snapshotPath, nil + return input, nil } type dgdrRunExecution struct { - GPUsPerNode int - SnapshotPath string - Input *runtime.RawExtension + GPUsPerNode int + SnapshotPath string + PlannerImage string + PlannerPullPolicy corev1.PullPolicy + RuntimeImageSource string + PublisherImage string + PublisherPullPolicy corev1.PullPolicy + DynamoVersion string + ImagePullSecrets []corev1.LocalObjectReference + Input *runtime.RawExtension +} + +func dgdrRunDynamoVersion(plannerImage, dynamoVersion string) (string, error) { + if version, err := runtimeversion.ParseImageVersion(plannerImage); err == nil { + return version.String(), nil + } + version, err := runtimeversion.Parse(dynamoVersion) + if err != nil { + return "", fmt.Errorf("resolve Dynamo version for planner image %q: %w", plannerImage, err) + } + return version.String(), nil +} + +func dgdrRunRuntimeImageSource(plannerImage, dynamoVersion string) (string, error) { + ref := strings.TrimSpace(plannerImage) + digest := strings.Index(ref, "@") + nameAndTag := ref + if digest >= 0 { + nameAndTag = ref[:digest] + } + lastSlash, lastColon := strings.LastIndex(nameAndTag, "/"), strings.LastIndex(nameAndTag, ":") + if lastColon > lastSlash && lastColon < len(nameAndTag)-1 { + return ref, nil + } + if digest >= 0 && nameAndTag != "" { + return nameAndTag + ":" + dynamoVersion, nil + } + return "", fmt.Errorf( + "resolve runtime images for planner image %q: a shared release tag or repository-specific digest is required", + plannerImage, + ) +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) dgdrRunImagePullSecrets( + namespace string, + images ...string, +) ([]corev1.LocalObjectReference, error) { + if r.DockerSecretRetriever == nil { + return nil, nil + } + + // Resolve every execution image against secrets visible in the Run namespace. + var refs []corev1.LocalObjectReference + for _, image := range images { + names, err := r.DockerSecretRetriever.GetSecrets(namespace, image) + if err != nil { + return nil, fmt.Errorf("resolve image pull secrets for %q in namespace %q: %w", image, namespace, err) + } + additional := make([]corev1.LocalObjectReference, 0, len(names)) + for _, name := range names { + additional = append(additional, corev1.LocalObjectReference{Name: name}) + } + refs = commoncontroller.AppendUniqueImagePullSecrets(refs, additional) + } + return refs, nil +} + +func dgdrRunExecutionFromConfigMap(configMap *corev1.ConfigMap) (dgdrRunExecution, error) { + execution := dgdrRunExecution{ + PlannerImage: configMap.Data[dgdrRunPlannerImageKey], + PlannerPullPolicy: corev1.PullPolicy(configMap.Data[dgdrRunPlannerPullPolicyKey]), + RuntimeImageSource: configMap.Data[dgdrRunRuntimeImageSourceKey], + PublisherImage: configMap.Data[dgdrRunPublisherImageKey], + PublisherPullPolicy: corev1.PullPolicy(configMap.Data[dgdrRunPublisherPullPolicyKey]), + DynamoVersion: configMap.Data[dgdrRunDynamoVersionKey], + SnapshotPath: configMap.Data[dgdrRunSnapshotPathKey], + } + for key, value := range map[string]string{ + dgdrRunPlannerImageKey: execution.PlannerImage, + dgdrRunPlannerPullPolicyKey: string(execution.PlannerPullPolicy), + dgdrRunRuntimeImageSourceKey: execution.RuntimeImageSource, + dgdrRunPublisherImageKey: execution.PublisherImage, + dgdrRunPublisherPullPolicyKey: string(execution.PublisherPullPolicy), + dgdrRunDynamoVersionKey: execution.DynamoVersion, + dgdrRunSnapshotPathKey: execution.SnapshotPath, + } { + if value == "" { + return dgdrRunExecution{}, fmt.Errorf("data key %q is empty", key) + } + } + if execution.PlannerPullPolicy != corev1.PullAlways && + execution.PlannerPullPolicy != corev1.PullIfNotPresent && + execution.PlannerPullPolicy != corev1.PullNever { + return dgdrRunExecution{}, fmt.Errorf("data key %q=%q is not a valid image pull policy", dgdrRunPlannerPullPolicyKey, execution.PlannerPullPolicy) + } + if execution.PublisherPullPolicy != corev1.PullAlways && + execution.PublisherPullPolicy != corev1.PullIfNotPresent && + execution.PublisherPullPolicy != corev1.PullNever { + return dgdrRunExecution{}, fmt.Errorf("data key %q=%q is not a valid image pull policy", dgdrRunPublisherPullPolicyKey, execution.PublisherPullPolicy) + } + + // Decode the immutable topology, credentials and effective input. + var err error + execution.GPUsPerNode, err = positiveInteger(configMap.Data[dgdrRunGPUsPerNodeKey], "data key "+dgdrRunGPUsPerNodeKey) + if err != nil { + return dgdrRunExecution{}, err + } + if err := json.Unmarshal([]byte(configMap.Data[dgdrRunImagePullSecretsKey]), &execution.ImagePullSecrets); err != nil { + return dgdrRunExecution{}, fmt.Errorf("decode data key %q: %w", dgdrRunImagePullSecretsKey, err) + } + execution.Input, err = dgdrRunStatusInput(configMap.Data[dgdrRunInputKey]) + if err != nil { + return dgdrRunExecution{}, fmt.Errorf("decode data key %q: %w", dgdrRunInputKey, err) + } + return execution, nil +} + +func validateDGDRRunJobExecution(job *batchv1.Job, execution dgdrRunExecution) error { + jobGPUs, err := positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) + if err != nil { + return err + } + if jobGPUs != execution.GPUsPerNode { + return errors.New("GPU counts differ") + } + if job.Annotations[dgdrRunSnapshotPathAnnotation] != execution.SnapshotPath { + return errors.New("snapshot paths differ") + } + + // Match both container identities and all credentials against the frozen artifact. + planner, found := dgdrRunContainer(job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + if !found || planner.Image != execution.PlannerImage || planner.ImagePullPolicy != execution.PlannerPullPolicy { + return errors.New("planner image identities differ") + } + publisher, found := dgdrRunContainer(job.Spec.Template.Spec.Containers, dgdrRunPublisherContainer) + if !found || publisher.Image != execution.PublisherImage || publisher.ImagePullPolicy != execution.PublisherPullPolicy { + return errors.New("publisher image identities differ") + } + if !apiequality.Semantic.DeepEqual(job.Spec.Template.Spec.ImagePullSecrets, execution.ImagePullSecrets) { + return errors.New("image pull secrets differ") + } + return nil +} + +func dgdrRunContainer(containers []corev1.Container, name string) (corev1.Container, bool) { + for _, container := range containers { + if container.Name == name { + return container, true + } + } + return corev1.Container{}, false } func dgdrRunStatusInput(contents string) (*runtime.RawExtension, error) { @@ -401,30 +597,22 @@ func dgdrRunStatusInput(contents string) (*runtime.RawExtension, error) { } func positiveAnnotation(annotations map[string]string, key string) (int, error) { - value := annotations[key] + return positiveInteger(annotations[key], "annotation "+key) +} + +func positiveInteger(value, field string) (int, error) { parsed, err := strconv.Atoi(value) if err != nil || parsed < 1 { - return 0, fmt.Errorf("annotation %s=%q is not a positive integer", key, value) + return 0, fmt.Errorf("%s=%q is not a positive integer", field, value) } return parsed, nil } -func annotationOrDefault(annotations map[string]string, key, defaultValue string) string { - if value := annotations[key]; value != "" { - return value - } - return defaultValue -} - func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( run *nvidiacomv1beta2.DynamoGraphDeploymentRun, name string, execution dgdrRunExecution, ) *batchv1.Job { - pullPolicy := r.OperatorImagePullPolicy - if pullPolicy == "" { - pullPolicy = corev1.PullIfNotPresent - } job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -442,11 +630,12 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( AutomountServiceAccountToken: ptr.To(false), RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{FSGroup: ptr.To[int64](65532)}, + ImagePullSecrets: append([]corev1.LocalObjectReference(nil), execution.ImagePullSecrets...), Containers: []corev1.Container{ { Name: dgdrRunSweeperContainer, - Image: r.PlannerImage, - ImagePullPolicy: corev1.PullIfNotPresent, + Image: execution.PlannerImage, + ImagePullPolicy: execution.PlannerPullPolicy, Command: []string{"aisimulate"}, Args: []string{ "recommend", "--stack", "dynamo", @@ -462,8 +651,8 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( }, { Name: dgdrRunPublisherContainer, - Image: r.OperatorImage, - ImagePullPolicy: pullPolicy, + Image: execution.PublisherImage, + ImagePullPolicy: execution.PublisherPullPolicy, Command: []string{dgdrRunPublisherBinary}, Args: []string{ "--snapshot-dir", execution.SnapshotPath, @@ -651,6 +840,9 @@ func dgdrRunResourceName(runName string) string { } func (r *DynamoGraphDeploymentRunV1beta2Reconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } if r.PlannerImage == "" { return errors.New("planner image is required for the DGDR Run sweeper") } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index d400a1b7868e..06ac076a21d1 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -8,6 +8,7 @@ package controller import ( "context" "encoding/json" + "errors" "reflect" "strings" "testing" @@ -23,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -38,6 +40,18 @@ type fixedRunHardware struct { calls []nvidiacomv1beta1.GPUSKUType } +type fixedRunSecrets struct { + byImage map[string][]string + err error +} + +func (f *fixedRunSecrets) GetSecrets(_ string, image string) ([]string, error) { + if f.err != nil { + return nil, f.err + } + return append([]string(nil), f.byImage[image]...), nil +} + func (f *fixedRunHardware) Discover(_ context.Context, sku nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { f.calls = append(f.calls, sku) if f.err != nil { @@ -72,6 +86,7 @@ func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects client: c, reconciler: &DynamoGraphDeploymentRunV1beta2Reconciler{ Client: c, + APIReader: c, Hardware: hardware, PlannerImage: "example.com/dynamo-planner:1.2.3", OperatorImage: "example.com/dynamo-operator:1.2.3", @@ -139,6 +154,7 @@ func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { "runtime_image": "example.com/custom-vllm-runtime:1.2.3", "name": "native-output-name", "snapshot_dir": "/work/results", + "namespace": "another-namespace", }, }) f := newDGDRRunFixture(t, hardware, run) @@ -155,6 +171,150 @@ func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { assertDGDRRunStatusInputMatchesConfigMap(t, f.client, stored, job) } +func TestV1beta2RunControllerFreezesExecutionBeforeCreatingTheJob(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + run := dgdrRunForTest(t, map[string]any{}) + f := newDGDRRunFixture(t, hardware, run) + f.reconciler.DockerSecretRetriever = &fixedRunSecrets{byImage: map[string][]string{ + "example.com/dynamo-planner:1.2.3": {"planner-secret", "shared-secret"}, + "example.com/dynamo-operator:1.2.3": {"shared-secret", "publisher-secret"}, + }} + + t.Log("Persist the immutable execution artifact before any Job exists") + execution, err := f.reconciler.reconcileInput(t.Context(), run, nil, dgdrRunResourceName(run.Name)) + if err != nil { + t.Fatalf("reconcile input: %v", err) + } + if execution.PlannerImage != "example.com/dynamo-planner:1.2.3" || execution.PublisherImage != "example.com/dynamo-operator:1.2.3" { + t.Fatalf("frozen images = %q/%q", execution.PlannerImage, execution.PublisherImage) + } + + t.Log("Simulate an operator upgrade before the next reconciliation") + f.reconciler.PlannerImage = "example.com/dynamo-planner:9.9.9" + f.reconciler.OperatorImage = "example.com/dynamo-operator:9.9.9" + f.reconciler.OperatorImagePullPolicy = corev1.PullNever + f.reconciler.DockerSecretRetriever = &fixedRunSecrets{byImage: map[string][]string{ + "example.com/dynamo-planner:9.9.9": {"new-secret"}, + }} + f.reconcile() + + t.Log("Verify the Job uses only the previously frozen identity and credentials") + job := f.job() + planner := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + publisher := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunPublisherContainer) + if planner.Image != "example.com/dynamo-planner:1.2.3" || publisher.Image != "example.com/dynamo-operator:1.2.3" { + t.Fatalf("Job images = %q/%q, want frozen images", planner.Image, publisher.Image) + } + wantSecrets := []corev1.LocalObjectReference{{Name: "planner-secret"}, {Name: "shared-secret"}, {Name: "publisher-secret"}} + if !reflect.DeepEqual(job.Spec.Template.Spec.ImagePullSecrets, wantSecrets) { + t.Fatalf("imagePullSecrets = %v, want %v", job.Spec.Template.Spec.ImagePullSecrets, wantSecrets) + } +} + +func TestV1beta2RunControllerFailsClosedWithoutExecutionConfig(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + + t.Log("Delete the immutable execution artifact while retaining the Job") + job := f.job() + configMap := &corev1.ConfigMap{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + if err := f.client.Delete(t.Context(), configMap); err != nil { + t.Fatalf("delete ConfigMap: %v", err) + } + + t.Log("Verify a real deletion fails closed instead of reconstructing or hot-looping") + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err == nil || !strings.Contains(err.Error(), "immutable execution ConfigMap") { + t.Fatalf("reconcile error = %v, want missing immutable execution ConfigMap", err) + } +} + +func TestV1beta2RunControllerWaitsForExecutionConfigCacheLag(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + + job := f.job() + configMap := &corev1.ConfigMap{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + directReader := fake.NewClientBuilder().WithScheme(f.client.Scheme()).WithObjects(configMap.DeepCopy()).Build() + if err := f.client.Delete(t.Context(), configMap); err != nil { + t.Fatalf("remove ConfigMap from simulated cache: %v", err) + } + f.reconciler.APIReader = directReader + + t.Log("Wait only when the uncached API confirms that the immutable artifact still exists") + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("reconcile result/error = %+v/%v, want bounded cache-lag retry", result, err) + } +} + +func TestV1beta2RunControllerRejectsJobExecutionSkew(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + + t.Log("Mutate the observed Job away from the immutable execution identity") + job := f.job() + container := &job.Spec.Template.Spec.Containers[0] + container.Image = "example.com/dynamo-planner:other" + if err := f.client.Update(t.Context(), job); err != nil { + t.Fatalf("update Job: %v", err) + } + + t.Log("Verify reconciliation detects the inconsistency instead of accepting skew") + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err == nil || !strings.Contains(err.Error(), "planner image identities differ") { + t.Fatalf("reconcile error = %v, want planner identity mismatch", err) + } +} + +func TestV1beta2RunControllerReportsImagePullSecretErrors(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f.reconciler.DockerSecretRetriever = &fixedRunSecrets{err: errors.New("registry index unavailable")} + + t.Log("Verify execution is not frozen without resolving its image credentials") + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err == nil || !strings.Contains(err.Error(), "registry index unavailable") { + t.Fatalf("reconcile error = %v, want image pull secret error", err) + } +} + +func TestV1beta2RunControllerKeepsPlannerDigestButDerivesRuntimesFromReleaseTag(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f.reconciler.PlannerImage = "example.com/dynamo-planner@sha256:0123" + f.reconciler.DynamoVersion = "1.7.0" + + t.Log("Reconcile a digest-pinned Planner through the frozen execution artifact") + f.reconcile() + + t.Log("Keep the Planner digest while giving sibling runtime derivation a shared release tag") + job := f.job() + planner := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + if planner.Image != "example.com/dynamo-planner@sha256:0123" { + t.Fatalf("planner image = %q, want configured digest", planner.Image) + } + configMap := &corev1.ConfigMap{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + if got := configMap.Data[dgdrRunRuntimeImageSourceKey]; got != "example.com/dynamo-planner:1.7.0" { + t.Fatalf("runtime image source = %q, want shared release tag", got) + } + var input map[string]any + if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &input); err != nil { + t.Fatalf("decode input: %v", err) + } + output := input["dgdr_run"].(map[string]any) + if output["runtime_image"] != "example.com/dynamo-planner:1.7.0" || output["runtime_version_override"] != "1.7.0" { + t.Fatalf("dgdr_run runtime identity = %v", output) + } +} + func assertDGDRRunJobIsolation( t *testing.T, job *batchv1.Job, @@ -187,7 +347,7 @@ func assertDGDRRunJobIsolation( "recommend", "--stack", "dynamo", "--config", dgdrRunInputPath, "--output", "dgdr_run", - "--output-dir", "/work/results", + "--output-dir", dgdrRunSnapshotPath, "--overwrite", }; !reflect.DeepEqual(sweeper.Args, want) { t.Fatalf("sweeper args = %q, want %q", sweeper.Args, want) @@ -195,15 +355,26 @@ func assertDGDRRunJobIsolation( if publisher.Image != operatorImage || publisher.ImagePullPolicy != corev1.PullAlways { t.Fatalf("publisher image = %q/%s, want configured operator image/Always", publisher.Image, publisher.ImagePullPolicy) } + if want := []string{dgdrRunPublisherBinary}; !reflect.DeepEqual(publisher.Command, want) { + t.Fatalf("publisher command = %q, want %q", publisher.Command, want) + } + if want := []string{ + "--snapshot-dir", dgdrRunSnapshotPath, + "--run-name", run.Name, + "--run-uid", string(run.UID), + "--sweeper-container", dgdrRunSweeperContainer, + }; !reflect.DeepEqual(publisher.Args, want) { + t.Fatalf("publisher args = %q, want %q", publisher.Args, want) + } if hasMount(sweeper.VolumeMounts, dgdrRunServiceAccountVolume) { t.Fatal("sweeper unexpectedly receives the Kubernetes service-account token") } if !hasMount(publisher.VolumeMounts, dgdrRunServiceAccountVolume) { t.Fatal("publisher does not receive its projected Kubernetes credentials") } - if !hasMountAt(sweeper.VolumeMounts, dgdrRunSnapshotVolume, "/work/results") || - !hasMountAt(publisher.VolumeMounts, dgdrRunSnapshotVolume, "/work/results") { - t.Fatal("sweeper and publisher do not share the parameter-overridden snapshot directory") + if !hasMountAt(sweeper.VolumeMounts, dgdrRunSnapshotVolume, dgdrRunSnapshotPath) || + !hasMountAt(publisher.VolumeMounts, dgdrRunSnapshotVolume, dgdrRunSnapshotPath) { + t.Fatal("sweeper and publisher do not share the frozen snapshot directory") } } @@ -243,16 +414,33 @@ func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardwar if configMap.Immutable == nil || !*configMap.Immutable { t.Fatal("input ConfigMap is not immutable") } + for key, want := range map[string]string{ + dgdrRunPlannerImageKey: "example.com/dynamo-planner:1.2.3", + dgdrRunPlannerPullPolicyKey: string(corev1.PullIfNotPresent), + dgdrRunRuntimeImageSourceKey: "example.com/dynamo-planner:1.2.3", + dgdrRunPublisherImageKey: "example.com/dynamo-operator:1.2.3", + dgdrRunPublisherPullPolicyKey: string(corev1.PullAlways), + dgdrRunDynamoVersionKey: "1.2.3", + dgdrRunGPUsPerNodeKey: "8", + dgdrRunSnapshotPathKey: dgdrRunSnapshotPath, + } { + if got := configMap.Data[key]; got != want { + t.Fatalf("ConfigMap data[%q] = %q, want %q", key, got, want) + } + } var input map[string]any if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &input); err != nil { t.Fatalf("decode input ConfigMap: %v", err) } output := input["dgdr_run"].(map[string]any) - if output["name"] != "native-output-name" || output["snapshot_dir"] != "/work/results" { - t.Fatalf("native parameter overrides were not preserved: %v", output) + if output["name"] != "search-1" || output["max_candidates"] != float64(2) { + t.Fatalf("operator-owned identity and candidate cap were not restored: %v", output) + } + if output["snapshot_dir"] != dgdrRunSnapshotPath || output["namespace"] != "default" { + t.Fatalf("operator-owned transport overrides were not restored: %v", output) } - if output["max_candidates"] != float64(7) || output["runtime_image"] != "example.com/custom-vllm-runtime:1.2.3" { - t.Fatalf("native parameter overrides were not preserved: %v", output) + if output["runtime_image"] != "example.com/dynamo-planner:1.2.3" || output["runtime_version_override"] != "1.2.3" { + t.Fatalf("operator-owned runtime identity was not restored: %v", output) } if output["derive_runtime_images"] != true { t.Fatalf("derive_runtime_images = %v, want true for a backend search", output["derive_runtime_images"]) @@ -275,31 +463,43 @@ func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { plannerImage string dynamoVersion string wantOverride string + wantRuntime string wantErr bool }{ - {name: "canonical planner tag", plannerImage: "example.com/dynamo-planner:1.7.0"}, - {name: "opaque CI tag", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7.0", wantOverride: "1.7.0"}, + {name: "canonical planner tag", plannerImage: "example.com/dynamo-planner:1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0"}, + {name: "v-prefixed planner tag", plannerImage: "example.com/dynamo-planner:v1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:v1.7.0"}, + {name: "prerelease planner tag", plannerImage: "example.com/dynamo-planner:1.7.0-rc.1", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0-rc.1"}, + {name: "build metadata planner tag", plannerImage: "example.com/dynamo-planner:1.7.0+build.5", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0+build.5"}, + {name: "tag and digest", plannerImage: "example.com/dynamo-planner:v1.7.0-rc.1@sha256:0123", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:v1.7.0-rc.1@sha256:0123"}, + {name: "digest only", plannerImage: "example.com/dynamo-planner@sha256:0123", dynamoVersion: "1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0"}, + {name: "opaque CI tag", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:git-sha"}, + {name: "untagged planner image", plannerImage: "example.com/dynamo-planner", dynamoVersion: "1.7.0", wantErr: true}, {name: "opaque tag without canonical version", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7", wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - run := dgdrRunForTest(t, map[string]any{}) - - input, _, err := dgdrRunInput(run, tt.plannerImage, tt.dynamoVersion, 8) + version, err := dgdrRunDynamoVersion(tt.plannerImage, tt.dynamoVersion) + var runtimeImage string + if err == nil { + runtimeImage, err = dgdrRunRuntimeImageSource(tt.plannerImage, version) + } if (err != nil) != tt.wantErr { - t.Fatalf("lower input error = %v, wantErr %t", err, tt.wantErr) + t.Fatalf("resolve execution identity error = %v, wantErr %t", err, tt.wantErr) } if tt.wantErr { return } - output := input["dgdr_run"].(map[string]any) - got, found := output["runtime_version_override"] - if tt.wantOverride == "" && found { - t.Fatalf("dgdr_run.runtime_version_override = %v, want omitted", got) + input, err := dgdrRunInput(dgdrRunForTest(t, map[string]any{}), runtimeImage, version, 8) + if err != nil { + t.Fatalf("lower input: %v", err) } - if tt.wantOverride != "" && got != tt.wantOverride { + output := input["dgdr_run"].(map[string]any) + if got := output["runtime_version_override"]; got != tt.wantOverride { t.Fatalf("dgdr_run.runtime_version_override = %v, want %s", got, tt.wantOverride) } + if got := output["runtime_image"]; got != tt.wantRuntime { + t.Fatalf("dgdr_run.runtime_image = %v, want %s", got, tt.wantRuntime) + } }) } } @@ -444,6 +644,25 @@ func TestDGDRRunResourceNameIsStableAndBounded(t *testing.T) { } } +func TestRequireDGDRV1beta2RESTMappingsIncludesCandidates(t *testing.T) { + groupVersion := nvidiacomv1beta2.GroupVersion + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{groupVersion}) + mapper.Add(groupVersion.WithKind("DynamoGraphDeploymentRequest"), meta.RESTScopeNamespace) + mapper.Add(groupVersion.WithKind("DynamoGraphDeploymentRun"), meta.RESTScopeNamespace) + + t.Log("Verify startup fails when Candidate publication cannot succeed") + err := requireDGDRV1beta2RESTMappings(mapper) + if err == nil || !strings.Contains(err.Error(), "DynamoGraphDeploymentCandidate") { + t.Fatalf("preflight error = %v, want missing Candidate API", err) + } + + t.Log("Register the complete execution API and verify preflight succeeds") + mapper.Add(groupVersion.WithKind("DynamoGraphDeploymentCandidate"), meta.RESTScopeNamespace) + if err := requireDGDRV1beta2RESTMappings(mapper); err != nil { + t.Fatalf("complete API preflight: %v", err) + } +} + func containerNamed(t *testing.T, containers []corev1.Container, name string) corev1.Container { t.Helper() for _, container := range containers { diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index 839e6893273e..bbc4c243e62c 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -47,6 +47,7 @@ type DynamoGraphDeploymentRunSetupOptions struct { SetupOptions GPUDiscoveryCache *gpu.GPUDiscoveryCache GPUDiscovery *gpu.GPUDiscovery + DockerSecretRetriever DockerSecretRetriever PlannerImage string DynamoVersion string OperatorImage string @@ -186,6 +187,7 @@ func SetupDynamoGraphDeploymentRunV1beta2(mgr ctrl.Manager, opts DynamoGraphDepl return opts.RuntimeConfig != nil && opts.RuntimeConfig.Gate.Enabled(features.GPUDiscovery) }, }, + DockerSecretRetriever: opts.DockerSecretRetriever, PlannerImage: opts.PlannerImage, DynamoVersion: opts.DynamoVersion, OperatorImage: opts.OperatorImage, diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 065eeba5bb3c..f541c552b44e 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3545,14 +3545,12 @@ DynamoGraphDeploymentRun is one immutable accepted DGDR request and its executio -DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. -Run creators resolve optional Request inputs and the final AISimulate parameters before -persisting the Run. Request admission defaulting does not apply to Runs. - -On a Run, spec.aisimulate.parameters is the complete resolved native AISimulate input, frozen when -the Request controller created the Run. The Run executes that object as it is. It is not lowered -from the typed fields again: they are copies of the Request, and lowering them would bring back -keys that a JSON merge patch null had deleted from the stored input. +DynamoGraphDeploymentRunSpec is an immutable snapshot of an accepted DGDR spec. +Request admission resolves its structural defaults before the Request controller copies it; +Request admission defaulting does not apply to Runs. + +The Run reconciler lowers this immutable typed snapshot plus inputOverrides exactly once and +records the resulting public recommendation input in status.aisimulate.input. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 0650a197f4bc..014e90d48076 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4989,14 +4989,12 @@ See [DynamoGraphDeploymentRunStatus](#dynamographdeploymentrunstatus). -DynamoGraphDeploymentRunSpec is an immutable, resolved copy of an accepted DGDR spec. -Run creators resolve optional Request inputs and the final AISimulate parameters before -persisting the Run. Request admission defaulting does not apply to Runs. - -On a Run, spec.aisimulate.parameters is the complete resolved native AISimulate input, frozen when -the Request controller created the Run. The Run executes that object as it is. It is not lowered -from the typed fields again: they are copies of the Request, and lowering them would bring back -keys that a JSON merge patch null had deleted from the stored input. +DynamoGraphDeploymentRunSpec is an immutable snapshot of an accepted DGDR spec. +Request admission resolves its structural defaults before the Request controller copies it; +Request admission defaulting does not apply to Runs. + +The Run reconciler lowers this immutable typed snapshot plus inputOverrides exactly once and +records the resulting public recommendation input in status.aisimulate.input. **Kind:** `type` diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index d48db24cfbd2..d09aa90176a2 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -51,7 +51,7 @@ # owned by github.com/ai-dynamo/snapshot. "nvidia.com/v1alpha1": 71, "nvidia.com/v1beta1": 76, - "nvidia.com/v1beta2": 32, + "nvidia.com/v1beta2": 35, "operator.config.dynamo.nvidia.com/v1alpha1": 29, } EXPECTED_OPERATOR_DEFAULT_SECTIONS = ( From 3c2f3b69de05beb87d81f183ea3928612bf725c6 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 11:11:41 +0100 Subject: [PATCH 049/100] pr-17195: satisfy current Go lint Signed-off-by: Dr. Stefan Schimanski --- deploy/operator/cmd/main.go | 16 ++++- ...mographdeploymentrun_v1beta2_controller.go | 60 ++++++++----------- ...phdeploymentrun_v1beta2_controller_test.go | 25 ++++---- 3 files changed, 50 insertions(+), 51 deletions(-) diff --git a/deploy/operator/cmd/main.go b/deploy/operator/cmd/main.go index d7b8d4e98eea..cbdf1d15e961 100644 --- a/deploy/operator/cmd/main.go +++ b/deploy/operator/cmd/main.go @@ -169,8 +169,13 @@ func main() { ) flag.StringVar(&operatorImagePullPolicy, "operator-image-pull-policy", string(corev1.PullIfNotPresent), "Image pull policy for operator helper init containers") - flag.StringVar(&dgdrDefaultImage, "dgdr-default-image", "", - "Planner image used by v1beta2 Runs and as the v1beta1 spec.image default; empty derives dynamo-planner:") + flag.StringVar( + &dgdrDefaultImage, + "dgdr-default-image", + "", + "Planner image used by v1beta2 Runs and as the v1beta1 spec.image default; "+ + "empty derives dynamo-planner:", + ) flag.BoolVar(&dgdrV1beta2Enabled, "dgdr-v1beta2-enabled", false, "Reconcile native v1beta2 DGDRs; requires the v1beta2 Request and Run CRDs to be installed") opts := zap.Options{ @@ -199,7 +204,12 @@ func main() { "provided", operatorVersion, "error", err.Error()) os.Exit(1) } - dynamoVersion := fmt.Sprintf("%d.%d.%d", parsedOperatorVersion.Major(), parsedOperatorVersion.Minor(), parsedOperatorVersion.Patch()) + dynamoVersion := fmt.Sprintf( + "%d.%d.%d", + parsedOperatorVersion.Major(), + parsedOperatorVersion.Minor(), + parsedOperatorVersion.Patch(), + ) setupLog.Info("Operator version configured", "version", operatorVersion) pullPolicy := corev1.PullPolicy(operatorImagePullPolicy) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 2d684532851b..5a8003f7b8af 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -186,14 +186,14 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( run *nvidiacomv1beta2.DynamoGraphDeploymentRun, name string, ) error { - sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + sa := &corev1.ServiceAccount{Name: name, Namespace: run.Namespace} if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { return controllerutil.SetControllerReference(run, sa, r.Scheme()) }); err != nil { return fmt.Errorf("reconcile ServiceAccount %q: %w", name, err) } - role := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + role := &rbacv1.Role{Name: name, Namespace: run.Namespace} if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { role.Rules = dgdrRunPublisherRules(run.Name) return controllerutil.SetControllerReference(run, role, r.Scheme()) @@ -201,7 +201,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( return fmt.Errorf("reconcile Role %q: %w", name, err) } - binding := &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: run.Namespace}} + binding := &rbacv1.RoleBinding{Name: name, Namespace: run.Namespace} if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, binding, func() error { binding.RoleRef = rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: name} binding.Subjects = []rbacv1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: run.Namespace}} @@ -338,15 +338,13 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( } configMap = &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: run.Namespace, - Annotations: map[string]string{ - dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), - dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, - }, + Name: name, + Namespace: run.Namespace, + Annotations: map[string]string{ + dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, }, - Immutable: ptr.To(true), + Immutable: new(true), Data: map[string]string{ dgdrRunInputKey: string(contents), dgdrRunPlannerImageKey: execution.PlannerImage, @@ -453,16 +451,12 @@ func dgdrRunDynamoVersion(plannerImage, dynamoVersion string) (string, error) { func dgdrRunRuntimeImageSource(plannerImage, dynamoVersion string) (string, error) { ref := strings.TrimSpace(plannerImage) - digest := strings.Index(ref, "@") - nameAndTag := ref - if digest >= 0 { - nameAndTag = ref[:digest] - } + nameAndTag, _, hasDigest := strings.Cut(ref, "@") lastSlash, lastColon := strings.LastIndex(nameAndTag, "/"), strings.LastIndex(nameAndTag, ":") if lastColon > lastSlash && lastColon < len(nameAndTag)-1 { return ref, nil } - if digest >= 0 && nameAndTag != "" { + if hasDigest && nameAndTag != "" { return nameAndTag + ":" + dynamoVersion, nil } return "", fmt.Errorf( @@ -614,20 +608,18 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( execution dgdrRunExecution, ) *batchv1.Job { job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: run.Namespace, - Annotations: map[string]string{ - dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), - dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, - }, + Name: name, + Namespace: run.Namespace, + Annotations: map[string]string{ + dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), + dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, }, Spec: batchv1.JobSpec{ BackoffLimit: ptr.To[int32](0), Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ ServiceAccountName: name, - AutomountServiceAccountToken: ptr.To(false), + AutomountServiceAccountToken: new(false), RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{FSGroup: ptr.To[int64](65532)}, ImagePullSecrets: append([]corev1.LocalObjectReference(nil), execution.ImagePullSecrets...), @@ -673,11 +665,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( Volumes: []corev1.Volume{ { Name: dgdrRunInputVolume, - VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{Name: name}, - }}, + ConfigMap: &corev1.ConfigMapVolumeSource{ + Name: name, + }, }, - {Name: dgdrRunSnapshotVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: dgdrRunSnapshotVolume, EmptyDir: &corev1.EmptyDirVolumeSource{}}, dgdrRunPublisherKubeAPIVolume(), }, }, @@ -690,20 +682,20 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( func dgdrRunPublisherKubeAPIVolume() corev1.Volume { return corev1.Volume{ Name: dgdrRunServiceAccountVolume, - VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ Sources: []corev1.VolumeProjection{ {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Path: "token", ExpirationSeconds: ptr.To(dgdrRunServiceAccountTokenTTL), + Path: "token", ExpirationSeconds: new(dgdrRunServiceAccountTokenTTL), }}, {ConfigMap: &corev1.ConfigMapProjection{ - LocalObjectReference: corev1.LocalObjectReference{Name: ConfigMapNameKubeRootCA}, - Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}, + Name: ConfigMapNameKubeRootCA, + Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}, }}, {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{ Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}, }}}}, }, - }}, + }, } } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 06ac076a21d1..01e890d677ff 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -97,9 +97,8 @@ func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects func (f *dgdrRunFixture) reconcile() { f.t.Helper() - if _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{NamespacedName: types.NamespacedName{ - Namespace: "default", Name: "search-1", - }}); err != nil { + if _, err := f.reconciler.Reconcile(f.t.Context(), ctrl.Request{ + Namespace: "default", Name: "search-1"}); err != nil { f.t.Fatalf("reconcile: %v", err) } } @@ -132,11 +131,11 @@ func dgdrRunForTest(t *testing.T, inputOverrides map[string]any) *nvidiacomv1bet t.Fatalf("marshal parameters: %v", err) } return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ - ObjectMeta: metav1.ObjectMeta{Name: "search-1", Namespace: "default", UID: "run-uid", Generation: 1}, + Name: "search-1", Namespace: "default", UID: "run-uid", Generation: 1, Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ - SKU: ptr.To("h200_sxm"), Budget: ptr.To[int32](16), + SKU: new("h200_sxm"), Budget: ptr.To[int32](16), }}, AISimulate: &nvidiacomv1beta2.AISimulateSpec{InputOverrides: &runtime.RawExtension{Raw: raw}}, Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 2}, @@ -590,15 +589,13 @@ func TestV1beta2RunControllerClassifiesPublisherProtocolFailure(t *testing.T) { f.reconcile() job := f.job() pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: job.Name + "-pod", - Namespace: job.Namespace, - Labels: map[string]string{batchv1.JobNameLabel: job.Name}, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", Name: job.Name, UID: job.UID, - Controller: ptr.To(true), - }}, - }, + Name: job.Name + "-pod", + Namespace: job.Namespace, + Labels: map[string]string{batchv1.JobNameLabel: job.Name}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", Name: job.Name, UID: job.UID, + Controller: new(true), + }}, } if err := f.client.Create(t.Context(), pod); err != nil { t.Fatalf("create Job pod: %v", err) From c79b97f936cb7c3ba771e65ef017a981842e1c7b Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 17:48:02 +0100 Subject: [PATCH 050/100] pr-17195: complete Run execution boundaries Signed-off-by: Dr. Stefan Schimanski --- .../components/operator/files/role.yaml | 10 +- ...mographdeploymentrun_v1beta2_controller.go | 14 ++- ...mentrun_v1beta2_controller_envtest_test.go | 103 ++++++++++++++++++ ...phdeploymentrun_v1beta2_controller_test.go | 13 +++ 4 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index f4420c3b53e8..039c9f9dc0c7 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -312,6 +312,7 @@ rules: - nvidia.com resources: - dynamocomponentdeployments/status + - dynamographdeploymentcandidates/status - dynamographdeploymentrequests/status - dynamographdeploymentruns/status - dynamographdeployments/status @@ -325,7 +326,8 @@ rules: - apiGroups: - nvidia.com resources: - - dynamographdeploymentruns + - dynamographdeploymentcandidates + - snapshotjobs verbs: - create - delete @@ -336,8 +338,9 @@ rules: - apiGroups: - nvidia.com resources: - - podsnapshots + - dynamographdeploymentruns verbs: + - create - delete - get - list @@ -346,9 +349,8 @@ rules: - apiGroups: - nvidia.com resources: - - snapshotjobs + - podsnapshots verbs: - - create - delete - get - list diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 5a8003f7b8af..fb8974e87416 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -107,6 +107,8 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates,verbs=get;list;watch;create;patch;delete +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates/status,verbs=get;update;patch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch @@ -576,17 +578,17 @@ func dgdrRunContainer(containers []corev1.Container, name string) (corev1.Contai } func dgdrRunStatusInput(contents string) (*runtime.RawExtension, error) { - var input map[string]any - if err := yaml.Unmarshal([]byte(contents), &input); err != nil { + raw, err := yaml.YAMLToJSON([]byte(contents)) + if err != nil { + return nil, fmt.Errorf("decode AISimulate input: %w", err) + } + var input map[string]json.RawMessage + if err := json.Unmarshal(raw, &input); err != nil { return nil, fmt.Errorf("decode AISimulate input: %w", err) } if input == nil { return nil, errors.New("decode AISimulate input: expected an object") } - raw, err := json.Marshal(input) - if err != nil { - return nil, fmt.Errorf("encode AISimulate status input: %w", err) - } return &runtime.RawExtension{Raw: raw}, nil } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go new file mode 100644 index 000000000000..d10bf30673d1 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go @@ -0,0 +1,103 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "errors" + "testing" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func TestDGDRRunPublisherRoleDoesNotEscalateManagerPermissions(t *testing.T) { + env := sharedEnv.ForTest(t) + admin := env.Client() + ctx := t.Context() + const managerName = "dgdr-run-manager" + + t.Log("Create a persisted Run and a restricted identity with every delegated permission except Candidate access") + run := dgdrRunForTest(t, map[string]any{}) + run.Name, run.Namespace, run.UID, run.Generation = "rbac-run", env.Namespace(), "", 0 + if err := admin.Create(ctx, run); err != nil { + t.Fatalf("create Run: %v", err) + } + if err := admin.Get(ctx, client.ObjectKeyFromObject(run), run); err != nil { + t.Fatalf("get Run: %v", err) + } + + managerRules := []rbacv1.PolicyRule{ + {APIGroups: []string{""}, Resources: []string{"serviceaccounts"}, Verbs: []string{"get", "list", "watch", "create", "update", "patch"}}, + {APIGroups: []string{rbacv1.GroupName}, Resources: []string{"roles", "rolebindings"}, Verbs: []string{"get", "list", "watch", "create", "update", "patch"}}, + } + for _, rule := range dgdrRunPublisherRules(run.Name) { + if len(rule.Resources) == 1 && (rule.Resources[0] == "dynamographdeploymentcandidates" || rule.Resources[0] == "dynamographdeploymentcandidates/status") { + continue + } + managerRules = append(managerRules, rule) + } + managerRole := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: managerName, Namespace: run.Namespace}, Rules: managerRules} + if err := admin.Create(ctx, managerRole); err != nil { + t.Fatalf("create manager Role: %v", err) + } + managerBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: managerName, Namespace: run.Namespace}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: managerRole.Name}, + Subjects: []rbacv1.Subject{{APIGroup: rbacv1.GroupName, Kind: rbacv1.UserKind, Name: managerName}}, + } + if err := admin.Create(ctx, managerBinding); err != nil { + t.Fatalf("create manager RoleBinding: %v", err) + } + managerConfig, err := env.AddUser(envtest.User{Name: managerName, Groups: []string{"system:authenticated"}}) + if err != nil { + t.Fatalf("create restricted manager identity: %v", err) + } + managerClient, err := client.New(managerConfig, client.Options{Scheme: env.Scheme()}) + if err != nil { + t.Fatalf("create restricted manager client: %v", err) + } + reconciler := &DynamoGraphDeploymentRunV1beta2Reconciler{Client: managerClient} + + t.Log("Kubernetes rejects the publisher Role while the manager lacks its Candidate permissions") + err = reconciler.reconcilePublisherAccess(ctx, run, dgdrRunResourceName(run.Name)) + var statusErr *apierrors.StatusError + if !errors.As(err, &statusErr) || statusErr.ErrStatus.Reason != metav1.StatusReasonForbidden { + t.Fatalf("reconcile publisher access error = %v, want RBAC escalation rejection", err) + } + + t.Log("Grant the manager every delegated permission and retry the same reconciliation") + managerRole.Rules = append(append([]rbacv1.PolicyRule{}, managerRules[:2]...), dgdrRunPublisherRules(run.Name)...) + if err := admin.Update(ctx, managerRole); err != nil { + t.Fatalf("grant publisher permissions to manager: %v", err) + } + if err := reconciler.reconcilePublisherAccess(ctx, run, dgdrRunResourceName(run.Name)); err != nil { + t.Fatalf("reconcile publisher access: %v", err) + } + + t.Log("The restricted manager created the complete publisher access bundle") + for _, object := range []client.Object{&corev1.ServiceAccount{}, &rbacv1.Role{}, &rbacv1.RoleBinding{}} { + object.SetName(dgdrRunResourceName(run.Name)) + object.SetNamespace(run.Namespace) + if err := admin.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + t.Fatalf("get %T: %v", object, err) + } + } + createdRole := &rbacv1.Role{} + key := client.ObjectKey{Namespace: run.Namespace, Name: dgdrRunResourceName(run.Name)} + if err := admin.Get(ctx, key, createdRole); err != nil { + t.Fatalf("get publisher Role: %v", err) + } + if want := dgdrRunPublisherRules(run.Name); !apiequality.Semantic.DeepEqual(createdRole.Rules, want) { + t.Fatalf("publisher Role rules = %#v, want %#v", createdRole.Rules, want) + } +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 01e890d677ff..2ad1f15cc49a 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -404,6 +404,19 @@ func assertDGDRRunStatusInputMatchesConfigMap( } } +func TestDGDRRunStatusInputPreservesLargeIntegers(t *testing.T) { + const seed = "9007199254740993" + + t.Log("Project the frozen YAML input into status without decoding numbers through float64") + input, err := dgdrRunStatusInput("optimizer:\n seed: " + seed + "\n") + if err != nil { + t.Fatalf("project status input: %v", err) + } + if got := string(input.Raw); !strings.Contains(got, `"seed":`+seed) { + t.Fatalf("status input lost integer precision: %s", got) + } +} + func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardware *fixedRunHardware) { t.Helper() configMap := &corev1.ConfigMap{} From 8f6bad5c72d4df41213d093f6b339a54dc95396c Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 14:07:19 +0100 Subject: [PATCH 051/100] pr-17195: grant Run owner-reference permissions Signed-off-by: Dr. Stefan Schimanski --- .../platform/components/operator/files/role.yaml | 1 + ...dynamographdeploymentrun_v1beta2_controller.go | 7 +++++++ ...ploymentrun_v1beta2_controller_envtest_test.go | 15 +++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 039c9f9dc0c7..7b5dfd770a62 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -303,6 +303,7 @@ rules: resources: - dynamocomponentdeployments/finalizers - dynamographdeploymentrequests/finalizers + - dynamographdeploymentruns/finalizers - dynamographdeployments/finalizers - dynamomodels/finalizers - lpxgraphdeployments/finalizers diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index fb8974e87416..7e45c897ca0b 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -106,6 +106,7 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { } // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns,verbs=get;list;watch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns/finalizers,verbs=update // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns/status,verbs=get;update;patch // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates,verbs=get;list;watch;create;patch;delete // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates/status,verbs=get;update;patch @@ -222,6 +223,12 @@ func dgdrRunPublisherRules(runName string) []rbacv1.PolicyRule { ResourceNames: []string{runName}, Verbs: []string{"get", "list", "watch"}, }, + { + APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, + Resources: []string{"dynamographdeploymentruns/finalizers"}, + ResourceNames: []string{runName}, + Verbs: []string{"update"}, + }, { APIGroups: []string{nvidiacomv1beta2.GroupVersion.Group}, Resources: []string{"dynamographdeploymentruns/status"}, diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go index d10bf30673d1..f8d2a574adca 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go @@ -100,4 +100,19 @@ func TestDGDRRunPublisherRoleDoesNotEscalateManagerPermissions(t *testing.T) { if want := dgdrRunPublisherRules(run.Name); !apiequality.Semantic.DeepEqual(createdRole.Rules, want) { t.Fatalf("publisher Role rules = %#v, want %#v", createdRole.Rules, want) } + if !roleAllowsRunFinalizerUpdate(createdRole.Rules, run.Name) { + t.Fatalf("publisher Role does not allow updating finalizers of Run %q: %#v", run.Name, createdRole.Rules) + } +} + +func roleAllowsRunFinalizerUpdate(rules []rbacv1.PolicyRule, runName string) bool { + for _, rule := range rules { + if len(rule.APIGroups) == 1 && rule.APIGroups[0] == "nvidia.com" && + len(rule.Resources) == 1 && rule.Resources[0] == "dynamographdeploymentruns/finalizers" && + len(rule.ResourceNames) == 1 && rule.ResourceNames[0] == runName && + len(rule.Verbs) == 1 && rule.Verbs[0] == "update" { + return true + } + } + return false } From 078f9ec049495903470e782e0994ec83a9fa5177 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 15:37:42 +0100 Subject: [PATCH 052/100] pr-17195: simplify Run RBAC envtest literals Signed-off-by: Dr. Stefan Schimanski --- ...graphdeploymentrun_v1beta2_controller_envtest_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go index f8d2a574adca..c80688bbec30 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go @@ -46,14 +46,15 @@ func TestDGDRRunPublisherRoleDoesNotEscalateManagerPermissions(t *testing.T) { } managerRules = append(managerRules, rule) } - managerRole := &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: managerName, Namespace: run.Namespace}, Rules: managerRules} + managerRole := &rbacv1.Role{Name: managerName, Namespace: run.Namespace, Rules: managerRules} if err := admin.Create(ctx, managerRole); err != nil { t.Fatalf("create manager Role: %v", err) } managerBinding := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{Name: managerName, Namespace: run.Namespace}, - RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: managerRole.Name}, - Subjects: []rbacv1.Subject{{APIGroup: rbacv1.GroupName, Kind: rbacv1.UserKind, Name: managerName}}, + Name: managerName, + Namespace: run.Namespace, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: managerRole.Name}, + Subjects: []rbacv1.Subject{{APIGroup: rbacv1.GroupName, Kind: rbacv1.UserKind, Name: managerName}}, } if err := admin.Create(ctx, managerBinding); err != nil { t.Fatalf("create manager RoleBinding: %v", err) From cd9f3fa39b793caccb2e2d84d15d596f8f4582da Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 17:44:17 +0100 Subject: [PATCH 053/100] pr-17195: harden immutable Run execution Signed-off-by: Dr. Stefan Schimanski --- .../components/operator/files/role.yaml | 12 +- .../v1beta2/dynamographdeploymentrun_types.go | 2 +- .../nvidia.com_dynamographdeploymentruns.yaml | 4 +- ...mographdeploymentrun_v1beta2_controller.go | 136 +++++++++++----- ...phdeploymentrun_v1beta2_controller_test.go | 147 +++++++++++++++--- .../dgdrrunpublisher/writer_envtest_test.go | 14 +- ...ploymentrequest_validation_envtest_test.go | 31 +++- 7 files changed, 264 insertions(+), 82 deletions(-) diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 7b5dfd770a62..596bbc21f5bf 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -328,18 +328,8 @@ rules: - nvidia.com resources: - dynamographdeploymentcandidates - - snapshotjobs - verbs: - - create - - delete - - get - - list - - patch - - watch -- apiGroups: - - nvidia.com - resources: - dynamographdeploymentruns + - snapshotjobs verbs: - create - delete diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go index 7a07c15eecae..1ca939cb6349 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go @@ -27,7 +27,7 @@ import ( // DynamoGraphDeploymentRunSpec is an immutable snapshot of an accepted DGDR spec. // Request admission resolves its structural defaults before the Request controller copies it; // Request admission defaulting does not apply to Runs. -// +kubebuilder:validation:XValidation:rule="has(self.model)",message="model is required" +// +kubebuilder:validation:XValidation:rule="has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.workload) && has(self.workload.static) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0",message="spec must be a fully resolved Request admission snapshot" // // The Run reconciler lowers this immutable typed snapshot plus inputOverrides exactly once and // records the resulting public recommendation input in status.aisimulate.input. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index c057b554069a..54396dd06bfd 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -324,8 +324,8 @@ spec: type: object type: object x-kubernetes-validations: - - message: model is required - rule: has(self.model) + - message: spec must be a fully resolved Request admission snapshot + rule: has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.workload) && has(self.workload.static) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0 status: description: DynamoGraphDeploymentRunStatus represents one search execution's observed state. properties: diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 7e45c897ca0b..4a409483e7a6 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -320,7 +320,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( if err != nil { return dgdrRunExecution{}, err } - execution.RuntimeImageSource, err = dgdrRunRuntimeImageSource(execution.PlannerImage, execution.DynamoVersion) + execution.RuntimeImageSource, err = dgdrRunRuntimeImageSource(execution.PlannerImage) if err != nil { return dgdrRunExecution{}, err } @@ -386,10 +386,21 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( if run.Spec.Hardware == nil || run.Spec.Hardware.GPU == nil || run.Spec.Hardware.GPU.SKU == nil { return 0, errors.New("discover GPUs per node: spec.hardware.gpu.sku was not resolved by Request admission") } - info, err := r.Hardware.Discover(ctx, nvidiacomv1beta1.GPUSKUType(*run.Spec.Hardware.GPU.SKU)) + sku := *run.Spec.Hardware.GPU.SKU + filterSKU := nvidiacomv1beta1.GPUSKUType("") + if aisimulate.IsKnownHardwareSKU(sku) { + filterSKU = nvidiacomv1beta1.GPUSKUType(sku) + } + info, err := r.Hardware.Discover(ctx, filterSKU) if err != nil { return 0, fmt.Errorf("discover GPUs per node: %w", err) } + if filterSKU == "" && info != nil && len(info.OtherModels) > 0 { + return 0, fmt.Errorf( + "discover GPUs per node for external AISimulate SKU %q: cluster discovery selected %q among multiple GPU models (%s)", + sku, info.Model, strings.Join(info.OtherModels, ", "), + ) + } if info == nil || info.GPUsPerNode < 1 { return 0, errors.New("discover GPUs per node: discovery returned no positive GPU count") } @@ -458,18 +469,21 @@ func dgdrRunDynamoVersion(plannerImage, dynamoVersion string) (string, error) { return version.String(), nil } -func dgdrRunRuntimeImageSource(plannerImage, dynamoVersion string) (string, error) { +func dgdrRunRuntimeImageSource(plannerImage string) (string, error) { ref := strings.TrimSpace(plannerImage) nameAndTag, _, hasDigest := strings.Cut(ref, "@") lastSlash, lastColon := strings.LastIndex(nameAndTag, "/"), strings.LastIndex(nameAndTag, ":") if lastColon > lastSlash && lastColon < len(nameAndTag)-1 { return ref, nil } - if hasDigest && nameAndTag != "" { - return nameAndTag + ":" + dynamoVersion, nil + if hasDigest { + return "", fmt.Errorf( + "resolve runtime images for planner image %q: a digest-only planner image cannot identify sibling runtime images; configure a shared release tag, optionally with a digest", + plannerImage, + ) } return "", fmt.Errorf( - "resolve runtime images for planner image %q: a shared release tag or repository-specific digest is required", + "resolve runtime images for planner image %q: a shared release tag is required, optionally with a repository-specific digest", plannerImage, ) } @@ -630,8 +644,14 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( ServiceAccountName: name, AutomountServiceAccountToken: new(false), RestartPolicy: corev1.RestartPolicyNever, - SecurityContext: &corev1.PodSecurityContext{FSGroup: ptr.To[int64](65532)}, - ImagePullSecrets: append([]corev1.LocalObjectReference(nil), execution.ImagePullSecrets...), + SecurityContext: &corev1.PodSecurityContext{ + FSGroup: ptr.To[int64](65532), + RunAsNonRoot: new(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + ImagePullSecrets: append([]corev1.LocalObjectReference(nil), execution.ImagePullSecrets...), Containers: []corev1.Container{ { Name: dgdrRunSweeperContainer, @@ -645,6 +665,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( "--output-dir", execution.SnapshotPath, "--overwrite", }, + SecurityContext: dgdrRunContainerSecurityContext(), VolumeMounts: []corev1.VolumeMount{ {Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true}, {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, @@ -665,6 +686,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, }, + SecurityContext: dgdrRunContainerSecurityContext(), VolumeMounts: []corev1.VolumeMount{ {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, {Name: dgdrRunServiceAccountVolume, MountPath: dgdrRunServiceAccountMountPath, ReadOnly: true}, @@ -688,6 +710,15 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( return job } +func dgdrRunContainerSecurityContext() *corev1.SecurityContext { + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: new(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + } +} + func dgdrRunPublisherKubeAPIVolume() corev1.Volume { return corev1.Volume{ Name: dgdrRunServiceAccountVolume, @@ -768,38 +799,22 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) dgdrRunFailure( log.FromContext(ctx).Error(err, "Could not inspect failed AISimulate Job pods", "job", job.Name) return dgdrRunReasonJobFailed, condition.Message } - for i := range pods.Items { - pod := &pods.Items[i] - if !metav1.IsControlledBy(pod, job) { - continue - } - for _, status := range pod.Status.ContainerStatuses { - terminated := status.State.Terminated - if terminated == nil || terminated.ExitCode == 0 { - continue - } - message := terminated.Message - if message == "" { - message = terminated.Reason - } - if message == "" { - message = fmt.Sprintf("container %s exited with code %d", status.Name, terminated.ExitCode) - } - if status.Name == dgdrRunPublisherContainer { - switch int(terminated.ExitCode) { - case dgdrrunpublisher.ExitRunFailed: - return dgdrRunReasonSearchFailed, message - case dgdrrunpublisher.ExitProtocolViolation: - return dgdrRunReasonProtocolViolation, message - case dgdrrunpublisher.ExitReconcileFailed: - return dgdrRunReasonPublicationFailed, message - } - } - if status.Name == dgdrRunSweeperContainer { - return dgdrRunReasonSearchFailed, message - } + if terminated := dgdrRunFailedContainer(pods.Items, job, dgdrRunPublisherContainer); terminated != nil { + message := dgdrRunPublisherFailureMessage(terminated) + switch int(terminated.ExitCode) { + case dgdrrunpublisher.ExitRunFailed: + return dgdrRunReasonSearchFailed, message + case dgdrrunpublisher.ExitProtocolViolation: + return dgdrRunReasonProtocolViolation, message + case dgdrrunpublisher.ExitReconcileFailed: + return dgdrRunReasonPublicationFailed, message + default: + return dgdrRunReasonJobFailed, message } } + if terminated := dgdrRunFailedContainer(pods.Items, job, dgdrRunSweeperContainer); terminated != nil { + return dgdrRunReasonSearchFailed, dgdrRunFailureMessage(dgdrRunSweeperContainer, terminated) + } message := condition.Message if message == "" { message = "AISimulate Job failed" @@ -807,6 +822,51 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) dgdrRunFailure( return dgdrRunReasonJobFailed, message } +func dgdrRunFailedContainer( + pods []corev1.Pod, + job *batchv1.Job, + containerName string, +) *corev1.ContainerStateTerminated { + for i := range pods { + if !metav1.IsControlledBy(&pods[i], job) { + continue + } + for j := range pods[i].Status.ContainerStatuses { + status := &pods[i].Status.ContainerStatuses[j] + if status.Name == containerName && status.State.Terminated != nil && status.State.Terminated.ExitCode != 0 { + return status.State.Terminated + } + } + } + return nil +} + +func dgdrRunPublisherFailureMessage(terminated *corev1.ContainerStateTerminated) string { + var outcome dgdrrunpublisher.TerminalOutcome + if err := json.Unmarshal([]byte(terminated.Message), &outcome); err == nil { + parts := make([]string, 0, 2) + for _, part := range []string{outcome.Message, outcome.Error} { + if part = strings.TrimSpace(part); part != "" { + parts = append(parts, part) + } + } + if len(parts) > 0 { + return strings.Join(parts, ": ") + } + } + return dgdrRunFailureMessage(dgdrRunPublisherContainer, terminated) +} + +func dgdrRunFailureMessage(containerName string, terminated *corev1.ContainerStateTerminated) string { + if terminated.Message != "" { + return terminated.Message + } + if terminated.Reason != "" { + return terminated.Reason + } + return fmt.Sprintf("container %s exited with code %d", containerName, terminated.ExitCode) +} + func dgdrRunCompletionTime(job *batchv1.Job) *metav1.Time { if job.Status.CompletionTime != nil { return job.Status.CompletionTime.DeepCopy() diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 2ad1f15cc49a..9a9be00abaf9 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -130,13 +130,19 @@ func dgdrRunForTest(t *testing.T, inputOverrides map[string]any) *nvidiacomv1bet if err != nil { t.Fatalf("marshal parameters: %v", err) } + metric := nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ Name: "search-1", Namespace: "default", UID: "run-uid", Generation: 1, Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ - Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, + Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM}, Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ SKU: new("h200_sxm"), Budget: ptr.To[int32](16), }}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, Metric: &metric, SLA: &nvidiacomv1beta2.ObjectiveSLASpec{}, + }, AISimulate: &nvidiacomv1beta2.AISimulateSpec{InputOverrides: &runtime.RawExtension{Raw: raw}}, Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 2}, }}, @@ -170,6 +176,36 @@ func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { assertDGDRRunStatusInputMatchesConfigMap(t, f.client, stored, job) } +func TestV1beta2RunControllerUsesUnfilteredTopologyForAnExternalHardwareSKU(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{Model: "Future Accelerator", GPUsPerNode: 8}} + run := dgdrRunForTest(t, map[string]any{}) + *run.Spec.Hardware.GPU.SKU = "future_accelerator" + f := newDGDRRunFixture(t, hardware, run) + + f.reconcile() + + if len(hardware.calls) != 1 || hardware.calls[0] != "" { + t.Fatalf("hardware discovery filters = %v, want one unfiltered topology lookup", hardware.calls) + } + if got := f.job().Spec.Template.Spec.Containers[0].Image; got != f.reconciler.PlannerImage { + t.Fatalf("sweeper image = %q, want %q", got, f.reconciler.PlannerImage) + } +} + +func TestV1beta2RunControllerRejectsAmbiguousTopologyForAnExternalHardwareSKU(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{ + Model: "Future Accelerator", GPUsPerNode: 8, OtherModels: []string{"Another Accelerator"}, + }} + run := dgdrRunForTest(t, map[string]any{}) + *run.Spec.Hardware.GPU.SKU = "future_accelerator" + f := newDGDRRunFixture(t, hardware, run) + + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err == nil || !strings.Contains(err.Error(), "among multiple GPU models") { + t.Fatalf("reconcile error = %v, want ambiguous external-SKU topology failure", err) + } +} + func TestV1beta2RunControllerFreezesExecutionBeforeCreatingTheJob(t *testing.T) { hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} run := dgdrRunForTest(t, map[string]any{}) @@ -283,34 +319,22 @@ func TestV1beta2RunControllerReportsImagePullSecretErrors(t *testing.T) { } } -func TestV1beta2RunControllerKeepsPlannerDigestButDerivesRuntimesFromReleaseTag(t *testing.T) { +func TestV1beta2RunControllerRejectsDigestOnlyPlannerImage(t *testing.T) { f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) f.reconciler.PlannerImage = "example.com/dynamo-planner@sha256:0123" f.reconciler.DynamoVersion = "1.7.0" - t.Log("Reconcile a digest-pinned Planner through the frozen execution artifact") - f.reconcile() - - t.Log("Keep the Planner digest while giving sibling runtime derivation a shared release tag") - job := f.job() - planner := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) - if planner.Image != "example.com/dynamo-planner@sha256:0123" { - t.Fatalf("planner image = %q, want configured digest", planner.Image) - } - configMap := &corev1.ConfigMap{} - if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { - t.Fatalf("get ConfigMap: %v", err) + t.Log("Reject the Planner before freezing an unrelated runtime release tag") + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err == nil || !strings.Contains(err.Error(), "digest-only planner image cannot identify sibling runtime images") { + t.Fatalf("reconcile error = %v, want digest-only Planner rejection", err) } - if got := configMap.Data[dgdrRunRuntimeImageSourceKey]; got != "example.com/dynamo-planner:1.7.0" { - t.Fatalf("runtime image source = %q, want shared release tag", got) + jobs := &batchv1.JobList{} + if err := f.client.List(t.Context(), jobs, client.InNamespace("default")); err != nil { + t.Fatalf("list Jobs: %v", err) } - var input map[string]any - if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &input); err != nil { - t.Fatalf("decode input: %v", err) - } - output := input["dgdr_run"].(map[string]any) - if output["runtime_image"] != "example.com/dynamo-planner:1.7.0" || output["runtime_version_override"] != "1.7.0" { - t.Fatalf("dgdr_run runtime identity = %v", output) + if len(jobs.Items) != 0 { + t.Fatalf("Jobs = %d, want none for an ambiguous runtime source", len(jobs.Items)) } } @@ -334,8 +358,20 @@ func assertDGDRRunJobIsolation( if job.Spec.Template.Spec.AutomountServiceAccountToken == nil || *job.Spec.Template.Spec.AutomountServiceAccountToken { t.Fatal("the pod must disable automatic service-account token mounting") } + podSecurity := job.Spec.Template.Spec.SecurityContext + if podSecurity == nil || podSecurity.RunAsNonRoot == nil || !*podSecurity.RunAsNonRoot || + podSecurity.SeccompProfile == nil || podSecurity.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Fatalf("pod security context = %+v, want runAsNonRoot and RuntimeDefault seccomp", podSecurity) + } sweeper := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) publisher := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunPublisherContainer) + for _, container := range []corev1.Container{sweeper, publisher} { + security := container.SecurityContext + if security == nil || security.AllowPrivilegeEscalation == nil || *security.AllowPrivilegeEscalation || + security.Capabilities == nil || !reflect.DeepEqual(security.Capabilities.Drop, []corev1.Capability{"ALL"}) { + t.Fatalf("container %q security context = %+v, want restricted defaults", container.Name, security) + } + } if sweeper.Image != plannerImage { t.Fatalf("sweeper image = %q, want operator-configured planner image %q", sweeper.Image, plannerImage) } @@ -483,7 +519,7 @@ func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { {name: "prerelease planner tag", plannerImage: "example.com/dynamo-planner:1.7.0-rc.1", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0-rc.1"}, {name: "build metadata planner tag", plannerImage: "example.com/dynamo-planner:1.7.0+build.5", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0+build.5"}, {name: "tag and digest", plannerImage: "example.com/dynamo-planner:v1.7.0-rc.1@sha256:0123", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:v1.7.0-rc.1@sha256:0123"}, - {name: "digest only", plannerImage: "example.com/dynamo-planner@sha256:0123", dynamoVersion: "1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0"}, + {name: "digest only", plannerImage: "example.com/dynamo-planner@sha256:0123", dynamoVersion: "1.7.0", wantErr: true}, {name: "opaque CI tag", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:git-sha"}, {name: "untagged planner image", plannerImage: "example.com/dynamo-planner", dynamoVersion: "1.7.0", wantErr: true}, {name: "opaque tag without canonical version", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7", wantErr: true}, @@ -493,7 +529,7 @@ func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { version, err := dgdrRunDynamoVersion(tt.plannerImage, tt.dynamoVersion) var runtimeImage string if err == nil { - runtimeImage, err = dgdrRunRuntimeImageSource(tt.plannerImage, version) + runtimeImage, err = dgdrRunRuntimeImageSource(tt.plannerImage) } if (err != nil) != tt.wantErr { t.Fatalf("resolve execution identity error = %v, wantErr %t", err, tt.wantErr) @@ -642,6 +678,67 @@ func TestV1beta2RunControllerClassifiesPublisherProtocolFailure(t *testing.T) { } } +func TestV1beta2RunControllerPrefersPublisherOutcomeWhenBothContainersFail(t *testing.T) { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + job := f.job() + outcome, err := json.Marshal(dgdrrunpublisher.TerminalOutcome{ + Phase: "Failed", Message: "terminal snapshot is invalid", Error: "candidate rank is duplicated", + }) + if err != nil { + t.Fatalf("encode terminal outcome: %v", err) + } + + t.Log("Report both a generic sweeper failure and the publisher's structured protocol outcome") + pod := &corev1.Pod{ + Name: job.Name + "-pod", + Namespace: job.Namespace, + Labels: map[string]string{batchv1.JobNameLabel: job.Name}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: batchv1.SchemeGroupVersion.String(), Kind: "Job", Name: job.Name, UID: job.UID, + Controller: new(true), + }}, + } + if err := f.client.Create(t.Context(), pod); err != nil { + t.Fatalf("create Job pod: %v", err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{ + { + Name: dgdrRunSweeperContainer, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: 1, Message: "planner process exited", + }}, + }, + { + Name: dgdrRunPublisherContainer, + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ + ExitCode: int32(dgdrrunpublisher.ExitProtocolViolation), Message: string(outcome), + }}, + }, + } + if err := f.client.Status().Update(t.Context(), pod); err != nil { + t.Fatalf("update Job pod status: %v", err) + } + failed := metav1.Now() + job.Status.Conditions = []batchv1.JobCondition{{ + Type: batchv1.JobFailed, Status: corev1.ConditionTrue, LastTransitionTime: failed, + }} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("fail Job: %v", err) + } + + t.Log("Prefer and decode the publisher outcome instead of reporting the sweeper exit") + f.reconcile() + run := f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Reason != dgdrRunReasonProtocolViolation { + t.Fatalf("Completed condition = %+v, want %s", condition, dgdrRunReasonProtocolViolation) + } + if condition.Message != "terminal snapshot is invalid: candidate rank is duplicated" { + t.Fatalf("failure message = %q, want decoded publisher diagnostics", condition.Message) + } +} + func TestDGDRRunResourceNameIsStableAndBounded(t *testing.T) { short := dgdrRunResourceName("search-1") if short != "search-1-aisimulate" { diff --git a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go index 4343732d9842..52777bd765f6 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go @@ -67,10 +67,22 @@ func newWriterFixture(t *testing.T) *writerFixture { ctx := context.Background() c := env.Client() + metric := v1beta2.ObjectiveMetric("ThroughputPerGPU") run := &v1beta2.DynamoGraphDeploymentRun{ Namespace: env.Namespace(), Name: "publisher-run", Spec: v1beta2.DynamoGraphDeploymentRunSpec{ - NativeSearchInput: v1beta2.NativeSearchInput{Model: &v1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}}, + NativeSearchInput: v1beta2.NativeSearchInput{ + Model: &v1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, + Backends: []v1beta2.Backend{v1beta2.BackendVLLM}, + Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{ + SKU: new("h200_sxm"), Budget: new(int32(8)), + }}, + Workload: &v1beta2.WorkloadSpec{Static: &v1beta2.StaticWorkloadSpec{}}, + Objective: &v1beta2.ObjectiveSpec{ + Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric, SLA: &v1beta2.ObjectiveSLASpec{}, + }, + Recommendation: &v1beta2.RecommendationSpec{MaxCandidates: 1}, + }, }, } if err := c.Create(ctx, run); err != nil { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index a05d02763135..8b0b1ee5fd58 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -822,6 +822,11 @@ func TestDynamoGraphDeploymentRunSpecIsImmutable(t *testing.T) { run: v1beta2RunForAdmission(alternateAdmissionModel, nil), wantCELErr: `: Invalid value: spec is immutable`, }, + { + name: "model-only Run is rejected before controller reconciliation", + run: v1beta2ModelOnlyRunForAdmission(v1beta2AdmissionModel), + wantCELErr: `spec: Invalid value: spec must be a fully resolved Request admission snapshot`, + }, } for _, tt := range tests { @@ -1036,16 +1041,35 @@ func beta2DGDRForAdmission( } func v1beta2RunForAdmission(model string, labels map[string]string) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + metric := nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ APIVersion: nvidiacomv1beta2.GroupVersion.String(), Kind: "DynamoGraphDeploymentRun", Name: "test-v1beta2-dgdr-run", Namespace: "default", Labels: labels, Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ - Model: &nvidiacomv1beta2.ModelSpec{Name: model}, + Model: &nvidiacomv1beta2.ModelSpec{Name: model}, + Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM}, + Hardware: &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: new("h200_sxm"), Budget: new(int32(8))}, + NumGPUsPerNode: new(int32(8)), + }, + Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}}, + Objective: &nvidiacomv1beta2.ObjectiveSpec{ + Mode: nvidiacomv1beta2.ObjectiveModeOptimize, Metric: &metric, SLA: &nvidiacomv1beta2.ObjectiveSLASpec{}, + }, + Recommendation: &nvidiacomv1beta2.RecommendationSpec{MaxCandidates: 1}, }}, } } +func v1beta2ModelOnlyRunForAdmission(model string) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := v1beta2RunForAdmission(model, nil) + run.Spec.NativeSearchInput = nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: model}, + } + return run +} + const v1beta2AdmissionModel = "Qwen/Qwen3-0.6B" // v1beta2DGDRForAdmission builds a v1beta2 Request as a client sends it, before the CRD @@ -1100,9 +1124,8 @@ func setV1beta2InputOverrides(spec map[string]any, overrides map[string]any) { // setBeta2AdmissionHardware names the hardware, so fixtures do not depend on GPU discovery. func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { - sku, budget, numGPUsPerNode := "h100_sxm", int32(4), int32(4) + sku, budget := "h100_sxm", int32(4) request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ - GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, - NumGPUsPerNode: &numGPUsPerNode, + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, } } From 684261dde5c5499b9609839d7c65cbc84f9ceeab Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 16:57:23 +0100 Subject: [PATCH 054/100] pr-17195: resolve publisher Run fixture [upstream: #17195] Signed-off-by: Dr. Stefan Schimanski --- .../internal/dgdrrunpublisher/writer_envtest_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go index 52777bd765f6..815d71557a05 100644 --- a/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go +++ b/deploy/operator/internal/dgdrrunpublisher/writer_envtest_test.go @@ -74,9 +74,10 @@ func newWriterFixture(t *testing.T) *writerFixture { NativeSearchInput: v1beta2.NativeSearchInput{ Model: &v1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, Backends: []v1beta2.Backend{v1beta2.BackendVLLM}, - Hardware: &v1beta2.HardwareSpec{GPU: &v1beta2.GPUHardwareSpec{ - SKU: new("h200_sxm"), Budget: new(int32(8)), - }}, + Hardware: &v1beta2.HardwareSpec{ + GPU: &v1beta2.GPUHardwareSpec{SKU: new("h200_sxm"), Budget: new(int32(8))}, + NumGPUsPerNode: new(int32(8)), + }, Workload: &v1beta2.WorkloadSpec{Static: &v1beta2.StaticWorkloadSpec{}}, Objective: &v1beta2.ObjectiveSpec{ Mode: v1beta2.ObjectiveModeOptimize, Metric: &metric, SLA: &v1beta2.ObjectiveSLASpec{}, From 1a0e03c058549e62ebe2b6fc4e40666b750c85ff Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 18:33:05 +0100 Subject: [PATCH 055/100] pr-17195: finish review test hygiene [upstream: #17195] Signed-off-by: Dr. Stefan Schimanski --- .../aisimulate/tests/dgdr_run/test_adapter.py | 1 + .../internal/gpu/dcgm_phase_context_test.go | 17 +++++++++++++++++ ...aphdeploymentrequest_carrier_envtest_test.go | 17 +++++++++++++++++ 3 files changed, 35 insertions(+) diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index d0723258f09d..53cac48ee38a 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -27,6 +27,7 @@ pytest.mark.pre_merge, pytest.mark.planner, pytest.mark.parallel, + pytest.mark.timeout(60), ] diff --git a/deploy/operator/internal/gpu/dcgm_phase_context_test.go b/deploy/operator/internal/gpu/dcgm_phase_context_test.go index ba4a86fe6fb6..ef4a15c59dda 100644 --- a/deploy/operator/internal/gpu/dcgm_phase_context_test.go +++ b/deploy/operator/internal/gpu/dcgm_phase_context_test.go @@ -1,3 +1,20 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package gpu import ( diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go index dd7123500a9e..7f431b046ad3 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -1,3 +1,20 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package validation_test import ( From 0ce597f78a56cefab5564d899cda14636c742c6a Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 19:03:08 +0100 Subject: [PATCH 056/100] pr-17195: close retained Run failure boundaries [upstream: #17195] Signed-off-by: Dr. Stefan Schimanski --- ...mographdeploymentrun_v1beta2_controller.go | 228 +++++++++-- ...phdeploymentrun_v1beta2_controller_test.go | 369 +++++++++++++++++- 2 files changed, 557 insertions(+), 40 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 4a409483e7a6..69429a501a46 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -39,6 +39,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" ) @@ -77,6 +78,9 @@ const ( dgdrRunReasonProtocolViolation = "ProtocolViolation" dgdrRunReasonPublicationFailed = "PublicationFailed" dgdrRunReasonJobFailed = "JobFailed" + dgdrRunReasonSuperseded = "Superseded" + dgdrRunReasonInvalidExecution = "InvalidExecution" + dgdrRunReasonInvalidResource = "InvalidResource" dgdrRunMessagePending = "AISimulate Job is waiting to start" dgdrRunMessageRunning = "AISimulate Job is running" dgdrRunMessageSucceeded = "AISimulate search completed successfully" @@ -85,6 +89,20 @@ const ( var errDGDRRunExecutionPending = errors.New("immutable execution ConfigMap is not visible") +type dgdrRunPermanentExecutionError struct { + cause error +} + +func (e *dgdrRunPermanentExecutionError) Error() string { return e.cause.Error() } +func (e *dgdrRunPermanentExecutionError) Unwrap() error { return e.cause } + +type dgdrRunRetryableError struct { + cause error +} + +func (e *dgdrRunRetryableError) Error() string { return e.cause.Error() } +func (e *dgdrRunRetryableError) Unwrap() error { return e.cause } + type dgdrRunHardwareDiscoverer interface { Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) } @@ -110,7 +128,7 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentruns/status,verbs=get;update;patch // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates,verbs=get;list;watch;create;patch;delete // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch @@ -127,18 +145,43 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( if run.DeletionTimestamp != nil || meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { return ctrl.Result{}, nil } + result, err := r.reconcileRun(ctx, run) + if err == nil || !dgdrRunIsPermanentError(err) { + return result, err + } + reason := dgdrRunReasonInvalidExecution + var ownershipConflict *commoncontroller.OwnershipConflictError + if errors.As(err, &ownershipConflict) { + reason = commoncontroller.EventReasonOwnershipConflict + } else if apierrors.IsInvalid(err) { + reason = dgdrRunReasonInvalidResource + } + if statusErr := r.completeRun(ctx, run, reason, err.Error()); statusErr != nil { + return ctrl.Result{}, statusErr + } + return ctrl.Result{}, reconcile.TerminalError(err) +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRun( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) (ctrl.Result, error) { jobs, err := r.ownedJobs(ctx, run) if err != nil { return ctrl.Result{}, err } if len(jobs) > 1 { - return ctrl.Result{}, fmt.Errorf("Run %q controls %d Jobs; expected at most one", run.Name, len(jobs)) + return ctrl.Result{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf( + "Run %q controls %d Jobs; expected at most one", run.Name, len(jobs))} } var job *batchv1.Job if len(jobs) == 1 { job = &jobs[0] } + if supersedingGeneration, cancelRequested := dgdrRunSupersedingGeneration(run); cancelRequested { + return ctrl.Result{}, r.cancelSupersededRun(ctx, run, job, supersedingGeneration) + } resourceName := dgdrRunResourceName(run.Name) if err := r.reconcilePublisherAccess(ctx, run, resourceName); err != nil { return ctrl.Result{}, err @@ -166,6 +209,81 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( return ctrl.Result{}, r.reconcileRunStatus(ctx, run, job, execution) } +func dgdrRunSupersedingGeneration(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) (string, bool) { + runGeneration, err := dgdrV1beta2RunGeneration(run) + if err != nil { + return "", false + } + cancelGenerationValue, ok := run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] + if !ok { + return "", false + } + cancelGeneration, err := strconv.ParseInt(cancelGenerationValue, 10, 64) + return cancelGenerationValue, err == nil && cancelGeneration > runGeneration +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) cancelSupersededRun( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, + supersedingGeneration string, +) error { + if job != nil { + if job.DeletionTimestamp.IsZero() { + err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("stop Job %q for superseded Run %q: %w", job.Name, run.Name, err) + } + if err == nil { + return nil + } + } else { + return nil + } + } + + return r.completeRun(ctx, run, dgdrRunReasonSuperseded, + fmt.Sprintf("Run was superseded by Request generation %s", supersedingGeneration)) +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) completeRun( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + reason, message string, +) error { + original := run.DeepCopy() + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseFailed + now := metav1.Now() + run.Status.CompletionTime = &now + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionTrue, + Reason: reason, + Message: message, + ObservedGeneration: run.Generation, + }) + if apiequality.Semantic.DeepEqual(original.Status, run.Status) { + return nil + } + if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { + return &dgdrRunRetryableError{cause: fmt.Errorf("complete Run %q: %w", run.Name, err)} + } + return nil +} + +func dgdrRunIsPermanentError(err error) bool { + var retryable *dgdrRunRetryableError + if errors.As(err, &retryable) { + return false + } + var permanent *dgdrRunPermanentExecutionError + if errors.As(err, &permanent) { + return true + } + var ownershipConflict *commoncontroller.OwnershipConflictError + return errors.As(err, &ownershipConflict) || apierrors.IsInvalid(err) +} + func (r *DynamoGraphDeploymentRunV1beta2Reconciler) ownedJobs( ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, @@ -191,6 +309,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( ) error { sa := &corev1.ServiceAccount{Name: name, Namespace: run.Namespace} if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { + if sa.ResourceVersion != "" { + if err := commoncontroller.CheckControllerOwnership(sa, run, r.Scheme()); err != nil { + return err + } + } return controllerutil.SetControllerReference(run, sa, r.Scheme()) }); err != nil { return fmt.Errorf("reconcile ServiceAccount %q: %w", name, err) @@ -198,6 +321,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( role := &rbacv1.Role{Name: name, Namespace: run.Namespace} if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, role, func() error { + if role.ResourceVersion != "" { + if err := commoncontroller.CheckControllerOwnership(role, run, r.Scheme()); err != nil { + return err + } + } role.Rules = dgdrRunPublisherRules(run.Name) return controllerutil.SetControllerReference(run, role, r.Scheme()) }); err != nil { @@ -206,6 +334,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( binding := &rbacv1.RoleBinding{Name: name, Namespace: run.Namespace} if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, binding, func() error { + if binding.ResourceVersion != "" { + if err := commoncontroller.CheckControllerOwnership(binding, run, r.Scheme()); err != nil { + return err + } + } binding.RoleRef = rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: name} binding.Subjects = []rbacv1.Subject{{Kind: "ServiceAccount", Name: name, Namespace: run.Namespace}} return controllerutil.SetControllerReference(run, binding, r.Scheme()) @@ -263,19 +396,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( key := types.NamespacedName{Namespace: run.Namespace, Name: name} err := r.Get(ctx, key, configMap) if err == nil { - if !metav1.IsControlledBy(configMap, run) { - return dgdrRunExecution{}, fmt.Errorf("ConfigMap %q is not controlled by Run %q", name, run.Name) - } - execution, parseErr := dgdrRunExecutionFromConfigMap(configMap) - if parseErr != nil { - return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q: %w", name, parseErr) - } - if job != nil { - if validateErr := validateDGDRRunJobExecution(job, execution); validateErr != nil { - return dgdrRunExecution{}, fmt.Errorf("Job %q and ConfigMap %q disagree: %w", job.Name, name, validateErr) - } - } - return execution, nil + return r.existingRunExecution(configMap, run, job) } if !apierrors.IsNotFound(err) { return dgdrRunExecution{}, fmt.Errorf("get ConfigMap %q: %w", name, err) @@ -295,10 +416,10 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( return dgdrRunExecution{}, fmt.Errorf("read ConfigMap %q directly from the API: %w", name, apiErr) } } - return dgdrRunExecution{}, fmt.Errorf( + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf( "immutable execution ConfigMap %q is missing while Job %q exists; refusing to reconstruct it from current operator configuration", name, job.Name, - ) + )} } // Resolve every execution value before creating the immutable artifact. @@ -318,11 +439,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( } execution.DynamoVersion, err = dgdrRunDynamoVersion(execution.PlannerImage, r.DynamoVersion) if err != nil { - return dgdrRunExecution{}, err + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: err} } execution.RuntimeImageSource, err = dgdrRunRuntimeImageSource(execution.PlannerImage) if err != nil { - return dgdrRunExecution{}, err + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: err} } execution.ImagePullSecrets, err = r.dgdrRunImagePullSecrets(run.Namespace, execution.PlannerImage, execution.PublisherImage) if err != nil { @@ -331,19 +452,19 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( input, err := dgdrRunInput(run, execution.RuntimeImageSource, execution.DynamoVersion, execution.GPUsPerNode) if err != nil { - return dgdrRunExecution{}, err + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: err} } contents, err := yaml.Marshal(input) if err != nil { - return dgdrRunExecution{}, fmt.Errorf("encode AISimulate input: %w", err) + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf("encode AISimulate input: %w", err)} } execution.Input, err = dgdrRunStatusInput(string(contents)) if err != nil { - return dgdrRunExecution{}, err + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: err} } pullSecrets, err := json.Marshal(execution.ImagePullSecrets) if err != nil { - return dgdrRunExecution{}, fmt.Errorf("encode image pull secrets: %w", err) + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf("encode image pull secrets: %w", err)} } configMap = &corev1.ConfigMap{ @@ -367,11 +488,43 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( dgdrRunImagePullSecretsKey: string(pullSecrets), }, } - if err := controllerutil.SetControllerReference(run, configMap, r.Scheme()); err != nil { - return dgdrRunExecution{}, fmt.Errorf("set ConfigMap owner: %w", err) + desired := configMap + configMap = &corev1.ConfigMap{Name: name, Namespace: run.Namespace} + operation, err := controllerutil.CreateOrUpdate(ctx, r.Client, configMap, func() error { + if configMap.ResourceVersion != "" { + return commoncontroller.CheckControllerOwnership(configMap, run, r.Scheme()) + } + configMap.Annotations = desired.Annotations + configMap.Immutable = desired.Immutable + configMap.Data = desired.Data + return controllerutil.SetControllerReference(run, configMap, r.Scheme()) + }) + if err != nil { + return dgdrRunExecution{}, fmt.Errorf("reconcile ConfigMap %q: %w", name, err) } - if err := r.Create(ctx, configMap); err != nil { - return dgdrRunExecution{}, fmt.Errorf("create ConfigMap %q: %w", name, err) + if operation != controllerutil.OperationResultCreated { + return r.existingRunExecution(configMap, run, job) + } + return execution, nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) existingRunExecution( + configMap *corev1.ConfigMap, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, +) (dgdrRunExecution, error) { + if err := commoncontroller.CheckControllerOwnership(configMap, run, r.Scheme()); err != nil { + return dgdrRunExecution{}, err + } + execution, err := dgdrRunExecutionFromConfigMap(configMap) + if err != nil { + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf("read ConfigMap %q: %w", configMap.Name, err)} + } + if job != nil { + if err := validateDGDRRunJobExecution(job, execution); err != nil { + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf( + "Job %q and ConfigMap %q disagree: %w", job.Name, configMap.Name, err)} + } } return execution, nil } @@ -381,10 +534,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( run *nvidiacomv1beta2.DynamoGraphDeploymentRun, ) (int, error) { if r.Hardware == nil { - return 0, errors.New("discover GPUs per node: hardware discoverer is not configured") + return 0, &dgdrRunRetryableError{cause: errors.New("discover GPUs per node: hardware discoverer is not configured")} } if run.Spec.Hardware == nil || run.Spec.Hardware.GPU == nil || run.Spec.Hardware.GPU.SKU == nil { - return 0, errors.New("discover GPUs per node: spec.hardware.gpu.sku was not resolved by Request admission") + return 0, &dgdrRunPermanentExecutionError{cause: errors.New( + "discover GPUs per node: spec.hardware.gpu.sku was not resolved by Request admission")} } sku := *run.Spec.Hardware.GPU.SKU filterSKU := nvidiacomv1beta1.GPUSKUType("") @@ -393,16 +547,16 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( } info, err := r.Hardware.Discover(ctx, filterSKU) if err != nil { - return 0, fmt.Errorf("discover GPUs per node: %w", err) + return 0, &dgdrRunRetryableError{cause: fmt.Errorf("discover GPUs per node: %w", err)} } if filterSKU == "" && info != nil && len(info.OtherModels) > 0 { - return 0, fmt.Errorf( + return 0, &dgdrRunRetryableError{cause: fmt.Errorf( "discover GPUs per node for external AISimulate SKU %q: cluster discovery selected %q among multiple GPU models (%s)", sku, info.Model, strings.Join(info.OtherModels, ", "), - ) + )} } if info == nil || info.GPUsPerNode < 1 { - return 0, errors.New("discover GPUs per node: discovery returned no positive GPU count") + return 0, &dgdrRunRetryableError{cause: errors.New("discover GPUs per node: discovery returned no positive GPU count")} } return info.GPUsPerNode, nil } @@ -501,7 +655,8 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) dgdrRunImagePullSecrets( for _, image := range images { names, err := r.DockerSecretRetriever.GetSecrets(namespace, image) if err != nil { - return nil, fmt.Errorf("resolve image pull secrets for %q in namespace %q: %w", image, namespace, err) + return nil, &dgdrRunRetryableError{cause: fmt.Errorf( + "resolve image pull secrets for %q in namespace %q: %w", image, namespace, err)} } additional := make([]corev1.LocalObjectReference, 0, len(names)) for _, name := range names { @@ -746,7 +901,8 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRunStatus( execution dgdrRunExecution, ) error { if execution.Input == nil { - return errors.New("update Run status: effective AISimulate input is missing") + return &dgdrRunPermanentExecutionError{cause: errors.New( + "update Run status: effective AISimulate input is missing")} } original := run.DeepCopy() run.Status.AISimulate = &nvidiacomv1beta2.AISimulateRunStatus{Input: execution.Input.DeepCopy()} @@ -784,7 +940,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRunStatus( return nil } if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { - return fmt.Errorf("update Run %q status: %w", run.Name, err) + return &dgdrRunRetryableError{cause: fmt.Errorf("update Run %q status: %w", run.Name, err)} } return nil } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 9a9be00abaf9..49fb31b6bba1 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -16,21 +16,27 @@ import ( nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" ) @@ -67,6 +73,16 @@ type dgdrRunFixture struct { } func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects ...client.Object) *dgdrRunFixture { + t.Helper() + return newDGDRRunFixtureWithInterceptor(t, hardware, interceptor.Funcs{}, objects...) +} + +func newDGDRRunFixtureWithInterceptor( + t *testing.T, + hardware dgdrRunHardwareDiscoverer, + funcs interceptor.Funcs, + objects ...client.Object, +) *dgdrRunFixture { t.Helper() scheme := runtime.NewScheme() if err := clientgoscheme.AddToScheme(scheme); err != nil { @@ -80,6 +96,7 @@ func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects WithObjects(objects...). WithIndex(&batchv1.Job{}, dgdrRunJobControllerUIDIndex, dgdrRunJobControllerUIDs). WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, &batchv1.Job{}, &corev1.Pod{}). + WithInterceptorFuncs(funcs). Build() return &dgdrRunFixture{ t: t, @@ -113,15 +130,21 @@ func (f *dgdrRunFixture) run() *nvidiacomv1beta2.DynamoGraphDeploymentRun { } func (f *dgdrRunFixture) job() *batchv1.Job { + f.t.Helper() + jobs := f.jobs() + if len(jobs) != 1 { + f.t.Fatalf("Jobs = %d, want 1", len(jobs)) + } + return &jobs[0] +} + +func (f *dgdrRunFixture) jobs() []batchv1.Job { f.t.Helper() jobs := &batchv1.JobList{} if err := f.client.List(f.t.Context(), jobs, client.InNamespace("default")); err != nil { f.t.Fatalf("list Jobs: %v", err) } - if len(jobs.Items) != 1 { - f.t.Fatalf("Jobs = %d, want 1", len(jobs.Items)) - } - return &jobs.Items[0] + return jobs.Items } func dgdrRunForTest(t *testing.T, inputOverrides map[string]any) *nvidiacomv1beta2.DynamoGraphDeploymentRun { @@ -133,6 +156,7 @@ func dgdrRunForTest(t *testing.T, inputOverrides map[string]any) *nvidiacomv1bet metric := nvidiacomv1beta2.ObjectiveMetric("ThroughputPerGPU") return &nvidiacomv1beta2.DynamoGraphDeploymentRun{ Name: "search-1", Namespace: "default", UID: "run-uid", Generation: 1, + Annotations: map[string]string{dgdrV1beta2GenerationAnnotation: "1"}, Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM}, @@ -176,6 +200,343 @@ func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { assertDGDRRunStatusInputMatchesConfigMap(t, f.client, stored, job) } +func TestV1beta2RunControllerRetainsAndTerminalizesASupersededRun(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + f := newDGDRRunFixture(t, hardware, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + f.reconcile() + + run := f.run() + run.Status.Progress = &nvidiacomv1beta2.DGDRRunProgress{ + Candidates: &nvidiacomv1beta2.CandidateProgress{Published: 1}, + } + run.Status.CandidateRefs = []corev1.LocalObjectReference{{Name: "candidate-a"}} + if err := f.client.Status().Update(t.Context(), run); err != nil { + t.Fatalf("publish candidate history: %v", err) + } + run = f.run() + if run.Annotations == nil { + run.Annotations = map[string]string{} + } + run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] = "3" + if err := f.client.Update(t.Context(), run); err != nil { + t.Fatalf("request cancellation: %v", err) + } + + t.Log("The first cancellation pass stops the Job but retains the Run and its history") + f.reconcile() + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want the execution stopped", len(jobs)) + } + run = f.run() + if meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { + t.Fatal("Run completed before Job deletion was observed") + } + if run.Status.Progress == nil || len(run.Status.CandidateRefs) != 1 { + t.Fatalf("candidate history was lost while stopping the Job: %+v", run.Status) + } + + t.Log("A coalesced Request generation becomes the terminal cancellation cause") + run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] = "4" + if err := f.client.Update(t.Context(), run); err != nil { + t.Fatalf("coalesce cancellation generation: %v", err) + } + f.reconcile() + run = f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed || condition == nil || + condition.Status != metav1.ConditionTrue || condition.Reason != dgdrRunReasonSuperseded { + t.Fatalf("superseded status = %+v", run.Status) + } + if !strings.Contains(condition.Message, "generation 4") { + t.Fatalf("superseded message = %q, want latest generation 4", condition.Message) + } + if run.Status.CompletionTime == nil || run.Status.Progress == nil || len(run.Status.CandidateRefs) != 1 { + t.Fatalf("terminal cancellation lost completion or candidate history: %+v", run.Status) + } + + t.Log("Retries are idempotent and never recreate the stopped Job") + completionTime := run.Status.CompletionTime.DeepCopy() + f.reconcile() + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d after retry, want none", len(jobs)) + } + if got := f.run().Status.CompletionTime; got == nil || !got.Equal(completionTime) { + t.Fatalf("completionTime = %v, want stable %v", got, completionTime) + } +} + +func TestV1beta2RunControllerCancelsBeforeCreatingExecutionResources(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + run := dgdrRunForTest(t, map[string]any{}) + run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] = "2" + f := newDGDRRunFixture(t, hardware, run) + + f.reconcile() + stored := f.run() + condition := meta.FindStatusCondition(stored.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != dgdrRunReasonSuperseded { + t.Fatalf("Completed condition = %+v, want True/%s", condition, dgdrRunReasonSuperseded) + } + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want none", len(jobs)) + } + if len(hardware.calls) != 0 { + t.Fatalf("hardware discovery calls = %v, want none for a canceled Run", hardware.calls) + } +} + +func TestV1beta2RunControllerRequiresANewerCancellationGeneration(t *testing.T) { + tests := []struct { + name string + runGeneration string + cancelGeneration string + wantCanceled bool + }{ + {name: "malformed Run generation", runGeneration: "malformed", cancelGeneration: "2"}, + {name: "zero Run generation", runGeneration: "0", cancelGeneration: "2"}, + {name: "malformed cancellation generation", runGeneration: "1", cancelGeneration: "malformed"}, + {name: "zero cancellation generation", runGeneration: "1", cancelGeneration: "0"}, + {name: "equal cancellation generation", runGeneration: "1", cancelGeneration: "1"}, + {name: "older cancellation generation", runGeneration: "2", cancelGeneration: "1"}, + {name: "newer cancellation generation", runGeneration: "1", cancelGeneration: "2", wantCanceled: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} + run := dgdrRunForTest(t, map[string]any{}) + run.Annotations[dgdrV1beta2GenerationAnnotation] = tt.runGeneration + run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] = tt.cancelGeneration + f := newDGDRRunFixture(t, hardware, run) + + t.Log("Reconcile the Run with the cancellation marker") + f.reconcile() + + completed := meta.IsStatusConditionTrue(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if completed != tt.wantCanceled { + t.Fatalf("Completed = %t, want %t", completed, tt.wantCanceled) + } + if tt.wantCanceled { + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want no execution for a superseded Run", len(jobs)) + } + if len(hardware.calls) != 0 { + t.Fatalf("hardware discovery calls = %v, want none", hardware.calls) + } + return + } + if jobs := f.jobs(); len(jobs) != 1 { + t.Fatalf("Jobs = %d, want normal execution", len(jobs)) + } + if len(hardware.calls) != 1 { + t.Fatalf("hardware discovery calls = %v, want normal execution", hardware.calls) + } + }) + } +} + +func TestV1beta2RunControllerRejectsSupportResourceOwnershipCollisionsWithoutMutation(t *testing.T) { + resourceKinds := []struct { + name string + new func(string) client.Object + }{ + {name: "ServiceAccount", new: func(name string) client.Object { + return &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "default", Labels: map[string]string{"existing": "service-account"}, + }} + }}, + {name: "Role", new: func(name string) client.Object { + return &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Rules: []rbacv1.PolicyRule{{APIGroups: []string{"existing.example"}, Resources: []string{"kept"}, Verbs: []string{"get"}}}} + }}, + {name: "RoleBinding", new: func(name string) client.Object { + return &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "default", Labels: map[string]string{"existing": "role-binding"}, + }} + }}, + {name: "ConfigMap", new: func(name string) client.Object { + return &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, + Data: map[string]string{"existing": "config-map"}} + }}, + } + owners := []struct { + name string + ref *metav1.OwnerReference + }{ + {name: "ownerless"}, + {name: "foreign", ref: &metav1.OwnerReference{ + APIVersion: "v1", Kind: "ConfigMap", Name: "foreign", UID: "foreign-uid", Controller: new(true), + }}, + {name: "stale Run UID", ref: &metav1.OwnerReference{ + APIVersion: nvidiacomv1beta2.GroupVersion.String(), Kind: "DynamoGraphDeploymentRun", + Name: "search-1", UID: "stale-run-uid", Controller: new(true), + }}, + } + + for _, resourceKind := range resourceKinds { + for _, owner := range owners { + t.Run(resourceKind.name+"/"+owner.name, func(t *testing.T) { + run := dgdrRunForTest(t, map[string]any{}) + collision := resourceKind.new(dgdrRunResourceName(run.Name)) + if owner.ref != nil { + collision.SetOwnerReferences([]metav1.OwnerReference{*owner.ref}) + } + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, run, collision) + + before := collision.DeepCopyObject().(client.Object) + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(collision), before); err != nil { + t.Fatalf("get collision before reconcile: %v", err) + } + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if !errors.Is(err, reconcile.TerminalError(nil)) { + t.Fatalf("reconcile error = %v, want terminal ownership conflict", err) + } + var ownershipConflict *commoncontroller.OwnershipConflictError + if !errors.As(err, &ownershipConflict) { + t.Fatalf("reconcile error = %T %v, want OwnershipConflictError", err, err) + } + + after := collision.DeepCopyObject().(client.Object) + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(collision), after); err != nil { + t.Fatalf("get collision after reconcile: %v", err) + } + if !apiequality.Semantic.DeepEqual(before, after) { + t.Fatalf("colliding resource was mutated\nbefore: %#v\nafter: %#v", before, after) + } + condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionTrue || + condition.Reason != commoncontroller.EventReasonOwnershipConflict { + t.Fatalf("Completed condition = %+v, want terminal ownership conflict", condition) + } + }) + } + } +} + +func TestV1beta2RunControllerClassifiesPermanentAndRetryableFailures(t *testing.T) { + tests := []struct { + name string + newFixture func(*testing.T) *dgdrRunFixture + wantTerminal bool + wantReason string + wantErrMessage string + }{ + { + name: "immutable lowering error is terminal", + newFixture: func(t *testing.T) *dgdrRunFixture { + run := dgdrRunForTest(t, map[string]any{}) + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, run) + f.reconciler.PlannerImage = "example.com/dynamo-planner@sha256:0123" + f.reconciler.DynamoVersion = "1.7.0" + return f + }, + wantTerminal: true, wantReason: dgdrRunReasonInvalidExecution, + wantErrMessage: "digest-only planner image", + }, + { + name: "invalid Kubernetes resource is terminal", + newFixture: func(t *testing.T) *dgdrRunFixture { + run := dgdrRunForTest(t, map[string]any{}) + return newDGDRRunFixtureWithInterceptor(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, object client.Object, opts ...client.CreateOption) error { + if _, isConfigMap := object.(*corev1.ConfigMap); isConfigMap { + return apierrors.NewInvalid(schema.GroupKind{Kind: "ConfigMap"}, object.GetName(), + field.ErrorList{field.Invalid(field.NewPath("data"), nil, "invalid execution")}) + } + return c.Create(ctx, object, opts...) + }, + }, run) + }, + wantTerminal: true, wantReason: dgdrRunReasonInvalidResource, + wantErrMessage: "invalid execution", + }, + { + name: "hardware discovery error is retryable", + newFixture: func(t *testing.T) *dgdrRunFixture { + return newDGDRRunFixture(t, &fixedRunHardware{err: errors.New("node inventory unavailable")}, + dgdrRunForTest(t, map[string]any{})) + }, + wantErrMessage: "node inventory unavailable", + }, + { + name: "registry secret error is retryable", + newFixture: func(t *testing.T) *dgdrRunFixture { + f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, + dgdrRunForTest(t, map[string]any{})) + f.reconciler.DockerSecretRetriever = &fixedRunSecrets{err: errors.New("registry index unavailable")} + return f + }, + wantErrMessage: "registry index unavailable", + }, + { + name: "terminal status write failure remains retryable", + newFixture: func(t *testing.T) *dgdrRunFixture { + run := dgdrRunForTest(t, map[string]any{}) + f := newDGDRRunFixtureWithInterceptor(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, interceptor.Funcs{ + SubResourcePatch: func( + _ context.Context, _ client.Client, _ string, _ client.Object, + _ client.Patch, _ ...client.SubResourcePatchOption, + ) error { + return errors.New("status writer unavailable") + }, + }, run) + f.reconciler.PlannerImage = "example.com/dynamo-planner@sha256:0123" + f.reconciler.DynamoVersion = "1.7.0" + return f + }, + wantErrMessage: "status writer unavailable", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := tt.newFixture(t) + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err == nil || !strings.Contains(err.Error(), tt.wantErrMessage) { + t.Fatalf("reconcile error = %v, want %q", err, tt.wantErrMessage) + } + if got := errors.Is(err, reconcile.TerminalError(nil)); got != tt.wantTerminal { + t.Fatalf("terminal error = %t, want %t: %v", got, tt.wantTerminal, err) + } + condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if tt.wantTerminal { + if condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != tt.wantReason { + t.Fatalf("Completed condition = %+v, want True/%s", condition, tt.wantReason) + } + } else if condition != nil && condition.Status == metav1.ConditionTrue { + t.Fatalf("retryable failure terminalized the Run: %+v", condition) + } + }) + } +} + +func TestDGDRRunPermanentErrorClassification(t *testing.T) { + invalid := apierrors.NewInvalid(schema.GroupKind{Kind: "Job"}, "search-1", field.ErrorList{ + field.Invalid(field.NewPath("spec"), nil, "invalid"), + }) + tests := []struct { + name string + err error + want bool + }{ + {name: "explicit immutable execution error", err: &dgdrRunPermanentExecutionError{cause: errors.New("invalid input")}, want: true}, + {name: "ownership conflict", err: &commoncontroller.OwnershipConflictError{Cause: errors.New("foreign owner")}, want: true}, + {name: "invalid API object", err: invalid, want: true}, + {name: "forbidden", err: apierrors.NewForbidden(schema.GroupResource{Group: "batch", Resource: "jobs"}, "search-1", errors.New("denied"))}, + {name: "conflict", err: apierrors.NewConflict(schema.GroupResource{Group: "batch", Resource: "jobs"}, "search-1", errors.New("changed"))}, + {name: "timeout", err: apierrors.NewTimeoutError("timed out", 1)}, + {name: "retryable invalid discovery error", err: &dgdrRunRetryableError{cause: invalid}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dgdrRunIsPermanentError(tt.err); got != tt.want { + t.Fatalf("permanent = %t, want %t for %T %v", got, tt.want, tt.err, tt.err) + } + }) + } +} + func TestV1beta2RunControllerUsesUnfilteredTopologyForAnExternalHardwareSKU(t *testing.T) { hardware := &fixedRunHardware{info: gpu.GPUInfo{Model: "Future Accelerator", GPUsPerNode: 8}} run := dgdrRunForTest(t, map[string]any{}) From 315da920ffcff49a17eb59eb39aac6437ab593a6 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 19:24:34 +0100 Subject: [PATCH 057/100] pr-17195: satisfy current operator lint [upstream: #17195] Signed-off-by: Dr. Stefan Schimanski --- ...mographdeploymentrun_v1beta2_controller.go | 13 +++----- ...phdeploymentrun_v1beta2_controller_test.go | 33 ++++++++++++------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 69429a501a46..bcdd4822b238 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -150,8 +150,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( return result, err } reason := dgdrRunReasonInvalidExecution - var ownershipConflict *commoncontroller.OwnershipConflictError - if errors.As(err, &ownershipConflict) { + if _, ok := errors.AsType[*commoncontroller.OwnershipConflictError](err); ok { reason = commoncontroller.EventReasonOwnershipConflict } else if apierrors.IsInvalid(err) { reason = dgdrRunReasonInvalidResource @@ -272,16 +271,14 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) completeRun( } func dgdrRunIsPermanentError(err error) bool { - var retryable *dgdrRunRetryableError - if errors.As(err, &retryable) { + if _, ok := errors.AsType[*dgdrRunRetryableError](err); ok { return false } - var permanent *dgdrRunPermanentExecutionError - if errors.As(err, &permanent) { + if _, ok := errors.AsType[*dgdrRunPermanentExecutionError](err); ok { return true } - var ownershipConflict *commoncontroller.OwnershipConflictError - return errors.As(err, &ownershipConflict) || apierrors.IsInvalid(err) + _, ownershipConflict := errors.AsType[*commoncontroller.OwnershipConflictError](err) + return ownershipConflict || apierrors.IsInvalid(err) } func (r *DynamoGraphDeploymentRunV1beta2Reconciler) ownedJobs( diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 49fb31b6bba1..8a4bda1759ab 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -342,22 +342,32 @@ func TestV1beta2RunControllerRejectsSupportResourceOwnershipCollisionsWithoutMut new func(string) client.Object }{ {name: "ServiceAccount", new: func(name string) client.Object { - return &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ - Name: name, Namespace: "default", Labels: map[string]string{"existing": "service-account"}, - }} + return &corev1.ServiceAccount{ + Name: name, + Namespace: "default", + Labels: map[string]string{"existing": "service-account"}, + } }}, {name: "Role", new: func(name string) client.Object { - return &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, - Rules: []rbacv1.PolicyRule{{APIGroups: []string{"existing.example"}, Resources: []string{"kept"}, Verbs: []string{"get"}}}} + return &rbacv1.Role{ + Name: name, + Namespace: "default", + Rules: []rbacv1.PolicyRule{{APIGroups: []string{"existing.example"}, Resources: []string{"kept"}, Verbs: []string{"get"}}}, + } }}, {name: "RoleBinding", new: func(name string) client.Object { - return &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{ - Name: name, Namespace: "default", Labels: map[string]string{"existing": "role-binding"}, - }} + return &rbacv1.RoleBinding{ + Name: name, + Namespace: "default", + Labels: map[string]string{"existing": "role-binding"}, + } }}, {name: "ConfigMap", new: func(name string) client.Object { - return &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}, - Data: map[string]string{"existing": "config-map"}} + return &corev1.ConfigMap{ + Name: name, + Namespace: "default", + Data: map[string]string{"existing": "config-map"}, + } }}, } owners := []struct { @@ -392,8 +402,7 @@ func TestV1beta2RunControllerRejectsSupportResourceOwnershipCollisionsWithoutMut if !errors.Is(err, reconcile.TerminalError(nil)) { t.Fatalf("reconcile error = %v, want terminal ownership conflict", err) } - var ownershipConflict *commoncontroller.OwnershipConflictError - if !errors.As(err, &ownershipConflict) { + if _, ok := errors.AsType[*commoncontroller.OwnershipConflictError](err); !ok { t.Fatalf("reconcile error = %T %v, want OwnershipConflictError", err, err) } From 50521176b7506c186e31991a784d309057ed8fb0 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 20:23:48 +0100 Subject: [PATCH 058/100] pr-17195: pin sweeper numeric non-root UID [upstream: #17195] Signed-off-by: Dr. Stefan Schimanski --- .../dynamographdeploymentrun_v1beta2_controller.go | 6 +++++- .../dynamographdeploymentrun_v1beta2_controller_test.go | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index bcdd4822b238..0c180d220a5a 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -782,6 +782,10 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( name string, execution dgdrRunExecution, ) *batchv1.Job { + // Pin the named planner-image user to its numeric UID so kubelet can enforce runAsNonRoot. + sweeperSecurityContext := dgdrRunContainerSecurityContext() + sweeperSecurityContext.RunAsUser = ptr.To[int64](1000) + job := &batchv1.Job{ Name: name, Namespace: run.Namespace, @@ -817,7 +821,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( "--output-dir", execution.SnapshotPath, "--overwrite", }, - SecurityContext: dgdrRunContainerSecurityContext(), + SecurityContext: sweeperSecurityContext, VolumeMounts: []corev1.VolumeMount{ {Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true}, {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 8a4bda1759ab..75ca7a5358ec 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -742,6 +742,9 @@ func assertDGDRRunJobIsolation( t.Fatalf("container %q security context = %+v, want restricted defaults", container.Name, security) } } + if sweeper.SecurityContext.RunAsUser == nil || *sweeper.SecurityContext.RunAsUser != 1000 { + t.Fatalf("sweeper runAsUser = %v, want planner image UID 1000", sweeper.SecurityContext.RunAsUser) + } if sweeper.Image != plannerImage { t.Fatalf("sweeper image = %q, want operator-configured planner image %q", sweeper.Image, plannerImage) } From 71ea04295b8f802d86294f2755cfd20ad8c8f6a7 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 15:01:24 +0100 Subject: [PATCH 059/100] pr-15895: pr-7: consolidate output adapter tests [upstream: devivasudevan/dynamo#7] Signed-off-by: Dr. Stefan Schimanski --- .../aisimulate/tests/dgdr_run/test_adapter.py | 984 +++++++++--------- 1 file changed, 479 insertions(+), 505 deletions(-) diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index 53cac48ee38a..a69355aff80b 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -79,13 +79,50 @@ def tags(snapshot: dict[str, Any]) -> list[str]: def wait_for(predicate: Any, timeout: float = 5.0) -> None: deadline = time.monotonic() + timeout + poll = threading.Event() while time.monotonic() < deadline: if predicate(): return - time.sleep(0.01) + poll.wait(min(0.01, deadline - time.monotonic())) raise AssertionError("condition not reached before the timeout") +class ControlledWait: + """Let a test release one writer delay without waiting for wall-clock time.""" + + def __init__(self) -> None: + self._condition = threading.Condition() + self._advanced = False + self._stopped = False + self.waiting = threading.Event() + self.timeouts: list[float] = [] + + def is_set(self) -> bool: + with self._condition: + return self._stopped + + def wait(self, timeout: float | None = None) -> bool: + with self._condition: + if timeout is None: + self._condition.wait_for(lambda: self._stopped) + else: + self.timeouts.append(timeout) + self.waiting.set() + self._condition.wait_for(lambda: self._advanced or self._stopped) + self._advanced = False + return self._stopped + + def advance(self) -> None: + with self._condition: + self._advanced = True + self._condition.notify_all() + + def set(self) -> None: + with self._condition: + self._stopped = True + self._condition.notify_all() + + @pytest.fixture def renders(monkeypatch: pytest.MonkeyPatch) -> list[str]: """Replace the shared materializer with a counting fake.""" @@ -121,44 +158,57 @@ def test_start_writes_the_initial_snapshot_immediately( assert snapshot["progress"] == {"round": 0, "evaluated": 0} -def test_projection_is_bounded_and_best_first_whatever_is_evaluated( - tmp_path: Path, renders: list[str] -) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - for index in range(500): - adapter.on_candidate(record(f"p{index:03d}", score=float(index))) - adapter.close() - - snapshot = load(tmp_path) - assert tags(snapshot) == ["p499", "p498", "p497"] - assert snapshot["progress"]["evaluated"] == 500 - # Only retained points are ever materialized, never all 500. - assert len(renders) == 3 - - -def test_ties_break_on_fewer_gpus_then_id_deterministically( - tmp_path: Path, renders: list[str] -) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path, max_candidates=4), None) - adapter.on_candidate(record("big", score=5.0, gpus=16)) - adapter.on_candidate(record("small", score=5.0, gpus=8)) - adapter.on_candidate(record("best", score=9.0, gpus=64)) - adapter.close() - - assert tags(load(tmp_path)) == ["best", "small", "big"] - - -def test_infeasible_candidates_are_counted_but_never_retained( - tmp_path: Path, renders: list[str] +@pytest.mark.parametrize( + ("records", "max_candidates", "expected"), + [ + pytest.param( + tuple(record(f"p{index:03d}", float(index)) for index in range(500)), + 3, + ["p499", "p498", "p497"], + id="bounded-best-first", + ), + pytest.param( + ( + record("big", 5.0, gpus=16), + record("small", 5.0, gpus=8), + record("best", 9.0, gpus=64), + ), + 4, + ["best", "small", "big"], + id="ties-use-fewer-gpus", + ), + pytest.param( + (record("ok", 1.0), record("nope", 99.0, status="infeasible")), + 3, + ["ok"], + id="infeasible-counted-not-retained", + ), + pytest.param( + (SimpleNamespace(status="feasible"), record("ok", 1.0)), + 3, + ["ok"], + id="malformed-callback-isolated", + ), + ], +) +def test_projection_is_bounded_ranked_and_feasible( + tmp_path: Path, + renders: list[str], + records: tuple[Any, ...], + max_candidates: int, + expected: list[str], ) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.on_candidate(record("ok", 1.0)) - adapter.on_candidate(record("nope", 99.0, status="infeasible")) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, max_candidates=max_candidates), workload=None + ) + for candidate in records: + adapter.on_candidate(candidate) adapter.close() snapshot = load(tmp_path) - assert tags(snapshot) == ["ok"] - assert snapshot["progress"]["evaluated"] == 2 + assert tags(snapshot) == expected + assert snapshot["progress"]["evaluated"] == len(records) + assert len(renders) == len(expected) def test_snapshot_preserves_the_evaluated_point_not_just_its_manifest( @@ -183,8 +233,36 @@ def test_snapshot_preserves_the_evaluated_point_not_just_its_manifest( assert "marker: a" in candidate["manifest"] -def test_each_candidate_uses_the_runtime_image_for_its_backend( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + ("runtime_image", "derive", "backends", "expected"), + [ + pytest.param( + "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", + True, + ("vllm", "sglang", "trtllm"), + { + "vllm": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", + "sglang": "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.3", + "trtllm": "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.3", + }, + id="derive-per-backend", + ), + pytest.param( + "example.com/my-runtime:1.2.3", + False, + ("sglang",), + {"sglang": "example.com/my-runtime:1.2.3"}, + id="explicit-image", + ), + ], +) +def test_runtime_image_selection_reaches_the_renderer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + runtime_image: str, + derive: bool, + backends: tuple[str, ...], + expected: dict[str, str], ) -> None: images: dict[str, str] = {} @@ -199,55 +277,18 @@ def capture_image( adapter = DGDRRunOutputAdapter( make_config( tmp_path, - runtime_image="nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.2.3", - derive_runtime_images=True, + runtime_image=runtime_image, + derive_runtime_images=derive, ), workload=None, ) - for index, backend in enumerate(("vllm", "sglang", "trtllm")): + for index, backend in enumerate(backends): candidate = record(backend, float(index)) candidate.config["backend"] = backend adapter.on_candidate(candidate) adapter.close() - assert images == { - "vllm": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", - "sglang": "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.2.3", - "trtllm": "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.2.3", - } - - -def test_explicit_runtime_image_stays_fixed_by_default( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - images: list[str] = [] - - def capture_image( - candidate: Any, workload: Any, options: Any, **kwargs: Any - ) -> str: - del candidate, workload, kwargs - images.append(options.runtime_image) - return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" - - monkeypatch.setattr(adapter_module, "render_dgd", capture_image) - adapter = DGDRRunOutputAdapter( - make_config(tmp_path, runtime_image="example.com/my-runtime:1.2.3"), - workload=None, - ) - candidate = record("sglang", 1.0) - candidate.config["backend"] = "sglang" - adapter.on_candidate(candidate) - adapter.close() - - assert images == ["example.com/my-runtime:1.2.3"] - - -def test_round_progress_is_monotonic(tmp_path: Path, renders: list[str]) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.on_round(3, []) - adapter.on_round(2, []) - adapter.close() - assert load(tmp_path)["progress"]["round"] == 3 + assert images == expected # -- identity, immutability, materialization ---------------------------------- @@ -279,8 +320,9 @@ def test_identity_depends_only_on_the_resolved_parameters() -> None: assert candidate_id_for({"a": 1}).startswith("evaluated-point-") -def test_aic_name_prefix_is_stable_bounded_and_candidate_specific( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize("run_name", ["a" * 63, "team.model"]) +def test_aic_name_prefix_is_stable_dns_safe_and_candidate_specific( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, run_name: str ) -> None: names: list[str] = [] @@ -292,7 +334,7 @@ def capture_name(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> monkeypatch.setattr(adapter_module, "render_dgd", capture_name) adapter = DGDRRunOutputAdapter( - make_config(tmp_path, name="a" * 63, max_candidates=2), workload=None + make_config(tmp_path, name=run_name, max_candidates=2), workload=None ) adapter.on_candidate(record("first", 2.0)) adapter.on_candidate(record("second", 1.0)) @@ -302,33 +344,9 @@ def capture_name(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> assert len(set(names)) == 2 assert all(len(name) <= 28 for name in names) assert all(name[0].isalnum() and name[-1].isalnum() for name in names) + assert all("." not in name for name in names) assert names[0] == adapter_module._dgd_name( - "a" * 63, candidate_id_for(record("first", 2.0).config) - ) - - -def test_aic_name_prefix_accepts_and_normalizes_a_dotted_run_name( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - names: list[str] = [] - - def capture_name(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: - del candidate, workload, options - names.append(kwargs["dgd_name"]) - return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" - - monkeypatch.setattr(adapter_module, "render_dgd", capture_name) - adapter = DGDRRunOutputAdapter( - make_config(tmp_path, name="team.model", max_candidates=1), workload=None - ) - adapter.on_candidate(record("first", 2.0)) - adapter.close() - - assert len(names) == 1 - assert "." not in names[0] - assert len(names[0]) <= 28 - assert names[0] == adapter_module._dgd_name( - "team.model", candidate_id_for(record("first", 2.0).config) + run_name, candidate_id_for(record("first", 2.0).config) ) @@ -343,6 +361,12 @@ def test_a_new_better_point_reorders_but_does_not_rematerialize_retained_ones( before = {c["id"]: c["manifest"] for c in load(tmp_path)["candidates"]} assert sorted(renders) == ["a", "b"] + for round_no in range(1, 20): + adapter.on_round(round_no, []) + adapter.on_round(18, []) + wait_for(lambda: load(tmp_path)["progress"]["round"] == 19) + assert sorted(renders) == ["a", "b"] + adapter.on_candidate(record("c", 3.0)) wait_for(lambda: tags(load(tmp_path)) == ["c", "b", "a"]) adapter.close() @@ -352,20 +376,6 @@ def test_a_new_better_point_reorders_but_does_not_rematerialize_retained_ones( assert sorted(renders) == ["a", "b", "c"] -def test_progress_updates_do_not_rematerialize( - tmp_path: Path, renders: list[str] -) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.start() - adapter.on_candidate(record("a", 1.0)) - wait_for(lambda: tags(load(tmp_path)) == ["a"]) - for round_no in range(1, 20): - adapter.on_round(round_no, []) - wait_for(lambda: load(tmp_path)["progress"]["round"] == 19) - adapter.close() - assert renders == ["a"] - - def test_a_point_is_immutable_once_observed(tmp_path: Path, renders: list[str]) -> None: adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) adapter.on_candidate(record("a", 1.0)) @@ -377,50 +387,42 @@ def test_a_point_is_immutable_once_observed(tmp_path: Path, renders: list[str]) assert load(tmp_path)["progress"]["evaluated"] == 2 -def test_a_persistent_materialization_failure_is_reported_and_retried_a_bounded_number_of_times( - tmp_path: Path, renders: list[str] -) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.start() - adapter.on_candidate(record("bad1", 5.0)) - wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) - for round_no in range(1, 10): - adapter.on_round(round_no, []) - wait_for(lambda: load(tmp_path)["progress"]["round"] == 9) - adapter.close() - - (candidate,) = load(tmp_path)["candidates"] - assert candidate["outcome"] == "materialization_failed" - assert "cannot render bad1" in candidate["error"] - assert "manifest" not in candidate - assert renders == ["bad1"] * adapter_module._MAX_RENDER_ATTEMPTS - - -# -- rate limiting and coalescing --------------------------------------------- +# -- lifecycle and concurrency ------------------------------------------------ -def test_many_callbacks_in_one_interval_produce_one_periodic_write( - tmp_path: Path, renders: list[str] +def test_callbacks_within_one_interval_coalesce_into_the_latest_snapshot( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch ) -> None: - config = make_config(tmp_path, snapshot_interval_seconds=0.5) - adapter = DGDRRunOutputAdapter(config, workload=None) + writes: list[Any] = [] + periodic_write = threading.Event() + real_write = adapter_module.write_snapshot + + def observe_write(directory: Path, snapshot: Any) -> Path: + path = real_write(directory, snapshot) + writes.append(snapshot) + if len(writes) == 2: + periodic_write.set() + return path + + monkeypatch.setattr(adapter_module, "write_snapshot", observe_write) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, snapshot_interval_seconds=60), workload=None + ) adapter.start() - assert adapter.snapshot_writes == 1 + interval = ControlledWait() + adapter._stop = interval # type: ignore[assignment] - for index in range(300): - adapter.on_candidate(record(f"p{index:03d}", score=float(index))) - time.sleep(0.15) - assert adapter.snapshot_writes == 1 + adapter.on_candidate(record("p000", 0.0)) + assert interval.waiting.wait(2), "writer did not enter the rate-limit delay" + for index in range(1, 300): + adapter.on_candidate(record(f"p{index:03d}", float(index))) + interval.advance() - wait_for(lambda: adapter.snapshot_writes == 2, timeout=5) + assert periodic_write.wait(2), "periodic snapshot was not written" snapshot = load(tmp_path) - # The one periodic write carries the latest state, not an arbitrary - # intermediate one. + assert len(writes) == 2 # initial + one coalesced periodic write assert snapshot["progress"]["evaluated"] == 300 assert tags(snapshot) == ["p299", "p298", "p297"] - - time.sleep(0.2) - assert adapter.snapshot_writes == 2 adapter.close() @@ -449,39 +451,63 @@ def test_terminal_state_bypasses_the_rate_limit_and_is_always_flushed( assert adapter.snapshot_writes == 2 # initial + terminal, nothing in between -def test_the_terminal_snapshot_contains_every_update_accepted_before_close( - tmp_path: Path, renders: list[str] +@pytest.mark.parametrize( + ("error", "phase", "error_name"), + [ + pytest.param(None, "Succeeded", None, id="success"), + pytest.param( + RuntimeError("search crashed"), "Failed", "RuntimeError", id="error" + ), + ], +) +def test_context_manager_records_terminal_outcome( + tmp_path: Path, + renders: list[str], + error: RuntimeError | None, + phase: str, + error_name: str | None, ) -> None: - config = make_config(tmp_path, snapshot_interval_seconds=3600) - with DGDRRunOutputAdapter(config, workload=None) as adapter: - for index in range(50): - adapter.on_candidate(record(f"w{index}", score=1.0 + index)) - adapter.on_candidate(record("winner", score=1000.0)) + adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) + if error is None: + with adapter: + pass + else: + with pytest.raises(RuntimeError, match="search crashed"), adapter: + raise error - snapshot = load(tmp_path) - assert snapshot["run"]["terminal"] is True - assert tags(snapshot)[0] == "winner" + run = load(tmp_path)["run"] + assert run["phase"] == phase + assert run.get("error") == error_name def test_callbacks_never_wait_for_a_slow_writer( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + rendering = threading.Event() release = threading.Event() def slow_render(candidate: Any, *args: Any, **kwargs: Any) -> str: - release.wait(5) + rendering.set() + assert release.wait(5), "test did not release the renderer" return "kind: DynamoGraphDeployment\n" monkeypatch.setattr(adapter_module, "render_dgd", slow_render) adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) adapter.start() adapter.on_candidate(record("a", 1.0)) - time.sleep(0.1) # the writer is now stuck rendering + assert rendering.wait(5), "writer did not enter the renderer" - started = time.monotonic() - for index in range(1000): - adapter.on_candidate(record(f"q{index}", score=float(index))) - assert time.monotonic() - started < 2 + callbacks_done = threading.Event() + + def send_callbacks() -> None: + for index in range(1000): + adapter.on_candidate(record(f"q{index}", score=float(index))) + callbacks_done.set() + + sender = threading.Thread(target=send_callbacks) + sender.start() + assert callbacks_done.wait(2), "callbacks blocked behind the writer" + sender.join() release.set() adapter.close() # Nothing was dropped while the writer was stuck. @@ -514,23 +540,6 @@ def test_close_rejects_a_non_terminal_phase(tmp_path: Path) -> None: # -- failure handling --------------------------------------------------------- -def test_context_manager_records_success_and_failure( - tmp_path: Path, renders: list[str] -) -> None: - ok = tmp_path / "ok" - with DGDRRunOutputAdapter(make_config(ok), workload=None): - pass - assert load(ok)["run"]["phase"] == "Succeeded" - - failed = tmp_path / "failed" - with pytest.raises(RuntimeError, match="search crashed"): - with DGDRRunOutputAdapter(make_config(failed), workload=None): - raise RuntimeError("search crashed") - snapshot = load(failed) - assert snapshot["run"]["phase"] == "Failed" - assert snapshot["run"]["error"] == "RuntimeError" - - def test_a_terminal_write_failure_is_raised_not_swallowed( tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch ) -> None: @@ -567,14 +576,33 @@ def flaky(directory: Path, snapshot: Any) -> Path: assert attempts["count"] >= 2 -def test_a_candidate_that_cannot_be_read_does_not_break_the_search( - tmp_path: Path, renders: list[str] +def test_a_persistent_write_failure_waits_for_backoff_instead_of_spinning( + tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch ) -> None: + monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 30) adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.on_candidate(SimpleNamespace(status="feasible")) # type: ignore[arg-type] - adapter.on_candidate(record("ok", 1.0)) + adapter.start() + backoff = ControlledWait() + adapter._stop = backoff # type: ignore[assignment] + real_write = adapter_module.write_snapshot + attempts: list[int] = [] + failed = threading.Event() + + def failing(directory: Path, snapshot: Any) -> Path: + del directory, snapshot + attempts.append(1) + failed.set() + raise OSError("disk full") + + monkeypatch.setattr(adapter_module, "write_snapshot", failing) + adapter.on_candidate(record("a", 1.0)) + assert failed.wait(2), "writer did not attempt the snapshot" + assert backoff.waiting.wait(2), "writer did not enter retry backoff" + assert len(attempts) == 1 + assert backoff.timeouts and backoff.timeouts[0] > 0 + + monkeypatch.setattr(adapter_module, "write_snapshot", real_write) adapter.close() - assert tags(load(tmp_path)) == ["ok"] # -- configuration and plugin ------------------------------------------------- @@ -586,32 +614,29 @@ def test_pareto_objectives_are_rejected_explicitly(tmp_path: Path) -> None: @pytest.mark.parametrize( - "overrides", + ("overrides", "valid"), [ - {"max_candidates": 0}, - {"max_candidates": 101}, - {"snapshot_interval_seconds": -1}, - {"name": "Not_DNS"}, - {"num_gpus_per_node": 0}, - {"unknown_field": 1}, + pytest.param({"max_candidates": 1}, True, id="min-candidates"), + pytest.param({"max_candidates": 100}, True, id="max-candidates"), + pytest.param({"max_candidates": 0}, False, id="too-few-candidates"), + pytest.param({"max_candidates": 101}, False, id="too-many-candidates"), + pytest.param({"snapshot_interval_seconds": -1}, False, id="negative-interval"), + pytest.param({"name": "Not_DNS"}, False, id="invalid-name"), + pytest.param({"num_gpus_per_node": 0}, False, id="zero-gpus"), + pytest.param({"unknown_field": 1}, False, id="unknown-field"), ], ) -def test_invalid_configuration_is_rejected( - tmp_path: Path, overrides: dict[str, Any] +def test_configuration_boundaries( + tmp_path: Path, overrides: dict[str, Any], valid: bool ) -> None: + if valid: + config = make_config(tmp_path, **overrides) + assert config.max_candidates == overrides["max_candidates"] + return with pytest.raises(ValueError): make_config(tmp_path, **overrides) -def test_max_candidates_follows_the_api_range(tmp_path: Path) -> None: - # spec.recommendation.maxCandidates is 1..100; the adapter accepts exactly that range. - assert make_config(tmp_path, max_candidates=1).max_candidates == 1 - assert make_config(tmp_path, max_candidates=100).max_candidates == 100 - for invalid in (0, 101, 200): - with pytest.raises(ValueError): - make_config(tmp_path, max_candidates=invalid) - - def test_max_candidates_has_no_default_of_its_own() -> None: # The operator supplies the already-defaulted API value; a missing one must fail. with pytest.raises(ValueError, match="max_candidates"): @@ -634,74 +659,40 @@ def test_a_snapshot_directory_is_required() -> None: DGDRRunOutputAdapter(config, workload=None) -def test_the_plugin_is_discoverable_and_builds_a_live_adapter(tmp_path: Path) -> None: +def test_plugin_discovery_validation_and_optional_live_subscription( + tmp_path: Path, +) -> None: plugin = create_adapter() assert plugin.name == "dgdr_run" - live = plugin.live( - { - "name": "sweep", - "snapshot_dir": str(tmp_path), - "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", - "num_gpus_per_node": 8, - "max_candidates": 5, - }, - workload=None, - ) - assert isinstance(live, DGDRRunOutputAdapter) - - -def test_plugin_subscribe_validates_config_before_search() -> None: - plugin = adapter_module.DGDRRunOutputPlugin() - good = { + with_dir = { "name": "run", + "snapshot_dir": str(tmp_path), "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", "num_gpus_per_node": 8, "max_candidates": 5, } - - assert plugin.subscribe(good) is None + assert isinstance(plugin.live(with_dir, workload=None), DGDRRunOutputAdapter) + assert plugin.subscribe(with_dir) is None + assert plugin.subscribe(with_dir, context=SimpleNamespace(workload=None)) is None + without_dir = { + key: value for key, value in with_dir.items() if key != "snapshot_dir" + } + assert ( + plugin.subscribe(without_dir, context=SimpleNamespace(workload="workload")) + is None + ) + assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() with pytest.raises(ValueError, match="num_gpus_per_node"): - plugin.subscribe({**good, "num_gpus_per_node": 0}) + plugin.subscribe({**with_dir, "num_gpus_per_node": 0}) with pytest.raises(ValueError, match="canonical"): plugin.subscribe( - {**good, "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest"} + { + **with_dir, + "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:latest", + } ) -def test_plugin_write_publishes_the_final_selection_as_a_terminal_snapshot( - tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch -) -> None: - class FakeSearchConfig: - @staticmethod - def model_validate(_: Any) -> Any: - return SimpleNamespace(workload="workload") - - monkeypatch.setattr(adapter_module, "SmartSearchConfig", FakeSearchConfig) - result = SimpleNamespace( - views=SimpleNamespace(pareto_front=[]), - provenance=SimpleNamespace(config={}), - selected_candidates=[ - record("a", 9.0), - record("b", 5.0), - ], - ) - written = create_adapter().write( - { - "name": "sweep", - "runtime_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.2.3", - "num_gpus_per_node": 8, - "max_candidates": 5, - }, - result=result, # type: ignore[arg-type] - output_dir=tmp_path, - ) - - assert written == [Path(SNAPSHOT_FILE_NAME)] - snapshot = load(tmp_path) - assert snapshot["run"]["terminal"] is True - assert tags(snapshot) == ["a", "b"] - - def test_plugin_write_keeps_sweepers_selection_order( tmp_path: Path, renders: list[str], fake_search_config: None ) -> None: @@ -713,8 +704,16 @@ def test_plugin_write_keeps_sweepers_selection_order( selected_candidates=[record("low", 1.0), record("high", 9.0)], counts=SimpleNamespace(evaluated=2), ) - create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) # type: ignore[arg-type] - assert tags(load(tmp_path)) == ["low", "high"] + written = create_adapter().write( + PLUGIN_CONFIG, + result=result, + output_dir=tmp_path, # type: ignore[arg-type] + ) + snapshot = load(tmp_path) + assert written == [Path(SNAPSHOT_FILE_NAME)] + assert snapshot["run"] == {"phase": "Succeeded", "terminal": True} + assert snapshot["progress"] == {"round": 0, "evaluated": 2} + assert tags(snapshot) == ["low", "high"] def test_plugin_write_refuses_pareto_and_empty_results(tmp_path: Path) -> None: @@ -743,37 +742,130 @@ def test_plugin_write_refuses_pareto_and_empty_results(tmp_path: Path) -> None: } -def test_plugin_subscribe_with_a_context_publishes_live( - tmp_path: Path, renders: list[str] +@pytest.mark.parametrize("search_fails", [False, True], ids=["complete", "failure"]) +def test_pinned_aisimulate_dispatches_the_complete_lifecycle_to_dgdr_run( + tmp_path: Path, + renders: list[str], + fake_search_config: None, + monkeypatch: pytest.MonkeyPatch, + search_fails: bool, ) -> None: - config = {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)} - callbacks = create_adapter().subscribe( - config, context=SimpleNamespace(workload="workload") - ) + import aisimulate.recommend as recommendation + from aisimulate.config import CoreRecommendationConfig + from aisimulate.sweeper import search - assert callbacks is not None - assert callbacks.on_candidate is not None - assert callbacks.on_round is not None - assert load(tmp_path)["run"] == {"phase": "Running", "terminal": False} - - callbacks.on_candidate(record("a", 9.0)) - wait_for(lambda: tags(load(tmp_path)) == ["a"]) - assert callbacks.on_complete is not None - callbacks.on_complete() + config = CoreRecommendationConfig.model_validate( + { + "engine": { + "model": "openai/gpt-oss-120b", + "hardware": "gb300", + "backend": "vllm", + "mode": "aggregated", + "context_length": 16384, + "workers": {"aggregated": {}}, + }, + "traffic": { + "source": { + "type": "synthetic", + "input_tokens": 10240, + "output_tokens": 1024, + }, + "load": { + "type": "concurrency", + "concurrency": {"choices": [8]}, + }, + "stop": {"requests_per_load_unit": 1}, + }, + "optimization": { + "target": "throughput_per_gpu", + "constraints": { + "min_candidate_gpus": 1, + "max_candidate_gpus": 8, + }, + }, + "optimizer": {"parallelism": 1, "max_trials": 1}, + } + ) + output_config = { + **PLUGIN_CONFIG, + "snapshot_dir": str(tmp_path), + # Keep candidate and round updates buffered so their presence after a + # successful search proves that on_complete forced the final live flush. + "snapshot_interval_seconds": 3600, + } + result = _final_result(1) + search_error = RuntimeError("search crashed") + # Use the real pinned AISimulate lifecycle dispatcher around a deterministic search. + monkeypatch.setattr( + recommendation, "resolve_budget", lambda *args, **kwargs: {"cpu_limit": 1} + ) -def test_plugin_subscribe_without_context_or_snapshot_dir_only_validates( - tmp_path: Path, -) -> None: - plugin = create_adapter() - with_dir = {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)} - assert plugin.subscribe(with_dir) is None - assert plugin.subscribe(with_dir, context=SimpleNamespace(workload=None)) is None - assert ( - plugin.subscribe(PLUGIN_CONFIG, context=SimpleNamespace(workload="workload")) - is None + class LifecycleSweeper: + def __init__(self, **kwargs: Any) -> None: + del kwargs + + def run( + self, + smart: Any, + *, + top_n: int | None, + on_candidate: Any = None, + on_round: Any = None, + ) -> Any: + del smart, top_n + assert on_candidate is not None + assert on_round is not None + on_candidate(record("a", 9.0)) + on_round(3, []) + if search_fails: + raise search_error + return result + + monkeypatch.setattr(search, "Sweeper", LifecycleSweeper) + + # A failed search must dispatch on_failure and leave a terminal failed snapshot. + if search_fails: + with pytest.raises(RuntimeError, match="search crashed") as raised: + recommendation._run_recommendation( + config, + stack="engine", + runner_factory=object(), + output_configs={"dgdr_run": output_config}, + output_adapters={"dgdr_run": create_adapter()}, + ) + assert raised.value is search_error + snapshot = load(tmp_path) + assert snapshot["run"] == { + "phase": "Failed", + "terminal": True, + "message": "search crashed", + "error": "RuntimeError", + } + assert snapshot["progress"] == {"round": 3, "evaluated": 1} + assert tags(snapshot) == ["a"] + return + + # A successful search must dispatch on_complete before the final writer takes over. + returned = recommendation._run_recommendation( + config, + stack="engine", + runner_factory=object(), + output_configs={"dgdr_run": output_config}, + output_adapters={"dgdr_run": create_adapter()}, ) - assert not (tmp_path / SNAPSHOT_FILE_NAME).exists() + assert returned is result + completed = load(tmp_path) + assert completed["run"] == {"phase": "Running", "terminal": False} + assert completed["progress"] == {"round": 3, "evaluated": 1} + assert tags(completed) == ["a"] + + # The separate final writer publishes the authoritative terminal selection. + create_adapter().write(output_config, result=returned, output_dir=tmp_path) + terminal = load(tmp_path) + assert terminal["run"] == {"phase": "Succeeded", "terminal": True} + assert terminal["progress"] == {"round": 3, "evaluated": 1} + assert tags(terminal) == ["a"] def _final_result(evaluated: int = 42) -> Any: @@ -814,41 +906,6 @@ def test_plugin_write_reports_no_artifact_for_a_snapshot_outside_output_dir( assert not (out / SNAPSHOT_FILE_NAME).exists() -def test_a_transient_materialization_failure_is_retried_and_recovers( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - attempts: list[int] = [] - - def flaky_render(candidate: Any, workload: Any, options: Any, **kwargs: Any) -> str: - attempts.append(1) - if len(attempts) == 1: - raise CandidateMaterializationError("cold start") - return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" - - monkeypatch.setattr(adapter_module, "render_dgd", flaky_render) - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.start() - adapter.on_candidate(record("a", 5.0)) - wait_for(lambda: len(load(tmp_path)["candidates"]) == 1) - assert load(tmp_path)["candidates"][0]["outcome"] == "materialization_failed" - - adapter.on_round(1, []) # any later write retries the failed render - wait_for(lambda: load(tmp_path)["candidates"][0]["outcome"] == "materialized") - adapter.close() - - (candidate,) = load(tmp_path)["candidates"] - assert candidate["outcome"] == "materialized" - assert "error" not in candidate - assert len(attempts) == 2 - - -def test_plugin_write_without_a_live_snapshot_starts_at_round_zero( - tmp_path: Path, renders: list[str], fake_search_config: None -) -> None: - create_adapter().write(PLUGIN_CONFIG, result=_final_result(2), output_dir=tmp_path) - assert load(tmp_path)["progress"] == {"round": 0, "evaluated": 2} - - # -- lifecycle: how the search ended -------------------------------------------- @@ -861,26 +918,6 @@ def _subscribed(tmp_path: Path) -> Any: return callbacks -def test_a_failed_search_leaves_a_terminal_failed_snapshot( - tmp_path: Path, renders: list[str] -) -> None: - callbacks = _subscribed(tmp_path) - callbacks.on_candidate(record("a", 9.0)) - callbacks.on_round(2, []) - - callbacks.on_failure(RuntimeError("replay crashed")) - - snapshot = load(tmp_path) - assert snapshot["run"] == { - "phase": "Failed", - "terminal": True, - "message": "replay crashed", - "error": "RuntimeError", - } - assert tags(snapshot) == ["a"] - assert snapshot["progress"]["round"] == 2 - - def test_a_long_failure_message_is_truncated( tmp_path: Path, renders: list[str] ) -> None: @@ -903,59 +940,6 @@ def test_a_search_that_finds_nothing_is_recorded_as_failed( assert run["message"] == "no feasible candidate found" -def test_a_completed_search_flushes_stops_the_writer_and_stays_non_terminal( - tmp_path: Path, renders: list[str] -) -> None: - config = make_config(tmp_path, snapshot_interval_seconds=60) - adapter = DGDRRunOutputAdapter(config, workload=None) - adapter.start() - adapter.on_candidate(record("a", 9.0)) - adapter.on_round(3, []) - - adapter.finish_search() - - snapshot = load(tmp_path) - assert snapshot["run"] == {"phase": "Running", "terminal": False} - assert tags(snapshot) == ["a"] - assert snapshot["progress"] == {"round": 3, "evaluated": 1} - # The writer is gone: nothing can overwrite the terminal snapshot written next. - assert adapter._thread is not None and not adapter._thread.is_alive() - writes = adapter.snapshot_writes - adapter.on_round(4, []) - time.sleep(0.1) - assert adapter.snapshot_writes == writes - - -def test_the_terminal_snapshot_after_a_completed_search_keeps_the_live_progress( - tmp_path: Path, renders: list[str], fake_search_config: None -) -> None: - callbacks = _subscribed(tmp_path) - callbacks.on_candidate(record("a", 9.0)) - callbacks.on_round(3, []) - callbacks.on_complete() - - create_adapter().write( - {**PLUGIN_CONFIG, "snapshot_dir": str(tmp_path)}, - result=_final_result(1), - output_dir=tmp_path, - ) - - snapshot = load(tmp_path) - assert snapshot["run"]["phase"] == "Succeeded" - assert snapshot["progress"] == {"round": 3, "evaluated": 1} - - -def test_failure_after_completion_is_a_noop_and_completion_after_failure_too( - tmp_path: Path, renders: list[str] -) -> None: - callbacks = _subscribed(tmp_path) - callbacks.on_candidate(record("a", 9.0)) - callbacks.on_failure(RuntimeError("first")) - callbacks.on_complete() - callbacks.on_failure(RuntimeError("second")) - assert load(tmp_path)["run"]["message"] == "first" - - # -- robustness ----------------------------------------------------------------- @@ -979,26 +963,6 @@ def broken(*_: Any, **__: Any) -> str: assert all(c["outcome"] != "materialization_failed" for c in snapshot["candidates"]) -def test_non_finite_scores_and_objectives_stay_valid_for_the_publisher( - tmp_path: Path, renders: list[str] -) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.on_candidate(record("a", float("-inf"), objectives={"e2e": float("nan")})) - adapter.on_candidate(record("b", float("nan"))) - adapter.close() - - text = (tmp_path / SNAPSHOT_FILE_NAME).read_text() - assert ( - "inf" not in text.lower().replace("information", "") - and "nan" not in text.lower() - ) - metrics = { - c["parameters"]["tag"]: c["metrics"] for c in load(tmp_path)["candidates"] - } - assert metrics["a"]["score"] is None and metrics["a"]["objectives"] == {"e2e": None} - assert metrics["b"]["score"] is None - - def test_a_candidate_error_message_is_bounded( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1014,37 +978,6 @@ def failing(*_: Any, **__: Any) -> str: assert 0 < len(candidate["error"]) <= 500 -def test_a_persistent_write_failure_does_not_spin_the_writer( - tmp_path: Path, renders: list[str], monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0.2) - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.start() - attempts = {"count": 0} - - def failing(directory: Path, snapshot: Any) -> Path: - attempts["count"] += 1 - raise OSError("disk full") - - monkeypatch.setattr(adapter_module, "write_snapshot", failing) - adapter.on_candidate(record("a", 1.0)) - time.sleep(0.6) - seen = attempts["count"] - monkeypatch.undo() - adapter.close() - assert 1 <= seen <= 5 - - -def test_plugin_write_keeps_the_objectives_of_the_selected_candidates( - tmp_path: Path, renders: list[str], fake_search_config: None -) -> None: - result = _final_result(1) - result.selected_candidates = [record("a", 9.0, objectives={"throughput": 12.5})] - create_adapter().write(PLUGIN_CONFIG, result=result, output_dir=tmp_path) - (candidate,) = load(tmp_path)["candidates"] - assert candidate["metrics"]["objectives"] == {"throughput": 12.5} - - def test_the_renderer_receives_the_candidates_gpu_count( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1087,13 +1020,28 @@ def test_every_finite_source_metric_survives_in_live_and_terminal_snapshots( "broken": None, "score": 87.7, "usedGpus": 4, + "objectives": {"goodput": 12.5, "invalid": None}, + } + source = { + **SOURCE_METRICS, + "score": 999.0, + "usedGpus": 99, + "objectives": 123.0, } live = tmp_path / "live" live.mkdir() adapter = DGDRRunOutputAdapter(make_config(live), workload=None) adapter.start() - adapter.on_candidate(record("a", 87.7, gpus=4, metrics=dict(SOURCE_METRICS))) + adapter.on_candidate( + record( + "a", + 87.7, + gpus=4, + metrics=source, + objectives={"goodput": 12.5, "invalid": float("nan")}, + ) + ) wait_for(lambda: tags(load(live)) == ["a"]) assert load(live)["candidates"][0]["metrics"] == expected adapter.close() @@ -1102,70 +1050,96 @@ def test_every_finite_source_metric_survives_in_live_and_terminal_snapshots( final.mkdir() result = _final_result(1) result.selected_candidates = [ - record("a", 87.7, gpus=4, metrics=dict(SOURCE_METRICS)) + record( + "a", + 87.7, + gpus=4, + metrics=source, + objectives={"goodput": 12.5, "invalid": float("nan")}, + ) ] create_adapter().write( {**PLUGIN_CONFIG, "snapshot_dir": str(final)}, result=result, output_dir=final ) assert load(final)["candidates"][0]["metrics"] == expected + for directory in (live, final): + text = (directory / SNAPSHOT_FILE_NAME).read_text().lower() + assert "nan" not in text and "inf" not in text.replace("information", "") -def test_reserved_metric_names_are_replaced_by_the_adapters_values( - tmp_path: Path, renders: list[str] -) -> None: - adapter = DGDRRunOutputAdapter(make_config(tmp_path), workload=None) - adapter.on_candidate( - record("a", 5.0, gpus=2, metrics={"score": 999.0, "usedGpus": 99, "ttft": 1.0}) - ) - adapter.close() - assert load(tmp_path)["candidates"][0]["metrics"] == { - "ttft": 1.0, - "score": 5.0, - "usedGpus": 2, - } - - -def test_a_selection_where_every_candidate_fails_to_materialize_is_a_failed_run( +@pytest.mark.parametrize( + ("failure_limits", "phase", "expected_outcomes", "expected_attempts"), + [ + pytest.param( + (99, 99), + "Failed", + ("materialization_failed", "materialization_failed"), + (3, 3), + id="all-fail-bounded", + ), + pytest.param( + (99, 0), + "Succeeded", + ("materialization_failed", "materialized"), + (3, 1), + id="partial-success", + ), + pytest.param( + (1,), + "Succeeded", + ("materialized",), + (2,), + id="transient-recovers", + ), + ], +) +def test_materialization_retry_and_terminal_policy( tmp_path: Path, - renders: list[str], fake_search_config: None, monkeypatch: pytest.MonkeyPatch, + failure_limits: tuple[int, ...], + phase: str, + expected_outcomes: tuple[str, ...], + expected_attempts: tuple[int, ...], ) -> None: monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0) - result = _final_result(2) - result.selected_candidates = [record("bad1", 9.0), record("bad2", 5.0)] - - with pytest.raises(CandidateMaterializationError, match="no selected candidate"): - create_adapter().write( - {**PLUGIN_CONFIG, "max_candidates": 2}, result=result, output_dir=tmp_path - ) - - snapshot = load(tmp_path) - assert snapshot["run"]["phase"] == "Failed" - assert snapshot["run"]["terminal"] is True - assert snapshot["run"]["error"] == "CandidateMaterializationError" - assert "cannot render bad1" in snapshot["run"]["message"] - assert len(snapshot["run"]["message"]) <= adapter_module._MAX_ERROR_MESSAGE - assert all(c["outcome"] == "materialization_failed" for c in snapshot["candidates"]) + attempts = [0] * len(failure_limits) + def flaky_render(candidate: Any, *_: Any, **__: Any) -> str: + index = int(candidate.config["tag"].removeprefix("candidate-")) + attempts[index] += 1 + if attempts[index] <= failure_limits[index]: + raise CandidateMaterializationError(f"cannot render candidate-{index}") + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" -def test_a_selection_with_one_materialized_candidate_still_succeeds( - tmp_path: Path, - renders: list[str], - fake_search_config: None, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(adapter_module, "_WRITE_RETRY_BACKOFF", 0) - result = _final_result(2) - result.selected_candidates = [record("bad1", 9.0), record("good", 5.0)] + monkeypatch.setattr(adapter_module, "render_dgd", flaky_render) + result = _final_result(len(failure_limits)) + result.selected_candidates = [ + record(f"candidate-{index}", float(len(failure_limits) - index)) + for index in range(len(failure_limits)) + ] - create_adapter().write( - {**PLUGIN_CONFIG, "max_candidates": 2}, result=result, output_dir=tmp_path - ) + if phase == "Failed": + with pytest.raises( + CandidateMaterializationError, match="no selected candidate" + ): + create_adapter().write( + {**PLUGIN_CONFIG, "max_candidates": len(failure_limits)}, + result=result, + output_dir=tmp_path, + ) + else: + create_adapter().write( + {**PLUGIN_CONFIG, "max_candidates": len(failure_limits)}, + result=result, + output_dir=tmp_path, + ) snapshot = load(tmp_path) - assert snapshot["run"]["phase"] == "Succeeded" - assert {c["parameters"]["tag"]: c["outcome"] for c in snapshot["candidates"]} == { - "bad1": "materialization_failed", - "good": "materialized", - } + assert snapshot["run"]["phase"] == phase + assert tuple(c["outcome"] for c in snapshot["candidates"]) == expected_outcomes + assert tuple(attempts) == expected_attempts + if phase == "Failed": + assert snapshot["run"]["error"] == "CandidateMaterializationError" + assert "cannot render candidate-0" in snapshot["run"]["message"] + assert len(snapshot["run"]["message"]) <= adapter_module._MAX_ERROR_MESSAGE From 61e0e58c8b8e0dbe7130497ccbf8f0b30f5a2c24 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 15:34:44 +0100 Subject: [PATCH 060/100] pr-15895: pr-7: resolve candidate topology [upstream: devivasudevan/dynamo#7] Signed-off-by: Dr. Stefan Schimanski --- .../aisimulate/output/dgdr_run/adapter.py | 97 +++++++- .../aisimulate/tests/dgdr_run/test_adapter.py | 212 ++++++++++++++++++ 2 files changed, 303 insertions(+), 6 deletions(-) diff --git a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py index 492b8c5030c7..0f82ffc7dcc9 100644 --- a/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py +++ b/components/src/dynamo/aisimulate/output/dgdr_run/adapter.py @@ -60,6 +60,7 @@ ) from aisimulate.sweeper.config import SmartSearchConfig from aisimulate.sweeper.result import SweepResult +from aisimulate_core.sdk.perf_database import load_system_spec from pydantic import BaseModel, ConfigDict, Field from dynamo.aisimulate.output.dgd.renderers import ( @@ -223,13 +224,79 @@ class DGDRRunOutputConfig(BaseModel): # the historical fixed-image behavior unless they opt into per-candidate derivation. derive_runtime_images: bool = False runtime_version_override: str | None = None - num_gpus_per_node: int = Field(gt=0) + # Standalone callers can continue supplying the node width directly; when both are + # present, the explicit value wins without consulting the candidate's system spec. + num_gpus_per_node: int | None = Field(default=None, gt=0) # The already-defaulted spec.recommendation.maxCandidates (API range 1..100). The adapter owns # no default of its own: a missing value must fail, not silently change the cardinality. max_candidates: int = Field(ge=1, le=100) snapshot_interval_seconds: float = Field(default=5.0, ge=0) - def generation_options(self, backend: str) -> DGDGenerationOptions: + def resolved_num_gpus_per_node( + self, candidate_parameters: Mapping[str, Any] | None = None + ) -> int: + """Return the explicit node width or resolve the candidate's concrete SKU.""" + if self.num_gpus_per_node is not None: + return self.num_gpus_per_node + + if candidate_parameters is None: + raise CandidateMaterializationError( + "candidate parameters are required when num_gpus_per_node is not provided" + ) + hardware_sku = candidate_parameters.get("hardware_sku") + if not isinstance(hardware_sku, str) or not hardware_sku.strip(): + raise CandidateMaterializationError( + "candidate parameters must contain a nonempty hardware_sku when " + "num_gpus_per_node is not provided" + ) + systems_paths = candidate_parameters.get("systems_paths") + if systems_paths is not None and ( + not isinstance(systems_paths, list) + or not systems_paths + or any( + not isinstance(path, str) or not path.strip() for path in systems_paths + ) + ): + raise CandidateMaterializationError( + "candidate systems_paths must be a nonempty list of nonempty strings" + ) + try: + system_spec = load_system_spec(hardware_sku, systems_paths=systems_paths) + except Exception as exc: + raise CandidateMaterializationError( + _bounded( + "could not load AISimulate system spec for hardware_sku " + f"{hardware_sku!r}" + ) + ) from exc + if not isinstance(system_spec, Mapping): + raise CandidateMaterializationError( + _bounded( + f"AISimulate system spec for hardware_sku {hardware_sku!r} " + "must be an object" + ) + ) + node_spec = system_spec.get("node") + if not isinstance(node_spec, Mapping): + raise CandidateMaterializationError( + _bounded( + f"AISimulate system spec for hardware_sku {hardware_sku!r} " + "must contain an object at node" + ) + ) + num_gpus_per_node = node_spec.get("num_gpus_per_node") + if type(num_gpus_per_node) is not int or num_gpus_per_node < 1: + raise CandidateMaterializationError( + _bounded( + f"AISimulate system spec for hardware_sku {hardware_sku!r} must " + "contain a positive integer at node.num_gpus_per_node" + ) + ) + return num_gpus_per_node + + def _generation_options( + self, backend: str, num_gpus_per_node: int + ) -> DGDGenerationOptions: runtime_image = ( derive_backend_image(self.runtime_image, backend) if self.derive_runtime_images @@ -239,7 +306,25 @@ def generation_options(self, backend: str) -> DGDGenerationOptions: runtime_image=runtime_image, runtime_version_override=self.runtime_version_override, namespace=self.namespace, - num_gpus_per_node=self.num_gpus_per_node, + num_gpus_per_node=num_gpus_per_node, + ) + + def validate_generation_options(self, backend: str) -> None: + """Fail fast on static rendering options before any candidate exists.""" + # A candidate's concrete topology arrives later; width 1 validates only image + # and version fields and is never passed to a renderer. + num_gpus_per_node = ( + self.num_gpus_per_node if self.num_gpus_per_node is not None else 1 + ) + self._generation_options(backend, num_gpus_per_node) + + def generation_options( + self, + backend: str, + candidate_parameters: Mapping[str, Any] | None = None, + ) -> DGDGenerationOptions: + return self._generation_options( + backend, self.resolved_num_gpus_per_node(candidate_parameters) ) @@ -275,7 +360,7 @@ def __init__( self._directory = Path(directory) self._config = config # Validate the source image and runtime version before the first callback. - config.generation_options("vllm") + config.validate_generation_options("vllm") self._workload = workload self._max_candidates = config.max_candidates self._interval = config.snapshot_interval_seconds @@ -611,7 +696,7 @@ def _materialize(self, item: _Retained) -> SnapshotCandidate: _RenderInput(config=item.parameters, used_gpus=item.used_gpus), self._workload, self._config.generation_options( - str(item.parameters.get("backend", "")) + str(item.parameters.get("backend", "")), item.parameters ), dgd_name=_dgd_name(self._config.name, item.id), renderer=self._config.generator, @@ -672,7 +757,7 @@ def subscribe( snapshot is still produced by ``write()``. """ resolved = DGDRRunOutputConfig.model_validate(dict(config)) - resolved.generation_options("vllm") + resolved.validate_generation_options("vllm") workload = getattr(context, "workload", None) if workload is None or resolved.snapshot_dir is None: return None diff --git a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py index a69355aff80b..c84e8cb14309 100644 --- a/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py +++ b/components/src/dynamo/aisimulate/tests/dgdr_run/test_adapter.py @@ -637,6 +637,218 @@ def test_configuration_boundaries( make_config(tmp_path, **overrides) +@pytest.mark.parametrize( + "candidate_parameters", + [ + pytest.param( + { + "hardware_sku": "concrete_candidate_sku", + "systems_paths": ["/external/systems"], + }, + id="operator-explicit-width", + ), + pytest.param({}, id="standalone-backwards-compatible"), + ], +) +def test_explicit_node_width_does_not_load_a_system_spec( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + candidate_parameters: dict[str, Any], +) -> None: + def unexpected_load(*_: Any, **__: Any) -> dict[str, Any]: + pytest.fail("an explicit num_gpus_per_node must bypass system-spec loading") + + monkeypatch.setattr(adapter_module, "load_system_spec", unexpected_load) + config = make_config(tmp_path, num_gpus_per_node=4) + + assert ( + config.generation_options("vllm", candidate_parameters).num_gpus_per_node == 4 + ) + + +@pytest.mark.parametrize( + ("hardware_sku", "num_gpus_per_node"), + [ + pytest.param("h200_sxm", 8, id="auto-resolved-concrete-catalog-sku"), + pytest.param("external_future_gpu", 16, id="external-systems-path-sku"), + ], +) +def test_concrete_candidate_sku_uses_aisimulates_system_spec_loader( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + hardware_sku: str, + num_gpus_per_node: int, +) -> None: + systems_paths = ( + ["/external/systems"] if hardware_sku.startswith("external") else None + ) + loaded: list[tuple[str, list[str] | None]] = [] + + def load(sku: str, systems_paths: list[str] | None = None) -> dict[str, Any]: + loaded.append((sku, systems_paths)) + return {"node": {"num_gpus_per_node": num_gpus_per_node}} + + monkeypatch.setattr(adapter_module, "load_system_spec", load) + config = make_config(tmp_path, num_gpus_per_node=None) + # AISimulate unrolls engine.hardware:auto and per-candidate choices before + # publishing the CandidateRecord, so its config always carries a concrete SKU. + candidate_parameters: dict[str, Any] = {"hardware_sku": hardware_sku} + if systems_paths is not None: + candidate_parameters["systems_paths"] = systems_paths + + assert ( + config.generation_options("vllm", candidate_parameters).num_gpus_per_node + == num_gpus_per_node + ) + assert loaded == [(hardware_sku, systems_paths)] + + +@pytest.mark.parametrize( + ("candidate_parameters", "message"), + [ + pytest.param(None, "candidate parameters are required", id="missing-candidate"), + pytest.param({}, "nonempty hardware_sku", id="missing-hardware-sku"), + pytest.param( + {"hardware_sku": ""}, "nonempty hardware_sku", id="empty-hardware-sku" + ), + pytest.param( + {"hardware_sku": "h200_sxm", "systems_paths": []}, + "candidate systems_paths", + id="empty-systems-paths", + ), + ], +) +def test_candidate_topology_source_is_required_at_materialization( + tmp_path: Path, + candidate_parameters: dict[str, Any] | None, + message: str, +) -> None: + config = make_config(tmp_path, num_gpus_per_node=None) + + with pytest.raises(CandidateMaterializationError, match=message): + config.generation_options("vllm", candidate_parameters) + + +@pytest.mark.parametrize( + "system_spec", + [ + pytest.param([], id="non-object-system-spec"), + pytest.param({}, id="missing-node"), + pytest.param({"node": None}, id="non-object-node"), + pytest.param({"node": {}}, id="missing-node-width"), + pytest.param({"node": {"num_gpus_per_node": 0}}, id="non-positive-node-width"), + pytest.param({"node": {"num_gpus_per_node": True}}, id="boolean-node-width"), + pytest.param({"node": {"num_gpus_per_node": "8"}}, id="string-node-width"), + ], +) +def test_malformed_system_spec_topology_is_rejected( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + system_spec: Any, +) -> None: + monkeypatch.setattr( + adapter_module, + "load_system_spec", + lambda _, systems_paths=None: system_spec, + ) + config = make_config(tmp_path, num_gpus_per_node=None) + + with pytest.raises( + CandidateMaterializationError, + match="AISimulate system spec.*(object|positive integer)", + ): + config.generation_options("vllm", {"hardware_sku": "malformed_system"}) + + +def test_system_spec_topology_errors_are_bounded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + adapter_module, + "load_system_spec", + lambda _, systems_paths=None: {}, + ) + config = make_config(tmp_path, num_gpus_per_node=None) + + with pytest.raises(CandidateMaterializationError) as raised: + config.generation_options("vllm", {"hardware_sku": "x" * 1000}) + + assert len(str(raised.value)) == adapter_module._MAX_ERROR_MESSAGE + + +def test_custom_candidate_system_root_reaches_candidate_materialization( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + materialized_node_widths: list[int] = [] + systems_root = tmp_path / "external-systems" + systems_root.mkdir() + (systems_root / "custom_twelve_gpu_node.yaml").write_text( + "node:\n num_gpus_per_node: 12\n", + encoding="utf-8", + ) + + def capture_node_width( + candidate: Any, workload: Any, options: Any, **kwargs: Any + ) -> str: + del candidate, workload, kwargs + materialized_node_widths.append(options.num_gpus_per_node) + return "apiVersion: nvidia.com/v1beta1\nkind: DynamoGraphDeployment\n" + + monkeypatch.setattr(adapter_module, "render_dgd", capture_node_width) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, num_gpus_per_node=None), + workload=None, + ) + candidate = record("a", 1.0) + candidate.config.update( + { + "hardware_sku": "custom_twelve_gpu_node", + "systems_paths": [str(systems_root)], + } + ) + adapter.on_candidate(candidate) + adapter.close() + + assert materialized_node_widths == [12] + assert load(tmp_path)["candidates"][0]["outcome"] == "materialized" + + +def test_invalid_candidate_topology_does_not_abort_other_candidates( + tmp_path: Path, + renders: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + def load_candidate_system( + sku: str, systems_paths: list[str] | None = None + ) -> dict[str, Any]: + del systems_paths + if sku == "broken_system": + return {"node": {}} + return {"node": {"num_gpus_per_node": 8}} + + monkeypatch.setattr(adapter_module, "load_system_spec", load_candidate_system) + adapter = DGDRRunOutputAdapter( + make_config(tmp_path, num_gpus_per_node=None, max_candidates=2), + workload=None, + ) + broken = record("broken-topology", 2.0) + broken.config["hardware_sku"] = "broken_system" + valid = record("valid-topology", 1.0) + valid.config["hardware_sku"] = "h200_sxm" + + adapter.on_candidate(broken) + adapter.on_candidate(valid) + adapter.close() + + snapshot = load(tmp_path) + assert snapshot["run"] == {"phase": "Succeeded", "terminal": True} + assert [candidate["outcome"] for candidate in snapshot["candidates"]] == [ + "materialization_failed", + "materialized", + ] + assert renders == ["valid-topology"] + + def test_max_candidates_has_no_default_of_its_own() -> None: # The operator supplies the already-defaulted API value; a missing one must fail. with pytest.raises(ValueError, match="max_candidates"): From b9f01391ecca3c543bb9790af000e86424143800 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 15:57:36 +0100 Subject: [PATCH 061/100] pr-17195: harden frozen Run execution [upstream: #17195] Signed-off-by: Dr. Stefan Schimanski --- ...mographdeploymentrun_v1beta2_controller.go | 573 +++++++++---- ...phdeploymentrun_v1beta2_controller_test.go | 764 +++++++++++++++--- deploy/operator/internal/controller/setup.go | 31 +- 3 files changed, 1065 insertions(+), 303 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 0c180d220a5a..f9b10fb3b046 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -17,12 +17,10 @@ import ( "time" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/aisimulate" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -54,7 +52,6 @@ const ( dgdrRunPublisherImageKey = "publisher-image" dgdrRunPublisherPullPolicyKey = "publisher-image-pull-policy" dgdrRunDynamoVersionKey = "dynamo-version" - dgdrRunGPUsPerNodeKey = "gpus-per-node" dgdrRunSnapshotPathKey = "snapshot-path" dgdrRunImagePullSecretsKey = "image-pull-secrets.json" dgdrRunInputMountPath = "/etc/aisimulate" @@ -67,7 +64,6 @@ const ( dgdrRunInputVolume = "input" dgdrRunSnapshotVolume = "snapshot" dgdrRunServiceAccountVolume = "publisher-kube-api" - dgdrRunGPUsPerNodeAnnotation = "nvidia.com/dgdr-gpus-per-node" dgdrRunSnapshotPathAnnotation = "nvidia.com/dgdr-snapshot-path" dgdrRunResourceComponent = "aisimulate" dgdrRunServiceAccountTokenTTL = int64(3600) @@ -103,10 +99,6 @@ type dgdrRunRetryableError struct { func (e *dgdrRunRetryableError) Error() string { return e.cause.Error() } func (e *dgdrRunRetryableError) Unwrap() error { return e.cause } -type dgdrRunHardwareDiscoverer interface { - Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) -} - // DynamoGraphDeploymentRunV1beta2Reconciler executes one immutable Run as an // AISimulate Job. The publisher sidecar owns progress and candidate publication; // this controller owns the Job reference, execution phase, timestamps and Completed. @@ -115,7 +107,6 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { APIReader client.Reader Config *configv1alpha1.OperatorConfiguration RuntimeConfig *commoncontroller.RuntimeConfig - Hardware dgdrRunHardwareDiscoverer PlannerImage string DynamoVersion string OperatorImage string @@ -130,7 +121,6 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates/status,verbs=get;update;patch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch @@ -145,6 +135,9 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( if run.DeletionTimestamp != nil || meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { return ctrl.Result{}, nil } + if condition := dgdrRunPendingPermanentFailure(run); condition != nil { + return r.finishPermanentFailure(ctx, run, condition.Reason, errors.New(condition.Message)) + } result, err := r.reconcileRun(ctx, run) if err == nil || !dgdrRunIsPermanentError(err) { return result, err @@ -155,10 +148,67 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( } else if apierrors.IsInvalid(err) { reason = dgdrRunReasonInvalidResource } - if statusErr := r.completeRun(ctx, run, reason, err.Error()); statusErr != nil { - return ctrl.Result{}, statusErr + return r.beginPermanentFailure(ctx, run, reason, err) +} + +func dgdrRunPendingPermanentFailure(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) *metav1.Condition { + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionFalse { + return nil + } + switch condition.Reason { + case dgdrRunReasonInvalidExecution, dgdrRunReasonInvalidResource, commoncontroller.EventReasonOwnershipConflict: + return condition + default: + return nil + } +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) beginPermanentFailure( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + reason string, + cause error, +) (ctrl.Result, error) { + original := run.DeepCopy() + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionFalse, + Reason: reason, + Message: cause.Error(), + ObservedGeneration: run.Generation, + }) + if !apiequality.Semantic.DeepEqual(original.Status, run.Status) { + if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { + return ctrl.Result{}, &dgdrRunRetryableError{cause: fmt.Errorf("record permanent failure for Run %q: %w", run.Name, err)} + } } - return ctrl.Result{}, reconcile.TerminalError(err) + // Deletion is a later observed-state transition. Returning here ensures the + // controller never acts on a marker that only exists in this local object. + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil +} + +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) finishPermanentFailure( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + reason string, + cause error, +) (ctrl.Result, error) { + jobs, err := r.ownedJobs(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + stopped, err := r.stopOwnedJobs(ctx, run, jobs, "after permanent failure") + if err != nil { + return ctrl.Result{}, err + } + if !stopped { + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil + } + if err := r.completeRun(ctx, run, reason, cause.Error()); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{}, reconcile.TerminalError(cause) } func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRun( @@ -179,7 +229,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRun( job = &jobs[0] } if supersedingGeneration, cancelRequested := dgdrRunSupersedingGeneration(run); cancelRequested { - return ctrl.Result{}, r.cancelSupersededRun(ctx, run, job, supersedingGeneration) + return r.cancelSupersededRun(ctx, run, job, supersedingGeneration) } resourceName := dgdrRunResourceName(run.Name) if err := r.reconcilePublisherAccess(ctx, run, resourceName); err != nil { @@ -226,25 +276,53 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) cancelSupersededRun( run *nvidiacomv1beta2.DynamoGraphDeploymentRun, job *batchv1.Job, supersedingGeneration string, -) error { +) (ctrl.Result, error) { + var cached []batchv1.Job if job != nil { - if job.DeletionTimestamp.IsZero() { - err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationForeground)) - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("stop Job %q for superseded Run %q: %w", job.Name, run.Name, err) - } - if err == nil { - return nil - } - } else { - return nil - } + cached = []batchv1.Job{*job} + } + stopped, err := r.stopOwnedJobs(ctx, run, cached, "for supersession") + if err != nil { + return ctrl.Result{}, err + } + if !stopped { + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil } - return r.completeRun(ctx, run, dgdrRunReasonSuperseded, + return ctrl.Result{}, r.completeRun(ctx, run, dgdrRunReasonSuperseded, fmt.Sprintf("Run was superseded by Request generation %s", supersedingGeneration)) } +// stopOwnedJobs stops every execution visible through either the informer cache or the API +// server. Completion is safe only after the uncached reader also confirms absence. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) stopOwnedJobs( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + cached []batchv1.Job, + action string, +) (bool, error) { + jobs := cached + if len(jobs) == 0 { + var err error + jobs, err = r.uncachedOwnedJobs(ctx, run) + if err != nil { + return false, err + } + } + if len(jobs) == 0 { + return true, nil + } + for i := range jobs { + if !jobs[i].DeletionTimestamp.IsZero() { + continue + } + if err := r.Delete(ctx, &jobs[i], client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !apierrors.IsNotFound(err) { + return false, fmt.Errorf("stop Job %q %s of Run %q: %w", jobs[i].Name, action, run.Name, err) + } + } + return false, nil +} + func (r *DynamoGraphDeploymentRunV1beta2Reconciler) completeRun( ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, @@ -299,6 +377,26 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) ownedJobs( return owned, nil } +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) uncachedOwnedJobs( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) ([]batchv1.Job, error) { + if r.APIReader == nil { + return nil, errors.New("API reader is required to confirm Job absence") + } + list := &batchv1.JobList{} + if err := r.APIReader.List(ctx, list, client.InNamespace(run.Namespace)); err != nil { + return nil, fmt.Errorf("list Jobs of Run %q from the API: %w", run.Name, err) + } + owned := make([]batchv1.Job, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], run) { + owned = append(owned, list.Items[i]) + } + } + return owned, nil +} + func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcilePublisherAccess( ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, @@ -430,15 +528,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( if execution.PublisherPullPolicy == "" { execution.PublisherPullPolicy = corev1.PullIfNotPresent } - execution.GPUsPerNode, err = r.discoverGPUsPerNode(ctx, run) - if err != nil { - return dgdrRunExecution{}, err - } execution.DynamoVersion, err = dgdrRunDynamoVersion(execution.PlannerImage, r.DynamoVersion) if err != nil { return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: err} } - execution.RuntimeImageSource, err = dgdrRunRuntimeImageSource(execution.PlannerImage) + execution.RuntimeImageSource, err = dgdrRunRuntimeImageSource(execution.PlannerImage, execution.DynamoVersion) if err != nil { return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: err} } @@ -447,7 +541,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( return dgdrRunExecution{}, err } - input, err := dgdrRunInput(run, execution.RuntimeImageSource, execution.DynamoVersion, execution.GPUsPerNode) + input, err := dgdrRunInput(run, execution.RuntimeImageSource, execution.DynamoVersion) if err != nil { return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: err} } @@ -468,7 +562,6 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( Name: name, Namespace: run.Namespace, Annotations: map[string]string{ - dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, }, Immutable: new(true), @@ -480,7 +573,6 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileInput( dgdrRunPublisherImageKey: execution.PublisherImage, dgdrRunPublisherPullPolicyKey: string(execution.PublisherPullPolicy), dgdrRunDynamoVersionKey: execution.DynamoVersion, - dgdrRunGPUsPerNodeKey: strconv.Itoa(execution.GPUsPerNode), dgdrRunSnapshotPathKey: execution.SnapshotPath, dgdrRunImagePullSecretsKey: string(pullSecrets), }, @@ -513,12 +605,16 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) existingRunExecution( if err := commoncontroller.CheckControllerOwnership(configMap, run, r.Scheme()); err != nil { return dgdrRunExecution{}, err } + if configMap.Immutable == nil || !*configMap.Immutable { + return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf( + "execution ConfigMap %q must be immutable", configMap.Name)} + } execution, err := dgdrRunExecutionFromConfigMap(configMap) if err != nil { return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf("read ConfigMap %q: %w", configMap.Name, err)} } if job != nil { - if err := validateDGDRRunJobExecution(job, execution); err != nil { + if err := validateDGDRRunJobExecution(job, run, configMap.Name, execution); err != nil { return dgdrRunExecution{}, &dgdrRunPermanentExecutionError{cause: fmt.Errorf( "Job %q and ConfigMap %q disagree: %w", job.Name, configMap.Name, err)} } @@ -526,43 +622,10 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) existingRunExecution( return execution, nil } -func (r *DynamoGraphDeploymentRunV1beta2Reconciler) discoverGPUsPerNode( - ctx context.Context, - run *nvidiacomv1beta2.DynamoGraphDeploymentRun, -) (int, error) { - if r.Hardware == nil { - return 0, &dgdrRunRetryableError{cause: errors.New("discover GPUs per node: hardware discoverer is not configured")} - } - if run.Spec.Hardware == nil || run.Spec.Hardware.GPU == nil || run.Spec.Hardware.GPU.SKU == nil { - return 0, &dgdrRunPermanentExecutionError{cause: errors.New( - "discover GPUs per node: spec.hardware.gpu.sku was not resolved by Request admission")} - } - sku := *run.Spec.Hardware.GPU.SKU - filterSKU := nvidiacomv1beta1.GPUSKUType("") - if aisimulate.IsKnownHardwareSKU(sku) { - filterSKU = nvidiacomv1beta1.GPUSKUType(sku) - } - info, err := r.Hardware.Discover(ctx, filterSKU) - if err != nil { - return 0, &dgdrRunRetryableError{cause: fmt.Errorf("discover GPUs per node: %w", err)} - } - if filterSKU == "" && info != nil && len(info.OtherModels) > 0 { - return 0, &dgdrRunRetryableError{cause: fmt.Errorf( - "discover GPUs per node for external AISimulate SKU %q: cluster discovery selected %q among multiple GPU models (%s)", - sku, info.Model, strings.Join(info.OtherModels, ", "), - )} - } - if info == nil || info.GPUsPerNode < 1 { - return 0, &dgdrRunRetryableError{cause: errors.New("discover GPUs per node: discovery returned no positive GPU count")} - } - return info.GPUsPerNode, nil -} - func dgdrRunInput( run *nvidiacomv1beta2.DynamoGraphDeploymentRun, runtimeImageSource string, dynamoVersion string, - gpusPerNode int, ) (map[string]any, error) { if run.Spec.Recommendation == nil || run.Spec.Recommendation.MaxCandidates < 1 { return nil, errors.New("spec.recommendation.maxCandidates must be defaulted before the Run is created") @@ -586,7 +649,9 @@ func dgdrRunInput( "runtime_image": runtimeImageSource, "runtime_version_override": dynamoVersion, "derive_runtime_images": true, - "num_gpus_per_node": int64(gpusPerNode), + } + if run.Spec.Hardware != nil && run.Spec.Hardware.NumGPUsPerNode != nil { + reserved["num_gpus_per_node"] = int64(*run.Spec.Hardware.NumGPUsPerNode) } for key, value := range reserved { if err := unstructured.SetNestedField(input, value, "dgdr_run", key); err != nil { @@ -597,7 +662,6 @@ func dgdrRunInput( } type dgdrRunExecution struct { - GPUsPerNode int SnapshotPath string PlannerImage string PlannerPullPolicy corev1.PullPolicy @@ -620,18 +684,15 @@ func dgdrRunDynamoVersion(plannerImage, dynamoVersion string) (string, error) { return version.String(), nil } -func dgdrRunRuntimeImageSource(plannerImage string) (string, error) { +func dgdrRunRuntimeImageSource(plannerImage, dynamoVersion string) (string, error) { ref := strings.TrimSpace(plannerImage) nameAndTag, _, hasDigest := strings.Cut(ref, "@") lastSlash, lastColon := strings.LastIndex(nameAndTag, "/"), strings.LastIndex(nameAndTag, ":") if lastColon > lastSlash && lastColon < len(nameAndTag)-1 { - return ref, nil + return nameAndTag, nil } if hasDigest { - return "", fmt.Errorf( - "resolve runtime images for planner image %q: a digest-only planner image cannot identify sibling runtime images; configure a shared release tag, optionally with a digest", - plannerImage, - ) + return nameAndTag + ":" + dynamoVersion, nil } return "", fmt.Errorf( "resolve runtime images for planner image %q: a shared release tag is required, optionally with a repository-specific digest", @@ -698,12 +759,8 @@ func dgdrRunExecutionFromConfigMap(configMap *corev1.ConfigMap) (dgdrRunExecutio return dgdrRunExecution{}, fmt.Errorf("data key %q=%q is not a valid image pull policy", dgdrRunPublisherPullPolicyKey, execution.PublisherPullPolicy) } - // Decode the immutable topology, credentials and effective input. + // Decode the immutable credentials and effective input. var err error - execution.GPUsPerNode, err = positiveInteger(configMap.Data[dgdrRunGPUsPerNodeKey], "data key "+dgdrRunGPUsPerNodeKey) - if err != nil { - return dgdrRunExecution{}, err - } if err := json.Unmarshal([]byte(configMap.Data[dgdrRunImagePullSecretsKey]), &execution.ImagePullSecrets); err != nil { return dgdrRunExecution{}, fmt.Errorf("decode data key %q: %w", dgdrRunImagePullSecretsKey, err) } @@ -714,33 +771,245 @@ func dgdrRunExecutionFromConfigMap(configMap *corev1.ConfigMap) (dgdrRunExecutio return execution, nil } -func validateDGDRRunJobExecution(job *batchv1.Job, execution dgdrRunExecution) error { - jobGPUs, err := positiveAnnotation(job.Annotations, dgdrRunGPUsPerNodeAnnotation) - if err != nil { +func validateDGDRRunJobExecution( + job *batchv1.Job, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + inputConfigMapName string, + execution dgdrRunExecution, +) error { + if job.Annotations[dgdrRunSnapshotPathAnnotation] != execution.SnapshotPath { + return errors.New("snapshot paths differ") + } + if err := validateDGDRRunJobSpec(&job.Spec); err != nil { return err } - if jobGPUs != execution.GPUsPerNode { - return errors.New("GPU counts differ") + pod := &job.Spec.Template.Spec + if pod.RestartPolicy != corev1.RestartPolicyNever { + return errors.New("pod restart policy differs") } - if job.Annotations[dgdrRunSnapshotPathAnnotation] != execution.SnapshotPath { - return errors.New("snapshot paths differ") + if pod.HostPID || pod.HostIPC || pod.HostNetwork || + (pod.ShareProcessNamespace != nil && *pod.ShareProcessNamespace) { + return errors.New("pod namespace isolation differs") + } + if len(pod.InitContainers) != 0 { + return errors.New("init containers are not allowed") + } + if err := validateDGDRRunPodSecurityContext(pod.SecurityContext); err != nil { + return err + } + if pod.ServiceAccountName != inputConfigMapName || pod.AutomountServiceAccountToken == nil || *pod.AutomountServiceAccountToken { + return errors.New("publisher service-account identities differ") + } + if len(pod.Containers) != 2 { + return fmt.Errorf("container count differs: got %d, want 2", len(pod.Containers)) } - // Match both container identities and all credentials against the frozen artifact. - planner, found := dgdrRunContainer(job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) - if !found || planner.Image != execution.PlannerImage || planner.ImagePullPolicy != execution.PlannerPullPolicy { + // Match both container identities, their immutable bindings and all credentials. + planner, found := dgdrRunContainer(pod.Containers, dgdrRunSweeperContainer) + if !found { return errors.New("planner image identities differ") } - publisher, found := dgdrRunContainer(job.Spec.Template.Spec.Containers, dgdrRunPublisherContainer) - if !found || publisher.Image != execution.PublisherImage || publisher.ImagePullPolicy != execution.PublisherPullPolicy { + if err := validateDGDRRunPlannerContainer(&planner, execution); err != nil { + return err + } + publisher, found := dgdrRunContainer(pod.Containers, dgdrRunPublisherContainer) + if !found { return errors.New("publisher image identities differ") } - if !apiequality.Semantic.DeepEqual(job.Spec.Template.Spec.ImagePullSecrets, execution.ImagePullSecrets) { + if err := validateDGDRRunPublisherContainer(&publisher, run, execution); err != nil { + return err + } + if err := validateDGDRRunVolumes(pod.Volumes, inputConfigMapName); err != nil { + return err + } + if !apiequality.Semantic.DeepEqual(pod.ImagePullSecrets, execution.ImagePullSecrets) { return errors.New("image pull secrets differ") } return nil } +func validateDGDRRunPlannerContainer(planner *corev1.Container, execution dgdrRunExecution) error { + if planner.Image != execution.PlannerImage || planner.ImagePullPolicy != execution.PlannerPullPolicy { + return errors.New("planner image identities differ") + } + if err := validateDGDRRunContainerSecurityContext(planner.SecurityContext, ptr.To[int64](1000)); err != nil { + return fmt.Errorf("planner security context differs: %w", err) + } + if !apiequality.Semantic.DeepEqual(planner.Command, dgdrRunSweeperCommand()) || + !apiequality.Semantic.DeepEqual(planner.Args, dgdrRunSweeperArgs(execution.SnapshotPath)) { + return errors.New("planner command or arguments differ") + } + if len(planner.Env) != 0 || len(planner.EnvFrom) != 0 { + return errors.New("planner environment differs") + } + if planner.Lifecycle != nil { + return errors.New("planner lifecycle differs") + } + if !apiequality.Semantic.DeepEqual(planner.VolumeMounts, dgdrRunSweeperVolumeMounts(execution.SnapshotPath)) { + return errors.New("planner volume mounts differ") + } + return nil +} + +func validateDGDRRunPublisherContainer( + publisher *corev1.Container, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + execution dgdrRunExecution, +) error { + if publisher.Image != execution.PublisherImage || publisher.ImagePullPolicy != execution.PublisherPullPolicy { + return errors.New("publisher image identities differ") + } + if err := validateDGDRRunContainerSecurityContext(publisher.SecurityContext, nil); err != nil { + return fmt.Errorf("publisher security context differs: %w", err) + } + if !apiequality.Semantic.DeepEqual(publisher.Command, dgdrRunPublisherCommand()) || + !apiequality.Semantic.DeepEqual(publisher.Args, dgdrRunPublisherArgs(run, execution.SnapshotPath)) { + return errors.New("publisher command, arguments or identity differ") + } + if err := validateDGDRRunPublisherEnv(publisher.Env); err != nil { + return err + } + if len(publisher.EnvFrom) != 0 { + return errors.New("publisher environment differs") + } + if publisher.Lifecycle != nil { + return errors.New("publisher lifecycle differs") + } + if !apiequality.Semantic.DeepEqual(publisher.VolumeMounts, dgdrRunPublisherVolumeMounts(execution.SnapshotPath)) { + return errors.New("publisher volume mounts differ") + } + return nil +} + +func validateDGDRRunJobSpec(spec *batchv1.JobSpec) error { + if spec.BackoffLimit == nil || *spec.BackoffLimit != 0 { + return errors.New("Job backoff limit differs") + } + if (spec.Parallelism != nil && *spec.Parallelism != 1) || + (spec.Completions != nil && *spec.Completions != 1) { + return errors.New("Job cardinality differs") + } + if spec.CompletionMode != nil && *spec.CompletionMode != batchv1.NonIndexedCompletion { + return errors.New("Job completion mode differs") + } + if (spec.Suspend != nil && *spec.Suspend) || (spec.ManualSelector != nil && *spec.ManualSelector) { + return errors.New("Job execution policy differs") + } + if spec.ActiveDeadlineSeconds != nil || spec.PodFailurePolicy != nil || spec.SuccessPolicy != nil || + spec.BackoffLimitPerIndex != nil || spec.MaxFailedIndexes != nil || + spec.TTLSecondsAfterFinished != nil || + (spec.PodReplacementPolicy != nil && *spec.PodReplacementPolicy != batchv1.TerminatingOrFailed) || + spec.ManagedBy != nil { + return errors.New("Job execution policy differs") + } + return nil +} + +func validateDGDRRunPodSecurityContext(security *corev1.PodSecurityContext) error { + if security == nil || security.FSGroup == nil || *security.FSGroup != 65532 { + return errors.New("pod fsGroup differs") + } + if security.RunAsNonRoot == nil || !*security.RunAsNonRoot { + return errors.New("pod runAsNonRoot differs") + } + if security.SeccompProfile == nil || security.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + return errors.New("pod seccomp profile differs") + } + return nil +} + +func validateDGDRRunContainerSecurityContext(security *corev1.SecurityContext, runAsUser *int64) error { + if security == nil || security.AllowPrivilegeEscalation == nil || *security.AllowPrivilegeEscalation { + return errors.New("allowPrivilegeEscalation must be false") + } + if security.Privileged != nil && *security.Privileged { + return errors.New("privileged must be false") + } + if security.RunAsNonRoot != nil && !*security.RunAsNonRoot { + return errors.New("runAsNonRoot cannot be disabled") + } + if security.SeccompProfile != nil && security.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + return errors.New("seccomp profile cannot weaken the pod default") + } + if security.ProcMount != nil && *security.ProcMount != corev1.DefaultProcMount { + return errors.New("procMount cannot weaken the container default") + } + if security.Capabilities == nil || len(security.Capabilities.Add) != 0 || + !apiequality.Semantic.DeepEqual(security.Capabilities.Drop, []corev1.Capability{"ALL"}) { + return errors.New("all capabilities must be dropped") + } + if !apiequality.Semantic.DeepEqual(security.RunAsUser, runAsUser) { + return errors.New("runAsUser differs") + } + return nil +} + +func validateDGDRRunPublisherEnv(env []corev1.EnvVar) error { + if len(env) != 2 { + return errors.New("publisher pod identity environment differs") + } + for name, fieldPath := range map[string]string{ + "POD_NAMESPACE": "metadata.namespace", + "POD_NAME": "metadata.name", + } { + found := false + for i := range env { + if env[i].Name == name && env[i].Value == "" && env[i].ValueFrom != nil && + env[i].ValueFrom.FieldRef != nil && env[i].ValueFrom.FieldRef.FieldPath == fieldPath { + found = true + break + } + } + if !found { + return errors.New("publisher pod identity environment differs") + } + } + return nil +} + +func validateDGDRRunVolumes(volumes []corev1.Volume, inputConfigMapName string) error { + if len(volumes) != 3 { + return errors.New("input, snapshot or publisher credential volumes differ") + } + input, found := dgdrRunVolume(volumes, dgdrRunInputVolume) + if !found || input.ConfigMap == nil || input.ConfigMap.Name != inputConfigMapName || + len(input.ConfigMap.Items) != 0 || input.ConfigMap.Optional != nil || !dgdrRunDefaultMode(input.ConfigMap.DefaultMode) { + return errors.New("input, snapshot or publisher credential volumes differ") + } + snapshot, found := dgdrRunVolume(volumes, dgdrRunSnapshotVolume) + if !found || snapshot.EmptyDir == nil || !apiequality.Semantic.DeepEqual(snapshot.EmptyDir, &corev1.EmptyDirVolumeSource{}) { + return errors.New("input, snapshot or publisher credential volumes differ") + } + publisher, found := dgdrRunVolume(volumes, dgdrRunServiceAccountVolume) + if !found || publisher.Projected == nil || + !dgdrRunDefaultMode(publisher.Projected.DefaultMode) || + !apiequality.Semantic.DeepEqual(publisher.Projected.Sources, dgdrRunPublisherKubeAPIVolume().Projected.Sources) { + return errors.New("input, snapshot or publisher credential volumes differ") + } + return nil +} + +func dgdrRunDefaultMode(mode *int32) bool { + return mode == nil || *mode == int32(0o644) +} + +func dgdrRunVolume(volumes []corev1.Volume, name string) (corev1.Volume, bool) { + var found *corev1.Volume + for i := range volumes { + if volumes[i].Name != name { + continue + } + if found != nil { + return corev1.Volume{}, false + } + found = &volumes[i] + } + if found == nil { + return corev1.Volume{}, false + } + return *found, true +} + func dgdrRunContainer(containers []corev1.Container, name string) (corev1.Container, bool) { for _, container := range containers { if container.Name == name { @@ -765,18 +1034,6 @@ func dgdrRunStatusInput(contents string) (*runtime.RawExtension, error) { return &runtime.RawExtension{Raw: raw}, nil } -func positiveAnnotation(annotations map[string]string, key string) (int, error) { - return positiveInteger(annotations[key], "annotation "+key) -} - -func positiveInteger(value, field string) (int, error) { - parsed, err := strconv.Atoi(value) - if err != nil || parsed < 1 { - return 0, fmt.Errorf("%s=%q is not a positive integer", field, value) - } - return parsed, nil -} - func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( run *nvidiacomv1beta2.DynamoGraphDeploymentRun, name string, @@ -790,7 +1047,6 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( Name: name, Namespace: run.Namespace, Annotations: map[string]string{ - dgdrRunGPUsPerNodeAnnotation: strconv.Itoa(execution.GPUsPerNode), dgdrRunSnapshotPathAnnotation: execution.SnapshotPath, }, Spec: batchv1.JobSpec{ @@ -813,52 +1069,23 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( Name: dgdrRunSweeperContainer, Image: execution.PlannerImage, ImagePullPolicy: execution.PlannerPullPolicy, - Command: []string{"aisimulate"}, - Args: []string{ - "recommend", "--stack", "dynamo", - "--config", dgdrRunInputPath, - "--output", "dgdr_run", - "--output-dir", execution.SnapshotPath, - "--overwrite", - }, + Command: dgdrRunSweeperCommand(), + Args: dgdrRunSweeperArgs(execution.SnapshotPath), SecurityContext: sweeperSecurityContext, - VolumeMounts: []corev1.VolumeMount{ - {Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true}, - {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, - }, + VolumeMounts: dgdrRunSweeperVolumeMounts(execution.SnapshotPath), }, { Name: dgdrRunPublisherContainer, Image: execution.PublisherImage, ImagePullPolicy: execution.PublisherPullPolicy, - Command: []string{dgdrRunPublisherBinary}, - Args: []string{ - "--snapshot-dir", execution.SnapshotPath, - "--run-name", run.Name, - "--run-uid", string(run.UID), - "--sweeper-container", dgdrRunSweeperContainer, - }, - Env: []corev1.EnvVar{ - {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, - {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, - }, + Command: dgdrRunPublisherCommand(), + Args: dgdrRunPublisherArgs(run, execution.SnapshotPath), + Env: dgdrRunPublisherEnv(), SecurityContext: dgdrRunContainerSecurityContext(), - VolumeMounts: []corev1.VolumeMount{ - {Name: dgdrRunSnapshotVolume, MountPath: execution.SnapshotPath}, - {Name: dgdrRunServiceAccountVolume, MountPath: dgdrRunServiceAccountMountPath, ReadOnly: true}, - }, + VolumeMounts: dgdrRunPublisherVolumeMounts(execution.SnapshotPath), }, }, - Volumes: []corev1.Volume{ - { - Name: dgdrRunInputVolume, - ConfigMap: &corev1.ConfigMapVolumeSource{ - Name: name, - }, - }, - {Name: dgdrRunSnapshotVolume, EmptyDir: &corev1.EmptyDirVolumeSource{}}, - dgdrRunPublisherKubeAPIVolume(), - }, + Volumes: dgdrRunVolumes(name), }, }, }, @@ -866,6 +1093,62 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( return job } +func dgdrRunSweeperCommand() []string { + return []string{"aisimulate"} +} + +func dgdrRunSweeperArgs(snapshotPath string) []string { + return []string{ + "recommend", "--stack", "dynamo", + "--config", dgdrRunInputPath, + "--output", "dgdr_run", + "--output-dir", snapshotPath, + "--overwrite", + } +} + +func dgdrRunPublisherCommand() []string { + return []string{dgdrRunPublisherBinary} +} + +func dgdrRunPublisherArgs(run *nvidiacomv1beta2.DynamoGraphDeploymentRun, snapshotPath string) []string { + return []string{ + "--snapshot-dir", snapshotPath, + "--run-name", run.Name, + "--run-uid", string(run.UID), + "--sweeper-container", dgdrRunSweeperContainer, + } +} + +func dgdrRunPublisherEnv() []corev1.EnvVar { + return []corev1.EnvVar{ + {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, + {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, + } +} + +func dgdrRunSweeperVolumeMounts(snapshotPath string) []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true}, + {Name: dgdrRunSnapshotVolume, MountPath: snapshotPath}, + } +} + +func dgdrRunPublisherVolumeMounts(snapshotPath string) []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: dgdrRunSnapshotVolume, MountPath: snapshotPath}, + {Name: dgdrRunServiceAccountVolume, MountPath: dgdrRunServiceAccountMountPath, ReadOnly: true}, + } +} + +func dgdrRunVolumes(inputConfigMapName string) []corev1.Volume { + return []corev1.Volume{ + {Name: dgdrRunInputVolume, ConfigMap: &corev1.ConfigMapVolumeSource{Name: inputConfigMapName}}, + {Name: dgdrRunSnapshotVolume, EmptyDir: &corev1.EmptyDirVolumeSource{}}, + dgdrRunPublisherKubeAPIVolume(), + } +} + func dgdrRunContainerSecurityContext() *corev1.SecurityContext { return &corev1.SecurityContext{ AllowPrivilegeEscalation: new(false), diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 75ca7a5358ec..f97d199f9fe6 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -14,11 +14,9 @@ import ( "testing" "time" - nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dgdrrunpublisher" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -40,12 +38,6 @@ import ( "sigs.k8s.io/yaml" ) -type fixedRunHardware struct { - info gpu.GPUInfo - err error - calls []nvidiacomv1beta1.GPUSKUType -} - type fixedRunSecrets struct { byImage map[string][]string err error @@ -58,28 +50,36 @@ func (f *fixedRunSecrets) GetSecrets(_ string, image string) ([]string, error) { return append([]string(nil), f.byImage[image]...), nil } -func (f *fixedRunHardware) Discover(_ context.Context, sku nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { - f.calls = append(f.calls, sku) - if f.err != nil { - return nil, f.err - } - return &f.info, nil -} - type dgdrRunFixture struct { t *testing.T client client.Client reconciler *DynamoGraphDeploymentRunV1beta2Reconciler } -func newDGDRRunFixture(t *testing.T, hardware dgdrRunHardwareDiscoverer, objects ...client.Object) *dgdrRunFixture { +type staleJobListOnceClient struct { + client.Client + hide bool +} + +func (c *staleJobListOnceClient) List( + ctx context.Context, + list client.ObjectList, + opts ...client.ListOption, +) error { + if _, ok := list.(*batchv1.JobList); ok && c.hide { + c.hide = false + return nil + } + return c.Client.List(ctx, list, opts...) +} + +func newDGDRRunFixture(t *testing.T, objects ...client.Object) *dgdrRunFixture { t.Helper() - return newDGDRRunFixtureWithInterceptor(t, hardware, interceptor.Funcs{}, objects...) + return newDGDRRunFixtureWithInterceptor(t, interceptor.Funcs{}, objects...) } func newDGDRRunFixtureWithInterceptor( t *testing.T, - hardware dgdrRunHardwareDiscoverer, funcs interceptor.Funcs, objects ...client.Object, ) *dgdrRunFixture { @@ -104,7 +104,6 @@ func newDGDRRunFixtureWithInterceptor( reconciler: &DynamoGraphDeploymentRunV1beta2Reconciler{ Client: c, APIReader: c, - Hardware: hardware, PlannerImage: "example.com/dynamo-planner:1.2.3", OperatorImage: "example.com/dynamo-operator:1.2.3", OperatorImagePullPolicy: corev1.PullAlways, @@ -160,9 +159,12 @@ func dgdrRunForTest(t *testing.T, inputOverrides map[string]any) *nvidiacomv1bet Spec: nvidiacomv1beta2.DynamoGraphDeploymentRunSpec{NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-0.6B"}, Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM}, - Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ - SKU: new("h200_sxm"), Budget: ptr.To[int32](16), - }}, + Hardware: &nvidiacomv1beta2.HardwareSpec{ + GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: new("h200_sxm"), Budget: ptr.To[int32](16), + }, + NumGPUsPerNode: ptr.To[int32](8), + }, Workload: &nvidiacomv1beta2.WorkloadSpec{Static: &nvidiacomv1beta2.StaticWorkloadSpec{}}, Objective: &nvidiacomv1beta2.ObjectiveSpec{ Mode: nvidiacomv1beta2.ObjectiveModeOptimize, Metric: &metric, SLA: &nvidiacomv1beta2.ObjectiveSLASpec{}, @@ -174,7 +176,6 @@ func dgdrRunForTest(t *testing.T, inputOverrides map[string]any) *nvidiacomv1bet } func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { - hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} run := dgdrRunForTest(t, map[string]any{ "engine": map[string]any{"hardware": "h100_sxm"}, "optimization": map[string]any{"constraints": map[string]any{"max_candidate_gpus": 12}}, @@ -186,14 +187,14 @@ func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { "namespace": "another-namespace", }, }) - f := newDGDRRunFixture(t, hardware, run) + f := newDGDRRunFixture(t, run) f.reconcile() f.reconcile() job := f.job() assertDGDRRunJobIsolation(t, job, run, f.reconciler.PlannerImage, f.reconciler.OperatorImage) - assertDGDRRunInput(t, f.client, job, hardware) + assertDGDRRunInput(t, f.client, job) assertDGDRRunSupportResources(t, f.client, job, run) stored := f.run() assertDGDRRunPendingStatus(t, stored, job.Name) @@ -201,8 +202,7 @@ func TestV1beta2RunControllerCreatesOneIsolatedAISimulateJob(t *testing.T) { } func TestV1beta2RunControllerRetainsAndTerminalizesASupersededRun(t *testing.T) { - hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} - f := newDGDRRunFixture(t, hardware, dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconcile() f.reconcile() @@ -267,10 +267,9 @@ func TestV1beta2RunControllerRetainsAndTerminalizesASupersededRun(t *testing.T) } func TestV1beta2RunControllerCancelsBeforeCreatingExecutionResources(t *testing.T) { - hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} run := dgdrRunForTest(t, map[string]any{}) run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] = "2" - f := newDGDRRunFixture(t, hardware, run) + f := newDGDRRunFixture(t, run) f.reconcile() stored := f.run() @@ -281,9 +280,6 @@ func TestV1beta2RunControllerCancelsBeforeCreatingExecutionResources(t *testing. if jobs := f.jobs(); len(jobs) != 0 { t.Fatalf("Jobs = %d, want none", len(jobs)) } - if len(hardware.calls) != 0 { - t.Fatalf("hardware discovery calls = %v, want none for a canceled Run", hardware.calls) - } } func TestV1beta2RunControllerRequiresANewerCancellationGeneration(t *testing.T) { @@ -304,11 +300,10 @@ func TestV1beta2RunControllerRequiresANewerCancellationGeneration(t *testing.T) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} run := dgdrRunForTest(t, map[string]any{}) run.Annotations[dgdrV1beta2GenerationAnnotation] = tt.runGeneration run.Annotations[dgdrV1beta2CancelRequestedByGenerationAnnotation] = tt.cancelGeneration - f := newDGDRRunFixture(t, hardware, run) + f := newDGDRRunFixture(t, run) t.Log("Reconcile the Run with the cancellation marker") f.reconcile() @@ -321,19 +316,14 @@ func TestV1beta2RunControllerRequiresANewerCancellationGeneration(t *testing.T) if jobs := f.jobs(); len(jobs) != 0 { t.Fatalf("Jobs = %d, want no execution for a superseded Run", len(jobs)) } - if len(hardware.calls) != 0 { - t.Fatalf("hardware discovery calls = %v, want none", hardware.calls) - } return } if jobs := f.jobs(); len(jobs) != 1 { t.Fatalf("Jobs = %d, want normal execution", len(jobs)) } - if len(hardware.calls) != 1 { - t.Fatalf("hardware discovery calls = %v, want normal execution", hardware.calls) - } }) } + } func TestV1beta2RunControllerRejectsSupportResourceOwnershipCollisionsWithoutMutation(t *testing.T) { @@ -392,19 +382,20 @@ func TestV1beta2RunControllerRejectsSupportResourceOwnershipCollisionsWithoutMut if owner.ref != nil { collision.SetOwnerReferences([]metav1.OwnerReference{*owner.ref}) } - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, run, collision) + f := newDGDRRunFixture(t, run, collision) before := collision.DeepCopyObject().(client.Object) if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(collision), before); err != nil { t.Fatalf("get collision before reconcile: %v", err) } - _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("marker result/error = %+v/%v, want durable ownership-failure marker", result, err) + } + _, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) if !errors.Is(err, reconcile.TerminalError(nil)) { t.Fatalf("reconcile error = %v, want terminal ownership conflict", err) } - if _, ok := errors.AsType[*commoncontroller.OwnershipConflictError](err); !ok { - t.Fatalf("reconcile error = %T %v, want OwnershipConflictError", err, err) - } after := collision.DeepCopyObject().(client.Object) if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(collision), after); err != nil { @@ -435,19 +426,19 @@ func TestV1beta2RunControllerClassifiesPermanentAndRetryableFailures(t *testing. name: "immutable lowering error is terminal", newFixture: func(t *testing.T) *dgdrRunFixture { run := dgdrRunForTest(t, map[string]any{}) - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, run) - f.reconciler.PlannerImage = "example.com/dynamo-planner@sha256:0123" + f := newDGDRRunFixture(t, run) + f.reconciler.PlannerImage = "example.com/dynamo-planner" f.reconciler.DynamoVersion = "1.7.0" return f }, wantTerminal: true, wantReason: dgdrRunReasonInvalidExecution, - wantErrMessage: "digest-only planner image", + wantErrMessage: "shared release tag is required", }, { name: "invalid Kubernetes resource is terminal", newFixture: func(t *testing.T) *dgdrRunFixture { run := dgdrRunForTest(t, map[string]any{}) - return newDGDRRunFixtureWithInterceptor(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, interceptor.Funcs{ + return newDGDRRunFixtureWithInterceptor(t, interceptor.Funcs{ Create: func(ctx context.Context, c client.WithWatch, object client.Object, opts ...client.CreateOption) error { if _, isConfigMap := object.(*corev1.ConfigMap); isConfigMap { return apierrors.NewInvalid(schema.GroupKind{Kind: "ConfigMap"}, object.GetName(), @@ -460,19 +451,10 @@ func TestV1beta2RunControllerClassifiesPermanentAndRetryableFailures(t *testing. wantTerminal: true, wantReason: dgdrRunReasonInvalidResource, wantErrMessage: "invalid execution", }, - { - name: "hardware discovery error is retryable", - newFixture: func(t *testing.T) *dgdrRunFixture { - return newDGDRRunFixture(t, &fixedRunHardware{err: errors.New("node inventory unavailable")}, - dgdrRunForTest(t, map[string]any{})) - }, - wantErrMessage: "node inventory unavailable", - }, { name: "registry secret error is retryable", newFixture: func(t *testing.T) *dgdrRunFixture { - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, - dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconciler.DockerSecretRetriever = &fixedRunSecrets{err: errors.New("registry index unavailable")} return f }, @@ -482,7 +464,7 @@ func TestV1beta2RunControllerClassifiesPermanentAndRetryableFailures(t *testing. name: "terminal status write failure remains retryable", newFixture: func(t *testing.T) *dgdrRunFixture { run := dgdrRunForTest(t, map[string]any{}) - f := newDGDRRunFixtureWithInterceptor(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, interceptor.Funcs{ + f := newDGDRRunFixtureWithInterceptor(t, interceptor.Funcs{ SubResourcePatch: func( _ context.Context, _ client.Client, _ string, _ client.Object, _ client.Patch, _ ...client.SubResourcePatchOption, @@ -490,7 +472,7 @@ func TestV1beta2RunControllerClassifiesPermanentAndRetryableFailures(t *testing. return errors.New("status writer unavailable") }, }, run) - f.reconciler.PlannerImage = "example.com/dynamo-planner@sha256:0123" + f.reconciler.PlannerImage = "example.com/dynamo-planner" f.reconciler.DynamoVersion = "1.7.0" return f }, @@ -501,7 +483,17 @@ func TestV1beta2RunControllerClassifiesPermanentAndRetryableFailures(t *testing. for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { f := tt.newFixture(t) - _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if tt.wantTerminal { + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("marker result/error = %+v/%v, want durable state retry", result, err) + } + condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != tt.wantReason { + t.Fatalf("marker condition = %+v, want False/%s", condition, tt.wantReason) + } + _, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + } if err == nil || !strings.Contains(err.Error(), tt.wantErrMessage) { t.Fatalf("reconcile error = %v, want %q", err, tt.wantErrMessage) } @@ -518,6 +510,7 @@ func TestV1beta2RunControllerClassifiesPermanentAndRetryableFailures(t *testing. } }) } + } func TestDGDRRunPermanentErrorClassification(t *testing.T) { @@ -535,7 +528,7 @@ func TestDGDRRunPermanentErrorClassification(t *testing.T) { {name: "forbidden", err: apierrors.NewForbidden(schema.GroupResource{Group: "batch", Resource: "jobs"}, "search-1", errors.New("denied"))}, {name: "conflict", err: apierrors.NewConflict(schema.GroupResource{Group: "batch", Resource: "jobs"}, "search-1", errors.New("changed"))}, {name: "timeout", err: apierrors.NewTimeoutError("timed out", 1)}, - {name: "retryable invalid discovery error", err: &dgdrRunRetryableError{cause: invalid}}, + {name: "explicitly retryable invalid error", err: &dgdrRunRetryableError{cause: invalid}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -546,40 +539,45 @@ func TestDGDRRunPermanentErrorClassification(t *testing.T) { } } -func TestV1beta2RunControllerUsesUnfilteredTopologyForAnExternalHardwareSKU(t *testing.T) { - hardware := &fixedRunHardware{info: gpu.GPUInfo{Model: "Future Accelerator", GPUsPerNode: 8}} +func TestV1beta2RunControllerOmitsUnspecifiedGPUsPerNode(t *testing.T) { run := dgdrRunForTest(t, map[string]any{}) - *run.Spec.Hardware.GPU.SKU = "future_accelerator" - f := newDGDRRunFixture(t, hardware, run) + run.Spec.Hardware.NumGPUsPerNode = nil + f := newDGDRRunFixture(t, run) f.reconcile() - if len(hardware.calls) != 1 || hardware.calls[0] != "" { - t.Fatalf("hardware discovery filters = %v, want one unfiltered topology lookup", hardware.calls) - } - if got := f.job().Spec.Template.Spec.Containers[0].Image; got != f.reconciler.PlannerImage { - t.Fatalf("sweeper image = %q, want %q", got, f.reconciler.PlannerImage) + input := dgdrRunInputFromConfigMap(t, f.client, f.job()) + output := input["dgdr_run"].(map[string]any) + if _, found := output["num_gpus_per_node"]; found { + t.Fatalf("dgdr_run.num_gpus_per_node = %v, want adapter-derived node width", output["num_gpus_per_node"]) } } -func TestV1beta2RunControllerRejectsAmbiguousTopologyForAnExternalHardwareSKU(t *testing.T) { - hardware := &fixedRunHardware{info: gpu.GPUInfo{ - Model: "Future Accelerator", GPUsPerNode: 8, OtherModels: []string{"Another Accelerator"}, - }} - run := dgdrRunForTest(t, map[string]any{}) - *run.Spec.Hardware.GPU.SKU = "future_accelerator" - f := newDGDRRunFixture(t, hardware, run) +func TestV1beta2RunControllerLowersHardwareOptOutFromInputOverrides(t *testing.T) { + run := dgdrRunForTest(t, map[string]any{ + "engine": map[string]any{"hardware": "future_accelerator"}, + "optimization": map[string]any{ + "constraints": map[string]any{"max_candidate_gpus": int64(24)}, + }, + }) + run.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{} + f := newDGDRRunFixture(t, run) - _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) - if err == nil || !strings.Contains(err.Error(), "among multiple GPU models") { - t.Fatalf("reconcile error = %v, want ambiguous external-SKU topology failure", err) + f.reconcile() + + input := dgdrRunInputFromConfigMap(t, f.client, f.job()) + output := input["dgdr_run"].(map[string]any) + if _, found := output["num_gpus_per_node"]; found { + t.Fatalf("dgdr_run.num_gpus_per_node = %v, want AISimulate system-spec resolution", output["num_gpus_per_node"]) + } + if got := input["engine"].(map[string]any)["hardware"]; got != "future_accelerator" { + t.Fatalf("engine.hardware = %v, want authoritative input override", got) } } func TestV1beta2RunControllerFreezesExecutionBeforeCreatingTheJob(t *testing.T) { - hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} run := dgdrRunForTest(t, map[string]any{}) - f := newDGDRRunFixture(t, hardware, run) + f := newDGDRRunFixture(t, run) f.reconciler.DockerSecretRetriever = &fixedRunSecrets{byImage: map[string][]string{ "example.com/dynamo-planner:1.2.3": {"planner-secret", "shared-secret"}, "example.com/dynamo-operator:1.2.3": {"shared-secret", "publisher-secret"}, @@ -617,7 +615,7 @@ func TestV1beta2RunControllerFreezesExecutionBeforeCreatingTheJob(t *testing.T) } func TestV1beta2RunControllerFailsClosedWithoutExecutionConfig(t *testing.T) { - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconcile() t.Log("Delete the immutable execution artifact while retaining the Job") @@ -630,15 +628,37 @@ func TestV1beta2RunControllerFailsClosedWithoutExecutionConfig(t *testing.T) { t.Fatalf("delete ConfigMap: %v", err) } - t.Log("Verify a real deletion fails closed instead of reconstructing or hot-looping") - _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + t.Log("Persist the permanent failure marker before taking cleanup action") + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("marker result/error = %+v/%v, want durable state retry", result, err) + } + if jobs := f.jobs(); len(jobs) != 1 { + t.Fatalf("Jobs = %d, want no deletion in the marker-writing reconciliation", len(jobs)) + } + condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != dgdrRunReasonInvalidExecution { + t.Fatalf("cleanup condition = %+v, want False/%s", condition, dgdrRunReasonInvalidExecution) + } + + t.Log("A later reconciliation observes the marker and stops the Job") + result, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("cleanup result/error = %+v/%v, want deletion-observation retry", result, err) + } + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want the invalid execution stopped", len(jobs)) + } + + t.Log("Publish Completed only after Job absence is observed") + _, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) if err == nil || !strings.Contains(err.Error(), "immutable execution ConfigMap") { t.Fatalf("reconcile error = %v, want missing immutable execution ConfigMap", err) } } func TestV1beta2RunControllerWaitsForExecutionConfigCacheLag(t *testing.T) { - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconcile() job := f.job() @@ -659,8 +679,25 @@ func TestV1beta2RunControllerWaitsForExecutionConfigCacheLag(t *testing.T) { } } +func TestV1beta2RunControllerRejectsMutableExecutionConfig(t *testing.T) { + run := dgdrRunForTest(t, map[string]any{}) + f := newDGDRRunFixture(t, run) + f.reconcile() + job := f.job() + configMap := &corev1.ConfigMap{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get execution ConfigMap: %v", err) + } + configMap.Immutable = new(false) + + _, err := f.reconciler.existingRunExecution(configMap, run, job) + if err == nil || !strings.Contains(err.Error(), "must be immutable") { + t.Fatalf("existing execution error = %v, want mutable ConfigMap rejection", err) + } +} + func TestV1beta2RunControllerRejectsJobExecutionSkew(t *testing.T) { - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconcile() t.Log("Mutate the observed Job away from the immutable execution identity") @@ -671,15 +708,458 @@ func TestV1beta2RunControllerRejectsJobExecutionSkew(t *testing.T) { t.Fatalf("update Job: %v", err) } - t.Log("Verify reconciliation detects the inconsistency instead of accepting skew") - _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + t.Log("Persist the failure marker without deleting the skewed Job") + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("marker result/error = %+v/%v, want durable state retry", result, err) + } + if jobs := f.jobs(); len(jobs) != 1 { + t.Fatalf("Jobs = %d, want no deletion in the marker-writing reconciliation", len(jobs)) + } + + t.Log("Observe the marker and stop the skewed Job") + result, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("cleanup result/error = %+v/%v, want deletion-observation retry", result, err) + } + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want the skewed execution stopped", len(jobs)) + } + + _, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) if err == nil || !strings.Contains(err.Error(), "planner image identities differ") { - t.Fatalf("reconcile error = %v, want planner identity mismatch", err) + t.Fatalf("terminal reconcile error = %v, want planner identity mismatch", err) + } +} + +func TestValidateDGDRRunJobExecutionRejectsImmutableBindingSkew(t *testing.T) { + run := dgdrRunForTest(t, map[string]any{}) + f := newDGDRRunFixture(t, run) + f.reconcile() + job := f.job() + configMap := &corev1.ConfigMap{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get execution ConfigMap: %v", err) + } + execution, err := dgdrRunExecutionFromConfigMap(configMap) + if err != nil { + t.Fatalf("decode execution: %v", err) + } + + tests := []struct { + name string + mutate func(*batchv1.Job) + wantErr string + }{ + { + name: "backoff limit", + mutate: func(job *batchv1.Job) { + job.Spec.BackoffLimit = ptr.To[int32](1) + }, + wantErr: "Job backoff limit differs", + }, + { + name: "parallel sweeps", + mutate: func(job *batchv1.Job) { + job.Spec.Parallelism = ptr.To[int32](2) + }, + wantErr: "Job cardinality differs", + }, + { + name: "multiple completions", + mutate: func(job *batchv1.Job) { + job.Spec.Completions = ptr.To[int32](2) + }, + wantErr: "Job cardinality differs", + }, + { + name: "indexed completion mode", + mutate: func(job *batchv1.Job) { + job.Spec.CompletionMode = ptr.To(batchv1.IndexedCompletion) + }, + wantErr: "Job completion mode differs", + }, + { + name: "suspended execution", + mutate: func(job *batchv1.Job) { + job.Spec.Suspend = new(true) + }, + wantErr: "Job execution policy differs", + }, + { + name: "active deadline", + mutate: func(job *batchv1.Job) { + job.Spec.ActiveDeadlineSeconds = ptr.To[int64](1) + }, + wantErr: "Job execution policy differs", + }, + { + name: "nondefault pod replacement policy", + mutate: func(job *batchv1.Job) { + job.Spec.PodReplacementPolicy = ptr.To(batchv1.Failed) + }, + wantErr: "Job execution policy differs", + }, + { + name: "restart policy", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyAlways + }, + wantErr: "pod restart policy differs", + }, + { + name: "host PID namespace", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.HostPID = true + }, + wantErr: "pod namespace isolation differs", + }, + { + name: "host IPC namespace", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.HostIPC = true + }, + wantErr: "pod namespace isolation differs", + }, + { + name: "host network namespace", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.HostNetwork = true + }, + wantErr: "pod namespace isolation differs", + }, + { + name: "shared process namespace", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.ShareProcessNamespace = new(true) + }, + wantErr: "pod namespace isolation differs", + }, + { + name: "init container", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.InitContainers = []corev1.Container{{Name: "unexpected", Image: "example.invalid/init"}} + }, + wantErr: "init containers are not allowed", + }, + { + name: "pod runAsNonRoot", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.SecurityContext.RunAsNonRoot = new(false) + }, + wantErr: "pod runAsNonRoot differs", + }, + { + name: "pod seccomp", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.SecurityContext.SeccompProfile.Type = corev1.SeccompProfileTypeUnconfined + }, + wantErr: "pod seccomp profile differs", + }, + { + name: "planner privilege escalation", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].SecurityContext.AllowPrivilegeEscalation = new(true) + }, + wantErr: "planner security context differs", + }, + { + name: "planner capabilities", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].SecurityContext.Capabilities.Add = []corev1.Capability{"NET_ADMIN"} + }, + wantErr: "planner security context differs", + }, + { + name: "planner user", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].SecurityContext.RunAsUser = ptr.To[int64](0) + }, + wantErr: "planner security context differs", + }, + { + name: "planner unmasked proc mount", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].SecurityContext.ProcMount = ptr.To(corev1.UnmaskedProcMount) + }, + wantErr: "planner security context differs", + }, + { + name: "publisher security", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[1].SecurityContext.Privileged = new(true) + }, + wantErr: "publisher security context differs", + }, + { + name: "publisher seccomp override", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[1].SecurityContext.SeccompProfile = &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeUnconfined, + } + }, + wantErr: "publisher security context differs", + }, + { + name: "publisher unmasked proc mount", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[1].SecurityContext.ProcMount = ptr.To(corev1.UnmaskedProcMount) + }, + wantErr: "publisher security context differs", + }, + { + name: "automatic service account token", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.AutomountServiceAccountToken = new(true) + }, + wantErr: "publisher service-account identities differ", + }, + { + name: "publisher token exposed to planner", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].VolumeMounts = append( + job.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: dgdrRunServiceAccountVolume, MountPath: dgdrRunServiceAccountMountPath}, + ) + }, + wantErr: "planner volume mounts differ", + }, + { + name: "planner command", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].Command = []string{"python"} + }, + wantErr: "planner command or arguments differ", + }, + { + name: "planner environment secret", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].EnvFrom = []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{}, + }} + }, + wantErr: "planner environment differs", + }, + { + name: "planner lifecycle hook", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].Lifecycle = &corev1.Lifecycle{ + PostStart: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"unexpected"}}}, + } + }, + wantErr: "planner lifecycle differs", + }, + { + name: "publisher Run UID", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[1].Args[5] = "other-uid" + }, + wantErr: "publisher command, arguments or identity differ", + }, + { + name: "publisher environment secret", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[1].EnvFrom = []corev1.EnvFromSource{{ + ConfigMapRef: &corev1.ConfigMapEnvSource{}, + }} + }, + wantErr: "publisher environment differs", + }, + { + name: "publisher lifecycle hook", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[1].Lifecycle = &corev1.Lifecycle{ + PreStop: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"unexpected"}}}, + } + }, + wantErr: "publisher lifecycle differs", + }, + { + name: "input ConfigMap source", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Volumes[0].ConfigMap.Name = "other-input" + }, + wantErr: "input, snapshot or publisher credential volumes differ", + }, + { + name: "snapshot mount", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].VolumeMounts[1].MountPath = "/other-output" + }, + wantErr: "planner volume mounts differ", + }, + { + name: "publisher token mount", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[1].VolumeMounts[1].Name = "other-token" + }, + wantErr: "publisher volume mounts differ", + }, + { + name: "service account", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.ServiceAccountName = "other-publisher" + }, + wantErr: "publisher service-account identities differ", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mutated := job.DeepCopy() + tt.mutate(mutated) + err := validateDGDRRunJobExecution(mutated, run, configMap.Name, execution) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("validation error = %v, want %q", err, tt.wantErr) + } + }) + } + + t.Run("API defaulted cardinality remains compatible", func(t *testing.T) { + defaulted := job.DeepCopy() + defaulted.Spec.Parallelism = ptr.To[int32](1) + defaulted.Spec.Completions = ptr.To[int32](1) + defaulted.Spec.CompletionMode = ptr.To(batchv1.NonIndexedCompletion) + defaulted.Spec.Suspend = new(false) + defaulted.Spec.ManualSelector = new(false) + defaulted.Spec.PodReplacementPolicy = ptr.To(batchv1.TerminatingOrFailed) + defaulted.Spec.Template.Spec.ShareProcessNamespace = new(false) + defaulted.Spec.Template.Spec.Containers[0].SecurityContext.ProcMount = ptr.To(corev1.DefaultProcMount) + defaulted.Spec.Template.Spec.Containers[1].SecurityContext.ProcMount = ptr.To(corev1.DefaultProcMount) + if err := validateDGDRRunJobExecution(defaulted, run, configMap.Name, execution); err != nil { + t.Fatalf("validation error = %v, want API defaults accepted", err) + } + }) +} + +func TestV1beta2RunControllerStopsAllJobsBeforePermanentCompletion(t *testing.T) { + run := dgdrRunForTest(t, map[string]any{}) + markerObservedByReconcile := false + deletedBeforeMarkerObservation := false + serveStaleMarkerOnce := false + f := newDGDRRunFixtureWithInterceptor(t, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, object, opts...); err != nil { + return err + } + if stored, ok := object.(*nvidiacomv1beta2.DynamoGraphDeploymentRun); ok { + if serveStaleMarkerOnce { + meta.RemoveStatusCondition(&stored.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + serveStaleMarkerOnce = false + } else if dgdrRunPendingPermanentFailure(stored) != nil { + markerObservedByReconcile = true + } + } + return nil + }, + Delete: func(ctx context.Context, c client.WithWatch, object client.Object, opts ...client.DeleteOption) error { + if _, ok := object.(*batchv1.Job); ok && !markerObservedByReconcile { + deletedBeforeMarkerObservation = true + } + return c.Delete(ctx, object, opts...) + }, + }, run) + f.reconcile() + duplicate := f.job().DeepCopy() + duplicate.Name += "-duplicate" + duplicate.ResourceVersion = "" + duplicate.UID = "" + if err := f.client.Create(t.Context(), duplicate); err != nil { + t.Fatalf("create duplicate Job: %v", err) + } + + t.Log("Record the permanent failure without acting on the locally patched marker") + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("marker result/error = %+v/%v, want durable state retry", result, err) + } + if jobs := f.jobs(); len(jobs) != 2 { + t.Fatalf("Jobs = %d, want no deletion before observing the marker", len(jobs)) + } + if markerObservedByReconcile || deletedBeforeMarkerObservation { + t.Fatalf("marker observed/deleted early = %t/%t, want an observed-state barrier", markerObservedByReconcile, deletedBeforeMarkerObservation) + } + serveStaleMarkerOnce = true + + t.Log("A reordered cache read cannot skip the marker-observation barrier") + result, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("stale-cache result/error = %+v/%v, want marker retry", result, err) + } + if jobs := f.jobs(); len(jobs) != 2 { + t.Fatalf("Jobs = %d, want no deletion from a stale marker read", len(jobs)) + } + if markerObservedByReconcile || deletedBeforeMarkerObservation { + t.Fatalf("marker observed/deleted early = %t/%t after stale read", markerObservedByReconcile, deletedBeforeMarkerObservation) + } + + t.Log("A later reconciliation observes the marker and deletes every owned Job") + result, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("cleanup result/error = %+v/%v, want deletion-observation retry", result, err) + } + if !markerObservedByReconcile || deletedBeforeMarkerObservation { + t.Fatalf("marker observed/deleted early = %t/%t, want deletion only after observation", markerObservedByReconcile, deletedBeforeMarkerObservation) + } + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want all conflicting executions stopped", len(jobs)) + } + + t.Log("Finish the recorded failure after observing Job absence") + _, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err == nil || !errors.Is(err, reconcile.TerminalError(nil)) { + t.Fatalf("terminal reconcile error = %v, want terminal failure", err) + } + condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("Completed condition = %+v, want True after cleanup", condition) + } +} + +func TestV1beta2RunControllerConfirmsJobAbsenceThroughAPIReader(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + + run := f.run() + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionFalse, + Reason: dgdrRunReasonInvalidExecution, + Message: "invalid immutable execution", + ObservedGeneration: run.Generation, + }) + if err := f.client.Status().Update(t.Context(), run); err != nil { + t.Fatalf("record permanent failure marker: %v", err) + } + + f.reconciler.Client = &staleJobListOnceClient{Client: f.client, hide: true} + + t.Log("The informer misses the Job, but the API reader still observes and stops it") + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("cleanup result/error = %+v/%v, want uncached deletion retry", result, err) + } + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want uncached live execution stopped", len(jobs)) + } + condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionFalse { + t.Fatalf("Completed condition = %+v, want False until uncached absence is observed", condition) + } + + t.Log("Only a later uncached absence observation permits terminal completion") + _, err = f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err == nil || !strings.Contains(err.Error(), "invalid immutable execution") { + t.Fatalf("terminal reconcile error = %v, want recorded permanent failure", err) + } + condition = meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("Completed condition = %+v, want True after uncached absence", condition) } } func TestV1beta2RunControllerReportsImagePullSecretErrors(t *testing.T) { - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconciler.DockerSecretRetriever = &fixedRunSecrets{err: errors.New("registry index unavailable")} t.Log("Verify execution is not frozen without resolving its image credentials") @@ -689,22 +1169,28 @@ func TestV1beta2RunControllerReportsImagePullSecretErrors(t *testing.T) { } } -func TestV1beta2RunControllerRejectsDigestOnlyPlannerImage(t *testing.T) { - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) +func TestV1beta2RunControllerSeparatesDigestPinnedPlannerFromRuntimeSource(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconciler.PlannerImage = "example.com/dynamo-planner@sha256:0123" f.reconciler.DynamoVersion = "1.7.0" - t.Log("Reject the Planner before freezing an unrelated runtime release tag") - _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) - if err == nil || !strings.Contains(err.Error(), "digest-only planner image cannot identify sibling runtime images") { - t.Fatalf("reconcile error = %v, want digest-only Planner rejection", err) + f.reconcile() + + job := f.job() + planner := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + if planner.Image != f.reconciler.PlannerImage { + t.Fatalf("planner image = %q, want digest-pinned %q", planner.Image, f.reconciler.PlannerImage) } - jobs := &batchv1.JobList{} - if err := f.client.List(t.Context(), jobs, client.InNamespace("default")); err != nil { - t.Fatalf("list Jobs: %v", err) + configMap := &corev1.ConfigMap{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get execution ConfigMap: %v", err) + } + if got := configMap.Data[dgdrRunRuntimeImageSourceKey]; got != "example.com/dynamo-planner:1.7.0" { + t.Fatalf("runtime image source = %q, want normalized release tag", got) } - if len(jobs.Items) != 0 { - t.Fatalf("Jobs = %d, want none for an ambiguous runtime source", len(jobs.Items)) + input := dgdrRunInputFromConfigMap(t, f.client, job) + if got := input["dgdr_run"].(map[string]any)["runtime_image"]; got != "example.com/dynamo-planner:1.7.0" { + t.Fatalf("dgdr_run.runtime_image = %v, want normalized release tag", got) } } @@ -719,9 +1205,6 @@ func assertDGDRRunJobIsolation( if !metav1.IsControlledBy(job, run) { t.Fatal("Job is not controlled by its Run") } - if got := job.Annotations[dgdrRunGPUsPerNodeAnnotation]; got != "8" { - t.Fatalf("GPUs-per-node annotation = %q, want 8", got) - } if job.Spec.BackoffLimit == nil || *job.Spec.BackoffLimit != 0 { t.Fatalf("backoffLimit = %v, want 0", job.Spec.BackoffLimit) } @@ -760,6 +1243,26 @@ func assertDGDRRunJobIsolation( }; !reflect.DeepEqual(sweeper.Args, want) { t.Fatalf("sweeper args = %q, want %q", sweeper.Args, want) } + assertDGDRRunPublisherIdentity(t, publisher, run, operatorImage) + if hasMount(sweeper.VolumeMounts, dgdrRunServiceAccountVolume) { + t.Fatal("sweeper unexpectedly receives the Kubernetes service-account token") + } + if !hasMount(publisher.VolumeMounts, dgdrRunServiceAccountVolume) { + t.Fatal("publisher does not receive its projected Kubernetes credentials") + } + if !hasMountAt(sweeper.VolumeMounts, dgdrRunSnapshotVolume, dgdrRunSnapshotPath) || + !hasMountAt(publisher.VolumeMounts, dgdrRunSnapshotVolume, dgdrRunSnapshotPath) { + t.Fatal("sweeper and publisher do not share the frozen snapshot directory") + } +} + +func assertDGDRRunPublisherIdentity( + t *testing.T, + publisher corev1.Container, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + operatorImage string, +) { + t.Helper() if publisher.Image != operatorImage || publisher.ImagePullPolicy != corev1.PullAlways { t.Fatalf("publisher image = %q/%s, want configured operator image/Always", publisher.Image, publisher.ImagePullPolicy) } @@ -774,16 +1277,6 @@ func assertDGDRRunJobIsolation( }; !reflect.DeepEqual(publisher.Args, want) { t.Fatalf("publisher args = %q, want %q", publisher.Args, want) } - if hasMount(sweeper.VolumeMounts, dgdrRunServiceAccountVolume) { - t.Fatal("sweeper unexpectedly receives the Kubernetes service-account token") - } - if !hasMount(publisher.VolumeMounts, dgdrRunServiceAccountVolume) { - t.Fatal("publisher does not receive its projected Kubernetes credentials") - } - if !hasMountAt(sweeper.VolumeMounts, dgdrRunSnapshotVolume, dgdrRunSnapshotPath) || - !hasMountAt(publisher.VolumeMounts, dgdrRunSnapshotVolume, dgdrRunSnapshotPath) { - t.Fatal("sweeper and publisher do not share the frozen snapshot directory") - } } func assertDGDRRunStatusInputMatchesConfigMap( @@ -826,7 +1319,7 @@ func TestDGDRRunStatusInputPreservesLargeIntegers(t *testing.T) { } } -func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardware *fixedRunHardware) { +func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job) { t.Helper() configMap := &corev1.ConfigMap{} if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { @@ -842,7 +1335,6 @@ func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardwar dgdrRunPublisherImageKey: "example.com/dynamo-operator:1.2.3", dgdrRunPublisherPullPolicyKey: string(corev1.PullAlways), dgdrRunDynamoVersionKey: "1.2.3", - dgdrRunGPUsPerNodeKey: "8", dgdrRunSnapshotPathKey: dgdrRunSnapshotPath, } { if got := configMap.Data[key]; got != want { @@ -866,6 +1358,9 @@ func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardwar if output["derive_runtime_images"] != true { t.Fatalf("derive_runtime_images = %v, want true for a backend search", output["derive_runtime_images"]) } + if output["num_gpus_per_node"] != float64(8) { + t.Fatalf("num_gpus_per_node = %v, want typed hardware topology 8", output["num_gpus_per_node"]) + } if got := input["engine"].(map[string]any)["hardware"]; got != "h100_sxm" { t.Fatalf("engine.hardware = %v, want authoritative input override h100_sxm", got) } @@ -873,9 +1368,19 @@ func assertDGDRRunInput(t *testing.T, c client.Client, job *batchv1.Job, hardwar if got := constraints["max_candidate_gpus"]; got != float64(12) { t.Fatalf("optimization.constraints.max_candidate_gpus = %v, want authoritative input override 12", got) } - if !reflect.DeepEqual(hardware.calls, []nvidiacomv1beta1.GPUSKUType{"h200_sxm"}) { - t.Fatalf("hardware discovery calls = %v, want the admitted typed SKU h200_sxm", hardware.calls) +} + +func dgdrRunInputFromConfigMap(t *testing.T, c client.Client, job *batchv1.Job) map[string]any { + t.Helper() + configMap := &corev1.ConfigMap{} + if err := c.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get input ConfigMap: %v", err) + } + var input map[string]any + if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &input); err != nil { + t.Fatalf("decode input ConfigMap: %v", err) } + return input } func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { @@ -891,8 +1396,8 @@ func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { {name: "v-prefixed planner tag", plannerImage: "example.com/dynamo-planner:v1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:v1.7.0"}, {name: "prerelease planner tag", plannerImage: "example.com/dynamo-planner:1.7.0-rc.1", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0-rc.1"}, {name: "build metadata planner tag", plannerImage: "example.com/dynamo-planner:1.7.0+build.5", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0+build.5"}, - {name: "tag and digest", plannerImage: "example.com/dynamo-planner:v1.7.0-rc.1@sha256:0123", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:v1.7.0-rc.1@sha256:0123"}, - {name: "digest only", plannerImage: "example.com/dynamo-planner@sha256:0123", dynamoVersion: "1.7.0", wantErr: true}, + {name: "tag and digest", plannerImage: "example.com/dynamo-planner:v1.7.0-rc.1@sha256:0123", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:v1.7.0-rc.1"}, + {name: "digest only", plannerImage: "example.com/dynamo-planner@sha256:0123", dynamoVersion: "1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:1.7.0"}, {name: "opaque CI tag", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7.0", wantOverride: "1.7.0", wantRuntime: "example.com/dynamo-planner:git-sha"}, {name: "untagged planner image", plannerImage: "example.com/dynamo-planner", dynamoVersion: "1.7.0", wantErr: true}, {name: "opaque tag without canonical version", plannerImage: "example.com/dynamo-planner:git-sha", dynamoVersion: "1.7", wantErr: true}, @@ -902,7 +1407,7 @@ func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { version, err := dgdrRunDynamoVersion(tt.plannerImage, tt.dynamoVersion) var runtimeImage string if err == nil { - runtimeImage, err = dgdrRunRuntimeImageSource(tt.plannerImage) + runtimeImage, err = dgdrRunRuntimeImageSource(tt.plannerImage, version) } if (err != nil) != tt.wantErr { t.Fatalf("resolve execution identity error = %v, wantErr %t", err, tt.wantErr) @@ -910,7 +1415,7 @@ func TestDGDRRunInputResolvesRuntimeVersion(t *testing.T) { if tt.wantErr { return } - input, err := dgdrRunInput(dgdrRunForTest(t, map[string]any{}), runtimeImage, version, 8) + input, err := dgdrRunInput(dgdrRunForTest(t, map[string]any{}), runtimeImage, version) if err != nil { t.Fatalf("lower input: %v", err) } @@ -956,8 +1461,7 @@ func assertDGDRRunPendingStatus(t *testing.T, stored *nvidiacomv1beta2.DynamoGra } func TestV1beta2RunControllerOwnsOnlyLifecycleStatus(t *testing.T) { - hardware := &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}} - f := newDGDRRunFixture(t, hardware, dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconcile() job := f.job() @@ -1007,7 +1511,7 @@ func TestV1beta2RunControllerOwnsOnlyLifecycleStatus(t *testing.T) { } func TestV1beta2RunControllerClassifiesPublisherProtocolFailure(t *testing.T) { - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconcile() job := f.job() pod := &corev1.Pod{ @@ -1052,7 +1556,7 @@ func TestV1beta2RunControllerClassifiesPublisherProtocolFailure(t *testing.T) { } func TestV1beta2RunControllerPrefersPublisherOutcomeWhenBothContainersFail(t *testing.T) { - f := newDGDRRunFixture(t, &fixedRunHardware{info: gpu.GPUInfo{GPUsPerNode: 8}}, dgdrRunForTest(t, map[string]any{})) + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconcile() job := f.job() outcome, err := json.Marshal(dgdrrunpublisher.TerminalOutcome{ diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index bbc4c243e62c..56355c42473d 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -11,7 +11,6 @@ import ( configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" lpxcontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller/lpx" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" "github.com/ai-dynamo/dynamo/deploy/operator/internal/modelendpoint" "github.com/ai-dynamo/dynamo/deploy/operator/internal/secret" @@ -45,8 +44,6 @@ type DynamoGraphDeploymentRequestSetupOptions struct { type DynamoGraphDeploymentRunSetupOptions struct { SetupOptions - GPUDiscoveryCache *gpu.GPUDiscoveryCache - GPUDiscovery *gpu.GPUDiscovery DockerSecretRetriever DockerSecretRetriever PlannerImage string DynamoVersion string @@ -73,20 +70,6 @@ func (o DynamoGraphDeploymentRequestSetupOptions) gpuDiscovery() *gpu.GPUDiscove return gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint) } -func (o DynamoGraphDeploymentRunSetupOptions) gpuDiscoveryCache() *gpu.GPUDiscoveryCache { - if o.GPUDiscoveryCache != nil { - return o.GPUDiscoveryCache - } - return gpu.NewGPUDiscoveryCache() -} - -func (o DynamoGraphDeploymentRunSetupOptions) gpuDiscovery() *gpu.GPUDiscovery { - if o.GPUDiscovery != nil { - return o.GPUDiscovery - } - return gpu.NewGPUDiscovery(gpu.ScrapeMetricsEndpoint) -} - func (o DynamoModelSetupOptions) modelEndpointClient() *modelendpoint.Client { if o.ModelEndpointClient != nil { return o.ModelEndpointClient @@ -176,17 +159,9 @@ func SetupDynamoGraphDeploymentRequestV1beta2(mgr ctrl.Manager, opts SetupOption // v1beta2 Runs as AISimulate Jobs and observes their lifecycle. func SetupDynamoGraphDeploymentRunV1beta2(mgr ctrl.Manager, opts DynamoGraphDeploymentRunSetupOptions) error { if err := (&DynamoGraphDeploymentRunV1beta2Reconciler{ - Client: mgr.GetClient(), - Config: opts.Config, - RuntimeConfig: opts.RuntimeConfig, - Hardware: &gpu.Discoverer{ - Reader: mgr.GetClient(), - DCGM: opts.gpuDiscovery(), - Cache: opts.gpuDiscoveryCache(), - NodeLabels: func() bool { - return opts.RuntimeConfig != nil && opts.RuntimeConfig.Gate.Enabled(features.GPUDiscovery) - }, - }, + Client: mgr.GetClient(), + Config: opts.Config, + RuntimeConfig: opts.RuntimeConfig, DockerSecretRetriever: opts.DockerSecretRetriever, PlannerImage: opts.PlannerImage, DynamoVersion: opts.DynamoVersion, From d2b683bf62cf504d0556e0ff114ce1dc74831b6f Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 16:11:12 +0100 Subject: [PATCH 062/100] pr-17195: enforce resolved Run snapshot shape [upstream: #17195] Signed-off-by: Dr. Stefan Schimanski --- .../v1beta2/dynamographdeploymentrun_types.go | 2 +- .../nvidia.com_dynamographdeploymentruns.yaml | 2 +- ...ploymentrequest_validation_envtest_test.go | 42 ++++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go index 1ca939cb6349..affb00815de3 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go @@ -27,7 +27,7 @@ import ( // DynamoGraphDeploymentRunSpec is an immutable snapshot of an accepted DGDR spec. // Request admission resolves its structural defaults before the Request controller copies it; // Request admission defaulting does not apply to Runs. -// +kubebuilder:validation:XValidation:rule="has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.workload) && has(self.workload.static) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0",message="spec must be a fully resolved Request admission snapshot" +// +kubebuilder:validation:XValidation:rule="has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && ((!has(self.hardware.gpu) && !has(self.hardware.numGpusPerNode)) || (has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.hardware.numGpusPerNode))) && has(self.workload) && has(self.workload.static) && ((!has(self.workload.static.inputTokens) && !has(self.workload.static.outputTokens) && !has(self.workload.static.load)) || (has(self.workload.static.inputTokens) && has(self.workload.static.outputTokens) && has(self.workload.static.load))) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.objective.sla) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0",message="spec must be a fully resolved Request admission snapshot" // // The Run reconciler lowers this immutable typed snapshot plus inputOverrides exactly once and // records the resulting public recommendation input in status.aisimulate.input. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 54396dd06bfd..5110346080b9 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -325,7 +325,7 @@ spec: type: object x-kubernetes-validations: - message: spec must be a fully resolved Request admission snapshot - rule: has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.workload) && has(self.workload.static) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0 + rule: has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && ((!has(self.hardware.gpu) && !has(self.hardware.numGpusPerNode)) || (has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.hardware.numGpusPerNode))) && has(self.workload) && has(self.workload.static) && ((!has(self.workload.static.inputTokens) && !has(self.workload.static.outputTokens) && !has(self.workload.static.load)) || (has(self.workload.static.inputTokens) && has(self.workload.static.outputTokens) && has(self.workload.static.load))) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.objective.sla) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0 status: description: DynamoGraphDeploymentRunStatus represents one search execution's observed state. properties: diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 8b0b1ee5fd58..2a7d18b5d322 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -827,6 +827,43 @@ func TestDynamoGraphDeploymentRunSpecIsImmutable(t *testing.T) { run: v1beta2ModelOnlyRunForAdmission(v1beta2AdmissionModel), wantCELErr: `spec: Invalid value: spec must be a fully resolved Request admission snapshot`, }, + { + name: "empty hardware is an accepted resolved opt-out", + run: func() *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := v1beta2RunForAdmission(v1beta2AdmissionModel, nil) + run.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{} + return run + }(), + }, + { + name: "partial hardware is rejected", + run: func() *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := v1beta2RunForAdmission(v1beta2AdmissionModel, nil) + run.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: new("h200_sxm"), Budget: new(int32(8)), + }} + return run + }(), + wantCELErr: `spec: Invalid value: spec must be a fully resolved Request admission snapshot`, + }, + { + name: "partial static workload is rejected", + run: func() *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := v1beta2RunForAdmission(v1beta2AdmissionModel, nil) + run.Spec.Workload.Static.InputTokens = new(int32(1024)) + return run + }(), + wantCELErr: `spec: Invalid value: spec must be a fully resolved Request admission snapshot`, + }, + { + name: "missing SLA snapshot is rejected", + run: func() *nvidiacomv1beta2.DynamoGraphDeploymentRun { + run := v1beta2RunForAdmission(v1beta2AdmissionModel, nil) + run.Spec.Objective.SLA = nil + return run + }(), + wantCELErr: `spec: Invalid value: spec must be a fully resolved Request admission snapshot`, + }, } for _, tt := range tests { @@ -1124,8 +1161,9 @@ func setV1beta2InputOverrides(spec map[string]any, overrides map[string]any) { // setBeta2AdmissionHardware names the hardware, so fixtures do not depend on GPU discovery. func setBeta2AdmissionHardware(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { - sku, budget := "h100_sxm", int32(4) + sku, budget, numGPUsPerNode := "h100_sxm", int32(4), int32(4) request.Spec.Hardware = &nvidiacomv1beta2.HardwareSpec{ - GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + GPU: &nvidiacomv1beta2.GPUHardwareSpec{SKU: &sku, Budget: &budget}, + NumGPUsPerNode: &numGPUsPerNode, } } From 32632ce25fcf85274c3310b96d77b67401d71360 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Tue, 27 Oct 2026 09:48:00 +0100 Subject: [PATCH 063/100] operator/dgdr: test v1beta2 candidates end to end Signed-off-by: Dr. Stefan Schimanski --- .../internal/controller/clusterenv_test.go | 117 ++++++++++++++++++ .../testdata/dgdr-v1beta2/vllm/input.yaml | 50 ++++++++ .../testdata/dgdr-v1beta2/vllm/output.yaml | 22 ++++ 3 files changed, 189 insertions(+) create mode 100644 deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/output.yaml diff --git a/deploy/operator/internal/controller/clusterenv_test.go b/deploy/operator/internal/controller/clusterenv_test.go index 65f9b8a3fde7..c43c752d7702 100644 --- a/deploy/operator/internal/controller/clusterenv_test.go +++ b/deploy/operator/internal/controller/clusterenv_test.go @@ -15,11 +15,14 @@ import ( "testing" "time" + semver "github.com/Masterminds/semver/v3" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gpu" dynamotesting "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/clusterenv" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/golden" @@ -34,6 +37,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -318,6 +322,119 @@ func TestClusterDynamoGraphDeploymentRequestProfilesAndCreatesWorkloadManifests( } } +func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing.T) { + inputs, err := filepath.Glob(filepath.Join("testdata/dgdr-v1beta2", "*", "input.yaml")) + if err != nil { + t.Fatalf("find v1beta2 DGDR cluster-test scenarios: %v", err) + } + if len(inputs) == 0 { + t.Fatal("no v1beta2 DGDR cluster-test scenarios found") + } + plannerImage := clusterenv.RequireEnv(t, "DYNAMO_CLUSTERTEST_PROFILER_IMAGE") + operatorImage := clusterenv.RequireEnv(t, "DYNAMO_CLUSTERTEST_OPERATOR_IMAGE") + operatorVersion, err := clusterTestOperatorVersion() + require.NoError(t, err) + parsedOperatorVersion, err := semver.NewVersion(operatorVersion) + require.NoError(t, err) + dynamoVersion := fmt.Sprintf("%d.%d.%d", parsedOperatorVersion.Major(), parsedOperatorVersion.Minor(), parsedOperatorVersion.Patch()) + for _, input := range inputs { + scenarioDir := filepath.Dir(input) + t.Run(filepath.Base(scenarioDir), func(t *testing.T) { + ctx := t.Context() + + t.Log("Create an isolated namespace in the explicitly unlocked cluster") + env := clusterTestEnv.RunT(t) + + t.Log("Allow the AISimulate Job Pod to run while preventing unrelated ReplicaSet workloads") + env.BlockReplicaSets() + + t.Log("Apply the native v1beta2 Request through production admission") + objects := golden.ApplyManifests(t, filepath.Join(scenarioDir, "input.yaml"), env.Client(), env.Namespace()) + if len(objects) != 1 || objects[0].GetKind() != "DynamoGraphDeploymentRequest" { + t.Fatalf("scenario input contains %d objects, want one DynamoGraphDeploymentRequest", len(objects)) + } + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + if err := env.Client().Get(ctx, client.ObjectKeyFromObject(&objects[0]), request); err != nil { + t.Fatalf("get admitted v1beta2 DGDR: %v", err) + } + + t.Log("Watch every Run transition before starting the production controllers") + watchClient, err := client.NewWithWatch(env.RESTConfig(), client.Options{Scheme: newClusterTestScheme()}) + require.NoError(t, err) + runWatch, err := watchClient.Watch(ctx, &nvidiacomv1beta2.DynamoGraphDeploymentRunList{}, client.InNamespace(env.Namespace())) + require.NoError(t, err) + defer runWatch.Stop() + + t.Log("Provide the Run controller the discovered node shape represented by the explicit Request hardware") + hardwareCache := gpu.NewGPUDiscoveryCache() + hardwareCache.Set(nvidiacomv1beta1.GPUSKUTypeH200SXM, &gpu.GPUInfo{ + System: nvidiacomv1beta1.GPUSKUTypeH200SXM, + GPUsPerNode: 8, + NodesWithGPUs: 1, + TotalGPUs: 8, + }, time.Hour) + + t.Log("Start the production Request, Run, AISimulate Job, and publisher chain") + operatorConfig := clusterTestRestrictedConfig(env.Namespace()) + runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{}} + env.StartManager(func(mgr ctrl.Manager) error { + setupOptions := SetupOptions{Config: operatorConfig, RuntimeConfig: runtimeConfig} + if err := SetupDynamoGraphDeploymentRequestV1beta2(mgr, setupOptions); err != nil { + return err + } + return SetupDynamoGraphDeploymentRunV1beta2(mgr, DynamoGraphDeploymentRunSetupOptions{ + SetupOptions: setupOptions, + GPUDiscoveryCache: hardwareCache, + PlannerImage: plannerImage, + DynamoVersion: dynamoVersion, + OperatorImage: operatorImage, + OperatorImagePullPolicy: corev1.PullIfNotPresent, + }) + }) + + t.Log("Observe the Run until the real AISimulate and publisher containers finish") + var completedRun nvidiacomv1beta2.DynamoGraphDeploymentRun + deadline := time.NewTimer(10 * time.Minute) + defer deadline.Stop() + for completedRun.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseSucceeded { + select { + case event, ok := <-runWatch.ResultChan(): + require.True(t, ok, "Run watch closed before the execution completed") + if event.Type == watch.Error { + t.Fatalf("Run watch failed: %v", apierrors.FromObject(event.Object)) + } + run, ok := event.Object.(*nvidiacomv1beta2.DynamoGraphDeploymentRun) + require.Truef(t, ok, "Run watch returned %T", event.Object) + if !metav1.IsControlledBy(run, request) { + continue + } + require.NotEqual(t, nvidiacomv1beta2.DGDRRunPhaseFailed, run.Status.Phase, + "Run failed: %v", run.Status.Conditions) + completedRun = *run.DeepCopy() + case <-deadline.C: + t.Fatal("v1beta2 DGDR did not complete with a candidate") + } + } + require.NotNil(t, completedRun.Status.Progress, "completed Run has no progress") + require.NotNil(t, completedRun.Status.Progress.Candidates, "completed Run has no candidate progress") + require.Equal(t, int32(1), completedRun.Status.Progress.Candidates.Published) + require.Len(t, completedRun.Status.CandidateRefs, 1) + + t.Log("Verify the Candidate is a real API object controlled by the immutable Run") + require.True(t, metav1.IsControlledBy(&completedRun, request), "Run is not controlled by its Request") + var candidates nvidiacomv1beta2.DynamoGraphDeploymentCandidateList + require.NoError(t, env.Client().List(ctx, &candidates, client.InNamespace(env.Namespace()))) + require.Len(t, candidates.Items, 1) + require.True(t, metav1.IsControlledBy(&candidates.Items[0], &completedRun), "Candidate is not controlled by its Run") + require.Equal(t, completedRun.Status.CandidateRefs[0].Name, candidates.Items[0].Name) + require.NotEmpty(t, candidates.Items[0].Spec.Components, "Candidate has no materialized DGD components") + + t.Log("Match the published Candidate API contract against the reviewed golden") + golden.EventuallyMatchManifests(t, env.Client(), env.Namespace(), filepath.Join(scenarioDir, "output.yaml")) + }) + } +} + func getOnlyClusterDGD(t *testing.T, ctx context.Context, c client.Client, namespace string) *nvidiacomv1beta1.DynamoGraphDeployment { t.Helper() diff --git a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/input.yaml b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/input.yaml new file mode 100644 index 000000000000..6df9ef9ce03d --- /dev/null +++ b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/input.yaml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nvidia.com/v1beta2 +kind: DynamoGraphDeploymentRequest +metadata: + name: aisimulate-vllm +spec: + model: + name: Qwen/Qwen3-32B + backends: + - vllm + hardware: + gpu: + sku: h200_sxm + budget: 2 + workload: + static: + inputTokens: 1024 + outputTokens: 128 + load: + concurrency: 10 + objective: + mode: Optimize + metric: ThroughputPerGPU + sla: {} + aisimulate: + inputOverrides: + engine: + mode: aggregated + context_length: 32768 + workers: + # TODO(#17581): Remove this explicit value after AISimulate preserves an unset runtime block size. + aggregated: + kv_cache: + block_size: 64 + prefill: null + decode: null + traffic: + stop: + requests: 8 + optimizer: + algorithm: random + max_trials: 3 + parallelism: 1 + candidate_timeout_seconds: 120 + dgdr_run: + snapshot_interval_seconds: 0 + recommendation: + maxCandidates: 1 diff --git a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/output.yaml b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/output.yaml new file mode 100644 index 000000000000..55d62bc91c25 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/output.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nvidia.com/v1beta2 +kind: DynamoGraphDeploymentCandidate +metadata: + $strict: false + name: $glob:aisimulate-vllm-1-*-evaluated-point-* + labels: + $strict: false + nvidia.com/dgdr-candidate-id: $pattern:^evaluated-point-[0-9a-f]{12}$ + nvidia.com/dgdr-run-uid: $ignore +spec: + $strict: false + parameters: + $strict: false + backend: vllm +status: + metrics: + $strict: false + score: $ignore + usedGpus: $ignore From ed4190b87b9589fbebf9e0a4a62e2be1c5561628 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 17:44:45 +0100 Subject: [PATCH 064/100] operator/dgdr: verify frozen v1beta2 input end to end Signed-off-by: Dr. Stefan Schimanski --- .../internal/controller/clusterenv_test.go | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/deploy/operator/internal/controller/clusterenv_test.go b/deploy/operator/internal/controller/clusterenv_test.go index c43c752d7702..e406e2138770 100644 --- a/deploy/operator/internal/controller/clusterenv_test.go +++ b/deploy/operator/internal/controller/clusterenv_test.go @@ -420,6 +420,35 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing require.Equal(t, int32(1), completedRun.Status.Progress.Candidates.Published) require.Len(t, completedRun.Status.CandidateRefs, 1) + t.Log("Verify the Job consumed the immutable input recorded in Run status") + require.NotNil(t, completedRun.Status.JobRef, "completed Run has no Job reference") + require.NotNil(t, completedRun.Status.AISimulate, "completed Run has no AISimulate status") + require.NotNil(t, completedRun.Status.AISimulate.Input, "completed Run has no effective AISimulate input") + key := types.NamespacedName{Namespace: env.Namespace(), Name: completedRun.Status.JobRef.Name} + var executionConfig corev1.ConfigMap + require.NoError(t, env.Client().Get(ctx, key, &executionConfig)) + require.True(t, metav1.IsControlledBy(&executionConfig, &completedRun), "execution ConfigMap is not controlled by its Run") + var job batchv1.Job + require.NoError(t, env.Client().Get(ctx, key, &job)) + require.True(t, metav1.IsControlledBy(&job, &completedRun), "AISimulate Job is not controlled by its Run") + var inputVolume *corev1.Volume + for i := range job.Spec.Template.Spec.Volumes { + if job.Spec.Template.Spec.Volumes[i].Name == dgdrRunInputVolume { + inputVolume = &job.Spec.Template.Spec.Volumes[i] + } + } + require.NotNil(t, inputVolume, "AISimulate Job has no input volume") + require.NotNil(t, inputVolume.ConfigMap, "AISimulate input volume is not a ConfigMap") + require.Equal(t, executionConfig.Name, inputVolume.ConfigMap.Name) + sweeper, found := dgdrRunContainer(job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + require.True(t, found, "AISimulate Job has no sweeper container") + require.Contains(t, sweeper.VolumeMounts, corev1.VolumeMount{ + Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true, + }) + configInput, err := dgdrRunStatusInput(executionConfig.Data[dgdrRunInputKey]) + require.NoError(t, err) + require.JSONEq(t, string(completedRun.Status.AISimulate.Input.Raw), string(configInput.Raw)) + t.Log("Verify the Candidate is a real API object controlled by the immutable Run") require.True(t, metav1.IsControlledBy(&completedRun, request), "Run is not controlled by its Request") var candidates nvidiacomv1beta2.DynamoGraphDeploymentCandidateList From b5379fc91a2b8c6c99c95c31237edcba463057b1 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Wed, 28 Oct 2026 20:15:55 +0100 Subject: [PATCH 065/100] operator/dgdr: verify selected-shape budget end to end Signed-off-by: Dr. Stefan Schimanski --- .../internal/controller/clusterenv_test.go | 75 +++++++++++++++++-- .../testdata/dgdr-v1beta2/vllm/input.yaml | 4 - 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/deploy/operator/internal/controller/clusterenv_test.go b/deploy/operator/internal/controller/clusterenv_test.go index e406e2138770..65c21b5b5987 100644 --- a/deploy/operator/internal/controller/clusterenv_test.go +++ b/deploy/operator/internal/controller/clusterenv_test.go @@ -9,6 +9,7 @@ package controller import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -29,6 +30,7 @@ import ( grovemock "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/mocks/grove" lwsmock "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/mocks/lws" "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/operatorchart" + webhookdefaulting "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/defaulting" webhooksetup "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/setup" "github.com/go-logr/logr" "github.com/stretchr/testify/require" @@ -36,6 +38,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" @@ -48,6 +51,19 @@ import ( var clusterTestEnv *clusterenv.Env +type clusterTestHardwareDiscoverer struct{} + +func (clusterTestHardwareDiscoverer) Discover(context.Context, nvidiacomv1beta1.GPUSKUType) (*gpu.GPUInfo, error) { + // Model two H200 node shapes where only the homogeneous 8-GPU shape is executable. + return &gpu.GPUInfo{ + System: nvidiacomv1beta1.GPUSKUTypeH200SXM, + GPUsPerNode: 8, + NodesWithGPUs: 2, + TotalGPUs: 12, + SelectedShapeGPUs: 8, + }, nil +} + func runClusterTests(m *testing.M, prepare func(context.Context, client.Client) (func(context.Context) error, error)) int { if prepare != nil { logf.SetLogger(zap.New(zap.UseDevMode(true))) @@ -62,7 +78,7 @@ func runClusterTests(m *testing.M, prepare func(context.Context, client.Client) operatorConfig := &configv1alpha1.OperatorConfiguration{} configv1alpha1.SetDefaultsOperatorConfiguration(operatorConfig) operatorConfig.GPU.DiscoveryEnabled = ptr.To(false) - runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true, DisaggregatedSet: true}} + runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{Grove: true, LWS: true, DisaggregatedSet: true, GPUDiscovery: true}} additionalAdmission := lwsmock.Configurations() groveAdmission := grovemock.Configurations() additionalAdmission.Mutating = append(additionalAdmission.Mutating, groveAdmission.Mutating...) @@ -89,6 +105,9 @@ func runClusterTests(m *testing.M, prepare func(context.Context, client.Client) RuntimeConfig: runtimeConfig, OperatorVersion: operatorVersion, OperatorPrincipal: "system:serviceaccount:dynamo-system:dynamo-operator", + DGDRV1beta2: webhookdefaulting.DGDRV1beta2Options{ + Hardware: clusterTestHardwareDiscoverer{}, + }, }); err != nil { return err } @@ -357,6 +376,11 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing if err := env.Client().Get(ctx, client.ObjectKeyFromObject(&objects[0]), request); err != nil { t.Fatalf("get admitted v1beta2 DGDR: %v", err) } + require.NotNil(t, request.Spec.Hardware, "admitted Request has no hardware default") + require.NotNil(t, request.Spec.Hardware.GPU, "admitted Request has no GPU default") + require.Equal(t, "h200_sxm", ptr.Deref(request.Spec.Hardware.GPU.SKU, "")) + require.Equal(t, int32(8), ptr.Deref(request.Spec.Hardware.GPU.Budget, 0), + "admission must use the selected 8-GPU node shape, not the 12-GPU inventory") t.Log("Watch every Run transition before starting the production controllers") watchClient, err := client.NewWithWatch(env.RESTConfig(), client.Options{Scheme: newClusterTestScheme()}) @@ -365,13 +389,14 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing require.NoError(t, err) defer runWatch.Stop() - t.Log("Provide the Run controller the discovered node shape represented by the explicit Request hardware") + t.Log("Provide the Run controller the same heterogeneous inventory resolved by Request admission") hardwareCache := gpu.NewGPUDiscoveryCache() hardwareCache.Set(nvidiacomv1beta1.GPUSKUTypeH200SXM, &gpu.GPUInfo{ - System: nvidiacomv1beta1.GPUSKUTypeH200SXM, - GPUsPerNode: 8, - NodesWithGPUs: 1, - TotalGPUs: 8, + System: nvidiacomv1beta1.GPUSKUTypeH200SXM, + GPUsPerNode: 8, + NodesWithGPUs: 2, + TotalGPUs: 12, + SelectedShapeGPUs: 8, }, time.Hour) t.Log("Start the production Request, Run, AISimulate Job, and publisher chain") @@ -419,6 +444,10 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing require.NotNil(t, completedRun.Status.Progress.Candidates, "completed Run has no candidate progress") require.Equal(t, int32(1), completedRun.Status.Progress.Candidates.Published) require.Len(t, completedRun.Status.CandidateRefs, 1) + require.NotNil(t, completedRun.Spec.Hardware) + require.NotNil(t, completedRun.Spec.Hardware.GPU) + require.Equal(t, int32(8), ptr.Deref(completedRun.Spec.Hardware.GPU.Budget, 0), + "immutable Run snapshot lost the admitted selected-shape budget") t.Log("Verify the Job consumed the immutable input recorded in Run status") require.NotNil(t, completedRun.Status.JobRef, "completed Run has no Job reference") @@ -448,6 +477,8 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing configInput, err := dgdrRunStatusInput(executionConfig.Data[dgdrRunInputKey]) require.NoError(t, err) require.JSONEq(t, string(completedRun.Status.AISimulate.Input.Raw), string(configInput.Raw)) + require.Equal(t, int64(8), aisimulateHardwareBudget(t, completedRun.Status.AISimulate.Input.Raw), + "lowered AISimulate input lost the selected-shape budget") t.Log("Verify the Candidate is a real API object controlled by the immutable Run") require.True(t, metav1.IsControlledBy(&completedRun, request), "Run is not controlled by its Request") @@ -457,6 +488,8 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing require.True(t, metav1.IsControlledBy(&candidates.Items[0], &completedRun), "Candidate is not controlled by its Run") require.Equal(t, completedRun.Status.CandidateRefs[0].Name, candidates.Items[0].Name) require.NotEmpty(t, candidates.Items[0].Spec.Components, "Candidate has no materialized DGD components") + require.LessOrEqual(t, candidateUsedGPUs(t, candidates.Items[0].Status.Metrics), int64(8), + "Candidate exceeds the selected-shape GPU budget") t.Log("Match the published Candidate API contract against the reviewed golden") golden.EventuallyMatchManifests(t, env.Client(), env.Namespace(), filepath.Join(scenarioDir, "output.yaml")) @@ -464,6 +497,36 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing } } +func aisimulateHardwareBudget(t *testing.T, raw []byte) int64 { + t.Helper() + + // Decode only the effective-input field whose value this end-to-end test follows. + var input struct { + Optimization struct { + Constraints struct { + MaxCandidateGPUs int64 `json:"max_candidate_gpus"` + } `json:"constraints"` + } `json:"optimization"` + } + require.NoError(t, json.Unmarshal(raw, &input)) + require.Positive(t, input.Optimization.Constraints.MaxCandidateGPUs, + "effective AISimulate input has no positive max_candidate_gpus") + return input.Optimization.Constraints.MaxCandidateGPUs +} + +func candidateUsedGPUs(t *testing.T, metrics *runtime.RawExtension) int64 { + t.Helper() + require.NotNil(t, metrics, "Candidate has no metrics") + + // Decode only the Candidate resource accounting needed to enforce the selected budget. + var decoded struct { + UsedGPUs int64 `json:"usedGpus"` + } + require.NoError(t, json.Unmarshal(metrics.Raw, &decoded)) + require.Positive(t, decoded.UsedGPUs, "Candidate metrics have no positive usedGpus") + return decoded.UsedGPUs +} + func getOnlyClusterDGD(t *testing.T, ctx context.Context, c client.Client, namespace string) *nvidiacomv1beta1.DynamoGraphDeployment { t.Helper() diff --git a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/input.yaml b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/input.yaml index 6df9ef9ce03d..83be9a7dd04c 100644 --- a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/input.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm/input.yaml @@ -10,10 +10,6 @@ spec: name: Qwen/Qwen3-32B backends: - vllm - hardware: - gpu: - sku: h200_sxm - budget: 2 workload: static: inputTokens: 1024 From a2f69a0e46a925257c9e10a8d75322c20b171cf9 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Thu, 29 Oct 2026 15:57:57 +0100 Subject: [PATCH 066/100] operator/dgdr: verify selected node width end to end Signed-off-by: Dr. Stefan Schimanski --- .../internal/controller/clusterenv_test.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/deploy/operator/internal/controller/clusterenv_test.go b/deploy/operator/internal/controller/clusterenv_test.go index 65c21b5b5987..a9333464cceb 100644 --- a/deploy/operator/internal/controller/clusterenv_test.go +++ b/deploy/operator/internal/controller/clusterenv_test.go @@ -381,6 +381,8 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing require.Equal(t, "h200_sxm", ptr.Deref(request.Spec.Hardware.GPU.SKU, "")) require.Equal(t, int32(8), ptr.Deref(request.Spec.Hardware.GPU.Budget, 0), "admission must use the selected 8-GPU node shape, not the 12-GPU inventory") + require.Equal(t, int32(8), ptr.Deref(request.Spec.Hardware.NumGPUsPerNode, 0), + "admission must persist the selected node width") t.Log("Watch every Run transition before starting the production controllers") watchClient, err := client.NewWithWatch(env.RESTConfig(), client.Options{Scheme: newClusterTestScheme()}) @@ -389,16 +391,6 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing require.NoError(t, err) defer runWatch.Stop() - t.Log("Provide the Run controller the same heterogeneous inventory resolved by Request admission") - hardwareCache := gpu.NewGPUDiscoveryCache() - hardwareCache.Set(nvidiacomv1beta1.GPUSKUTypeH200SXM, &gpu.GPUInfo{ - System: nvidiacomv1beta1.GPUSKUTypeH200SXM, - GPUsPerNode: 8, - NodesWithGPUs: 2, - TotalGPUs: 12, - SelectedShapeGPUs: 8, - }, time.Hour) - t.Log("Start the production Request, Run, AISimulate Job, and publisher chain") operatorConfig := clusterTestRestrictedConfig(env.Namespace()) runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{}} @@ -409,7 +401,6 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing } return SetupDynamoGraphDeploymentRunV1beta2(mgr, DynamoGraphDeploymentRunSetupOptions{ SetupOptions: setupOptions, - GPUDiscoveryCache: hardwareCache, PlannerImage: plannerImage, DynamoVersion: dynamoVersion, OperatorImage: operatorImage, @@ -448,6 +439,8 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing require.NotNil(t, completedRun.Spec.Hardware.GPU) require.Equal(t, int32(8), ptr.Deref(completedRun.Spec.Hardware.GPU.Budget, 0), "immutable Run snapshot lost the admitted selected-shape budget") + require.Equal(t, int32(8), ptr.Deref(completedRun.Spec.Hardware.NumGPUsPerNode, 0), + "immutable Run snapshot lost the admitted node width") t.Log("Verify the Job consumed the immutable input recorded in Run status") require.NotNil(t, completedRun.Status.JobRef, "completed Run has no Job reference") From ab4b32b8e45608863e43ecae3dea586492051c2a Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Thu, 29 Oct 2026 09:17:25 -0700 Subject: [PATCH 067/100] feat(operator): replay recorded traces from a PVC in DGDR v1beta2 runs Squash of hzhou/dgdr-trace-pvc at 410f5dd4b5 onto sttts-dgdr-v1beta2-integration (a2f69a0e46). The branch history carried a copy of the integration stack for every force-push of the base, which put the pull request over the 250-commit limit of the GitHub API that the DCO app and copy-pr-bot read. The tree is identical to 410f5dd4b5. Signed-off-by: hongkuanz Co-Authored-By: Claude Opus 5.5 --- deploy/operator/api/roundtrip_fuzz_test.go | 26 +- .../dynamographdeploymentrequest_types.go | 102 +- .../v1beta2/dynamographdeploymentrun_types.go | 2 +- .../api/v1beta2/zz_generated.deepcopy.go | 81 ++ ...dia.com_dynamographdeploymentrequests.yaml | 95 +- .../nvidia.com_dynamographdeploymentruns.yaml | 97 +- .../operator/internal/aisimulate/defaults.go | 4 +- deploy/operator/internal/aisimulate/doc.go | 15 +- deploy/operator/internal/aisimulate/input.go | 16 + deploy/operator/internal/aisimulate/paths.go | 2 + deploy/operator/internal/aisimulate/trace.go | 49 + .../internal/aisimulate/trace_test.go | 83 ++ .../internal/controller/clusterenv_test.go | 27 + .../dgdr_v1beta2_trace_clustertest_test.go | 350 ++++++ ...aphdeploymentrequest_v1beta2_controller.go | 18 + ...ploymentrequest_v1beta2_controller_test.go | 66 + ...mographdeploymentrun_v1beta2_controller.go | 460 ++++++- ...phdeploymentrun_v1beta2_controller_test.go | 1099 +++++++++++++++++ .../dgdr-v1beta2/vllm-trace/input.yaml | 46 + .../dgdr-v1beta2/vllm-trace/output.yaml | 22 + .../dgdr-v1beta2/vllm-trace/trace.jsonl | 40 + .../dynamographdeploymentrequest_v1beta2.go | 9 +- .../webhook/defaulting/suite_envtest_test.go | 31 + ...hdeploymentrequest_carrier_envtest_test.go | 31 +- .../dynamographdeploymentrequest_v1beta2.go | 5 +- ...ploymentrequest_v1beta2_input_overrides.go | 50 +- ...ploymentrequest_validation_envtest_test.go | 435 +++++++ .../webhook/validation/shared_helpers.go | 14 + .../webhook/validation/shared_v1beta2.go | 106 ++ .../additional-resources/api-reference-k8s.md | 101 +- .../dynamo-graph-deployment-request.mdx | 94 ++ .../kubernetes-api/full-api-reference.mdx | 110 +- .../scripts/tests/test_gen_kubernetes_api.py | 2 +- 33 files changed, 3597 insertions(+), 91 deletions(-) create mode 100644 deploy/operator/internal/aisimulate/trace.go create mode 100644 deploy/operator/internal/aisimulate/trace_test.go create mode 100644 deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go create mode 100644 deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml create mode 100644 deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/output.yaml create mode 100644 deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/trace.jsonl create mode 100644 deploy/operator/internal/webhook/validation/shared_v1beta2.go diff --git a/deploy/operator/api/roundtrip_fuzz_test.go b/deploy/operator/api/roundtrip_fuzz_test.go index 6fc6158d09eb..bee73c544229 100644 --- a/deploy/operator/api/roundtrip_fuzz_test.go +++ b/deploy/operator/api/roundtrip_fuzz_test.go @@ -563,7 +563,31 @@ func fuzzBeta2Hardware(hardware *v1beta2.HardwareSpec, c randfill.Continue) { } func fuzzBeta2Workload(workload *v1beta2.WorkloadSpec, c randfill.Continue) { - if workload == nil || workload.Static == nil { + if workload == nil { + return + } + + // A trace is exclusive with static and names one admitted claim-relative file. When both are + // filled, keep one of them at random so static and trace are equally likely. + if workload.Trace != nil && workload.Static != nil { + if c.Bool() { + workload.Static = nil + } else { + workload.Trace = nil + } + } + if workload.Trace != nil { + workload.Trace.Format = v1beta2.TraceFormatDynamo + workload.Trace.Source.PVC = &v1beta2.TracePVCSource{ + ClaimName: fmt.Sprintf("traces-%d", c.Uint32()), + Path: fmt.Sprintf("retune/window-%d.jsonl.gz", c.Uint32()), + } + if workload.Trace.Replay != nil && workload.Trace.Replay.ArrivalSpeedupRatio != nil { + workload.Trace.Replay.ArrivalSpeedupRatio = oneOfPtr(c, 0.5, 1.0, 4.0) + } + return + } + if workload.Static == nil { return } diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 25536741ca97..742befb25a5f 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -154,13 +154,89 @@ type StaticWorkloadSpec struct { Load *StaticLoadSpec `json:"load,omitempty"` } -// WorkloadSpec defines the synthetic traffic model used for every candidate evaluation. +// TraceFormat identifies the decoder of a recorded request trace. +// +kubebuilder:validation:Enum=dynamo +type TraceFormat string + +const ( + // TraceFormatDynamo is the Dynamo request trace (dynamo.request.trace.v1) that the frontend + // records, as JSON Lines or gzip-compressed JSON Lines. + TraceFormatDynamo TraceFormat = "dynamo" +) + +// TracePVCSource names one trace file on a PersistentVolumeClaim in the Request namespace. +type TracePVCSource struct { + // ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. + // The Run Job mounts only the trace file from it, read-only, and only into the AISimulate + // container. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + ClaimName string `json:"claimName"` + + // Path names one trace file relative to the root of the claim. It is neither a directory + // nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' + // segments or a NUL character, which no file name can hold. The file must be complete before + // the Request is created and must never be rewritten: every Run of the Request reads it when + // its Job starts. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=1024 + // +kubebuilder:validation:XValidation:rule="!self.startsWith('/') && !self.endsWith('/') && !self.contains('//') && !self.matches('(^|/)[.][.]?(/|$)')",message="path must be relative to the claim root without empty, '.' or '..' segments" + // +kubebuilder:validation:XValidation:rule="!self.contains('\\u0000')",message="path must not contain a NUL character" + Path string `json:"path"` +} + +// TraceSourceSpec identifies exactly one trace artifact. A PersistentVolumeClaim is the only +// source in this release. +// +kubebuilder:validation:XValidation:rule="has(self.pvc)",message="pvc is required" +type TraceSourceSpec struct { + // PVC reads the trace from a PersistentVolumeClaim in the Request namespace. + // +optional + PVC *TracePVCSource `json:"pvc,omitempty"` +} + +// TraceReplaySpec configures how AISimulate replays a decoded trace. +type TraceReplaySpec struct { + // ArrivalSpeedupRatio multiplies the recorded request-arrival rate. 1.0 preserves it. + // When omitted, the recorded arrival times are replayed unchanged. + // +optional + // +kubebuilder:validation:ExclusiveMinimum=true + // +kubebuilder:validation:Minimum=0 + ArrivalSpeedupRatio *float64 `json:"arrivalSpeedupRatio,omitempty"` +} + +// TraceWorkloadSpec replays a recorded request trace open loop at its recorded arrival times. +type TraceWorkloadSpec struct { + // Format identifies the trace decoder. It has no default; only dynamo is admitted in this + // release. + // +kubebuilder:validation:Required + Format TraceFormat `json:"format,omitempty"` + + // Source identifies the trace artifact. + Source TraceSourceSpec `json:"source"` + + // Replay configures how the decoded trace is replayed. + // +optional + Replay *TraceReplaySpec `json:"replay,omitempty"` +} + +// WorkloadSpec defines the traffic model used for every candidate evaluation. static and +// trace are mutually exclusive. +// +kubebuilder:validation:XValidation:rule="!(has(self.static) && has(self.trace))",message="static and trace are mutually exclusive" type WorkloadSpec struct { - // Static defines AISimulate synthetic traffic. When omitted, admission supplies the - // complete static default. An explicit empty object opts out; any nonempty object is - // completed with omitted inputTokens, outputTokens and load defaults. + // Static defines AISimulate synthetic traffic. When omitted without a trace, admission + // supplies the complete static default. An explicit empty object opts out; any nonempty + // object is completed with omitted inputTokens, outputTokens and load defaults. // +optional Static *StaticWorkloadSpec `json:"static,omitempty"` + + // Trace replays a recorded request trace from a PersistentVolumeClaim in the Request + // namespace. Admission supplies no static default when it is set. The Run Job mounts only the + // trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, + // or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change + // traffic.source.type, format or paths. + // +optional + Trace *TraceWorkloadSpec `json:"trace,omitempty"` } // ObjectiveMode selects the optimization mode. Pareto is reserved for a future @@ -239,9 +315,13 @@ type AISimulateSpec struct { // runtime_image, runtime_version_override, derive_runtime_images, // num_gpus_per_node, and namespace are operator-owned. Request admission // rejects them, and the Run controller enforces its values while lowering the - // immutable snapshot. The operator runs the AISimulate version from its - // configured planner image; Requests and Runs cannot select a different image - // or version. + // immutable snapshot. When spec.workload.trace is set, traffic and + // traffic.source must remain objects, and the trace owns traffic.source.type, + // format, and paths the same way: Request admission rejects them, and the Run + // controller enforces the typed values. Other traffic overrides are AISimulate + // input, which AISimulate validates when the Run's Job starts. The operator runs + // the AISimulate version from its configured planner image; Requests and Runs + // cannot select a different image or version. // +optional // +kubebuilder:pruning:PreserveUnknownFields // +kubebuilder:validation:Type=object @@ -279,12 +359,14 @@ type NativeSearchInput struct { // +optional Hardware *HardwareSpec `json:"hardware,omitempty"` - // Workload defines the synthetic traffic model used for every candidate evaluation. + // Workload defines the traffic model used for every candidate evaluation: synthetic static + // traffic or a recorded trace. // When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; // an explicit empty workload receives the same shape. workload.static: {} opts out of all - // traffic defaults. Any other static object is completed with its missing defaults. + // traffic defaults. Any other static object is completed with its missing defaults, and a + // trace workload receives no static default. // spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over - // every lowered traffic field it names. + // every lowered traffic field it names, except the traffic.source fields a trace owns. // The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction // search. Lowering independently defaults optimizer.max_trials to 320. // +optional diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go index affb00815de3..1c363bc39090 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go @@ -27,7 +27,7 @@ import ( // DynamoGraphDeploymentRunSpec is an immutable snapshot of an accepted DGDR spec. // Request admission resolves its structural defaults before the Request controller copies it; // Request admission defaulting does not apply to Runs. -// +kubebuilder:validation:XValidation:rule="has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && ((!has(self.hardware.gpu) && !has(self.hardware.numGpusPerNode)) || (has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.hardware.numGpusPerNode))) && has(self.workload) && has(self.workload.static) && ((!has(self.workload.static.inputTokens) && !has(self.workload.static.outputTokens) && !has(self.workload.static.load)) || (has(self.workload.static.inputTokens) && has(self.workload.static.outputTokens) && has(self.workload.static.load))) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.objective.sla) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0",message="spec must be a fully resolved Request admission snapshot" +// +kubebuilder:validation:XValidation:rule="has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && ((!has(self.hardware.gpu) && !has(self.hardware.numGpusPerNode)) || (has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.hardware.numGpusPerNode))) && has(self.workload) && (has(self.workload.trace) || (has(self.workload.static) && ((!has(self.workload.static.inputTokens) && !has(self.workload.static.outputTokens) && !has(self.workload.static.load)) || (has(self.workload.static.inputTokens) && has(self.workload.static.outputTokens) && has(self.workload.static.load))))) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.objective.sla) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0",message="spec must be a fully resolved Request admission snapshot" // // The Run reconciler lowers this immutable typed snapshot plus inputOverrides exactly once and // records the resulting public recommendation input in status.aisimulate.input. diff --git a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go index 899c5121053e..7167809dd96e 100644 --- a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go @@ -854,6 +854,82 @@ func (in *StaticWorkloadSpec) DeepCopy() *StaticWorkloadSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TracePVCSource) DeepCopyInto(out *TracePVCSource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TracePVCSource. +func (in *TracePVCSource) DeepCopy() *TracePVCSource { + if in == nil { + return nil + } + out := new(TracePVCSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TraceReplaySpec) DeepCopyInto(out *TraceReplaySpec) { + *out = *in + if in.ArrivalSpeedupRatio != nil { + in, out := &in.ArrivalSpeedupRatio, &out.ArrivalSpeedupRatio + *out = new(float64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TraceReplaySpec. +func (in *TraceReplaySpec) DeepCopy() *TraceReplaySpec { + if in == nil { + return nil + } + out := new(TraceReplaySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TraceSourceSpec) DeepCopyInto(out *TraceSourceSpec) { + *out = *in + if in.PVC != nil { + in, out := &in.PVC, &out.PVC + *out = new(TracePVCSource) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TraceSourceSpec. +func (in *TraceSourceSpec) DeepCopy() *TraceSourceSpec { + if in == nil { + return nil + } + out := new(TraceSourceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TraceWorkloadSpec) DeepCopyInto(out *TraceWorkloadSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + if in.Replay != nil { + in, out := &in.Replay, &out.Replay + *out = new(TraceReplaySpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TraceWorkloadSpec. +func (in *TraceWorkloadSpec) DeepCopy() *TraceWorkloadSpec { + if in == nil { + return nil + } + out := new(TraceWorkloadSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadSpec) DeepCopyInto(out *WorkloadSpec) { *out = *in @@ -862,6 +938,11 @@ func (in *WorkloadSpec) DeepCopyInto(out *WorkloadSpec) { *out = new(StaticWorkloadSpec) (*in).DeepCopyInto(*out) } + if in.Trace != nil { + in, out := &in.Trace, &out.Trace + *out = new(TraceWorkloadSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadSpec. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 5a1d6bb9d9c9..35e489f174b0 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9610,9 +9610,13 @@ spec: runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the - immutable snapshot. The operator runs the AISimulate version from its - configured planner image; Requests and Runs cannot select a different image - or version. + immutable snapshot. When spec.workload.trace is set, traffic and + traffic.source must remain objects, and the trace owns traffic.source.type, + format, and paths the same way: Request admission rejects them, and the Run + controller enforces the typed values. Other traffic overrides are AISimulate + input, which AISimulate validates when the Run's Job starts. The operator runs + the AISimulate version from its configured planner image; Requests and Runs + cannot select a different image or version. type: object x-kubernetes-preserve-unknown-fields: true type: object @@ -9780,20 +9784,22 @@ spec: x-kubernetes-preserve-unknown-fields: true workload: description: |- - Workload defines the synthetic traffic model used for every candidate evaluation. + Workload defines the traffic model used for every candidate evaluation: synthetic static + traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: {} opts out of all - traffic defaults. Any other static object is completed with its missing defaults. + traffic defaults. Any other static object is completed with its missing defaults, and a + trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over - every lowered traffic field it names. + every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. properties: static: description: |- - Static defines AISimulate synthetic traffic. When omitted, admission supplies the - complete static default. An explicit empty object opts out; any nonempty object is - completed with omitted inputTokens, outputTokens and load defaults. + Static defines AISimulate synthetic traffic. When omitted without a trace, admission + supplies the complete static default. An explicit empty object opts out; any nonempty + object is completed with omitted inputTokens, outputTokens and load defaults. properties: inputTokens: description: InputTokens is the synthetic request input length in tokens. @@ -9862,7 +9868,78 @@ spec: minimum: 1 type: integer type: object + trace: + description: |- + Trace replays a recorded request trace from a PersistentVolumeClaim in the Request + namespace. Admission supplies no static default when it is set. The Run Job mounts only the + trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, + or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change + traffic.source.type, format or paths. + properties: + format: + description: |- + Format identifies the trace decoder. It has no default; only dynamo is admitted in this + release. + enum: + - dynamo + type: string + replay: + description: Replay configures how the decoded trace is replayed. + properties: + arrivalSpeedupRatio: + description: |- + ArrivalSpeedupRatio multiplies the recorded request-arrival rate. 1.0 preserves it. + When omitted, the recorded arrival times are replayed unchanged. + exclusiveMinimum: true + minimum: 0 + type: number + type: object + source: + description: Source identifies the trace artifact. + properties: + pvc: + description: PVC reads the trace from a PersistentVolumeClaim in the Request namespace. + properties: + claimName: + description: |- + ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. + The Run Job mounts only the trace file from it, read-only, and only into the AISimulate + container. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path names one trace file relative to the root of the claim. It is neither a directory + nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' + segments or a NUL character, which no file name can hold. The file must be complete before + the Request is created and must never be rewritten: every Run of the Request reads it when + its Job starts. + maxLength: 1024 + minLength: 1 + type: string + x-kubernetes-validations: + - message: path must be relative to the claim root without empty, '.' or '..' segments + rule: '!self.startsWith(''/'') && !self.endsWith(''/'') && !self.contains(''//'') && !self.matches(''(^|/)[.][.]?(/|$)'')' + - message: path must not contain a NUL character + rule: '!self.contains(''\u0000'')' + required: + - claimName + - path + type: object + type: object + x-kubernetes-validations: + - message: pvc is required + rule: has(self.pvc) + required: + - format + - source + type: object type: object + x-kubernetes-validations: + - message: static and trace are mutually exclusive + rule: '!(has(self.static) && has(self.trace))' type: object x-kubernetes-validations: - message: exactly one of native v1beta2 fields or v1beta1 must be specified diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index 5110346080b9..e5c7afd8799a 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -77,9 +77,13 @@ spec: runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the - immutable snapshot. The operator runs the AISimulate version from its - configured planner image; Requests and Runs cannot select a different image - or version. + immutable snapshot. When spec.workload.trace is set, traffic and + traffic.source must remain objects, and the trace owns traffic.source.type, + format, and paths the same way: Request admission rejects them, and the Run + controller enforces the typed values. Other traffic overrides are AISimulate + input, which AISimulate validates when the Run's Job starts. The operator runs + the AISimulate version from its configured planner image; Requests and Runs + cannot select a different image or version. type: object x-kubernetes-preserve-unknown-fields: true type: object @@ -239,20 +243,22 @@ spec: type: object workload: description: |- - Workload defines the synthetic traffic model used for every candidate evaluation. + Workload defines the traffic model used for every candidate evaluation: synthetic static + traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: {} opts out of all - traffic defaults. Any other static object is completed with its missing defaults. + traffic defaults. Any other static object is completed with its missing defaults, and a + trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over - every lowered traffic field it names. + every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. properties: static: description: |- - Static defines AISimulate synthetic traffic. When omitted, admission supplies the - complete static default. An explicit empty object opts out; any nonempty object is - completed with omitted inputTokens, outputTokens and load defaults. + Static defines AISimulate synthetic traffic. When omitted without a trace, admission + supplies the complete static default. An explicit empty object opts out; any nonempty + object is completed with omitted inputTokens, outputTokens and load defaults. properties: inputTokens: description: InputTokens is the synthetic request input length in tokens. @@ -321,11 +327,82 @@ spec: minimum: 1 type: integer type: object + trace: + description: |- + Trace replays a recorded request trace from a PersistentVolumeClaim in the Request + namespace. Admission supplies no static default when it is set. The Run Job mounts only the + trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, + or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change + traffic.source.type, format or paths. + properties: + format: + description: |- + Format identifies the trace decoder. It has no default; only dynamo is admitted in this + release. + enum: + - dynamo + type: string + replay: + description: Replay configures how the decoded trace is replayed. + properties: + arrivalSpeedupRatio: + description: |- + ArrivalSpeedupRatio multiplies the recorded request-arrival rate. 1.0 preserves it. + When omitted, the recorded arrival times are replayed unchanged. + exclusiveMinimum: true + minimum: 0 + type: number + type: object + source: + description: Source identifies the trace artifact. + properties: + pvc: + description: PVC reads the trace from a PersistentVolumeClaim in the Request namespace. + properties: + claimName: + description: |- + ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. + The Run Job mounts only the trace file from it, read-only, and only into the AISimulate + container. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + path: + description: |- + Path names one trace file relative to the root of the claim. It is neither a directory + nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' + segments or a NUL character, which no file name can hold. The file must be complete before + the Request is created and must never be rewritten: every Run of the Request reads it when + its Job starts. + maxLength: 1024 + minLength: 1 + type: string + x-kubernetes-validations: + - message: path must be relative to the claim root without empty, '.' or '..' segments + rule: '!self.startsWith(''/'') && !self.endsWith(''/'') && !self.contains(''//'') && !self.matches(''(^|/)[.][.]?(/|$)'')' + - message: path must not contain a NUL character + rule: '!self.contains(''\u0000'')' + required: + - claimName + - path + type: object + type: object + x-kubernetes-validations: + - message: pvc is required + rule: has(self.pvc) + required: + - format + - source + type: object type: object + x-kubernetes-validations: + - message: static and trace are mutually exclusive + rule: '!(has(self.static) && has(self.trace))' type: object x-kubernetes-validations: - message: spec must be a fully resolved Request admission snapshot - rule: has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && ((!has(self.hardware.gpu) && !has(self.hardware.numGpusPerNode)) || (has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.hardware.numGpusPerNode))) && has(self.workload) && has(self.workload.static) && ((!has(self.workload.static.inputTokens) && !has(self.workload.static.outputTokens) && !has(self.workload.static.load)) || (has(self.workload.static.inputTokens) && has(self.workload.static.outputTokens) && has(self.workload.static.load))) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.objective.sla) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0 + rule: has(self.model) && has(self.backends) && size(self.backends) > 0 && has(self.hardware) && ((!has(self.hardware.gpu) && !has(self.hardware.numGpusPerNode)) || (has(self.hardware.gpu) && has(self.hardware.gpu.sku) && has(self.hardware.gpu.budget) && has(self.hardware.numGpusPerNode))) && has(self.workload) && (has(self.workload.trace) || (has(self.workload.static) && ((!has(self.workload.static.inputTokens) && !has(self.workload.static.outputTokens) && !has(self.workload.static.load)) || (has(self.workload.static.inputTokens) && has(self.workload.static.outputTokens) && has(self.workload.static.load))))) && has(self.objective) && has(self.objective.mode) && has(self.objective.metric) && has(self.objective.sla) && has(self.recommendation) && has(self.recommendation.maxCandidates) && self.recommendation.maxCandidates > 0 status: description: DynamoGraphDeploymentRunStatus represents one search execution's observed state. properties: diff --git a/deploy/operator/internal/aisimulate/defaults.go b/deploy/operator/internal/aisimulate/defaults.go index d26bbd54fea5..ba2be457abbc 100644 --- a/deploy/operator/internal/aisimulate/defaults.go +++ b/deploy/operator/internal/aisimulate/defaults.go @@ -57,7 +57,9 @@ func applyDefaults(underlay map[string]any) error { b := builder{root: underlay} if has(underlay, trafficPath...) { - if !has(underlay, trafficStopRequests...) { + // A trace accepts only max_virtual_time_seconds; a request count would be rejected. + sourceType, _, _ := unstructured.NestedString(underlay, trafficSourceType...) + if sourceType != "trace" && !has(underlay, trafficStopRequests...) { b.set(float64(DefaultRequestCount), trafficStopRequests) } } diff --git a/deploy/operator/internal/aisimulate/doc.go b/deploy/operator/internal/aisimulate/doc.go index a41ef4bee3d7..158e0dcd2947 100644 --- a/deploy/operator/internal/aisimulate/doc.go +++ b/deploy/operator/internal/aisimulate/doc.go @@ -27,12 +27,15 @@ // Lower starts with the public engine-role underlay required for AISimulate's default // aggregated-and-disaggregated search. It then translates the typed spec (model, backends, // hardware, workload, objective) into that input. The derived defaults that no API field -// carries are added next: the request count, a bounded trial budget for a -// KV-load range search, and optimization.strict_sla when an SLA bound is set. Last, -// spec.aisimulate.inputOverrides is applied on top as a JSON Merge Patch (RFC 7386): upstream -// objects merge recursively, arrays and scalars replace the value, and null deletes it. The -// input overrides are authoritative: they win on every path they name, including the derived -// defaults, and are applied once, last. +// carries are added next: the request count (not for a trace, which accepts only a virtual-time +// stop), a bounded trial budget for a KV-load range search, and optimization.strict_sla when an +// SLA bound is set. Last, spec.aisimulate.inputOverrides is applied on top as a JSON Merge +// Patch (RFC 7386): upstream objects merge recursively, arrays and scalars replace the value, +// and null deletes it. The input overrides are authoritative: they win on every path they name, +// including the derived defaults, and are applied once, last. Only the operator-owned paths are +// exempt: Request admission rejects them and the Run controller restores them after this +// lowering. They are the operator-owned dgdr_run fields and, for a trace, traffic.source.type, +// format and paths. // // The document stays unstructured (map[string]any) because its schema belongs to the // AISimulate release bundled in the operator's Planner image, not to this package. Reads and writes go diff --git a/deploy/operator/internal/aisimulate/input.go b/deploy/operator/internal/aisimulate/input.go index 54d42bc1c679..8010ce0fb96d 100644 --- a/deploy/operator/internal/aisimulate/input.go +++ b/deploy/operator/internal/aisimulate/input.go @@ -161,6 +161,8 @@ func Underlay(spec *v1beta2.NativeSearchInput) (map[string]any, error) { } } + b.setTraceWorkload(spec.Workload) + if objective := spec.Objective; objective != nil { if objective.Mode == v1beta2.ObjectiveModePareto { if len(objective.Metrics) > 0 && !isDefaultParetoMetrics(objective.Metrics) { @@ -191,6 +193,20 @@ func Underlay(spec *v1beta2.NativeSearchInput) (map[string]any, error) { return b.root, b.err } +// setTraceWorkload lowers workload.trace: it replays the recorded arrival times, scaled by the +// speedup only when one is set. workload may be nil; a workload without a trace lowers nothing. +func (b *builder) setTraceWorkload(workload *v1beta2.WorkloadSpec) { + if workload == nil || workload.Trace == nil { + return + } + trace := workload.Trace + b.set(TraceSource(trace), trafficSource) + b.set("trace_timestamps", trafficLoadType) + if replay := trace.Replay; replay != nil && replay.ArrivalSpeedupRatio != nil { + b.set(*replay.ArrivalSpeedupRatio, trafficLoadSpeedup) + } +} + func hasStaticWorkloadFields(static *v1beta2.StaticWorkloadSpec) bool { return static.InputTokens != nil || static.OutputTokens != nil || static.Load != nil } diff --git a/deploy/operator/internal/aisimulate/paths.go b/deploy/operator/internal/aisimulate/paths.go index 25ef533ecd6c..82a5b4ab5b08 100644 --- a/deploy/operator/internal/aisimulate/paths.go +++ b/deploy/operator/internal/aisimulate/paths.go @@ -37,6 +37,7 @@ var ( optimizerMaxTrials = []string{"optimizer", "max_trials"} trafficPath = []string{"traffic"} + trafficSource = []string{"traffic", "source"} trafficSourceType = []string{"traffic", "source", "type"} trafficSourceInputTokens = []string{"traffic", "source", "input_tokens"} trafficSourceOutputTokens = []string{"traffic", "source", "output_tokens"} @@ -44,6 +45,7 @@ var ( trafficLoadConcurrency = []string{"traffic", "load", "concurrency"} trafficLoadRequestsPerSecond = []string{"traffic", "load", "requests_per_second"} trafficLoadFraction = []string{"traffic", "load", "fraction"} + trafficLoadSpeedup = []string{"traffic", "load", "speedup"} trafficStopRequests = []string{"traffic", "stop", "requests"} ) diff --git a/deploy/operator/internal/aisimulate/trace.go b/deploy/operator/internal/aisimulate/trace.go new file mode 100644 index 000000000000..275411798ac6 --- /dev/null +++ b/deploy/operator/internal/aisimulate/trace.go @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "strings" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" +) + +// TraceMountPath is the directory of the Run Job's sweeper container that holds the trace file. +const TraceMountPath = "/var/run/aisimulate/trace" + +// TracePath is where the Run Job mounts the trace file at claimPath: one fixed file under +// TraceMountPath. The Job mounts only that file, through a subPath of the claim, so the kubelet +// resolves claimPath and refuses a symlink that resolves outside the claim. The name keeps the +// ".gz" suffix, by which AISimulate recognizes gzip-compressed JSON Lines. +func TracePath(claimPath string) string { + if strings.HasSuffix(claimPath, ".gz") { + return TraceMountPath + "/trace.jsonl.gz" + } + return TraceMountPath + "/trace.jsonl" +} + +// TraceSource returns the traffic.source object that trace owns: a trace of its format read from +// its one claim file under TraceMountPath. trace must not be nil. A trace without a pvc source has +// no paths; the CRD schema rejects that shape. +func TraceSource(trace *v1beta2.TraceWorkloadSpec) map[string]any { + source := map[string]any{"type": "trace", "format": string(trace.Format)} + if pvc := trace.Source.PVC; pvc != nil { + source["paths"] = []any{TracePath(pvc.Path)} + } + return source +} diff --git a/deploy/operator/internal/aisimulate/trace_test.go b/deploy/operator/internal/aisimulate/trace_test.go new file mode 100644 index 000000000000..b74752dddbfb --- /dev/null +++ b/deploy/operator/internal/aisimulate/trace_test.go @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package aisimulate + +import ( + "testing" + + v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" +) + +func withTrace(spec *v1beta2.DynamoGraphDeploymentRequestSpec, speedup *float64) *v1beta2.DynamoGraphDeploymentRequestSpec { + spec.Workload = &v1beta2.WorkloadSpec{Trace: &v1beta2.TraceWorkloadSpec{ + Format: v1beta2.TraceFormatDynamo, + Source: v1beta2.TraceSourceSpec{PVC: &v1beta2.TracePVCSource{ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz"}}, + }} + if speedup != nil { + spec.Workload.Trace.Replay = &v1beta2.TraceReplaySpec{ArrivalSpeedupRatio: speedup} + } + return spec +} + +func TestLowerTraceWorkload(t *testing.T) { + tests := []struct { + name string + spec *v1beta2.DynamoGraphDeploymentRequestSpec + wantTraffic string + }{ + { + name: "recorded arrival times without a request stop", + spec: withTrace(defaultedSpec(), nil), + wantTraffic: `{"load":{"type":"trace_timestamps"},` + + `"source":{"format":"dynamo","paths":["/var/run/aisimulate/trace/trace.jsonl.gz"],"type":"trace"}}`, + }, + { + name: "an arrival speedup lowers to traffic.load.speedup", + spec: withTrace(defaultedSpec(), new(2.5)), + wantTraffic: `{"load":{"speedup":2.5,"type":"trace_timestamps"},` + + `"source":{"format":"dynamo","paths":["/var/run/aisimulate/trace/trace.jsonl.gz"],"type":"trace"}}`, + }, + { + name: "a stop override still applies", + spec: withInputOverrides(withTrace(defaultedSpec(), nil), `{"traffic":{"stop":{"max_virtual_time_seconds":600}}}`), + wantTraffic: `{"load":{"type":"trace_timestamps"},` + + `"source":{"format":"dynamo","paths":["/var/run/aisimulate/trace/trace.jsonl.gz"],"type":"trace"},` + + `"stop":{"max_virtual_time_seconds":600}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + traffic := nested(t, lower(t, tt.spec), "traffic").(map[string]any) + if got := encode(t, traffic); got != tt.wantTraffic { + t.Fatalf("traffic\n got: %s\nwant: %s", got, tt.wantTraffic) + } + }) + } +} + +func TestTracePathIsOneFixedFileUnderTheMount(t *testing.T) { + for claimPath, want := range map[string]string{ + "retune/w-1.jsonl.gz": "/var/run/aisimulate/trace/trace.jsonl.gz", + "retune/w-1.jsonl": "/var/run/aisimulate/trace/trace.jsonl", + "../../etc/passwd": "/var/run/aisimulate/trace/trace.jsonl", + } { + if got := TracePath(claimPath); got != want { + t.Errorf("TracePath(%q) = %q, want %q", claimPath, got, want) + } + } +} diff --git a/deploy/operator/internal/controller/clusterenv_test.go b/deploy/operator/internal/controller/clusterenv_test.go index a9333464cceb..9d921162729d 100644 --- a/deploy/operator/internal/controller/clusterenv_test.go +++ b/deploy/operator/internal/controller/clusterenv_test.go @@ -47,6 +47,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/yaml" ) var clusterTestEnv *clusterenv.Env @@ -367,6 +368,26 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing t.Log("Allow the AISimulate Job Pod to run while preventing unrelated ReplicaSet workloads") env.BlockReplicaSets() + t.Log("Write a trace scenario's trace to the claim file its Request names before applying it") + traceFixture := filepath.Join(scenarioDir, "trace.jsonl") + var trace clusterTestTraceFile + traceBefore := "" + if _, err := os.Stat(traceFixture); err == nil { + scenario := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{} + data, err := os.ReadFile(input) + require.NoError(t, err) + require.NoError(t, yaml.Unmarshal(data, scenario)) + require.NotNil(t, scenario.Spec.Workload, "trace scenario input has no workload") + require.NotNil(t, scenario.Spec.Workload.Trace, "trace scenario input has no trace workload") + require.NotNil(t, scenario.Spec.Workload.Trace.Source.PVC, "trace scenario input has no trace claim") + trace = clusterTestTraceFile{ + claim: scenario.Spec.Workload.Trace.Source.PVC.ClaimName, + path: scenario.Spec.Workload.Trace.Source.PVC.Path, + } + createClusterTestTraceClaim(t, env, trace) + traceBefore = writeClusterTestTrace(t, env, trace, traceFixture, plannerImage) + } + t.Log("Apply the native v1beta2 Request through production admission") objects := golden.ApplyManifests(t, filepath.Join(scenarioDir, "input.yaml"), env.Client(), env.Namespace()) if len(objects) != 1 || objects[0].GetKind() != "DynamoGraphDeploymentRequest" { @@ -486,6 +507,12 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing t.Log("Match the published Candidate API contract against the reviewed golden") golden.EventuallyMatchManifests(t, env.Client(), env.Namespace(), filepath.Join(scenarioDir, "output.yaml")) + + if traceBefore != "" { + t.Log("The Run replayed the mounted trace file and left its contents, owner and mode unchanged") + requireClusterTestTraceInput(t, &completedRun, trace) + require.Equal(t, traceBefore, statClusterTestTrace(t, env, trace, "trace-after", plannerImage)) + } }) } } diff --git a/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go b/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go new file mode 100644 index 000000000000..e8ed309447e1 --- /dev/null +++ b/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go @@ -0,0 +1,350 @@ +//go:build clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + semver "github.com/Masterminds/semver/v3" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/aisimulate" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/testing/clusterenv" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +// The writer gives a trace file an owner the Job's users do not have and the trace group, so the +// sweeper can read it only through the supplemental group of the Run Job's pod. +const ( + clusterTestTraceUID = 4321 + clusterTestTraceGID = dgdrRunTraceGroup + clusterTestTraceMode = "640" +) + +// clusterTestTraceFile is the claim and claim-relative path a trace Request reads. +type clusterTestTraceFile struct { + claim, path string +} + +// clusterTestTraceWriter gzips the fixture to the claim file and gives it the requested owner and mode. +const clusterTestTraceWriter = ` +import gzip, os, shutil, sys +source, target, uid, gid, mode = sys.argv[1:] +os.makedirs(os.path.dirname(target), exist_ok=True) +with open(source, "rb") as src, gzip.GzipFile(target, "wb", mtime=0) as dst: + shutil.copyfileobj(src, dst) +os.chown(target, int(uid), int(gid)) +os.chmod(target, int(mode, 8)) +` + +// clusterTestTraceLinker places a symlink at the claim file that points outside the claim. +const clusterTestTraceLinker = ` +import os, sys +link, target = sys.argv[1:] +os.makedirs(os.path.dirname(link), exist_ok=True) +os.symlink(target, link) +` + +// clusterTestTraceStat reports the claim file's owner, mode and digest as its termination message. +const clusterTestTraceStat = ` +import hashlib, os, sys +st = os.lstat(sys.argv[1]) +with open(sys.argv[1], "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() +with open("/dev/termination-log", "w") as log: + log.write(f"{st.st_uid} {st.st_gid} {st.st_mode & 0o7777:o} {digest}") +` + +// createClusterTestTraceClaim creates the static filesystem claim of trace in env's namespace. Kind +// installs no dynamic provisioner the test may rely on, so the claim binds a PersistentVolume of its +// own on one node, which is deleted with the test. +func createClusterTestTraceClaim(t *testing.T, env *clusterenv.TestEnv, trace clusterTestTraceFile) { + t.Helper() + ctx := t.Context() + + t.Log("Pin the trace volume to one node, so every pod that mounts it sees the same files") + var nodes corev1.NodeList + require.NoError(t, env.Client().List(ctx, &nodes)) + require.NotEmpty(t, nodes.Items, "the cluster has no nodes") + hostname := nodes.Items[0].Labels[corev1.LabelHostname] + require.NotEmpty(t, hostname, "node %q has no %s label", nodes.Items[0].Name, corev1.LabelHostname) + + t.Log("Create the PersistentVolume and the claim bound to it") + volume := &corev1.PersistentVolume{ + Name: "dgdr-trace-" + env.Namespace(), + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("16Mi")}, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + ClaimRef: &corev1.ObjectReference{Namespace: env.Namespace(), Name: trace.claim}, + PersistentVolumeSource: corev1.PersistentVolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/local/dynamo-clustertest/" + env.Namespace(), + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }}, + NodeAffinity: &corev1.VolumeNodeAffinity{Required: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: corev1.LabelHostname, Operator: corev1.NodeSelectorOpIn, Values: []string{hostname}, + }}, + }}}}, + }, + } + require.NoError(t, env.Client().Create(ctx, volume)) + t.Cleanup(func() { + if err := env.Client().Delete(context.Background(), volume); client.IgnoreNotFound(err) != nil { + t.Errorf("delete trace PersistentVolume %q: %v", volume.Name, err) + } + }) + claim := &corev1.PersistentVolumeClaim{ + Name: trace.claim, Namespace: env.Namespace(), + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + VolumeMode: ptr.To(corev1.PersistentVolumeFilesystem), + StorageClassName: new(""), + VolumeName: volume.Name, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("16Mi")}, + }, + }, + } + require.NoError(t, env.Client().Create(ctx, claim)) +} + +// runClusterTestTraceJob runs script with args in image, with claim mounted at /claim (read-only +// unless writable) and the files of configMap, if named, at /fixture. It waits for the Job to +// succeed and returns its termination message. +func runClusterTestTraceJob( + t *testing.T, + env *clusterenv.TestEnv, + name, image, claim, configMap string, + writable bool, + script string, + args ...string, +) string { + t.Helper() + ctx := t.Context() + + t.Logf("Run the %s Job as root with the trace claim mounted", name) + pod := corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + SecurityContext: &corev1.PodSecurityContext{RunAsUser: ptr.To[int64](0), RunAsGroup: ptr.To[int64](0)}, + Containers: []corev1.Container{{ + Name: "trace", + Image: image, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: append([]string{"python3", "-c", script}, args...), + VolumeMounts: []corev1.VolumeMount{{Name: "claim", MountPath: "/claim", ReadOnly: !writable}}, + }}, + Volumes: []corev1.Volume{{ + Name: "claim", + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: claim, ReadOnly: !writable}, + }}, + } + if configMap != "" { + pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "fixture", MountPath: "/fixture", ReadOnly: true}) + pod.Volumes = append(pod.Volumes, corev1.Volume{ + Name: "fixture", + ConfigMap: &corev1.ConfigMapVolumeSource{Name: configMap}, + }) + } + job := &batchv1.Job{ + Name: name, Namespace: env.Namespace(), + Spec: batchv1.JobSpec{BackoffLimit: ptr.To[int32](0), Template: corev1.PodTemplateSpec{Spec: pod}}, + } + require.NoError(t, env.Client().Create(ctx, job)) + + t.Logf("Wait for the %s Job to succeed", name) + require.Eventually(t, func() bool { + require.NoError(t, env.Client().Get(ctx, client.ObjectKeyFromObject(job), job)) + require.Zero(t, job.Status.Failed, "%s Job failed: %+v", name, job.Status.Conditions) + return job.Status.Succeeded == 1 + }, 5*time.Minute, time.Second) + + t.Logf("Read the %s Job's termination message", name) + var pods corev1.PodList + require.NoError(t, env.Client().List(ctx, &pods, client.InNamespace(env.Namespace()), + client.MatchingLabels{batchv1.JobNameLabel: name})) + require.Len(t, pods.Items, 1) + require.Len(t, pods.Items[0].Status.ContainerStatuses, 1) + terminated := pods.Items[0].Status.ContainerStatuses[0].State.Terminated + require.NotNil(t, terminated, "%s pod did not terminate", name) + return terminated.Message +} + +// writeClusterTestTrace gzips the fixture at fixture into trace and returns the file's owner, mode +// and digest as the claim holds it. +func writeClusterTestTrace(t *testing.T, env *clusterenv.TestEnv, trace clusterTestTraceFile, fixture, image string) string { + t.Helper() + ctx := t.Context() + + t.Log("Carry the checked-in trace fixture into the cluster") + content, err := os.ReadFile(fixture) + require.NoError(t, err) + require.NoError(t, env.Client().Create(ctx, &corev1.ConfigMap{ + Name: "trace-fixture", Namespace: env.Namespace(), + Data: map[string]string{"trace.jsonl": string(content)}, + })) + + runClusterTestTraceJob(t, env, "trace-writer", image, trace.claim, "trace-fixture", true, clusterTestTraceWriter, + "/fixture/trace.jsonl", filepath.Join("/claim", trace.path), + fmt.Sprint(clusterTestTraceUID), fmt.Sprint(clusterTestTraceGID), clusterTestTraceMode) + return statClusterTestTrace(t, env, trace, "trace-before", image) +} + +// statClusterTestTrace returns the owner, mode and digest of trace as its claim holds it, read by a +// Job named name. +func statClusterTestTrace(t *testing.T, env *clusterenv.TestEnv, trace clusterTestTraceFile, name, image string) string { + t.Helper() + stat := runClusterTestTraceJob(t, env, name, image, trace.claim, "", false, clusterTestTraceStat, + filepath.Join("/claim", trace.path)) + require.Regexp(t, fmt.Sprintf(`^%d %d %s [0-9a-f]{64}$`, clusterTestTraceUID, clusterTestTraceGID, clusterTestTraceMode), stat) + return stat +} + +// requireClusterTestTraceInput checks that run replayed trace through its fixed mount path. +func requireClusterTestTraceInput(t *testing.T, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, trace clusterTestTraceFile) { + t.Helper() + require.NotNil(t, run.Status.AISimulate, "completed Run has no AISimulate status") + var input struct { + Traffic struct { + Source map[string]any `json:"source"` + Load map[string]any `json:"load"` + } `json:"traffic"` + } + require.NoError(t, yaml.Unmarshal(run.Status.AISimulate.Input.Raw, &input)) + require.Equal(t, map[string]any{ + "type": "trace", "format": "dynamo", "paths": []any{aisimulate.TracePath(trace.path)}, + }, input.Traffic.Source) + require.Equal(t, "trace_timestamps", input.Traffic.Load["type"]) +} + +func TestClusterDynamoGraphDeploymentRequestV1beta2RefusesATraceSymlinkThatLeavesItsClaim(t *testing.T) { + ctx := t.Context() + plannerImage := clusterenv.RequireEnv(t, "DYNAMO_CLUSTERTEST_PROFILER_IMAGE") + operatorImage := clusterenv.RequireEnv(t, "DYNAMO_CLUSTERTEST_OPERATOR_IMAGE") + + t.Log("Create an isolated namespace in the explicitly unlocked cluster") + env := clusterTestEnv.RunT(t) + env.BlockReplicaSets() + + t.Log("Place a symlink at the trace path that points at a file outside the claim") + trace := clusterTestTraceFile{claim: "linked-traces", path: "retune/escape.jsonl.gz"} + createClusterTestTraceClaim(t, env, trace) + runClusterTestTraceJob(t, env, "trace-linker", plannerImage, trace.claim, "", true, clusterTestTraceLinker, + filepath.Join("/claim", trace.path), "/etc/passwd") + + t.Log("Apply a Request that reads the linked trace path through production admission") + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ + Name: "aisimulate-trace-symlink", Namespace: env.Namespace(), + Spec: nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec{ + DynamoGraphDeploymentRequestNativeSpec: nvidiacomv1beta2.DynamoGraphDeploymentRequestNativeSpec{ + NativeSearchInput: nvidiacomv1beta2.NativeSearchInput{ + Model: &nvidiacomv1beta2.ModelSpec{Name: "Qwen/Qwen3-32B"}, + Backends: []nvidiacomv1beta2.Backend{nvidiacomv1beta2.BackendVLLM}, + Hardware: &nvidiacomv1beta2.HardwareSpec{GPU: &nvidiacomv1beta2.GPUHardwareSpec{ + SKU: new("h200_sxm"), Budget: new(int32(2)), + }}, + Workload: &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ + Format: nvidiacomv1beta2.TraceFormatDynamo, + Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: trace.claim, Path: trace.path, + }}, + }}, + }, + }, + }, + } + require.NoError(t, env.Client().Create(ctx, request)) + + t.Log("Watch every Run transition before starting the production controllers") + watchClient, err := client.NewWithWatch(env.RESTConfig(), client.Options{Scheme: newClusterTestScheme()}) + require.NoError(t, err) + runWatch, err := watchClient.Watch(ctx, &nvidiacomv1beta2.DynamoGraphDeploymentRunList{}, client.InNamespace(env.Namespace())) + require.NoError(t, err) + defer runWatch.Stop() + startClusterTestDGDRV1beta2Controllers(t, env, plannerImage, operatorImage) + + t.Log("The kubelet refuses the subPath, so the Run fails before AISimulate reads anything") + var failed nvidiacomv1beta2.DynamoGraphDeploymentRun + deadline := time.NewTimer(5 * time.Minute) + defer deadline.Stop() + for failed.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed { + select { + case event, ok := <-runWatch.ResultChan(): + require.True(t, ok, "Run watch closed before the Run failed") + require.NotEqual(t, watch.Error, event.Type, "Run watch failed") + run, ok := event.Object.(*nvidiacomv1beta2.DynamoGraphDeploymentRun) + require.Truef(t, ok, "Run watch returned %T", event.Object) + require.NotEqual(t, nvidiacomv1beta2.DGDRRunPhaseSucceeded, run.Status.Phase, "Run read a trace outside its claim") + failed = *run.DeepCopy() + case <-deadline.C: + t.Fatal("the Run did not fail for its escaping trace symlink") + } + } + condition := meta.FindStatusCondition(failed.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + require.NotNil(t, condition) + require.Equal(t, dgdrRunReasonTraceSourceUnavailable, condition.Reason) + require.Contains(t, condition.Message, `cannot be mounted: failed to prepare subPath for volumeMount "trace"`) + + t.Log("No Candidate was published, and the completed Run deletes its Job") + var candidates nvidiacomv1beta2.DynamoGraphDeploymentCandidateList + require.NoError(t, env.Client().List(ctx, &candidates, client.InNamespace(env.Namespace()))) + require.Empty(t, candidates.Items) + require.Eventually(t, func() bool { + var jobs batchv1.JobList + require.NoError(t, env.Client().List(ctx, &jobs, client.InNamespace(env.Namespace()))) + for i := range jobs.Items { + if metav1.IsControlledBy(&jobs.Items[i], &failed) { + return false + } + } + return true + }, time.Minute, time.Second, "the failed Run's Job was not deleted") +} + +// startClusterTestDGDRV1beta2Controllers starts the production Request and Run controllers for env. +func startClusterTestDGDRV1beta2Controllers(t *testing.T, env *clusterenv.TestEnv, plannerImage, operatorImage string) { + t.Helper() + operatorVersion, err := clusterTestOperatorVersion() + require.NoError(t, err) + parsedOperatorVersion, err := semver.NewVersion(operatorVersion) + require.NoError(t, err) + dynamoVersion := fmt.Sprintf("%d.%d.%d", parsedOperatorVersion.Major(), parsedOperatorVersion.Minor(), parsedOperatorVersion.Patch()) + + t.Log("Start the production Request, Run, AISimulate Job, and publisher chain") + operatorConfig := clusterTestRestrictedConfig(env.Namespace()) + runtimeConfig := &commoncontroller.RuntimeConfig{Gate: features.Gates{}} + env.StartManager(func(mgr ctrl.Manager) error { + setupOptions := SetupOptions{Config: operatorConfig, RuntimeConfig: runtimeConfig} + if err := SetupDynamoGraphDeploymentRequestV1beta2(mgr, setupOptions); err != nil { + return err + } + return SetupDynamoGraphDeploymentRunV1beta2(mgr, DynamoGraphDeploymentRunSetupOptions{ + SetupOptions: setupOptions, + PlannerImage: plannerImage, + DynamoVersion: dynamoVersion, + OperatorImage: operatorImage, + OperatorImagePullPolicy: corev1.PullIfNotPresent, + }) + }) +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go index 4cd79565f8e1..3fce0c179186 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller.go @@ -207,6 +207,13 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) reconcileMissingRun( } marker := meta.FindStatusCondition(request.Status.Conditions, nvidiacomv1beta2.DGDRRequestConditionAccepted) + if request.Status.ActiveRunRef == nil && marker != nil && marker.ObservedGeneration == request.Generation && + marker.Reason == dgdrReasonInvalidSpec { + // The Run CRD rejected this generation's Run; only a new generation can change that. + condition := dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, marker.Message) + return &condition, ctrl.Result{}, nil + } + markerMatches := request.Status.ActiveRunRef != nil && request.Status.ActiveRunRef.Name == wantName && marker != nil && marker.ObservedGeneration == request.Generation if markerMatches && (marker.Reason == dgdrReasonRunCreating || marker.Reason == dgdrReasonRunCreated) { @@ -298,6 +305,17 @@ func (r *DynamoGraphDeploymentRequestV1beta2Reconciler) createReservedRun( original *nvidiacomv1beta2.DynamoGraphDeploymentRequest, ) (ctrl.Result, error) { err := r.Create(ctx, run) + + // The Run CRD rejects the frozen spec, such as a trace that reached the Request without + // admission. Retrying the same spec cannot succeed, so the reservation is released and the + // Request reports the invalid spec until a new generation. + if apierrors.IsInvalid(err) { + log.FromContext(ctx).Info("Run CRD rejects the Request spec", "reason", err.Error()) + request.Status.ActiveRunRef = nil + meta.SetStatusCondition(&request.Status.Conditions, + dgdrV1beta2Condition(request, metav1.ConditionFalse, dgdrReasonInvalidSpec, err.Error())) + return ctrl.Result{}, r.patchStatus(ctx, request, original) + } if err != nil && !apierrors.IsAlreadyExists(err) { return ctrl.Result{}, fmt.Errorf("create Run %q: %w", run.Name, err) } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index 22c4b313ac45..9e60fd469fdd 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "testing" @@ -32,7 +33,9 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -740,3 +743,66 @@ func TestV1beta2RequestControllerPreservesInputOverridesForRunLowering(t *testin t.Fatalf("Run inputOverrides changed with the Request: %s", got) } } + +func TestV1beta2RequestControllerReportsARunTheCRDRejectsInsteadOfRetryingIt(t *testing.T) { + invalid := apierrors.NewInvalid( + schema.GroupKind{Group: nvidiacomv1beta2.GroupVersion.Group, Kind: "DynamoGraphDeploymentRun"}, "search-1", + field.ErrorList{field.Invalid(field.NewPath("spec", "workload", "trace", "source", "pvc", "path"), "../escape.jsonl", + "path must be relative to the claim root without empty, '.' or '..' segments")}, + ) + tests := []struct { + name string + createErr error + wantRuns []string + wantReason string + wantMessage string + }{ + { + name: "a Run the API server accepts is created", + wantRuns: []string{"search-1"}, + wantReason: dgdrReasonRunCreated, + }, + { + name: "a Run the CRD rejects is reported once as an invalid spec", + createErr: invalid, + wantReason: dgdrReasonInvalidSpec, + wantMessage: invalid.Error(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + creates := 0 + f := newDGDRV1beta2FixtureWithInterceptor(t, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*nvidiacomv1beta2.DynamoGraphDeploymentRun); ok { + creates++ + if tt.createErr != nil { + return tt.createErr + } + } + return c.Create(ctx, obj, opts...) + }, + }, dgdrV1beta2Request(true)) + + t.Log("Reconcile twice: an invalid spec is a condition for this generation, not a create to retry") + f.reconcile() + f.reconcile() + if names := f.runNames(); !slices.Equal(names, tt.wantRuns) { + t.Fatalf("runs = %v, want %v", names, tt.wantRuns) + } + if creates != 1 { + t.Fatalf("Run creates = %d, want 1", creates) + } + if request := f.request(); tt.createErr != nil && request.Status.ActiveRunRef != nil { + t.Fatalf("activeRunRef = %+v, want the reservation released", request.Status.ActiveRunRef) + } + condition := dgdrV1beta2AcceptedCondition(t, f.request()) + if condition.Reason != tt.wantReason { + t.Fatalf("Accepted = %+v, want reason %s", condition, tt.wantReason) + } + if tt.wantMessage != "" && condition.Message != tt.wantMessage { + t.Fatalf("Accepted message = %q, want %q", condition.Message, tt.wantMessage) + } + }) + } +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index f9b10fb3b046..2b808127cae0 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -12,6 +12,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "strconv" "strings" "time" @@ -34,15 +35,19 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" ) const ( dgdrRunJobControllerUIDIndex = ".metadata.dgdrRunControllerUID" + dgdrRunTraceClaimIndex = ".spec.workload.trace.source.pvc.claimName" dgdrRunMaxJobNameLength = 63 dgdrRunInputKey = "input.yaml" @@ -63,10 +68,12 @@ const ( dgdrRunPublisherContainer = "publisher" dgdrRunInputVolume = "input" dgdrRunSnapshotVolume = "snapshot" + dgdrRunTraceVolume = "trace" dgdrRunServiceAccountVolume = "publisher-kube-api" dgdrRunSnapshotPathAnnotation = "nvidia.com/dgdr-snapshot-path" dgdrRunResourceComponent = "aisimulate" dgdrRunServiceAccountTokenTTL = int64(3600) + dgdrRunTraceGroup = int64(65532) dgdrRunReasonPending = "JobPending" dgdrRunReasonRunning = "JobRunning" dgdrRunReasonSucceeded = "SearchSucceeded" @@ -121,6 +128,7 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates/status,verbs=get;update;patch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch @@ -132,9 +140,21 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( if err := r.Get(ctx, req.NamespacedName, run); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } - if run.DeletionTimestamp != nil || meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { + if run.DeletionTimestamp != nil { return ctrl.Result{}, nil } + + // A Run fails for its trace while its Job may still run, because the failure is recorded before + // the Job is deleted: a failed or stale write then leaves the Job in place rather than gone and + // recreated. Deleting it is the one step left for such a completed Run. + if completed := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted); completed != nil && + completed.Status == metav1.ConditionTrue { + if completed.Reason != dgdrRunReasonTraceSourceUnavailable { + return ctrl.Result{}, nil + } + return ctrl.Result{}, r.deleteUnfinishedJobs(ctx, run) + } + if condition := dgdrRunPendingPermanentFailure(run); condition != nil { return r.finishPermanentFailure(ctx, run, condition.Reason, errors.New(condition.Message)) } @@ -178,8 +198,10 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) beginPermanentFailure( Message: cause.Error(), ObservedGeneration: run.Generation, }) + // Like every Run status write, the marker carries the resourceVersion the cache showed, so a + // stale Run cannot overwrite a newer status, such as a recorded trace failure. if !apiequality.Semantic.DeepEqual(original.Status, run.Status) { - if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { + if err := r.Status().Patch(ctx, run, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return ctrl.Result{}, &dgdrRunRetryableError{cause: fmt.Errorf("record permanent failure for Run %q: %w", run.Name, err)} } } @@ -231,7 +253,42 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRun( if supersedingGeneration, cancelRequested := dgdrRunSupersedingGeneration(run); cancelRequested { return r.cancelSupersededRun(ctx, run, job, supersedingGeneration) } + + // A Job that can never mount its trace would keep the Run active forever, so a Run that has not + // created its Job checks the claim before anything is created for it. A recorded jobRef, or a Job + // the API server already holds, means the Job was created even if the cache does not show it yet; + // it is checked below once the cache shows it. resourceName := dgdrRunResourceName(run.Name) + if job == nil && run.Status.JobRef == nil { + unavailable, err := r.traceSourceUnavailable(ctx, run, false) + if err != nil { + return ctrl.Result{}, err + } + if unavailable != "" { + created, err := r.runJobExists(ctx, run, resourceName) + if err != nil { + return ctrl.Result{}, err + } + if !created { + return ctrl.Result{}, r.completeRun(ctx, run, dgdrRunReasonTraceSourceUnavailable, unavailable) + } + } + } + + // A Job whose sweeper can never read the trace would stay pending with nothing to end it: the + // Run fails, and the completed Run then deletes the Job. Claim and pod events wake the Run for + // this check. + if job != nil && dgdrRunTraceClaim(run) != nil && !dgdrRunJobFinished(job) { + unavailable, err := r.jobTraceUnavailable(ctx, run, job) + if err != nil { + return ctrl.Result{}, err + } + if unavailable != "" { + return ctrl.Result{}, r.completeRun(ctx, run, dgdrRunReasonTraceSourceUnavailable, + fmt.Sprintf("%s; Job %q is deleted", unavailable, job.Name)) + } + } + if err := r.reconcilePublisherAccess(ctx, run, resourceName); err != nil { return ctrl.Result{}, err } @@ -342,7 +399,9 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) completeRun( if apiequality.Semantic.DeepEqual(original.Status, run.Status) { return nil } - if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { + // The write carries the resourceVersion the cache showed, so a stale Run conflicts instead of + // overwriting a newer status. + if err := r.Status().Patch(ctx, run, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return &dgdrRunRetryableError{cause: fmt.Errorf("complete Run %q: %w", run.Name, err)} } return nil @@ -377,6 +436,267 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) ownedJobs( return owned, nil } +const ( + dgdrRunReasonTraceSourceUnavailable = "TraceSourceUnavailable" + dgdrRunClaimProtectionFinalizer = "kubernetes.io/pvc-protection" +) + +// traceSourceUnavailable returns why a Job could not mount the Run's trace claim, or "" when it +// can or the Run replays no trace. scheduled tells whether a pod of the Job is already bound to a +// node: the kubelet still mounts a claim being deleted while the pvc-protection finalizer keeps it +// for that pod. The answer fails the Run terminally, so the claim is read through the uncached API +// reader: claim events only wake the Run, and a claim the cache has not observed yet, such as one +// created just before the Run, must not end it. A Pending claim is accepted: a WaitForFirstConsumer +// claim binds only once its first pod is scheduled. Access modes are not checked: whether another +// pod holds a ReadWriteOnce or ReadWriteOncePod claim is a scheduling question, not a terminal one. +// Other read errors are returned for a retry. run must not be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) traceSourceUnavailable( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + scheduled bool, +) (string, error) { + source := dgdrRunTraceClaim(run) + if source == nil { + return "", nil + } + name := source.ClaimName + claim := &corev1.PersistentVolumeClaim{} + err := r.APIReader.Get(ctx, types.NamespacedName{Namespace: run.Namespace, Name: name}, claim) + if apierrors.IsNotFound(err) { + return fmt.Sprintf("trace PersistentVolumeClaim %q does not exist in namespace %q", name, run.Namespace), nil + } + if err != nil { + return "", fmt.Errorf("get trace PersistentVolumeClaim %q in namespace %q: %w", name, run.Namespace, err) + } + + // Each state below keeps the Job from ever mounting the claim read-only as a filesystem. + reason := "" + switch { + case claim.DeletionTimestamp != nil && !(scheduled && controllerutil.ContainsFinalizer(claim, dgdrRunClaimProtectionFinalizer)): + reason = "is being deleted" + case claim.Spec.VolumeMode != nil && *claim.Spec.VolumeMode == corev1.PersistentVolumeBlock: + reason = "is a raw block volume, not a filesystem" + case claim.Status.Phase == corev1.ClaimLost: + reason = "has lost its PersistentVolume" + default: + return "", nil + } + return fmt.Sprintf("trace PersistentVolumeClaim %q in namespace %q %s", name, run.Namespace, reason), nil +} + +// runJobExists reports whether the API server holds the Job this Run controls under name. Creating +// the Job and recording its jobRef are separate writes, so a Job can exist while both the cache and +// status.jobRef miss it. The Job is therefore read through the uncached API reader, and only right +// before the Run would be failed terminally for its trace claim. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) runJobExists( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + name string, +) (bool, error) { + job := &batchv1.Job{} + err := r.APIReader.Get(ctx, types.NamespacedName{Namespace: run.Namespace, Name: name}, job) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("get Job %q of Run %q: %w", name, run.Name, err) + } + return metav1.IsControlledBy(job, run), nil +} + +// dgdrRunTraceClaim returns the claim the Run's trace is read from, or nil when it replays no trace. +// run must not be nil. +func dgdrRunTraceClaim(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) *nvidiacomv1beta2.TracePVCSource { + if run.Spec.Workload == nil || run.Spec.Workload.Trace == nil { + return nil + } + return run.Spec.Workload.Trace.Source.PVC +} + +// dgdrRunTraceClaimNames indexes a Run by the claim its trace is read from. +func dgdrRunTraceClaimNames(obj client.Object) []string { + run, ok := obj.(*nvidiacomv1beta2.DynamoGraphDeploymentRun) + if !ok { + return nil + } + if source := dgdrRunTraceClaim(run); source != nil { + return []string{source.ClaimName} + } + return nil +} + +// runsForTraceClaim maps a claim event to the Runs in the claim's namespace that read their trace +// from it. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) runsForTraceClaim(ctx context.Context, claim client.Object) []reconcile.Request { + runs := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + if err := r.List(ctx, runs, client.InNamespace(claim.GetNamespace()), + client.MatchingFields{dgdrRunTraceClaimIndex: claim.GetName()}); err != nil { + log.FromContext(ctx).Error(err, "Could not list the Runs that read a trace claim", "claim", claim.GetName()) + return nil + } + requests := make([]reconcile.Request, 0, len(runs.Items)) + for i := range runs.Items { + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&runs.Items[i])}) + } + return requests +} + +// dgdrRunJobFinished reports whether job reports a finished pod or a terminal or interim terminal +// condition, so its outcome no longer depends on the trace. The Job runs one pod, because its +// backoffLimit is 0. job must not be nil. +func dgdrRunJobFinished(job *batchv1.Job) bool { + uncounted := job.Status.UncountedTerminatedPods + if job.Status.Succeeded > 0 || job.Status.Failed > 0 || + (uncounted != nil && len(uncounted.Succeeded)+len(uncounted.Failed) > 0) { + return true + } + for _, conditionType := range []batchv1.JobConditionType{ + batchv1.JobComplete, batchv1.JobFailed, batchv1.JobSuccessCriteriaMet, batchv1.JobFailureTarget, + } { + if condition := dgdrRunJobCondition(job, conditionType); condition != nil && condition.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +// jobTraceUnavailable returns why job, which has not finished, can never read the Run's trace, or +// "" when it can or still may. +// +// A sweeper that cannot mount the trace file waits in CreateContainerConfigError until the pod is +// deleted, so that is reported whatever the claim does. The claim itself matters until the sweeper +// has started. Before its pod is scheduled, the pod cannot get a claim that is unusable. A scheduled +// pod still cannot mount a claim that is gone, lost or a block volume, or one being deleted without +// pvc-protection, as when the claim goes while the pod is bound; pvc-protection keeps a claim being +// deleted for the pods bound to it, and the kubelet mounts it. While the cache shows fewer pods than +// the Job counts as active, whether one is scheduled or started is unknown and nothing is reported; +// the pod's event wakes the Run again. The pod cache can lag a binding by a moment, and a claim +// deleted in that moment fails the Run although its pod just got the claim. run and job must not +// be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) jobTraceUnavailable( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + job *batchv1.Job, +) (string, error) { + pods, err := r.jobPods(ctx, job) + if err != nil { + return "", err + } + + // A trace file the sweeper cannot mount, whether or not the claim is usable. + source := dgdrRunTraceClaim(run) + for i := range pods { + if message := dgdrRunTraceMountFailure(&pods[i]); message != "" { + return fmt.Sprintf("trace file %q on PersistentVolumeClaim %q cannot be mounted: %s", + source.Path, source.ClaimName, message), nil + } + } + + // The claim, until the sweeper has started. + if ptr.Deref(job.Status.Ready, 0) > 0 || int(job.Status.Active) > len(pods) { + return "", nil + } + scheduled := false + for i := range pods { + if dgdrRunSweeperStarted(&pods[i]) { + return "", nil + } + scheduled = scheduled || pods[i].Spec.NodeName != "" + } + return r.traceSourceUnavailable(ctx, run, scheduled) +} + +// jobPods returns the pods job controls, as the cache shows them. job must not be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) jobPods(ctx context.Context, job *batchv1.Job) ([]corev1.Pod, error) { + list := &corev1.PodList{} + if err := r.List(ctx, list, client.InNamespace(job.Namespace), client.MatchingLabels{batchv1.JobNameLabel: job.Name}); err != nil { + return nil, fmt.Errorf("list pods of Job %q: %w", job.Name, err) + } + pods := make([]corev1.Pod, 0, len(list.Items)) + for i := range list.Items { + if metav1.IsControlledBy(&list.Items[i], job) { + pods = append(pods, list.Items[i]) + } + } + return pods, nil +} + +// dgdrRunSweeperStarted reports whether pod's sweeper container has been created and started, so it +// mounted the trace. The pod cache drops a running state, so a reported sweeper that is not waiting +// has started. pod must not be nil. +func dgdrRunSweeperStarted(pod *corev1.Pod) bool { + for _, status := range pod.Status.ContainerStatuses { + if status.Name == dgdrRunSweeperContainer && + (status.State.Waiting == nil || status.LastTerminationState.Terminated != nil) { + return true + } + } + return false +} + +// dgdrRunTraceMountFailure returns the kubelet's message when pod's sweeper cannot start because +// the trace file cannot be mounted, such as a missing file on a read-only claim or a symlink that +// leaves the claim, or "" otherwise. pod must not be nil. +func dgdrRunTraceMountFailure(pod *corev1.Pod) string { + for _, status := range pod.Status.ContainerStatuses { + waiting := status.State.Waiting + if status.Name == dgdrRunSweeperContainer && waiting != nil && waiting.Reason == "CreateContainerConfigError" && + strings.Contains(waiting.Message, fmt.Sprintf("volumeMount %q", dgdrRunTraceVolume)) { + return waiting.Message + } + } + return "" +} + +// deleteUnfinishedJobs deletes the Jobs run controls that have not finished, with their pods. run +// failed because those Jobs could never read its trace. The UID precondition keeps a Job created +// under the same name since the cache showed it. run must not be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) deleteUnfinishedJobs(ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun) error { + jobs, err := r.ownedJobs(ctx, run) + if err != nil { + return err + } + for i := range jobs { + job := &jobs[i] + if dgdrRunJobFinished(job) { + continue + } + err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationBackground), client.Preconditions{UID: &job.UID}) + if client.IgnoreNotFound(err) != nil { + return fmt.Errorf("delete Job %q: %w", job.Name, err) + } + } + return nil +} + +// dgdrRunJobPod reports whether obj is a pod of a Job named like a Run's AISimulate Job. +func dgdrRunJobPod(obj client.Object) bool { + owner := metav1.GetControllerOf(obj) + return owner != nil && owner.Kind == "Job" && owner.APIVersion == batchv1.SchemeGroupVersion.String() && + strings.HasSuffix(owner.Name, "-"+dgdrRunResourceComponent) +} + +// runForJobPod maps a pod event to the Run that controls the pod's Job. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) runForJobPod(ctx context.Context, pod client.Object) []reconcile.Request { + owner := metav1.GetControllerOf(pod) + if owner == nil { + return nil + } + job := &batchv1.Job{} + if err := r.Get(ctx, types.NamespacedName{Namespace: pod.GetNamespace(), Name: owner.Name}, job); err != nil { + if !apierrors.IsNotFound(err) { + log.FromContext(ctx).Error(err, "Could not get the Job of a pod", "pod", pod.GetName(), "job", owner.Name) + } + return nil + } + run := metav1.GetControllerOf(job) + if job.UID != owner.UID || run == nil || + run.Kind != "DynamoGraphDeploymentRun" || run.APIVersion != nvidiacomv1beta2.GroupVersion.String() { + return nil + } + return []reconcile.Request{{Namespace: job.Namespace, Name: run.Name}} +} + func (r *DynamoGraphDeploymentRunV1beta2Reconciler) uncachedOwnedJobs( ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, @@ -658,6 +978,24 @@ func dgdrRunInput( return nil, fmt.Errorf("set operator-owned dgdr_run.%s: %w", key, err) } } + + // The typed trace owns traffic.source.type, format and paths the same way. Restoring them keeps a + // Run created without Request admission reading only the file on its read-only mount. Such a Run + // can also replace traffic or traffic.source with a non-object; that parent is rebuilt rather than + // failing every reconcile. + if run.Spec.Workload != nil && run.Spec.Workload.Trace != nil { + traffic, ok := input["traffic"].(map[string]any) + if !ok { + traffic = map[string]any{} + input["traffic"] = traffic + } + source, ok := traffic["source"].(map[string]any) + if !ok { + source = map[string]any{} + traffic["source"] = source + } + maps.Copy(source, aisimulate.TraceSource(run.Spec.Workload.Trace)) + } return input, nil } @@ -794,7 +1132,7 @@ func validateDGDRRunJobExecution( if len(pod.InitContainers) != 0 { return errors.New("init containers are not allowed") } - if err := validateDGDRRunPodSecurityContext(pod.SecurityContext); err != nil { + if err := validateDGDRRunPodSecurityContext(pod.SecurityContext, run); err != nil { return err } if pod.ServiceAccountName != inputConfigMapName || pod.AutomountServiceAccountToken == nil || *pod.AutomountServiceAccountToken { @@ -809,7 +1147,7 @@ func validateDGDRRunJobExecution( if !found { return errors.New("planner image identities differ") } - if err := validateDGDRRunPlannerContainer(&planner, execution); err != nil { + if err := validateDGDRRunPlannerContainer(&planner, run, execution); err != nil { return err } publisher, found := dgdrRunContainer(pod.Containers, dgdrRunPublisherContainer) @@ -819,7 +1157,7 @@ func validateDGDRRunJobExecution( if err := validateDGDRRunPublisherContainer(&publisher, run, execution); err != nil { return err } - if err := validateDGDRRunVolumes(pod.Volumes, inputConfigMapName); err != nil { + if err := validateDGDRRunVolumes(pod.Volumes, run, inputConfigMapName); err != nil { return err } if !apiequality.Semantic.DeepEqual(pod.ImagePullSecrets, execution.ImagePullSecrets) { @@ -828,7 +1166,11 @@ func validateDGDRRunJobExecution( return nil } -func validateDGDRRunPlannerContainer(planner *corev1.Container, execution dgdrRunExecution) error { +func validateDGDRRunPlannerContainer( + planner *corev1.Container, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + execution dgdrRunExecution, +) error { if planner.Image != execution.PlannerImage || planner.ImagePullPolicy != execution.PlannerPullPolicy { return errors.New("planner image identities differ") } @@ -845,7 +1187,7 @@ func validateDGDRRunPlannerContainer(planner *corev1.Container, execution dgdrRu if planner.Lifecycle != nil { return errors.New("planner lifecycle differs") } - if !apiequality.Semantic.DeepEqual(planner.VolumeMounts, dgdrRunSweeperVolumeMounts(execution.SnapshotPath)) { + if !apiequality.Semantic.DeepEqual(planner.VolumeMounts, dgdrRunSweeperVolumeMounts(run, execution.SnapshotPath)) { return errors.New("planner volume mounts differ") } return nil @@ -905,10 +1247,13 @@ func validateDGDRRunJobSpec(spec *batchv1.JobSpec) error { return nil } -func validateDGDRRunPodSecurityContext(security *corev1.PodSecurityContext) error { - if security == nil || security.FSGroup == nil || *security.FSGroup != 65532 { +func validateDGDRRunPodSecurityContext(security *corev1.PodSecurityContext, run *nvidiacomv1beta2.DynamoGraphDeploymentRun) error { + if security == nil || security.FSGroup != nil { return errors.New("pod fsGroup differs") } + if !apiequality.Semantic.DeepEqual(security.SupplementalGroups, dgdrRunPodSecurityContext(run).SupplementalGroups) { + return errors.New("pod supplemental groups differ") + } if security.RunAsNonRoot == nil || !*security.RunAsNonRoot { return errors.New("pod runAsNonRoot differs") } @@ -967,8 +1312,12 @@ func validateDGDRRunPublisherEnv(env []corev1.EnvVar) error { return nil } -func validateDGDRRunVolumes(volumes []corev1.Volume, inputConfigMapName string) error { - if len(volumes) != 3 { +func validateDGDRRunVolumes( + volumes []corev1.Volume, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + inputConfigMapName string, +) error { + if len(volumes) != len(dgdrRunVolumes(run, inputConfigMapName)) { return errors.New("input, snapshot or publisher credential volumes differ") } input, found := dgdrRunVolume(volumes, dgdrRunInputVolume) @@ -986,6 +1335,14 @@ func validateDGDRRunVolumes(volumes []corev1.Volume, inputConfigMapName string) !apiequality.Semantic.DeepEqual(publisher.Projected.Sources, dgdrRunPublisherKubeAPIVolume().Projected.Sources) { return errors.New("input, snapshot or publisher credential volumes differ") } + + // A Run that replays a trace mounts exactly its claim, read-only. + if source := dgdrRunTraceClaim(run); source != nil { + trace, found := dgdrRunVolume(volumes, dgdrRunTraceVolume) + if !found || !apiequality.Semantic.DeepEqual(trace, dgdrRunTraceClaimVolume(source)) { + return errors.New("trace volume differs") + } + } return nil } @@ -1056,14 +1413,8 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( ServiceAccountName: name, AutomountServiceAccountToken: new(false), RestartPolicy: corev1.RestartPolicyNever, - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To[int64](65532), - RunAsNonRoot: new(true), - SeccompProfile: &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - }, - }, - ImagePullSecrets: append([]corev1.LocalObjectReference(nil), execution.ImagePullSecrets...), + SecurityContext: dgdrRunPodSecurityContext(run), + ImagePullSecrets: append([]corev1.LocalObjectReference(nil), execution.ImagePullSecrets...), Containers: []corev1.Container{ { Name: dgdrRunSweeperContainer, @@ -1072,7 +1423,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( Command: dgdrRunSweeperCommand(), Args: dgdrRunSweeperArgs(execution.SnapshotPath), SecurityContext: sweeperSecurityContext, - VolumeMounts: dgdrRunSweeperVolumeMounts(execution.SnapshotPath), + VolumeMounts: dgdrRunSweeperVolumeMounts(run, execution.SnapshotPath), }, { Name: dgdrRunPublisherContainer, @@ -1085,7 +1436,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) buildJob( VolumeMounts: dgdrRunPublisherVolumeMounts(execution.SnapshotPath), }, }, - Volumes: dgdrRunVolumes(name), + Volumes: dgdrRunVolumes(run, name), }, }, }, @@ -1127,11 +1478,37 @@ func dgdrRunPublisherEnv() []corev1.EnvVar { } } -func dgdrRunSweeperVolumeMounts(snapshotPath string) []corev1.VolumeMount { - return []corev1.VolumeMount{ +// dgdrRunPodSecurityContext returns the pod security context of run's Job. The pod sets no +// fsGroup, for which kubelet and CSI drivers may chown and chmod a volume recursively; a Run that +// replays a trace gets the trace group as a supplemental group instead, which changes no volume. +// run must not be nil. +func dgdrRunPodSecurityContext(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) *corev1.PodSecurityContext { + security := &corev1.PodSecurityContext{ + RunAsNonRoot: new(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + } + if dgdrRunTraceClaim(run) != nil { + security.SupplementalGroups = []int64{dgdrRunTraceGroup} + } + return security +} + +// dgdrRunSweeperVolumeMounts returns the sweeper's mounts. Only AISimulate reads the trace: it +// mounts the one trace file read-only through a subPath, which the kubelet resolves inside the +// claim, refusing a symlink that resolves outside it. run must not be nil. +func dgdrRunSweeperVolumeMounts(run *nvidiacomv1beta2.DynamoGraphDeploymentRun, snapshotPath string) []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ {Name: dgdrRunInputVolume, MountPath: dgdrRunInputMountPath, ReadOnly: true}, {Name: dgdrRunSnapshotVolume, MountPath: snapshotPath}, } + if source := dgdrRunTraceClaim(run); source != nil { + mounts = append(mounts, corev1.VolumeMount{ + Name: dgdrRunTraceVolume, MountPath: aisimulate.TracePath(source.Path), SubPath: source.Path, ReadOnly: true, + }) + } + return mounts } func dgdrRunPublisherVolumeMounts(snapshotPath string) []corev1.VolumeMount { @@ -1141,12 +1518,30 @@ func dgdrRunPublisherVolumeMounts(snapshotPath string) []corev1.VolumeMount { } } -func dgdrRunVolumes(inputConfigMapName string) []corev1.Volume { - return []corev1.Volume{ +// dgdrRunVolumes returns the volumes of run's Job, with the trace claim mounted read-only when run +// replays a trace. run must not be nil. +func dgdrRunVolumes(run *nvidiacomv1beta2.DynamoGraphDeploymentRun, inputConfigMapName string) []corev1.Volume { + volumes := []corev1.Volume{ {Name: dgdrRunInputVolume, ConfigMap: &corev1.ConfigMapVolumeSource{Name: inputConfigMapName}}, {Name: dgdrRunSnapshotVolume, EmptyDir: &corev1.EmptyDirVolumeSource{}}, dgdrRunPublisherKubeAPIVolume(), } + if source := dgdrRunTraceClaim(run); source != nil { + volumes = append(volumes, dgdrRunTraceClaimVolume(source)) + } + return volumes +} + +// dgdrRunTraceClaimVolume returns the read-only volume of the claim source names. source must not +// be nil. +func dgdrRunTraceClaimVolume(source *nvidiacomv1beta2.TracePVCSource) corev1.Volume { + return corev1.Volume{ + Name: dgdrRunTraceVolume, + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: source.ClaimName, + ReadOnly: true, + }, + } } func dgdrRunContainerSecurityContext() *corev1.SecurityContext { @@ -1220,10 +1615,13 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRunStatus( Message: message, ObservedGeneration: run.Generation, }) + + // A stale Run must not overwrite a newer status, such as a trace failure recorded while the Job + // was still pending, so the write carries the resourceVersion the cache showed. if apiequality.Semantic.DeepEqual(original.Status, run.Status) { return nil } - if err := r.Status().Patch(ctx, run, client.MergeFrom(original)); err != nil { + if err := r.Status().Patch(ctx, run, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return &dgdrRunRetryableError{cause: fmt.Errorf("update Run %q status: %w", run.Name, err)} } return nil @@ -1355,6 +1753,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) SetupWithManager(mgr ctrl.Ma ); err != nil { return fmt.Errorf("index Jobs by Run controller UID: %w", err) } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), &nvidiacomv1beta2.DynamoGraphDeploymentRun{}, dgdrRunTraceClaimIndex, dgdrRunTraceClaimNames, + ); err != nil { + return fmt.Errorf("index Runs by trace claim: %w", err) + } return ctrl.NewControllerManagedBy(mgr). Named("dynamographdeploymentrun-v1beta2"). For(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}). @@ -1363,6 +1766,9 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) SetupWithManager(mgr ctrl.Ma Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Watches(&corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(r.runsForTraceClaim)). + Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(r.runForJobPod), + builder.WithPredicates(predicate.NewPredicateFuncs(dgdrRunJobPod))). WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.Config, r.RuntimeConfig)). Complete(r) } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index f97d199f9fe6..70a99935988a 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "reflect" "strings" "testing" @@ -95,6 +96,7 @@ func newDGDRRunFixtureWithInterceptor( WithScheme(scheme). WithObjects(objects...). WithIndex(&batchv1.Job{}, dgdrRunJobControllerUIDIndex, dgdrRunJobControllerUIDs). + WithIndex(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, dgdrRunTraceClaimIndex, dgdrRunTraceClaimNames). WithStatusSubresource(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, &batchv1.Job{}, &corev1.Pod{}). WithInterceptorFuncs(funcs). Build() @@ -856,6 +858,20 @@ func TestValidateDGDRRunJobExecutionRejectsImmutableBindingSkew(t *testing.T) { }, wantErr: "pod seccomp profile differs", }, + { + name: "pod fsGroup", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.SecurityContext.FSGroup = ptr.To[int64](65532) + }, + wantErr: "pod fsGroup differs", + }, + { + name: "pod supplemental group", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.SecurityContext.SupplementalGroups = []int64{65532} + }, + wantErr: "pod supplemental groups differ", + }, { name: "planner privilege escalation", mutate: func(job *batchv1.Job) { @@ -1032,6 +1048,97 @@ func TestValidateDGDRRunJobExecutionRejectsImmutableBindingSkew(t *testing.T) { }) } +func TestValidateDGDRRunJobExecutionBindsTheTraceMount(t *testing.T) { + run := dgdrRunWithTraceForTest(t, map[string]any{}) + f := newDGDRRunFixture(t, run, traceClaimForTest()) + f.reconcile() + job := f.job() + configMap := &corev1.ConfigMap{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get execution ConfigMap: %v", err) + } + execution, err := dgdrRunExecutionFromConfigMap(configMap) + if err != nil { + t.Fatalf("decode execution: %v", err) + } + + t.Log("The Job built for a trace Run matches its frozen execution") + if err := validateDGDRRunJobExecution(job, run, configMap.Name, execution); err != nil { + t.Fatalf("validation error = %v, want the built trace Job accepted", err) + } + + t.Log("Any change to the trace claim, its mount or its group is skew") + tests := []struct { + name string + mutate func(*batchv1.Job) + wantErr string + }{ + { + name: "missing trace group", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.SecurityContext.SupplementalGroups = nil + }, + wantErr: "pod supplemental groups differ", + }, + { + name: "fsGroup in place of the trace group", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.SecurityContext.FSGroup = ptr.To[int64](65532) + }, + wantErr: "pod fsGroup differs", + }, + { + name: "writable trace claim", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Volumes[3].PersistentVolumeClaim.ReadOnly = false + }, + wantErr: "trace volume differs", + }, + { + name: "other trace claim", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Volumes[3].PersistentVolumeClaim.ClaimName = "other-traces" + }, + wantErr: "trace volume differs", + }, + { + name: "missing trace claim", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Volumes = job.Spec.Template.Spec.Volumes[:3] + }, + wantErr: "input, snapshot or publisher credential volumes differ", + }, + { + name: "whole claim mounted", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].VolumeMounts[2].SubPath = "" + }, + wantErr: "planner volume mounts differ", + }, + { + name: "trace exposed to the publisher", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[1].VolumeMounts = append( + job.Spec.Template.Spec.Containers[1].VolumeMounts, + job.Spec.Template.Spec.Containers[0].VolumeMounts[2], + ) + }, + wantErr: "publisher volume mounts differ", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mutated := job.DeepCopy() + tt.mutate(mutated) + err := validateDGDRRunJobExecution(mutated, run, configMap.Name, execution) + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("validation error = %v, want %q", err, tt.wantErr) + } + }) + } +} + func TestV1beta2RunControllerStopsAllJobsBeforePermanentCompletion(t *testing.T) { run := dgdrRunForTest(t, map[string]any{}) markerObservedByReconcile := false @@ -1254,6 +1361,9 @@ func assertDGDRRunJobIsolation( !hasMountAt(publisher.VolumeMounts, dgdrRunSnapshotVolume, dgdrRunSnapshotPath) { t.Fatal("sweeper and publisher do not share the frozen snapshot directory") } + if hasMount(publisher.VolumeMounts, dgdrRunTraceVolume) { + t.Fatal("publisher unexpectedly mounts the trace claim") + } } func assertDGDRRunPublisherIdentity( @@ -1460,6 +1570,17 @@ func assertDGDRRunPendingStatus(t *testing.T, stored *nvidiacomv1beta2.DynamoGra } } +func assertDGDRRunRunningStatus(t *testing.T, stored *nvidiacomv1beta2.DynamoGraphDeploymentRun, jobName string) { + t.Helper() + if stored.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseRunning || stored.Status.JobRef == nil || stored.Status.JobRef.Name != jobName { + t.Fatalf("Run status = %+v, want Running and JobRef %q", stored.Status, jobName) + } + condition := meta.FindStatusCondition(stored.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != dgdrRunReasonRunning { + t.Fatalf("Completed condition = %+v, want False/%s", condition, dgdrRunReasonRunning) + } +} + func TestV1beta2RunControllerOwnsOnlyLifecycleStatus(t *testing.T) { f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) f.reconcile() @@ -1675,3 +1796,981 @@ func hasMountAt(mounts []corev1.VolumeMount, name, path string) bool { } return false } + +func dgdrRunWithTraceForTest(t *testing.T, inputOverrides map[string]any) *nvidiacomv1beta2.DynamoGraphDeploymentRun { + t.Helper() + run := dgdrRunForTest(t, inputOverrides) + run.Spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ + Format: nvidiacomv1beta2.TraceFormatDynamo, + Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz", + }}, + }} + return run +} + +func traceClaimForTest() *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + Name: "request-traces", Namespace: "default", + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + VolumeMode: ptr.To(corev1.PersistentVolumeFilesystem), + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } +} + +func TestV1beta2RunControllerMountsTheTraceFileReadOnlyIntoTheSweeperOnly(t *testing.T) { + run := dgdrRunWithTraceForTest(t, map[string]any{}) + f := newDGDRRunFixture(t, run, traceClaimForTest()) + f.reconcile() + + t.Log("The pod mounts the claim read-only as the trace volume") + job := f.job() + var trace *corev1.Volume + for i := range job.Spec.Template.Spec.Volumes { + if job.Spec.Template.Spec.Volumes[i].Name == dgdrRunTraceVolume { + trace = &job.Spec.Template.Spec.Volumes[i] + } + } + if trace == nil || trace.PersistentVolumeClaim == nil || + trace.PersistentVolumeClaim.ClaimName != "request-traces" || !trace.PersistentVolumeClaim.ReadOnly { + t.Fatalf("trace volume = %#v, want the read-only request-traces claim", trace) + } + + t.Log("Only the sweeper mounts the one trace file, read-only through a subPath at the fixed path") + sweeper := containerNamed(t, job.Spec.Template.Spec.Containers, dgdrRunSweeperContainer) + want := corev1.VolumeMount{ + Name: dgdrRunTraceVolume, MountPath: "/var/run/aisimulate/trace/trace.jsonl.gz", + SubPath: "retune/w-1.jsonl.gz", ReadOnly: true, + } + var traceMounts []corev1.VolumeMount + for _, mount := range sweeper.VolumeMounts { + if mount.Name == dgdrRunTraceVolume { + traceMounts = append(traceMounts, mount) + } + } + if !reflect.DeepEqual(traceMounts, []corev1.VolumeMount{want}) { + t.Fatalf("sweeper trace mounts = %#v, want exactly %#v", traceMounts, want) + } + assertDGDRRunJobIsolation(t, job, run, f.reconciler.PlannerImage, f.reconciler.OperatorImage) + + t.Log("The pod grants the trace group as a supplemental group and requests no fsGroup") + security := job.Spec.Template.Spec.SecurityContext + if security == nil || security.FSGroup != nil || !reflect.DeepEqual(security.SupplementalGroups, []int64{65532}) { + t.Fatalf("pod securityContext = %#v, want supplemental group 65532 and no fsGroup", security) + } + + t.Log("The frozen input reads the mounted trace file") + configMap := &corev1.ConfigMap{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(job), configMap); err != nil { + t.Fatalf("get ConfigMap: %v", err) + } + var input map[string]any + if err := yaml.Unmarshal([]byte(configMap.Data[dgdrRunInputKey]), &input); err != nil { + t.Fatalf("decode input: %v", err) + } + source := input["traffic"].(map[string]any)["source"].(map[string]any) + if !reflect.DeepEqual(source["paths"], []any{"/var/run/aisimulate/trace/trace.jsonl.gz"}) { + t.Fatalf("traffic.source.paths = %v", source["paths"]) + } +} + +func TestV1beta2RunControllerRequestsNoPodGroupsWithoutATrace(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunForTest(t, map[string]any{})) + f.reconcile() + + t.Log("A Run without a trace requests neither an fsGroup nor a supplemental group") + security := f.job().Spec.Template.Spec.SecurityContext + if security == nil || security.FSGroup != nil || len(security.SupplementalGroups) != 0 { + t.Fatalf("pod securityContext = %#v, want no fsGroup and no supplemental groups", security) + } +} + +func TestDGDRRunInputEnforcesTheTypedTraceSource(t *testing.T) { + t.Log("Lower a Run whose overrides try to replace the trace source and change the load") + run := dgdrRunWithTraceForTest(t, map[string]any{"traffic": map[string]any{ + "source": map[string]any{"type": "synthetic", "format": "mooncake", "paths": []any{"/etc/passwd"}}, + "load": map[string]any{"speedup": 4}, + }}) + input, err := dgdrRunInput(run, "example.com/dynamo-planner:1.2.3", "1.2.3") + if err != nil { + t.Fatalf("lower input: %v", err) + } + + t.Log("The typed trace wins for the source; the load override is kept") + traffic := input["traffic"].(map[string]any) + wantSource := map[string]any{"type": "trace", "format": "dynamo", "paths": []any{"/var/run/aisimulate/trace/trace.jsonl.gz"}} + if !reflect.DeepEqual(traffic["source"], wantSource) { + t.Fatalf("traffic.source = %#v, want %#v", traffic["source"], wantSource) + } + load := traffic["load"].(map[string]any) + if load["type"] != "trace_timestamps" || load["speedup"] != json.Number("4") { + t.Fatalf("traffic.load = %#v, want trace_timestamps with the override speedup", load) + } + if _, found := traffic["stop"]; found { + t.Fatalf("traffic.stop = %#v, want none for a trace", traffic["stop"]) + } +} + +func TestDGDRRunInputRebuildsANonObjectTraceSourceParent(t *testing.T) { + wantSource := map[string]any{"type": "trace", "format": "dynamo", "paths": []any{"/var/run/aisimulate/trace/trace.jsonl.gz"}} + tests := []struct { + name string + overrides map[string]any + wantLoad map[string]any + }{ + { + name: "scalar traffic", + overrides: map[string]any{"traffic": "invalid"}, + }, + { + name: "scalar traffic.source", + overrides: map[string]any{"traffic": map[string]any{"source": "invalid"}}, + wantLoad: map[string]any{"type": "trace_timestamps"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Lower a Run created without admission whose override replaces a trace source parent") + input, err := dgdrRunInput(dgdrRunWithTraceForTest(t, tt.overrides), "example.com/dynamo-planner:1.2.3", "1.2.3") + if err != nil { + t.Fatalf("lower input: %v", err) + } + + t.Log("The parent is an object again and carries the typed trace source") + traffic := input["traffic"].(map[string]any) + if !reflect.DeepEqual(traffic["source"], wantSource) { + t.Fatalf("traffic.source = %#v, want %#v", traffic["source"], wantSource) + } + if load, _ := traffic["load"].(map[string]any); !reflect.DeepEqual(load, tt.wantLoad) { + t.Fatalf("traffic.load = %#v, want %#v", traffic["load"], tt.wantLoad) + } + }) + } +} + +func TestV1beta2RunControllerFailsARunWhoseTraceClaimCannotBeMounted(t *testing.T) { + tests := []struct { + name string + missing bool + mutate func(*corev1.PersistentVolumeClaim) + reason string + }{ + { + name: "missing claim", + missing: true, + reason: `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`, + }, + { + name: "claim being deleted", + mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.DeletionTimestamp = new(metav1.Now()) + claim.Finalizers = []string{"kubernetes.io/pvc-protection"} + }, + reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is being deleted`, + }, + { + name: "raw block claim", + mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.VolumeMode = ptr.To(corev1.PersistentVolumeBlock) + }, + reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is a raw block volume`, + }, + { + name: "lost claim", + mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Status.Phase = corev1.ClaimLost + }, + reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" has lost its PersistentVolume`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := []client.Object{dgdrRunWithTraceForTest(t, map[string]any{})} + if !tt.missing { + claim := traceClaimForTest() + tt.mutate(claim) + objects = append(objects, claim) + } + f := newDGDRRunFixture(t, objects...) + + t.Log("Reconcile twice: the first pass fails the Run, the second sees it completed") + f.reconcile() + f.reconcile() + + t.Log("The Run is Failed and Completed with the claim named in the message") + run := f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed || condition == nil || + condition.Status != metav1.ConditionTrue || condition.Reason != dgdrRunReasonTraceSourceUnavailable { + t.Fatalf("status = %+v, want Failed and Completed=True/%s", run.Status, dgdrRunReasonTraceSourceUnavailable) + } + if !strings.HasPrefix(condition.Message, tt.reason) || run.Status.CompletionTime == nil || run.Status.JobRef != nil { + t.Fatalf("condition message = %q, completionTime = %v, jobRef = %v; want message prefix %q", + condition.Message, run.Status.CompletionTime, run.Status.JobRef, tt.reason) + } + if condition.ObservedGeneration != run.Generation { + t.Fatalf("condition observedGeneration = %d, want %d", condition.ObservedGeneration, run.Generation) + } + + t.Log("Nothing was created for the Run") + assertNoDGDRRunResources(t, f.client) + }) + } +} + +func assertNoDGDRRunResources(t *testing.T, c client.Client) { + t.Helper() + for _, list := range []client.ObjectList{ + &batchv1.JobList{}, &corev1.ConfigMapList{}, &corev1.ServiceAccountList{}, + &rbacv1.RoleList{}, &rbacv1.RoleBindingList{}, + } { + if err := c.List(t.Context(), list, client.InNamespace("default")); err != nil { + t.Fatalf("list %T: %v", list, err) + } + if n := meta.LenList(list); n != 0 { + t.Fatalf("%T has %d items, want none", list, n) + } + } +} + +func TestV1beta2RunControllerAcceptsAnUnboundOrSharedTraceClaim(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + }{ + {name: "WaitForFirstConsumer claim still Pending", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Status.Phase = corev1.ClaimPending + }}, + {name: "ReadWriteOnce claim", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }}, + {name: "ReadWriteOncePod claim", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOncePod} + }}, + {name: "claim without a volume mode", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.VolumeMode = nil + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + claim := traceClaimForTest() + tt.mutate(claim) + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + f.reconcile() + assertDGDRRunPendingStatus(t, f.run(), f.job().Name) + }) + } +} + +func TestV1beta2RunControllerRetriesATransientTraceClaimReadError(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + f.reconciler.APIReader = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.PersistentVolumeClaim); ok { + return errors.New("API server unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + + t.Log("A read error other than NotFound is returned for a retry") + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err == nil || !strings.Contains(err.Error(), "API server unavailable") { + t.Fatalf("reconcile error = %v, want the claim read error", err) + } + + t.Log("The Run stays open, and nothing was created for it") + if run := f.run(); run.Status.Phase != "" || len(run.Status.Conditions) != 0 { + t.Fatalf("status = %+v, want untouched", run.Status) + } + assertNoDGDRRunResources(t, f.client) +} + +func TestV1beta2RunControllerKeepsFollowingAStartedJobAfterItsTraceClaimDisappears(t *testing.T) { + tests := []struct { + name string + status batchv1.JobStatus + podNode string + podPhase corev1.PodPhase + wantPhase nvidiacomv1beta2.DGDRRunPhase + }{ + { + name: "pod scheduled while its images are pulled", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, + podNode: "node-a", + podPhase: corev1.PodPending, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "a container exited after the pod was ready", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, + podNode: "node-a", + podPhase: corev1.PodRunning, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "pod ready while the pod cache still shows it unscheduled", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, + podPhase: corev1.PodPending, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "Job tracks a succeeded pod it has not counted", + status: batchv1.JobStatus{UncountedTerminatedPods: &batchv1.UncountedTerminatedPods{Succeeded: []types.UID{"pod"}}}, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "Job tracks a failed pod it has not counted", + status: batchv1.JobStatus{UncountedTerminatedPods: &batchv1.UncountedTerminatedPods{Failed: []types.UID{"pod"}}}, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "Job met its success criteria", + status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobSuccessCriteriaMet, Status: corev1.ConditionTrue}}}, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "Job reached its failure target", + status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailureTarget, Status: corev1.ConditionTrue}}}, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "Job counted a succeeded pod", + status: batchv1.JobStatus{Succeeded: 1}, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "Job counted a failed pod", + status: batchv1.JobStatus{Failed: 1}, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + }, + { + name: "Job complete", + status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseSucceeded, + }, + { + name: "Job failed", + status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseFailed, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + claim := traceClaimForTest() + claim.Finalizers = []string{"kubernetes.io/pvc-protection"} + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + f.reconcile() + + t.Log("The Job, and its pod where the row has one, report the pod past the claim") + job := f.job() + job.Status = tt.status + job.Status.StartTime = new(metav1.Now()) + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("update Job status: %v", err) + } + if tt.podPhase != "" { + createDGDRRunJobPodForTest(t, f.client, job, tt.podNode, corev1.PodStatus{Phase: tt.podPhase}) + } + + t.Log("Delete the claim") + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) + } + + t.Log("Hide the Job from the cached List, as a lagging informer would") + cached := f.reconciler.Client + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*batchv1.JobList); ok { + return nil + } + return c.List(ctx, list, opts...) + }, + }) + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if !apierrors.IsAlreadyExists(err) { + t.Fatalf("reconcile error = %v, want the Job create to hit AlreadyExists until the cache catches up", err) + } + + t.Log("Once the cache shows the Job again, the Job, not the claim, decides the Run") + f.reconciler.Client = cached + f.reconcile() + if kept := f.job(); kept.UID != job.UID { + t.Fatalf("Job UID = %q, want the started Job %q kept", kept.UID, job.UID) + } + run := f.run() + if run.Status.Phase != tt.wantPhase { + t.Fatalf("phase = %q, want %q", run.Status.Phase, tt.wantPhase) + } + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if condition == nil || condition.Reason == dgdrRunReasonTraceSourceUnavailable { + t.Fatalf("Completed condition = %+v, want the Job, not the claim, to decide it", condition) + } + }) + } +} + +// createDGDRRunJobPodForTest stores the pod the Job controller would create for job, bound to node +// unless node is empty, with status. +func createDGDRRunJobPodForTest(t *testing.T, c client.Client, job *batchv1.Job, node string, status corev1.PodStatus) { + t.Helper() + pod := &corev1.Pod{ + Name: job.Name + "-pod", Namespace: job.Namespace, + Labels: map[string]string{batchv1.JobNameLabel: job.Name}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(job, batchv1.SchemeGroupVersion.WithKind("Job"))}, + Spec: corev1.PodSpec{NodeName: node}, + } + if err := c.Create(t.Context(), pod); err != nil { + t.Fatalf("create Job pod: %v", err) + } + pod.Status = status + if err := c.Status().Update(t.Context(), pod); err != nil { + t.Fatalf("update Job pod status: %v", err) + } +} + +// assertDGDRRunFailedForTrace checks that the Run failed for its trace with a message that starts +// with reason and names the Job, and that the completed Run deleted that Job. +func assertDGDRRunFailedForTrace(t *testing.T, c client.Client, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, jobName, reason string) { + t.Helper() + jobs := &batchv1.JobList{} + if err := c.List(t.Context(), jobs, client.InNamespace("default")); err != nil { + t.Fatalf("list Jobs: %v", err) + } + if len(jobs.Items) != 0 { + t.Fatalf("Jobs = %d, want the Job that could never read the trace deleted", len(jobs.Items)) + } + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed || condition == nil || + condition.Status != metav1.ConditionTrue || condition.Reason != dgdrRunReasonTraceSourceUnavailable { + t.Fatalf("status = %+v, want Failed and Completed=True/%s", run.Status, dgdrRunReasonTraceSourceUnavailable) + } + wantSuffix := fmt.Sprintf(`; Job %q is deleted`, jobName) + if !strings.HasPrefix(condition.Message, reason) || !strings.HasSuffix(condition.Message, wantSuffix) { + t.Fatalf("condition message = %q, want prefix %q and suffix %q", condition.Message, reason, wantSuffix) + } + if run.Status.CompletionTime == nil || run.Status.JobRef == nil || run.Status.JobRef.Name != jobName { + t.Fatalf("completionTime = %v, jobRef = %+v; want a completion time and the Job's jobRef", + run.Status.CompletionTime, run.Status.JobRef) + } +} + +func TestV1beta2RunControllerDeletesAnUnscheduledJobWhoseTraceClaimBecomesUnusable(t *testing.T) { + tests := []struct { + name string + finalizers []string + deleteClaim bool + mutate func(*corev1.PersistentVolumeClaim) + updateStatus bool + reason string + }{ + { + name: "claim deleted", + deleteClaim: true, + reason: `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`, + }, + { + name: "claim being deleted", + finalizers: []string{"kubernetes.io/pvc-protection"}, + deleteClaim: true, + reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is being deleted`, + }, + { + name: "claim replaced by a raw block claim", + mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.VolumeMode = ptr.To(corev1.PersistentVolumeBlock) + }, + reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is a raw block volume`, + }, + { + name: "claim lost its volume", + mutate: func(claim *corev1.PersistentVolumeClaim) { claim.Status.Phase = corev1.ClaimLost }, + updateStatus: true, + reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" has lost its PersistentVolume`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + claim := traceClaimForTest() + claim.Finalizers = tt.finalizers + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + f.reconcile() + job := f.job() + + t.Log("The Job's pod is created but not scheduled yet") + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("update Job status: %v", err) + } + createDGDRRunJobPodForTest(t, f.client, job, "", corev1.PodStatus{Phase: corev1.PodPending}) + f.reconcile() + assertDGDRRunRunningStatus(t, f.run(), job.Name) + + t.Log("The claim becomes unusable before the pod is scheduled") + if tt.deleteClaim { + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) + } + } + if tt.mutate != nil { + stored := &corev1.PersistentVolumeClaim{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(claim), stored); err != nil { + t.Fatalf("get claim: %v", err) + } + tt.mutate(stored) + var err error + if tt.updateStatus { + err = f.client.Status().Update(t.Context(), stored) + } else { + err = f.client.Update(t.Context(), stored) + } + if err != nil { + t.Fatalf("update claim: %v", err) + } + } + + t.Log("Reconcile twice: the first pass fails the Run, the second deletes the Job of the completed Run") + f.reconcile() + f.reconcile() + assertDGDRRunFailedForTrace(t, f.client, f.run(), job.Name, tt.reason) + }) + } +} + +func TestV1beta2RunControllerDeletesAJobWhoseSweeperCannotMountTheTraceFile(t *testing.T) { + const prefix = `trace file "retune/w-1.jsonl.gz" on PersistentVolumeClaim "request-traces" cannot be mounted: ` + tests := []struct { + name string + container string + reason string + message string + wantFailed bool + }{ + { + name: "a subPath that resolves outside the claim", + container: dgdrRunSweeperContainer, + reason: "CreateContainerConfigError", + message: `failed to prepare subPath for volumeMount "trace" of container "sweeper"`, + wantFailed: true, + }, + { + name: "a missing file on a read-only claim", + container: dgdrRunSweeperContainer, + reason: "CreateContainerConfigError", + message: `failed to create subPath directory for volumeMount "trace" of container "sweeper"`, + wantFailed: true, + }, + { + name: "another volume of the sweeper is left to the Job", + container: dgdrRunSweeperContainer, + reason: "CreateContainerConfigError", + message: `failed to prepare subPath for volumeMount "input" of container "sweeper"`, + }, + { + name: "the publisher is left to the Job", + container: dgdrRunPublisherContainer, + reason: "CreateContainerConfigError", + message: `failed to prepare subPath for volumeMount "trace" of container "publisher"`, + }, + { + name: "a sweeper image that cannot be pulled is left to the Job", + container: dgdrRunSweeperContainer, + reason: "ErrImagePull", + message: `failed to pull image`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + f.reconcile() + job := f.job() + + t.Log("The scheduled pod's container waits with the row's reason while the claim is usable") + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("update Job status: %v", err) + } + createDGDRRunJobPodForTest(t, f.client, job, "node-a", corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: tt.container, + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: tt.reason, Message: tt.message}}, + }}, + }) + + t.Log("Reconcile twice and expect only a trace mount failure to fail the Run and delete its Job") + f.reconcile() + f.reconcile() + if !tt.wantFailed { + if kept := f.job(); kept.UID != job.UID { + t.Fatalf("Job UID = %q, want the Job %q kept", kept.UID, job.UID) + } + assertDGDRRunRunningStatus(t, f.run(), job.Name) + return + } + assertDGDRRunFailedForTrace(t, f.client, f.run(), job.Name, prefix+tt.message) + }) + } +} + +func TestV1beta2RunControllerFailsAScheduledJobOnlyForAClaimItCanNeverMount(t *testing.T) { + tests := []struct { + name string + finalizers []string + deleteClaim bool + lose bool + sweeperStarted bool + reason string + }{ + { + name: "claim being deleted while pvc-protection keeps it for the pod", + finalizers: []string{"kubernetes.io/pvc-protection"}, + deleteClaim: true, + }, + { + name: "claim gone while the pod was bound", + deleteClaim: true, + reason: `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`, + }, + { + name: "claim being deleted without pvc-protection", + finalizers: []string{"example.com/hold"}, + deleteClaim: true, + reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is being deleted`, + }, + { + name: "claim lost its volume", + lose: true, + reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" has lost its PersistentVolume`, + }, + { + name: "sweeper started before the claim went", + deleteClaim: true, + sweeperStarted: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + claim := traceClaimForTest() + claim.Finalizers = tt.finalizers + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + f.reconcile() + job := f.job() + + t.Log("The Job's pod is bound to a node; its sweeper is created or still waits") + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("update Job status: %v", err) + } + sweeper := corev1.ContainerStatus{Name: dgdrRunSweeperContainer, State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{Reason: "ContainerCreating"}, + }} + if tt.sweeperStarted { + sweeper.State = corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} + } + createDGDRRunJobPodForTest(t, f.client, job, "node-a", corev1.PodStatus{ + Phase: corev1.PodPending, ContainerStatuses: []corev1.ContainerStatus{sweeper}, + }) + + t.Log("The claim changes after the binding") + if tt.deleteClaim { + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) + } + } + if tt.lose { + stored := &corev1.PersistentVolumeClaim{} + if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(claim), stored); err != nil { + t.Fatalf("get claim: %v", err) + } + stored.Status.Phase = corev1.ClaimLost + if err := f.client.Status().Update(t.Context(), stored); err != nil { + t.Fatalf("lose claim: %v", err) + } + } + + t.Log("Reconcile twice and expect only a claim the pod can never mount to end the Run") + f.reconcile() + f.reconcile() + if tt.reason == "" { + if kept := f.job(); kept.UID != job.UID { + t.Fatalf("Job UID = %q, want the Job %q kept", kept.UID, job.UID) + } + assertDGDRRunRunningStatus(t, f.run(), job.Name) + return + } + assertDGDRRunFailedForTrace(t, f.client, f.run(), job.Name, tt.reason) + }) + } +} + +func TestV1beta2RunControllerDoesNotLetAStaleRunUndoATraceFailure(t *testing.T) { + tests := []struct { + name string + // jobCondition, when set, finishes the Job after the trace failure was recorded. + jobCondition batchv1.JobConditionType + }{ + {name: "the trace is still unreadable"}, + {name: "the Job completes after the failure", jobCondition: batchv1.JobComplete}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + f.reconcile() + job := f.job() + + t.Log("The Job's pod waits unscheduled while the Run is Running") + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("update Job status: %v", err) + } + createDGDRRunJobPodForTest(t, f.client, job, "", corev1.PodStatus{Phase: corev1.PodPending}) + f.reconcile() + stale := f.run() + assertDGDRRunRunningStatus(t, stale, job.Name) + + t.Log("The claim goes and the trace failure is recorded") + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) + } + f.reconcile() + failed := f.run() + if condition := meta.FindStatusCondition(failed.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted); condition == nil || + condition.Reason != dgdrRunReasonTraceSourceUnavailable { + t.Fatalf("Completed condition = %+v, want %s", condition, dgdrRunReasonTraceSourceUnavailable) + } + if tt.jobCondition != "" { + stored := f.job() + stored.Status.Conditions = []batchv1.JobCondition{{Type: tt.jobCondition, Status: corev1.ConditionTrue}} + if err := f.client.Status().Update(t.Context(), stored); err != nil { + t.Fatalf("finish Job: %v", err) + } + } + + t.Log("A pass that still reads the Running Run conflicts instead of overwriting the failure") + cached := f.reconciler.Client + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if run, ok := obj.(*nvidiacomv1beta2.DynamoGraphDeploymentRun); ok { + stale.DeepCopyInto(run) + return nil + } + return c.Get(ctx, key, obj, opts...) + }, + }) + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(stale)}) + if !apierrors.IsConflict(err) { + t.Fatalf("reconcile error = %v, want a status conflict", err) + } + f.reconciler.Client = cached + if got := f.run(); !reflect.DeepEqual(got.Status, failed.Status) { + t.Fatalf("status = %+v, want the recorded failure %+v", got.Status, failed.Status) + } + + t.Log("The next pass deletes the Job only if it has not finished, and keeps the failure") + f.reconcile() + if tt.jobCondition == "" { + assertDGDRRunFailedForTrace(t, f.client, f.run(), job.Name, + `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`) + return + } + if kept := f.job(); kept.UID != job.UID { + t.Fatalf("Job UID = %q, want the finished Job %q kept", kept.UID, job.UID) + } + if got := f.run(); !reflect.DeepEqual(got.Status, failed.Status) { + t.Fatalf("status = %+v, want the recorded failure %+v", got.Status, failed.Status) + } + }) + } +} + +func TestV1beta2RunControllerWaitsUntilTheCacheShowsAnActiveJobsPod(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + f.reconcile() + + t.Log("The Job counts an active pod that the pod cache does not show yet") + job := f.job() + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("update Job status: %v", err) + } + + t.Log("The claim disappears; whether the pod is scheduled is unknown, so the Job is kept for now") + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) + } + f.reconcile() + if kept := f.job(); kept.UID != job.UID { + t.Fatalf("Job UID = %q, want the Job %q kept", kept.UID, job.UID) + } + assertDGDRRunRunningStatus(t, f.run(), job.Name) +} + +func TestV1beta2RunControllerKeepsTheJobUntilTheTraceFailureIsRecorded(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + f.reconcile() + f.reconcile() + jobName := f.job().Name + assertDGDRRunPendingStatus(t, f.run(), jobName) + + t.Log("Delete the claim, then fail the status patch that records the trace failure") + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) + } + cached := f.reconciler.Client + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + SubResourcePatch: func(context.Context, client.Client, string, client.Object, client.Patch, ...client.SubResourcePatchOption) error { + return errors.New("injected status patch failure") + }, + }) + if _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}); err == nil { + t.Fatal("reconcile error = nil, want the injected status patch failure") + } + + t.Log("The Job stays until the failure is recorded, so no pass can create another one") + if job := f.job(); job.Name != jobName { + t.Fatalf("Job = %q, want %q kept", job.Name, jobName) + } + assertDGDRRunPendingStatus(t, f.run(), jobName) + + t.Log("The next passes record the failure and then delete the Job") + f.reconciler.Client = cached + f.reconcile() + f.reconcile() + assertDGDRRunFailedForTrace(t, f.client, f.run(), jobName, + `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`) +} + +func TestV1beta2RunControllerWakesTheRunsThatReadAChangedTraceClaim(t *testing.T) { + reader := dgdrRunWithTraceForTest(t, map[string]any{}) + otherClaim := dgdrRunWithTraceForTest(t, map[string]any{}) + otherClaim.Name = "search-2" + otherClaim.Spec.Workload.Trace.Source.PVC.ClaimName = "other-traces" + otherNamespace := dgdrRunWithTraceForTest(t, map[string]any{}) + otherNamespace.Namespace = "elsewhere" + withoutTrace := dgdrRunForTest(t, map[string]any{}) + withoutTrace.Name = "search-3" + f := newDGDRRunFixture(t, reader, otherClaim, otherNamespace, withoutTrace) + + t.Log("An event on request-traces in default wakes only the Run that reads it") + got := f.reconciler.runsForTraceClaim(t.Context(), traceClaimForTest()) + want := []reconcile.Request{{NamespacedName: client.ObjectKeyFromObject(reader)}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("requests = %v, want %v", got, want) + } +} + +func TestV1beta2RunControllerWakesTheRunOfAJobPod(t *testing.T) { + run := dgdrRunWithTraceForTest(t, map[string]any{}) + f := newDGDRRunFixture(t, run, traceClaimForTest()) + f.reconcile() + job := f.job() + otherJob := &batchv1.Job{Name: "other-aisimulate", Namespace: "default", UID: "other-job"} + if err := f.client.Create(t.Context(), otherJob); err != nil { + t.Fatalf("create Job without a Run: %v", err) + } + tests := []struct { + name string + owner metav1.OwnerReference + wantWatch bool + want []reconcile.Request + }{ + { + name: "a pod of the Run's Job", + owner: *metav1.NewControllerRef(job, batchv1.SchemeGroupVersion.WithKind("Job")), + wantWatch: true, + want: []reconcile.Request{{NamespacedName: client.ObjectKeyFromObject(f.run())}}, + }, + { + name: "a pod of a Job no Run controls", + owner: *metav1.NewControllerRef(otherJob, batchv1.SchemeGroupVersion.WithKind("Job")), + wantWatch: true, + }, + { + name: "a pod of a Job that no longer exists", + owner: metav1.OwnerReference{APIVersion: "batch/v1", Kind: "Job", Name: "gone-aisimulate", UID: "gone", Controller: new(true)}, + wantWatch: true, + }, + { + name: "a pod of a Job with the same name but another UID", + owner: metav1.OwnerReference{APIVersion: "batch/v1", Kind: "Job", Name: job.Name, UID: "replaced", Controller: new(true)}, + wantWatch: true, + }, + { + name: "a pod of a Job named unlike a Run's Job", + owner: metav1.OwnerReference{APIVersion: "batch/v1", Kind: "Job", Name: "nightly", UID: "nightly", Controller: new(true)}, + }, + { + name: "a pod of a ReplicaSet", + owner: metav1.OwnerReference{APIVersion: "apps/v1", Kind: "ReplicaSet", Name: "web-aisimulate", UID: "rs", Controller: new(true)}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pod := &corev1.Pod{Name: "pod", Namespace: "default", OwnerReferences: []metav1.OwnerReference{tt.owner}} + + t.Log("Only pods of Jobs named like a Run's Job pass the watch filter") + if got := dgdrRunJobPod(pod); got != tt.wantWatch { + t.Fatalf("dgdrRunJobPod = %v, want %v", got, tt.wantWatch) + } + + t.Log("Only a pod of the Job a Run controls wakes that Run") + if got := f.reconciler.runForJobPod(t.Context(), pod); !reflect.DeepEqual(got, tt.want) { + t.Fatalf("requests = %v, want %v", got, tt.want) + } + }) + } +} + +func TestV1beta2RunControllerKeepsAJobWhoseJobRefWasNeverRecorded(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + cached := f.reconciler.Client + + t.Log("Create the Job but fail the status patch that would record its jobRef") + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + SubResourcePatch: func(context.Context, client.Client, string, client.Object, client.Patch, ...client.SubResourcePatchOption) error { + return errors.New("injected status patch failure") + }, + }) + if _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}); err == nil { + t.Fatal("reconcile error = nil, want the injected status patch failure") + } + job := f.job() + if f.run().Status.JobRef != nil { + t.Fatalf("jobRef = %+v, want none after the failed status patch", f.run().Status.JobRef) + } + + t.Log("The Job's pod becomes ready") + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(1)), StartTime: new(metav1.Now())} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("update Job status: %v", err) + } + + t.Log("Delete the claim and hide the Job from the cached List") + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) + } + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*batchv1.JobList); ok { + return nil + } + return c.List(ctx, list, opts...) + }, + }) + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if !apierrors.IsAlreadyExists(err) { + t.Fatalf("reconcile error = %v, want the Job create to hit AlreadyExists instead of failing the Run", err) + } + if condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted); condition != nil { + t.Fatalf("Completed condition = %+v, want none while the Job exists", condition) + } + + t.Log("Once the cache shows the Job, the Job drives the Run") + f.reconciler.Client = cached + f.reconcile() + assertDGDRRunRunningStatus(t, f.run(), job.Name) +} diff --git a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml new file mode 100644 index 000000000000..87d43e666645 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The test writes trace.jsonl, gzip-compressed, to retune/w-1.jsonl.gz on the request-traces claim +# before it applies this Request. The direct generator renders the Candidate without the AIC +# generator bridge. +apiVersion: nvidia.com/v1beta2 +kind: DynamoGraphDeploymentRequest +metadata: + name: aisimulate-vllm-trace +spec: + model: + name: Qwen/Qwen3-32B + backends: + - vllm + workload: + trace: + format: dynamo + source: + pvc: + claimName: request-traces + path: retune/w-1.jsonl.gz + replay: + arrivalSpeedupRatio: 2.0 + objective: + mode: Optimize + metric: ThroughputPerGPU + sla: {} + aisimulate: + inputOverrides: + engine: + mode: aggregated + context_length: 32768 + workers: + prefill: null + decode: null + optimizer: + algorithm: random + max_trials: 3 + parallelism: 1 + candidate_timeout_seconds: 120 + dgdr_run: + generator: direct + snapshot_interval_seconds: 0 + recommendation: + maxCandidates: 1 diff --git a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/output.yaml b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/output.yaml new file mode 100644 index 000000000000..81ce298dedb0 --- /dev/null +++ b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/output.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nvidia.com/v1beta2 +kind: DynamoGraphDeploymentCandidate +metadata: + $strict: false + name: $glob:aisimulate-vllm-trace-1-*-evaluated-point-* + labels: + $strict: false + nvidia.com/dgdr-candidate-id: $pattern:^evaluated-point-[0-9a-f]{12}$ + nvidia.com/dgdr-run-uid: $ignore +spec: + $strict: false + parameters: + $strict: false + backend: vllm +status: + metrics: + $strict: false + score: $ignore + usedGpus: $ignore diff --git a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/trace.jsonl b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/trace.jsonl new file mode 100644 index 000000000000..e7b9ff7a1b9f --- /dev/null +++ b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/trace.jsonl @@ -0,0 +1,40 @@ +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000001147, "request": {"request_id": "r0", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000000247, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 256, "input_sequence_hashes": [3000, 3001, 1000000000, 1000000001]}, "output_tokens": 49}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000001458, "request": {"request_id": "r1", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000000558, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 640, "input_sequence_hashes": [3000, 3001, 3002, 3003, 3004, 1000000100, 1000000101, 1000000102, 1000000103, 1000000104]}, "output_tokens": 54}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000001752, "request": {"request_id": "r2", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000000852, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 832, "input_sequence_hashes": [2000, 2001, 2002, 2003, 2004, 2005, 1000000200, 1000000201, 1000000202, 1000000203, 1000000204, 1000000205, 1000000206]}, "output_tokens": 43}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000002060, "request": {"request_id": "r3", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000001160, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 512, "input_sequence_hashes": [1000, 1001, 1002, 1003, 1000000300, 1000000301, 1000000302, 1000000303]}, "output_tokens": 33}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000002158, "request": {"request_id": "r4", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000001258, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 768, "input_sequence_hashes": [2000, 2001, 2002, 2003, 2004, 2005, 1000000400, 1000000401, 1000000402, 1000000403, 1000000404, 1000000405]}, "output_tokens": 93}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000002283, "request": {"request_id": "r5", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000001383, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 320, "input_sequence_hashes": [2000, 2001, 1000000500, 1000000501, 1000000502]}, "output_tokens": 25}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000002683, "request": {"request_id": "r6", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000001783, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 704, "input_sequence_hashes": [2000, 2001, 2002, 2003, 2004, 1000000600, 1000000601, 1000000602, 1000000603, 1000000604, 1000000605]}, "output_tokens": 87}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000002784, "request": {"request_id": "r7", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000001884, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 640, "input_sequence_hashes": [2000, 2001, 2002, 2003, 2004, 1000000700, 1000000701, 1000000702, 1000000703, 1000000704]}, "output_tokens": 56}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000003146, "request": {"request_id": "r8", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000002246, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 768, "input_sequence_hashes": [1000, 1001, 1002, 1003, 1004, 1005, 1000000800, 1000000801, 1000000802, 1000000803, 1000000804, 1000000805]}, "output_tokens": 77}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000003422, "request": {"request_id": "r9", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000002522, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 256, "input_sequence_hashes": [2000, 2001, 1000000900, 1000000901]}, "output_tokens": 86}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000003479, "request": {"request_id": "r10", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000002579, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 960, "input_sequence_hashes": [0, 1, 2, 3, 4, 5, 6, 1000001000, 1000001001, 1000001002, 1000001003, 1000001004, 1000001005, 1000001006, 1000001007]}, "output_tokens": 67}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000003871, "request": {"request_id": "r11", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000002971, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 832, "input_sequence_hashes": [0, 1, 2, 3, 4, 5, 1000001100, 1000001101, 1000001102, 1000001103, 1000001104, 1000001105, 1000001106]}, "output_tokens": 79}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000004091, "request": {"request_id": "r12", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000003191, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 960, "input_sequence_hashes": [1000, 1001, 1002, 1003, 1004, 1005, 1006, 1000001200, 1000001201, 1000001202, 1000001203, 1000001204, 1000001205, 1000001206, 1000001207]}, "output_tokens": 57}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000004173, "request": {"request_id": "r13", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000003273, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 832, "input_sequence_hashes": [1000, 1001, 1002, 1003, 1004, 1005, 1000001300, 1000001301, 1000001302, 1000001303, 1000001304, 1000001305, 1000001306]}, "output_tokens": 44}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000004345, "request": {"request_id": "r14", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000003445, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 1024, "input_sequence_hashes": [1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1000001400, 1000001401, 1000001402, 1000001403, 1000001404, 1000001405, 1000001406, 1000001407]}, "output_tokens": 85}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000004624, "request": {"request_id": "r15", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000003724, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 320, "input_sequence_hashes": [0, 1, 1000001500, 1000001501, 1000001502]}, "output_tokens": 56}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000004934, "request": {"request_id": "r16", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000004034, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 320, "input_sequence_hashes": [3000, 3001, 1000001600, 1000001601, 1000001602]}, "output_tokens": 54}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000005266, "request": {"request_id": "r17", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000004366, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 960, "input_sequence_hashes": [2000, 2001, 2002, 2003, 2004, 2005, 2006, 1000001700, 1000001701, 1000001702, 1000001703, 1000001704, 1000001705, 1000001706, 1000001707]}, "output_tokens": 31}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000005596, "request": {"request_id": "r18", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000004696, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 768, "input_sequence_hashes": [2000, 2001, 2002, 2003, 2004, 2005, 1000001800, 1000001801, 1000001802, 1000001803, 1000001804, 1000001805]}, "output_tokens": 42}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000005954, "request": {"request_id": "r19", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000005054, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 704, "input_sequence_hashes": [2000, 2001, 2002, 2003, 2004, 1000001900, 1000001901, 1000001902, 1000001903, 1000001904, 1000001905]}, "output_tokens": 27}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000006309, "request": {"request_id": "r20", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000005409, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 576, "input_sequence_hashes": [3000, 3001, 3002, 3003, 1000002000, 1000002001, 1000002002, 1000002003, 1000002004]}, "output_tokens": 89}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000006482, "request": {"request_id": "r21", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000005582, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 384, "input_sequence_hashes": [2000, 2001, 2002, 1000002100, 1000002101, 1000002102]}, "output_tokens": 40}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000006627, "request": {"request_id": "r22", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000005727, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 832, "input_sequence_hashes": [0, 1, 2, 3, 4, 5, 1000002200, 1000002201, 1000002202, 1000002203, 1000002204, 1000002205, 1000002206]}, "output_tokens": 49}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000006920, "request": {"request_id": "r23", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000006020, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 320, "input_sequence_hashes": [0, 1, 1000002300, 1000002301, 1000002302]}, "output_tokens": 32}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000007046, "request": {"request_id": "r24", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000006146, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 320, "input_sequence_hashes": [0, 1, 1000002400, 1000002401, 1000002402]}, "output_tokens": 85}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000007445, "request": {"request_id": "r25", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000006545, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 960, "input_sequence_hashes": [3000, 3001, 3002, 3003, 3004, 3005, 3006, 1000002500, 1000002501, 1000002502, 1000002503, 1000002504, 1000002505, 1000002506, 1000002507]}, "output_tokens": 83}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000007636, "request": {"request_id": "r26", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000006736, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 448, "input_sequence_hashes": [1000, 1001, 1002, 1000002600, 1000002601, 1000002602, 1000002603]}, "output_tokens": 91}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000007900, "request": {"request_id": "r27", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000007000, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 704, "input_sequence_hashes": [2000, 2001, 2002, 2003, 2004, 1000002700, 1000002701, 1000002702, 1000002703, 1000002704, 1000002705]}, "output_tokens": 79}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000008288, "request": {"request_id": "r28", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000007388, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 320, "input_sequence_hashes": [2000, 2001, 1000002800, 1000002801, 1000002802]}, "output_tokens": 57}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000008651, "request": {"request_id": "r29", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000007751, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 704, "input_sequence_hashes": [0, 1, 2, 3, 4, 1000002900, 1000002901, 1000002902, 1000002903, 1000002904, 1000002905]}, "output_tokens": 91}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000009023, "request": {"request_id": "r30", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000008123, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 448, "input_sequence_hashes": [2000, 2001, 2002, 1000003000, 1000003001, 1000003002, 1000003003]}, "output_tokens": 47}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000009081, "request": {"request_id": "r31", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000008181, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 320, "input_sequence_hashes": [2000, 2001, 1000003100, 1000003101, 1000003102]}, "output_tokens": 44}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000009321, "request": {"request_id": "r32", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000008421, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 576, "input_sequence_hashes": [1000, 1001, 1002, 1003, 1000003200, 1000003201, 1000003202, 1000003203, 1000003204]}, "output_tokens": 70}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000009402, "request": {"request_id": "r33", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000008502, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 1024, "input_sequence_hashes": [0, 1, 2, 3, 4, 5, 6, 7, 1000003300, 1000003301, 1000003302, 1000003303, 1000003304, 1000003305, 1000003306, 1000003307]}, "output_tokens": 34}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000009564, "request": {"request_id": "r34", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000008664, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 832, "input_sequence_hashes": [0, 1, 2, 3, 4, 5, 1000003400, 1000003401, 1000003402, 1000003403, 1000003404, 1000003405, 1000003406]}, "output_tokens": 84}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000009922, "request": {"request_id": "r35", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000009022, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 256, "input_sequence_hashes": [0, 1, 1000003500, 1000003501]}, "output_tokens": 31}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000010297, "request": {"request_id": "r36", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000009397, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 832, "input_sequence_hashes": [1000, 1001, 1002, 1003, 1004, 1005, 1000003600, 1000003601, 1000003602, 1000003603, 1000003604, 1000003605, 1000003606]}, "output_tokens": 89}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000010408, "request": {"request_id": "r37", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000009508, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 320, "input_sequence_hashes": [3000, 3001, 1000003700, 1000003701, 1000003702]}, "output_tokens": 63}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000010517, "request": {"request_id": "r38", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000009617, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 832, "input_sequence_hashes": [0, 1, 2, 3, 4, 5, 1000003800, 1000003801, 1000003802, 1000003803, 1000003804, 1000003805, 1000003806]}, "output_tokens": 18}} +{"schema": "dynamo.request.trace.v1", "event_type": "request_end", "event_time_unix_ms": 1761000010666, "request": {"request_id": "r39", "model": "Qwen/Qwen3-32B", "request_received_ms": 1761000009766, "total_time_ms": 900.0, "replay": {"trace_block_size": 64, "input_length": 960, "input_sequence_hashes": [1000, 1001, 1002, 1003, 1004, 1005, 1006, 1000003900, 1000003901, 1000003902, 1000003903, 1000003904, 1000003905, 1000003906, 1000003907]}, "output_tokens": 31}} diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index 612b4fb9eb71..18cae8332cd5 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -54,8 +54,9 @@ const ( // objective.mode, the scalar metric, objective.sla, recommendation.maxCandidates, // workload.static and discovered cluster hardware. workload.static: {} opts out of the static // workload; a nonempty static object receives any missing inputTokens, outputTokens and load -// defaults. objective.sla: {} opts out of the latency defaults. hardware: {} opts out of -// cluster discovery and typed hardware defaults; a nonempty object is completed. +// defaults, and workload.trace suppresses the static default. objective.sla: {} opts out of the +// latency defaults. hardware: {} opts out of cluster discovery and typed hardware defaults; a +// nonempty object is completed. // spec.aisimulate.inputOverrides never changes a default. It overrides the defaulted typed // values later, when the input is lowered. @@ -211,6 +212,10 @@ func defaultGPUHardwareSpecV1beta2(gpu *nvidiacomv1beta2.GPUHardwareSpec, defaul } func defaultWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec) { + // A trace is the complete traffic model; it receives no static default and no format default. + if workload.Trace != nil { + return + } if workload.Static == nil { workload.Static = &nvidiacomv1beta2.StaticWorkloadSpec{} } else if workload.Static.InputTokens == nil && workload.Static.OutputTokens == nil && workload.Static.Load == nil { diff --git a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go index 3001d912142d..6135cc8595e8 100644 --- a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go @@ -367,6 +367,20 @@ func TestDynamoGraphDeploymentRequestV1beta2DefaultingAdmission(t *testing.T) { discovered: discoveredH200, check: expectV1beta2ExplicitValues, }, + { + name: "v1beta2 trace workload receives no static default", + request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { + spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ + Format: nvidiacomv1beta2.TraceFormatDynamo, + Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz", + }}, + }} + }), + discovered: discoveredH200, + check: expectV1beta2TraceWithoutStaticDefault, + idempotent: true, + }, { name: "v1beta2 explicit node width wins while admission discovers a missing budget", request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { @@ -570,6 +584,23 @@ func expectV1beta2ExplicitValues(t *testing.T, request *nvidiacomv1beta2.DynamoG expectV1beta2H200x8Hardware(t, request) } +func expectV1beta2TraceWithoutStaticDefault(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + t.Helper() + want := &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ + Format: nvidiacomv1beta2.TraceFormatDynamo, + Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz", + }}, + }} + if !reflect.DeepEqual(request.Spec.Workload, want) { + t.Fatalf("workload = %#v, want the trace alone", request.Spec.Workload) + } + if request.Spec.Objective == nil || request.Spec.Recommendation == nil { + t.Fatalf("objective = %#v, recommendation = %#v, want the other defaults", request.Spec.Objective, request.Spec.Recommendation) + } + expectV1beta2H200x8Hardware(t, request) +} + func expectV1beta2MetadataOnlyUpdate(t *testing.T, request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { t.Helper() expectV1beta2H200x8Hardware(t, request) diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go index 7f431b046ad3..3d4b1698ae54 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -24,10 +24,12 @@ import ( "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" ) @@ -45,9 +47,35 @@ func TestDynamoGraphDeploymentRequestCarrierObjectsStayWritable(t *testing.T) { t.Run("native v1beta2 written through v1alpha1", func(t *testing.T) { carrierRoundTrip(t, beta2DGDRForAdmission(setBeta2AdmissionHardware), alphaDGDRForAdmission(nil), "v1beta2") }) + t.Run("native v1beta2 trace written through v1beta1", func(t *testing.T) { + foreign := carrierRoundTrip(t, beta2DGDRForAdmission(setBeta2AdmissionTrace), betaDGDRForAdmission(nil), "v1beta2") + + t.Log("The v1beta1 carrier holds the complete trace and no static default") + trace, found, err := unstructured.NestedMap(foreign.Object, "spec", "v1beta2", "workload", "trace") + if err != nil || !found { + t.Fatalf("spec.v1beta2.workload.trace found=%v err=%v", found, err) + } + path, _, _ := unstructured.NestedString(trace, "source", "pvc", "path") + if trace["format"] != "dynamo" || path != "retune/w-1.jsonl.gz" { + t.Fatalf("spec.v1beta2.workload.trace = %#v", trace) + } + if _, found, _ := unstructured.NestedFieldNoCopy(foreign.Object, "spec", "v1beta2", "workload", "static"); found { + t.Fatal("the carrier gained a static workload beside the trace") + } + }) +} + +func setBeta2AdmissionTrace(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { + setBeta2AdmissionHardware(request) + request.Spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ + Format: nvidiacomv1beta2.TraceFormatDynamo, + Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz", + }}, + }} } -func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, carrierKey string) { +func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, carrierKey string) *unstructured.Unstructured { t.Helper() env := admissionEnv.ForTest(t) admissionGate.set(features.Gates{GPUDiscovery: true}) @@ -119,4 +147,5 @@ func carrierRoundTrip(t *testing.T, ownerFixture, otherFixture runtime.Object, c if updated.GetGeneration() != created.GetGeneration() { t.Fatalf("generation = %d, want %d: an unchanged update rewrote the spec", updated.GetGeneration(), created.GetGeneration()) } + return foreign } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index 8f8c0e14eab9..60bb57cf789f 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -60,10 +60,13 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe if spec.Hardware != nil { allErrs = append(allErrs, v.validateHardwareSpecV1beta2(spec.Hardware, fldPath.Child("hardware"))...) } + if spec.Workload != nil { + allErrs = append(allErrs, validateWorkloadSpec(spec.Workload, fldPath.Child("workload"))...) + } if spec.AISimulate == nil { return allErrs } - return append(allErrs, validateAISimulateSpec(spec.AISimulate, fldPath.Child("aisimulate"))...) + return append(allErrs, validateAISimulateSpec(spec.AISimulate, spec.Workload, fldPath.Child("aisimulate"))...) } // validateHardwareSpecV1beta2 validates hardware. spec and fldPath must not be nil. diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go index bd955a3e54b5..201576e1f9a0 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go @@ -23,9 +23,11 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) -// validateAISimulateSpec validates the Request's native AISimulate overrides. +// validateAISimulateSpec validates the Request's native AISimulate overrides. workload may be +// nil; a workload with a trace owns the trace source the overrides may not change. func validateAISimulateSpec( input *nvidiacomv1beta2.AISimulateSpec, + workload *nvidiacomv1beta2.WorkloadSpec, fldPath *field.Path, ) field.ErrorList { // Use the lowering package's decoder as the single interpretation of RawExtension. @@ -35,16 +37,21 @@ func validateAISimulateSpec( return nil } + allErrs := field.ErrorList{} + if workload != nil && workload.Trace != nil { + allErrs = validateTraceOverrides(overrides, fldPath.Child("inputOverrides")) + } + dgdrRunValue, found := overrides["dgdr_run"] if !found { - return nil + return allErrs } dgdrRun, ok := dgdrRunValue.(map[string]any) if !ok { - return field.ErrorList{field.Forbidden( + return append(allErrs, field.Forbidden( fldPath.Child("inputOverrides", "dgdr_run"), "must be an object because the operator owns required execution fields within it", - )} + )) } // Search overrides remain authoritative; only execution values coupled to the @@ -59,7 +66,6 @@ func validateAISimulateSpec( "runtime_version_override", "snapshot_dir", } - allErrs := field.ErrorList{} for _, key := range reserved { if _, found := dgdrRun[key]; found { allErrs = append(allErrs, field.Forbidden( @@ -70,3 +76,37 @@ func validateAISimulateSpec( } return allErrs } + +// validateTraceOverrides rejects the traffic overrides that would replace what a trace workload +// owns: traffic.source.type, format and paths, or a traffic or traffic.source value that is not an +// object and would delete them. Every other traffic override is AISimulate input, which AISimulate +// validates itself. +func validateTraceOverrides(overrides map[string]any, fldPath *field.Path) field.ErrorList { + const owned = "is owned by spec.workload.trace and cannot be overridden" + traffic, found := overrides["traffic"] + if !found { + return nil + } + trafficObject, ok := traffic.(map[string]any) + if !ok { + return field.ErrorList{field.Forbidden(fldPath.Child("traffic"), + "must be an object because spec.workload.trace owns traffic.source")} + } + + // The source fields the trace owns, or a source that is not an object. + source, found := trafficObject["source"] + if !found { + return nil + } + sourceObject, ok := source.(map[string]any) + if !ok { + return field.ErrorList{field.Forbidden(fldPath.Child("traffic", "source"), owned)} + } + allErrs := field.ErrorList{} + for _, key := range []string{"format", "paths", "type"} { + if _, found := sourceObject[key]; found { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("traffic", "source", key), owned)) + } + } + return allErrs +} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 2a7d18b5d322..922b8aa0a1fb 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -751,6 +751,370 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { discovered: discoveredH200, wantCELErr: `spec.workload.static.load: Invalid value: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified`, }, + // v1beta2 recorded-trace workload: the shared schema rules and the trace-owned source. + { + name: "v1beta2 trace workload is kept and receives no static default", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune/w-1.jsonl.gz")} + }), + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if _, found := v1beta2Field(t, result, "spec", "workload", "static"); found { + t.Fatal("workload.static was defaulted beside a trace") + } + if got := v1beta2String(t, result, "spec", "workload", "trace", "source", "pvc", "path"); got != "retune/w-1.jsonl.gz" { + t.Fatalf("workload.trace.source.pvc.path = %q", got) + } + if got := v1beta2String(t, result, "spec", "workload", "trace", "format"); got != "dynamo" { + t.Fatalf("workload.trace.format = %q, want dynamo", got) + } + }, + }, + { + name: "v1beta2 trace path that leaves the claim is rejected by CEL", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune/../../w.jsonl")} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + { + name: "v1beta2 absolute trace path is rejected by CEL", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("/w.jsonl")} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + { + name: "v1beta2 trace path with an empty segment is rejected by CEL", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune//w.jsonl")} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + { + name: "v1beta2 trace path with a NUL character is rejected by CEL", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune/w\x00.jsonl")} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must not contain a NUL character`, + }, + { + name: "v1beta2 trace claim name must be a DNS subdomain", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + trace := v1beta2TraceForAdmission("w.jsonl") + trace["source"] = map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "w.jsonl"}} + spec["workload"] = map[string]any{"trace": trace} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.workload.trace.source.pvc.claimName: Invalid value: "Bad_Claim": spec.workload.trace.source.pvc.claimName in body should match '^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$'`, + }, + { + name: "v1beta2 trace format is required", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + trace := v1beta2TraceForAdmission("w.jsonl") + delete(trace, "format") + spec["workload"] = map[string]any{"trace": trace} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.workload.trace.format: Required value`, + }, + { + name: "v1beta2 trace format admits only dynamo", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + trace := v1beta2TraceForAdmission("w.jsonl") + trace["format"] = "mooncake" + spec["workload"] = map[string]any{"trace": trace} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantSchemaErr: `spec.workload.trace.format: Unsupported value: "mooncake": supported values: "dynamo"`, + }, + { + name: "v1beta2 trace source requires a pvc", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + trace := v1beta2TraceForAdmission("w.jsonl") + trace["source"] = map[string]any{} + spec["workload"] = map[string]any{"trace": trace} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload.trace.source: Invalid value: pvc is required`, + }, + { + name: "v1beta2 static and trace are mutually exclusive", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}, "trace": v1beta2TraceForAdmission("w.jsonl")} + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantCELErr: `spec.workload: Invalid value: static and trace are mutually exclusive`, + }, + { + name: "v1beta2 input overrides cannot change the trace source", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl")} + setV1beta2InputOverrides(spec, map[string]any{"traffic": map[string]any{ + "source": map[string]any{"type": "synthetic", "format": "mooncake", "paths": []any{"/etc/passwd"}}, + "load": map[string]any{"speedup": 2.0}, + }}) + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.inputOverrides.traffic.source.format: Forbidden: is owned by spec.workload.trace and cannot be overridden", + "spec.aisimulate.inputOverrides.traffic.source.paths: Forbidden: is owned by spec.workload.trace and cannot be overridden", + "spec.aisimulate.inputOverrides.traffic.source.type: Forbidden: is owned by spec.workload.trace and cannot be overridden", + }, + }, + // A v1beta2 trace in a v1beta1 carrier stored without admission: CEL ratchets the unchanged + // trace on the next v1beta2 write, so the structural webhook recursion rejects it. + { + name: "v1beta2 carrier trace path that leaves the claim", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("../escape.jsonl")}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("../escape.jsonl")} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload.trace.source.pvc.path: Invalid value: "../escape.jsonl": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + // CEL ratchets the unchanged trace and reports what it tolerated as a warning. + wantWarnings: []string{ + `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + }, + { + name: "v1beta2 carrier absolute trace path", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("/w.jsonl")}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("/w.jsonl")} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload.trace.source.pvc.path: Invalid value: "/w.jsonl": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + // CEL ratchets the unchanged trace and reports what it tolerated as a warning. + wantWarnings: []string{ + `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + }, + { + name: "v1beta2 carrier trace path with an empty segment", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("retune//w.jsonl")}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune//w.jsonl")} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload.trace.source.pvc.path: Invalid value: "retune//w.jsonl": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + // CEL ratchets the unchanged trace and reports what it tolerated as a warning. + wantWarnings: []string{ + `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + }, + { + name: "v1beta2 carrier trace path with a NUL character", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("retune/w\x00.jsonl")}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune/w\x00.jsonl")} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.workload.trace.source.pvc.path: Invalid value: \"retune/w\\x00.jsonl\": path must not contain a NUL character", + }, + // CEL ratchets the unchanged trace and reports what it tolerated as a warning. + wantWarnings: []string{ + `spec.workload.trace.source.pvc.path: Invalid value: "string": path must not contain a NUL character`, + }, + }, + { + name: "v1beta2 carrier trace path longer than the claim path limit", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission(strings.Repeat("a", 1025))}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission(strings.Repeat("a", 1025))} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.workload.trace.source.pvc.path: Too long: may not be more than 1024 characters", + }, + }, + { + name: "v1beta2 carrier trace claim that is not a DNS subdomain", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "w.jsonl"}}}}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "w.jsonl"}}}} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload.trace.source.pvc.claimName: Invalid value: "Bad_Claim": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, + }, + }, + { + name: "v1beta2 carrier trace format that is not admitted", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "mooncake", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": map[string]any{"format": "mooncake", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload.trace.format: Unsupported value: "mooncake": supported values: "dynamo"`, + }, + }, + { + name: "v1beta2 carrier trace without a format", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": map[string]any{"source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload.trace.format: Required value`, + }, + }, + { + name: "v1beta2 carrier trace source without a pvc", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{}}}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{}}} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload.trace.source.pvc: Required value`, + }, + // CEL ratchets the unchanged trace and reports what it tolerated as a warning. + wantWarnings: []string{ + `spec.workload.trace.source: Invalid value: "object": pvc is required`, + }, + }, + { + name: "v1beta2 carrier trace beside a static workload", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload: Forbidden: static and trace are mutually exclusive`, + }, + // CEL ratchets the unchanged trace and reports what it tolerated as a warning. + wantWarnings: []string{ + `spec.workload: Invalid value: "object": static and trace are mutually exclusive`, + }, + }, + { + name: "v1beta2 carrier trace arrival speedup that is not positive", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload.trace.replay.arrivalSpeedupRatio: Invalid value: 0: must be greater than 0`, + }, + }, + { + name: "v1beta2 carrier trace errors are reported together in field order", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": -1.0}}}), + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": -1.0}}} + }), + seedWithoutWebhook: true, + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + `spec.workload: Forbidden: static and trace are mutually exclusive`, + `spec.workload.trace.format: Unsupported value: "mooncake": supported values: "dynamo"`, + `spec.workload.trace.source.pvc.claimName: Invalid value: "Bad_Claim": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, + `spec.workload.trace.source.pvc.path: Invalid value: "../w.jsonl": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.replay.arrivalSpeedupRatio: Invalid value: -1: must be greater than 0`, + }, + // CEL ratchets the unchanged trace and reports what it tolerated as a warning. + wantWarnings: []string{ + `spec.workload: Invalid value: "object": static and trace are mutually exclusive`, + `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + }, + { + name: "v1beta2 a trace leaves traffic overrides it does not own to AISimulate", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl")} + setV1beta2InputOverrides(spec, map[string]any{"traffic": map[string]any{ + "load": map[string]any{"speedup": 2.0}, + "source": map[string]any{"block_size": 64}, + "stop": map[string]any{"requests": 8}, + }}) + }), + gpuDiscovery: true, + discovered: discoveredH200, + }, + { + name: "v1beta2 source overrides stay allowed without a trace", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + setV1beta2InputOverrides(spec, map[string]any{"traffic": map[string]any{ + "source": map[string]any{"type": "trace", "format": "dynamo", "paths": []any{"/x"}}, + }}) + }), + gpuDiscovery: true, + discovered: discoveredH200, + }, + { + name: "v1beta2 a null traffic override would delete the trace source", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl")} + setV1beta2InputOverrides(spec, map[string]any{"traffic": nil}) + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.inputOverrides.traffic: Forbidden: must be an object because spec.workload.trace owns traffic.source", + }, + }, + { + name: "v1beta2 a null source override would delete the trace source", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl")} + setV1beta2InputOverrides(spec, map[string]any{"traffic": map[string]any{"source": nil}}) + }), + gpuDiscovery: true, + discovered: discoveredH200, + wantWebhook: []string{ + "spec.aisimulate.inputOverrides.traffic.source: Forbidden: is owned by spec.workload.trace and cannot be overridden", + }, + }, { name: "v1beta2 update to duplicate backends is rejected", oldRequest: v1beta2DGDRForAdmission(nil), @@ -883,6 +1247,55 @@ func TestDynamoGraphDeploymentRunSpecIsImmutable(t *testing.T) { } } +// A Run has no admission webhook, so the trace rules it shares with the Request must hold in its +// own CRD schema: a directly created Run cannot name a file outside its claim. +func TestDynamoGraphDeploymentRunTraceIsValidatedByTheCRD(t *testing.T) { + tests := []struct { + name string + tracePath string + withStatic bool + wantCELErr string + }{ + {name: "a valid trace is stored", tracePath: "retune/w-1.jsonl.gz"}, + { + name: "a path that leaves the claim is rejected", + tracePath: "../w.jsonl", + wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + }, + { + name: "a path with a NUL character is rejected", + tracePath: "retune/w\x00.jsonl", + wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must not contain a NUL character`, + }, + { + name: "static beside a trace is rejected", + tracePath: "retune/w-1.jsonl.gz", + withStatic: true, + wantCELErr: `spec.workload: Invalid value: static and trace are mutually exclusive`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := v1beta2RunForAdmission(v1beta2AdmissionModel, nil) + run.Spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ + Format: nvidiacomv1beta2.TraceFormatDynamo, + Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", Path: tt.tracePath, + }}, + }} + if tt.withStatic { + run.Spec.Workload.Static = &nvidiacomv1beta2.StaticWorkloadSpec{} + } + actual := runAdmissionTest(t, admissionTestCase{object: run, wantCELError: tt.wantCELErr}) + if tt.wantCELErr == "" { + if got := v1beta2String(t, actual, "spec", "workload", "trace", "source", "pvc", "path"); got != "retune/w-1.jsonl.gz" { + t.Fatalf("stored workload.trace.source.pvc.path = %q, want the trace kept", got) + } + } + }) + } +} + func TestDynamoGraphDeploymentRequestStatusRepresentationMatchesSpec(t *testing.T) { tests := []struct { name string @@ -1127,6 +1540,28 @@ func v1beta2DGDRForAdmission(mutate func(spec map[string]any)) *unstructured.Uns }} } +// v1beta1CarrierForAdmission is a v1beta1 Request carrying a v1beta2 spec with workload under the +// name v1beta2DGDRForAdmission uses. Only conversion writes such a carrier, so seed it without the +// webhook. +func v1beta1CarrierForAdmission(workload map[string]any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeploymentRequest", + "metadata": map[string]any{"name": "test-v1beta2-dgdr", "namespace": "default"}, + "spec": map[string]any{"v1beta2": map[string]any{ + "model": map[string]any{"name": v1beta2AdmissionModel}, + "workload": workload, + }}, + }} +} + +func v1beta2TraceForAdmission(path string) map[string]any { + return map[string]any{ + "format": "dynamo", + "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": path}}, + } +} + func v1beta2Field(t *testing.T, obj *unstructured.Unstructured, path ...string) (any, bool) { t.Helper() value, found, err := unstructured.NestedFieldNoCopy(obj.Object, path...) diff --git a/deploy/operator/internal/webhook/validation/shared_helpers.go b/deploy/operator/internal/webhook/validation/shared_helpers.go index a5780d059a14..be801a8accaa 100644 --- a/deploy/operator/internal/webhook/validation/shared_helpers.go +++ b/deploy/operator/internal/webhook/validation/shared_helpers.go @@ -405,6 +405,20 @@ func groveNativeMinimumPath(component *nvidiacomv1beta1.DynamoComponentDeploymen return fldPath.Child("providerOverride", "value", "spec", "minAvailable") } +// traceClaimPathHasOnlyNamedSegments reports whether claimPath is relative to the claim root and +// every segment names an entry: none is empty, "." or "..". +func traceClaimPathHasOnlyNamedSegments(claimPath string) bool { + if claimPath == "" { + return false + } + for segment := range strings.SplitSeq(claimPath, "/") { + if segment == "" || segment == "." || segment == ".." { + return false + } + } + return true +} + // hasRoleSidecarV1Alpha1 resolves component-wide mode from the authored leader. // spec must not be nil. func hasRoleSidecarV1Alpha1(spec *nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec) bool { diff --git a/deploy/operator/internal/webhook/validation/shared_v1beta2.go b/deploy/operator/internal/webhook/validation/shared_v1beta2.go new file mode 100644 index 000000000000..426094a5a965 --- /dev/null +++ b/deploy/operator/internal/webhook/validation/shared_v1beta2.go @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package validation + +import ( + "strings" + "unicode/utf8" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +// maxTraceClaimPathLength is the CRD maxLength of spec.workload.trace.source.pvc.path. +const maxTraceClaimPathLength = 1024 + +// validateWorkloadSpec validates workload, a type that Requests and Runs share. The CRD schema and +// CEL enforce the same rules on every v1beta2 write; this recursion reaches a Request whose v1beta1 +// carrier was stored without admission and that CEL ratcheting then lets through. workload and +// fldPath must not be nil. +func validateWorkloadSpec(workload *nvidiacomv1beta2.WorkloadSpec, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if workload.Trace == nil { + return allErrs + } + if workload.Static != nil { + allErrs = append(allErrs, field.Forbidden(fldPath, "static and trace are mutually exclusive")) + } + return append(allErrs, validateTraceWorkloadSpec(workload.Trace, fldPath.Child("trace"))...) +} + +// validateTraceWorkloadSpec validates trace. trace and fldPath must not be nil. +func validateTraceWorkloadSpec(trace *nvidiacomv1beta2.TraceWorkloadSpec, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + // The format has no default: AISimulate and the DEP default to mooncake, which is not admitted. + switch trace.Format { + case nvidiacomv1beta2.TraceFormatDynamo: + case "": + allErrs = append(allErrs, field.Required(fldPath.Child("format"), "")) + default: + allErrs = append(allErrs, field.NotSupported(fldPath.Child("format"), trace.Format, + []nvidiacomv1beta2.TraceFormat{nvidiacomv1beta2.TraceFormatDynamo})) + } + + allErrs = append(allErrs, validateTraceSourceSpec(&trace.Source, fldPath.Child("source"))...) + if trace.Replay != nil { + allErrs = append(allErrs, validateTraceReplaySpec(trace.Replay, fldPath.Child("replay"))...) + } + return allErrs +} + +// validateTraceSourceSpec validates source. source and fldPath must not be nil. +func validateTraceSourceSpec(source *nvidiacomv1beta2.TraceSourceSpec, fldPath *field.Path) field.ErrorList { + if source.PVC == nil { + return field.ErrorList{field.Required(fldPath.Child("pvc"), "")} + } + return validateTracePVCSource(source.PVC, fldPath.Child("pvc")) +} + +// validateTracePVCSource validates pvc: a PersistentVolumeClaim name and one nameable file inside +// it. pvc and fldPath must not be nil. +func validateTracePVCSource(pvc *nvidiacomv1beta2.TracePVCSource, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + for _, msg := range validation.IsDNS1123Subdomain(pvc.ClaimName) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("claimName"), pvc.ClaimName, msg)) + } + + // The path names one file below the claim root: relative, with only named segments. + pathPath := fldPath.Child("path") + if utf8.RuneCountInString(pvc.Path) > maxTraceClaimPathLength { + allErrs = append(allErrs, field.TooLongCharacters(pathPath, pvc.Path, maxTraceClaimPathLength)) + } + if !traceClaimPathHasOnlyNamedSegments(pvc.Path) { + allErrs = append(allErrs, field.Invalid(pathPath, pvc.Path, + "path must be relative to the claim root without empty, '.' or '..' segments")) + } + if strings.ContainsRune(pvc.Path, 0) { + allErrs = append(allErrs, field.Invalid(pathPath, pvc.Path, "path must not contain a NUL character")) + } + return allErrs +} + +// validateTraceReplaySpec validates replay. replay and fldPath must not be nil. +func validateTraceReplaySpec(replay *nvidiacomv1beta2.TraceReplaySpec, fldPath *field.Path) field.ErrorList { + if replay.ArrivalSpeedupRatio != nil && *replay.ArrivalSpeedupRatio <= 0 { + return field.ErrorList{field.Invalid(fldPath.Child("arrivalSpeedupRatio"), *replay.ArrivalSpeedupRatio, + "must be greater than 0")} + } + return nil +} diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index f541c552b44e..9278dc89c194 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3259,7 +3259,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `inputOverrides` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from
the typed fields. It uses the upstream AISimulate JSON contract, including
snake_case names. Values here take precedence, and null deletes a lowered
field. The dgdr_run member must remain an object, and its operator-owned
transport, identity, and runtime fields name, max_candidates, snapshot_dir,
runtime_image, runtime_version_override, derive_runtime_images,
num_gpus_per_node, and namespace are operator-owned. Request admission
rejects them, and the Run controller enforces its values while lowering the
immutable snapshot. The operator runs the AISimulate version from its
configured planner image; Requests and Runs cannot select a different image
or version. | | Type: object
Optional: \{\}
| +| `inputOverrides` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from
the typed fields. It uses the upstream AISimulate JSON contract, including
snake_case names. Values here take precedence, and null deletes a lowered
field. The dgdr_run member must remain an object, and its operator-owned
transport, identity, and runtime fields name, max_candidates, snapshot_dir,
runtime_image, runtime_version_override, derive_runtime_images,
num_gpus_per_node, and namespace are operator-owned. Request admission
rejects them, and the Run controller enforces its values while lowering the
immutable snapshot. When spec.workload.trace is set, traffic and
traffic.source must remain objects, and the trace owns traffic.source.type,
format, and paths the same way: Request admission rejects them, and the Run
controller enforces the typed values. Other traffic overrides are AISimulate
input, which AISimulate validates when the Run's Job starts. The operator runs
the AISimulate version from its configured planner image; Requests and Runs
cannot select a different image or version. | | Type: object
Optional: \{\}
| #### Backend @@ -3453,7 +3453,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, except the traffic.source fields a trace owns.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3495,7 +3495,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, except the traffic.source fields a trace owns.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3562,7 +3562,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, except the traffic.source fields a trace owns.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3724,7 +3724,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the synthetic traffic model used for every candidate evaluation.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, except the traffic.source fields a trace owns.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3894,11 +3894,97 @@ _Appears in:_ | `load` _[StaticLoadSpec](#staticloadspec)_ | Load selects the synthetic load model. When omitted from a nonempty static
workload, admission defaults it to concurrency 10. | | Optional: \{\}
| +#### TraceFormat + +_Underlying type:_ _string_ + +TraceFormat identifies the decoder of a recorded request trace. + +_Validation:_ +- Enum: [dynamo] + +_Appears in:_ +- [TraceWorkloadSpec](#traceworkloadspec) + +| Field | Description | +| --- | --- | +| `dynamo` | TraceFormatDynamo is the Dynamo request trace (dynamo.request.trace.v1) that the frontend
records, as JSON Lines or gzip-compressed JSON Lines.
| + + +#### TracePVCSource + + + +TracePVCSource names one trace file on a PersistentVolumeClaim in the Request namespace. + + + +_Appears in:_ +- [TraceSourceSpec](#tracesourcespec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `claimName` _string_ | ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace.
The Run Job mounts only the trace file from it, read-only, and only into the AISimulate
container. | | MaxLength: 253
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
| +| `path` _string_ | Path names one trace file relative to the root of the claim. It is neither a directory
nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..'
segments or a NUL character, which no file name can hold. The file must be complete before
the Request is created and must never be rewritten: every Run of the Request reads it when
its Job starts. | | MaxLength: 1024
MinLength: 1
| + + +#### TraceReplaySpec + + + +TraceReplaySpec configures how AISimulate replays a decoded trace. + + + +_Appears in:_ +- [TraceWorkloadSpec](#traceworkloadspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `arrivalSpeedupRatio` _float_ | ArrivalSpeedupRatio multiplies the recorded request-arrival rate. 1.0 preserves it.
When omitted, the recorded arrival times are replayed unchanged. | | ExclusiveMinimum: true
Minimum: 0
Optional: \{\}
| + + +#### TraceSourceSpec + + + +TraceSourceSpec identifies exactly one trace artifact. A PersistentVolumeClaim is the only +source in this release. + + + +_Appears in:_ +- [TraceWorkloadSpec](#traceworkloadspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `pvc` _[TracePVCSource](#tracepvcsource)_ | PVC reads the trace from a PersistentVolumeClaim in the Request namespace. | | Optional: \{\}
| + + +#### TraceWorkloadSpec + + + +TraceWorkloadSpec replays a recorded request trace open loop at its recorded arrival times. + + + +_Appears in:_ +- [WorkloadSpec](#v1beta2-workloadspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `format` _[TraceFormat](#traceformat)_ | Format identifies the trace decoder. It has no default; only dynamo is admitted in this
release. | | Enum: [dynamo]
Required: \{\}
| +| `source` _[TraceSourceSpec](#tracesourcespec)_ | Source identifies the trace artifact. | | | +| `replay` _[TraceReplaySpec](#tracereplayspec)_ | Replay configures how the decoded trace is replayed. | | Optional: \{\}
| + + #### v1beta2 WorkloadSpec -WorkloadSpec defines the synthetic traffic model used for every candidate evaluation. +WorkloadSpec defines the traffic model used for every candidate evaluation. static and +trace are mutually exclusive. @@ -3910,7 +3996,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines AISimulate synthetic traffic. When omitted, admission supplies the
complete static default. An explicit empty object opts out; any nonempty object is
completed with omitted inputTokens, outputTokens and load defaults. | | Optional: \{\}
| +| `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines AISimulate synthetic traffic. When omitted without a trace, admission
supplies the complete static default. An explicit empty object opts out; any nonempty
object is completed with omitted inputTokens, outputTokens and load defaults. | | Optional: \{\}
| +| `trace` _[TraceWorkloadSpec](#traceworkloadspec)_ | Trace replays a recorded request trace from a PersistentVolumeClaim in the Request
namespace. Admission supplies no static default when it is set. The Run Job mounts only the
trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl,
or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change
traffic.source.type, format or paths. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index 530994cd155c..088e8fcfb625 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -1114,6 +1114,100 @@ kubectl get dgdr my-model -n $NAMESPACE \ - The relationship is tracked via labels: `dgdr.nvidia.com/name` and `dgdr.nvidia.com/namespace`. - Additional resources (planner ConfigMaps) are created in the same namespace and labeled with `dgdr.nvidia.com/name`. +## Replay a recorded trace (v1beta2) + + +The `nvidia.com/v1beta2` API is experimental and disabled by default. See the +[installation guide](../../kubernetes/installation/install-dynamo.md) before you enable it. + + +A native `v1beta2` request can evaluate every candidate against a recorded request trace instead +of synthetic traffic. Set `spec.workload.trace` instead of `spec.workload.static`; the two are +mutually exclusive, and admission adds no static default when `trace` is set. To switch an existing +request from static traffic to a trace, set `spec.workload.static` to `null` in the same update. + +```yaml +apiVersion: nvidia.com/v1beta2 +kind: DynamoGraphDeploymentRequest +metadata: + name: qwen3-trace-replay +spec: + model: + name: Qwen/Qwen3-0.6B + workload: + trace: + format: dynamo # required; no default + source: + pvc: + claimName: request-traces # PersistentVolumeClaim in this namespace + path: windows/2026-10-27T10-00.jsonl.gz + replay: + arrivalSpeedupRatio: 1.0 # optional; 2.0 replays arrivals twice as fast +``` + +Before you create the request: + +- Create the PersistentVolumeClaim in the request's namespace. Use a filesystem volume, not + `volumeMode: Block`. When the process that writes the trace keeps running, use `ReadWriteMany`. + With `ReadWriteOnce`, a Job scheduled to another node than the writer waits in + `ContainerCreating`, and with `ReadWriteOncePod`, a Job waits in `Pending` until the writer's pod + is gone. The Run reports `Running` while its Job waits. +- Finish writing the file. `path` names one file relative to the claim root; directories and globs + are not expanded. Every Run of the request reads the file when its Job starts, so do not rewrite + it afterwards. To replay several rolled `jsonl_gz` segments, concatenate them into one `.jsonl.gz` + file. A relative symlink to another file in the claim works. An absolute symlink does not, + because the kubelet resolves it on the node rather than inside the claim, and neither does a + symlink that leaves the claim. +- Make the file readable by others (for example, mode `0644` in `0755` directories) or by group + `65532`, which the Job's pod has as a supplemental group. The pod sets no `fsGroup`, so neither + the kubelet nor a CSI driver changes the ownership or permissions of anything on the claim. + +The operator mounts only this file, read-only and only into the AISimulate container, through a +`subPath` of the claim at `/var/run/aisimulate/trace/trace.jsonl.gz`, or at +`/var/run/aisimulate/trace/trace.jsonl` when `path` does not end in `.gz`. The kubelet resolves +`path` inside the claim and refuses to mount a file outside it. If the file does not exist when +the Job starts, the kubelet cannot mount it either. On a volume type whose node directory the +kubelet can write, such as `hostPath`, the kubelet instead creates an empty directory at `path` on +the claim, and the Run fails when AISimulate cannot read it. + +`format: dynamo` is the only admitted format. It reads the Dynamo request trace +(`dynamo.request.trace.v1`) written by the frontend, as JSON Lines or as gzip-compressed JSON Lines +when the file name ends in `.gz`. See the [Request Trace Reference](../observability/request-traces.mdx). + +The operator lowers the trace to the AISimulate input `traffic.source` (`type: trace`, `format`, and +`paths` with the mounted file) and `traffic.load.type: trace_timestamps`, with +`traffic.load.speedup` when `arrivalSpeedupRatio` is set. It adds no `traffic.stop.requests`. When +`trace` is set, `spec.aisimulate.inputOverrides` cannot set `traffic.source.type`, +`traffic.source.format`, or `traffic.source.paths`, or replace `traffic` or `traffic.source` with a +non-object value. Admission rejects these overrides, and the Run controller restores the typed +values. Other `traffic` overrides, such as `traffic.load` and `traffic.stop`, go to AISimulate, +which validates the complete input when the Run's Job starts; an input it rejects fails the Run. + +The Run fails, and its `Completed` condition has reason `TraceSourceUnavailable`, when its Job could +never read the trace: + +- The claim does not exist, is being deleted, is a block volume, or has lost its volume. The + operator checks this before it creates the Job, and again whenever the claim or the Job's pod + changes until the AISimulate container starts. Once a pod is scheduled, Kubernetes keeps a claim + being deleted until the pod ends, and the pod can still mount it. +- The kubelet cannot mount the trace file, for example because it does not exist, is an absolute + symlink, or is a symlink that leaves the claim. + +The operator records the failure on the Run and then deletes such a Job. A claim that is still +`Pending`, such as one that waits for its first consumer, is accepted. The request starts one Run +per `metadata.generation`, so after you fix the claim or the file, change the request's `spec` to +start a new Run. + +```bash +# Active and recent Runs of the request +kubectl get dynamographdeploymentrequests.v1beta2.nvidia.com qwen3-trace-replay -n "$NAMESPACE" \ + -o jsonpath='{.status.activeRunRef}{"\n"}{.status.recentRunRefs}{"\n"}' + +# Why a Run failed +kubectl get dynamographdeploymentrun qwen3-trace-replay-1 -n "$NAMESPACE" \ + -o jsonpath='{.status.conditions}' +``` + ## Known issues - **PCIe profiler data not yet available.** See the PCIe callout under [SKU format](#sku-format). diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 014e90d48076..d07629435f64 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -10,12 +10,12 @@ subtitle: Full Kubernetes CRD + operator-config API reference, generated from th > [!WARNING] > **Auto-generated from source.** This page is regenerated from the Dynamo operator CRDs by `docs/fern/scripts/gen_kubernetes_api.py` and covers **every** type across all three API packages: the deprecated `nvidia.com/v1alpha1` surface, the supported `nvidia.com/v1beta1` surface, and the operator's own `operator.config.dynamo.nvidia.com/v1alpha1` configuration. The trimmed [DGD](dynamo-graph-deployment.mdx), [DGDR](dynamo-graph-deployment-request.mdx), and [DCD](dynamo-component-deployment.mdx) references cover only user-facing `v1beta1` fields. To edit the surface, change the Go types under `deploy/operator/api/` and let CI regenerate this page. -Dynamo publishes 4 Kubernetes API packages with 211 typed CRD and config sections. +Dynamo publishes 4 Kubernetes API packages with 216 typed CRD and config sections. 71 types 76 types -35 types +40 types 29 types @@ -4546,7 +4546,7 @@ planner image. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered field. The dgdr_run member must remain an object, and its operator-owned transport, identity, and runtime fields name, max_candidates, snapshot_dir, runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the immutable snapshot. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. +InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered field. The dgdr_run member must remain an object, and its operator-owned transport, identity, and runtime fields name, max_candidates, snapshot_dir, runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the immutable snapshot. When spec.workload.trace is set, traffic and traffic.source must remain objects, and the trace owns traffic.source.type, format, and paths the same way: Request admission rejects them, and the Run controller enforces the typed values. Other traffic overrides are AISimulate input, which AISimulate validates when the Run's Job starts. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. See [RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg). **Validation:** Type: object Optional: \{\} @@ -4811,7 +4811,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. +Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -4893,7 +4893,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. +Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5019,7 +5019,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. +Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5256,7 +5256,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the synthetic traffic model used for every candidate evaluation. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. +Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5458,18 +5458,110 @@ See [StaticLoadSpec](#staticloadspec). + +TraceFormat identifies the decoder of a recorded request trace. + + +**Kind:** `enum` +**Underlying type:** `string` +**Validation:** Enum: [dynamo] +**Appears in:** [TraceWorkloadSpec](#traceworkloadspec) + +**Allowed values** + +- dynamo TraceFormatDynamo is the Dynamo request trace (dynamo.request.trace.v1) that the frontend records, as JSON Lines or gzip-compressed JSON Lines. + + + +TracePVCSource names one trace file on a PersistentVolumeClaim in the Request namespace. + + +**Kind:** `type` +**Appears in:** [TraceSourceSpec](#tracesourcespec) + + +ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate container. +**Validation:** MaxLength: 253 MinLength: 1 Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + + + +Path names one trace file relative to the root of the claim. It is neither a directory nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' segments or a NUL character, which no file name can hold. The file must be complete before the Request is created and must never be rewritten: every Run of the Request reads it when its Job starts. +**Validation:** MaxLength: 1024 MinLength: 1 + + + + +TraceReplaySpec configures how AISimulate replays a decoded trace. + + +**Kind:** `type` +**Appears in:** [TraceWorkloadSpec](#traceworkloadspec) + + +ArrivalSpeedupRatio multiplies the recorded request-arrival rate. 1.0 preserves it. When omitted, the recorded arrival times are replayed unchanged. +**Validation:** ExclusiveMinimum: true Minimum: 0 Optional: \{\} + + + + +TraceSourceSpec identifies exactly one trace artifact. A PersistentVolumeClaim is the only +source in this release. + + +**Kind:** `type` +**Appears in:** [TraceWorkloadSpec](#traceworkloadspec) + + +PVC reads the trace from a PersistentVolumeClaim in the Request namespace. +See [TracePVCSource](#tracepvcsource). +**Validation:** Optional: \{\} + + + + +TraceWorkloadSpec replays a recorded request trace open loop at its recorded arrival times. + + +**Kind:** `type` +**Appears in:** [WorkloadSpec](#v1beta2-workloadspec) + + +Format identifies the trace decoder. It has no default; only dynamo is admitted in this release. +See [TraceFormat](#traceformat). +**Validation:** Enum: [dynamo] Required: \{\} + + + +Source identifies the trace artifact. +See [TraceSourceSpec](#tracesourcespec). + + + +Replay configures how the decoded trace is replayed. +See [TraceReplaySpec](#tracereplayspec). +**Validation:** Optional: \{\} + + + -WorkloadSpec defines the synthetic traffic model used for every candidate evaluation. +WorkloadSpec defines the traffic model used for every candidate evaluation. static and +trace are mutually exclusive. **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -Static defines AISimulate synthetic traffic. When omitted, admission supplies the complete static default. An explicit empty object opts out; any nonempty object is completed with omitted inputTokens, outputTokens and load defaults. +Static defines AISimulate synthetic traffic. When omitted without a trace, admission supplies the complete static default. An explicit empty object opts out; any nonempty object is completed with omitted inputTokens, outputTokens and load defaults. See [StaticWorkloadSpec](#staticworkloadspec). **Validation:** Optional: \{\} + + +Trace replays a recorded request trace from a PersistentVolumeClaim in the Request namespace. Admission supplies no static default when it is set. The Run Job mounts only the trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change traffic.source.type, format or paths. +See [TraceWorkloadSpec](#traceworkloadspec). +**Validation:** Optional: \{\} +
diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index d09aa90176a2..e6c035481d74 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -51,7 +51,7 @@ # owned by github.com/ai-dynamo/snapshot. "nvidia.com/v1alpha1": 71, "nvidia.com/v1beta1": 76, - "nvidia.com/v1beta2": 35, + "nvidia.com/v1beta2": 40, "operator.config.dynamo.nvidia.com/v1alpha1": 29, } EXPECTED_OPERATOR_DEFAULT_SECTIONS = ( From a95d1f3e2f2e0cba854cffc7419e9fe5666ff6e6 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Thu, 29 Oct 2026 09:26:08 -0700 Subject: [PATCH 068/100] fix(operator): keep legacy non-trace Run Jobs valid and cover the stale marker The integration base built every Run Job with fsGroup 65532 and the merged Job validation required none, so an operator upgrade would fail an in-flight non-trace sweep as InvalidExecution and delete its Job. Validation now accepts that legacy fsGroup for a Run without a trace; a trace Run still must request none, and any other fsGroup is still skew. Also cover the optimistic lock on the permanent-failure marker: a stale pass that sees a second Run-controlled Job after the trace failure was recorded must conflict rather than overwrite TraceSourceUnavailable. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- ...mographdeploymentrun_v1beta2_controller.go | 10 ++++++- ...phdeploymentrun_v1beta2_controller_test.go | 27 ++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 2b808127cae0..040a3e3af9ab 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -74,6 +74,7 @@ const ( dgdrRunResourceComponent = "aisimulate" dgdrRunServiceAccountTokenTTL = int64(3600) dgdrRunTraceGroup = int64(65532) + dgdrRunLegacyFSGroup = int64(65532) dgdrRunReasonPending = "JobPending" dgdrRunReasonRunning = "JobRunning" dgdrRunReasonSucceeded = "SearchSucceeded" @@ -1248,7 +1249,14 @@ func validateDGDRRunJobSpec(spec *batchv1.JobSpec) error { } func validateDGDRRunPodSecurityContext(security *corev1.PodSecurityContext, run *nvidiacomv1beta2.DynamoGraphDeploymentRun) error { - if security == nil || security.FSGroup != nil { + if security == nil { + return errors.New("pod fsGroup differs") + } + + // The pod requests no fsGroup. Earlier operators set fsGroup 65532 on every Run Job; such a Job + // mounts no trace claim, so a Run without a trace keeps accepting it and an operator upgrade does + // not fail its in-flight sweep. + if security.FSGroup != nil && (dgdrRunTraceClaim(run) != nil || *security.FSGroup != dgdrRunLegacyFSGroup) { return errors.New("pod fsGroup differs") } if !apiequality.Semantic.DeepEqual(security.SupplementalGroups, dgdrRunPodSecurityContext(run).SupplementalGroups) { diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 70a99935988a..33e0ab666fc5 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -861,7 +861,7 @@ func TestValidateDGDRRunJobExecutionRejectsImmutableBindingSkew(t *testing.T) { { name: "pod fsGroup", mutate: func(job *batchv1.Job) { - job.Spec.Template.Spec.SecurityContext.FSGroup = ptr.To[int64](65532) + job.Spec.Template.Spec.SecurityContext.FSGroup = ptr.To[int64](1000) }, wantErr: "pod fsGroup differs", }, @@ -1046,6 +1046,14 @@ func TestValidateDGDRRunJobExecutionRejectsImmutableBindingSkew(t *testing.T) { t.Fatalf("validation error = %v, want API defaults accepted", err) } }) + + t.Run("a Job from an operator that set fsGroup 65532 remains compatible", func(t *testing.T) { + legacy := job.DeepCopy() + legacy.Spec.Template.Spec.SecurityContext.FSGroup = ptr.To[int64](65532) + if err := validateDGDRRunJobExecution(legacy, run, configMap.Name, execution); err != nil { + t.Fatalf("validation error = %v, want the legacy fsGroup accepted for a Run without a trace", err) + } + }) } func TestValidateDGDRRunJobExecutionBindsTheTraceMount(t *testing.T) { @@ -2512,9 +2520,13 @@ func TestV1beta2RunControllerDoesNotLetAStaleRunUndoATraceFailure(t *testing.T) name string // jobCondition, when set, finishes the Job after the trace failure was recorded. jobCondition batchv1.JobConditionType + // duplicateJob adds a second, unfinished Job the Run controls, so the stale pass fails the + // Run permanently and writes the permanent-failure marker. + duplicateJob bool }{ {name: "the trace is still unreadable"}, {name: "the Job completes after the failure", jobCondition: batchv1.JobComplete}, + {name: "a second Job appears after the Job completes", jobCondition: batchv1.JobComplete, duplicateJob: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -2549,6 +2561,15 @@ func TestV1beta2RunControllerDoesNotLetAStaleRunUndoATraceFailure(t *testing.T) t.Fatalf("finish Job: %v", err) } } + if tt.duplicateJob { + duplicate := f.job().DeepCopy() + duplicate.Name += "-duplicate" + duplicate.ResourceVersion, duplicate.UID = "", "" + duplicate.Status = batchv1.JobStatus{} + if err := f.client.Create(t.Context(), duplicate); err != nil { + t.Fatalf("create duplicate Job: %v", err) + } + } t.Log("A pass that still reads the Running Run conflicts instead of overwriting the failure") cached := f.reconciler.Client @@ -2577,8 +2598,8 @@ func TestV1beta2RunControllerDoesNotLetAStaleRunUndoATraceFailure(t *testing.T) `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`) return } - if kept := f.job(); kept.UID != job.UID { - t.Fatalf("Job UID = %q, want the finished Job %q kept", kept.UID, job.UID) + if jobs := f.jobs(); len(jobs) != 1 || jobs[0].UID != job.UID { + t.Fatalf("Jobs = %d, want only the finished Job %q kept", len(jobs), job.UID) } if got := f.run(); !reflect.DeepEqual(got.Status, failed.Status) { t.Fatalf("status = %+v, want the recorded failure %+v", got.Status, failed.Status) From 8f4a34011c8b69b3c811ae6922138068aff34f62 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Thu, 29 Oct 2026 10:26:52 -0700 Subject: [PATCH 069/100] fix(planner): satisfy mypy and CodeQL in live retune - metrics: pass the collector registry by keyword instead of through a dict splat, which mypy matched against the positional parameters of Counter and Gauge. - dgdr: read the current attempt's Request name through _ShardRun.require_name(), so the get_request calls get a str rather than Optional[str]. - dgdr, state: derive Request names, label values, spec digests and the state fingerprint from SHA-256 instead of SHA-1. They are identifiers, not secrets; the lengths are unchanged. - window_cut: publish window files with mode 0600 instead of 0644. The Run Job reads them as the planner image user (UID 1000), which is also the Planner's default user, so neither group nor others need access; CodeQL flags group-readable modes too. The live retune guide states the requirement. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../src/dynamo/planner/retune/live/dgdr.py | 33 ++++++++++++++----- .../src/dynamo/planner/retune/live/metrics.py | 33 +++++++++---------- .../src/dynamo/planner/retune/live/state.py | 2 +- .../dynamo/planner/retune/live/window_cut.py | 6 ++-- .../tests/unit/test_live_retune_window_cut.py | 6 ++-- .../auto-deployment/planner-live-retune.md | 4 ++- 6 files changed, 52 insertions(+), 32 deletions(-) diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py index f44eed905346..f7761f6b5621 100644 --- a/components/src/dynamo/planner/retune/live/dgdr.py +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -337,10 +337,10 @@ def need_seconds(shard: Shard, cfg: DgdrExecutorConfig, *, needs_cut: bool) -> f def spec_hash(spec: Mapping[str, Any]) -> str: - """First 16 hex digits of the SHA-1 of the canonical JSON of ``spec``.""" + """First 16 hex digits of the SHA-256 of the canonical JSON of ``spec``.""" canonical = json.dumps(spec, sort_keys=True, separators=(",", ":"), allow_nan=False) - return hashlib.sha1(canonical.encode("utf-8")).hexdigest()[:16] + return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16] def window_claim_path(cfg: DgdrExecutorConfig, dgd: DgdRef, window_end_ms: int) -> str: @@ -352,10 +352,10 @@ def window_claim_path(cfg: DgdrExecutorConfig, dgd: DgdRef, window_end_ms: int) def request_name( dgd: DgdRef, window_end_ms: int, shard_id: str, digest: str, attempt: int ) -> str: - """``-rt-``.""" + """``-rt-``.""" key = f"{dgd.uid}|{int(window_end_ms)}|{shard_id}|{digest}|{attempt}" - suffix = hashlib.sha1(key.encode("utf-8")).hexdigest()[:10] + suffix = hashlib.sha256(key.encode("utf-8")).hexdigest()[:10] prefix = dgd.name[:40].rstrip("-.") or "dgd" return f"{prefix}-rt-{suffix}" @@ -367,7 +367,7 @@ def _label_value(value: str) -> str: r"([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?)?", value ): return value - digest = hashlib.sha1(value.encode("utf-8")).hexdigest()[:10] + digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:10] head = re.sub(r"[^-A-Za-z0-9_.]", "-", value)[:52].strip("-_.") return f"{head}-{digest}" if head else digest @@ -842,6 +842,19 @@ class _ShardRun: rows: Optional[dict] = None published: int = 0 + def require_name(self) -> str: + """The current attempt's Request name. + + It is set once a create is sent or a Request is adopted, so a shard in + the ``unknown`` or ``wait`` state, where it is read, always has it. + """ + + if self.name is None: + raise RuntimeError( + f"shard {self.shard.id} has no Request name in state {self.state}" + ) + return self.name + def summary(self) -> dict[str, Any]: return { "kind": self.shard.kind, @@ -1317,7 +1330,9 @@ def _adopt_existing(self, run: _ShardRun) -> None: existing = self._retrying( f"get Request {run.name}", - lambda: self.ctx.kube.get_request(run.name, timeout_s=self._timeout()), + lambda: self.ctx.kube.get_request( + run.require_name(), timeout_s=self._timeout() + ), ) if existing is None: # Deleted in between; create it again on the next poll. @@ -1434,7 +1449,9 @@ def _resolve_unknown(self, run: _ShardRun) -> None: """After a create whose outcome is unknown: adopt it or create again.""" try: - existing = self.ctx.kube.get_request(run.name, timeout_s=self._timeout()) + existing = self.ctx.kube.get_request( + run.require_name(), timeout_s=self._timeout() + ) except KubeError as error: self._read_failed(f"get Request {run.name}", error) return @@ -1467,7 +1484,7 @@ def _observe(self, run: _ShardRun) -> None: kube = self.ctx.kube try: - obj = kube.get_request(run.name, timeout_s=self._timeout()) + obj = kube.get_request(run.require_name(), timeout_s=self._timeout()) except KubeError as error: self._read_failed(f"get Request {run.name}", error) return diff --git a/components/src/dynamo/planner/retune/live/metrics.py b/components/src/dynamo/planner/retune/live/metrics.py index cd28e1c21248..15a559acc4a7 100644 --- a/components/src/dynamo/planner/retune/live/metrics.py +++ b/components/src/dynamo/planner/retune/live/metrics.py @@ -42,93 +42,92 @@ class _Collectors: """The metric objects of one registry.""" def __init__(self, registry: CollectorRegistry) -> None: - kw = {"registry": registry} self.decisions_total = Counter( f"{PREFIX}_decisions_total", "Retune decisions, counted once the state write persisted them.", labelnames=["outcome"], - **kw, + registry=registry, ) self.failures_total = Counter( f"{PREFIX}_failures_total", "Failed retune windows and supervisor failures, by reason.", labelnames=["reason"], - **kw, + registry=registry, ) self.skipped_total = Counter( f"{PREFIX}_skipped_total", "Due windows the launch gate held back, once per window and reason.", labelnames=["reason"], - **kw, + registry=registry, ) self.leader = Gauge( f"{PREFIX}_leader", "1 while this planner holds the retune leadership Lease.", - **kw, + registry=registry, ) self.ready = Gauge( f"{PREFIX}_ready", "1 while every spec and health premise holds, so windows launch.", - **kw, + registry=registry, ) self.initialized = Gauge( f"{PREFIX}_initialized", "1 once the leader finished initializing from the persisted state.", - **kw, + registry=registry, ) self.rollback_pending = Gauge( f"{PREFIX}_rollback_pending", "1 while routers are being reset to their startup parameters.", - **kw, + registry=registry, ) self.controller_generation = Gauge( f"{PREFIX}_controller_generation", "Generation of the installed retune controller; it grows on each rebuild.", - **kw, + registry=registry, ) self.baseline_score = Gauge( f"{PREFIX}_baseline_score", "Objective score of the current parameters on the latest decided " "window (NaN when not scored).", - **kw, + registry=registry, ) self.best_score = Gauge( f"{PREFIX}_best_score", "Best candidate score on the latest decided window (NaN when none).", - **kw, + registry=registry, ) self.sweep_seconds = Gauge( f"{PREFIX}_sweep_seconds", "Wall-clock duration of the latest sweep (NaN when it did not run).", - **kw, + registry=registry, ) self.missing_candidates = Gauge( f"{PREFIX}_missing_candidates", "Candidates of the latest sweep without a score.", - **kw, + registry=registry, ) self.router_converged = Gauge( f"{PREFIX}_router_converged", "1 when every frontend router holds the target policy parameters.", - **kw, + registry=registry, ) self.router_patch_errors_total = Counter( f"{PREFIX}_router_patch_errors_total", "Failed /router_policy PATCH or reset calls, by HTTP status or kind.", labelnames=["code"], - **kw, + registry=registry, ) self.trace_sink_missing = Gauge( f"{PREFIX}_trace_sink_missing", "1 for a Ready frontend pod without its own trace segment directory.", labelnames=["pod"], - **kw, + registry=registry, ) self.trace_dropped_members_total = Counter( f"{PREFIX}_trace_dropped_members_total", "Trace gzip members dropped by window cuts for a bad CRC, size or " "truncation.", - **kw, + registry=registry, ) for outcome in (*RETUNE_OUTCOMES, *_LIVE_OUTCOMES): self.decisions_total.labels(outcome=outcome) diff --git a/components/src/dynamo/planner/retune/live/state.py b/components/src/dynamo/planner/retune/live/state.py index 598fc2f9996d..640b4fd031a1 100644 --- a/components/src/dynamo/planner/retune/live/state.py +++ b/components/src/dynamo/planner/retune/live/state.py @@ -164,7 +164,7 @@ def fingerprint(config: PlannerConfig, startup: Optional[FrontendStartup]) -> st "sku": None if dgdr is None else dgdr.sku, "startup": None if startup is None else startup.fingerprint_source, } - digest = hashlib.sha1(canonical_json(payload).encode("utf-8")).hexdigest() + digest = hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() return digest[:16] diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py index 026261454064..54e22b1921df 100644 --- a/components/src/dynamo/planner/retune/live/window_cut.py +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -37,7 +37,7 @@ that evicts the member needed furthest in the future. A member or row that no longer matches pass 1 fails the cut (the segment changed). * **Atomic output.** The file is written to ``.tmp--`` - next to ``out_path``, fsynced, made 0644 and hard-linked into place; an + next to ``out_path``, fsynced, made 0600 and hard-linked into place; an existing ``out_path`` is reused (another process cut the same window). * **Child process.** ``cut_window`` runs the cut in a ``spawn`` child, like the replay sweep's workers, and kills it at the deadline or on cancel, so @@ -1089,7 +1089,9 @@ def _publish(out_path: str, temp_path: str, write: Callable[[_ByteSink], None]) write(sink) raw.flush() os.fsync(raw.fileno()) - os.chmod(temp_path, 0o644) + # Only the owner reads the window. The Run Job reads it as the planner + # image user (uid 1000), which the Planner also runs as by default. + os.chmod(temp_path, 0o600) try: os.link(temp_path, out_path) created = True diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py index 0dfe1ee34c76..6587cd9afede 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -392,7 +392,7 @@ def test_cut_matches_read_trace_window(tmp_path, seed): assert result.dropped_members == 0 assert result.dropped_bytes == 0 assert not result.reused - assert stat.S_IMODE(out.stat().st_mode) == 0o644 + assert stat.S_IMODE(out.stat().st_mode) == 0o600 assert _temp_files(out.parent) == [] @@ -974,7 +974,7 @@ def vanished_link(source, target): assert _temp_files(out.parent) == [] -def test_output_is_deterministic_and_0644_under_any_umask(tmp_path): +def test_output_is_deterministic_and_0600_under_any_umask(tmp_path): root = _random_trace(tmp_path / "segments", 8) first, second = _out(tmp_path, "a.jsonl.gz"), _out(tmp_path, "b.jsonl.gz") previous = os.umask(0o077) @@ -985,7 +985,7 @@ def test_output_is_deterministic_and_0644_under_any_umask(tmp_path): os.umask(previous) assert first.read_bytes() == second.read_bytes() - assert stat.S_IMODE(first.stat().st_mode) == 0o644 + assert stat.S_IMODE(first.stat().st_mode) == 0o600 with gzip.open(first, "rb") as source: assert source.read(1) # No file name or mtime in the gzip header. diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index ea10836c8160..40a3034d90a1 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -64,7 +64,9 @@ This applies to all DGDR trace Runs, not only those the Planner creates. - **A public model.** The Run Job has no Hugging Face credentials and no model mount, so `retune.dgdr.model` must be a public Hugging Face repository id that AISimulate can resolve. - **A ReadWriteMany claim** in the DGD namespace, dedicated to traces. Create `segments/` and - `windows/` at its root, writable by the user the frontend and Planner containers run as. + `windows/` at its root, writable by the user the frontend and Planner containers run as. The + Planner writes each window file with mode `0600`, and the Run Job reads it as the planner image + user (UID 1000), so run the Planner as that user, which is its default. - **A KV-router frontend with default routing fields**, and workers that do not set their own router mode. See [Deployment rules](#deployment-rules). - **One Planner replica.** From 3ecbb634c1c6b7d9f5bf466f8c2532742e78f01f Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 30 Oct 2026 13:36:12 -0700 Subject: [PATCH 070/100] docs(planner): fix the live retune engine example and note zero credit The Planner configuration example pinned backend_version "0.11.0" for h200_sxm, which the pinned AISimulate does not accept for that GPU (UnlistedQueryVersionError), so every Run would fail. The example now leaves it out, and the engine section says when to set it. The limitations table now says that a frontend starting with overlap_score_credit 0 keeps it at 0: a live router rejects a positive credit then, so an omitted credit searches only 0 and a written positive value fails P4. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../pages/kubernetes/auto-deployment/planner-live-retune.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 40a3034d90a1..07845191d4c4 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -150,7 +150,6 @@ retune: engine: mode: aggregated backend: vllm - backend_version: "0.11.0" context_length: 32768 workers: aggregated: @@ -213,6 +212,9 @@ AISimulate searches by default, would make the Run search engine settings as wel sets `spec.hardware.gpu.budget` to their GPU total. - Write roles the mode does not use as `null`, or leave them out. - Set the model and GPU with `retune.dgdr.model` and `retune.dgdr.sku`, not in the engine. +- Leave `backend_version` out to let the pinned AISimulate pick its version for + `retune.dgdr.sku`. If you set it, it must be a version AISimulate accepts for that GPU; + otherwise every Run fails with `UnlistedQueryVersionError`. ### Time budget @@ -371,6 +373,7 @@ The Planner exports these on `metric_reporting_prometheus_port`: | Retune with load or throughput scaling, or of Planner parameters | A DGDR Run has no base Planner configuration, warm-up or predictor pre-sweep | | Frontends with non-default routing fields beyond the three knobs, or `kv_event_lag_ms` other than 0 | The DGDR router section expresses only the three knobs, and the Run does not pass a lag | | Workers that set their own router mode (`--router-mode` or `DYN_ROUTER_MODE`) | A worker's advertised router configuration replaces the frontend's for its workers, and the Run replays only the frontend's | +| Raising `overlap_score_credit` on a frontend that starts it at 0: an omitted credit then searches only 0, and a written positive value fails P4 | A live router rejects a positive credit after starting at 0 | | Private, fine-tuned or local-path models | The Run Job has no Hugging Face credentials or model mount | | Several frontends without router replica sync | AISimulate simulates one router | | EPP or Gateway API Inference Extension frontends, `frontendSidecar`, TLS frontends | No `/router_policy` there, different addressing, or certificates | From bdf026f517b68ef01aacf09876a321284906f1c2 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 30 Oct 2026 14:00:27 -0700 Subject: [PATCH 071/100] fix(operator): protect trace claims per Run and cancel Runs on claim deletion Give the trace PersistentVolumeClaim an operator-owned lifetime tied to the Runs that read it, and stop deriving Run outcomes from cached pod startup state or kubelet messages. - Before a Run creates its Job it patches its own finalizer, nvidia.com/dgdr-run-, onto the claim with an optimistic lock, so a stale cache or a claim being deleted never gets a Job. Patch errors are retryable; the API server reports a refused finalizer as Invalid, which would otherwise fail the Run permanently. - A claim with a deletionTimestamp, or one gone after the Run created its Job, cancels the Run whatever its pod has started: the Job is stopped through the foreground stop path shared with Request supersession (stopAndCompleteRun), and only after the API server confirms it is gone does the Run fail with TraceSourceDeleted. A Job that already has JobComplete or JobFailed decides the Run itself. - A completed Run stops any unfinished Job a stale pass created and then releases its finalizer after an uncached confirmation. A claim-keyed reconciler releases the finalizers of deleted Runs once no Job they control exists, so release converges after Run deletion, namespace deletion and operator restarts. - A missing claim keeps the Run open with TraceSourceMissing; a Block or Lost claim fails it with TraceSourceUnavailable through the durable permanent-failure marker, and only before the first Job. - Waiting containers are diagnostics only: the Run stays open with ContainerWaiting and the container's reason and message verbatim. The message-based trace mount classification, the scheduled/started inference, and the operator-initiated Job deletion are removed. - Grant patch/update on persistentvolumeclaims through the RBAC marker, pin the pod cache fields the diagnostic reads, verify the API server fencing in envtest, and document the lifecycle and manual finalizer removal. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../components/operator/files/role.yaml | 11 +- .../dynamographdeploymentrequest_types.go | 3 +- ...dia.com_dynamographdeploymentrequests.yaml | 3 +- .../nvidia.com_dynamographdeploymentruns.yaml | 3 +- .../dgdr_v1beta2_trace_clustertest_test.go | 121 +- ...mographdeploymentrun_v1beta2_controller.go | 468 +++--- ...mentrun_v1beta2_controller_envtest_test.go | 72 + ...phdeploymentrun_v1beta2_controller_test.go | 1261 +++++++++++------ ...ymentrun_v1beta2_trace_claim_controller.go | 172 +++ .../controller/pod_cache_contract_test.go | 40 + deploy/operator/internal/controller/setup.go | 10 +- .../additional-resources/api-reference-k8s.md | 2 +- .../dynamo-graph-deployment-request.mdx | 84 +- .../kubernetes-api/full-api-reference.mdx | 2 +- 14 files changed, 1562 insertions(+), 690 deletions(-) create mode 100644 deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_trace_claim_controller.go diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index f667b372eac5..09548c4a7cd8 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -11,6 +11,7 @@ rules: resources: - configmaps - events + - persistentvolumeclaims - services verbs: - create @@ -28,16 +29,6 @@ rules: - get - list - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - watch - apiGroups: - "" resources: diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index 742befb25a5f..e51a958f546f 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -168,7 +168,8 @@ const ( type TracePVCSource struct { // ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. // The Run Job mounts only the trace file from it, read-only, and only into the AISimulate - // container. + // container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` + // until its execution stops; deleting the claim cancels the Runs that read it. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 35e489f174b0..f0c3c0ece371 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9904,7 +9904,8 @@ spec: description: |- ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate - container. + container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` + until its execution stops; deleting the claim cancels the Runs that read it. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index e5c7afd8799a..c5cc3b66259c 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -363,7 +363,8 @@ spec: description: |- ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate - container. + container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` + until its execution stops; deleting the claim cancels the Runs that read it. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ diff --git a/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go b/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go index e8ed309447e1..7575f40d37f2 100644 --- a/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go +++ b/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go @@ -12,6 +12,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "testing" "time" @@ -24,6 +25,7 @@ import ( "github.com/stretchr/testify/require" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -284,42 +286,111 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2RefusesATraceSymlinkThatLeave defer runWatch.Stop() startClusterTestDGDRV1beta2Controllers(t, env, plannerImage, operatorImage) - t.Log("The kubelet refuses the subPath, so the Run fails before AISimulate reads anything") - var failed nvidiacomv1beta2.DynamoGraphDeploymentRun + t.Log("The kubelet refuses the subPath, so the sweeper waits and the open Run reports it verbatim") + var waiting nvidiacomv1beta2.DynamoGraphDeploymentRun + awaitClusterTestRun(t, runWatch, "the Run did not report its waiting sweeper", func(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { + waiting = *run.DeepCopy() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + return condition != nil && condition.Reason == dgdrRunReasonContainerWaiting + }) + condition := meta.FindStatusCondition(waiting.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + require.Equal(t, metav1.ConditionFalse, condition.Status) + require.NotNil(t, waiting.Status.JobRef) + require.Nil(t, waiting.Status.CompletionTime) + + t.Log("The sweeper never started: its pod reports the kubelet's waiting state, which the Run carries verbatim") + var pods corev1.PodList + require.NoError(t, env.Client().List(ctx, &pods, client.InNamespace(env.Namespace()), + client.MatchingLabels{batchv1.JobNameLabel: waiting.Status.JobRef.Name})) + require.Len(t, pods.Items, 1) + var sweeper *corev1.ContainerStatus + for i := range pods.Items[0].Status.ContainerStatuses { + if pods.Items[0].Status.ContainerStatuses[i].Name == dgdrRunSweeperContainer { + sweeper = &pods.Items[0].Status.ContainerStatuses[i] + } + } + require.NotNil(t, sweeper) + require.NotNil(t, sweeper.State.Waiting) + require.Nil(t, sweeper.State.Running) + require.Nil(t, sweeper.LastTerminationState.Terminated) + require.Zero(t, sweeper.RestartCount) + require.Contains(t, condition.Message, fmt.Sprintf("container %q is waiting: %s: ", dgdrRunSweeperContainer, sweeper.State.Waiting.Reason)) + + t.Log("The waiting container alone neither ends the Run nor makes the operator delete its Job") + require.Never(t, func() bool { + job := &batchv1.Job{} + err := env.Client().Get(ctx, client.ObjectKey{Namespace: env.Namespace(), Name: waiting.Status.JobRef.Name}, job) + run := &nvidiacomv1beta2.DynamoGraphDeploymentRun{} + require.NoError(t, env.Client().Get(ctx, client.ObjectKeyFromObject(&waiting), run)) + return err != nil || job.DeletionTimestamp != nil || + meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + }, 20*time.Second, time.Second) + + t.Log("The Run's finalizer protects the claim; deleting the claim cancels the Run") + claim := &corev1.PersistentVolumeClaim{} + require.NoError(t, env.Client().Get(ctx, client.ObjectKey{Namespace: env.Namespace(), Name: trace.claim}, claim)) + require.Contains(t, claim.Finalizers, dgdrRunTraceClaimFinalizer(waiting.UID)) + require.NoError(t, env.Client().Delete(ctx, claim)) + var canceled nvidiacomv1beta2.DynamoGraphDeploymentRun + awaitClusterTestRun(t, runWatch, "the Run was not canceled with its claim", func(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { + canceled = *run.DeepCopy() + return meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + }) + + t.Log("The Job and its pod were gone before the Run failed with TraceSourceDeleted") + var jobs batchv1.JobList + require.NoError(t, env.Client().List(ctx, &jobs, client.InNamespace(env.Namespace()))) + for i := range jobs.Items { + require.False(t, metav1.IsControlledBy(&jobs.Items[i], &canceled), "Job %q of the canceled Run still exists", jobs.Items[i].Name) + } + require.NoError(t, env.Client().List(ctx, &pods, client.InNamespace(env.Namespace()), + client.MatchingLabels{batchv1.JobNameLabel: waiting.Status.JobRef.Name})) + require.Empty(t, pods.Items) + condition = meta.FindStatusCondition(canceled.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + require.Equal(t, nvidiacomv1beta2.DGDRRunPhaseFailed, canceled.Status.Phase) + require.Equal(t, dgdrRunReasonTraceSourceDeleted, condition.Reason) + + t.Log("No Candidate was published, and the completed Run releases the claim") + var candidates nvidiacomv1beta2.DynamoGraphDeploymentCandidateList + require.NoError(t, env.Client().List(ctx, &candidates, client.InNamespace(env.Namespace()))) + require.Empty(t, candidates.Items) + require.Eventually(t, func() bool { + stored := &corev1.PersistentVolumeClaim{} + err := env.Client().Get(ctx, client.ObjectKeyFromObject(claim), stored) + if apierrors.IsNotFound(err) { + return true + } + require.NoError(t, err) + return !slices.Contains(stored.Finalizers, dgdrRunTraceClaimFinalizer(waiting.UID)) + }, time.Minute, time.Second, "the canceled Run did not release its claim") +} + +// awaitClusterTestRun reads Run events from runWatch until done accepts one, failing with message +// after five minutes. No Run may ever succeed: none of them may read a trace outside its claim. +func awaitClusterTestRun( + t *testing.T, + runWatch watch.Interface, + message string, + done func(*nvidiacomv1beta2.DynamoGraphDeploymentRun) bool, +) { + t.Helper() deadline := time.NewTimer(5 * time.Minute) defer deadline.Stop() - for failed.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed { + for { select { case event, ok := <-runWatch.ResultChan(): - require.True(t, ok, "Run watch closed before the Run failed") + require.True(t, ok, "Run watch closed: %s", message) require.NotEqual(t, watch.Error, event.Type, "Run watch failed") run, ok := event.Object.(*nvidiacomv1beta2.DynamoGraphDeploymentRun) require.Truef(t, ok, "Run watch returned %T", event.Object) require.NotEqual(t, nvidiacomv1beta2.DGDRRunPhaseSucceeded, run.Status.Phase, "Run read a trace outside its claim") - failed = *run.DeepCopy() + if done(run) { + return + } case <-deadline.C: - t.Fatal("the Run did not fail for its escaping trace symlink") + t.Fatal(message) } } - condition := meta.FindStatusCondition(failed.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) - require.NotNil(t, condition) - require.Equal(t, dgdrRunReasonTraceSourceUnavailable, condition.Reason) - require.Contains(t, condition.Message, `cannot be mounted: failed to prepare subPath for volumeMount "trace"`) - - t.Log("No Candidate was published, and the completed Run deletes its Job") - var candidates nvidiacomv1beta2.DynamoGraphDeploymentCandidateList - require.NoError(t, env.Client().List(ctx, &candidates, client.InNamespace(env.Namespace()))) - require.Empty(t, candidates.Items) - require.Eventually(t, func() bool { - var jobs batchv1.JobList - require.NoError(t, env.Client().List(ctx, &jobs, client.InNamespace(env.Namespace()))) - for i := range jobs.Items { - if metav1.IsControlledBy(&jobs.Items[i], &failed) { - return false - } - } - return true - }, time.Minute, time.Second, "the failed Run's Job was not deleted") } // startClusterTestDGDRV1beta2Controllers starts the production Request and Run controllers for env. diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index 040a3e3af9ab..d0c9b0a2fcdd 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -129,7 +129,7 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates/status,verbs=get;update;patch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;patch;update // +kubebuilder:rbac:groups="",resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch @@ -142,18 +142,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) Reconcile( return ctrl.Result{}, client.IgnoreNotFound(err) } if run.DeletionTimestamp != nil { + // The trace claim reconciler releases the claim once the Jobs of the deleted Run are gone. return ctrl.Result{}, nil } - - // A Run fails for its trace while its Job may still run, because the failure is recorded before - // the Job is deleted: a failed or stale write then leaves the Job in place rather than gone and - // recreated. Deleting it is the one step left for such a completed Run. - if completed := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted); completed != nil && - completed.Status == metav1.ConditionTrue { - if completed.Reason != dgdrRunReasonTraceSourceUnavailable { - return ctrl.Result{}, nil - } - return ctrl.Result{}, r.deleteUnfinishedJobs(ctx, run) + if meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { + return r.retireCompletedRun(ctx, run) } if condition := dgdrRunPendingPermanentFailure(run); condition != nil { @@ -178,7 +171,8 @@ func dgdrRunPendingPermanentFailure(run *nvidiacomv1beta2.DynamoGraphDeploymentR return nil } switch condition.Reason { - case dgdrRunReasonInvalidExecution, dgdrRunReasonInvalidResource, commoncontroller.EventReasonOwnershipConflict: + case dgdrRunReasonInvalidExecution, dgdrRunReasonInvalidResource, commoncontroller.EventReasonOwnershipConflict, + dgdrRunReasonTraceSourceUnavailable: return condition default: return nil @@ -200,7 +194,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) beginPermanentFailure( ObservedGeneration: run.Generation, }) // Like every Run status write, the marker carries the resourceVersion the cache showed, so a - // stale Run cannot overwrite a newer status, such as a recorded trace failure. + // stale Run cannot overwrite a newer status, such as a terminal cancellation. if !apiequality.Semantic.DeepEqual(original.Status, run.Status) { if err := r.Status().Patch(ctx, run, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return ctrl.Result{}, &dgdrRunRetryableError{cause: fmt.Errorf("record permanent failure for Run %q: %w", run.Name, err)} @@ -221,16 +215,13 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) finishPermanentFailure( if err != nil { return ctrl.Result{}, err } - stopped, err := r.stopOwnedJobs(ctx, run, jobs, "after permanent failure") + completed, err := r.stopAndCompleteRun(ctx, run, jobs, "after permanent failure", reason, cause.Error()) if err != nil { return ctrl.Result{}, err } - if !stopped { + if !completed { return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil } - if err := r.completeRun(ctx, run, reason, cause.Error()); err != nil { - return ctrl.Result{}, err - } return ctrl.Result{}, reconcile.TerminalError(cause) } @@ -252,44 +243,43 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRun( job = &jobs[0] } if supersedingGeneration, cancelRequested := dgdrRunSupersedingGeneration(run); cancelRequested { - return r.cancelSupersededRun(ctx, run, job, supersedingGeneration) - } - - // A Job that can never mount its trace would keep the Run active forever, so a Run that has not - // created its Job checks the claim before anything is created for it. A recorded jobRef, or a Job - // the API server already holds, means the Job was created even if the cache does not show it yet; - // it is checked below once the cache shows it. - resourceName := dgdrRunResourceName(run.Name) - if job == nil && run.Status.JobRef == nil { - unavailable, err := r.traceSourceUnavailable(ctx, run, false) - if err != nil { + completed, err := r.stopAndCompleteRun(ctx, run, jobs, "for supersession", dgdrRunReasonSuperseded, + fmt.Sprintf("Run was superseded by Request generation %s", supersedingGeneration)) + if err != nil || completed { return ctrl.Result{}, err } - if unavailable != "" { - created, err := r.runJobExists(ctx, run, resourceName) - if err != nil { - return ctrl.Result{}, err - } - if !created { - return ctrl.Result{}, r.completeRun(ctx, run, dgdrRunReasonTraceSourceUnavailable, unavailable) - } - } + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil } - // A Job whose sweeper can never read the trace would stay pending with nothing to end it: the - // Run fails, and the completed Run then deletes the Job. Claim and pod events wake the Run for - // this check. - if job != nil && dgdrRunTraceClaim(run) != nil && !dgdrRunJobFinished(job) { - unavailable, err := r.jobTraceUnavailable(ctx, run, job) + // The trace claim decides whether the Run may execute, before anything is created for it. + var claim *corev1.PersistentVolumeClaim + if source := dgdrRunTraceClaim(run); source != nil { + claim, err = r.traceClaim(ctx, run, source) if err != nil { return ctrl.Result{}, err } - if unavailable != "" { - return ctrl.Result{}, r.completeRun(ctx, run, dgdrRunReasonTraceSourceUnavailable, - fmt.Sprintf("%s; Job %q is deleted", unavailable, job.Name)) + + // Deleting the claim cancels the Run whatever its pod has started, unless its Job already + // finished: the Job's own outcome then decides the Run. A claim that is gone after the Run + // created its Job was protected by the Run, so it was deleted, too. Before the first Job, a + // missing claim keeps the Run open, and only a claim no Job could ever mount fails it. + executed := job != nil || run.Status.JobRef != nil + switch { + case (claim != nil && claim.DeletionTimestamp != nil) || (claim == nil && executed): + if job == nil || !dgdrRunJobFinished(job) { + return r.cancelForDeletedTraceClaim(ctx, run, jobs, source) + } + case claim == nil: + return ctrl.Result{}, r.reportMissingTraceClaim(ctx, run, source) + case !executed: + if unusable := dgdrRunUnusableTraceClaim(claim); unusable != "" { + return r.beginPermanentFailure(ctx, run, dgdrRunReasonTraceSourceUnavailable, fmt.Errorf( + "trace PersistentVolumeClaim %q in namespace %q %s", source.ClaimName, run.Namespace, unusable)) + } } } + resourceName := dgdrRunResourceName(run.Name) if err := r.reconcilePublisherAccess(ctx, run, resourceName); err != nil { return ctrl.Result{}, err } @@ -301,6 +291,13 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRun( return ctrl.Result{}, err } + // The Run protects its trace claim before it creates the Job and while the Job runs. claim is + // nil here for a Run without a trace, or for one whose claim is gone after its Job finished. + if claim != nil { + if err := r.protectTraceClaim(ctx, run, claim, job); err != nil { + return ctrl.Result{}, err + } + } if job == nil { job = r.buildJob(run, resourceName, execution) if err := controllerutil.SetControllerReference(run, job, r.Scheme()); err != nil { @@ -329,26 +326,24 @@ func dgdrRunSupersedingGeneration(run *nvidiacomv1beta2.DynamoGraphDeploymentRun return cancelGenerationValue, err == nil && cancelGeneration > runGeneration } -func (r *DynamoGraphDeploymentRunV1beta2Reconciler) cancelSupersededRun( +// stopAndCompleteRun stops every execution of run and, once the API server confirms that none is +// left, completes run as Failed with reason and message. It reports whether run was completed; +// otherwise the caller retries after the stop. jobs are the Jobs of run the cache shows. run must +// not be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) stopAndCompleteRun( ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, - job *batchv1.Job, - supersedingGeneration string, -) (ctrl.Result, error) { - var cached []batchv1.Job - if job != nil { - cached = []batchv1.Job{*job} - } - stopped, err := r.stopOwnedJobs(ctx, run, cached, "for supersession") - if err != nil { - return ctrl.Result{}, err + jobs []batchv1.Job, + action, reason, message string, +) (bool, error) { + stopped, err := r.stopOwnedJobs(ctx, run, jobs, action) + if err != nil || !stopped { + return false, err } - if !stopped { - return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil + if err := r.completeRun(ctx, run, reason, message); err != nil { + return false, err } - - return ctrl.Result{}, r.completeRun(ctx, run, dgdrRunReasonSuperseded, - fmt.Sprintf("Run was superseded by Request generation %s", supersedingGeneration)) + return true, nil } // stopOwnedJobs stops every execution visible through either the informer cache or the API @@ -439,70 +434,192 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) ownedJobs( const ( dgdrRunReasonTraceSourceUnavailable = "TraceSourceUnavailable" - dgdrRunClaimProtectionFinalizer = "kubernetes.io/pvc-protection" + dgdrRunReasonTraceSourceMissing = "TraceSourceMissing" + dgdrRunReasonTraceSourceDeleted = "TraceSourceDeleted" + dgdrRunReasonContainerWaiting = "ContainerWaiting" + // dgdrRunTraceClaimFinalizerPrefix and the Run UID name the finalizer by which one Run protects + // its trace claim: nvidia.com/dgdr-run-. + dgdrRunTraceClaimFinalizerPrefix = "nvidia.com/dgdr-run-" ) -// traceSourceUnavailable returns why a Job could not mount the Run's trace claim, or "" when it -// can or the Run replays no trace. scheduled tells whether a pod of the Job is already bound to a -// node: the kubelet still mounts a claim being deleted while the pvc-protection finalizer keeps it -// for that pod. The answer fails the Run terminally, so the claim is read through the uncached API -// reader: claim events only wake the Run, and a claim the cache has not observed yet, such as one -// created just before the Run, must not end it. A Pending claim is accepted: a WaitForFirstConsumer -// claim binds only once its first pod is scheduled. Access modes are not checked: whether another -// pod holds a ReadWriteOnce or ReadWriteOncePod claim is a scheduling question, not a terminal one. -// Other read errors are returned for a retry. run must not be nil. -func (r *DynamoGraphDeploymentRunV1beta2Reconciler) traceSourceUnavailable( +// dgdrRunTraceClaimFinalizer returns the finalizer by which the Run with uid protects its trace +// claim. Each Run holds its own, so Runs that share a claim never release each other's protection. +func dgdrRunTraceClaimFinalizer(uid types.UID) string { + return dgdrRunTraceClaimFinalizerPrefix + string(uid) +} + +// traceClaim returns the claim source names, as the cache shows it, or nil when the cache shows no +// such claim. A claim that has not appeared yet keeps the Run pending, and the claim's events wake +// the Run, so the cache answers. run and source must not be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) traceClaim( ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, - scheduled bool, -) (string, error) { - source := dgdrRunTraceClaim(run) - if source == nil { - return "", nil - } - name := source.ClaimName + source *nvidiacomv1beta2.TracePVCSource, +) (*corev1.PersistentVolumeClaim, error) { claim := &corev1.PersistentVolumeClaim{} - err := r.APIReader.Get(ctx, types.NamespacedName{Namespace: run.Namespace, Name: name}, claim) + err := r.Get(ctx, types.NamespacedName{Namespace: run.Namespace, Name: source.ClaimName}, claim) if apierrors.IsNotFound(err) { - return fmt.Sprintf("trace PersistentVolumeClaim %q does not exist in namespace %q", name, run.Namespace), nil + return nil, nil } if err != nil { - return "", fmt.Errorf("get trace PersistentVolumeClaim %q in namespace %q: %w", name, run.Namespace, err) + return nil, fmt.Errorf("get trace PersistentVolumeClaim %q in namespace %q: %w", source.ClaimName, run.Namespace, err) } + return claim, nil +} - // Each state below keeps the Job from ever mounting the claim read-only as a filesystem. - reason := "" +// dgdrRunUnusableTraceClaim returns why a Job could never mount claim read-only as a filesystem, or +// "" when it can. Only structured fields decide: a raw block claim, or a claim that lost its +// volume. A Pending claim is usable: a WaitForFirstConsumer claim binds only once its first pod is +// scheduled. Access modes are not checked: whether another pod holds a ReadWriteOnce or +// ReadWriteOncePod claim is a scheduling question. claim must not be nil. +func dgdrRunUnusableTraceClaim(claim *corev1.PersistentVolumeClaim) string { switch { - case claim.DeletionTimestamp != nil && !(scheduled && controllerutil.ContainsFinalizer(claim, dgdrRunClaimProtectionFinalizer)): - reason = "is being deleted" case claim.Spec.VolumeMode != nil && *claim.Spec.VolumeMode == corev1.PersistentVolumeBlock: - reason = "is a raw block volume, not a filesystem" + return "is a raw block volume, not a filesystem" case claim.Status.Phase == corev1.ClaimLost: - reason = "has lost its PersistentVolume" + return "has lost its PersistentVolume" default: - return "", nil + return "" } - return fmt.Sprintf("trace PersistentVolumeClaim %q in namespace %q %s", name, run.Namespace, reason), nil } -// runJobExists reports whether the API server holds the Job this Run controls under name. Creating -// the Job and recording its jobRef are separate writes, so a Job can exist while both the cache and -// status.jobRef miss it. The Job is therefore read through the uncached API reader, and only right -// before the Run would be failed terminally for its trace claim. -func (r *DynamoGraphDeploymentRunV1beta2Reconciler) runJobExists( +// reportMissingTraceClaim keeps run open while its trace claim does not exist. The claim's creation +// wakes the Run, which then starts. run and source must not be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reportMissingTraceClaim( ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, - name string, -) (bool, error) { - job := &batchv1.Job{} - err := r.APIReader.Get(ctx, types.NamespacedName{Namespace: run.Namespace, Name: name}, job) - if apierrors.IsNotFound(err) { - return false, nil + source *nvidiacomv1beta2.TracePVCSource, +) error { + original := run.DeepCopy() + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhasePending + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, + Status: metav1.ConditionFalse, + Reason: dgdrRunReasonTraceSourceMissing, + Message: fmt.Sprintf("trace PersistentVolumeClaim %q does not exist in namespace %q; the Run starts once it is created", + source.ClaimName, run.Namespace), + ObservedGeneration: run.Generation, + }) + if apiequality.Semantic.DeepEqual(original.Status, run.Status) { + return nil + } + if err := r.Status().Patch(ctx, run, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { + return &dgdrRunRetryableError{cause: fmt.Errorf("report the missing trace claim of Run %q: %w", run.Name, err)} + } + return nil +} + +// protectTraceClaim adds the finalizer of run to claim, which the cache shows not being deleted. +// Before the Run creates its Job, job is nil and the finalizer is patched even when the cache shows +// it already: the patch carries the resourceVersion the cache showed, so the API server fails it +// when the cached claim is stale, such as when the cache has not shown the claim's deletion yet, and +// a successful patch confirms that the claim is not being deleted. The API server also refuses a new +// finalizer on a claim being deleted. Every failure is retryable: the refused finalizer is reported +// as Invalid, which would otherwise fail the Run permanently, and the next pass reads the claim again +// and cancels the Run if the claim is being deleted. While job runs, a finalizer removed by hand is +// restored; a finished job needs no protection. claim is updated to the stored object. run and claim +// must not be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) protectTraceClaim( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + claim *corev1.PersistentVolumeClaim, + job *batchv1.Job, +) error { + if job != nil && (dgdrRunJobFinished(job) || controllerutil.ContainsFinalizer(claim, dgdrRunTraceClaimFinalizer(run.UID))) { + return nil + } + original := claim.DeepCopy() + controllerutil.AddFinalizer(claim, dgdrRunTraceClaimFinalizer(run.UID)) + if err := r.Patch(ctx, claim, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { + return &dgdrRunRetryableError{cause: fmt.Errorf("protect trace PersistentVolumeClaim %q for Run %q: %w", claim.Name, run.Name, err)} + } + return nil +} + +// cancelForDeletedTraceClaim stops the execution of run because its trace claim is being deleted or +// is gone, and then fails run with TraceSourceDeleted. jobs are the Jobs of run the cache shows. +// When the cache shows none, a Job the API server already holds is stopped as well, unless it +// finished: its own outcome decides the Run once the cache shows it. The Run keeps its finalizer on +// the claim until it completes. run and source must not be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) cancelForDeletedTraceClaim( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, + jobs []batchv1.Job, + source *nvidiacomv1beta2.TracePVCSource, +) (ctrl.Result, error) { + if len(jobs) == 0 { + live, err := r.uncachedOwnedJobs(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + for i := range live { + if dgdrRunJobFinished(&live[i]) { + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil + } + } + jobs = live + } + + completed, err := r.stopAndCompleteRun(ctx, run, jobs, "for its deleted trace claim", dgdrRunReasonTraceSourceDeleted, + fmt.Sprintf("trace PersistentVolumeClaim %q in namespace %q was deleted, which cancels the Run", source.ClaimName, run.Namespace)) + if err != nil || completed { + return ctrl.Result{}, err } + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil +} + +// retireCompletedRun keeps a completed Run free of live executions and then releases its trace +// claim. Completed=True means that no execution of the Run is left, but a stale pass can still +// create a Job after the Run completed; such an unfinished Job is stopped. Releasing the claim is an +// absence decision the cache cannot make alone, because the Job cache can miss a Job that was just +// created, so the API server confirms that no unfinished Job of the Run is left first. run must not +// be nil. +func (r *DynamoGraphDeploymentRunV1beta2Reconciler) retireCompletedRun( + ctx context.Context, + run *nvidiacomv1beta2.DynamoGraphDeploymentRun, +) (ctrl.Result, error) { + jobs, err := r.ownedJobs(ctx, run) + if err != nil { + return ctrl.Result{}, err + } + unfinished := make([]batchv1.Job, 0, len(jobs)) + for i := range jobs { + if !dgdrRunJobFinished(&jobs[i]) { + unfinished = append(unfinished, jobs[i]) + } + } + if len(unfinished) > 0 { + if _, err := r.stopOwnedJobs(ctx, run, unfinished, "of a completed Run"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil + } + + // Only a trace claim that still carries the Run's finalizer needs the uncached confirmation. + source := dgdrRunTraceClaim(run) + if source == nil { + return ctrl.Result{}, nil + } + claim, err := r.traceClaim(ctx, run, source) + if err != nil || claim == nil || !controllerutil.ContainsFinalizer(claim, dgdrRunTraceClaimFinalizer(run.UID)) { + return ctrl.Result{}, err + } + live, err := r.uncachedOwnedJobs(ctx, run) if err != nil { - return false, fmt.Errorf("get Job %q of Run %q: %w", name, run.Name, err) + return ctrl.Result{}, err + } + for i := range live { + if !dgdrRunJobFinished(&live[i]) { + return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil + } } - return metav1.IsControlledBy(job, run), nil + + original := claim.DeepCopy() + controllerutil.RemoveFinalizer(claim, dgdrRunTraceClaimFinalizer(run.UID)) + if err := r.Patch(ctx, claim, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{}, fmt.Errorf("release trace PersistentVolumeClaim %q of Run %q: %w", claim.Name, run.Name, err) + } + return ctrl.Result{}, nil } // dgdrRunTraceClaim returns the claim the Run's trace is read from, or nil when it replays no trace. @@ -542,18 +659,11 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) runsForTraceClaim(ctx contex return requests } -// dgdrRunJobFinished reports whether job reports a finished pod or a terminal or interim terminal -// condition, so its outcome no longer depends on the trace. The Job runs one pod, because its -// backoffLimit is 0. job must not be nil. +// dgdrRunJobFinished reports whether job has a terminal JobComplete or JobFailed condition. The Job +// controller adds either only after the Job's pod terminated, so a finished Job no longer uses the +// trace claim and its outcome is final. job must not be nil. func dgdrRunJobFinished(job *batchv1.Job) bool { - uncounted := job.Status.UncountedTerminatedPods - if job.Status.Succeeded > 0 || job.Status.Failed > 0 || - (uncounted != nil && len(uncounted.Succeeded)+len(uncounted.Failed) > 0) { - return true - } - for _, conditionType := range []batchv1.JobConditionType{ - batchv1.JobComplete, batchv1.JobFailed, batchv1.JobSuccessCriteriaMet, batchv1.JobFailureTarget, - } { + for _, conditionType := range []batchv1.JobConditionType{batchv1.JobComplete, batchv1.JobFailed} { if condition := dgdrRunJobCondition(job, conditionType); condition != nil && condition.Status == corev1.ConditionTrue { return true } @@ -561,52 +671,6 @@ func dgdrRunJobFinished(job *batchv1.Job) bool { return false } -// jobTraceUnavailable returns why job, which has not finished, can never read the Run's trace, or -// "" when it can or still may. -// -// A sweeper that cannot mount the trace file waits in CreateContainerConfigError until the pod is -// deleted, so that is reported whatever the claim does. The claim itself matters until the sweeper -// has started. Before its pod is scheduled, the pod cannot get a claim that is unusable. A scheduled -// pod still cannot mount a claim that is gone, lost or a block volume, or one being deleted without -// pvc-protection, as when the claim goes while the pod is bound; pvc-protection keeps a claim being -// deleted for the pods bound to it, and the kubelet mounts it. While the cache shows fewer pods than -// the Job counts as active, whether one is scheduled or started is unknown and nothing is reported; -// the pod's event wakes the Run again. The pod cache can lag a binding by a moment, and a claim -// deleted in that moment fails the Run although its pod just got the claim. run and job must not -// be nil. -func (r *DynamoGraphDeploymentRunV1beta2Reconciler) jobTraceUnavailable( - ctx context.Context, - run *nvidiacomv1beta2.DynamoGraphDeploymentRun, - job *batchv1.Job, -) (string, error) { - pods, err := r.jobPods(ctx, job) - if err != nil { - return "", err - } - - // A trace file the sweeper cannot mount, whether or not the claim is usable. - source := dgdrRunTraceClaim(run) - for i := range pods { - if message := dgdrRunTraceMountFailure(&pods[i]); message != "" { - return fmt.Sprintf("trace file %q on PersistentVolumeClaim %q cannot be mounted: %s", - source.Path, source.ClaimName, message), nil - } - } - - // The claim, until the sweeper has started. - if ptr.Deref(job.Status.Ready, 0) > 0 || int(job.Status.Active) > len(pods) { - return "", nil - } - scheduled := false - for i := range pods { - if dgdrRunSweeperStarted(&pods[i]) { - return "", nil - } - scheduled = scheduled || pods[i].Spec.NodeName != "" - } - return r.traceSourceUnavailable(ctx, run, scheduled) -} - // jobPods returns the pods job controls, as the cache shows them. job must not be nil. func (r *DynamoGraphDeploymentRunV1beta2Reconciler) jobPods(ctx context.Context, job *batchv1.Job) ([]corev1.Pod, error) { list := &corev1.PodList{} @@ -622,52 +686,28 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) jobPods(ctx context.Context, return pods, nil } -// dgdrRunSweeperStarted reports whether pod's sweeper container has been created and started, so it -// mounted the trace. The pod cache drops a running state, so a reported sweeper that is not waiting -// has started. pod must not be nil. -func dgdrRunSweeperStarted(pod *corev1.Pod) bool { - for _, status := range pod.Status.ContainerStatuses { - if status.Name == dgdrRunSweeperContainer && - (status.State.Waiting == nil || status.LastTerminationState.Terminated != nil) { - return true - } - } - return false -} - -// dgdrRunTraceMountFailure returns the kubelet's message when pod's sweeper cannot start because -// the trace file cannot be mounted, such as a missing file on a read-only claim or a symlink that -// leaves the claim, or "" otherwise. pod must not be nil. -func dgdrRunTraceMountFailure(pod *corev1.Pod) string { - for _, status := range pod.Status.ContainerStatuses { - waiting := status.State.Waiting - if status.Name == dgdrRunSweeperContainer && waiting != nil && waiting.Reason == "CreateContainerConfigError" && - strings.Contains(waiting.Message, fmt.Sprintf("volumeMount %q", dgdrRunTraceVolume)) { - return waiting.Message - } - } - return "" -} - -// deleteUnfinishedJobs deletes the Jobs run controls that have not finished, with their pods. run -// failed because those Jobs could never read its trace. The UID precondition keeps a Job created -// under the same name since the cache showed it. run must not be nil. -func (r *DynamoGraphDeploymentRunV1beta2Reconciler) deleteUnfinishedJobs(ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun) error { - jobs, err := r.ownedJobs(ctx, run) - if err != nil { - return err - } - for i := range jobs { - job := &jobs[i] - if dgdrRunJobFinished(job) { - continue - } - err := r.Delete(ctx, job, client.PropagationPolicy(metav1.DeletePropagationBackground), client.Preconditions{UID: &job.UID}) - if client.IgnoreNotFound(err) != nil { - return fmt.Errorf("delete Job %q: %w", job.Name, err) +// dgdrRunWaitingContainer returns the first sweeper, then publisher, container of pods that waits +// for a reason other than its ordinary start, such as a mount, configuration or image error. The +// result is only a diagnostic: a waiting container may start once its cause is fixed, so it never +// ends the Run. Only the container name and its waiting reason and message are read, which the +// operator's pod cache keeps. +func dgdrRunWaitingContainer(pods []corev1.Pod) (string, corev1.ContainerStateWaiting, bool) { + for _, name := range []string{dgdrRunSweeperContainer, dgdrRunPublisherContainer} { + for i := range pods { + for _, status := range pods[i].Status.ContainerStatuses { + waiting := status.State.Waiting + if status.Name != name || waiting == nil { + continue + } + switch waiting.Reason { + case "", "ContainerCreating", "PodInitializing": + continue + } + return name, *waiting, true + } } } - return nil + return "", corev1.ContainerStateWaiting{}, false } // dgdrRunJobPod reports whether obj is a pod of a Job named like a Run's AISimulate Job. @@ -1616,6 +1656,20 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRunStatus( reason, message = dgdrRunReasonRunning, dgdrRunMessageRunning } + // A container of the unfinished Job that waits on an error is reported verbatim. It keeps the + // Run open: the container may start once its cause is fixed, and only the Job's conditions or an + // explicit cancellation end the Run. + if completed == metav1.ConditionFalse { + pods, err := r.jobPods(ctx, job) + if err != nil { + return err + } + if container, waiting, found := dgdrRunWaitingContainer(pods); found { + reason = dgdrRunReasonContainerWaiting + message = fmt.Sprintf("container %q is waiting: %s: %s", container, waiting.Reason, waiting.Message) + } + } + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ Type: nvidiacomv1beta2.DGDRRunConditionCompleted, Status: completed, @@ -1624,8 +1678,8 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRunStatus( ObservedGeneration: run.Generation, }) - // A stale Run must not overwrite a newer status, such as a trace failure recorded while the Job - // was still pending, so the write carries the resourceVersion the cache showed. + // A stale Run must not overwrite a newer status, such as a terminal cancellation, so the write + // carries the resourceVersion the cache showed. if apiequality.Semantic.DeepEqual(original.Status, run.Status) { return nil } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go index c80688bbec30..d44552c82a05 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_envtest_test.go @@ -8,15 +8,19 @@ package controller import ( + "context" "errors" "testing" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/envtest" ) @@ -117,3 +121,71 @@ func roleAllowsRunFinalizerUpdate(rules []rbacv1.PolicyRule, runName string) boo } return false } + +func TestDGDRRunTraceClaimProtectionIsFencedByTheAPIServer(t *testing.T) { + env := sharedEnv.ForTest(t) + admin := env.Client() + ctx := t.Context() + reconciler := &DynamoGraphDeploymentRunV1beta2Reconciler{Client: admin} + first := &nvidiacomv1beta2.DynamoGraphDeploymentRun{Name: "first", UID: "first-uid"} + second := &nvidiacomv1beta2.DynamoGraphDeploymentRun{Name: "second", UID: "second-uid"} + + t.Log("Create a trace claim that another finalizer keeps while it is deleted") + claim := &corev1.PersistentVolumeClaim{ + Name: "request-traces", Namespace: env.Namespace(), Finalizers: []string{"example.com/hold"}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Mi")}}, + }, + } + if err := admin.Create(ctx, claim); err != nil { + t.Fatalf("create claim: %v", err) + } + t.Cleanup(func() { + stored := &corev1.PersistentVolumeClaim{} + if err := admin.Get(context.Background(), client.ObjectKeyFromObject(claim), stored); err != nil { + return + } + stored.Finalizers = nil + if err := admin.Update(context.Background(), stored); client.IgnoreNotFound(err) != nil { + t.Errorf("release claim: %v", err) + } + }) + stale := claim.DeepCopy() + + t.Log("The first Run protects the claim") + if err := reconciler.protectTraceClaim(ctx, first, claim, nil); err != nil { + t.Fatalf("protect claim: %v", err) + } + if !controllerutil.ContainsFinalizer(claim, dgdrRunTraceClaimFinalizer(first.UID)) { + t.Fatalf("finalizers = %v, want the first Run's", claim.Finalizers) + } + protected := claim.DeepCopy() + + t.Log("A protection read from a stale claim conflicts and is retried") + err := reconciler.protectTraceClaim(ctx, second, stale, nil) + if !apierrors.IsConflict(err) || dgdrRunIsPermanentError(err) { + t.Fatalf("stale protection error = %v, want a retryable conflict", err) + } + + t.Log("Delete the claim") + if err := admin.Delete(ctx, claim); err != nil { + t.Fatalf("delete claim: %v", err) + } + + t.Log("A protected claim whose deletion the cache has not shown fails the fence before a Job is created") + err = reconciler.protectTraceClaim(ctx, first, protected, nil) + if !apierrors.IsConflict(err) || dgdrRunIsPermanentError(err) { + t.Fatalf("fence error for a stale protected claim = %v, want a retryable conflict", err) + } + + t.Log("The API server refuses a new finalizer on the claim being deleted, and the refusal is retried") + deleting := &corev1.PersistentVolumeClaim{} + if err := admin.Get(ctx, client.ObjectKeyFromObject(claim), deleting); err != nil { + t.Fatalf("get claim: %v", err) + } + err = reconciler.protectTraceClaim(ctx, second, deleting, nil) + if !apierrors.IsInvalid(err) || dgdrRunIsPermanentError(err) { + t.Fatalf("protection error for a claim being deleted = %v, want a retryable Invalid", err) + } +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 33e0ab666fc5..4d090a5af166 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "reflect" + "slices" "strings" "testing" "time" @@ -28,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" @@ -35,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/yaml" ) @@ -1960,24 +1963,10 @@ func TestDGDRRunInputRebuildsANonObjectTraceSourceParent(t *testing.T) { func TestV1beta2RunControllerFailsARunWhoseTraceClaimCannotBeMounted(t *testing.T) { tests := []struct { - name string - missing bool - mutate func(*corev1.PersistentVolumeClaim) - reason string + name string + mutate func(*corev1.PersistentVolumeClaim) + reason string }{ - { - name: "missing claim", - missing: true, - reason: `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`, - }, - { - name: "claim being deleted", - mutate: func(claim *corev1.PersistentVolumeClaim) { - claim.DeletionTimestamp = new(metav1.Now()) - claim.Finalizers = []string{"kubernetes.io/pvc-protection"} - }, - reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is being deleted`, - }, { name: "raw block claim", mutate: func(claim *corev1.PersistentVolumeClaim) { @@ -1995,17 +1984,19 @@ func TestV1beta2RunControllerFailsARunWhoseTraceClaimCannotBeMounted(t *testing. } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - objects := []client.Object{dgdrRunWithTraceForTest(t, map[string]any{})} - if !tt.missing { - claim := traceClaimForTest() - tt.mutate(claim) - objects = append(objects, claim) - } - f := newDGDRRunFixture(t, objects...) + claim := traceClaimForTest() + tt.mutate(claim) + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + request := ctrl.Request{Namespace: "default", Name: "search-1"} - t.Log("Reconcile twice: the first pass fails the Run, the second sees it completed") - f.reconcile() - f.reconcile() + t.Log("The first pass records the failure marker, the second confirms no Job exists and fails the Run") + result, err := f.reconciler.Reconcile(t.Context(), request) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("marker result/error = %+v/%v, want a durable marker and a retry", result, err) + } + if _, err := f.reconciler.Reconcile(t.Context(), request); !errors.Is(err, reconcile.TerminalError(nil)) { + t.Fatalf("reconcile error = %v, want a terminal failure", err) + } t.Log("The Run is Failed and Completed with the claim named in the message") run := f.run() @@ -2022,12 +2013,61 @@ func TestV1beta2RunControllerFailsARunWhoseTraceClaimCannotBeMounted(t *testing. t.Fatalf("condition observedGeneration = %d, want %d", condition.ObservedGeneration, run.Generation) } - t.Log("Nothing was created for the Run") + t.Log("Nothing was created for the Run, and the claim was never protected") assertNoDGDRRunResources(t, f.client) + if stored, found := f.claim(); !found || len(stored.Finalizers) != 0 { + t.Fatalf("claim = %+v (found %t), want it without finalizers", stored, found) + } }) } } +func TestV1beta2RunControllerKeepsARunOpenUntilItsTraceClaimExists(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{})) + + t.Log("Without the claim the Run stays open with a diagnostic, and nothing is created for it") + f.reconcile() + f.reconcile() + run := f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhasePending || condition == nil || + condition.Status != metav1.ConditionFalse || condition.Reason != dgdrRunReasonTraceSourceMissing { + t.Fatalf("status = %+v, want Pending and Completed=False/%s", run.Status, dgdrRunReasonTraceSourceMissing) + } + want := `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"; the Run starts once it is created` + if condition.Message != want || run.Status.CompletionTime != nil { + t.Fatalf("condition message = %q, completionTime = %v; want %q and no completion", condition.Message, run.Status.CompletionTime, want) + } + assertNoDGDRRunResources(t, f.client) + + t.Log("The claim's creation wakes the Run, which protects the claim and starts") + if err := f.client.Create(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("create claim: %v", err) + } + if got := f.reconciler.runsForTraceClaim(t.Context(), traceClaimForTest()); len(got) != 1 || got[0].Name != "search-1" { + t.Fatalf("claim event requests = %v, want the Run", got) + } + f.reconcile() + assertDGDRRunPendingStatus(t, f.run(), f.job().Name) + assertDGDRRunTraceClaimFinalizers(t, f, dgdrRunTraceClaimFinalizer("run-uid")) +} + +func TestV1beta2RunControllerCreatesNothingForATraceClaimBeingDeleted(t *testing.T) { + claim := traceClaimForTest() + claim.DeletionTimestamp = new(metav1.Now()) + claim.Finalizers = []string{"kubernetes.io/pvc-protection"} + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + + t.Log("Deleting the claim before the first Job cancels the Run like a deletion after it") + f.reconcile() + f.reconcile() + assertDGDRRunCanceledForTrace(t, f.run(), "") + + t.Log("Nothing was created, and no finalizer was added to the claim being deleted") + assertNoDGDRRunResources(t, f.client) + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection") +} + func assertNoDGDRRunResources(t *testing.T, c client.Client) { t.Helper() for _, list := range []client.ObjectList{ @@ -2043,6 +2083,43 @@ func assertNoDGDRRunResources(t *testing.T, c client.Client) { } } +// claim returns the stored trace claim and whether it exists. +func (f *dgdrRunFixture) claim() (*corev1.PersistentVolumeClaim, bool) { + f.t.Helper() + claim := &corev1.PersistentVolumeClaim{} + err := f.client.Get(f.t.Context(), types.NamespacedName{Namespace: "default", Name: "request-traces"}, claim) + if apierrors.IsNotFound(err) { + return nil, false + } + if err != nil { + f.t.Fatalf("get claim: %v", err) + } + return claim, true +} + +// assertDGDRRunTraceClaimFinalizers checks that the stored trace claim carries exactly want. +func assertDGDRRunTraceClaimFinalizers(t *testing.T, f *dgdrRunFixture, want ...string) { + t.Helper() + claim, found := f.claim() + if !found { + t.Fatalf("claim is gone, want finalizers %v", want) + } + if !slices.Equal(claim.Finalizers, want) { + t.Fatalf("claim finalizers = %v, want %v", claim.Finalizers, want) + } +} + +func TestDGDRRunTraceClaimFinalizerIsAQualifiedName(t *testing.T) { + t.Log("A Run UID completes the finalizer to a qualified name within the length limits") + finalizer := dgdrRunTraceClaimFinalizer("0f8fad5b-d9cb-469f-a165-70867728950e") + if finalizer != "nvidia.com/dgdr-run-0f8fad5b-d9cb-469f-a165-70867728950e" { + t.Fatalf("finalizer = %q", finalizer) + } + if errs := validation.IsQualifiedName(finalizer); len(errs) != 0 { + t.Fatalf("finalizer %q is not a qualified name: %v", finalizer, errs) + } +} + func TestV1beta2RunControllerAcceptsAnUnboundOrSharedTraceClaim(t *testing.T) { tests := []struct { name string @@ -2060,6 +2137,9 @@ func TestV1beta2RunControllerAcceptsAnUnboundOrSharedTraceClaim(t *testing.T) { {name: "claim without a volume mode", mutate: func(claim *corev1.PersistentVolumeClaim) { claim.Spec.VolumeMode = nil }}, + {name: "claim protected by Kubernetes and another controller", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Finalizers = []string{"kubernetes.io/pvc-protection", "example.com/hold"} + }}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -2068,16 +2148,19 @@ func TestV1beta2RunControllerAcceptsAnUnboundOrSharedTraceClaim(t *testing.T) { f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) f.reconcile() assertDGDRRunPendingStatus(t, f.run(), f.job().Name) + + t.Log("The claim keeps its finalizers and gains the Run's") + assertDGDRRunTraceClaimFinalizers(t, f, append(claim.Finalizers, dgdrRunTraceClaimFinalizer("run-uid"))...) }) } } func TestV1beta2RunControllerRetriesATransientTraceClaimReadError(t *testing.T) { f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) - f.reconciler.APIReader = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { if _, ok := obj.(*corev1.PersistentVolumeClaim); ok { - return errors.New("API server unavailable") + return errors.New("cache unavailable") } return c.Get(ctx, key, obj, opts...) }, @@ -2085,7 +2168,7 @@ func TestV1beta2RunControllerRetriesATransientTraceClaimReadError(t *testing.T) t.Log("A read error other than NotFound is returned for a retry") _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) - if err == nil || !strings.Contains(err.Error(), "API server unavailable") { + if err == nil || !strings.Contains(err.Error(), "cache unavailable") { t.Fatalf("reconcile error = %v, want the claim read error", err) } @@ -2096,127 +2179,238 @@ func TestV1beta2RunControllerRetriesATransientTraceClaimReadError(t *testing.T) assertNoDGDRRunResources(t, f.client) } -func TestV1beta2RunControllerKeepsFollowingAStartedJobAfterItsTraceClaimDisappears(t *testing.T) { +func TestV1beta2RunControllerProtectsTheTraceClaimBeforeCreatingTheJob(t *testing.T) { tests := []struct { - name string - status batchv1.JobStatus - podNode string - podPhase corev1.PodPhase - wantPhase nvidiacomv1beta2.DGDRRunPhase + name string + // patchErr, when set, fails the claim patch that protects the claim. + patchErr error }{ + {name: "the protection is persisted"}, + {name: "the claim patch fails", patchErr: errors.New("API server unavailable")}, + { + name: "the claim patch conflicts with a newer claim", + patchErr: apierrors.NewConflict(schema.GroupResource{Resource: "persistentvolumeclaims"}, "request-traces", + errors.New("the object has been modified")), + }, { - name: "pod scheduled while its images are pulled", - status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, - podNode: "node-a", - podPhase: corev1.PodPending, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the API server refuses a new finalizer on a claim being deleted", + patchErr: apierrors.NewInvalid(schema.GroupKind{Kind: "PersistentVolumeClaim"}, "request-traces", field.ErrorList{ + field.Forbidden(field.NewPath("metadata", "finalizers"), "no new finalizers can be added if the object is being deleted"), + }), }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var calls []string + f := newDGDRRunFixtureWithInterceptor(t, interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*corev1.PersistentVolumeClaim); ok { + calls = append(calls, "protect claim") + if tt.patchErr != nil { + return tt.patchErr + } + } + return c.Patch(ctx, obj, patch, opts...) + }, + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*batchv1.Job); ok { + calls = append(calls, "create Job") + } + return c.Create(ctx, obj, opts...) + }, + }, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + + t.Log("Reconcile the Run once") + _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{Namespace: "default", Name: "search-1"}) + + if tt.patchErr == nil { + t.Log("The claim is protected before the Job is created") + if err != nil { + t.Fatalf("reconcile: %v", err) + } + if want := []string{"protect claim", "create Job"}; !reflect.DeepEqual(calls, want) { + t.Fatalf("calls = %v, want %v", calls, want) + } + assertDGDRRunTraceClaimFinalizers(t, f, dgdrRunTraceClaimFinalizer("run-uid")) + return + } + + t.Log("A failed protection is retried: no Job is created and the Run is not failed") + if err == nil || dgdrRunIsPermanentError(err) { + t.Fatalf("reconcile error = %v, want a retryable error", err) + } + if want := []string{"protect claim"}; !reflect.DeepEqual(calls, want) { + t.Fatalf("calls = %v, want %v", calls, want) + } + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want none without protection", len(jobs)) + } + if run := f.run(); len(run.Status.Conditions) != 0 { + t.Fatalf("status = %+v, want untouched", run.Status) + } + }) + } +} + +func TestV1beta2RunControllerCancelsARunWhoseTraceClaimIsDeleted(t *testing.T) { + waitingSweeper := []corev1.ContainerStatus{{Name: dgdrRunSweeperContainer, State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{Reason: "CreateContainerConfigError", Message: "failed to prepare subPath"}, + }}} + creatingSweeper := []corev1.ContainerStatus{{Name: dgdrRunSweeperContainer, State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{Reason: "ContainerCreating"}, + }}} + runningSweeper := []corev1.ContainerStatus{{Name: dgdrRunSweeperContainer, State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{}, + }}} + tests := []struct { + name string + status batchv1.JobStatus + // pod, when set, is the Job's pod as the cache shows it. + pod *corev1.PodStatus + // podNode binds the pod to a node. + podNode string + // removeClaim removes the claim completely, as when its finalizers were removed by hand. + removeClaim bool + }{ { - name: "a container exited after the pod was ready", - status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, - podNode: "node-a", - podPhase: corev1.PodRunning, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the Job counts a pod the pod cache does not show", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, }, { - name: "pod ready while the pod cache still shows it unscheduled", - status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, - podPhase: corev1.PodPending, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the pod is not scheduled", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, + pod: &corev1.PodStatus{Phase: corev1.PodPending}, }, { - name: "Job tracks a succeeded pod it has not counted", - status: batchv1.JobStatus{UncountedTerminatedPods: &batchv1.UncountedTerminatedPods{Succeeded: []types.UID{"pod"}}}, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the pod is scheduled while its images are pulled", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, + pod: &corev1.PodStatus{Phase: corev1.PodPending, ContainerStatuses: creatingSweeper}, + podNode: "node-a", }, { - name: "Job tracks a failed pod it has not counted", - status: batchv1.JobStatus{UncountedTerminatedPods: &batchv1.UncountedTerminatedPods{Failed: []types.UID{"pod"}}}, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the sweeper waits on a mount error", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, + pod: &corev1.PodStatus{Phase: corev1.PodPending, ContainerStatuses: waitingSweeper}, + podNode: "node-a", }, { - name: "Job met its success criteria", - status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobSuccessCriteriaMet, Status: corev1.ConditionTrue}}}, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the sweeper runs", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, + pod: &corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: runningSweeper}, + podNode: "node-a", }, { - name: "Job reached its failure target", - status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailureTarget, Status: corev1.ConditionTrue}}}, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the pod is ready while the pod cache still shows it unscheduled", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, + pod: &corev1.PodStatus{Phase: corev1.PodPending}, }, { - name: "Job counted a succeeded pod", - status: batchv1.JobStatus{Succeeded: 1}, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the Job tracks a succeeded pod it has not counted", + status: batchv1.JobStatus{UncountedTerminatedPods: &batchv1.UncountedTerminatedPods{Succeeded: []types.UID{"pod"}}}, }, { - name: "Job counted a failed pod", - status: batchv1.JobStatus{Failed: 1}, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseRunning, + name: "the Job met its success criteria", + status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobSuccessCriteriaMet, Status: corev1.ConditionTrue}}}, }, { - name: "Job complete", - status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseSucceeded, + name: "the Job reached its failure target", + status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailureTarget, Status: corev1.ConditionTrue}}}, }, { - name: "Job failed", - status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}, - wantPhase: nvidiacomv1beta2.DGDRRunPhaseFailed, + name: "the claim is gone after its finalizers were removed by hand", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, + pod: &corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: runningSweeper}, + podNode: "node-a", + removeClaim: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { claim := traceClaimForTest() claim.Finalizers = []string{"kubernetes.io/pvc-protection"} - f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + var propagation []metav1.DeletionPropagation + f := newDGDRRunFixtureWithInterceptor(t, interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*batchv1.Job); ok { + options := &client.DeleteOptions{} + options.ApplyOptions(opts) + propagation = append(propagation, ptr.Deref(options.PropagationPolicy, "")) + } + return c.Delete(ctx, obj, opts...) + }, + }, dgdrRunWithTraceForTest(t, map[string]any{}), claim) f.reconcile() + finalizer := dgdrRunTraceClaimFinalizer("run-uid") + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection", finalizer) - t.Log("The Job, and its pod where the row has one, report the pod past the claim") + t.Log("The Job reports the row's execution state; a test finalizer stands in for its pods' cleanup") job := f.job() + job.Finalizers = []string{"example.com/pods-terminating"} + if err := f.client.Update(t.Context(), job); err != nil { + t.Fatalf("hold Job deletion: %v", err) + } job.Status = tt.status job.Status.StartTime = new(metav1.Now()) if err := f.client.Status().Update(t.Context(), job); err != nil { t.Fatalf("update Job status: %v", err) } - if tt.podPhase != "" { - createDGDRRunJobPodForTest(t, f.client, job, tt.podNode, corev1.PodStatus{Phase: tt.podPhase}) + if tt.pod != nil { + createDGDRRunJobPodForTest(t, f.client, job, tt.podNode, *tt.pod) } + f.reconcile() t.Log("Delete the claim") if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { t.Fatalf("delete claim: %v", err) } + if tt.removeClaim { + stored, _ := f.claim() + stored.Finalizers = nil + if err := f.client.Update(t.Context(), stored); err != nil { + t.Fatalf("remove claim finalizers: %v", err) + } + } - t.Log("Hide the Job from the cached List, as a lagging informer would") - cached := f.reconciler.Client - f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ - List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { - if _, ok := list.(*batchv1.JobList); ok { - return nil - } - return c.List(ctx, list, opts...) - }, - }) - _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) - if !apierrors.IsAlreadyExists(err) { - t.Fatalf("reconcile error = %v, want the Job create to hit AlreadyExists until the cache catches up", err) + t.Log("The Run stops its Job in the foreground and stays open while the Job is deleted") + f.reconcile() + f.reconcile() + if stopped := f.job(); stopped.UID != job.UID || stopped.DeletionTimestamp == nil { + t.Fatalf("Job = %+v, want the Job %q being deleted", stopped.ObjectMeta, job.UID) + } + if !reflect.DeepEqual(propagation, []metav1.DeletionPropagation{metav1.DeletePropagationForeground}) { + t.Fatalf("Job deletions = %v, want one foreground deletion", propagation) + } + if meta.IsStatusConditionTrue(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { + t.Fatalf("status = %+v, want the Run open until its Job is gone", f.run().Status) + } + if !tt.removeClaim { + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection", finalizer) } - t.Log("Once the cache shows the Job again, the Job, not the claim, decides the Run") - f.reconciler.Client = cached + t.Log("Once the Job and its pods are gone, the Run fails with TraceSourceDeleted and keeps the claim until then") + job = f.job() + job.Finalizers = nil + if err := f.client.Update(t.Context(), job); err != nil { + t.Fatalf("finish Job deletion: %v", err) + } f.reconcile() - if kept := f.job(); kept.UID != job.UID { - t.Fatalf("Job UID = %q, want the started Job %q kept", kept.UID, job.UID) + assertDGDRRunCanceledForTrace(t, f.run(), job.Name) + if !tt.removeClaim { + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection", finalizer) } - run := f.run() - if run.Status.Phase != tt.wantPhase { - t.Fatalf("phase = %q, want %q", run.Status.Phase, tt.wantPhase) + + t.Log("The completed Run releases the claim and never recreates its Job") + f.reconcile() + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want none", len(jobs)) } - condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) - if condition == nil || condition.Reason == dgdrRunReasonTraceSourceUnavailable { - t.Fatalf("Completed condition = %+v, want the Job, not the claim, to decide it", condition) + if tt.removeClaim { + if _, found := f.claim(); found { + t.Fatal("claim exists, want it gone") + } + return } + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection") }) } } @@ -2240,338 +2434,319 @@ func createDGDRRunJobPodForTest(t *testing.T, c client.Client, job *batchv1.Job, } } -// assertDGDRRunFailedForTrace checks that the Run failed for its trace with a message that starts -// with reason and names the Job, and that the completed Run deleted that Job. -func assertDGDRRunFailedForTrace(t *testing.T, c client.Client, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, jobName, reason string) { +// assertDGDRRunCanceledForTrace checks that run failed because its trace claim was deleted, with the +// jobRef of jobName, or none when jobName is empty. +func assertDGDRRunCanceledForTrace(t *testing.T, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, jobName string) { t.Helper() - jobs := &batchv1.JobList{} - if err := c.List(t.Context(), jobs, client.InNamespace("default")); err != nil { - t.Fatalf("list Jobs: %v", err) - } - if len(jobs.Items) != 0 { - t.Fatalf("Jobs = %d, want the Job that could never read the trace deleted", len(jobs.Items)) - } condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseFailed || condition == nil || - condition.Status != metav1.ConditionTrue || condition.Reason != dgdrRunReasonTraceSourceUnavailable { - t.Fatalf("status = %+v, want Failed and Completed=True/%s", run.Status, dgdrRunReasonTraceSourceUnavailable) + condition.Status != metav1.ConditionTrue || condition.Reason != dgdrRunReasonTraceSourceDeleted { + t.Fatalf("status = %+v, want Failed and Completed=True/%s", run.Status, dgdrRunReasonTraceSourceDeleted) } - wantSuffix := fmt.Sprintf(`; Job %q is deleted`, jobName) - if !strings.HasPrefix(condition.Message, reason) || !strings.HasSuffix(condition.Message, wantSuffix) { - t.Fatalf("condition message = %q, want prefix %q and suffix %q", condition.Message, reason, wantSuffix) + want := `trace PersistentVolumeClaim "request-traces" in namespace "default" was deleted, which cancels the Run` + if condition.Message != want || run.Status.CompletionTime == nil { + t.Fatalf("condition message = %q, completionTime = %v; want %q and a completion time", + condition.Message, run.Status.CompletionTime, want) } - if run.Status.CompletionTime == nil || run.Status.JobRef == nil || run.Status.JobRef.Name != jobName { - t.Fatalf("completionTime = %v, jobRef = %+v; want a completion time and the Job's jobRef", - run.Status.CompletionTime, run.Status.JobRef) + if (jobName == "" && run.Status.JobRef != nil) || (jobName != "" && (run.Status.JobRef == nil || run.Status.JobRef.Name != jobName)) { + t.Fatalf("jobRef = %+v, want %q", run.Status.JobRef, jobName) } } -func TestV1beta2RunControllerDeletesAnUnscheduledJobWhoseTraceClaimBecomesUnusable(t *testing.T) { +func TestV1beta2RunControllerLetsAFinishedJobDecideAfterItsTraceClaimIsDeleted(t *testing.T) { tests := []struct { - name string - finalizers []string - deleteClaim bool - mutate func(*corev1.PersistentVolumeClaim) - updateStatus bool - reason string + name string + condition batchv1.JobConditionType + wantPhase nvidiacomv1beta2.DGDRRunPhase + wantReason string + // hideJob hides the Job from the cached List for the first pass after the deletion. + hideJob bool }{ - { - name: "claim deleted", - deleteClaim: true, - reason: `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`, - }, - { - name: "claim being deleted", - finalizers: []string{"kubernetes.io/pvc-protection"}, - deleteClaim: true, - reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is being deleted`, - }, - { - name: "claim replaced by a raw block claim", - mutate: func(claim *corev1.PersistentVolumeClaim) { - claim.Spec.VolumeMode = ptr.To(corev1.PersistentVolumeBlock) - }, - reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is a raw block volume`, - }, - { - name: "claim lost its volume", - mutate: func(claim *corev1.PersistentVolumeClaim) { claim.Status.Phase = corev1.ClaimLost }, - updateStatus: true, - reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" has lost its PersistentVolume`, - }, + {name: "the Job completed", condition: batchv1.JobComplete, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseSucceeded, wantReason: dgdrRunReasonSucceeded}, + {name: "the Job failed", condition: batchv1.JobFailed, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseFailed, wantReason: dgdrRunReasonJobFailed}, + {name: "the Job completed but the cache does not show it yet", condition: batchv1.JobComplete, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseSucceeded, wantReason: dgdrRunReasonSucceeded, hideJob: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { claim := traceClaimForTest() - claim.Finalizers = tt.finalizers + claim.Finalizers = []string{"kubernetes.io/pvc-protection"} f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) f.reconcile() - job := f.job() - t.Log("The Job's pod is created but not scheduled yet") - job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} + t.Log("The Job finishes, and then the claim is deleted") + job := f.job() + job.Status.StartTime = new(metav1.Now()) + job.Status.Conditions = []batchv1.JobCondition{{Type: tt.condition, Status: corev1.ConditionTrue}} if err := f.client.Status().Update(t.Context(), job); err != nil { - t.Fatalf("update Job status: %v", err) + t.Fatalf("finish Job: %v", err) } - createDGDRRunJobPodForTest(t, f.client, job, "", corev1.PodStatus{Phase: corev1.PodPending}) - f.reconcile() - assertDGDRRunRunningStatus(t, f.run(), job.Name) - - t.Log("The claim becomes unusable before the pod is scheduled") - if tt.deleteClaim { - if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { - t.Fatalf("delete claim: %v", err) - } + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) } - if tt.mutate != nil { - stored := &corev1.PersistentVolumeClaim{} - if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(claim), stored); err != nil { - t.Fatalf("get claim: %v", err) - } - tt.mutate(stored) - var err error - if tt.updateStatus { - err = f.client.Status().Update(t.Context(), stored) - } else { - err = f.client.Update(t.Context(), stored) - } - if err != nil { - t.Fatalf("update claim: %v", err) + + if tt.hideJob { + t.Log("A pass that finds the finished Job only in the API server waits for the cache") + cached := f.reconciler.Client + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*batchv1.JobList); ok { + return nil + } + return c.List(ctx, list, opts...) + }, + }) + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("result/error = %+v/%v, want a retry", result, err) } + f.reconciler.Client = cached } - t.Log("Reconcile twice: the first pass fails the Run, the second deletes the Job of the completed Run") + t.Log("The finished Job is kept and decides the Run") f.reconcile() + if kept := f.job(); kept.UID != job.UID || kept.DeletionTimestamp != nil { + t.Fatalf("Job = %+v, want the finished Job %q kept", kept.ObjectMeta, job.UID) + } + run := f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != tt.wantPhase || condition == nil || condition.Status != metav1.ConditionTrue || + condition.Reason != tt.wantReason { + t.Fatalf("status = %+v, want %s and Completed=True/%s", run.Status, tt.wantPhase, tt.wantReason) + } + + t.Log("The completed Run releases the claim") f.reconcile() - assertDGDRRunFailedForTrace(t, f.client, f.run(), job.Name, tt.reason) + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection") }) } } -func TestV1beta2RunControllerDeletesAJobWhoseSweeperCannotMountTheTraceFile(t *testing.T) { - const prefix = `trace file "retune/w-1.jsonl.gz" on PersistentVolumeClaim "request-traces" cannot be mounted: ` - tests := []struct { - name string - container string - reason string - message string - wantFailed bool - }{ - { - name: "a subPath that resolves outside the claim", - container: dgdrRunSweeperContainer, - reason: "CreateContainerConfigError", - message: `failed to prepare subPath for volumeMount "trace" of container "sweeper"`, - wantFailed: true, - }, - { - name: "a missing file on a read-only claim", - container: dgdrRunSweeperContainer, - reason: "CreateContainerConfigError", - message: `failed to create subPath directory for volumeMount "trace" of container "sweeper"`, - wantFailed: true, - }, - { - name: "another volume of the sweeper is left to the Job", - container: dgdrRunSweeperContainer, - reason: "CreateContainerConfigError", - message: `failed to prepare subPath for volumeMount "input" of container "sweeper"`, - }, - { - name: "the publisher is left to the Job", - container: dgdrRunPublisherContainer, - reason: "CreateContainerConfigError", - message: `failed to prepare subPath for volumeMount "trace" of container "publisher"`, - }, - { - name: "a sweeper image that cannot be pulled is left to the Job", - container: dgdrRunSweeperContainer, - reason: "ErrImagePull", - message: `failed to pull image`, - }, +func TestV1beta2RunControllerReleasesTheTraceClaimOnlyAfterTheRunCompletes(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + f.reconcile() + finalizer := dgdrRunTraceClaimFinalizer("run-uid") + + t.Log("The running Run keeps its protection") + job := f.job() + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(1)), StartTime: new(metav1.Now())} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("start Job: %v", err) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) - f.reconcile() - job := f.job() + f.reconcile() + assertDGDRRunRunningStatus(t, f.run(), job.Name) + assertDGDRRunTraceClaimFinalizers(t, f, finalizer) - t.Log("The scheduled pod's container waits with the row's reason while the claim is usable") - job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} - if err := f.client.Status().Update(t.Context(), job); err != nil { - t.Fatalf("update Job status: %v", err) - } - createDGDRRunJobPodForTest(t, f.client, job, "node-a", corev1.PodStatus{ - Phase: corev1.PodPending, - ContainerStatuses: []corev1.ContainerStatus{{ - Name: tt.container, - State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: tt.reason, Message: tt.message}}, - }}, - }) + t.Log("The Job completes; the pass that records the outcome keeps the protection") + job = f.job() + job.Status.Active, job.Status.Ready = 0, new(int32(0)) + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("complete Job: %v", err) + } + f.reconcile() + if f.run().Status.Phase != nvidiacomv1beta2.DGDRRunPhaseSucceeded { + t.Fatalf("status = %+v, want Succeeded", f.run().Status) + } + assertDGDRRunTraceClaimFinalizers(t, f, finalizer) - t.Log("Reconcile twice and expect only a trace mount failure to fail the Run and delete its Job") - f.reconcile() - f.reconcile() - if !tt.wantFailed { - if kept := f.job(); kept.UID != job.UID { - t.Fatalf("Job UID = %q, want the Job %q kept", kept.UID, job.UID) - } - assertDGDRRunRunningStatus(t, f.run(), job.Name) - return - } - assertDGDRRunFailedForTrace(t, f.client, f.run(), job.Name, prefix+tt.message) - }) + t.Log("The completed Run releases the claim and keeps its finished Job") + f.reconcile() + assertDGDRRunTraceClaimFinalizers(t, f) + if kept := f.job(); kept.UID != job.UID { + t.Fatalf("Job UID = %q, want the finished Job %q kept", kept.UID, job.UID) } } -func TestV1beta2RunControllerFailsAScheduledJobOnlyForAClaimItCanNeverMount(t *testing.T) { +func TestV1beta2RunControllerStopsAnUnfinishedJobOfACompletedRunBeforeReleasingItsClaim(t *testing.T) { tests := []struct { - name string - finalizers []string - deleteClaim bool - lose bool - sweeperStarted bool - reason string + name string + // hideJob hides the Job from the cached List until the last step. + hideJob bool }{ - { - name: "claim being deleted while pvc-protection keeps it for the pod", - finalizers: []string{"kubernetes.io/pvc-protection"}, - deleteClaim: true, - }, - { - name: "claim gone while the pod was bound", - deleteClaim: true, - reason: `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`, - }, - { - name: "claim being deleted without pvc-protection", - finalizers: []string{"example.com/hold"}, - deleteClaim: true, - reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" is being deleted`, - }, - { - name: "claim lost its volume", - lose: true, - reason: `trace PersistentVolumeClaim "request-traces" in namespace "default" has lost its PersistentVolume`, - }, - { - name: "sweeper started before the claim went", - deleteClaim: true, - sweeperStarted: true, - }, + {name: "the cache shows the Job"}, + {name: "only the API server shows the Job", hideJob: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - claim := traceClaimForTest() - claim.Finalizers = tt.finalizers - f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) f.reconcile() job := f.job() + finalizer := dgdrRunTraceClaimFinalizer("run-uid") - t.Log("The Job's pod is bound to a node; its sweeper is created or still waits") - job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} - if err := f.client.Status().Update(t.Context(), job); err != nil { - t.Fatalf("update Job status: %v", err) - } - sweeper := corev1.ContainerStatus{Name: dgdrRunSweeperContainer, State: corev1.ContainerState{ - Waiting: &corev1.ContainerStateWaiting{Reason: "ContainerCreating"}, - }} - if tt.sweeperStarted { - sweeper.State = corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} - } - createDGDRRunJobPodForTest(t, f.client, job, "node-a", corev1.PodStatus{ - Phase: corev1.PodPending, ContainerStatuses: []corev1.ContainerStatus{sweeper}, + t.Log("The Run is completed while its Job still runs, as after a stale pass created the Job") + run := f.run() + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, Status: metav1.ConditionTrue, + Reason: dgdrRunReasonSuperseded, Message: "Run was superseded by Request generation 2", }) - - t.Log("The claim changes after the binding") - if tt.deleteClaim { - if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { - t.Fatalf("delete claim: %v", err) - } + run.Status.Phase = nvidiacomv1beta2.DGDRRunPhaseFailed + if err := f.client.Status().Update(t.Context(), run); err != nil { + t.Fatalf("complete Run: %v", err) } - if tt.lose { - stored := &corev1.PersistentVolumeClaim{} - if err := f.client.Get(t.Context(), client.ObjectKeyFromObject(claim), stored); err != nil { - t.Fatalf("get claim: %v", err) - } - stored.Status.Phase = corev1.ClaimLost - if err := f.client.Status().Update(t.Context(), stored); err != nil { - t.Fatalf("lose claim: %v", err) + + if tt.hideJob { + t.Log("While the cache misses the Job, the API server keeps the claim protected") + cached := f.reconciler.Client + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*batchv1.JobList); ok { + return nil + } + return c.List(ctx, list, opts...) + }, + }) + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(run)}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("result/error = %+v/%v, want a retry", result, err) } + assertDGDRRunTraceClaimFinalizers(t, f, finalizer) + f.reconciler.Client = cached } - t.Log("Reconcile twice and expect only a claim the pod can never mount to end the Run") + t.Log("The completed Run stops the Job before it releases the claim") f.reconcile() + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want the unfinished Job %q stopped", len(jobs), job.UID) + } + assertDGDRRunTraceClaimFinalizers(t, f, finalizer) f.reconcile() - if tt.reason == "" { - if kept := f.job(); kept.UID != job.UID { - t.Fatalf("Job UID = %q, want the Job %q kept", kept.UID, job.UID) - } - assertDGDRRunRunningStatus(t, f.run(), job.Name) - return + assertDGDRRunTraceClaimFinalizers(t, f) + if condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted); condition == nil || + condition.Reason != dgdrRunReasonSuperseded { + t.Fatalf("Completed condition = %+v, want the recorded outcome kept", condition) } - assertDGDRRunFailedForTrace(t, f.client, f.run(), job.Name, tt.reason) }) } } -func TestV1beta2RunControllerDoesNotLetAStaleRunUndoATraceFailure(t *testing.T) { +func TestV1beta2RunControllerKeepsAnotherRunsProtectionOfASharedTraceClaim(t *testing.T) { + first := dgdrRunWithTraceForTest(t, map[string]any{}) + second := dgdrRunWithTraceForTest(t, map[string]any{}) + second.Name, second.UID = "search-2", "run-uid-2" + f := newDGDRRunFixture(t, first, second, traceClaimForTest()) + + t.Log("Each Run protects the shared claim with its own finalizer") + f.reconcile() + if _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(second)}); err != nil { + t.Fatalf("reconcile second Run: %v", err) + } + assertDGDRRunTraceClaimFinalizers(t, f, dgdrRunTraceClaimFinalizer("run-uid"), dgdrRunTraceClaimFinalizer("run-uid-2")) + + t.Log("The first Run completes and releases only its own protection") + var job *batchv1.Job + jobs := f.jobs() + for i := range jobs { + if metav1.IsControlledBy(&jobs[i], first) { + job = &jobs[i] + } + } + if job == nil { + t.Fatalf("Jobs = %d, want one of the first Run", len(jobs)) + } + job.Status.StartTime = new(metav1.Now()) + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("complete first Job: %v", err) + } + f.reconcile() + f.reconcile() + assertDGDRRunTraceClaimFinalizers(t, f, dgdrRunTraceClaimFinalizer("run-uid-2")) +} + +func TestV1beta2RunControllerNeverRecreatesACanceledJobWhileItsOutcomeIsUnrecorded(t *testing.T) { + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + f.reconcile() + f.reconcile() + jobName := f.job().Name + assertDGDRRunPendingStatus(t, f.run(), jobName) + + t.Log("Delete the claim; the first pass stops the Job") + if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("delete claim: %v", err) + } + f.reconcile() + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want the Job stopped", len(jobs)) + } + + t.Log("Fail the status patch that records the cancellation") + cached := f.reconciler.Client + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + SubResourcePatch: func(context.Context, client.Client, string, client.Object, client.Patch, ...client.SubResourcePatchOption) error { + return errors.New("injected status patch failure") + }, + }) + if _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}); err == nil { + t.Fatal("reconcile error = nil, want the injected status patch failure") + } + + t.Log("The claim is still being deleted, so no pass creates another Job") + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want none", len(jobs)) + } + assertDGDRRunPendingStatus(t, f.run(), jobName) + + t.Log("The next pass records the cancellation") + f.reconciler.Client = cached + f.reconcile() + assertDGDRRunCanceledForTrace(t, f.run(), jobName) + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want none", len(jobs)) + } +} + +func TestV1beta2RunControllerDoesNotLetAStaleRunUndoATraceCancellation(t *testing.T) { tests := []struct { name string - // jobCondition, when set, finishes the Job after the trace failure was recorded. - jobCondition batchv1.JobConditionType - // duplicateJob adds a second, unfinished Job the Run controls, so the stale pass fails the - // Run permanently and writes the permanent-failure marker. - duplicateJob bool + // recreateClaim replaces the deleted claim with a new one under the same name before the stale + // pass, so the stale pass protects it and creates a Job for the completed Run. + recreateClaim bool }{ - {name: "the trace is still unreadable"}, - {name: "the Job completes after the failure", jobCondition: batchv1.JobComplete}, - {name: "a second Job appears after the Job completes", jobCondition: batchv1.JobComplete, duplicateJob: true}, + {name: "the claim is still being deleted"}, + {name: "the claim was recreated under the same name", recreateClaim: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + claim := traceClaimForTest() + claim.Finalizers = []string{"kubernetes.io/pvc-protection"} + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) f.reconcile() job := f.job() - t.Log("The Job's pod waits unscheduled while the Run is Running") - job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} + t.Log("The Job runs while the Run is Running") + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(1)), StartTime: new(metav1.Now())} if err := f.client.Status().Update(t.Context(), job); err != nil { t.Fatalf("update Job status: %v", err) } - createDGDRRunJobPodForTest(t, f.client, job, "", corev1.PodStatus{Phase: corev1.PodPending}) f.reconcile() stale := f.run() assertDGDRRunRunningStatus(t, stale, job.Name) - t.Log("The claim goes and the trace failure is recorded") + t.Log("The claim is deleted, the Job is stopped and the cancellation is recorded") if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { t.Fatalf("delete claim: %v", err) } f.reconcile() - failed := f.run() - if condition := meta.FindStatusCondition(failed.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted); condition == nil || - condition.Reason != dgdrRunReasonTraceSourceUnavailable { - t.Fatalf("Completed condition = %+v, want %s", condition, dgdrRunReasonTraceSourceUnavailable) - } - if tt.jobCondition != "" { - stored := f.job() - stored.Status.Conditions = []batchv1.JobCondition{{Type: tt.jobCondition, Status: corev1.ConditionTrue}} - if err := f.client.Status().Update(t.Context(), stored); err != nil { - t.Fatalf("finish Job: %v", err) + f.reconcile() + canceled := f.run() + assertDGDRRunCanceledForTrace(t, canceled, job.Name) + if tt.recreateClaim { + f.reconcile() + if stored, found := f.claim(); found { + stored.Finalizers = nil + if err := f.client.Update(t.Context(), stored); err != nil { + t.Fatalf("finish claim deletion: %v", err) + } } - } - if tt.duplicateJob { - duplicate := f.job().DeepCopy() - duplicate.Name += "-duplicate" - duplicate.ResourceVersion, duplicate.UID = "", "" - duplicate.Status = batchv1.JobStatus{} - if err := f.client.Create(t.Context(), duplicate); err != nil { - t.Fatalf("create duplicate Job: %v", err) + if err := f.client.Create(t.Context(), traceClaimForTest()); err != nil { + t.Fatalf("recreate claim: %v", err) } } - t.Log("A pass that still reads the Running Run conflicts instead of overwriting the failure") + t.Log("A pass that still reads the Running Run conflicts instead of overwriting the cancellation") cached := f.reconciler.Client f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { @@ -2587,82 +2762,154 @@ func TestV1beta2RunControllerDoesNotLetAStaleRunUndoATraceFailure(t *testing.T) t.Fatalf("reconcile error = %v, want a status conflict", err) } f.reconciler.Client = cached - if got := f.run(); !reflect.DeepEqual(got.Status, failed.Status) { - t.Fatalf("status = %+v, want the recorded failure %+v", got.Status, failed.Status) + if got := f.run(); !reflect.DeepEqual(got.Status, canceled.Status) { + t.Fatalf("status = %+v, want the recorded cancellation %+v", got.Status, canceled.Status) + } + wantStaleJobs := 0 + if tt.recreateClaim { + wantStaleJobs = 1 + } + if jobs := f.jobs(); len(jobs) != wantStaleJobs { + t.Fatalf("Jobs = %d after the stale pass, want %d", len(jobs), wantStaleJobs) } - t.Log("The next pass deletes the Job only if it has not finished, and keeps the failure") + t.Log("The completed Run stops any Job the stale pass created, then releases the claim") f.reconcile() - if tt.jobCondition == "" { - assertDGDRRunFailedForTrace(t, f.client, f.run(), job.Name, - `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`) - return + f.reconcile() + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want none left", len(jobs)) } - if jobs := f.jobs(); len(jobs) != 1 || jobs[0].UID != job.UID { - t.Fatalf("Jobs = %d, want only the finished Job %q kept", len(jobs), job.UID) + if tt.recreateClaim { + assertDGDRRunTraceClaimFinalizers(t, f) + } else { + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection") } - if got := f.run(); !reflect.DeepEqual(got.Status, failed.Status) { - t.Fatalf("status = %+v, want the recorded failure %+v", got.Status, failed.Status) + if got := f.run(); !reflect.DeepEqual(got.Status, canceled.Status) { + t.Fatalf("status = %+v, want the recorded cancellation %+v", got.Status, canceled.Status) } }) } } -func TestV1beta2RunControllerWaitsUntilTheCacheShowsAnActiveJobsPod(t *testing.T) { - f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) - f.reconcile() - - t.Log("The Job counts an active pod that the pod cache does not show yet") - job := f.job() - job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} - if err := f.client.Status().Update(t.Context(), job); err != nil { - t.Fatalf("update Job status: %v", err) - } - - t.Log("The claim disappears; whether the pod is scheduled is unknown, so the Job is kept for now") - if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { - t.Fatalf("delete claim: %v", err) - } - f.reconcile() - if kept := f.job(); kept.UID != job.UID { - t.Fatalf("Job UID = %q, want the Job %q kept", kept.UID, job.UID) +func TestV1beta2RunControllerReportsAWaitingContainerWithoutEndingTheRun(t *testing.T) { + tests := []struct { + name string + container string + reason string + message string + // wantReason is the Run's Completed reason; a diagnostic carries the container's state verbatim. + wantReason string + }{ + { + name: "a subPath that resolves outside the claim", + container: dgdrRunSweeperContainer, + reason: "CreateContainerConfigError", + message: `failed to prepare subPath for volumeMount "trace" of container "sweeper"`, + wantReason: dgdrRunReasonContainerWaiting, + }, + { + name: "a missing file on a read-only claim", + container: dgdrRunSweeperContainer, + reason: "CreateContainerConfigError", + message: `failed to create subPath directory for volumeMount "trace" of container "sweeper"`, + wantReason: dgdrRunReasonContainerWaiting, + }, + { + name: "a kubelet message in other words", + container: dgdrRunSweeperContainer, + reason: "CreateContainerConfigError", + message: "the trace mount could not be prepared", + wantReason: dgdrRunReasonContainerWaiting, + }, + { + name: "another volume of the sweeper", + container: dgdrRunSweeperContainer, + reason: "CreateContainerConfigError", + message: `failed to prepare subPath for volumeMount "input" of container "sweeper"`, + wantReason: dgdrRunReasonContainerWaiting, + }, + { + name: "the publisher", + container: dgdrRunPublisherContainer, + reason: "CreateContainerConfigError", + message: `failed to prepare subPath for volumeMount "trace" of container "publisher"`, + wantReason: dgdrRunReasonContainerWaiting, + }, + { + name: "a sweeper image that cannot be pulled", + container: dgdrRunSweeperContainer, + reason: "ErrImagePull", + message: "failed to pull image", + wantReason: dgdrRunReasonContainerWaiting, + }, + { + name: "a container that is being created", + container: dgdrRunSweeperContainer, + reason: "ContainerCreating", + wantReason: dgdrRunReasonRunning, + }, + { + name: "a container that waits for its pod to initialize", + container: dgdrRunPublisherContainer, + reason: "PodInitializing", + wantReason: dgdrRunReasonRunning, + }, } - assertDGDRRunRunningStatus(t, f.run(), job.Name) -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var deleted bool + f := newDGDRRunFixtureWithInterceptor(t, interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + deleted = true + return c.Delete(ctx, obj, opts...) + }, + }, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) + f.reconcile() + job := f.job() -func TestV1beta2RunControllerKeepsTheJobUntilTheTraceFailureIsRecorded(t *testing.T) { - f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) - f.reconcile() - f.reconcile() - jobName := f.job().Name - assertDGDRRunPendingStatus(t, f.run(), jobName) + t.Log("The scheduled pod's container waits with the row's reason") + job.Status = batchv1.JobStatus{Active: 1, Ready: new(int32(0)), StartTime: new(metav1.Now())} + if err := f.client.Status().Update(t.Context(), job); err != nil { + t.Fatalf("update Job status: %v", err) + } + createDGDRRunJobPodForTest(t, f.client, job, "node-a", corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: tt.container, + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: tt.reason, Message: tt.message}}, + }}, + }) - t.Log("Delete the claim, then fail the status patch that records the trace failure") - if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { - t.Fatalf("delete claim: %v", err) - } - cached := f.reconciler.Client - f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ - SubResourcePatch: func(context.Context, client.Client, string, client.Object, client.Patch, ...client.SubResourcePatchOption) error { - return errors.New("injected status patch failure") - }, - }) - if _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}); err == nil { - t.Fatal("reconcile error = nil, want the injected status patch failure") - } + t.Log("Reconcile twice: the Run stays open with the diagnostic and its Job is kept") + f.reconcile() + f.reconcile() + if kept := f.job(); kept.UID != job.UID || kept.DeletionTimestamp != nil || deleted { + t.Fatalf("Job = %+v (deleted %t), want the Job %q kept", kept.ObjectMeta, deleted, job.UID) + } + run := f.run() + condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) + if run.Status.Phase != nvidiacomv1beta2.DGDRRunPhaseRunning || run.Status.CompletionTime != nil || condition == nil || + condition.Status != metav1.ConditionFalse || condition.Reason != tt.wantReason { + t.Fatalf("status = %+v, want Running and Completed=False/%s", run.Status, tt.wantReason) + } + if want := fmt.Sprintf("container %q is waiting: %s: %s", tt.container, tt.reason, tt.message); tt.wantReason == dgdrRunReasonContainerWaiting && + condition.Message != want { + t.Fatalf("condition message = %q, want %q", condition.Message, want) + } - t.Log("The Job stays until the failure is recorded, so no pass can create another one") - if job := f.job(); job.Name != jobName { - t.Fatalf("Job = %q, want %q kept", job.Name, jobName) + t.Log("Once the container starts, the Run reports the running Job again") + pod := &corev1.Pod{} + if err := f.client.Get(t.Context(), types.NamespacedName{Namespace: "default", Name: job.Name + "-pod"}, pod); err != nil { + t.Fatalf("get pod: %v", err) + } + pod.Status.ContainerStatuses[0].State = corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} + if err := f.client.Status().Update(t.Context(), pod); err != nil { + t.Fatalf("start container: %v", err) + } + f.reconcile() + assertDGDRRunRunningStatus(t, f.run(), job.Name) + }) } - assertDGDRRunPendingStatus(t, f.run(), jobName) - - t.Log("The next passes record the failure and then delete the Job") - f.reconciler.Client = cached - f.reconcile() - f.reconcile() - assertDGDRRunFailedForTrace(t, f.client, f.run(), jobName, - `trace PersistentVolumeClaim "request-traces" does not exist in namespace "default"`) } func TestV1beta2RunControllerWakesTheRunsThatReadAChangedTraceClaim(t *testing.T) { @@ -2746,7 +2993,7 @@ func TestV1beta2RunControllerWakesTheRunOfAJobPod(t *testing.T) { } } -func TestV1beta2RunControllerKeepsAJobWhoseJobRefWasNeverRecorded(t *testing.T) { +func TestV1beta2RunControllerCancelsAJobWhoseJobRefWasNeverRecorded(t *testing.T) { f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), traceClaimForTest()) cached := f.reconciler.Client @@ -2770,7 +3017,7 @@ func TestV1beta2RunControllerKeepsAJobWhoseJobRefWasNeverRecorded(t *testing.T) t.Fatalf("update Job status: %v", err) } - t.Log("Delete the claim and hide the Job from the cached List") + t.Log("Delete the claim and hide the Job from the cached List; the API server still shows the Job, which is stopped") if err := f.client.Delete(t.Context(), traceClaimForTest()); err != nil { t.Fatalf("delete claim: %v", err) } @@ -2782,16 +3029,182 @@ func TestV1beta2RunControllerKeepsAJobWhoseJobRefWasNeverRecorded(t *testing.T) return c.List(ctx, list, opts...) }, }) - _, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) - if !apierrors.IsAlreadyExists(err) { - t.Fatalf("reconcile error = %v, want the Job create to hit AlreadyExists instead of failing the Run", err) + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("result/error = %+v/%v, want the stop and a retry", result, err) + } + if jobs := f.jobs(); len(jobs) != 0 { + t.Fatalf("Jobs = %d, want the Job stopped", len(jobs)) } if condition := meta.FindStatusCondition(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted); condition != nil { - t.Fatalf("Completed condition = %+v, want none while the Job exists", condition) + t.Fatalf("Completed condition = %+v, want none until the stop is confirmed", condition) } - t.Log("Once the cache shows the Job, the Job drives the Run") + t.Log("Once the API server confirms the Job is gone, the Run is canceled") f.reconciler.Client = cached f.reconcile() - assertDGDRRunRunningStatus(t, f.run(), job.Name) + assertDGDRRunCanceledForTrace(t, f.run(), "") +} + +// newDGDRRunTraceClaimFixture returns a trace claim reconciler over a fake client holding objects, +// whose cached Job List the returned hide flag can make empty. +func newDGDRRunTraceClaimFixture( + t *testing.T, + objects ...client.Object, +) (*DynamoGraphDeploymentRunTraceClaimReconciler, client.Client, *bool) { + t.Helper() + f := newDGDRRunFixture(t, objects...) + hide := new(bool) + cached := interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*batchv1.JobList); ok && *hide { + return nil + } + return c.List(ctx, list, opts...) + }, + }) + return &DynamoGraphDeploymentRunTraceClaimReconciler{Client: cached, APIReader: f.client}, f.client, hide +} + +func TestDGDRRunTraceClaimReconcilerReleasesOnlyRunsThatCanNoLongerExecute(t *testing.T) { + const otherFinalizer = "kubernetes.io/pvc-protection" + live := dgdrRunWithTraceForTest(t, map[string]any{}) + live.Name, live.UID = "live", "live-uid" + deleting := dgdrRunWithTraceForTest(t, map[string]any{}) + deleting.Name, deleting.UID = "deleting", "deleting-uid" + deleting.DeletionTimestamp = new(metav1.Now()) + deleting.Finalizers = []string{metav1.FinalizerDeleteDependents} + tests := []struct { + name string + // runs exist besides the claim; jobFor lists the UIDs that control a Job. + runs []*nvidiacomv1beta2.DynamoGraphDeploymentRun + jobFor []types.UID + // hideJobs makes the cache miss every Job, as a lagging informer would. + hideJobs bool + uids []types.UID + claimDeleted bool + want []string + }{ + {name: "a Run that exists", runs: []*nvidiacomv1beta2.DynamoGraphDeploymentRun{live}, + uids: []types.UID{"live-uid"}, want: []string{dgdrRunTraceClaimFinalizer("live-uid")}}, + {name: "a deleted Run whose Job remains", jobFor: []types.UID{"gone-uid"}, + uids: []types.UID{"gone-uid"}, want: []string{dgdrRunTraceClaimFinalizer("gone-uid")}}, + {name: "a deleted Run whose Job only the API server shows", jobFor: []types.UID{"gone-uid"}, hideJobs: true, + uids: []types.UID{"gone-uid"}, want: []string{dgdrRunTraceClaimFinalizer("gone-uid")}}, + {name: "a deleted Run whose Job is gone", uids: []types.UID{"gone-uid"}}, + {name: "a Run being deleted whose Job remains", runs: []*nvidiacomv1beta2.DynamoGraphDeploymentRun{deleting}, + jobFor: []types.UID{"deleting-uid"}, uids: []types.UID{"deleting-uid"}, + want: []string{dgdrRunTraceClaimFinalizer("deleting-uid")}}, + {name: "a Run being deleted whose Job is gone", runs: []*nvidiacomv1beta2.DynamoGraphDeploymentRun{deleting}, + uids: []types.UID{"deleting-uid"}}, + {name: "a deleted Run and a Run that exists share the claim", runs: []*nvidiacomv1beta2.DynamoGraphDeploymentRun{live}, + uids: []types.UID{"gone-uid", "live-uid"}, want: []string{dgdrRunTraceClaimFinalizer("live-uid")}}, + {name: "a claim being deleted after the operator restarted", uids: []types.UID{"gone-uid"}, claimDeleted: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + claim := traceClaimForTest() + claim.Finalizers = []string{otherFinalizer} + for _, uid := range tt.uids { + claim.Finalizers = append(claim.Finalizers, dgdrRunTraceClaimFinalizer(uid)) + } + if tt.claimDeleted { + claim.DeletionTimestamp = new(metav1.Now()) + } + objects := []client.Object{claim} + for _, run := range tt.runs { + objects = append(objects, run.DeepCopy()) + } + for i, uid := range tt.jobFor { + owner := &nvidiacomv1beta2.DynamoGraphDeploymentRun{Name: string(uid), UID: uid} + objects = append(objects, &batchv1.Job{ + Name: fmt.Sprintf("job-%d-aisimulate", i), Namespace: "default", + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(owner, nvidiacomv1beta2.GroupVersion.WithKind("DynamoGraphDeploymentRun")), + }, + }) + } + reconciler, c, hide := newDGDRRunTraceClaimFixture(t, objects...) + *hide = tt.hideJobs + + t.Log("Reconcile the claim") + if _, err := reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(claim)}); err != nil { + t.Fatalf("reconcile: %v", err) + } + + t.Log("Only the finalizers of Runs that can still execute remain, beside foreign ones") + stored := &corev1.PersistentVolumeClaim{} + if err := c.Get(t.Context(), client.ObjectKeyFromObject(claim), stored); err != nil { + t.Fatalf("get claim: %v", err) + } + want := append([]string{otherFinalizer}, tt.want...) + if !reflect.DeepEqual(stored.Finalizers, want) { + t.Fatalf("finalizers = %v, want %v", stored.Finalizers, want) + } + }) + } +} + +func TestDGDRRunTraceClaimReconcilerReleasesTheClaimOfADeletedRun(t *testing.T) { + claim := traceClaimForTest() + claim.Finalizers = []string{"kubernetes.io/pvc-protection"} + f := newDGDRRunFixture(t, dgdrRunWithTraceForTest(t, map[string]any{}), claim) + claimReconciler := &DynamoGraphDeploymentRunTraceClaimReconciler{Client: f.client, APIReader: f.client} + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(claim)} + f.reconcile() + job := f.job() + + t.Log("The Run is deleted while its Job still runs; the Job keeps the claim protected") + if err := f.client.Delete(t.Context(), f.run()); err != nil { + t.Fatalf("delete Run: %v", err) + } + if _, err := claimReconciler.Reconcile(t.Context(), request); err != nil { + t.Fatalf("reconcile claim: %v", err) + } + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection", dgdrRunTraceClaimFinalizer("run-uid")) + + t.Log("The Job's deletion event maps to the claim, which is then released") + if got := dgdrRunTraceClaimOfJob(t.Context(), job); !reflect.DeepEqual(got, []reconcile.Request{request}) { + t.Fatalf("Job event requests = %v, want %v", got, []reconcile.Request{request}) + } + if err := f.client.Delete(t.Context(), job); err != nil { + t.Fatalf("delete Job: %v", err) + } + if _, err := claimReconciler.Reconcile(t.Context(), request); err != nil { + t.Fatalf("reconcile claim: %v", err) + } + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection") +} + +func TestDGDRRunTraceClaimReconcilerWatchesOnlyWhatReleasesAClaim(t *testing.T) { + run := dgdrRunWithTraceForTest(t, map[string]any{}) + withoutTrace := dgdrRunForTest(t, map[string]any{}) + claimRequest := []reconcile.Request{{Namespace: "default", Name: "request-traces"}} + unowned := &batchv1.Job{Name: "nightly", Namespace: "default", Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{Volumes: []corev1.Volume{dgdrRunTraceClaimVolume(run.Spec.Workload.Trace.Source.PVC)}}, + }}} + + t.Log("Claims pass the filter only while a Run protects them") + protected := traceClaimForTest() + protected.Finalizers = []string{"kubernetes.io/pvc-protection", dgdrRunTraceClaimFinalizer("run-uid")} + if !dgdrRunProtectsTraceClaim(protected) || dgdrRunProtectsTraceClaim(traceClaimForTest()) { + t.Fatal("only a claim with a Run finalizer passes the filter") + } + + t.Log("Run and Job deletions map to the claim the Run reads") + if got := dgdrRunTraceClaimOfRun(t.Context(), run); !reflect.DeepEqual(got, claimRequest) { + t.Fatalf("Run requests = %v, want %v", got, claimRequest) + } + if got := dgdrRunTraceClaimOfRun(t.Context(), withoutTrace); len(got) != 0 { + t.Fatalf("Run without a trace requests = %v, want none", got) + } + if got := dgdrRunTraceClaimOfJob(t.Context(), unowned); len(got) != 0 { + t.Fatalf("Job no Run controls requests = %v, want none", got) + } + + t.Log("Only deletions pass the Run and Job event filter") + if dgdrRunDeleteEvents.Create(event.CreateEvent{Object: run}) || dgdrRunDeleteEvents.Update(event.UpdateEvent{ObjectOld: run, ObjectNew: run}) || + !dgdrRunDeleteEvents.Delete(event.DeleteEvent{Object: run}) || dgdrRunDeleteEvents.Generic(event.GenericEvent{Object: run}) { + t.Fatal("the event filter must pass deletions only") + } } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_trace_claim_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_trace_claim_controller.go new file mode 100644 index 000000000000..c6db3a4392c6 --- /dev/null +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_trace_claim_controller.go @@ -0,0 +1,172 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "context" + "fmt" + "slices" + "strings" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// DynamoGraphDeploymentRunTraceClaimReconciler releases the finalizers by which Runs protect their +// trace claims when the Run reconciler cannot: a Run has no finalizer of its own, so it can be +// deleted, by its Request or with its namespace, before it released its claim. The claim keeps a +// Run's finalizer while that Run exists and is not being deleted, or while a Job the Run controls +// exists. The Run reconciler releases the finalizer of a completed Run itself. +type DynamoGraphDeploymentRunTraceClaimReconciler struct { + client.Client + APIReader client.Reader + Config *configv1alpha1.OperatorConfiguration + RuntimeConfig *commoncontroller.RuntimeConfig +} + +func (r *DynamoGraphDeploymentRunTraceClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + claim := &corev1.PersistentVolumeClaim{} + if err := r.Get(ctx, req.NamespacedName, claim); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Drop the finalizer of every Run that can no longer execute against the claim. + original := claim.DeepCopy() + for _, finalizer := range original.Finalizers { + uid, ok := strings.CutPrefix(finalizer, dgdrRunTraceClaimFinalizerPrefix) + if !ok { + continue + } + held, err := r.runHoldsClaim(ctx, claim, types.UID(uid)) + if err != nil { + return ctrl.Result{}, err + } + if !held { + controllerutil.RemoveFinalizer(claim, finalizer) + } + } + if len(claim.Finalizers) == len(original.Finalizers) { + return ctrl.Result{}, nil + } + if err := r.Patch(ctx, claim, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{}, fmt.Errorf("release trace PersistentVolumeClaim %q: %w", claim.Name, err) + } + return ctrl.Result{}, nil +} + +// runHoldsClaim reports whether the Run with uid may still execute against claim: the Run exists +// and is not being deleted, or a Job it controls exists. A Run never appears after its finalizer, so +// the cache answers for the Run. The Job cache can miss a Job the Run created just before it was +// deleted, and releasing the claim cannot be undone, so the API server confirms that no such Job is +// left before the claim is released. claim must not be nil. +func (r *DynamoGraphDeploymentRunTraceClaimReconciler) runHoldsClaim( + ctx context.Context, + claim *corev1.PersistentVolumeClaim, + uid types.UID, +) (bool, error) { + runs := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} + if err := r.List(ctx, runs, client.InNamespace(claim.Namespace), + client.MatchingFields{dgdrRunTraceClaimIndex: claim.Name}); err != nil { + return false, fmt.Errorf("list the Runs that read trace PersistentVolumeClaim %q: %w", claim.Name, err) + } + for i := range runs.Items { + if runs.Items[i].UID == uid && runs.Items[i].DeletionTimestamp == nil { + return true, nil + } + } + + // A Job of the Run, as the cache and then the API server show it. + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace(claim.Namespace), + client.MatchingFields{dgdrRunJobControllerUIDIndex: string(uid)}); err != nil { + return false, fmt.Errorf("list the Jobs of Run %q: %w", uid, err) + } + if len(jobs.Items) > 0 { + return true, nil + } + live := &batchv1.JobList{} + if err := r.APIReader.List(ctx, live, client.InNamespace(claim.Namespace)); err != nil { + return false, fmt.Errorf("list the Jobs of Run %q from the API: %w", uid, err) + } + for i := range live.Items { + if slices.Contains(dgdrRunJobControllerUIDs(&live.Items[i]), string(uid)) { + return true, nil + } + } + return false, nil +} + +// dgdrRunProtectsTraceClaim reports whether obj carries the finalizer of a Run. +func dgdrRunProtectsTraceClaim(obj client.Object) bool { + return slices.ContainsFunc(obj.GetFinalizers(), func(finalizer string) bool { + return strings.HasPrefix(finalizer, dgdrRunTraceClaimFinalizerPrefix) + }) +} + +// dgdrRunTraceClaimOfRun maps a Run event to the trace claim the Run reads, if any. +func dgdrRunTraceClaimOfRun(_ context.Context, obj client.Object) []reconcile.Request { + run, ok := obj.(*nvidiacomv1beta2.DynamoGraphDeploymentRun) + if !ok { + return nil + } + source := dgdrRunTraceClaim(run) + if source == nil { + return nil + } + return []reconcile.Request{{Namespace: run.Namespace, Name: source.ClaimName}} +} + +// dgdrRunTraceClaimOfJob maps an event of a Job a Run controls to the trace claim the Job mounts, if +// any. +func dgdrRunTraceClaimOfJob(_ context.Context, obj client.Object) []reconcile.Request { + job, ok := obj.(*batchv1.Job) + if !ok || len(dgdrRunJobControllerUIDs(job)) == 0 { + return nil + } + trace, found := dgdrRunVolume(job.Spec.Template.Spec.Volumes, dgdrRunTraceVolume) + if !found || trace.PersistentVolumeClaim == nil { + return nil + } + return []reconcile.Request{{Namespace: job.Namespace, Name: trace.PersistentVolumeClaim.ClaimName}} +} + +// dgdrRunDeleteEvents passes only deletions: a Run or Job releases a claim only by going away, and +// claim events cover every other state, including the initial list after a restart. +var dgdrRunDeleteEvents = predicate.Funcs{ + CreateFunc: func(event.CreateEvent) bool { return false }, + UpdateFunc: func(event.UpdateEvent) bool { return false }, + DeleteFunc: func(event.DeleteEvent) bool { return true }, + GenericFunc: func(event.GenericEvent) bool { return false }, +} + +// SetupWithManager registers the reconciler. It relies on the Run and Job indexes that the Run +// reconciler registers, so it is set up after that reconciler. +func (r *DynamoGraphDeploymentRunTraceClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { + if r.APIReader == nil { + r.APIReader = mgr.GetAPIReader() + } + return ctrl.NewControllerManagedBy(mgr). + Named("dynamographdeploymentrun-v1beta2-trace-claim"). + For(&corev1.PersistentVolumeClaim{}, builder.WithPredicates(predicate.NewPredicateFuncs(dgdrRunProtectsTraceClaim))). + Watches(&nvidiacomv1beta2.DynamoGraphDeploymentRun{}, handler.EnqueueRequestsFromMapFunc(dgdrRunTraceClaimOfRun), + builder.WithPredicates(dgdrRunDeleteEvents)). + Watches(&batchv1.Job{}, handler.EnqueueRequestsFromMapFunc(dgdrRunTraceClaimOfJob), + builder.WithPredicates(dgdrRunDeleteEvents)). + WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(r.Config, r.RuntimeConfig)). + Complete(r) +} diff --git a/deploy/operator/internal/controller/pod_cache_contract_test.go b/deploy/operator/internal/controller/pod_cache_contract_test.go index 91f4c9422fd7..6cdea36e8d6a 100644 --- a/deploy/operator/internal/controller/pod_cache_contract_test.go +++ b/deploy/operator/internal/controller/pod_cache_contract_test.go @@ -138,6 +138,46 @@ func TestProjectedPodSupportsControllerContract(t *testing.T) { assert.Contains(t, detail, "profiling failed") }) + t.Run("DGDR Run diagnostics", func(t *testing.T) { + t.Log("Project a pod of a Run's Job whose sweeper waits on a mount error") + job := &batchv1.Job{Name: "search-1-aisimulate", Namespace: pod.Namespace, UID: types.UID("job-uid")} + runPod := podcache.Project(&corev1.Pod{ + Name: "search-1-aisimulate-abcde", + Namespace: pod.Namespace, + Labels: map[string]string{batchv1.JobNameLabel: job.Name}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(job, batchv1.SchemeGroupVersion.WithKind("Job"))}, + Spec: corev1.PodSpec{ + NodeName: "node-a", + Containers: []corev1.Container{{Name: dgdrRunSweeperContainer}, {Name: dgdrRunPublisherContainer}}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + {Name: dgdrRunPublisherContainer, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, + {Name: dgdrRunSweeperContainer, State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{ + Reason: "CreateContainerConfigError", Message: `failed to prepare subPath for volumeMount "trace"`, + }}}, + }, + }, + }) + + t.Log("The Run controller finds the pod of its Job and reports the waiting sweeper verbatim") + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + reconciler := &DynamoGraphDeploymentRunV1beta2Reconciler{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(runPod).Build(), + } + pods, err := reconciler.jobPods(context.Background(), job) + require.NoError(t, err) + require.Len(t, pods, 1) + container, waiting, found := dgdrRunWaitingContainer(pods) + assert.True(t, found) + assert.Equal(t, dgdrRunSweeperContainer, container) + assert.Equal(t, corev1.ContainerStateWaiting{ + Reason: "CreateContainerConfigError", Message: `failed to prepare subPath for volumeMount "trace"`, + }, waiting) + }) + t.Run("Recreate", func(t *testing.T) { // PR #11909 lists old worker Pods by these labels and treats only // Failed/Succeeded phases (or cache disappearance) as terminated. diff --git a/deploy/operator/internal/controller/setup.go b/deploy/operator/internal/controller/setup.go index 56355c42473d..d11d48583225 100644 --- a/deploy/operator/internal/controller/setup.go +++ b/deploy/operator/internal/controller/setup.go @@ -156,7 +156,8 @@ func SetupDynamoGraphDeploymentRequestV1beta2(mgr ctrl.Manager, opts SetupOption } // SetupDynamoGraphDeploymentRunV1beta2 registers the controller that executes immutable -// v1beta2 Runs as AISimulate Jobs and observes their lifecycle. +// v1beta2 Runs as AISimulate Jobs and observes their lifecycle, and the controller that releases +// the trace claims of deleted Runs. func SetupDynamoGraphDeploymentRunV1beta2(mgr ctrl.Manager, opts DynamoGraphDeploymentRunSetupOptions) error { if err := (&DynamoGraphDeploymentRunV1beta2Reconciler{ Client: mgr.GetClient(), @@ -170,6 +171,13 @@ func SetupDynamoGraphDeploymentRunV1beta2(mgr ctrl.Manager, opts DynamoGraphDepl }).SetupWithManager(mgr); err != nil { return fmt.Errorf("unable to create DynamoGraphDeploymentRun v1beta2 controller: %w", err) } + if err := (&DynamoGraphDeploymentRunTraceClaimReconciler{ + Client: mgr.GetClient(), + Config: opts.Config, + RuntimeConfig: opts.RuntimeConfig, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create DynamoGraphDeploymentRun v1beta2 trace claim controller: %w", err) + } return nil } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index fad4e5095676..b5130e2e1571 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3986,7 +3986,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `claimName` _string_ | ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace.
The Run Job mounts only the trace file from it, read-only, and only into the AISimulate
container. | | MaxLength: 253
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
| +| `claimName` _string_ | ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace.
The Run Job mounts only the trace file from it, read-only, and only into the AISimulate
container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-`
until its execution stops; deleting the claim cancels the Runs that read it. | | MaxLength: 253
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
| | `path` _string_ | Path names one trace file relative to the root of the claim. It is neither a directory
nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..'
segments or a NUL character, which no file name can hold. The file must be complete before
the Request is created and must never be rewritten: every Run of the Request reads it when
its Job starts. | | MaxLength: 1024
MinLength: 1
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index 088e8fcfb625..8bd9af5abccb 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -1166,9 +1166,10 @@ The operator mounts only this file, read-only and only into the AISimulate conta `subPath` of the claim at `/var/run/aisimulate/trace/trace.jsonl.gz`, or at `/var/run/aisimulate/trace/trace.jsonl` when `path` does not end in `.gz`. The kubelet resolves `path` inside the claim and refuses to mount a file outside it. If the file does not exist when -the Job starts, the kubelet cannot mount it either. On a volume type whose node directory the -kubelet can write, such as `hostPath`, the kubelet instead creates an empty directory at `path` on -the claim, and the Run fails when AISimulate cannot read it. +the Job starts, the kubelet cannot mount it either, and the AISimulate container waits; see +[Trace claim lifecycle](#trace-claim-lifecycle). On a volume type whose node directory the kubelet +can write, such as `hostPath`, the kubelet instead creates an empty directory at `path` on the +claim, and the Run fails when AISimulate cannot read it. `format: dynamo` is the only admitted format. It reads the Dynamo request trace (`dynamo.request.trace.v1`) written by the frontend, as JSON Lines or as gzip-compressed JSON Lines @@ -1183,31 +1184,78 @@ non-object value. Admission rejects these overrides, and the Run controller rest values. Other `traffic` overrides, such as `traffic.load` and `traffic.stop`, go to AISimulate, which validates the complete input when the Run's Job starts; an input it rejects fails the Run. -The Run fails, and its `Completed` condition has reason `TraceSourceUnavailable`, when its Job could -never read the trace: - -- The claim does not exist, is being deleted, is a block volume, or has lost its volume. The - operator checks this before it creates the Job, and again whenever the claim or the Job's pod - changes until the AISimulate container starts. Once a pod is scheduled, Kubernetes keeps a claim - being deleted until the pod ends, and the pod can still mount it. -- The kubelet cannot mount the trace file, for example because it does not exist, is an absolute - symlink, or is a symlink that leaves the claim. - -The operator records the failure on the Run and then deletes such a Job. A claim that is still -`Pending`, such as one that waits for its first consumer, is accepted. The request starts one Run -per `metadata.generation`, so after you fix the claim or the file, change the request's `spec` to -start a new Run. +### Trace claim lifecycle + +The operator ties the claim's lifetime to the Runs that read it. It decides from structured state +only: the claim's `deletionTimestamp`, `volumeMode`, and phase, the Job's conditions, and the +request's `spec`. It copies container status messages into the Run but never interprets them. + +- **Protection.** Before a Run creates its Job, the operator adds the finalizer + `nvidia.com/dgdr-run-` to the claim, where `` is the Run's `metadata.uid`. Every + Run that reads the claim adds its own finalizer, so Runs of several requests can share a claim. The + operator removes a Run's finalizer once the Run is complete and none of its Jobs is still running, + or once the Run and its Jobs are deleted, for example when the request retires old Runs or is + deleted. +- **Deleting the claim cancels its Runs.** When the claim has a `deletionTimestamp`, the operator + stops the Job of every Run that reads it through foreground deletion, whether or not the Job's pod + has started. Once the Job and its pod are gone, the Run fails with reason `TraceSourceDeleted`, and + the operator then removes the Run's finalizer. A Job that already completed or failed is not + stopped; its own outcome decides the Run. A Run whose claim is already being deleted before it + creates its Job fails with `TraceSourceDeleted` without creating anything. +- **A missing claim keeps the Run open.** When the claim does not exist, the Run stays `Pending` + with `Completed=False` and reason `TraceSourceMissing`, and starts once you create the claim. +- **A claim no Job can mount fails the Run before it starts.** Before the Run creates its Job, a + `volumeMode: Block` claim or a claim in phase `Lost` fails the Run with reason + `TraceSourceUnavailable`. After the Job exists, the operator does not check the claim's volume mode + or phase again; the Job's pod waits until the volume is usable, the Job fails, or you cancel the + Run. +- **A waiting container is a diagnostic, not a failure.** While a container of the Run's Job waits + on an error, such as a trace file the kubelet cannot mount (a missing file, an absolute symlink, + or a symlink that leaves the claim) or an image it cannot pull, the Run stays open with + `Completed=False`, reason `ContainerWaiting`, and the message + `container "" is waiting: : `, where `` and `` are copied + from the container's status. The operator does not delete the Job, and there is no timeout. The + kubelet keeps retrying the container, so fixing the cause, for example writing the missing file, + lets the same Run continue. Waiting in `ContainerCreating` or `PodInitializing` is not reported: + volume errors that keep a pod there, such as a `ReadWriteOnce` claim attached to another node, + appear only in the pod's events. + +To stop a Run that cannot continue, cancel it explicitly: change the request's `spec` (the Run +fails with reason `Superseded`), delete the request, or delete the claim (`TraceSourceDeleted`). A +Pending claim, such as one that waits for its first consumer, is accepted. The request starts one +Run per `metadata.generation`, so after a Run failed, change the request's `spec` to start a new +Run. ```bash # Active and recent Runs of the request kubectl get dynamographdeploymentrequests.v1beta2.nvidia.com qwen3-trace-replay -n "$NAMESPACE" \ -o jsonpath='{.status.activeRunRef}{"\n"}{.status.recentRunRefs}{"\n"}' -# Why a Run failed +# Why a Run failed or waits kubectl get dynamographdeploymentrun qwen3-trace-replay-1 -n "$NAMESPACE" \ -o jsonpath='{.status.conditions}' ``` + +Kubernetes keeps a deleted claim `Terminating` while any pod that mounts it exists +(`kubernetes.io/pvc-protection`), including the completed pods of finished Jobs. The operator keeps +the Runs, Jobs, and pods of a request's five most recent finished Runs, so a deleted trace claim can +stay `Terminating` after every Run released it, until those Runs are retired or the request is +deleted. + + +If the operator is uninstalled, or the `v1beta2` API is disabled, while Runs are in flight, their +finalizers stay on the claim and keep it from being deleted. List them, then remove only the +`nvidia.com/dgdr-run-` finalizers (requires `jq`): + +```bash +kubectl get pvc request-traces -n "$NAMESPACE" -o jsonpath='{.metadata.finalizers}{"\n"}' + +kubectl get pvc request-traces -n "$NAMESPACE" -o json \ + | jq '.metadata.finalizers |= map(select(startswith("nvidia.com/dgdr-run-") | not))' \ + | kubectl replace -f - +``` + ## Known issues - **PCIe profiler data not yet available.** See the PCIe callout under [SKU format](#sku-format). diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index caf6c3712f52..7bac788cb0f1 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -5568,7 +5568,7 @@ TracePVCSource names one trace file on a PersistentVolumeClaim in the Request na **Appears in:** [TraceSourceSpec](#tracesourcespec) -ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate container. +ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` until its execution stops; deleting the claim cancels the Runs that read it. **Validation:** MaxLength: 253 MinLength: 1 Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` From bde7f19ff3048166c250442fa567a29f93fe88c1 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 30 Oct 2026 14:01:10 -0700 Subject: [PATCH 072/100] test(operator): render the trace scenario through the default AIC generator The vllm-trace ClusterEnv scenario forced dgdr_run.generator: direct, so it could pass while an ordinary trace Request failed to render its Candidate through the default AIC generator. The AIC renderer forwards the trace workload since #17474 and #17523, so drop the override and keep the scenario on the default path. The pinned AISimulate still rejects a candidate without an explicit block size (#17581), so the scenario takes the same TODO(#17581) block-size workaround as the static vllm scenario. The Candidate golden is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../testdata/dgdr-v1beta2/vllm-trace/input.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml index 87d43e666645..3dc482a2bf68 100644 --- a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml @@ -2,8 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # The test writes trace.jsonl, gzip-compressed, to retune/w-1.jsonl.gz on the request-traces claim -# before it applies this Request. The direct generator renders the Candidate without the AIC -# generator bridge. +# before it applies this Request. apiVersion: nvidia.com/v1beta2 kind: DynamoGraphDeploymentRequest metadata: @@ -32,6 +31,10 @@ spec: mode: aggregated context_length: 32768 workers: + # TODO(#17581): Remove this explicit value after AISimulate preserves an unset runtime block size. + aggregated: + kv_cache: + block_size: 64 prefill: null decode: null optimizer: @@ -40,7 +43,6 @@ spec: parallelism: 1 candidate_timeout_seconds: 120 dgdr_run: - generator: direct snapshot_interval_seconds: 0 recommendation: maxCandidates: 1 From f73f4bbd0bcff6ea2cc8e566d21e68f940efdbac Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 30 Oct 2026 14:22:12 -0700 Subject: [PATCH 073/100] fix(operator): finish a started trace-claim cancellation and never stop a Job on a stale read A Job the operator foreground-deletes for a deleted trace claim can be marked JobFailed by the Job controller while its pod is removed. The finished-Job exemption then took that failure as the Job's own outcome and the Run never reported TraceSourceDeleted. The exemption now applies only to a Job that finished while nothing was deleting it. stopOwnedJobs now fences each deletion with the UID and resourceVersion of the Job as it was read. A Job that finished after the cache last showed it is no longer stopped on that stale view: the API server reports a Conflict and the next pass decides again from the finished Job, which keeps its outcome. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- ...mographdeploymentrun_v1beta2_controller.go | 26 +++++++--- ...phdeploymentrun_v1beta2_controller_test.go | 51 +++++++++++++++++++ .../dynamo-graph-deployment-request.mdx | 6 ++- 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index d0c9b0a2fcdd..e3dcb415e0db 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -260,13 +260,15 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) reconcileRun( } // Deleting the claim cancels the Run whatever its pod has started, unless its Job already - // finished: the Job's own outcome then decides the Run. A claim that is gone after the Run + // finished on its own: the Job's own outcome then decides the Run. A Job that is being deleted + // is cancelled even when it finished, because the Job controller fails a Job whose pod the + // deletion removes, and a started cancellation must end as one. A claim that is gone after the Run // created its Job was protected by the Run, so it was deleted, too. Before the first Job, a // missing claim keeps the Run open, and only a claim no Job could ever mount fails it. executed := job != nil || run.Status.JobRef != nil switch { case (claim != nil && claim.DeletionTimestamp != nil) || (claim == nil && executed): - if job == nil || !dgdrRunJobFinished(job) { + if job == nil || !dgdrRunJobFinishedOnItsOwn(job) { return r.cancelForDeletedTraceClaim(ctx, run, jobs, source) } case claim == nil: @@ -347,7 +349,10 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) stopAndCompleteRun( } // stopOwnedJobs stops every execution visible through either the informer cache or the API -// server. Completion is safe only after the uncached reader also confirms absence. +// server. Completion is safe only after the uncached reader also confirms absence. Each deletion is +// fenced by the UID and resourceVersion of the Job as it was read, so a Job that changed since, such +// as one that finished meanwhile, is not stopped on a stale decision: the API server reports a +// Conflict, and the next pass decides again from the changed Job. func (r *DynamoGraphDeploymentRunV1beta2Reconciler) stopOwnedJobs( ctx context.Context, run *nvidiacomv1beta2.DynamoGraphDeploymentRun, @@ -369,7 +374,9 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) stopOwnedJobs( if !jobs[i].DeletionTimestamp.IsZero() { continue } - if err := r.Delete(ctx, &jobs[i], client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil && !apierrors.IsNotFound(err) { + err := r.Delete(ctx, &jobs[i], client.PropagationPolicy(metav1.DeletePropagationForeground), + client.Preconditions{UID: &jobs[i].UID, ResourceVersion: &jobs[i].ResourceVersion}) + if err != nil && !apierrors.IsNotFound(err) && !apierrors.IsConflict(err) { return false, fmt.Errorf("stop Job %q %s of Run %q: %w", jobs[i].Name, action, run.Name, err) } } @@ -539,7 +546,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) protectTraceClaim( // cancelForDeletedTraceClaim stops the execution of run because its trace claim is being deleted or // is gone, and then fails run with TraceSourceDeleted. jobs are the Jobs of run the cache shows. // When the cache shows none, a Job the API server already holds is stopped as well, unless it -// finished: its own outcome decides the Run once the cache shows it. The Run keeps its finalizer on +// finished on its own: its own outcome decides the Run once the cache shows it. The Run keeps its finalizer on // the claim until it completes. run and source must not be nil. func (r *DynamoGraphDeploymentRunV1beta2Reconciler) cancelForDeletedTraceClaim( ctx context.Context, @@ -553,7 +560,7 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) cancelForDeletedTraceClaim( return ctrl.Result{}, err } for i := range live { - if dgdrRunJobFinished(&live[i]) { + if dgdrRunJobFinishedOnItsOwn(&live[i]) { return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil } } @@ -671,6 +678,13 @@ func dgdrRunJobFinished(job *batchv1.Job) bool { return false } +// dgdrRunJobFinishedOnItsOwn reports whether job finished while nothing was deleting it. A Job +// that is being deleted can still be marked failed, because the Job controller counts the pods the +// deletion removes as failed, so its outcome would only echo the deletion. job must not be nil. +func dgdrRunJobFinishedOnItsOwn(job *batchv1.Job) bool { + return job.DeletionTimestamp.IsZero() && dgdrRunJobFinished(job) +} + // jobPods returns the pods job controls, as the cache shows them. job must not be nil. func (r *DynamoGraphDeploymentRunV1beta2Reconciler) jobPods(ctx context.Context, job *batchv1.Job) ([]corev1.Pod, error) { list := &corev1.PodList{} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 4d090a5af166..21925d744b57 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -2271,6 +2271,9 @@ func TestV1beta2RunControllerCancelsARunWhoseTraceClaimIsDeleted(t *testing.T) { podNode string // removeClaim removes the claim completely, as when its finalizers were removed by hand. removeClaim bool + // failDuringDeletion marks the Job failed while it is being deleted, as the Job controller + // does once the deletion removed its pod. + failDuringDeletion bool }{ { name: "the Job counts a pod the pod cache does not show", @@ -2323,6 +2326,13 @@ func TestV1beta2RunControllerCancelsARunWhoseTraceClaimIsDeleted(t *testing.T) { podNode: "node-a", removeClaim: true, }, + { + name: "the Job is marked failed while it is being deleted", + status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, + pod: &corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: runningSweeper}, + podNode: "node-a", + failDuringDeletion: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -2387,6 +2397,23 @@ func TestV1beta2RunControllerCancelsARunWhoseTraceClaimIsDeleted(t *testing.T) { assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection", finalizer) } + if tt.failDuringDeletion { + t.Log("A Job marked failed during its deletion is still cancelled, not taken as its own outcome") + deleting := f.job() + deleting.Status.Conditions = []batchv1.JobCondition{ + {Type: batchv1.JobFailureTarget, Status: corev1.ConditionTrue}, + {Type: batchv1.JobFailed, Status: corev1.ConditionTrue}, + } + if err := f.client.Status().Update(t.Context(), deleting); err != nil { + t.Fatalf("fail the deleting Job: %v", err) + } + f.reconcile() + if meta.IsStatusConditionTrue(f.run().Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) { + t.Fatalf("status = %+v, want the Run open until its Job is gone", f.run().Status) + } + assertDGDRRunTraceClaimFinalizers(t, f, "kubernetes.io/pvc-protection", finalizer) + } + t.Log("Once the Job and its pods are gone, the Run fails with TraceSourceDeleted and keeps the claim until then") job = f.job() job.Finalizers = nil @@ -2461,6 +2488,8 @@ func TestV1beta2RunControllerLetsAFinishedJobDecideAfterItsTraceClaimIsDeleted(t wantReason string // hideJob hides the Job from the cached List for the first pass after the deletion. hideJob bool + // staleJob shows the Job unfinished in the cached List for the first pass after the deletion. + staleJob bool }{ {name: "the Job completed", condition: batchv1.JobComplete, wantPhase: nvidiacomv1beta2.DGDRRunPhaseSucceeded, wantReason: dgdrRunReasonSucceeded}, @@ -2468,6 +2497,8 @@ func TestV1beta2RunControllerLetsAFinishedJobDecideAfterItsTraceClaimIsDeleted(t wantPhase: nvidiacomv1beta2.DGDRRunPhaseFailed, wantReason: dgdrRunReasonJobFailed}, {name: "the Job completed but the cache does not show it yet", condition: batchv1.JobComplete, wantPhase: nvidiacomv1beta2.DGDRRunPhaseSucceeded, wantReason: dgdrRunReasonSucceeded, hideJob: true}, + {name: "the Job completed but the cache still shows it running", condition: batchv1.JobComplete, + wantPhase: nvidiacomv1beta2.DGDRRunPhaseSucceeded, wantReason: dgdrRunReasonSucceeded, staleJob: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -2478,6 +2509,7 @@ func TestV1beta2RunControllerLetsAFinishedJobDecideAfterItsTraceClaimIsDeleted(t t.Log("The Job finishes, and then the claim is deleted") job := f.job() + unfinished := job.DeepCopy() job.Status.StartTime = new(metav1.Now()) job.Status.Conditions = []batchv1.JobCondition{{Type: tt.condition, Status: corev1.ConditionTrue}} if err := f.client.Status().Update(t.Context(), job); err != nil { @@ -2505,6 +2537,25 @@ func TestV1beta2RunControllerLetsAFinishedJobDecideAfterItsTraceClaimIsDeleted(t f.reconciler.Client = cached } + if tt.staleJob { + t.Log("A pass that sees the Job unfinished in a stale cache cannot stop the finished Job") + cached := f.reconciler.Client + f.reconciler.Client = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if jobs, ok := list.(*batchv1.JobList); ok { + jobs.Items = []batchv1.Job{*unfinished.DeepCopy()} + return nil + } + return c.List(ctx, list, opts...) + }, + }) + result, err := f.reconciler.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(f.run())}) + if err != nil || result.RequeueAfter != dgdrRunExecutionCacheRetry { + t.Fatalf("result/error = %+v/%v, want a retry", result, err) + } + f.reconciler.Client = cached + } + t.Log("The finished Job is kept and decides the Run") f.reconcile() if kept := f.job(); kept.UID != job.UID || kept.DeletionTimestamp != nil { diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index 8bd9af5abccb..c6251b17ef44 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -1199,8 +1199,10 @@ request's `spec`. It copies container status messages into the Run but never int - **Deleting the claim cancels its Runs.** When the claim has a `deletionTimestamp`, the operator stops the Job of every Run that reads it through foreground deletion, whether or not the Job's pod has started. Once the Job and its pod are gone, the Run fails with reason `TraceSourceDeleted`, and - the operator then removes the Run's finalizer. A Job that already completed or failed is not - stopped; its own outcome decides the Run. A Run whose claim is already being deleted before it + the operator then removes the Run's finalizer. A Job that already completed or failed before the + claim's deletion, and is not itself being deleted, is not stopped; its own outcome decides the Run. + Once the operator has started stopping a Job, the Run fails with `TraceSourceDeleted` even if the + Job is marked failed while its pod is removed. A Run whose claim is already being deleted before it creates its Job fails with `TraceSourceDeleted` without creating anything. - **A missing claim keeps the Run open.** When the claim does not exist, the Run stays `Pending` with `Completed=False` and reason `TraceSourceMissing`, and starts once you create the claim. From e9ab6afc24fd01cb1f12b31b991d4ef9230c2ea8 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 30 Oct 2026 15:48:31 -0700 Subject: [PATCH 074/100] feat(replay): replay DGDR trace Runs with common random numbers DynamoReplayRunner now replays every candidate of a trace sweep with _deterministic=True: ordinal request ids and the canonical router seed. Candidates of one trace then differ only by their configuration, so a candidate's score does not depend on the other candidates, the Run or the parallelism. Live planner retune compares candidates that different DGDR Runs of one window scored on that basis. The synthetic-workload path is unchanged. First layer of the live retune stack split from #17562. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/replay/simulation.py | 9 ++ .../replay/tests/test_simulation_crn.py | 140 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 components/src/dynamo/replay/tests/test_simulation_crn.py diff --git a/components/src/dynamo/replay/simulation.py b/components/src/dynamo/replay/simulation.py index 5b1a5bd49936..ca04ea4b9874 100644 --- a/components/src/dynamo/replay/simulation.py +++ b/components/src/dynamo/replay/simulation.py @@ -409,6 +409,13 @@ def _run_trace( # embedded value instead of imposing the synthetic-workload default. if trace_format not in {"weka", "dynamo"} and trace_block_size is None: trace_block_size = self.trace_block_size + # Common random numbers: every candidate of a trace sweep replays the + # same trace with ordinal request ids and the canonical router seed, + # so candidates differ only by their configuration and a score does + # not depend on the other candidates, the Run or the parallelism. + # Live planner retune compares candidates scored by different DGDR + # Runs of one window on that basis. The synthetic path has no such + # option. if deployment.deployment_mode == "agg": return run_trace_replay( trace_files=trace_files, @@ -423,6 +430,7 @@ def _run_trace( execution_model=execution_model, extra_engine_args=self._engine_args(deployment, "aggregated"), num_workers=deployment.num_workers, + _deterministic=True, **common, ) return run_trace_replay( @@ -440,6 +448,7 @@ def _run_trace( decode_engine_args=self._engine_args(deployment, "decode"), num_prefill_workers=deployment.num_prefill_workers, num_decode_workers=deployment.num_decode_workers, + _deterministic=True, **common, ) diff --git a/components/src/dynamo/replay/tests/test_simulation_crn.py b/components/src/dynamo/replay/tests/test_simulation_crn.py new file mode 100644 index 000000000000..ea45aeabdf05 --- /dev/null +++ b/components/src/dynamo/replay/tests/test_simulation_crn.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the simulation imports. + +"""Common random numbers in the Sweeper replay runner. + +Trace replays (what a DGDR trace Run evaluates) are deterministic, so live +planner retune can compare candidates scored by different Runs of one +window; synthetic replays have no such option and must not receive it. +""" + +from __future__ import annotations + +import inspect + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from aisimulate.sweeper.replay import BackendDeploymentSpec, ReplaySpec + +from dynamo.replay import ReplayReport +from dynamo.replay import api as replay_api +from dynamo.replay import simulation + +pytestmark = [ + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.planner, +] + +_REPORT = ReplayReport( + summary={"completed_requests": 1, "output_throughput_tok_s": 1.0}, + per_request=None, + coverage={}, + planner=None, +) + + +def _deployment(mode: str) -> BackendDeploymentSpec: + if mode == "agg": + return BackendDeploymentSpec( + deployment_mode="agg", + backend="vllm", + backend_version="0.11.0", + agg_engine_args={"timing_model": {"type": "polynomial"}}, + num_workers=2, + ) + return BackendDeploymentSpec( + deployment_mode="disagg", + backend="vllm", + backend_version="0.11.0", + prefill_engine_args={ + "worker_type": "prefill", + "timing_model": {"type": "polynomial"}, + }, + decode_engine_args={ + "worker_type": "decode", + "timing_model": {"type": "polynomial"}, + }, + num_prefill_workers=1, + num_decode_workers=2, + ) + + +def _strict_fake(real, seen: dict): + """A replay entry point that binds its kwargs to ``real``'s signature.""" + + signature = inspect.signature(real) + + def fake(*args, **kwargs): + signature.bind(*args, **kwargs) + seen.update(kwargs) + return _REPORT + + return fake + + +@pytest.mark.parametrize("mode", ["agg", "disagg"]) +@pytest.mark.parametrize( + "workload", + [ + {"trace_path": "window.jsonl.gz", "trace_format": "dynamo"}, + {"trace_paths": ["a.jsonl", "b.jsonl"], "trace_format": "mooncake"}, + ], +) +def test_trace_replays_are_deterministic(monkeypatch, mode, workload): + seen: dict = {} + monkeypatch.setattr( + simulation, + "run_trace_replay", + _strict_fake(replay_api.run_trace_replay, seen), + ) + spec = ReplaySpec( + backend_deployment=_deployment(mode), + workload=workload, + goal={"target": "throughput"}, + ) + + simulation.DynamoReplayRunnerFactory().create(0).run(spec) + + assert seen["_deterministic"] is True + + +@pytest.mark.parametrize("mode", ["agg", "disagg"]) +def test_synthetic_replays_do_not_receive_the_flag(monkeypatch, mode): + # The synthetic entry point has no such parameter; the strict fake fails + # if the flag leaks into the shared kwargs. + assert ( + "_deterministic" + not in inspect.signature(replay_api.run_synthetic_trace_replay).parameters + ) + seen: dict = {} + monkeypatch.setattr( + simulation, + "run_synthetic_trace_replay", + _strict_fake(replay_api.run_synthetic_trace_replay, seen), + ) + spec = ReplaySpec( + backend_deployment=_deployment(mode), + workload={ + "trace_path": None, + "isl": 512, + "osl": 128, + "num_request_ratio": 10.0, + "concurrency": None, + "request_rate": 20.0, + }, + goal={"target": "throughput"}, + ) + + simulation.DynamoReplayRunnerFactory().create(0).run(spec) + + assert seen # the synthetic path ran + assert "_deterministic" not in seen From 30dd62618b74357fef0ab9f198b1feb3ccb066b5 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 30 Oct 2026 16:23:59 -0700 Subject: [PATCH 075/100] feat(planner): add the live retune configuration and deployment premises Add retune.executor ('local' | 'dgdr') and the retune.dgdr section (DgdrExecutorConfig, DgdrRunConfig) to the planner config, with the configuration premises of live retune: P1 (Kubernetes, decision latency, explicit timeout), P2 (retune-only, router-only), P3 (DGDR router knobs), P5 (a pinned, scalar AISimulate engine), safe claim paths and P13 (the largest grid shard finishes within the sweep deadline). Add the dependency-free live retune contracts (retune/live/contracts.py), the frontend premise checks that parse the startup router config from the DGD (retune/live/frontend_config.py), the live fields of SweepRequest and SweepResult (router_domain, deadline_s, spec_replicas, replicas_verified, generation; failure_reason, details), which replay leaves at their defaults, and the shared test fakes. Nothing runs a live retune yet: a live Planner still rejects any retune at startup (NativePlannerBase), and the planner.retune description still says replay-only; the last layer (#17562) wires the live Planner and updates that description. Replay's _retune_inputs now rejects executor 'dgdr' (with a test): replay runs local sweeps only, and before this check it accepted a dgdr configuration and silently ran local sweeps instead. Second layer of the live retune stack split from #17562. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/config/planner_config.py | 561 ++++- .../dynamo/planner/retune/live/__init__.py | 11 + .../dynamo/planner/retune/live/contracts.py | 470 ++++ .../planner/retune/live/frontend_config.py | 1794 ++++++++++++++++ components/src/dynamo/planner/retune/sweep.py | 26 +- .../dynamo/planner/tests/live_retune_fakes.py | 1098 ++++++++++ .../tests/unit/test_live_retune_config.py | 619 ++++++ .../unit/test_live_retune_frontend_config.py | 1889 +++++++++++++++++ components/src/dynamo/replay/api.py | 8 + .../dynamo/replay/tests/test_retune_replay.py | 32 + 10 files changed, 6504 insertions(+), 4 deletions(-) create mode 100644 components/src/dynamo/planner/retune/live/__init__.py create mode 100644 components/src/dynamo/planner/retune/live/contracts.py create mode 100644 components/src/dynamo/planner/retune/live/frontend_config.py create mode 100644 components/src/dynamo/planner/tests/live_retune_fakes.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_config.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index c2b2903af8c5..fcdcdedbc8d6 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -23,7 +23,7 @@ from dataclasses import asdict from enum import Enum from functools import lru_cache -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Annotated, Any, Dict, Literal, Optional, Protocol, cast from urllib.parse import parse_qsl, urlsplit @@ -847,6 +847,375 @@ def replay_kwargs( ) +class DgdrRunConfig(BaseModel): + """How each DGDR v1beta2 Run of a live retune sweep evaluates its points.""" + + model_config = ConfigDict(extra="forbid") + + parallelism: int = Field( + default=8, + ge=1, + description=( + "AISimulate optimizer.parallelism of each Run. The Run Job's CPU " + "and memory may lower the effective value." + ), + ) + candidate_timeout_seconds: float = Field( + default=1800.0, + gt=0, + allow_inf_nan=False, + description="AISimulate optimizer.candidate_timeout_seconds of each Run.", + ) + max_shards: int = Field( + default=3, + ge=1, + le=16, + description=( + "Most grid shards (Requests) per window. A shard is a Cartesian " + "sub-grid of at most 100 points; off-grid points and the CRN probe " + "are extra single-point Requests and do not count." + ), + ) + + +class DgdrExecutorConfig(BaseModel): + """Live retune through DGDR v1beta2 (``retune.executor: dgdr``). + + The planner cuts each window from the frontend request traces on a shared + RWX claim, creates DGDR v1beta2 Requests that replay it, scores their + Candidates and applies the winner to the frontends' routers. + """ + + model_config = ConfigDict(extra="forbid") + + phase_seconds: float = Field( + default=0.0, + ge=0, + allow_inf_nan=False, + description=( + "Window phase: windows are [phase + k*interval, phase + (k+1)*interval) " + "in epoch seconds, so they survive planner restarts." + ), + ) + claim_name: str = Field( + min_length=1, + description="RWX PersistentVolumeClaim, in the DGD namespace, holding traces.", + ) + mount_path: str = Field( + default="/trace", + min_length=1, + description="Where the planner container mounts claim_name.", + ) + segments_dir: str = Field( + default="segments", + min_length=1, + description="Claim-relative directory of frontend trace segments: ///.", + ) + windows_dir: str = Field( + default="windows", + min_length=1, + description="Claim-relative directory of window files: //.", + ) + startup_router_config: dict[str, Any] | None = Field( + default=None, + description=( + "Optional startup KvRouterConfig JSON. It must equal what the planner " + "parses from the frontend's args and env; parsing is the source." + ), + ) + model: str = Field( + min_length=1, + description="HF repo id the workers serve (worker --model); AISimulate resolves it.", + ) + sku: str = Field(min_length=1, description="spec.hardware.gpu.sku of each Request.") + engine: dict[str, Any] = Field( + description=( + "AISimulate public engine section with every leaf a scalar " + "(mode, backend, backend_version, context_length, " + "workers..parallelism.preset, scheduler, kv_cache). Replicas " + "come from the DGD spec at each launch and must not be set here." + ), + ) + cut_budget_seconds: float = Field( + default=900.0, + gt=0, + allow_inf_nan=False, + description="Deadline of the subprocess that cuts a window file.", + ) + job_startup_seconds: float = Field( + default=300.0, + ge=0, + allow_inf_nan=False, + description="Run Job start-up time reserved when deciding whether a shard can still finish.", + ) + run: DgdrRunConfig = Field(default_factory=DgdrRunConfig) + keep_requests: int = Field( + default=5, + ge=0, + description="Windows whose terminal Requests are kept for audit.", + ) + max_window_rows: int = Field( + default=5_000_000, + ge=1, + description="A window with more requests fails (window_too_large).", + ) + promote_last_advisory: bool = Field( + default=False, + description=( + "When switching from advisory to active, apply the last advisory " + "recommendation if it is recent and its fingerprint still matches." + ), + ) + lease_duration_seconds: float = Field( + default=60.0, + ge=15, + allow_inf_nan=False, + description="Leadership Lease duration; renewed every third of it.", + ) + actuator_interval_seconds: float = Field( + default=15.0, + gt=0, + allow_inf_nan=False, + description="Period of the router actuator and reconciler rounds.", + ) + trace_incomplete_threshold: float = Field( + default=0.05, + ge=0, + le=1, + description=( + "With a Prometheus endpoint, a window whose request count is below " + "the frontend's by more than this fraction fails (trace_incomplete)." + ), + ) + + +# Live retune (``retune.executor: dgdr``) limits. A sweep must end this long +# before the next window's launch: harvest, the decision write and clean-up +# run in that margin. +DGDR_TIMEOUT_MARGIN_SECONDS = 600.0 +# DGDR v1beta2 ``spec.recommendation.maxCandidates`` maximum: one Request (a +# grid shard) carries at most this many router points. +DGDR_MAX_SHARD_POINTS = 100 +# Router knobs a DGDR v1beta2 Request can search, by public name +# (``contracts.DGDR_ROUTER_KNOBS``); the Request's router section has no +# place for the retune extension knobs. +_DGDR_ROUTER_SECTION_KEYS = frozenset( + { + "overlap_score_credit", + "prefill_load_scale", + "temperature", + "policy", + "prefill_load_model", + } +) +_DGDR_ENGINE_MODES = {"agg": "aggregated", "disagg": "disaggregated"} +_DGDR_ENGINE_ROLES = {"agg": ("aggregated",), "disagg": ("prefill", "decode")} +# ``parallelism.preset`` entry keys: AISimulate requires all of these plus +# ``replicas``, which the executor sets from the DGD spec at each launch. +_DGDR_PRESET_KEYS = ("tensor", "pipeline", "attention_data", "moe_tensor", "moe_expert") +_DGDR_PRESET_OPTIONAL_KEYS = ("prefill_context", "decode_context") +# Scheduler knobs AISimulate searches over a default domain when omitted. +_DGDR_SCHEDULER_KNOBS = ("max_batched_tokens", "max_sequences") + + +def dgdr_need_seconds( + points: int, dgdr: DgdrExecutorConfig, *, needs_cut: bool +) -> float: + """Wall-clock seconds one DGDR shard of ``points`` router points needs. + + ``cut_budget`` (when the window file must still be cut) + Run Job + start-up + one round per ``run.parallelism`` candidates, plus one: the + AISimulate optimizer evaluates its first candidate alone. ``parallelism`` + is the configured value; Job resources may lower it, so this is + optimistic. + """ + + if points < 1: + raise ValueError(f"a DGDR shard has at least one point, got {points}") + rounds = 1 + math.ceil((points - 1) / dgdr.run.parallelism) + cut = dgdr.cut_budget_seconds if needs_cut else 0.0 + return cut + dgdr.job_startup_seconds + rounds * dgdr.run.candidate_timeout_seconds + + +def balanced_shard_parts(sizes: Sequence[int], max_shards: int) -> tuple[int, ...]: + """How many chunks to cut each grid dimension into for DGDR sharding. + + ``sizes`` are the value counts of the router grid dimensions. Cutting + dimension ``i`` into ``parts[i]`` contiguous chunks of at most + ``ceil(sizes[i] / parts[i])`` values gives ``prod(parts)`` Cartesian + shards, the largest with ``largest_shard_points(sizes, parts)`` points. + Returns the parts with ``prod(parts) <= max_shards`` that minimize the + largest shard, then the shard count, then lexicographically, so every + caller derives the same split: the default 3 x 9 x 4 grid with + ``max_shards=3`` gives ``(1, 3, 1)``, three shards of 36 points. + """ + + if max_shards < 1: + raise ValueError(f"max_shards must be at least 1, got {max_shards}") + if any(size < 1 for size in sizes): + raise ValueError(f"every grid dimension needs a value, got sizes {sizes}") + + def splits(index: int, budget: int): + if index == len(sizes): + yield () + return + for part in range(1, min(sizes[index], budget) + 1): + for rest in splits(index + 1, budget // part): + yield (part, *rest) + + return min( + splits(0, max_shards), + key=lambda parts: ( + largest_shard_points(sizes, parts), + math.prod(parts), + parts, + ), + ) + + +def largest_shard_points(sizes: Sequence[int], parts: Sequence[int]) -> int: + """Points in the largest shard of ``balanced_shard_parts``' split.""" + + return math.prod( + math.ceil(size / part) for size, part in zip(sizes, parts, strict=True) + ) + + +def _validate_dgdr_engine( + engine: Mapping[str, Any], planner_mode: str, backend: str +) -> None: + """P5: ``retune.dgdr.engine`` pins one engine with scalars only. + + The section is the Request's ``inputOverrides.engine`` merge patch, so + any AISimulate search domain left in it (a ``choices`` or ``range`` + mapping, a parallelism preset list or a default domain) would make the + Run search engine settings next to the router knobs. A written + ``backend`` must be the deployed one (``PlannerConfig.backend``), which + every Request names and which fills an omitted ``backend``. + """ + + name = "retune.dgdr.engine" + _check_dgdr_engine_scalars(engine, (name,)) + written = engine.get("backend") + if written is not None and written != backend: + raise ValueError( + f"{name}.backend {written!r} differs from the deployed backend " + f"{backend!r} (P5); remove it or set it to {backend!r}" + ) + expected_mode = _DGDR_ENGINE_MODES[planner_mode] + if engine.get("mode") != expected_mode: + raise ValueError( + f"{name}.mode must be {expected_mode!r} for planner mode " + f"{planner_mode!r} (P5), got {engine.get('mode')!r}; an omitted mode " + "searches both aggregated and disaggregated engines" + ) + for key, owner in (("model", "model"), ("hardware", "sku")): + if key in engine: + raise ValueError( + f"{name}.{key} would override the Request's typed field (P5); " + f"set retune.dgdr.{owner} instead" + ) + workers = engine.get("workers") + if not isinstance(workers, Mapping): + raise ValueError( + f"{name}.workers must be a mapping that pins every worker role of " + f"planner mode {planner_mode!r} (P5)" + ) + used = _DGDR_ENGINE_ROLES[planner_mode] + for role, worker in workers.items(): + if role not in used and worker is not None: + raise ValueError( + f"{name}.workers.{role} is not a worker role of planner mode " + f"{planner_mode!r}; write it as null or remove it (P5)" + ) + for role in used: + path = f"{name}.workers.{role}" + worker = workers.get(role) + if not isinstance(worker, Mapping): + raise ValueError( + f"{path} is required for planner mode {planner_mode!r} (P5)" + ) + _validate_dgdr_parallelism(worker.get("parallelism"), f"{path}.parallelism") + scheduler = worker.get("scheduler") + scheduler = scheduler if isinstance(scheduler, Mapping) else {} + for knob in _DGDR_SCHEDULER_KNOBS: + if not _is_positive_int(scheduler.get(knob)): + raise ValueError( + f"{path}.scheduler.{knob} must be a positive integer (P5); " + "omitted, AISimulate searches its default domain" + ) + + +def _validate_dgdr_parallelism(parallelism: Any, path: str) -> None: + preset = parallelism.get("preset") if isinstance(parallelism, Mapping) else None + if not ( + isinstance(preset, list) and len(preset) == 1 and isinstance(preset[0], Mapping) + ): + raise ValueError( + f"{path}.preset must be a list of exactly one mapping with " + f"{', '.join(_DGDR_PRESET_KEYS)} (P5); got {preset!r}" + ) + others = sorted(set(parallelism) - {"preset"}) + if others: + raise ValueError( + f"{path} sets {others} next to preset; pin the parallelism in the " + "single preset entry (P5)" + ) + entry = preset[0] + missing = [key for key in _DGDR_PRESET_KEYS if key not in entry] + unknown = sorted( + set(entry) - set(_DGDR_PRESET_KEYS) - set(_DGDR_PRESET_OPTIONAL_KEYS) + ) + if missing or unknown: + raise ValueError( + f"{path}.preset[0] must set exactly {', '.join(_DGDR_PRESET_KEYS)} " + f"(P5); missing={missing}, unknown={unknown}" + ) + for key, value in entry.items(): + if not _is_positive_int(value): + raise ValueError( + f"{path}.preset[0].{key} must be a positive integer (P5), got {value!r}" + ) + + +def _check_dgdr_engine_scalars(value: Any, path: tuple[str, ...]) -> None: + dotted = ".".join(path) + if isinstance(value, Mapping): + domain = sorted({"choices", "range"} & set(value)) + if domain: + raise ValueError( + f"{dotted} is a search domain ({domain[0]}); every " + "retune.dgdr.engine value must be a scalar (P5)" + ) + for key, item in value.items(): + if key == "replicas": + raise ValueError( + f"{dotted}.replicas must not be set: the DGD spec provides " + "replicas at each launch (P5)" + ) + _check_dgdr_engine_scalars(item, (*path, str(key))) + elif isinstance(value, list): + # Only workers..parallelism.preset is a list; its shape is + # checked per used role. + if not ( + len(path) == 5 + and path[1] == "workers" + and path[3:] == ("parallelism", "preset") + ): + raise ValueError( + f"{dotted} is a list; every retune.dgdr.engine value except " + "workers..parallelism.preset must be a scalar (P5)" + ) + for index, item in enumerate(value): + _check_dgdr_engine_scalars(item, (*path[:-1], f"{path[-1]}[{index}]")) + elif value is not None and not isinstance(value, (str, int, float)): + raise ValueError(f"{dotted} must be a scalar, got {type(value).__name__} (P5)") + + +def _is_positive_int(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + class RetuneConfig(BaseModel): """Periodic replay-based retune of router and planner runtime parameters. @@ -981,6 +1350,17 @@ class RetuneConfig(BaseModel): "AISimulate default domain; null does not search the router." ), ) + executor: Literal["local", "dgdr"] = Field( + default="local", + description=( + "Where sweeps run: local worker processes (replay) or DGDR v1beta2 " + "Requests (live; requires dgdr)." + ), + ) + dgdr: DgdrExecutorConfig | None = Field( + default=None, + description="Live DGDR executor settings; required iff executor is dgdr.", + ) planner: Literal["default"] | dict[str, Any] | None = Field( default=None, description=( @@ -1019,6 +1399,12 @@ def _validate_domain_syntax( parse_retune_section(section, value) # type: ignore[call-overload] return value + @model_validator(mode="after") + def _validate_executor(self) -> "RetuneConfig": + if (self.executor == "dgdr") != (self.dgdr is not None): + raise ValueError("retune.dgdr is required iff retune.executor is 'dgdr'") + return self + @model_validator(mode="after") def _validate_timing(self) -> "RetuneConfig": timeout = self.effective_timeout_seconds @@ -1456,8 +1842,8 @@ def _validate_ca_bundle_path(cls, v: Optional[str]) -> Optional[str]: ) # Advisory mode: compute, log and report scaling decisions without - # executing them, live and in planner-in-the-loop replay; replay records - # retune decisions (replay-only) the same way. + # executing them, live and in planner-in-the-loop replay; retune records + # a decision that would apply as advisory and applies nothing. advisory: bool = SLAPlannerDefaults.advisory # --- Power-aware budget (read-only caps; DGD-owned) --- @@ -1952,6 +2338,9 @@ def _validate_retune(self, retune: RetuneConfig) -> None: raise ValueError( f"retune requires mode='agg' or mode='disagg', got {self.mode!r}" ) + if retune.executor == "dgdr": + # Before the domain expansion, whose errors assume replay. + self._validate_live_retune(retune) if self.scaling_enabled() and retune.objective not in RETUNE_PER_GPU_OBJECTIVES: raise ValueError( f"retune.objective {retune.objective!r} does not divide by " @@ -1996,6 +2385,172 @@ def _validate_retune(self, retune: RetuneConfig) -> None: "omitted AISimulate knob searches its default domain, so pin " "knobs with a scalar to shrink the grid" ) + if retune.executor == "dgdr": + self._validate_live_retune_budget(retune, router_domain) + + def _validate_live_retune(self, retune: RetuneConfig) -> None: + """Configuration premises P1, P2, P3 and P5 of live retune. + + A live planner sweeps through DGDR v1beta2 Requests in the Kubernetes + namespace of its DGD, retunes the frontend router only (Scenario 1), + and decides when a sweep returns; the frontend-side premises are + checked at runtime against the DGD. + """ + + dgdr = retune.dgdr + if dgdr is None: # RetuneConfig requires it with executor dgdr + raise ValueError("retune.dgdr is required iff retune.executor is 'dgdr'") + if self.environment != "kubernetes": + raise ValueError( + "retune.executor 'dgdr' requires environment='kubernetes' (P1), " + f"got {self.environment!r}" + ) + latency = retune.decision_latency_seconds + if latency == "measured": + if retune.decision_latency_scale != 1.0: + raise ValueError( + "retune.executor 'dgdr' decides when the sweep returns, so " + "decision_latency_seconds: measured requires " + "decision_latency_scale 1 (P1), got " + f"{retune.decision_latency_scale}" + ) + elif latency != 0: + raise ValueError( + "retune.executor 'dgdr' decides when the sweep returns, so " + "retune.decision_latency_seconds must be 0 or 'measured' (P1), " + f"got {latency}" + ) + if retune.timeout_seconds is None: + raise ValueError( + "retune.executor 'dgdr' requires an explicit " + "retune.timeout_seconds (P1): the sweep deadline, window cut " + "included" + ) + limit = retune.interval_seconds - DGDR_TIMEOUT_MARGIN_SECONDS + if retune.timeout_seconds > limit: + raise ValueError( + f"retune.timeout_seconds must be <= interval_seconds - " + f"{DGDR_TIMEOUT_MARGIN_SECONDS:g} = {limit:g} with executor " + f"'dgdr' (P1), got {retune.timeout_seconds:g}" + ) + if self.scaling_enabled(): + raise ValueError( + "retune.executor 'dgdr' supports retune-only mode (P2): set " + "enable_load_scaling and enable_throughput_scaling both to false" + ) + if retune.planner is not None: + raise ValueError( + "retune.executor 'dgdr' retunes the router only (P2); remove " + "retune.planner" + ) + self._validate_live_retune_router(retune) + _validate_dgdr_engine(dgdr.engine, self.mode, self.backend) + # Claim paths: the planner, the frontends and the Run Jobs mount the + # same claim, and clean-up deletes under these directories. A DGDR + # trace path rejects empty, '.' and '..' segments. + dirs: dict[str, list[str]] = {} + for name in ("segments_dir", "windows_dir"): + value = getattr(dgdr, name) + dirs[name] = value.split("/") + if any(segment in ("", ".", "..") for segment in dirs[name]): + raise ValueError( + f"retune.dgdr.{name} must be a relative directory inside " + "the claim: '/'-separated names without empty, '.' or " + f"'..' segments, got {value!r}" + ) + if not PurePosixPath(dgdr.mount_path).is_absolute(): + raise ValueError( + f"retune.dgdr.mount_path must be absolute, got {dgdr.mount_path!r}" + ) + segments, windows = dirs["segments_dir"], dirs["windows_dir"] + shorter = min(len(segments), len(windows)) + if segments[:shorter] == windows[:shorter]: + raise ValueError( + "retune.dgdr.segments_dir and retune.dgdr.windows_dir must not " + f"contain one another, got {dgdr.segments_dir!r} and " + f"{dgdr.windows_dir!r}" + ) + + @staticmethod + def _validate_live_retune_router(retune: RetuneConfig) -> None: + """P3: the router domain fits the DGDR v1beta2 router section. + + A Request searches ``overlap_score_credit``, ``prefill_load_scale`` + and ``temperature`` of a ``kv_router`` with ``prefill_load_model: + none``; ``RouterRetuneConfig`` already pins the policy. + """ + + spec = retune.router + if not isinstance(spec, Mapping): + return + extra = sorted(set(spec) - _DGDR_ROUTER_SECTION_KEYS) + if extra: + raise ValueError( + f"retune.router {extra} cannot be searched with executor 'dgdr' " + "(P3): a DGDR Request searches only overlap_score_credit, " + "prefill_load_scale and temperature" + ) + if "prefill_load_model" not in spec: + return + # Imported on use: the adapter schema imports this module. + from aisimulate.config.common import Choices + + from dynamo.planner.retune.domains import parse_retune_section + + model = parse_retune_section("router", spec).prefill_load_model + if "type" not in model.model_fields_set: + return + written = ( + list(model.type.choices) + if isinstance(model.type, Choices) + else [model.type] + ) + if written != ["none"]: + raise ValueError( + "retune.router.prefill_load_model.type must be 'none' with " + f"executor 'dgdr' (P3), got {written}" + ) + + @staticmethod + def _validate_live_retune_budget( + retune: RetuneConfig, router_domain: Mapping[str, Sequence[float]] + ) -> None: + """P13: the largest grid shard can finish within the sweep deadline. + + The baseline can fall in any grid shard, and a window whose baseline + shard cannot finish fails with ``launch_too_late``, so the largest + shard must fit. The domain is the widest one: a zero startup credit + only narrows it at runtime. + """ + + dgdr = retune.dgdr + if dgdr is None or retune.timeout_seconds is None: + # _validate_live_retune rejects both first. + raise ValueError("retune.executor 'dgdr' requires dgdr and timeout_seconds") + sizes = [len(values) for values in router_domain.values()] + parts = balanced_shard_parts(sizes, dgdr.run.max_shards) + points = largest_shard_points(sizes, parts) + if points > DGDR_MAX_SHARD_POINTS: + raise ValueError( + f"the retune router grid of {math.prod(sizes)} points cannot be " + f"split into at most retune.dgdr.run.max_shards=" + f"{dgdr.run.max_shards} Cartesian shards of at most " + f"{DGDR_MAX_SHARD_POINTS} points (P13); the largest would have " + f"{points}. Raise max_shards or pin router knobs with a scalar" + ) + need = dgdr_need_seconds(points, dgdr, needs_cut=True) + if need > retune.timeout_seconds: + rounds = 1 + math.ceil((points - 1) / dgdr.run.parallelism) + raise ValueError( + f"the largest retune grid shard ({points} points) needs " + f"{need:g}s: cut_budget_seconds {dgdr.cut_budget_seconds:g} + " + f"job_startup_seconds {dgdr.job_startup_seconds:g} + {rounds} " + f"rounds x candidate_timeout_seconds " + f"{dgdr.run.candidate_timeout_seconds:g}, more than " + f"retune.timeout_seconds={retune.timeout_seconds:g} (P13), so " + "every window would fail launch_too_late. Raise timeout_seconds " + "or run.parallelism, add run.max_shards, or shrink the grid" + ) def _validate_retune_planner_grid( self, diff --git a/components/src/dynamo/planner/retune/live/__init__.py b/components/src/dynamo/planner/retune/live/__init__.py new file mode 100644 index 000000000000..452ce8294de1 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/__init__.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live retune: run the retune sweep through DGDR v1beta2 and apply the winner. + +The replay retune loop (``RetuneController`` and ``_decide``) is unchanged; +this package supplies its live pieces: the DGDR sweep executor, the +supervisor that owns leadership and persisted state, the router actuator, +the reconciler and the trace window cutter. ``contracts`` holds the types +they share. +""" diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py new file mode 100644 index 000000000000..cfa462a592b1 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -0,0 +1,470 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Types shared by the live retune modules. + +Every module in ``dynamo.planner.retune.live`` depends on this one and on +nothing else in the package except through the protocols defined here, so +each can be built and tested with fakes. +""" + +from __future__ import annotations + +import threading +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal, Optional, Protocol + +# The eight builtin default-policy parameters a frontend router exposes on +# GET/PATCH /router_policy, by their live (PATCH) names. +POLICY_PARAMETER_KEYS: tuple[str, ...] = ( + "decode_active_request_weight", + "disk_cache_hit_weight", + "host_cache_hit_weight", + "overlap_score_credit", + "overlap_score_credit_decay", + "prefill_load_scale", + "router_temperature", + "shared_cache_multiplier", +) + +# Router knobs a DGDR v1beta2 Request can search, AISimulate public name -> +# live name. ``dynamo.router.simulation.config.ROUTER_PUBLIC_TO_LIVE`` is the +# source of truth; this mirrors it for modules that must not import the +# router simulation stack. +DGDR_ROUTER_KNOBS: dict[str, str] = { + "overlap_score_credit": "overlap_score_credit", + "prefill_load_scale": "prefill_load_scale", + "temperature": "router_temperature", +} + +LABEL_PREFIX = "nvidia.com/planner-retune-" +LABEL_DGD = LABEL_PREFIX + "dgd" +LABEL_DGD_UID = LABEL_PREFIX + "dgd-uid" +LABEL_WINDOW = LABEL_PREFIX + "window" +LABEL_SHARD = LABEL_PREFIX + "shard" +LABEL_SPEC = LABEL_PREFIX + "spec" +LABEL_ATTEMPT = LABEL_PREFIX + "attempt" + +# Pod labels the operator sets (deploy/operator/internal/consts/consts.go). +POD_LABEL_DGD_NAME = "nvidia.com/dynamo-graph-deployment-name" +POD_LABEL_COMPONENT_TYPE = "nvidia.com/dynamo-component-type" + +DGDR_GROUP = "nvidia.com" +DGDR_VERSION = "v1beta2" +DGDR_REQUESTS = "dynamographdeploymentrequests" +DGDR_RUNS = "dynamographdeploymentruns" +DGDR_CANDIDATES = "dynamographdeploymentcandidates" +RUN_UID_LABEL = "nvidia.com/dgdr-run-uid" + +# SweepResult.failure_reason values that mean "this process stopped working +# on the window", not "the window failed": no decision is recorded and the +# controller rewinds so the window can be adopted (§4.7.5). +NON_DECISION_FAILURES = frozenset({"shutdown", "lost_leadership"}) + +GateReason = Literal["not_leader", "not_ready", "already_decided"] + + +# --------------------------------------------------------------- controller + + +@dataclass(frozen=True) +class GateResult: + """What ``launch_gate`` tells the controller about a due window. + + ``reason`` is ``None`` to let the window launch (or record ``busy``). + ``not_leader``/``not_ready`` keep ``_last_k`` so the window is retried on + a later tick; ``already_decided`` advances it without a decision. + ``spec_replicas`` (worker type -> DGD spec replicas) and ``abort`` (set + by the supervisor when this process stops leading) are only meaningful + when ``reason`` is ``None``. + """ + + reason: Optional[GateReason] = None + spec_replicas: Optional[dict[str, int]] = None + abort: Optional[threading.Event] = None + + +# (k, (window_start_s, window_end_s, as_of_s)) -> GateResult +LaunchGate = Callable[[int, tuple[float, float, float]], GateResult] +# controller generation of the sweep's controller -> may it be harvested? +HarvestAllowed = Callable[[int], bool] + + +# ------------------------------------------------------------ kubernetes + + +class KubeError(Exception): + """A Kubernetes call failed (network, timeout, 5xx, unexpected status).""" + + +class KubeNotFound(KubeError): + pass + + +class KubeConflict(KubeError): + """409 on update (stale resourceVersion).""" + + +class KubeAlreadyExists(KubeError): + """409 on create (the name is taken).""" + + +class KubeInvalid(KubeError): + """422 / 400: the API server rejected the object (e.g. Strict pruning).""" + + +@dataclass(frozen=True) +class DgdRef: + """The DynamoGraphDeployment this planner serves.""" + + name: str + namespace: str # Kubernetes namespace + uid: str + dynamo_namespace: str # the routers' ``namespace`` on GET /router_policy + + +@dataclass(frozen=True) +class PodInfo: + name: str + uid: str + ip: Optional[str] + ready: bool + terminating: bool + labels: Mapping[str, str] = field(default_factory=dict) + # Container port named ``http`` of the frontend container, if any. + http_port: Optional[int] = None + # ``status.phase`` (Pending, Running, Succeeded, Failed, Unknown). + phase: Optional[str] = None + + +class RetuneKube(Protocol): + """Synchronous Kubernetes access for live retune. + + Every method takes an optional ``timeout_s`` (connect+read, applied as + the kubernetes client's ``_request_timeout`` tuple of ints) and raises + the ``Kube*`` errors above instead of client exceptions. Callers on the + event loop wrap calls in ``asyncio.to_thread``. + """ + + namespace: str + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + ... + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + ... + + def create_request( + self, + body: dict, + *, + dry_run: bool = False, + timeout_s: float = 10.0, + ) -> dict: + """Create with ``field_validation='Strict'`` (and ``dry_run='All'``).""" + ... + + def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def list_requests( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + ... + + def delete_request( + self, + name: str, + *, + propagation: Literal["Background", "Foreground"] = "Background", + timeout_s: float = 10.0, + ) -> None: + """Missing objects are not an error.""" + ... + + def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def list_candidates( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + ... + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + ... + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + """Update with ``body['metadata']['resourceVersion']``; 409 -> KubeConflict.""" + ... + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + ... + + +# ------------------------------------------------------- frontend config + + +@dataclass(frozen=True) +class FrontendStartup: + """The frontend router configuration parsed from the DGD spec. + + ``router_config`` is the startup ``KvRouterConfig`` JSON (only keys the + frontend sets); ``startup_maps`` is ``policy_parameters(wt)`` for each + worker type of the planner mode. ``fingerprint_source`` is a canonical + JSON string of everything parsed that the state fingerprint covers. + """ + + router_mode: str # "kv_router" or "round_robin" + router_config: dict[str, Any] + startup_maps: dict[str, dict[str, float]] + frontend_component: str # DGD component name of the frontend + replicas: int + replica_sync: bool + fingerprint_source: str + + +@dataclass(frozen=True) +class PremiseReport: + """Spec-class premise evaluation (P4, P7-P10; P14 is the actuator's). + + ``p4_ok`` alone decides whether a controller can be built; ``spec_ok`` + is the conjunction of every spec premise. ``reasons`` explain each + failed premise as ``"P: ..."``; ``warnings`` carry warn-only ones. + """ + + p4_ok: bool + spec_ok: bool + reasons: tuple[str, ...] = () + warnings: tuple[str, ...] = () + startup: Optional[FrontendStartup] = None + + +# ---------------------------------------------------------- router policy + + +@dataclass(frozen=True) +class RouterInfo: + """One router on GET /router_policy.""" + + router_id: str + namespace: str + worker_type: str + revision: int + # The 8 PolicyParameters; None for a custom policy without runtime config. + configuration: Optional[dict[str, float]] + model: Optional[str] = None + + +@dataclass(frozen=True) +class PolicyCallResult: + """Outcome of one GET or PATCH. ``status`` is the HTTP status (0 on a + network error or timeout).""" + + status: int + routers: tuple[RouterInfo, ...] = () # GET + router: Optional[RouterInfo] = None # PATCH/reset response, when 200 + error: Optional[str] = None + + +class RouterPolicyClient(Protocol): + async def get(self, pod: PodInfo) -> PolicyCallResult: + ... + + async def patch( + self, + pod: PodInfo, + router_id: str, + expected_revision: int, + parameters: Mapping[str, float], + ) -> PolicyCallResult: + ... + + async def reset( + self, pod: PodInfo, router_id: str, expected_revision: int + ) -> PolicyCallResult: + """PATCH ``{router_id, expected_revision, reset: true}``.""" + ... + + +ActuationMode = Literal["apply", "rollback", "observe"] + + +@dataclass(frozen=True) +class ActuationTarget: + """What one actuator round converges to. + + ``apply``: PATCH explicit values so every router's 8 parameters equal + ``desired_by_wt[worker_type]``. ``rollback``: reset every router whose + configuration differs from its latched initial (never by revision). + ``observe``: no writes; latch initials and check parity only. + ``include_not_ready`` also targets NotReady, non-terminating pods. + ``learn_unknown`` lets the actuator reset a router whose initial was + never latched (revision > 0) once to learn it; that is active mode only + (§4.8). With False (advisory) such a router is never written: rollback + leaves it alone and does not wait for it to converge. + """ + + mode: ActuationMode + desired_by_wt: Optional[dict[str, dict[str, float]]] = None + include_not_ready: bool = False + learn_unknown: bool = True + + +@dataclass(frozen=True) +class ActuationReport: + converged: bool + # router_id -> configuration latched this round (revision 0 or reset). + new_initials: dict[str, dict[str, float]] = field(default_factory=dict) + # router_ids seen this round (to prune stale initials). + seen_router_ids: frozenset[str] = frozenset() + # P14: every known initial equals startup_maps[worker_type]. + parity_ok: Optional[bool] = None + parity_mismatches: tuple[str, ...] = () + admin_api_disabled: bool = False + errors: dict[str, int] = field(default_factory=dict) # by status code / kind + # P12: enough Ready frontends, each with a configured router for every + # worker type, whatever their values. Gates launches; convergence does not. + routers_present: Optional[bool] = None + + +# ----------------------------------------------------------- reconciler + + +class SweepRegistry(Protocol): + """Thread-safe link between a running sweep and the reconciler. + + Protection is reference-counted per window: every ``protect`` call + takes one reference and needs one matching ``release``. A window, with + the union of every Request name given to it, stays protected while any + reference is held, so two sweeps of one window (a rebuilt controller + relaunching it while the old sweep winds down) cannot drop each + other's protection. + """ + + def protect(self, window_end_ms: int, request_names: Iterable[str]) -> None: + """Mark a window file and Requests as in use by this process (one + reference).""" + ... + + def release(self, window_end_ms: int) -> None: + """Drop one reference taken by ``protect``.""" + ... + + def enqueue_delete(self, request_names: Iterable[str]) -> None: + ... + + +# ---------------------------------------------------------------- window + + +@dataclass(frozen=True) +class WindowCutResult: + path: Optional[str] # None for an empty window (no file written) + num_requests: int + trace_block_size: Optional[int] + row_counts: dict[str, int] + dropped_members: int = 0 + dropped_bytes: int = 0 + reused: bool = False # the final file already existed (EEXIST) + + +class WindowCutter(Protocol): + def __call__( + self, + segments_root: str, + start_ms: int, + end_ms: int, + as_of_ms: int, + out_path: str, + *, + max_rows: int, + deadline_s: float, + cancel: Optional[threading.Event] = None, + ) -> WindowCutResult: + ... + + +# --------------------------------------------------------------- metrics + + +class RetuneMetricsSink(Protocol): + """``dynamo_planner_retune_*`` metrics; every method is cheap and safe.""" + + def decision(self, outcome: str) -> None: + ... + + def failure(self, reason: str) -> None: + ... + + def skipped(self, reason: str) -> None: + ... + + def supervisor_state( + self, + *, + leader: bool, + ready: bool, + initialized: bool, + rollback_pending: bool, + generation: int, + ) -> None: + ... + + def scores( + self, + *, + baseline: Optional[float], + best: Optional[float], + sweep_seconds: Optional[float], + missing_candidates: int, + ) -> None: + ... + + def router_converged(self, converged: bool) -> None: + ... + + def patch_error(self, code: str) -> None: + ... + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + ... + + def dropped_members(self, count: int) -> None: + ... + + +class NullMetrics: + """A ``RetuneMetricsSink`` that records nothing (tests, disabled metrics).""" + + def decision(self, outcome: str) -> None: + pass + + def failure(self, reason: str) -> None: + pass + + def skipped(self, reason: str) -> None: + pass + + def supervisor_state(self, **_: Any) -> None: + pass + + def scores(self, **_: Any) -> None: + pass + + def router_converged(self, converged: bool) -> None: + pass + + def patch_error(self, code: str) -> None: + pass + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + pass + + def dropped_members(self, count: int) -> None: + pass diff --git a/components/src/dynamo/planner/retune/live/frontend_config.py b/components/src/dynamo/planner/retune/live/frontend_config.py new file mode 100644 index 000000000000..76485dbb7f10 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/frontend_config.py @@ -0,0 +1,1794 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Frontend router startup parsed from the DGD spec, and the spec premises. + +Live retune replays each window in DGDR with AISimulate's router defaults +plus the searched knobs, while the live frontends start from whatever their +argv and env say. The two agree only when the frontend leaves every +unsearched routing field at its default (premise P4), so the planner reads +the frontend's startup from the DynamoGraphDeployment spec, never from a +running pod (design §4.3): + +- ``find_frontend`` picks the one standalone frontend component. +- ``parse_frontend`` rebuilds the frontend's effective argv (the operator's + ``command``/``args`` defaulting) and env (operator-injected, graph ``env`` + and the main container's, with Kubernetes ``$(VAR)`` expansion), then + parses them with the frontend's own argument groups. Those groups read + ``os.environ`` while registering their arguments; here they run as copies + of their functions whose ``os`` sees only the frontend's env, so the + planner process's environment is neither read nor modified. A + shell-wrapped command, a ``--router-policy-config`` file, router env set + through ``valueFrom`` and an argument whose unreadable ``$(VAR)`` could + set a router flag make the frontend unparseable. ``envFrom`` sources are + assumed to set no router env; ``evaluate_premises`` warns about them. +- ``evaluate_premises`` checks P4 and P7-P10 and warns on P6 and P7; P14 + is the router actuator's. P4 also parses every worker component's router + flags and env the same way: a worker that resolves a router mode + advertises its own router configuration, which replaces the frontend's + for its worker set. ``p4_ok`` alone decides whether a retune controller + can be built. + +Every function is pure (a dict in, a value out, no Kubernetes calls) and +never reads the planner's own environment. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import posixpath +import re +import shlex +import types +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from functools import lru_cache +from typing import TYPE_CHECKING, Any, Optional + +from dynamo.common.configuration.groups.kv_router_args import ( + _KV_ROUTER_FIELDS, + KvRouterConfigBase, +) +from dynamo.common.configuration.groups.router_args import RouterConfigBase +from dynamo.llm import KvRouterConfig +from dynamo.planner.retune.live.contracts import FrontendStartup, PremiseReport + +if TYPE_CHECKING: + from dynamo.planner.config.planner_config import PlannerConfig, RetuneConfig + +logger = logging.getLogger(__name__) + +FRONTEND_TYPE = "frontend" +PLANNER_TYPE = "planner" +EPP_TYPE = "epp" +# Component types that run inference workers; v1beta1 ``type`` is optional. +WORKER_TYPES = frozenset({"worker", "prefill", "decode", ""}) + +MAIN_CONTAINER = "main" +# The operator's frontend invocation (component_frontend.go GetBaseContainer). +DEFAULT_FRONTEND_COMMAND = ("python3",) +DEFAULT_FRONTEND_ARGS = ("-m", "dynamo.frontend") +FRONTEND_MODULE = "dynamo.frontend" + +# Env the operator injects into every component (component_common.go +# getCommonContainer); user env of the same name overrides it. +ENV_PARENT_DGD_NAME = "DYN_PARENT_DGD_K8S_NAME" +ENV_PARENT_DGD_NAMESPACE = "DYN_PARENT_DGD_K8S_NAMESPACE" +_OPERATOR_FIELD_REF_ENV = ("POD_NAME", "POD_NAMESPACE", "POD_UID") + +# The trace recipe of design §4.4.1 (P10). +ENV_POD_NAME = "DYN_POD_NAME" +ENV_TRACE = "DYN_REQUEST_TRACE" +ENV_TRACE_SINKS = "DYN_REQUEST_TRACE_SINKS" +ENV_TRACE_FORMAT = "DYN_REQUEST_TRACE_FILE_FORMAT" +ENV_TRACE_RECORDS = "DYN_REQUEST_TRACE_RECORDS" +ENV_TRACE_PATH = "DYN_REQUEST_TRACE_FILE_PATH" +# lib/truthy is_truthy; lib/llm/src/request_trace/config.rs parse_file_format. +_TRUTHY = frozenset({"1", "true", "on", "yes"}) +_GZ_FORMATS = frozenset({"jsonl_gz", "jsonl.gz"}) + +ENV_WORKER_SELECTION_POLICY = "DYN_ROUTER_WORKER_SELECTION_POLICY" +ENV_LOAD_AWARE = "DYN_ROUTER_LOAD_AWARE" +# A worker that resolves a router mode (router_args.add_worker_router_arguments) +# advertises a RouterConfig in its model card. +ENV_ROUTER_MODE = "DYN_ROUTER_MODE" +_ROUTER_MODE_DEST = "router_mode" +_ROUTER_ENV_PREFIX = "DYN_ROUTER_" +# Router env the Rust router reads directly (no frontend flag) that tunes +# transport or startup only, never which worker is picked. +_TRANSPORT_ROUTER_ENV = frozenset( + { + "DYN_ROUTER_ACTIVE_SEQUENCE_DIRECT_ZMQ", + "DYN_ROUTER_ACTIVE_SEQUENCE_ZMQ_RCVHWM", + "DYN_ROUTER_MODEL_CARD_WAIT_SECS", + "DYN_ROUTER_ZMQ_ENDPOINTS_PER_SUB", + } +) +# The deprecated overlap_score_weight: KvRouterArgGroup reads the first name +# when it is set and the second otherwise. +_OVERLAP_WEIGHT_ENV = ("DYN_ROUTER_KV_OVERLAP_SCORE_WEIGHT", "DYN_OVERLAP_SCORE_WEIGHT") + +# RouterConfig (not KvRouterConfig) fields that change routing or admission: +# session affinity pins requests and the busy thresholds reject them. The +# DGDR replay models neither. +_ROUTER_ADMISSION_FIELDS = ( + "session_affinity_ttl_secs", + "active_decode_blocks_threshold", + "active_prefill_tokens_threshold", + "active_prefill_tokens_threshold_frac", +) +# KvRouterConfig fields serde skips (to_json drops them and from_json +# rejects them), so they are checked on the parsed flags. +_SERDE_SKIPPED_DEFAULTS: dict[str, Any] = { + "router_approximate_cache_policy": "ttl", + "router_prefill_policy": None, + "router_decode_policy": None, +} +# KvRouterConfig JSON keys a frontend may set to a non-default value: they +# do not change which worker the router picks, or another premise covers +# them. +TOLERATED_ROUTER_FIELDS = frozenset( + { + "router_replica_sync", # P8 + "router_event_threads", # indexer thread count + # Hashing of router-derived active-sequence identities. + "router_tracking_hash", + "router_tracking_key_file", + "router_tracking_key_id", + "serve_indexer", # serves this router's indexer to other routers + "enable_session_prefix_index", # tracked, not consumed by routing + "skip_initial_worker_wait", # startup only + } +) +# Fields with their own P4 message. +_LOAD_MODEL = "router_prefill_load_model" +_CONDITIONAL_DISAGG = "conditional_disagg_enabled" + +_TLS_DESTS = ("tls_cert_path", "tls_key_path") +# Parsed destinations whose value the startup depends on. +_RELEVANT_DESTS = frozenset( + { + *_KV_ROUTER_FIELDS, + *_ROUTER_ADMISSION_FIELDS, + *_TLS_DESTS, + "router_mode", + "session_affinity_mode", + "load_aware", + "conditional_disagg_config", + } +) +# runtime_status.go ResolveComponentRuntimeStatus, in its priority order. +_MODEL_FLAGS = ("--served-model-name", "--model-name", "--model", "--model-path") +_SHELLS = frozenset({"sh", "bash", "dash", "zsh", "ash", "ksh"}) +# Python options that may precede ``-m``; the second set takes a value. +_PYTHON_FLAGS = frozenset({"-u", "-B", "-E", "-I", "-O", "-OO", "-P", "-q", "-s", "-S"}) +_PYTHON_FLAGS_WITH_VALUE = frozenset({"-X", "-W"}) +# Marks an argument part that references env the planner cannot read. +_OPAQUE = "\x00opaque:" +_OPAQUE_REPR = repr(_OPAQUE)[1:-1] + +# Receives the frontend argument groups' own log lines (deprecation and +# removed-flag notices), which would otherwise repeat on every parse. +_SILENT_LOGGER = logging.getLogger(f"{__name__}.frontend_args") +_SILENT_LOGGER.disabled = True + + +class FrontendParseError(ValueError): + """The frontend's router configuration cannot be derived from the spec.""" + + +# ------------------------------------------------------------------ API + + +def worker_types(planner_mode: str) -> tuple[str, ...]: + """Router worker types of a planner mode, as the retune controller uses them.""" + + return ("prefill", "decode") if planner_mode == "disagg" else ("aggregated",) + + +def canonical_json(value: Any) -> str: + """Deterministic JSON: sorted keys, no whitespace.""" + + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def find_frontend(dgd: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: + """Return the name and spec of the DGD's one standalone frontend component. + + v1beta1 lists components in ``spec.components``; a v1alpha1 + ``spec.services`` map is converted to the same shape (``type``, + ``replicas``, ``podTemplate.spec`` with a ``main`` container), so callers + read one layout. Raises ``FrontendParseError`` ("P7: ...") unless there + is exactly one standalone frontend, naming an EPP or ``frontendSidecar`` + layout when that is why. + """ + + components = _components(dgd) + frontends = [c for c in components if _type(c) == FRONTEND_TYPE] + if len(frontends) == 1: + return str(frontends[0].get("name", "")), dict(frontends[0]) + if frontends: + names = ", ".join(sorted(str(c.get("name")) for c in frontends)) + raise FrontendParseError( + f"P7: the DGD has {len(frontends)} frontend components ({names}); " + "live retune needs exactly one" + ) + sidecars = _sidecar_components(components) + if sidecars: + raise FrontendParseError( + f"P7: the frontend runs as a frontendSidecar of {', '.join(sidecars)}; " + "live retune needs a standalone frontend component" + ) + if any(_type(c) == EPP_TYPE for c in components): + raise FrontendParseError( + "P7: the DGD routes through an EPP component and has no standalone " + "frontend component" + ) + raise FrontendParseError("P7: the DGD has no frontend component") + + +def served_model_names( + dgd: Mapping[str, Any], +) -> tuple[dict[str, str], tuple[str, ...]]: + """Served model name of each worker component, and the undetermined ones. + + Uses the operator's ``status.components[].servedModelName`` while + the status is current, else the first of ``--served-model-name``, + ``--model-name``, ``--model`` and ``--model-path`` in the main + container's command and args (the operator's order), after Kubernetes + ``$(VAR)`` expansion against the component's env, skipping values that + reference env the planner cannot read. + """ + + projected = _projected_status(dgd) + models: dict[str, str] = {} + unknown: list[str] = [] + for component in _components(dgd): + if _type(component) not in WORKER_TYPES: + continue + name = str(component.get("name", "")) + model = (projected.get(name) or {}).get("servedModelName") or _flag_model( + _model_flags(component, _component_env(dgd, component)) + ) + if model: + models[name] = str(model) + else: + unknown.append(name) + return models, tuple(sorted(unknown)) + + +def served_model_name(dgd: Mapping[str, Any]) -> Optional[str]: + """The DGD's served model when exactly one name is known, else ``None``.""" + + names = set(served_model_names(dgd)[0].values()) + return next(iter(names)) if len(names) == 1 else None + + +def parse_frontend( + dgd: Mapping[str, Any], *, planner_mode: str +) -> FrontendStartup | str: + """Parse the frontend's startup router configuration from the DGD spec. + + Returns the startup, or a ``"P4: ..."`` / ``"P7: ..."`` reason when it + cannot be derived. ``router_config`` holds only the keys whose serde + value differs from ``KvRouterConfig()``; ``startup_maps`` resolves + ``policy_parameters`` for each worker type of ``planner_mode``. A + frontend that parses but violates other P4 conditions (a non-KV router + mode, a non-default field) still returns its startup; see + ``evaluate_premises``. + """ + + analysis = _analyze(dgd, planner_mode) + if analysis.startup is None: + return analysis.error or "P4: the frontend configuration cannot be parsed" + return analysis.startup + + +def evaluate_premises( + dgd: Mapping[str, Any], + *, + planner_config: PlannerConfig, + startup: FrontendStartup | None, + parse_error: str | None, + planner_replicas: int | None, +) -> PremiseReport: + """Evaluate the spec premises P4 and P7-P10, and the P6 and P7 warnings. + + ``startup``/``parse_error`` are ``parse_frontend``'s result for ``dgd``; + what a ``FrontendStartup`` cannot carry (serde-skipped flags, session + affinity, unread router env, TLS, the trace recipe) is re-derived from + ``dgd`` (cached, so cheap). P4 also fails for a worker component that + advertises its own router configuration. ``planner_replicas``, when + given, replaces the planner component replicas read from the DGD (P9). + A malformed DGD never raises; every problem becomes a reason. Raises + ``ValueError`` only when ``planner_config.retune.dgdr`` is unset. + """ + + retune = planner_config.retune + if retune is None or retune.dgdr is None: + raise ValueError("evaluate_premises requires planner_config.retune.dgdr") + dgdr = retune.dgdr + mode = str(planner_config.mode) + analysis = _analyze(dgd, mode) + components = _components(dgd) + p4: list[str] = [] + others: list[str] = [] + warnings: list[str] = [] + + # P4: a parsed KV router whose unsearched fields are the defaults and + # whose startup passes the retune controller's constructor checks. + if startup is None: + p4.append(parse_error or analysis.error or "P4: the frontend was not parsed") + else: + p4.extend(analysis.p4_violations) + p4.extend(_startup_errors(startup, retune, mode)) + if dgdr.startup_router_config is not None: + mismatch = _startup_config_mismatch(dgdr.startup_router_config, startup) + if mismatch is not None: + p4.append(mismatch) + # A worker's advertised router configuration replaces the frontend's for + # its worker set (watcher.rs effective_router_config), whatever the + # frontend's flags say. + p4.extend(_worker_router_errors(dgd, components, warnings)) + + # P7: one served model, one standalone frontend, no EPP, no TLS. + models, unknown = served_model_names(dgd) + distinct = sorted(set(models.values())) + if len(distinct) > 1: + others.append( + f"P7: the DGD serves {len(distinct)} models ({', '.join(distinct)}); " + "live retune needs one" + ) + if unknown: + warnings.append( + f"P7: cannot determine the served model of {', '.join(unknown)}; " + "live retune needs the DGD to serve one model" + ) + # Every DGDR Run simulates retune.dgdr.model; an alias given with + # --served-model-name can hide the HF id, so a miss only warns. + names = _worker_model_names(dgd, components) + if names and dgdr.model not in names: + warnings.append( + f"P7: retune.dgdr.model {dgdr.model!r} is not among the workers' " + f"--model/--model-path/--served-model-name values " + f"({', '.join(sorted(names))}); every DGDR Run simulates " + "retune.dgdr.model, so make sure it is the model the DGD serves" + ) + if analysis.component is None: + if analysis.error is not None and analysis.error.startswith("P7"): + others.append(analysis.error) + else: + if any(_type(c) == EPP_TYPE for c in components): + others.append("P7: the DGD routes through an EPP component (GAIE)") + sidecars = _sidecar_components(components) + if sidecars: + others.append(f"P7: {', '.join(sidecars)} run a frontendSidecar") + if analysis.tls: + others.append( + "P7: the frontend serves TLS (--tls-cert-path / DYN_TLS_CERT_PATH)" + ) + + if analysis.env is not None: + for source in analysis.env.env_from: + if _env_from_may_set_router_env(source): + warnings.append( + f"P4: the frontend container uses envFrom {_describe_env_from(source)}; " + "the planner cannot read it and assumes it sets no router env " + "(P14 still checks the routers' policy parameters)" + ) + + # P8: one frontend replica, or synchronized router replicas. + if startup is not None and startup.replicas > 1 and not startup.replica_sync: + others.append( + f"P8: the frontend has {startup.replicas} replicas without " + "router_replica_sync; AISimulate models a single router" + ) + + # P9: worker replicas only change through the DGD spec, and one planner. + for component in components: + if _type(component) in WORKER_TYPES and _scaling_adapter(component): + others.append( + f"P9: worker component {component.get('name')} has a " + "scalingAdapter; its replicas could change during a window" + ) + others.extend(_planner_replica_errors(components, planner_replicas, warnings)) + + # P10: the frontend records request_end traces onto the claim. + if analysis.component is not None and analysis.env is not None: + others.extend( + _trace_recipe_errors( + analysis, + dgd_name=_dgd_name(dgd), + claim_name=dgdr.claim_name, + segments_dir=dgdr.segments_dir, + warnings=warnings, + ) + ) + + # P6 (warn only): AISimulate resolves the model by its HF id. + warning = _model_warning(dgdr.model) + if warning is not None: + warnings.append(warning) + + reasons = tuple(_unique([*p4, *others])) + return PremiseReport( + p4_ok=not p4, + spec_ok=not reasons, + reasons=reasons, + warnings=tuple(_unique(warnings)), + startup=startup, + ) + + +# --------------------------------------------------------- frontend facts + + +@dataclass(frozen=True) +class _Env: + """A component main container's env as far as the planner can know it. + + ``entries`` is the merged raw env keyed by name (operator-injected, then + graph ``env``, then the container's; later wins). ``values`` holds each + name whose value the planner can compute, after ``$(VAR)`` expansion; + ``opaque`` the names it cannot (``valueFrom``, or a reference to one). + """ + + entries: Mapping[str, Mapping[str, Any]] + values: Mapping[str, str] + opaque: frozenset[str] + env_from: tuple[Mapping[str, Any], ...] + + +@dataclass(frozen=True) +class _Analysis: + component: Optional[str] = None + spec: Optional[Mapping[str, Any]] = None + env: Optional[_Env] = None + startup: Optional[FrontendStartup] = None + error: Optional[str] = None + # P4 violations a FrontendStartup cannot show. + p4_violations: tuple[str, ...] = () + tls: bool = False + + +@dataclass(frozen=True) +class _Parsed: + """The cacheable outcome of parsing one frontend argv and env.""" + + error: Optional[str] = None + router_mode: str = "" + router_config: Optional[Mapping[str, Any]] = None + startup_maps: Optional[Mapping[str, Mapping[str, float]]] = None + replica_sync: bool = False + p4_violations: tuple[str, ...] = () + tls: bool = False + + +def _analyze(dgd: Mapping[str, Any], planner_mode: str) -> _Analysis: + try: + name, spec = find_frontend(dgd) + except FrontendParseError as error: + return _Analysis(error=str(error)) + env = _component_env(dgd, spec) + try: + argv = _effective_argv(spec) + except FrontendParseError as error: + return _Analysis(component=name, spec=spec, env=env, error=str(error)) + parsed = _parse_cached( + canonical_json( + { + "argv": list(argv), + "values": dict(env.values), + "opaque": sorted(env.opaque), + "env_from": [dict(source) for source in env.env_from], + } + ), + worker_types(planner_mode), + ) + if parsed.error is not None: + return _Analysis(component=name, spec=spec, env=env, error=parsed.error) + assert parsed.router_config is not None and parsed.startup_maps is not None + router_config = dict(parsed.router_config) + startup_maps = {wt: dict(params) for wt, params in parsed.startup_maps.items()} + startup = FrontendStartup( + router_mode=parsed.router_mode, + router_config=router_config, + startup_maps=startup_maps, + frontend_component=name, + replicas=_replicas(spec), + replica_sync=parsed.replica_sync, + # Replicas stay out: scaling the frontend is P8's concern and must not + # reset the persisted state. + fingerprint_source=canonical_json( + { + "router_mode": parsed.router_mode, + "router_config": router_config, + "startup_maps": startup_maps, + } + ), + ) + return _Analysis( + component=name, + spec=spec, + env=env, + startup=startup, + p4_violations=parsed.p4_violations, + tls=parsed.tls, + ) + + +@lru_cache(maxsize=32) +def _parse_cached(key: str, wts: tuple[str, ...]) -> _Parsed: + """Parse once per distinct frontend (revalidation re-parses every round).""" + + inputs = json.loads(key) + try: + return _parse( + inputs["argv"], + inputs["values"], + frozenset(inputs["opaque"]), + inputs["env_from"], + wts, + ) + except FrontendParseError as error: + return _Parsed(error=str(error)) + except Exception as error: # noqa: BLE001 - fail closed, never crash the planner + logger.exception("live retune could not parse the frontend configuration") + return _Parsed( + error=( + "P4: the planner failed to parse the frontend configuration " + f"({type(error).__name__}: {error})" + ) + ) + + +def _parse( + argv: Sequence[str], + values: Mapping[str, str], + opaque: frozenset[str], + env_from: Sequence[Mapping[str, Any]], + wts: tuple[str, ...], +) -> _Parsed: + args = _frontend_args(argv) + parser, registered = _frontend_parser(values) + + def relevant(name: str) -> bool: + if name in registered: + return registered[name] in _RELEVANT_DESTS + return name in _OVERLAP_WEIGHT_ENV or ( + name.startswith(_ROUTER_ENV_PREFIX) and name not in _TRANSPORT_ROUTER_ENV + ) + + # Router env the planner cannot read makes the startup unknowable. + hidden = sorted(name for name in opaque if relevant(name)) + if hidden: + raise FrontendParseError( + f"P4: frontend router env {', '.join(hidden)} is set through valueFrom " + "(or references such a value); the planner cannot read it, so set " + "it with a literal value" + ) + # envFrom sources (typically an HF token secret) are assumed not to set + # router env; evaluate_premises warns, and P14 compares the routers' + # policy parameters with the parsed startup. + del env_from + + # An unreadable $(VAR) becomes a marker the parser carries as a value: a + # non-router flag keeps it unconverted, a router flag fails the parse, + # and one left over where a flag could stand fails below. + expanded = [_expand_arg(arg, values, opaque) for arg in args] + namespace, unknown = parser.parse_known_args(expanded) + hidden_flag = _opaque_flag_position(unknown) + if hidden_flag is not None: + original = dict(zip(expanded, args)).get(hidden_flag, hidden_flag) + raise FrontendParseError( + f"P4: frontend arg {original!r} references env the planner cannot " + "read and is not the value of a frontend flag, so it could set a " + "router flag" + ) + unresolved = sorted( + dest + for dest, value in vars(namespace).items() + if dest in _RELEVANT_DESTS and _OPAQUE in str(value) + ) + if unresolved: + raise FrontendParseError( + f"P4: frontend args set {', '.join(unresolved)} from env the planner " + "cannot read" + ) + + view = _RouterView.from_cli_args(namespace) + # FrontendConfig.validate: the load-aware preset implies KV routing. + if view.load_aware: + view.router_mode = "kv" + if view.router_policy_config is not None: + raise FrontendParseError( + "P4: the frontend sets --router-policy-config " + f"({view.router_policy_config}); the planner cannot read that file, " + "and live retune needs the builtin default policy" + ) + try: + view.apply_load_aware_preset() + view.apply_conditional_disagg_config() + kwargs = {name: getattr(view, name) for name in _KV_ROUTER_FIELDS} + config = KvRouterConfig(**kwargs) + except (ValueError, TypeError) as error: + raise FrontendParseError( + f"P4: the frontend's router configuration is invalid: {error}" + ) from None + + full = json.loads(config.to_json()) + defaults = json.loads(KvRouterConfig().to_json()) + router_config = { + key: value + for key, value in full.items() + if key not in defaults or defaults[key] != value + } + try: + maps = _policy_maps(canonical_json(router_config), wts) + except ValueError as error: + raise FrontendParseError( + f"P4: cannot resolve the frontend's startup policy parameters: {error}" + ) from None + + violations: list[str] = [] + if kwargs["overlap_score_weight"] is not None: + violations.append( + "P4: the frontend sets the deprecated overlap_score_weight, which " + "overrides prefill_load_scale" + ) + for name, default in _SERDE_SKIPPED_DEFAULTS.items(): + if kwargs[name] != default: + violations.append( + f"P4: the frontend sets {name}={kwargs[name]!r} (default " + f"{default!r}), which live retune cannot model" + ) + if view.load_aware: + violations.append( + f"P4: the frontend enables the load-aware preset (--load-aware / " + f"{ENV_LOAD_AWARE})" + ) + for name in _ROUTER_ADMISSION_FIELDS: + value = getattr(view, name) + if value is not None: + violations.append( + f"P4: the frontend sets {name}={value!r}; live retune cannot " + "model session affinity or busy-worker rejection" + ) + for name in sorted(values): + if ( + not name.startswith(_ROUTER_ENV_PREFIX) + or name in registered + or name in _TRANSPORT_ROUTER_ENV + ): + continue + if name == ENV_WORKER_SELECTION_POLICY: + violations.append( + f"P4: the frontend sets {name}, which selects a worker-selection " + "policy instance instead of the builtin default policy" + ) + else: + violations.append( + f"P4: the frontend sets {name}, a router setting live retune " + "cannot model" + ) + + return _Parsed( + router_mode=_router_mode(str(view.router_mode)), + router_config=router_config, + startup_maps=maps, + replica_sync=bool(full.get("router_replica_sync", False)), + p4_violations=tuple(violations), + tls=any(getattr(view, dest, None) for dest in _TLS_DESTS), + ) + + +class _RouterView(RouterConfigBase, KvRouterConfigBase): + """The router fields of ``FrontendConfig``, without its other checks.""" + + tls_cert_path: Optional[str] = None + tls_key_path: Optional[str] = None + + +@lru_cache(maxsize=256) +def _policy_maps( + router_config_json: str, wts: tuple[str, ...] +) -> dict[str, dict[str, float]]: + config = KvRouterConfig.from_json(router_config_json) + return { + wt: {key: float(value) for key, value in config.policy_parameters(wt).items()} + for wt in wts + } + + +def _router_mode(cli_mode: str) -> str: + """``--router-mode`` in the ``RetuneEngineSpec.router_mode`` spelling.""" + + return "kv_router" if cli_mode == "kv" else cli_mode.replace("-", "_") + + +# ----------------------------------------------------------- argv and env + + +def _effective_argv(spec: Mapping[str, Any]) -> tuple[str, ...]: + """The frontend container's argv after the operator's defaulting. + + graph.go: an authored frontend ``command`` drops the default args; with + none, the command stays ``python3`` and the args default to + ``-m dynamo.frontend`` unless authored. + """ + + if spec.get("roles"): + raise FrontendParseError( + "P4: the frontend component uses roles; live retune needs a " + "single-pod frontend" + ) + main = _main_container(spec) + command = [str(part) for part in main.get("command") or []] + args = [str(part) for part in main.get("args") or []] + if command: + return tuple(command + args) + return (*DEFAULT_FRONTEND_COMMAND, *(args or DEFAULT_FRONTEND_ARGS)) + + +def _frontend_args(argv: Sequence[str]) -> list[str]: + """The arguments after ``python -m dynamo.frontend``; anything else fails.""" + + if not argv: + raise FrontendParseError("P4: the frontend container has an empty command") + program = posixpath.basename(argv[0]) + if program in _SHELLS or (len(argv) > 1 and argv[1] == "-c"): + raise FrontendParseError( + f"P4: the frontend command is shell-wrapped ({' '.join(argv[:2])} ...); " + "write it as command: [python3] and args: [-m, dynamo.frontend, ...]" + ) + if not program.startswith("python"): + raise FrontendParseError( + f"P4: the frontend command starts with {argv[0]!r}, not a python " + f"interpreter running -m {FRONTEND_MODULE}" + ) + index = 1 + while index < len(argv): + token = argv[index] + if token in _PYTHON_FLAGS: + index += 1 + continue + if token in _PYTHON_FLAGS_WITH_VALUE: + index += 2 + continue + if token == "-m" and index + 1 < len(argv): + module, rest = argv[index + 1], index + 2 + elif token.startswith("-m") and len(token) > 2: + module, rest = token[2:], index + 1 + else: + break + if module == FRONTEND_MODULE: + return list(argv[rest:]) + break + raise FrontendParseError( + f"P4: the frontend command {' '.join(argv[:4])!r} does not run " + f"python -m {FRONTEND_MODULE}" + ) + + +def _component_env(dgd: Mapping[str, Any], spec: Mapping[str, Any]) -> _Env: + """Merge a component's env the way the operator renders it. + + The operator injects ``DYN_PARENT_DGD_K8S_*`` and ``POD_*``, MergeEnvs + overlays graph ``env`` and the main container's ``env`` and sorts the + result by name; Kubernetes expands ``$(VAR)`` against earlier names. + """ + + metadata = dgd.get("metadata") or {} + entries: dict[str, Mapping[str, Any]] = { + ENV_PARENT_DGD_NAME: { + "name": ENV_PARENT_DGD_NAME, + "value": str(metadata.get("name", "")), + }, + ENV_PARENT_DGD_NAMESPACE: { + "name": ENV_PARENT_DGD_NAMESPACE, + "value": str(metadata.get("namespace", "")), + }, + } + for name in _OPERATOR_FIELD_REF_ENV: + entries[name] = {"name": name, "valueFrom": {"fieldRef": {}}} + main = _main_container(spec) + for entry in [*_graph_env(dgd), *(main.get("env") or [])]: + if isinstance(entry, Mapping) and entry.get("name"): + entries[str(entry["name"])] = entry + + values: dict[str, str] = {} + opaque: set[str] = set() + for name in sorted(entries): + entry = entries[name] + expanded = ( + None + if entry.get("valueFrom") is not None + else _expand(str(entry.get("value") or ""), values, opaque) + ) + if expanded is None: + opaque.add(name) + else: + values[name] = expanded + env_from = tuple( + source for source in main.get("envFrom") or [] if isinstance(source, Mapping) + ) + return _Env( + entries=entries, + values=values, + opaque=frozenset(opaque), + env_from=env_from, + ) + + +def _expand( + value: str, + values: Mapping[str, str], + opaque: Iterable[str], + *, + mark_opaque: bool = False, +) -> Optional[str]: + """Kubernetes ``$(VAR)`` expansion. + + ``$$`` escapes a ``$`` and a reference to an undefined name stays as + written. A reference to an opaque name makes the result ``None``, or, + with ``mark_opaque``, is replaced by a marker. + """ + + hidden = set(opaque) + out: list[str] = [] + index = 0 + while index < len(value): + if value.startswith("$$", index): + out.append("$") + index += 2 + continue + if value.startswith("$(", index): + close = value.find(")", index + 2) + if close != -1: + name = value[index + 2 : close] + if name in hidden: + if not mark_opaque: + return None + out.append(f"{_OPAQUE}{name}\x00") + else: + out.append(values.get(name, value[index : close + 1])) + index = close + 1 + continue + out.append(value[index]) + index += 1 + return "".join(out) + + +def _expand_arg(arg: str, values: Mapping[str, str], opaque: Iterable[str]) -> str: + """Expand one argument, marking opaque references for the parser.""" + + expanded = _expand(arg, values, opaque, mark_opaque=True) + assert expanded is not None + return expanded + + +def _opaque_flag_position(unknown: Sequence[str]) -> Optional[str]: + """A left-over argument whose unreadable env could stand for a flag. + + The frontend parser consumes the value of every flag it knows, so a + marked argument it leaves over sits where a flag could, unless it + follows a left-over option (a flag the frontend hands to its chat + processor, and this its value). A left-over option whose name is + marked always could. + """ + + previous: Optional[str] = None + for token in unknown: + if token.startswith("-"): + if _OPAQUE in token.split("=", 1)[0]: + return token + elif token.startswith(_OPAQUE) and not ( + previous is not None + and previous.startswith("-") + and "=" not in previous + and _OPAQUE not in previous + ): + return token + previous = token + return None + + +def _opaque_reference(text: str) -> Optional[str]: + """``$(NAME)`` for the first opaque marker in ``text``, or ``None``. + + argparse quotes a rejected value with ``repr``, so the marker may appear + escaped. + """ + + for marker, end_marker in ((_OPAQUE, "\x00"), (_OPAQUE_REPR, "\\x00")): + index = text.find(marker) + if index != -1: + start = index + len(marker) + end = text.find(end_marker, start) + return f"$({text[start:end] if end != -1 else text[start:]})" + return None + + +def _env_from_may_set_router_env(source: Mapping[str, Any]) -> bool: + """Whether an envFrom source could define a ``DYN_*`` name.""" + + prefix = str(source.get("prefix") or "") + return "DYN_".startswith(prefix) or prefix.startswith("DYN_") + + +def _describe_env_from(source: Mapping[str, Any]) -> str: + for kind in ("configMapRef", "secretRef"): + reference = source.get(kind) + if isinstance(reference, Mapping): + return f"{kind} {reference.get('name')!r}" + return canonical_json(dict(source)) + + +# ---------------------------------------------------- frontend arg parser + + +def _parse_error_message(message: str) -> str: + reference = _opaque_reference(message) + if reference is None: + return f"P4: cannot parse the frontend args: {message.strip()}" + return ( + f"P4: a frontend router arg references {reference}, which the planner " + "cannot read" + ) + + +def _worker_parse_error_message(message: str) -> str: + reference = _opaque_reference(message) + if reference is None: + return f"has router flags the planner cannot parse ({message.strip()})" + return ( + f"has a router flag that references {reference}, which the planner " + "cannot read" + ) + + +class _ArgsParser(argparse.ArgumentParser): + """An ``ArgumentParser`` that raises instead of printing and exiting. + + A value carrying the opaque marker skips type conversion and the + choices check unless its destination is in ``relevant``, so an + unreadable value of a flag that does not matter here parses, while one + of a flag that does fails with a message naming the reference. + """ + + def __init__( + self, + *, + relevant: frozenset[str] = _RELEVANT_DESTS, + describe: Callable[[str], str] = _parse_error_message, + ) -> None: + super().__init__(add_help=False) + self._relevant = relevant + self._describe = describe + + def error(self, message: str): # type: ignore[override] + raise FrontendParseError(self._describe(message)) + + def exit(self, status: int = 0, message: Optional[str] = None): # type: ignore[override] + raise FrontendParseError( + self._describe(message or f"argument parsing exited with {status}") + ) + + def _opaque_irrelevant(self, action: argparse.Action, value: Any) -> bool: + return ( + isinstance(value, str) + and _OPAQUE in value + and action.dest not in self._relevant + ) + + def _get_value(self, action, arg_string): # type: ignore[no-untyped-def] + if self._opaque_irrelevant(action, arg_string): + return arg_string + return super()._get_value(action, arg_string) + + def _check_value(self, action, value): # type: ignore[no-untyped-def] + if self._opaque_irrelevant(action, value): + return None + return super()._check_value(action, value) + + +class _OsWithEnv: + """``os`` with ``environ`` replaced by the frontend's env.""" + + def __init__(self, environ: Mapping[str, str]) -> None: + self.environ = dict(environ) + + def __getattr__(self, name: str) -> Any: + return getattr(os, name) + + +class _QuietIgnore(argparse.Action): + """Accept and drop a removed flag, without the frontend's warning.""" + + def __call__(self, parser, namespace, values, option_string=None) -> None: # type: ignore[no-untyped-def] + return None + + +class _NoArguments: + """Replaces an argument group none of whose flags matter here.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + def add_arguments(self, parser: Any) -> None: + return None + + +def _rebind( + function: Callable[..., Any], overrides: Mapping[str, Any] +) -> Callable[..., Any]: + """A copy of ``function`` whose module globals are overlaid with ``overrides``. + + The module is untouched, so other threads keep running the original. + """ + + rebound = types.FunctionType( + function.__code__, + {**function.__globals__, **overrides}, + function.__name__, + function.__defaults__, + function.__closure__, + ) + rebound.__kwdefaults__ = function.__kwdefaults__ + return rebound + + +def _frontend_parser( + env: Mapping[str, str], +) -> tuple[argparse.ArgumentParser, dict[str, str]]: + """The frontend's argument parser with every env default read from ``env``. + + Runs ``FrontendArgGroup.add_arguments`` and the router groups it adds as + copies whose ``os``, ``env_or_default``, ``add_argument`` and + ``add_negatable_bool_argument`` see only ``env``. Deprecation warnings, + the AISimulate perf-model flags (whose type reads files) and the + ``--model-path`` directory check are dropped. Returns the parser and the + env name -> dest map of everything it registered. + """ + + from dynamo.common.configuration.groups import kv_router_args, router_args + from dynamo.frontend import frontend_args + + overrides, bound, registered = _env_bound(env) + add_frontend_arguments = _rebind( + frontend_args.FrontendArgGroup.add_arguments, + { + **overrides, + "RouterArgGroup": bound(router_args.RouterArgGroup), + "KvRouterArgGroup": bound(kv_router_args.KvRouterArgGroup), + "AisPerfArgGroup": _NoArguments, + "validate_model_path": str, + }, + ) + parser = _ArgsParser() + try: + add_frontend_arguments(frontend_args.FrontendArgGroup(), parser) + except (ValueError, TypeError, argparse.ArgumentTypeError) as error: + # The frontend itself fails to start on such env. + raise FrontendParseError(f"P4: invalid frontend env: {error}") from None + + # The removed flags only log when used; keep their arity, drop the noise. + for action in parser._actions: + if isinstance(action, router_args._DeprecatedEnforceDisaggAction): + action.__class__ = argparse.BooleanOptionalAction # type: ignore[assignment] + elif isinstance(action, router_args._IgnoredAdmissionControlAction): + action.__class__ = _QuietIgnore # type: ignore[assignment] + return parser, registered + + +def _worker_router_parser( + env: Mapping[str, str], +) -> tuple[argparse.ArgumentParser, dict[str, str]]: + """A worker's router-advertisement parser with env defaults from ``env``. + + Runs ``router_args.add_worker_router_arguments``, which every backend + (vLLM, SGLang, TensorRT-LLM, mocker) uses, the way ``_frontend_parser`` + runs the frontend's groups. Only ``router_mode`` is relevant: it decides + whether the worker advertises a router configuration at all. + """ + + from dynamo.common.configuration.groups import kv_router_args, router_args + + overrides, bound, registered = _env_bound(env) + add_worker_arguments = _rebind( + router_args.add_worker_router_arguments, + { + "RouterArgGroup": bound(router_args.RouterArgGroup), + "KvRouterArgGroup": bound(kv_router_args.KvRouterArgGroup), + }, + ) + parser = _ArgsParser( + relevant=frozenset({_ROUTER_MODE_DEST}), + describe=_worker_parse_error_message, + ) + try: + add_worker_arguments(parser) + except (ValueError, TypeError, argparse.ArgumentTypeError) as error: + # The worker itself fails to start on such env. + raise FrontendParseError(f"has invalid router env ({error})") from None + return parser, registered + + +def _env_bound( + env: Mapping[str, str], +) -> tuple[dict[str, Any], Callable[[type], type], dict[str, str]]: + """Globals that make an argument group read only ``env``. + + Returns the overrides for ``_rebind``, a function that subclasses an + ``ArgGroup`` with its ``add_arguments`` rebound to them, and the env + name -> dest map the bound groups fill while registering. + """ + + from dynamo.common.configuration import utils + + fake_os = _OsWithEnv(env) + registered: dict[str, str] = {} + env_or_default = _rebind(utils.env_or_default, {"os": fake_os}) + base_add = _rebind( + utils.add_argument, {"os": fake_os, "env_or_default": env_or_default} + ) + + def add_argument(parser: Any, **kwargs: Any) -> None: + kwargs.pop("deprecated", None) + flag = str(kwargs["flag_name"]) + registered[str(kwargs["env_var"])] = str( + kwargs.get("dest") or flag.lstrip("-").replace("-", "_") + ) + base_add(parser, **kwargs) + + overrides = { + "os": fake_os, + "env_or_default": env_or_default, + "add_argument": add_argument, + "add_negatable_bool_argument": _rebind( + utils.add_negatable_bool_argument, {"add_argument": add_argument} + ), + "logger": _SILENT_LOGGER, + } + + def bound(group: Any) -> type: + return type( + group.__name__, + (group,), + {"add_arguments": _rebind(group.add_arguments, overrides)}, + ) + + return overrides, bound, registered + + +# --------------------------------------------------------- premise checks + + +def _startup_errors( + startup: FrontendStartup, retune: RetuneConfig, planner_mode: str +) -> list[str]: + """P4 on a parsed startup: KV mode, defaults, the controller's checks.""" + + if startup.router_mode != "kv_router": + return [ + f"P4: the frontend routes with {startup.router_mode!r}; live retune " + "needs --router-mode kv" + ] + try: + searched = set(_router_domain_keys(retune)) + except ValueError as error: + return [f"P4: cannot expand retune.router: {error}"] + config = startup.router_config + errors: list[str] = [] + load_model = config.get(_LOAD_MODEL, "none") + if load_model != "none": + errors.append( + f"P4: the frontend's {_LOAD_MODEL} is {load_model!r}; live retune " + "needs 'none'" + ) + if config.get(_CONDITIONAL_DISAGG): + errors.append( + "P4: the frontend enables conditional disaggregation, which replay " + "and live routers resolve differently" + ) + for key in sorted(config): + if ( + key in searched + or key in TOLERATED_ROUTER_FIELDS + or key in (_LOAD_MODEL, _CONDITIONAL_DISAGG) + ): + continue + errors.append( + f"P4: the frontend sets {key}={config[key]!r}; only the searched " + f"router knobs ({', '.join(sorted(searched)) or 'none'}) may differ " + "from their defaults" + ) + if errors: + return errors + return _controller_errors(retune, config, worker_types(planner_mode)) + + +def _worker_router_errors( + dgd: Mapping[str, Any], + components: Sequence[Mapping[str, Any]], + warnings: list[str], +) -> list[str]: + """P4 on the workers: none advertises its own router configuration. + + A worker that resolves a router mode (``--router-mode`` or + ``DYN_ROUTER_MODE``, from its container or the graph env) puts a + ``RouterConfig`` in its model card, and the frontend routes that worker + set with it instead of its own flags; replay would model the frontend's. + """ + + errors: list[str] = [] + env_from: list[str] = [] + for component in components: + if _type(component) not in WORKER_TYPES: + continue + name = str(component.get("name", "")) + env = _component_env(dgd, component) + argv, shell = _worker_argv(component, env) + detail = _worker_router_cached( + canonical_json( + { + "argv": argv, + "shell": shell, + "values": dict(env.values), + "opaque": sorted(env.opaque), + } + ) + ) + if detail is not None: + errors.append( + f"P4: worker component {name} {detail}; a worker's router " + "configuration replaces the frontend's for its workers, so set " + f"--router-mode and {ENV_ROUTER_MODE} on the frontend only (a " + "graph-level env entry reaches every worker)" + ) + if any(_env_from_may_set_router_env(source) for source in env.env_from): + env_from.append(name) + if env_from: + warnings.append( + f"P4: worker components {', '.join(sorted(env_from))} use envFrom; " + "the planner cannot read it and assumes it sets no router env (a " + f"{ENV_ROUTER_MODE} there would make the worker advertise its own " + "router configuration)" + ) + return errors + + +@lru_cache(maxsize=64) +def _worker_router_cached(key: str) -> Optional[str]: + """Why one worker advertises a router configuration, or ``None``.""" + + inputs = json.loads(key) + try: + return _worker_router_mode( + inputs["argv"], + bool(inputs["shell"]), + inputs["values"], + frozenset(inputs["opaque"]), + ) + except FrontendParseError as error: + return str(error) + except Exception as error: # noqa: BLE001 - fail closed, never crash the planner + logger.exception("live retune could not parse a worker's router flags") + return ( + "could not be checked for a router configuration " + f"({type(error).__name__}: {error})" + ) + + +def _worker_router_mode( + argv: Sequence[str], + shell: bool, + values: Mapping[str, str], + opaque: frozenset[str], +) -> Optional[str]: + parser, registered = _worker_router_parser(values) + mode_env = sorted( + name for name, dest in registered.items() if dest == _ROUTER_MODE_DEST + ) + hidden = [name for name in mode_env if name in opaque] + if hidden: + return ( + f"sets {', '.join(hidden)} through valueFrom (or a reference to such " + "a value), so the planner cannot tell whether it advertises a router " + "configuration" + ) + # A shell script can also export the env inline. + inline = sorted( + {token.split("=", 1)[0] for token in argv if shell and "=" in token} + & set(mode_env) + ) + if inline: + return f"sets {', '.join(inline)} in its shell command" + namespace, _ = parser.parse_known_args(list(argv)) + mode = getattr(namespace, _ROUTER_MODE_DEST, None) + if mode is None: + return None + return ( + f"advertises its own router configuration (router mode {mode!r} from " + f"--router-mode or {ENV_ROUTER_MODE})" + ) + + +def _worker_argv(component: Mapping[str, Any], env: _Env) -> tuple[list[str], bool]: + """A worker's argv after ``$(VAR)`` expansion, and whether a shell runs it. + + Unreadable references stay marked. A shell-wrapped command is split + into the script's words, so the flags of the command it runs are seen. + """ + + main = _main_container(component) + parts: list[str] = [] + for part in [*(main.get("command") or []), *(main.get("args") or [])]: + expanded = _expand(str(part), env.values, env.opaque, mark_opaque=True) + assert expanded is not None + parts.append(expanded) + shell = bool(parts) and ( + posixpath.basename(parts[0]) in _SHELLS or parts[1:2] == ["-c"] + ) + if not shell: + return parts, False + words: list[str] = [] + for part in parts: + try: + words.extend(shlex.split(part)) + except ValueError: + words.extend(part.split()) + return words, True + + +def _router_domain_keys(retune: RetuneConfig) -> list[str]: + if retune.router is None: + return [] + from dynamo.planner.retune.domains import expand_router_domain + + return list(expand_router_domain(retune)) + + +def _controller_errors( + retune: RetuneConfig, router_config: Mapping[str, Any], wts: tuple[str, ...] +) -> list[str]: + """The startup-dependent checks of ``RetuneController._router_domain``. + + A startup failing them would make the controller constructor raise, so + P4 rejects it before a controller is built. + """ + + if retune.router is None: + return [] + from dynamo.planner.retune.domains import ( + OVERLAP_SCORE_CREDIT, + expand_router_domain, + grid_points, + ) + + startup_json = canonical_json(dict(router_config)) + try: + startup_maps = _policy_maps(startup_json, wts) + load_model = json.loads(KvRouterConfig.from_json(startup_json).to_json())[ + _LOAD_MODEL + ] + except ValueError as error: + return [ + f"P4: the frontend startup is not a valid router configuration: {error}" + ] + # A worker type that starts with no overlap credit can never get it. + zero_credit = sorted( + wt + for wt, params in startup_maps.items() + if wt != "decode" and params[OVERLAP_SCORE_CREDIT] == 0 + ) + try: + domain = expand_router_domain( + retune, + startup_load_model=load_model, + zero_startup_credit=bool(zero_credit), + ) + except ValueError as error: + return [f"P4: retune.router does not fit the frontend startup: {error}"] + for point in grid_points(domain, {}): + try: + maps = _policy_maps(canonical_json({**router_config, **point.router}), wts) + except ValueError as error: + return [ + f"P4: retune.router candidate {point.router} is not a valid " + f"router configuration for this frontend: {error}" + ] + for wt in zero_credit: + if maps[wt][OVERLAP_SCORE_CREDIT] != 0: + return [ + f"P4: retune.router candidate {point.router} gives the {wt} " + "router overlap_score_credit > 0, but the frontend starts it " + "at 0 (a live router rejects that update)" + ] + return [] + + +def _startup_config_mismatch( + declared: Mapping[str, Any], startup: FrontendStartup +) -> Optional[str]: + """``retune.dgdr.startup_router_config`` must mean the parsed startup.""" + + try: + declared_full = json.loads( + KvRouterConfig.from_json(json.dumps(dict(declared))).to_json() + ) + except ValueError as error: + return ( + "P4: retune.dgdr.startup_router_config is not a KvRouterConfig " + f"JSON: {error}" + ) + parsed_full = json.loads( + KvRouterConfig.from_json(json.dumps(dict(startup.router_config))).to_json() + ) + keys = sorted( + key + for key in set(declared_full) | set(parsed_full) + if declared_full.get(key) != parsed_full.get(key) + ) + if not keys: + return None + diffs = "; ".join( + f"{key}: declared {declared_full.get(key)!r}, frontend {parsed_full.get(key)!r}" + for key in keys + ) + return f"P4: retune.dgdr.startup_router_config differs from the frontend ({diffs})" + + +def _planner_replica_errors( + components: Sequence[Mapping[str, Any]], + planner_replicas: Optional[int], + warnings: list[str], +) -> list[str]: + if planner_replicas is not None: + if planner_replicas != 1: + return [ + f"P9: the planner has {planner_replicas} replicas; live retune needs 1" + ] + return [] + planners = [c for c in components if _type(c) == PLANNER_TYPE] + if not planners: + warnings.append( + "P9: the DGD has no planner component; its replicas cannot be checked" + ) + return [ + f"P9: planner component {c.get('name')} has {_replicas(c)} replicas; " + "live retune needs 1" + for c in planners + if _replicas(c) != 1 + ] + + +def _trace_recipe_errors( + analysis: _Analysis, + *, + dgd_name: str, + claim_name: str, + segments_dir: str, + warnings: list[str], +) -> list[str]: + """P10: the frontend writes request_end segments onto the claim (§4.4.1).""" + + assert analysis.spec is not None and analysis.env is not None + entries = analysis.env.entries + errors: list[str] = [] + + def literal(name: str) -> Optional[str]: + entry = entries.get(name) + if entry is None: + return None + if entry.get("valueFrom") is not None: + errors.append(f"P10: frontend env {name} must be a literal value") + return None + return str(entry.get("value") or "") + + trace = literal(ENV_TRACE) + if trace is None or trace.strip().lower() not in _TRUTHY: + errors.append(f"P10: the frontend must set {ENV_TRACE}=1") + + sinks = _csv(literal(ENV_TRACE_SINKS)) + if "file" not in sinks or {"jsonl", "jsonl_gz"} & set(sinks): + errors.append(f"P10: the frontend must set {ENV_TRACE_SINKS}=file") + elif set(sinks) != {"file"}: + extra = ", ".join(sorted(set(sinks) - {"file"})) + warnings.append(f"P10: {ENV_TRACE_SINKS} also enables {extra}") + + file_format = literal(ENV_TRACE_FORMAT) + if file_format is not None and file_format.strip().lower() not in _GZ_FORMATS: + errors.append(f"P10: {ENV_TRACE_FORMAT} must be jsonl_gz or unset") + + if set(_csv(literal(ENV_TRACE_RECORDS))) != {"request_end"}: + errors.append(f"P10: the frontend must set {ENV_TRACE_RECORDS}=request_end") + + pod_name = entries.get(ENV_POD_NAME) or {} + field_ref = (pod_name.get("valueFrom") or {}).get("fieldRef") or {} + if field_ref.get("fieldPath") != "metadata.name": + errors.append( + f"P10: the frontend must set {ENV_POD_NAME} from fieldRef metadata.name" + ) + parent = entries.get(ENV_PARENT_DGD_NAME) or {} + if parent.get("valueFrom") is not None or parent.get("value") != dgd_name: + errors.append( + f"P10: the frontend overrides {ENV_PARENT_DGD_NAME}; it must stay the " + f"DGD name {dgd_name!r}" + ) + + mounts, mount_errors = _claim_mounts(analysis.spec, claim_name) + errors.extend(mount_errors) + path = literal(ENV_TRACE_PATH) + if path is None: + errors.append(f"P10: the frontend must set {ENV_TRACE_PATH}") + elif mounts: + segments = segments_dir.strip("/") + prefixes = [ + _join_dir(mount, segments, dgd_dir, f"$({ENV_POD_NAME})") + for mount in mounts + for dgd_dir in (f"$({ENV_PARENT_DGD_NAME})", dgd_name) + ] + # Directly in the pod's directory: the reconciler prunes only the + # segments at its top level. + if not any( + path.startswith(prefix) + and len(path) > len(prefix) + and "/" not in path[len(prefix) :] + for prefix in prefixes + ): + errors.append( + f"P10: {ENV_TRACE_PATH}={path!r} must be a file prefix directly " + f"under {prefixes[0]!r} (no further '/')" + ) + return errors + + +def _claim_mounts( + spec: Mapping[str, Any], claim_name: str +) -> tuple[list[str], list[str]]: + """Writable main-container mount paths of the claim's root, and errors.""" + + pod_spec = _pod_spec(spec) + volumes = { + str(volume.get("name")) + for volume in pod_spec.get("volumes") or [] + if isinstance(volume, Mapping) + and (volume.get("persistentVolumeClaim") or {}).get("claimName") == claim_name + } + if not volumes: + return [], [f"P10: the frontend pod has no volume for claim {claim_name!r}"] + candidates = [ + mount + for mount in _main_container(spec).get("volumeMounts") or [] + if isinstance(mount, Mapping) and mount.get("name") in volumes + ] + if not candidates: + return [], [ + f"P10: the frontend main container does not mount claim {claim_name!r}" + ] + mounts = [ + str(mount["mountPath"]) + for mount in candidates + if mount.get("mountPath") + and not mount.get("readOnly") + and not mount.get("subPath") + and not mount.get("subPathExpr") + ] + if not mounts: + return [], [ + f"P10: the frontend mounts claim {claim_name!r} read-only or through a " + "subPath; mount the claim root writable" + ] + return mounts, [] + + +def _join_dir(mount: str, *parts: str) -> str: + return "/".join([mount.rstrip("/"), *(part for part in parts if part)]) + "/" + + +def _model_warning(model: str) -> Optional[str]: + try: + from aisimulate_core.sdk.common import DefaultHFModels + except Exception: # noqa: BLE001 - optional dependency of a warn-only premise + return None + if model in DefaultHFModels: + return None + return ( + f"P6: retune.dgdr.model {model!r} is not an AISimulate default HF model; " + "each DGDR Run must be able to download it" + ) + + +# ------------------------------------------------------------ DGD helpers + + +def _components(dgd: Mapping[str, Any]) -> list[Mapping[str, Any]]: + spec = dgd.get("spec") or {} + components = spec.get("components") + if isinstance(components, list): + return [c for c in components if isinstance(c, Mapping)] + services = spec.get("services") + if isinstance(services, Mapping): + return [ + _from_v1alpha1(str(name), service) + for name, service in services.items() + if isinstance(service, Mapping) + ] + return [] + + +def _from_v1alpha1(name: str, service: Mapping[str, Any]) -> dict[str, Any]: + """A v1alpha1 service in the v1beta1 component shape this module reads.""" + + extra = service.get("extraPodSpec") or {} + main = dict(extra.get("mainContainer") or {}) + main["name"] = MAIN_CONTAINER + env: dict[str, Any] = {} + for entry in [*(service.get("envs") or []), *(main.get("env") or [])]: + if isinstance(entry, Mapping) and entry.get("name"): + env[str(entry["name"])] = entry + main["env"] = list(env.values()) + env_from = list(main.get("envFrom") or []) + if service.get("envFromSecret"): + env_from.append({"secretRef": {"name": service["envFromSecret"]}}) + main["envFrom"] = env_from + # v1alpha1 volumeMounts name a PVC and mount it at mountPoint. + mounts = list(main.get("volumeMounts") or []) + volumes = list(extra.get("volumes") or []) + for mount in service.get("volumeMounts") or []: + if isinstance(mount, Mapping) and mount.get("name"): + claim = str(mount["name"]) + mounts.append({"name": claim, "mountPath": mount.get("mountPoint")}) + volumes.append( + {"name": claim, "persistentVolumeClaim": {"claimName": claim}} + ) + main["volumeMounts"] = mounts + component: dict[str, Any] = { + "name": name, + "type": service.get("componentType") or "", + "podTemplate": {"spec": {"containers": [main], "volumes": volumes}}, + } + for key in ("replicas", "roles", "modelRef"): + if service.get(key) is not None: + component[key] = service[key] + if service.get("frontendSidecar"): + component["frontendSidecar"] = "sidecar-frontend" + # v1alpha1 needs enabled: true; v1beta1 opts in with any object. + if (service.get("scalingAdapter") or {}).get("enabled"): + component["scalingAdapter"] = {} + return component + + +def _graph_env(dgd: Mapping[str, Any]) -> list[Any]: + spec = dgd.get("spec") or {} + return list(spec.get("env") or spec.get("envs") or []) + + +def _pod_spec(spec: Mapping[str, Any]) -> Mapping[str, Any]: + template = spec.get("podTemplate") + pod_spec = template.get("spec") if isinstance(template, Mapping) else None + return pod_spec if isinstance(pod_spec, Mapping) else {} + + +def _main_container(spec: Mapping[str, Any]) -> Mapping[str, Any]: + for container in _pod_spec(spec).get("containers") or []: + if isinstance(container, Mapping) and container.get("name") == MAIN_CONTAINER: + return container + return {} + + +def _sidecar_components(components: Sequence[Mapping[str, Any]]) -> list[str]: + return sorted(str(c.get("name")) for c in components if c.get("frontendSidecar")) + + +def _type(component: Mapping[str, Any]) -> str: + return str(component.get("type") or "") + + +def _replicas(component: Mapping[str, Any]) -> int: + """Spec replicas; an omitted value is the workload default of 1.""" + + replicas = component.get("replicas") + try: + return 1 if replicas is None else int(replicas) + except (TypeError, ValueError): + return 1 + + +def _scaling_adapter(component: Mapping[str, Any]) -> bool: + return component.get("scalingAdapter") is not None + + +def _dgd_name(dgd: Mapping[str, Any]) -> str: + return str((dgd.get("metadata") or {}).get("name", "")) + + +def _projected_status(dgd: Mapping[str, Any]) -> Mapping[str, Any]: + """``status.components`` while the status is current, else empty.""" + + status = dgd.get("status") or {} + generation = (dgd.get("metadata") or {}).get("generation") + current = generation is not None and generation == status.get("observedGeneration") + return (status.get("components") or {}) if current else {} + + +def _model_flags( + component: Mapping[str, Any], env: _Env +) -> list[tuple[str, Optional[str]]]: + """Each model flag in the main container's command and args, in order. + + Values are ``$(VAR)``-expanded against the component's env; a value + that still references env (or looks like a flag) is ``None``. + """ + + main = _main_container(component) + tokens: list[str] = [] + for part in [*(main.get("command") or []), *(main.get("args") or [])]: + text = _expand(str(part), env.values, env.opaque, mark_opaque=True) + assert text is not None + try: + tokens.extend(shlex.split(text)) + except ValueError: + tokens.append(text) + found: list[tuple[str, Optional[str]]] = [] + for index, token in enumerate(tokens): + for flag in _MODEL_FLAGS: + if token == flag and index + 1 < len(tokens): + value = tokens[index + 1] + elif token.startswith(flag + "="): + value = token[len(flag) + 1 :] + else: + continue + resolved = ( + value + and "$" not in value + and _OPAQUE not in value + and not value.startswith("-") + ) + found.append((flag, value if resolved else None)) + return found + + +def _flag_model(flags: Sequence[tuple[str, Optional[str]]]) -> Optional[str]: + """The operator's served model: the first flag of ``_MODEL_FLAGS`` given.""" + + for flag in _MODEL_FLAGS: + for name, value in flags: + if name == flag: + return value + return None + + +def _worker_model_names( + dgd: Mapping[str, Any], components: Sequence[Mapping[str, Any]] +) -> set[str]: + """Every model name or path the workers give, and their served names.""" + + projected = _projected_status(dgd) + names: set[str] = set() + for component in components: + if _type(component) not in WORKER_TYPES: + continue + served = (projected.get(str(component.get("name", ""))) or {}).get( + "servedModelName" + ) + if served: + names.add(str(served)) + for _, value in _model_flags(component, _component_env(dgd, component)): + if value: + names.update(part for part in re.split(r"[\s,]+", value) if part) + return names + + +def _csv(value: Optional[str]) -> list[str]: + if not value: + return [] + return [part.strip().lower() for part in value.split(",") if part.strip()] + + +def _unique(items: Iterable[str]) -> list[str]: + return list(dict.fromkeys(items)) diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 1dc508dac62f..d8cf1521f201 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -171,6 +171,17 @@ class SweepRequest: ``retune.domains`` expands ``retune.planner.load_predictor``; its winner's predictor fields are fixed into every non-baseline candidate. ``objective`` is the metric ``score_summary`` scores each replay on. + + The remaining fields are set only by a live controller (``launch_gate`` + set) and keep their defaults in replay, where ``run_sweep`` ignores them: + ``router_domain`` is the controller's router domain (live knob name -> + values), ``deadline_s`` the absolute epoch deadline ``D_k = L_k + + timeout``, ``spec_replicas`` the DGD spec replicas by worker type the + gate reported, ``replicas_verified`` whether the recorded ready counts + cover the whole window (``False`` when the window began before the + controller started), and ``generation`` the launching controller's + generation. Abort events are passed to the sweep function, never stored + here, so the request stays picklable. """ window_index: int @@ -192,6 +203,11 @@ class SweepRequest: max_parallel: int timeout_s: float history_paths: tuple[str, ...] = () + router_domain: dict[str, tuple[float, ...]] | None = None + deadline_s: float | None = None + spec_replicas: dict[str, int] | None = None + replicas_verified: bool = True + generation: int = 0 def __post_init__(self) -> None: keys = [candidate.key() for candidate in self.candidates] @@ -207,6 +223,8 @@ def __post_init__(self) -> None: raise ValueError("max_parallel must be at least 1") if not self.timeout_s > 0: raise ValueError("timeout_s must be positive") + if self.deadline_s is not None and not math.isfinite(self.deadline_s): + raise ValueError("deadline_s must be finite") if self.predictor_candidates: labels = [label for label, _ in self.predictor_candidates] if len(set(labels)) != len(labels): @@ -238,7 +256,11 @@ class SweepResult: prepares it on ``warmup_observations`` as a host does before a swap; ``predictor_warmup_error`` is that preparation's error, if any. ``row_counts`` holds the window reader's counts and, when the history - was read, ``history_requests``. + was read, ``history_requests``. ``failure_reason`` is a stable, + machine-readable reason a live executor sets next to the free-text + ``error`` (``retune.live.contracts.NON_DECISION_FAILURES`` mean the + window was abandoned, not failed); ``details`` carries JSON-shaped + executor facts for the decision record. ``run_sweep`` sets neither. """ status: SweepStatus @@ -251,6 +273,8 @@ class SweepResult: predictor_warmup_error: str | None = None wall_s: float = 0.0 error: str | None = None + failure_reason: str | None = None + details: dict[str, Any] = field(default_factory=dict) def with_predictor_fields( diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py new file mode 100644 index 000000000000..0429ec30f493 --- /dev/null +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -0,0 +1,1098 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""In-memory fakes of the Kubernetes side of live retune. + +``FakeRetuneKube`` implements ``contracts.RetuneKube`` over plain dicts with +the API-server behaviour the live modules depend on: ``resourceVersion`` +compare-and-swap on Lease replace, ``AlreadyExists`` on create, Strict field +validation and silent pruning on Request create, label selectors, cascading +Request deletion, and injectable faults (including writes that time out but +land). ``FakeDgdrOperator`` drives the Requests in a ``FakeRetuneKube`` +through the DGDR v1beta2 lifecycle (Accepted -> Run -> Completed with +Candidates) the way the operator at #17483 does, lowering each Request into +``Run.status.aisimulate.input`` with the same merge-patch rules. + +This is a plain module, not a conftest: tests import what they need. +""" + +from __future__ import annotations + +import copy +import functools +import itertools +import json +import re +import threading +import time +import uuid +from collections import deque +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field, replace +from typing import Any, Literal, Optional + +from dynamo.planner.retune.live.contracts import ( + DGDR_ROUTER_KNOBS, + POD_LABEL_COMPONENT_TYPE, + POD_LABEL_DGD_NAME, + RUN_UID_LABEL, + KubeAlreadyExists, + KubeConflict, + KubeError, + KubeInvalid, + KubeNotFound, + PodInfo, +) + +# Operator constants (deploy/operator/internal/controller at #17483). +REQUEST_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" +CANDIDATE_ID_LABEL = "nvidia.com/dgdr-candidate-id" +TRACE_MOUNT_PATH = "/var/run/aisimulate/trace" +DEFAULT_RUNTIME_IMAGE = "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.8.0" +DEFAULT_DYNAMO_VERSION = "1.8.0" +ROUTER_ADAPTER = "dynamo.router" + +_METRIC_TARGETS = { + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", +} + +KubeMethod = Literal[ + "get_dgd", + "list_pods", + "create_request", + "get_request", + "list_requests", + "delete_request", + "get_run", + "list_candidates", + "get_lease", + "create_lease", + "replace_lease", + "delete_lease", +] + + +# ------------------------------------------------------------------ helpers + + +def merge_patch(target: Any, patch: Any) -> Any: + """RFC 7386 JSON merge patch; returns a new value, never mutates.""" + + if not isinstance(patch, Mapping): + return copy.deepcopy(patch) + result = dict(target) if isinstance(target, Mapping) else {} + for key, value in patch.items(): + if value is None: + result.pop(key, None) + else: + result[key] = merge_patch(result.get(key), value) + return result + + +def nested_get(obj: Any, *path: str, default: Any = None) -> Any: + for key in path: + if not isinstance(obj, Mapping) or key not in obj: + return default + obj = obj[key] + return obj + + +def _nested_set(obj: dict, path: tuple[str, ...], value: Any) -> None: + for key in path[:-1]: + child = obj.get(key) + if not isinstance(child, dict): + child = {} + obj[key] = child + obj = child + obj[path[-1]] = value + + +def _nested_pop(obj: Any, path: tuple[str, ...]) -> bool: + for key in path[:-1]: + if not isinstance(obj, dict) or key not in obj: + return False + obj = obj[key] + if isinstance(obj, dict) and path[-1] in obj: + del obj[path[-1]] + return True + return False + + +def _split(path: str) -> tuple[str, ...]: + return tuple(path.split(".")) + + +def rfc3339(epoch_s: float) -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch_s)) + + +_SELECTOR_TERM = re.compile( + r"^\s*(!?)\s*([A-Za-z0-9._/-]+)\s*(?:(==|=|!=)\s*(\S*))?\s*$" +) + + +def match_labels(labels: Mapping[str, str] | None, selector: str) -> bool: + """Equality-based label selector: ``k=v``, ``k==v``, ``k!=v``, ``k``, ``!k``.""" + + labels = labels or {} + if not selector.strip(): + return True + for term in selector.split(","): + found = _SELECTOR_TERM.match(term) + if found is None: + raise ValueError(f"unsupported label selector term {term!r}") + negate, key, op, value = found.groups() + if op is None: + present = key in labels + if present == bool(negate): + return False + elif negate: + raise ValueError(f"unsupported label selector term {term!r}") + elif op in ("=", "=="): + if labels.get(key) != value: + return False + elif labels.get(key) == value: # != + return False + return True + + +def lower_request_input( + request: Mapping[str, Any], + *, + run_name: str, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + gpus_per_node: int = 8, +) -> dict[str, Any]: + """``Run.status.aisimulate.input`` for a Request, as the operator lowers it. + + Mirrors ``aisimulate.LowerWithUnderlay`` and ``dgdrRunInput`` at #17483: + the public role underlay, the ``dgdr_run`` static underlay, the typed + underlay, the derived defaults, ``inputOverrides`` as the final merge + patch, then the operator-owned ``dgdr_run`` fields and the trace source. + """ + + spec = request.get("spec") or {} + max_candidates = int(nested_get(spec, "recommendation", "maxCandidates", default=1)) + underlay: dict[str, Any] = { + "engine": {"workers": {"aggregated": {}, "prefill": {}, "decode": {}}} + } + underlay = merge_patch( + underlay, {"dgdr_run": {"name": run_name, "max_candidates": max_candidates}} + ) + typed: dict[str, Any] = {} + model = nested_get(spec, "model", "name") + if model is not None: + _nested_set(typed, ("engine", "model"), model) + if spec.get("backends"): + _nested_set(typed, ("engine", "backend", "choices"), list(spec["backends"])) + sku = nested_get(spec, "hardware", "gpu", "sku") + if sku: + _nested_set(typed, ("engine", "hardware"), sku) + budget = nested_get(spec, "hardware", "gpu", "budget") + if budget is not None: + _nested_set( + typed, ("optimization", "constraints", "max_candidate_gpus"), int(budget) + ) + trace = nested_get(spec, "workload", "trace") + trace_source = None + if trace is not None: + path = nested_get(trace, "source", "pvc", "path", default="") + mounted = TRACE_MOUNT_PATH + ( + "/trace.jsonl.gz" if path.endswith(".gz") else "/trace.jsonl" + ) + trace_source = { + "type": "trace", + "format": trace.get("format"), + "paths": [mounted], + } + _nested_set(typed, ("traffic", "source"), dict(trace_source)) + _nested_set(typed, ("traffic", "load", "type"), "trace_timestamps") + speedup = nested_get(trace, "replay", "arrivalSpeedupRatio") + if speedup is not None: + _nested_set(typed, ("traffic", "load", "speedup"), speedup) + objective = spec.get("objective") or {} + if objective.get("metric") is not None: + _nested_set( + typed, ("optimization", "target"), _METRIC_TARGETS[objective["metric"]] + ) + sla = objective.get("sla") or {} + for api_key, input_key in ( + ("ttftMs", "ttft_ms"), + ("itlMs", "itl_ms"), + ("e2eMs", "e2e_ms"), + ): + if sla.get(api_key) is not None: + _nested_set(typed, ("evaluation", "sla", input_key), sla[api_key]) + underlay = merge_patch(underlay, typed) + if "traffic" in underlay: + source_type = nested_get(underlay, "traffic", "source", "type") + if source_type != "trace" and nested_get(underlay, "traffic", "stop") is None: + _nested_set(underlay, ("traffic", "stop", "requests"), 1000.0) + if nested_get(underlay, "optimizer", "max_trials") is None: + _nested_set(underlay, ("optimizer", "max_trials"), 320.0) + if nested_get(underlay, "evaluation", "sla") and ( + nested_get(underlay, "optimization", "strict_sla") is None + ): + _nested_set(underlay, ("optimization", "strict_sla"), True) + overrides = nested_get(spec, "aisimulate", "inputOverrides", default={}) or {} + effective = merge_patch(underlay, overrides) + reserved = { + "name": run_name, + "max_candidates": max_candidates, + "snapshot_dir": "/var/run/aisimulate/output", + "namespace": nested_get(request, "metadata", "namespace"), + "runtime_image": runtime_image, + "runtime_version_override": dynamo_version, + "derive_runtime_images": True, + "num_gpus_per_node": gpus_per_node, + } + for key, value in reserved.items(): + _nested_set(effective, ("dgdr_run", key), value) + if trace_source is not None: + traffic = effective.setdefault("traffic", {}) + traffic.setdefault("source", {}).update(trace_source) + # Round-trip through JSON like the API server does. + return json.loads(json.dumps(effective)) + + +def router_points(effective_input: Mapping[str, Any]) -> list[dict[str, float]]: + """The router points (public knob names) an input's router section spans.""" + + router = effective_input.get("router") or {} + axes: list[tuple[str, list[float]]] = [] + for public in DGDR_ROUTER_KNOBS: + value = router.get(public) + if isinstance(value, Mapping): + axes.append((public, [float(v) for v in value["choices"]])) + elif value is not None: + axes.append((public, [float(value)])) + return [ + dict(zip([name for name, _ in axes], values)) + for values in itertools.product(*(values for _, values in axes)) + ] + + +def engine_parameters(effective_input: Mapping[str, Any]) -> dict[str, Any]: + """The non-router parameters every Candidate of a pinned engine shares.""" + + engine = effective_input.get("engine") or {} + workers = engine.get("workers") or {} + params: dict[str, Any] = { + "backend": engine.get("backend"), + "deployment_mode": "disagg" if engine.get("mode") == "disaggregated" else "agg", + "model": engine.get("model"), + "hardware": engine.get("hardware"), + } + for role, worker in sorted(workers.items()): + if isinstance(worker, Mapping): + preset = nested_get(worker, "parallelism", "preset", default=[{}]) + params[f"{role}_parallelism"] = dict(preset[0]) if preset else {} + return json.loads(json.dumps(params)) + + +def candidate_parameters( + effective_input: Mapping[str, Any], point_public: Mapping[str, float] +) -> dict[str, Any]: + """``Candidate.spec.parameters`` of one evaluated router point.""" + + router = effective_input.get("router") or {} + return { + **engine_parameters(effective_input), + "adapters": { + ROUTER_ADAPTER: { + "policy": router.get("policy", "kv_router"), + "prefill_load_model": dict( + router.get("prefill_load_model") or {"type": "none"} + ), + **{name: float(value) for name, value in point_public.items()}, + } + }, + } + + +def default_metrics(point_public: Mapping[str, float]) -> dict[str, float]: + """Deterministic replay summary metrics of a router point (for tests).""" + + credit = point_public.get("overlap_score_credit", 1.0) + scale = point_public.get("prefill_load_scale", 1.0) + temperature = point_public.get("temperature", 0.0) + goodput = 1000.0 + 100.0 * credit - 3.0 * abs(scale - 2.0) - 10.0 * temperature + return { + "goodput_output_throughput_tok_s": goodput, + "output_throughput_tok_s": goodput * 1.1, + "gpu_hours": 2.0, + "duration_ms": 3_600_000.0, + "mean_ttft_ms": 100.0 + scale, + "num_ttft_samples": 1000.0, + "mean_e2e_latency_ms": 1000.0 + scale, + "num_e2e_latency_samples": 1000.0, + "mean_output_token_throughput_per_user": 50.0 + credit, + } + + +# ------------------------------------------------------------------- faults + + +@dataclass +class Fault: + """One injected failure of a ``FakeRetuneKube`` method. + + ``landed=True`` performs the call and then raises (a write that timed + out but reached the API server). ``match`` limits the fault to calls + whose first positional argument (a name, selector or body) satisfies it. + """ + + error: Exception + times: int = 1 + landed: bool = False + match: Optional[Callable[[Any], bool]] = None + + +@dataclass +class KubeCall: + method: str + target: Any + kwargs: dict[str, Any] = field(default_factory=dict) + + +# --------------------------------------------------------------------- kube + + +class FakeRetuneKube: + """In-memory ``contracts.RetuneKube`` (thread-safe). + + Stores DGDs, pods (``PodInfo``), Requests, Runs, Candidates and Leases + as plain dicts. Objects come back as deep copies, so callers can never + alias the store. ``now`` stamps ``creationTimestamp``. + + Request create honours ``field_validation='Strict'``: a path listed in + ``unknown_fields`` (dotted, e.g. ``spec.workload.trace``) is rejected + with ``KubeInvalid``; a path in ``pruned_fields`` is dropped silently + (an API server that ignores the validation directive). ``reject_create`` + rejects every create with ``KubeInvalid``. ``on_request_created`` hooks + run after each stored (non dry-run) create. + """ + + def __init__( + self, + namespace: str = "test-ns", + *, + now: Callable[[], float] = time.time, + ) -> None: + self.namespace = namespace + self.now = now + self.dgds: dict[str, dict] = {} + self.pods: dict[str, PodInfo] = {} + self.requests: dict[str, dict] = {} + self.runs: dict[str, dict] = {} + # Each Run's lowered input, recorded in its status by ``create_job``. + self.run_inputs: dict[str, dict] = {} + self.candidates: dict[str, dict] = {} + self.leases: dict[str, dict] = {} + self.calls: list[KubeCall] = [] + self.deleted_requests: list[tuple[str, str]] = [] + self.unknown_fields: set[str] = set() + self.pruned_fields: set[str] = set() + self.reject_create: Optional[str] = None + self.on_request_created: list[Callable[[FakeRetuneKube, dict], None]] = [] + self.cascade_deletes = True + self._faults: dict[str, deque[Fault]] = {} + self._rv = itertools.count(1) + self._lock = threading.RLock() + + # ------------------------------------------------------------ faults + + def fail( + self, + method: KubeMethod, + error: Exception | None = None, + *, + times: int = 1, + landed: bool = False, + match: Optional[Callable[[Any], bool]] = None, + ) -> None: + """Make the next ``times`` matching calls of ``method`` raise ``error``.""" + + with self._lock: + self._faults.setdefault(method, deque()).append( + Fault( + error=error or KubeError(f"injected {method} failure"), + times=times, + landed=landed, + match=match, + ) + ) + + def clear_faults(self) -> None: + with self._lock: + self._faults.clear() + + def calls_of(self, method: str) -> list[KubeCall]: + with self._lock: + return [call for call in self.calls if call.method == method] + + def _call( + self, + method: str, + target: Any, + kwargs: dict[str, Any], + operation: Callable[[], Any], + ) -> Any: + with self._lock: + self.calls.append(KubeCall(method, copy.deepcopy(target), dict(kwargs))) + fault = self._take_fault(method, target) + if fault is not None and not fault.landed: + raise fault.error + result = operation() + if fault is not None: + raise fault.error + return result + + def _take_fault(self, method: str, target: Any) -> Optional[Fault]: + queue = self._faults.get(method) + if not queue: + return None + for fault in list(queue): + if fault.match is None or fault.match(target): + fault.times -= 1 + if fault.times <= 0: + queue.remove(fault) + return fault + return None + + def _stamp(self, obj: dict, *, kind: str, api_version: str) -> dict: + metadata = obj.setdefault("metadata", {}) + obj.setdefault("apiVersion", api_version) + obj.setdefault("kind", kind) + metadata.setdefault("namespace", self.namespace) + metadata.setdefault("uid", str(uuid.uuid4())) + metadata.setdefault("creationTimestamp", rfc3339(self.now())) + metadata.setdefault("generation", 1) + metadata["resourceVersion"] = str(next(self._rv)) + return obj + + def bump(self, obj: dict) -> None: + """Give a stored object a new ``resourceVersion``.""" + + obj["metadata"]["resourceVersion"] = str(next(self._rv)) + + # --------------------------------------------------------------- DGD + + def put_dgd(self, dgd: dict) -> dict: + with self._lock: + stored = copy.deepcopy(dgd) + self._stamp( + stored, kind="DynamoGraphDeployment", api_version="nvidia.com/v1beta1" + ) + self.dgds[stored["metadata"]["name"]] = stored + return copy.deepcopy(stored) + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + if name not in self.dgds: + raise KubeNotFound(f"DynamoGraphDeployment {name!r} not found") + return copy.deepcopy(self.dgds[name]) + + return self._call("get_dgd", name, {"timeout_s": timeout_s}, operation) + + # -------------------------------------------------------------- pods + + def add_pod( + self, + name: str, + *, + dgd: str | None = None, + component_type: str = "frontend", + ready: bool = True, + terminating: bool = False, + ip: str | None = None, + http_port: int | None = 8000, + labels: Mapping[str, str] | None = None, + ) -> PodInfo: + all_labels = dict(labels or {}) + if dgd is not None: + all_labels.setdefault(POD_LABEL_DGD_NAME, dgd) + if component_type: + all_labels.setdefault(POD_LABEL_COMPONENT_TYPE, component_type) + pod = PodInfo( + name=name, + uid=str(uuid.uuid4()), + ip=ip if ip is not None else f"10.0.0.{len(self.pods) + 1}", + ready=ready, + terminating=terminating, + labels=all_labels, + http_port=http_port, + ) + with self._lock: + self.pods[name] = pod + return pod + + def update_pod(self, name: str, **changes: Any) -> PodInfo: + with self._lock: + pod = replace(self.pods[name], **changes) + self.pods[name] = pod + return pod + + def remove_pod(self, name: str) -> None: + with self._lock: + self.pods.pop(name, None) + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + def operation() -> list[PodInfo]: + return [ + pod + for pod in self.pods.values() + if match_labels(pod.labels, label_selector) + ] + + return self._call( + "list_pods", label_selector, {"timeout_s": timeout_s}, operation + ) + + # ----------------------------------------------------------- Requests + + def create_request( + self, + body: dict, + *, + dry_run: bool = False, + timeout_s: float = 10.0, + ) -> dict: + def operation() -> dict: + if self.reject_create is not None: + raise KubeInvalid(self.reject_create) + stored = copy.deepcopy(body) + for path in sorted(self.unknown_fields): + if nested_get(stored, *_split(path)) is not None: + raise KubeInvalid(f'strict decoding error: unknown field "{path}"') + for path in self.pruned_fields: + _nested_pop(stored, _split(path)) + name = nested_get(stored, "metadata", "name") + if not name: + raise KubeInvalid("metadata.name is required") + if name in self.requests: + raise KubeAlreadyExists( + f"dynamographdeploymentrequests {name!r} already exists" + ) + self._stamp( + stored, + kind="DynamoGraphDeploymentRequest", + api_version="nvidia.com/v1beta2", + ) + if dry_run: + return stored + self.requests[name] = stored + for hook in list(self.on_request_created): + hook(self, copy.deepcopy(stored)) + return copy.deepcopy(stored) + + return self._call( + "create_request", + body, + {"dry_run": dry_run, "timeout_s": timeout_s}, + operation, + ) + + def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.requests.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_request", name, {"timeout_s": timeout_s}, operation) + + def list_requests( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + def operation() -> list[dict]: + return [ + copy.deepcopy(obj) + for obj in self.requests.values() + if match_labels(obj["metadata"].get("labels"), label_selector) + ] + + return self._call( + "list_requests", label_selector, {"timeout_s": timeout_s}, operation + ) + + def delete_request( + self, + name: str, + *, + propagation: Literal["Background", "Foreground"] = "Background", + timeout_s: float = 10.0, + ) -> None: + def operation() -> None: + request = self.requests.pop(name, None) + if request is None: + return + self.deleted_requests.append((name, propagation)) + if not self.cascade_deletes: + return + request_uid = request["metadata"]["uid"] + for run_name, run in list(self.runs.items()): + owners = run["metadata"].get("ownerReferences") or [] + if any(owner.get("uid") == request_uid for owner in owners): + self._delete_run(run_name) + + return self._call( + "delete_request", + name, + {"propagation": propagation, "timeout_s": timeout_s}, + operation, + ) + + # --------------------------------------------------- Runs, Candidates + + def _delete_run(self, run_name: str) -> None: + run = self.runs.pop(run_name, None) + self.run_inputs.pop(run_name, None) + if run is None: + return + run_uid = run["metadata"]["uid"] + for candidate_name, candidate in list(self.candidates.items()): + labels = candidate["metadata"].get("labels") or {} + if labels.get(RUN_UID_LABEL) == run_uid: + del self.candidates[candidate_name] + + def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.runs.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_run", name, {"timeout_s": timeout_s}, operation) + + def list_candidates( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + def operation() -> list[dict]: + return [ + copy.deepcopy(obj) + for obj in self.candidates.values() + if match_labels(obj["metadata"].get("labels"), label_selector) + ] + + return self._call( + "list_candidates", label_selector, {"timeout_s": timeout_s}, operation + ) + + # ------------------------------------------------------------- Leases + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.leases.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_lease", name, {"timeout_s": timeout_s}, operation) + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + stored = copy.deepcopy(body) + name = nested_get(stored, "metadata", "name") + if not name: + raise KubeInvalid("metadata.name is required") + if name in self.leases: + raise KubeAlreadyExists(f"leases {name!r} already exists") + stored["metadata"].pop("resourceVersion", None) + self._stamp(stored, kind="Lease", api_version="coordination.k8s.io/v1") + self.leases[name] = stored + return copy.deepcopy(stored) + + return self._call("create_lease", body, {"timeout_s": timeout_s}, operation) + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + current = self.leases.get(name) + if current is None: + raise KubeNotFound(f"leases {name!r} not found") + sent = nested_get(body, "metadata", "resourceVersion") + if sent != current["metadata"]["resourceVersion"]: + raise KubeConflict( + f"leases {name!r}: resourceVersion {sent!r} is stale " + f"(current {current['metadata']['resourceVersion']!r})" + ) + stored = copy.deepcopy(body) + metadata = stored.setdefault("metadata", {}) + for key in ("uid", "creationTimestamp", "namespace", "generation"): + metadata[key] = current["metadata"].get(key) + metadata["name"] = name + stored.setdefault("apiVersion", current.get("apiVersion")) + stored.setdefault("kind", current.get("kind")) + self.bump(stored) + self.leases[name] = stored + return copy.deepcopy(stored) + + return self._call( + "replace_lease", (name, body), {"timeout_s": timeout_s}, operation + ) + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + def operation() -> None: + self.leases.pop(name, None) + + return self._call("delete_lease", name, {"timeout_s": timeout_s}, operation) + + # ------------------------------------------------- operator simulation + + def set_request_condition( + self, + name: str, + *, + status: str, + reason: str, + message: str = "", + observed_generation: int | None = None, + ) -> None: + """Set the Request's ``Accepted`` condition.""" + + with self._lock: + request = self.requests[name] + generation = request["metadata"].get("generation", 1) + conditions = [ + condition + for condition in request.setdefault("status", {}).get("conditions", []) + if condition.get("type") != "Accepted" + ] + conditions.append( + { + "type": "Accepted", + "status": status, + "reason": reason, + "message": message, + "observedGeneration": ( + generation + if observed_generation is None + else observed_generation + ), + "lastTransitionTime": rfc3339(self.now()), + } + ) + request["status"]["conditions"] = conditions + self.bump(request) + + def accept_request( + self, + name: str, + *, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + mutate_input: Optional[Callable[[dict], None]] = None, + ) -> dict: + """Create the Request's Run (``-``) and mark it Accepted. + + As with the operator, the new Run has no status yet: ``create_job`` + later records its lowered input (kept in ``run_inputs`` until then) + together with ``jobRef``. + """ + + with self._lock: + request = self.requests[name] + generation = int(request["metadata"].get("generation", 1)) + run_name = f"{name}-{generation}" + effective = lower_request_input( + request, + run_name=run_name, + runtime_image=runtime_image, + dynamo_version=dynamo_version, + ) + if mutate_input is not None: + mutate_input(effective) + run: dict[str, Any] = { + "metadata": { + "name": run_name, + "annotations": {REQUEST_GENERATION_ANNOTATION: str(generation)}, + "ownerReferences": [ + { + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRequest", + "name": name, + "uid": request["metadata"]["uid"], + "controller": True, + "blockOwnerDeletion": True, + } + ], + }, + "spec": copy.deepcopy( + {k: v for k, v in request.get("spec", {}).items() if k != "v1beta1"} + ), + "status": {}, + } + self._stamp( + run, kind="DynamoGraphDeploymentRun", api_version="nvidia.com/v1beta2" + ) + self.runs[run_name] = run + self.run_inputs[run_name] = effective + status = request.setdefault("status", {}) + status["observedGeneration"] = generation + status["activeRunRef"] = {"name": run_name} + self.set_request_condition( + name, + status="True", + reason="RunCreated", + message=f"Run {run_name!r} exists for generation {generation}", + ) + return copy.deepcopy(run) + + def create_job(self, run_name: str, *, record_input: bool = True) -> None: + """Create the Run's Job, as the operator's ``reconcileRunStatus`` does. + + One status write records ``status.aisimulate.input`` and ``jobRef`` + with ``Completed=False`` (``JobPending``). ``record_input=False`` + leaves the input out (an operator that does not record it). + """ + + with self._lock: + run = self.runs[run_name] + status = run.setdefault("status", {}) + if record_input: + status["aisimulate"] = { + "input": copy.deepcopy(self.run_inputs[run_name]) + } + status["jobRef"] = {"name": f"{run_name}-job", "uid": str(uuid.uuid4())} + status["phase"] = "Pending" + self._set_completed( + run, "False", "JobPending", "AISimulate Job is waiting to start" + ) + + def start_run(self, run_name: str, *, record_input: bool = True) -> None: + """Start the Run's Job (creating it first, see ``create_job``).""" + + with self._lock: + run = self.runs[run_name] + if not run["status"].get("jobRef"): + self.create_job(run_name, record_input=record_input) + status = run["status"] + status["startTime"] = rfc3339(self.now()) + status["phase"] = "Running" + self._set_completed(run, "False", "JobRunning", "AISimulate Job is running") + + def add_candidate( + self, + run_name: str, + parameters: Mapping[str, Any], + metrics: Mapping[str, Any], + ) -> dict: + with self._lock: + run = self.runs[run_name] + canonical = json.dumps(parameters, sort_keys=True, separators=(",", ":")) + candidate_id = ( + "evaluated-point-" + uuid.uuid5(uuid.NAMESPACE_OID, canonical).hex[:12] + ) + candidate = { + "metadata": { + "name": f"{run_name}-{candidate_id}", + "labels": { + RUN_UID_LABEL: run["metadata"]["uid"], + CANDIDATE_ID_LABEL: candidate_id, + }, + "ownerReferences": [ + { + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRun", + "name": run_name, + "uid": run["metadata"]["uid"], + "controller": True, + } + ], + }, + "spec": {"parameters": json.loads(canonical)}, + "status": {"metrics": json.loads(json.dumps(dict(metrics)))}, + } + self._stamp( + candidate, + kind="DynamoGraphDeploymentCandidate", + api_version="nvidia.com/v1beta2", + ) + self.candidates[candidate["metadata"]["name"]] = candidate + return copy.deepcopy(candidate) + + def complete_run( + self, + run_name: str, + *, + reason: str = "SearchSucceeded", + message: str = "", + candidates: Iterable[tuple[Mapping[str, Any], Mapping[str, Any]]] = (), + ) -> None: + """Publish ``candidates`` (parameters, metrics) and complete the Run.""" + + with self._lock: + run = self.runs[run_name] + refs = [ + { + "name": self.add_candidate(run_name, params, metrics)["metadata"][ + "name" + ] + } + for params, metrics in candidates + ] + status = run["status"] + status["candidateRefs"] = refs + status["phase"] = "Succeeded" if reason == "SearchSucceeded" else "Failed" + status["completionTime"] = rfc3339(self.now()) + self._set_completed(run, "True", reason, message) + owner = (run["metadata"].get("ownerReferences") or [{}])[0] + request = self.requests.get(owner.get("name", "")) + if request is not None: + request_status = request.setdefault("status", {}) + active = request_status.get("activeRunRef") or {} + if active.get("name") == run_name: + request_status.pop("activeRunRef", None) + recent = [ + ref + for ref in request_status.get("recentRunRefs", []) + if ref.get("name") != run_name + ] + request_status["recentRunRefs"] = [{"name": run_name}, *recent] + self.bump(request) + + def _set_completed(self, run: dict, status: str, reason: str, message: str) -> None: + conditions = [ + condition + for condition in run["status"].get("conditions", []) + if condition.get("type") != "Completed" + ] + conditions.append( + { + "type": "Completed", + "status": status, + "reason": reason, + "message": message, + "lastTransitionTime": rfc3339(self.now()), + } + ) + run["status"]["conditions"] = conditions + self.bump(run) + + def run_of(self, request_name: str) -> Optional[dict]: + """The stored Run a Request owns, if any (the newest).""" + + with self._lock: + request = self.requests.get(request_name) + if request is None: + return None + owned = [ + run + for run in self.runs.values() + if any( + owner.get("uid") == request["metadata"]["uid"] + for owner in run["metadata"].get("ownerReferences") or [] + ) + ] + return copy.deepcopy(owned[-1]) if owned else None + + +Evaluator = Callable[[dict, Mapping[str, float]], Optional[Mapping[str, Any]]] + + +class FakeDgdrOperator: + """Drives every Request of a ``FakeRetuneKube`` one phase per ``step()``. + + Phases: accept (a Run with no status, as before its Job exists), start + (the lowered input, jobRef and startTime; the operator records the + input together with jobRef), complete (one Candidate per router point + the input spans, metrics from ``evaluate(request, point_public)``; + ``None`` leaves the point unpublished, as AISimulate drops an + infeasible point). ``hold`` keeps + named Requests in their current phase; ``fail_runs`` maps a Request + name to the ``Completed`` reason its Run ends with. ``mutate_input`` + edits a Run's lowered input and ``mutate_parameters`` a Candidate's + parameters (both get the Request name), to simulate version skew. + ``max_phase`` stops every Request at ``accepted`` or ``started``. + """ + + def __init__( + self, + kube: FakeRetuneKube, + *, + evaluate: Optional[Evaluator] = None, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + mutate_input: Optional[Callable[[str, dict], None]] = None, + mutate_parameters: Optional[Callable[[str, dict], None]] = None, + ) -> None: + self.kube = kube + self.evaluate: Evaluator = evaluate or ( + lambda _request, point: default_metrics(point) + ) + self.runtime_image = runtime_image + self.dynamo_version = dynamo_version + self.mutate_input = mutate_input + self.mutate_parameters = mutate_parameters + self.max_phase: Optional[Literal["accepted", "started"]] = None + self.hold: set[str] = set() + self.fail_runs: dict[str, str] = {} + self.steps = 0 + + def step(self) -> int: + """Advance every Request one phase; returns how many moved.""" + + moved = 0 + with self.kube._lock: + self.steps += 1 + for name in list(self.kube.requests): + if name in self.hold: + continue + run = self.kube.run_of(name) + if run is None: + hook = self.mutate_input + self.kube.accept_request( + name, + runtime_image=self.runtime_image, + dynamo_version=self.dynamo_version, + mutate_input=( + None if hook is None else functools.partial(hook, name) + ), + ) + moved += 1 + continue + run_name = run["metadata"]["name"] + completed = next( + ( + condition + for condition in run["status"].get("conditions", []) + if condition.get("type") == "Completed" + ), + None, + ) + if completed is not None and completed.get("status") == "True": + continue + if self.max_phase == "accepted": + continue + if not run["status"].get("startTime"): + self.kube.start_run(run_name) + moved += 1 + continue + if self.max_phase == "started": + continue + reason = self.fail_runs.get(name) + if reason is not None: + self.kube.complete_run(run_name, reason=reason, message="injected") + moved += 1 + continue + effective = run["status"]["aisimulate"]["input"] + request = self.kube.requests[name] + published = [] + for point in router_points(effective): + metrics = self.evaluate(copy.deepcopy(request), point) + if metrics is not None: + parameters = candidate_parameters(effective, point) + if self.mutate_parameters is not None: + self.mutate_parameters(name, parameters) + published.append((parameters, metrics)) + self.kube.complete_run(run_name, candidates=published) + moved += 1 + return moved + + def run_to_completion(self, max_steps: int = 10) -> None: + for _ in range(max_steps): + if not self.step(): + return diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py new file mode 100644 index 000000000000..7ce2be19ba81 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py @@ -0,0 +1,619 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the retune imports. + +"""Configuration premises of live retune (``retune.executor: dgdr``). + +P1 (Kubernetes, decision latency, explicit timeout within the interval), P2 +(retune-only, router only), P3 (the DGDR router knobs), P5 (a pinned engine +section) and P13 (the largest grid shard fits the sweep deadline), plus the +shard-split and need-seconds helpers the executor shares with validation. +""" + +from __future__ import annotations + +import copy +import math + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from pydantic import ValidationError + +from dynamo.planner.config.planner_config import ( + DGDR_MAX_SHARD_POINTS, + DgdrExecutorConfig, + PlannerConfig, + balanced_shard_parts, + dgdr_need_seconds, + largest_shard_points, +) +from dynamo.planner.retune.domains import expand_router_domain + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +PRESET = { + "tensor": 2, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, +} +SCHEDULER = {"max_batched_tokens": 8192, "max_sequences": 256} + + +def _worker(**overrides): + return { + "parallelism": {"preset": [dict(PRESET)]}, + "scheduler": dict(SCHEDULER), + "kv_cache": {"block_size": 64, "capacity": {"memory_fraction": 0.9}}, + **overrides, + } + + +ENGINE_AGG = { + "mode": "aggregated", + "backend": "vllm", + "backend_version": "0.11.0", + "context_length": 32768, + "workers": {"aggregated": _worker(), "prefill": None, "decode": None}, +} +ENGINE_DISAGG = { + "mode": "disaggregated", + "backend": "vllm", + "backend_version": {"vllm": "0.11.0"}, + "context_length": "max", + "workers": {"prefill": _worker(), "decode": _worker()}, +} +# Design §4.1 defaults: T = 6 h, timeout = T - 20 min. +INTERVAL = 21600.0 +TIMEOUT = 20400.0 +# The default router domain is 3 x 9 x 4 = 108 points; with max_shards 3 the +# largest shard has 36: 900 + 300 + (1 + ceil(35 / 8)) x 1800 = 12000 s. +DEFAULT_NEED = 12000.0 + + +def _dgdr(**overrides): + return { + "claim_name": "request-traces", + "model": "Qwen/Qwen3-32B", + "sku": "h200_sxm", + "engine": copy.deepcopy(ENGINE_AGG), + **overrides, + } + + +def _retune(**overrides): + return { + "interval_seconds": INTERVAL, + "timeout_seconds": TIMEOUT, + "router": "default", + "executor": "dgdr", + "dgdr": _dgdr(), + **overrides, + } + + +def _config(retune=None, **overrides) -> PlannerConfig: + values = { + "namespace": "test-ns", + "environment": "kubernetes", + "mode": "agg", + "enable_load_scaling": False, + "enable_throughput_scaling": False, + "retune": _retune() if retune is None else retune, + **overrides, + } + return PlannerConfig(**values) + + +def _rejects(match: str, retune=None, **overrides) -> None: + with pytest.raises(ValidationError, match=match): + _config(retune, **overrides) + + +def _engine(mode: str = "agg") -> dict: + return copy.deepcopy(ENGINE_AGG if mode == "agg" else ENGINE_DISAGG) + + +# ------------------------------------------------------------------ valid + + +@pytest.mark.parametrize("mode", ["agg", "disagg"]) +def test_live_retune_config_accepts_the_design_example(mode): + retune = _retune(dgdr=_dgdr(engine=_engine(mode))) + config = _config(retune, mode=mode) + + assert config.retune is not None + assert config.retune.executor == "dgdr" + assert config.retune.dgdr is not None + assert config.retune.dgdr.engine == _engine(mode) + # Validation never rewrites the sections it checks. + assert config.retune.router == "default" + + +def test_live_retune_accepts_a_measured_decision_latency_at_scale_one(): + config = _config(_retune(decision_latency_seconds="measured")) + assert config.retune is not None + assert config.retune.decision_latency_seconds == "measured" + + +def test_live_retune_accepts_the_largest_timeout_the_interval_allows(): + config = _config(_retune(timeout_seconds=INTERVAL - 600)) + assert config.retune is not None + + +def test_live_retune_accepts_the_three_dgdr_router_knobs_and_none_load_model(): + router = { + "overlap_score_credit": {"choices": [0.0, 1.0]}, + "prefill_load_scale": {"range": {"min": 0.5, "max": 2.0, "step": 0.5}}, + "temperature": 0.0, + "policy": "kv_router", + "prefill_load_model": {"type": "none"}, + } + assert _config(_retune(router=router)).retune is not None + router["prefill_load_model"] = {"type": {"choices": ["none"]}} + assert _config(_retune(router=router)).retune is not None + + +def test_replay_retune_keeps_its_rules(): + # executor local: none of the live premises apply. + config = _config( + {"interval_seconds": 600, "router": "default"}, + environment="virtual", + ) + assert config.retune is not None + assert config.retune.executor == "local" + + +# --------------------------------------------------------------------- P1 + + +def test_p1_requires_kubernetes(): + _rejects(r"environment='kubernetes' \(P1\)", environment="virtual") + + +@pytest.mark.parametrize("latency", [30.0, 0.5]) +def test_p1_rejects_a_fixed_nonzero_decision_latency(latency): + _rejects( + r"decision_latency_seconds must be 0 or 'measured' \(P1\)", + _retune(decision_latency_seconds=latency), + ) + + +def test_p1_rejects_a_scaled_measured_decision_latency(): + # Valid for RetuneConfig (2 x 10000 <= 21600), rejected by P1 before the + # time budget is checked. + _rejects( + r"decision_latency_scale 1 \(P1\)", + _retune( + decision_latency_seconds="measured", + decision_latency_scale=2.0, + timeout_seconds=10000.0, + ), + ) + + +def test_p1_requires_an_explicit_timeout(): + retune = _retune() + del retune["timeout_seconds"] + _rejects(r"explicit retune.timeout_seconds \(P1\)", retune) + + +def test_p1_rejects_a_timeout_within_ten_minutes_of_the_interval(): + _rejects( + r"timeout_seconds must be <= interval_seconds - 600 = 21000 .*\(P1\)", + _retune(timeout_seconds=INTERVAL - 599), + ) + + +# --------------------------------------------------------------------- P2 + + +@pytest.mark.parametrize( + "scaling", + [ + {"enable_load_scaling": True, "enable_throughput_scaling": False}, + { + "enable_load_scaling": False, + "enable_throughput_scaling": True, + "optimization_target": "sla", + }, + ], +) +def test_p2_requires_retune_only(scaling): + _rejects(r"retune-only mode \(P2\)", **scaling) + + +def test_p2_requires_both_scaling_flags_explicitly(): + # Without both flags written, easy mode keeps load scaling on. + with pytest.raises(ValidationError, match=r"retune-only mode \(P2\)"): + PlannerConfig( + namespace="test-ns", + environment="kubernetes", + mode="agg", + enable_throughput_scaling=False, + retune=_retune(), + ) + + +def test_p2_rejects_a_planner_domain(): + _rejects( + r"retunes the router only \(P2\)", + _retune(planner={"load_sensitivity": {"preset": ["default"]}}), + ) + + +# --------------------------------------------------------------------- P3 + + +@pytest.mark.parametrize( + "knob", ["overlap_score_credit_decay", "decode_active_request_weight"] +) +def test_p3_rejects_retune_extension_knobs(knob): + _rejects( + rf"retune.router \['{knob}'\] cannot be searched with executor 'dgdr' \(P3\)", + _retune(router={knob: {"choices": [0.0, 1.0]}}), + ) + + +@pytest.mark.parametrize( + "load_model", + [{"type": "ais"}, {"type": {"choices": ["none", "ais"]}}], +) +def test_p3_requires_the_none_prefill_load_model(load_model): + _rejects( + r"prefill_load_model.type must be 'none' with executor 'dgdr' \(P3\)", + _retune(router={"temperature": 0.0, "prefill_load_model": load_model}), + ) + + +def test_p3_keeps_the_schema_errors_for_other_policies(): + # RouterRetuneConfig already pins the policy; its error stays first. + _rejects( + r"retune.router.policy is fixed at startup", + _retune(router={"policy": "round_robin"}), + ) + + +# --------------------------------------------------------------------- P5 + + +def _rejects_engine(match: str, engine: dict, mode: str = "agg") -> None: + _rejects(match, _retune(dgdr=_dgdr(engine=engine)), mode=mode) + + +@pytest.mark.parametrize( + ("path", "value", "match"), + [ + ( + ("workers", "aggregated", "scheduler", "max_sequences"), + {"choices": [128, 256]}, + r"engine.workers.aggregated.scheduler.max_sequences is a search " + r"domain \(choices\); .* \(P5\)", + ), + ( + ("workers", "aggregated", "kv_cache", "capacity", "memory_fraction"), + {"range": {"min": 0.8, "max": 0.9}}, + r"memory_fraction is a search domain \(range\)", + ), + ( + ("backend",), + {"choices": ["vllm", "sglang"]}, + r"engine.backend is a search domain", + ), + ( + ("workers", "aggregated", "timing", "systems_paths"), + ["/opt/systems"], + r"engine.workers.aggregated.timing.systems_paths is a list; .* \(P5\)", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "replicas"), + 2, + r"engine.workers.aggregated.parallelism.preset\[0\].replicas must " + r"not be set: the DGD spec provides replicas at each launch \(P5\)", + ), + ( + ("workers", "aggregated", "parallelism", "replicas"), + 2, + r"parallelism.replicas must not be set: the DGD spec", + ), + ( + ("workers", "aggregated", "parallelism", "preset"), + [dict(PRESET), dict(PRESET, tensor=4)], + r"parallelism.preset must be a list of exactly one mapping .* \(P5\)", + ), + ( + ("workers", "aggregated", "parallelism", "preset"), + "default", + r"parallelism.preset must be a list of exactly one mapping", + ), + ( + ("workers", "aggregated", "parallelism", "tensor"), + 2, + r"parallelism sets \['tensor'\] next to preset", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "tensor"), + True, + r"preset\[0\].tensor must be a positive integer \(P5\)", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "expert"), + 1, + r"preset\[0\] must set exactly .*unknown=\['expert'\]", + ), + ( + ("workers", "aggregated", "scheduler", "max_batched_tokens"), + None, + r"scheduler.max_batched_tokens must be a positive integer \(P5\); " + r"omitted, AISimulate searches its default domain", + ), + ( + ("mode",), + "disaggregated", + r"engine.mode must be 'aggregated' for planner mode 'agg' \(P5\)", + ), + ( + ("model",), + "Qwen/Qwen3-32B", + r"engine.model would override the Request's typed field \(P5\); " + r"set retune.dgdr.model", + ), + ( + ("hardware",), + "h200_sxm", + r"set retune.dgdr.sku instead", + ), + ( + ("workers", "decode"), + _worker(), + r"engine.workers.decode is not a worker role of planner mode 'agg'", + ), + ( + ("workers", "encoder"), + {"tensor": 1}, + r"engine.workers.encoder is not a worker role", + ), + ], +) +def test_p5_rejects_an_unpinned_engine(path, value, match): + engine = _engine() + target = engine + for key in path[:-1]: + target = target.setdefault(key, {}) if isinstance(key, str) else target[key] + target[path[-1]] = value + _rejects_engine(match, engine) + + +def test_p5_rejects_a_missing_preset_knob(): + engine = _engine() + del engine["workers"]["aggregated"]["parallelism"]["preset"][0]["moe_expert"] + _rejects_engine(r"missing=\['moe_expert'\]", engine) + + +def test_p5_requires_mode(): + engine = _engine() + del engine["mode"] + _rejects_engine(r"an omitted mode searches both", engine) + + +def test_p5_requires_every_role_of_the_planner_mode(): + engine = _engine("disagg") + del engine["workers"]["decode"] + _rejects_engine( + r"engine.workers.decode is required for planner mode 'disagg' \(P5\)", + engine, + mode="disagg", + ) + + +def test_p5_requires_the_disaggregated_engine_mode_in_disagg(): + engine = _engine("disagg") + engine["mode"] = "aggregated" + _rejects_engine( + r"engine.mode must be 'disaggregated' for planner mode 'disagg'", + engine, + mode="disagg", + ) + + +def test_p5_engine_backend_must_be_the_deployed_backend(): + engine = _engine() + engine["backend"] = "sglang" + # PlannerConfig.backend defaults to vllm; every Run would fail its build. + _rejects_engine( + r"retune.dgdr.engine.backend 'sglang' differs from the deployed " + r"backend 'vllm' \(P5\)", + engine, + ) + config = _config(_retune(dgdr=_dgdr(engine=engine)), backend="sglang") + assert config.retune is not None and config.retune.dgdr is not None + assert config.retune.dgdr.engine["backend"] == "sglang" + _rejects( + r"retune.dgdr.engine.backend 'vllm' differs from the deployed backend " + r"'trtllm' \(P5\)", + _retune(dgdr=_dgdr(engine=_engine())), + backend="trtllm", + ) + + +def test_p5_accepts_an_omitted_engine_backend(): + # The Request fills it with the deployed backend. + engine = _engine() + del engine["backend"] + for backend in ("vllm", "sglang"): + config = _config( + _retune(dgdr=_dgdr(engine=copy.deepcopy(engine))), backend=backend + ) + assert config.retune is not None + + +def test_live_retune_accepts_nested_claim_directories(): + dgdr = _dgdr(segments_dir="retune/segments", windows_dir="retune/windows") + assert _config(_retune(dgdr=dgdr)).retune is not None + + +def test_p5_accepts_unused_roles_absent_or_null(): + engine = _engine() + del engine["workers"]["prefill"] + engine["workers"]["encoder"] = None + assert _config(_retune(dgdr=_dgdr(engine=engine))).retune is not None + + +def test_p5_accepts_the_context_parallel_preset_knobs(): + engine = _engine() + engine["workers"]["aggregated"]["parallelism"]["preset"][0]["decode_context"] = 1 + assert _config(_retune(dgdr=_dgdr(engine=engine))).retune is not None + + +# -------------------------------------------------------------- claim paths + + +@pytest.mark.parametrize( + ("overrides", "match"), + [ + ({"segments_dir": "/trace/segments"}, r"segments_dir must be a relative"), + ({"windows_dir": "../windows"}, r"windows_dir must be a relative"), + ({"windows_dir": "."}, r"windows_dir must be a relative"), + ({"windows_dir": "a/./windows"}, r"windows_dir must be a relative"), + ({"windows_dir": "windows/"}, r"windows_dir must be a relative"), + ({"segments_dir": "a//segments"}, r"segments_dir must be a relative"), + ({"mount_path": "trace"}, r"mount_path must be absolute"), + ({"windows_dir": "segments"}, r"must not contain one another"), + ({"windows_dir": "segments/windows"}, r"must not contain one another"), + ], +) +def test_live_retune_rejects_unsafe_claim_paths(overrides, match): + _rejects(match, _retune(dgdr=_dgdr(**overrides))) + + +# -------------------------------------------------------------------- P13 + + +def test_p13_accepts_a_shard_that_exactly_fits(): + assert _config(_retune(timeout_seconds=DEFAULT_NEED)).retune is not None + + +def test_p13_rejects_a_largest_shard_that_cannot_finish(): + _rejects( + r"largest retune grid shard \(36 points\) needs 12000s: " + r"cut_budget_seconds 900 \+ job_startup_seconds 300 \+ 6 rounds x " + r"candidate_timeout_seconds 1800, more than " + r"retune.timeout_seconds=11999 \(P13\)", + _retune(timeout_seconds=DEFAULT_NEED - 1), + ) + + +def test_p13_counts_the_largest_shard_of_fewer_max_shards(): + # max_shards 2: 3 x 9 x 2 = 54 points -> 1 + ceil(53 / 8) = 8 rounds. + dgdr = _dgdr(run={"max_shards": 2}) + need = 900 + 300 + 8 * 1800 + assert _config(_retune(dgdr=dgdr, timeout_seconds=need)).retune is not None + _rejects(r"\(54 points\)", _retune(dgdr=dgdr, timeout_seconds=need - 1)) + + +def test_p13_rejects_a_grid_that_needs_more_shards(): + _rejects( + r"router grid of 108 points cannot be split into at most " + r"retune.dgdr.run.max_shards=1 Cartesian shards of at most 100 points " + r"\(P13\); the largest would have 108", + _retune(dgdr=_dgdr(run={"max_shards": 1})), + ) + + +def test_p13_uses_the_configured_run_parallelism_and_timeouts(): + dgdr = _dgdr( + cut_budget_seconds=60, + job_startup_seconds=0, + run={"parallelism": 36, "candidate_timeout_seconds": 100}, + ) + # 60 + 0 + (1 + ceil(35 / 36)) x 100 = 260. + assert _config(_retune(dgdr=dgdr, timeout_seconds=260)).retune is not None + _rejects(r"needs 260s", _retune(dgdr=dgdr, timeout_seconds=259)) + + +# --------------------------------------------------------------- helpers + + +@pytest.mark.parametrize( + ("sizes", "max_shards", "parts", "largest"), + [ + ((3, 9, 4), 3, (1, 3, 1), 36), + ((3, 9, 4), 2, (1, 1, 2), 54), + ((3, 9, 4), 1, (1, 1, 1), 108), + # 15 shards of 1 x 2 x 4 beat 12 of 1 x 9 x 1. + ((3, 9, 4), 16, (3, 5, 1), 8), + ((10,), 3, (3,), 4), + ((2, 2), 16, (2, 2), 1), + ((), 3, (), 1), + ], +) +def test_balanced_shard_parts(sizes, max_shards, parts, largest): + assert balanced_shard_parts(sizes, max_shards) == parts + assert largest_shard_points(sizes, parts) == largest + assert math.prod(parts) <= max_shards + + +def test_balanced_shard_parts_minimizes_the_largest_shard_exhaustively(): + for sizes in ((3, 9, 4), (5, 7), (2, 3, 5), (12,), (1, 1, 30)): + for max_shards in range(1, 17): + parts = balanced_shard_parts(sizes, max_shards) + best = min( + largest_shard_points(sizes, (a, b, c)[: len(sizes)]) + for a in range(1, max_shards + 1) + for b in range(1, max_shards + 1) + for c in range(1, max_shards + 1) + if math.prod((a, b, c)[: len(sizes)]) <= max_shards + and all(p <= n for p, n in zip((a, b, c)[: len(sizes)], sizes)) + ) + assert largest_shard_points(sizes, parts) == best + + +@pytest.mark.parametrize( + ("sizes", "max_shards"), + [((3, 0), 3), ((3,), 0)], +) +def test_balanced_shard_parts_rejects_bad_input(sizes, max_shards): + with pytest.raises(ValueError): + balanced_shard_parts(sizes, max_shards) + + +def test_default_router_grid_splits_into_three_shards_of_36(): + retune = _config().retune + assert retune is not None + sizes = [len(values) for values in expand_router_domain(retune).values()] + assert sizes == [3, 9, 4] + parts = balanced_shard_parts(sizes, 3) + assert largest_shard_points(sizes, parts) == 36 <= DGDR_MAX_SHARD_POINTS + + +@pytest.mark.parametrize( + ("points", "needs_cut", "expected"), + [ + (1, True, 900 + 300 + 1 * 1800), + (1, False, 300 + 1 * 1800), + (9, True, 900 + 300 + 2 * 1800), + (10, True, 900 + 300 + 3 * 1800), + (36, False, 300 + 6 * 1800), + ], +) +def test_dgdr_need_seconds(points, needs_cut, expected): + dgdr = DgdrExecutorConfig.model_validate(_dgdr()) + assert dgdr_need_seconds(points, dgdr, needs_cut=needs_cut) == expected + + +def test_dgdr_need_seconds_rejects_an_empty_shard(): + dgdr = DgdrExecutorConfig.model_validate(_dgdr()) + with pytest.raises(ValueError): + dgdr_need_seconds(0, dgdr, needs_cut=True) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py new file mode 100644 index 000000000000..ce64e08551eb --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py @@ -0,0 +1,1889 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the retune imports. + +"""Frontend startup parsing and the spec premises of live retune. + +Covers ``find_frontend`` (v1beta1 and v1alpha1 layouts), ``parse_frontend`` +(the operator's argv defaulting, env merging and ``$(VAR)`` expansion, the +frontend's own argument groups run against an explicit env, a golden +comparison with the real frontend parser, disaggregated maps) and +``evaluate_premises`` (P4, P6-P10) on table-driven DGD fixtures, including +P4 on workers that advertise their own router configuration (compared with +the real worker parser) and the P7 check of ``retune.dgdr.model``. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +import warnings +from typing import Any + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from dynamo.common.configuration.groups.kv_router_args import _KV_ROUTER_FIELDS +from dynamo.frontend.frontend_args import FrontendArgGroup, FrontendConfig +from dynamo.llm import KvRouterConfig +from dynamo.planner.config.planner_config import ( + DgdrExecutorConfig, + PlannerConfig, + RetuneConfig, +) +from dynamo.planner.retune.live import frontend_config as fc +from dynamo.planner.retune.live.contracts import FrontendStartup, PremiseReport + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +DGD = "llm" +NAMESPACE = "team-a" +CLAIM = "request-traces" +MODEL = "Qwen/Qwen3-32B" +TRACE_PATH = "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" +KV_ARGS = ("-m", "dynamo.frontend", "--router-mode", "kv") +DEFAULT_AGG_MAP = { + "decode_active_request_weight": 0.0, + "disk_cache_hit_weight": 0.25, + "host_cache_hit_weight": 0.75, + "overlap_score_credit": 1.0, + "overlap_score_credit_decay": 0.0, + "prefill_load_scale": 1.0, + "router_temperature": 0.0, + "shared_cache_multiplier": 0.0, +} + +PRESET = { + "tensor": 2, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, +} +ENGINE_AGG = { + "mode": "aggregated", + "backend": "vllm", + "backend_version": "0.11.0", + "context_length": 32768, + "workers": { + "aggregated": { + "parallelism": {"preset": [dict(PRESET)]}, + "scheduler": {"max_batched_tokens": 8192, "max_sequences": 256}, + "kv_cache": {"block_size": 64, "capacity": {"memory_fraction": 0.9}}, + }, + "prefill": None, + "decode": None, + }, +} + + +# ------------------------------------------------------------- fixtures + + +def trace_env(**overrides: Any) -> list[dict[str, Any]]: + """The recipe env of design §4.4.1; an override of None drops the entry.""" + + entries: dict[str, dict[str, Any]] = { + "DYN_POD_NAME": { + "name": "DYN_POD_NAME", + "valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}}, + }, + "DYN_REQUEST_TRACE": {"name": "DYN_REQUEST_TRACE", "value": "1"}, + "DYN_REQUEST_TRACE_RECORDS": { + "name": "DYN_REQUEST_TRACE_RECORDS", + "value": "request_end", + }, + "DYN_REQUEST_TRACE_SINKS": {"name": "DYN_REQUEST_TRACE_SINKS", "value": "file"}, + "DYN_REQUEST_TRACE_FILE_FORMAT": { + "name": "DYN_REQUEST_TRACE_FILE_FORMAT", + "value": "jsonl_gz", + }, + "DYN_REQUEST_TRACE_FILE_PATH": { + "name": "DYN_REQUEST_TRACE_FILE_PATH", + "value": TRACE_PATH, + }, + "DYN_REQUEST_TRACE_FILE_ROLL_BYTES": { + "name": "DYN_REQUEST_TRACE_FILE_ROLL_BYTES", + "value": "67108864", + }, + "DYN_REQUEST_TRACE_CAPACITY": { + "name": "DYN_REQUEST_TRACE_CAPACITY", + "value": "8192", + }, + } + for name, entry in overrides.items(): + if entry is None: + entries.pop(name, None) + elif isinstance(entry, dict): + entries[name] = {"name": name, **entry} + else: + entries[name] = {"name": name, "value": entry} + return list(entries.values()) + + +def frontend( + *, + args: tuple[str, ...] | list[str] | None = KV_ARGS, + command: tuple[str, ...] | list[str] | None = ("python3",), + env: list[dict[str, Any]] | None = None, + replicas: int | None = 1, + mounts: list[dict[str, Any]] | None = None, + volumes: list[dict[str, Any]] | None = None, + name: str = "Frontend", + **extra: Any, +) -> dict[str, Any]: + main: dict[str, Any] = { + "name": "main", + "image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0", + "env": trace_env() if env is None else env, + "volumeMounts": ( + [{"name": "traces", "mountPath": "/trace"}] if mounts is None else mounts + ), + } + if command is not None: + main["command"] = list(command) + if args is not None: + main["args"] = list(args) + component: dict[str, Any] = { + "name": name, + "type": "frontend", + "podTemplate": { + "spec": { + "containers": [main], + "volumes": ( + [{"name": "traces", "persistentVolumeClaim": {"claimName": CLAIM}}] + if volumes is None + else volumes + ), + } + }, + **extra, + } + if replicas is not None: + component["replicas"] = replicas + return component + + +def worker( + name: str = "agg", *, type_: str = "worker", model: str = MODEL, **extra: Any +) -> dict[str, Any]: + return { + "name": name, + "type": type_, + "replicas": 2, + "podTemplate": { + "spec": { + "containers": [ + { + "name": "main", + "command": ["python3", "-m", "dynamo.vllm"], + "args": [f"--model={model}", "--tensor-parallel-size=2"], + } + ] + } + }, + **extra, + } + + +def planner(replicas: int = 1) -> dict[str, Any]: + return { + "name": "Planner", + "type": "planner", + "replicas": replicas, + "podTemplate": {"spec": {"containers": [{"name": "main"}]}}, + } + + +def dgd(*components: dict[str, Any], **spec: Any) -> dict[str, Any]: + if not components: + components = (frontend(), worker(), planner()) + return { + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeployment", + "metadata": {"name": DGD, "namespace": NAMESPACE, "uid": "u-1"}, + "spec": {"components": list(components), **spec}, + } + + +def retune_config(router: Any = "default", **dgdr: Any) -> RetuneConfig: + return RetuneConfig( + interval_seconds=21600.0, + timeout_seconds=20400.0, + router=router, + executor="dgdr", + dgdr=DgdrExecutorConfig( + **{ + "claim_name": CLAIM, + "model": MODEL, + "sku": "h200_sxm", + "engine": copy.deepcopy(ENGINE_AGG), + **dgdr, + } + ), + ) + + +def planner_config( + mode: str = "agg", router: Any = "default", **dgdr: Any +) -> PlannerConfig: + # model_construct keeps these tests independent of the live-config + # validators (see test_live_retune_config.py); evaluate_premises reads + # only mode and retune. + return PlannerConfig.model_construct( + mode=mode, retune=retune_config(router, **dgdr) + ) + + +def parse(deployment: dict[str, Any], mode: str = "agg") -> FrontendStartup: + startup = fc.parse_frontend(deployment, planner_mode=mode) + assert isinstance(startup, FrontendStartup), startup + return startup + + +def parse_error(deployment: dict[str, Any], mode: str = "agg") -> str: + result = fc.parse_frontend(deployment, planner_mode=mode) + assert isinstance(result, str), result + return result + + +def evaluate( + deployment: dict[str, Any], + *, + config: PlannerConfig | None = None, + planner_replicas: int | None = None, +) -> PremiseReport: + config = config or planner_config() + result = fc.parse_frontend(deployment, planner_mode=str(config.mode)) + return fc.evaluate_premises( + deployment, + planner_config=config, + startup=result if isinstance(result, FrontendStartup) else None, + parse_error=result if isinstance(result, str) else None, + planner_replicas=planner_replicas, + ) + + +def reasons_matching(report: PremiseReport, prefix: str) -> list[str]: + return [reason for reason in report.reasons if reason.startswith(prefix)] + + +def with_frontend(**kwargs: Any) -> dict[str, Any]: + return dgd(frontend(**kwargs), worker(), planner()) + + +# --------------------------------------------------------- find_frontend + + +def test_find_frontend_v1beta1(): + name, spec = fc.find_frontend(dgd()) + assert name == "Frontend" + assert spec["type"] == "frontend" + + +def test_find_frontend_v1alpha1_services_map(): + deployment = { + "apiVersion": "nvidia.com/v1alpha1", + "kind": "DynamoGraphDeployment", + "metadata": {"name": DGD, "namespace": NAMESPACE}, + "spec": { + "envs": [{"name": "DYN_ROUTER_TEMPERATURE", "value": "0.5"}], + "services": { + "Frontend": { + "componentType": "frontend", + "replicas": 1, + "envs": trace_env(), + "volumeMounts": [{"name": CLAIM, "mountPoint": "/trace"}], + "extraPodSpec": { + "mainContainer": { + "command": ["python3"], + "args": list(KV_ARGS), + } + }, + }, + "VllmWorker": { + "componentType": "worker", + "replicas": 2, + "extraPodSpec": {"mainContainer": {"args": [f"--model {MODEL}"]}}, + }, + "Planner": {"componentType": "planner", "replicas": 1}, + }, + }, + } + name, spec = fc.find_frontend(deployment) + assert name == "Frontend" + assert spec["podTemplate"]["spec"]["containers"][0]["args"] == list(KV_ARGS) + + startup = parse(deployment) + assert startup.router_mode == "kv_router" + assert startup.router_config == {"router_temperature": 0.5} + report = evaluate(deployment) + assert report.spec_ok, report.reasons + assert fc.served_model_name(deployment) == MODEL + + +@pytest.mark.parametrize( + ("components", "match"), + [ + ((worker(), planner()), "no frontend component"), + ( + (frontend(), frontend(name="Frontend2"), worker()), + "2 frontend components", + ), + ((worker(frontendSidecar="sidecar-frontend"),), "frontendSidecar of agg"), + (({"name": "Epp", "type": "epp"}, worker()), "EPP component"), + ], +) +def test_find_frontend_rejects(components, match): + with pytest.raises(fc.FrontendParseError, match=match): + fc.find_frontend(dgd(*components)) + assert parse_error(dgd(*components)).startswith("P7:") + + +# ----------------------------------------------------- argv and parsing + + +def test_parse_recipe_frontend(): + startup = parse(dgd()) + assert startup.router_mode == "kv_router" + assert startup.router_config == {} + assert startup.startup_maps == {"aggregated": DEFAULT_AGG_MAP} + assert startup.frontend_component == "Frontend" + assert startup.replicas == 1 + assert startup.replica_sync is False + assert json.loads(startup.fingerprint_source) == { + "router_mode": "kv_router", + "router_config": {}, + "startup_maps": {"aggregated": DEFAULT_AGG_MAP}, + } + assert startup.fingerprint_source == fc.canonical_json( + json.loads(startup.fingerprint_source) + ) + + +@pytest.mark.parametrize( + ("command", "args", "mode"), + [ + # Operator default: python3 -m dynamo.frontend (round-robin). + (None, None, "round_robin"), + # Authored args keep the default python3 command. + (None, KV_ARGS, "kv_router"), + (("python3",), KV_ARGS, "kv_router"), + (("python3", "-m", "dynamo.frontend"), ("--router-mode", "kv"), "kv_router"), + ( + ("/usr/bin/python3", "-u", "-X", "dev", "-m", "dynamo.frontend"), + ("--router-mode=kv",), + "kv_router", + ), + (("python", "-mdynamo.frontend", "--router-mode", "kv"), None, "kv_router"), + (("python3",), ("-m", "dynamo.frontend", "--router-mode", "random"), "random"), + ( + ("python3",), + ("-m", "dynamo.frontend", "--router-mode", "power-of-two"), + "power_of_two", + ), + ], +) +def test_effective_argv(command, args, mode): + startup = parse(with_frontend(command=command, args=args)) + assert startup.router_mode == mode + + +def test_authored_command_drops_default_args(): + # graph.go: an authored frontend command clears the default args, so the + # command alone must name the module. + error = parse_error(with_frontend(command=("python3",), args=())) + assert "does not run python -m dynamo.frontend" in error + + +@pytest.mark.parametrize( + "command,args", + [ + (("/bin/sh", "-c"), ("python3 -m dynamo.frontend --router-mode kv",)), + (("bash", "-lc", "exec python3 -m dynamo.frontend --router-mode kv"), None), + (("python3", "-c", "import runpy"), None), + ], +) +def test_shell_wrapped_command_is_unparseable(command, args): + error = parse_error(with_frontend(command=command, args=args)) + assert error.startswith("P4:") + assert "shell-wrapped" in error + + +@pytest.mark.parametrize( + "command,args,match", + [ + (("dynamo-frontend",), ("--router-mode", "kv"), "not a python interpreter"), + (("python3",), ("-m", "dynamo.vllm"), "does not run python -m dynamo.frontend"), + (("python3",), ("serve.py",), "does not run python -m dynamo.frontend"), + ], +) +def test_non_frontend_command_is_unparseable(command, args, match): + assert match in parse_error(with_frontend(command=command, args=args)) + + +def test_frontend_roles_are_unparseable(): + error = parse_error(dgd(frontend(roles=[{"name": "leader"}]), worker())) + assert "uses roles" in error + + +def test_env_sets_router_mode_and_knobs(): + env = trace_env( + DYN_ROUTER_MODE="kv", + DYN_ROUTER_TEMPERATURE="0.5", + DYN_ROUTER_KV_OVERLAP_SCORE_CREDIT="0.25", + ) + startup = parse(with_frontend(args=("-m", "dynamo.frontend"), env=env)) + assert startup.router_mode == "kv_router" + assert startup.router_config == { + "router_temperature": 0.5, + "overlap_score_credit": 0.25, + } + assert startup.startup_maps["aggregated"]["router_temperature"] == 0.5 + assert startup.startup_maps["aggregated"]["overlap_score_credit"] == 0.25 + + +def test_cli_wins_over_env_and_container_env_over_graph_env(): + env = trace_env(DYN_ROUTER_TEMPERATURE="0.5", DYN_ROUTER_PREFILL_LOAD_SCALE="2") + deployment = dgd( + frontend(args=(*KV_ARGS, "--router-temperature", "0.2"), env=env), + worker(), + env=[ + {"name": "DYN_ROUTER_PREFILL_LOAD_SCALE", "value": "3"}, + {"name": "DYN_ROUTER_KV_OVERLAP_SCORE_CREDIT", "value": "0.5"}, + ], + ) + assert parse(deployment).router_config == { + "router_temperature": 0.2, + "prefill_load_scale": 2.0, + "overlap_score_credit": 0.5, + } + + +def test_planner_process_env_is_ignored(monkeypatch): + monkeypatch.setenv("DYN_ROUTER_MODE", "kv") + monkeypatch.setenv("DYN_ROUTER_TEMPERATURE", "0.9") + monkeypatch.setenv("DYN_ROUTER_REPLICA_SYNC", "1") + before = dict(os.environ) + startup = parse(with_frontend(args=("-m", "dynamo.frontend"))) + assert startup.router_mode == "round_robin" + assert startup.router_config == {} + assert startup.replica_sync is False + assert dict(os.environ) == before + + +def test_abbreviated_flag_matches_frontend_parser(): + # The frontend parses with argparse abbreviations enabled. + startup = parse(with_frontend(args=(*KV_ARGS, "--router-temp", "0.4"))) + assert startup.router_config == {"router_temperature": 0.4} + + +def test_unknown_frontend_args_are_ignored(): + args = ( + *KV_ARGS, + "--http-port", + "8000", + "--kv-cache-block-size", + "64", + "--trust-remote-code", + "--some-vllm-processor-flag", + "x", + ) + assert parse(with_frontend(args=args)).router_config == {} + + +def test_legacy_and_removed_flags_parse_quietly(caplog): + args = ( + *KV_ARGS, + "--no-kv-events", + "--admission-control", + "none", + "--enforce-disagg", + ) + startup = parse(with_frontend(args=args)) + assert startup.router_config == {"use_kv_events": False} + assert not [r for r in caplog.records if r.levelname == "WARNING"] + + +def test_replica_sync_and_disagg_maps(): + startup = parse( + with_frontend( + args=(*KV_ARGS, "--router-replica-sync", "--router-temperature", "0.3") + ), + mode="disagg", + ) + assert startup.replica_sync is True + assert startup.router_config == { + "router_replica_sync": True, + "router_temperature": 0.3, + } + assert set(startup.startup_maps) == {"prefill", "decode"} + prefill, decode = startup.startup_maps["prefill"], startup.startup_maps["decode"] + assert prefill["overlap_score_credit"] == 1.0 + assert prefill["host_cache_hit_weight"] == 0.75 + # Plain decode routers report zero cache credits. + assert decode["overlap_score_credit"] == 0.0 + assert decode["host_cache_hit_weight"] == 0.0 + assert decode["disk_cache_hit_weight"] == 0.0 + assert decode["router_temperature"] == 0.3 + + +def test_zero_credit_links_cache_weights(): + startup = parse( + with_frontend(args=(*KV_ARGS, "--router-kv-overlap-score-credit", "0")) + ) + agg = startup.startup_maps["aggregated"] + assert agg["overlap_score_credit"] == 0.0 + assert agg["host_cache_hit_weight"] == 0.0 + assert agg["disk_cache_hit_weight"] == 0.0 + + +def test_load_aware_implies_kv_mode(): + startup = parse(with_frontend(args=("-m", "dynamo.frontend", "--load-aware"))) + assert startup.router_mode == "kv_router" + assert startup.router_config["use_kv_events"] is False + assert startup.router_config["overlap_score_credit"] == 0.0 + + +def test_fingerprint_ignores_replicas_but_not_router_config(): + base = parse(dgd(frontend(replicas=1), worker())) + scaled = parse(dgd(frontend(replicas=3), worker())) + tuned = parse( + dgd(frontend(args=(*KV_ARGS, "--router-temperature", "0.1")), worker()) + ) + assert scaled.replicas == 3 + assert base.fingerprint_source == scaled.fingerprint_source + assert base.fingerprint_source != tuned.fingerprint_source + + +def test_omitted_replicas_default_to_one(): + assert parse(dgd(frontend(replicas=None), worker())).replicas == 1 + + +# -------------------------------------------- env references and sources + + +def test_arg_expands_literal_env(): + env = trace_env(TEMP="0.3") + args = (*KV_ARGS, "--router-temperature", "$(TEMP)") + assert parse(with_frontend(args=args, env=env)).router_config == { + "router_temperature": 0.3 + } + + +def test_escaped_reference_stays_literal(): + # $$(TEMP) is the literal "$(TEMP)", which is not a float. + env = trace_env(TEMP="0.3") + args = (*KV_ARGS, "--router-temperature", "$$(TEMP)") + error = parse_error(with_frontend(args=args, env=env)) + assert "cannot parse the frontend args" in error + + +def test_arg_referencing_value_from_is_unparseable(): + env = trace_env(TEMP={"valueFrom": {"configMapKeyRef": {"name": "c", "key": "t"}}}) + args = (*KV_ARGS, "--router-temperature", "$(TEMP)") + error = parse_error(with_frontend(args=args, env=env)) + assert "$(TEMP)" in error and "cannot read" in error + + +def test_opaque_string_router_arg_is_unparseable(): + env = trace_env(MODE={"valueFrom": {"secretKeyRef": {"name": "s", "key": "k"}}}) + args = ("-m", "dynamo.frontend", "--router-tracking-key-id", "id-$(MODE)") + error = parse_error(with_frontend(args=args, env=env)) + assert "router_tracking_key_id" in error + + +def test_arg_that_is_an_opaque_reference_is_unparseable(): + env = trace_env(EXTRA={"valueFrom": {"secretKeyRef": {"name": "s", "key": "k"}}}) + error = parse_error(with_frontend(args=(*KV_ARGS, "$(EXTRA)"), env=env)) + assert "$(EXTRA)" in error + + +def test_opaque_reference_in_a_non_router_arg_is_fine(): + args = (*KV_ARGS, "--metrics-prefix", "fe-$(POD_NAME)") + assert parse(with_frontend(args=args)).router_config == {} + + +OPAQUE_PORT = {"valueFrom": {"configMapKeyRef": {"name": "c", "key": "port"}}} + + +@pytest.mark.parametrize( + "args", + [ + # Typed non-router flags keep the unreadable value unconverted. + (*KV_ARGS, "--http-port", "$(HTTP_PORT)"), + (*KV_ARGS, "--http-port=$(HTTP_PORT)"), + (*KV_ARGS, "--grpc-metrics-port", "$(POD_UID)"), + # A non-router choices flag. + (*KV_ARGS, "--dyn-chat-processor", "$(HTTP_PORT)"), + # The value of a flag the frontend hands to its chat processor. + ( + *KV_ARGS, + "--dyn-chat-processor", + "vllm", + "--tool-call-parser", + "$(HTTP_PORT)", + ), + (*KV_ARGS, "--dyn-chat-processor", "vllm", "--tool-call-parser=$(HTTP_PORT)"), + ], +) +def test_opaque_reference_in_a_typed_non_router_arg_is_fine(args): + startup = parse(with_frontend(args=args, env=trace_env(HTTP_PORT=OPAQUE_PORT))) + assert startup.router_mode == "kv_router" + assert startup.router_config == {} + + +@pytest.mark.parametrize( + ("args", "match"), + [ + # Router flags whose value is unreadable. + ( + ("-m", "dynamo.frontend", "--router-mode", "$(HTTP_PORT)"), + "router arg references $(HTTP_PORT)", + ), + ( + (*KV_ARGS, "--router-temp", "$(HTTP_PORT)"), + "router arg references $(HTTP_PORT)", + ), + ( + (*KV_ARGS, "--router-ttl-secs=$(HTTP_PORT)"), + "router arg references $(HTTP_PORT)", + ), + # Left over where a flag could stand. + ( + (*KV_ARGS, "--router-kv-events", "$(HTTP_PORT)"), + "could set a router flag", + ), + ((*KV_ARGS, "--$(HTTP_PORT)"), "could set a router flag"), + ((*KV_ARGS, "--$(HTTP_PORT)=1"), "could set a router flag"), + ( + (*KV_ARGS, "--tool-call-parser", "--$(HTTP_PORT)", "$(HTTP_PORT)"), + "could set a router flag", + ), + ], +) +def test_opaque_reference_that_could_set_a_router_flag_is_unparseable(args, match): + error = parse_error(with_frontend(args=args, env=trace_env(HTTP_PORT=OPAQUE_PORT))) + assert error.startswith("P4:") + assert match in error + + +def test_router_env_from_value_from_is_unparseable(): + env = trace_env( + DYN_ROUTER_TEMPERATURE={ + "valueFrom": {"configMapKeyRef": {"name": "c", "key": "t"}} + } + ) + error = parse_error(with_frontend(env=env)) + assert "DYN_ROUTER_TEMPERATURE" in error and "valueFrom" in error + + +def test_router_env_referencing_value_from_is_unparseable(): + env = trace_env( + A_TEMP={"valueFrom": {"configMapKeyRef": {"name": "c", "key": "t"}}}, + DYN_ROUTER_TEMPERATURE="$(A_TEMP)", + ) + assert "DYN_ROUTER_TEMPERATURE" in parse_error(with_frontend(env=env)) + + +def test_router_env_references_earlier_literal(): + env = trace_env(A_TEMP="0.7", DYN_ROUTER_TEMPERATURE="$(A_TEMP)") + assert parse(with_frontend(env=env)).router_config == {"router_temperature": 0.7} + + +def test_non_router_value_from_is_fine(): + env = trace_env( + HF_TOKEN={"valueFrom": {"secretKeyRef": {"name": "hf", "key": "t"}}} + ) + assert parse(with_frontend(env=env)).router_mode == "kv_router" + + +@pytest.mark.parametrize( + ("source", "warned"), + [ + ({"secretRef": {"name": "hf-token-secret"}}, True), + ({"configMapRef": {"name": "router"}, "prefix": "DYN_"}, True), + ({"configMapRef": {"name": "router"}, "prefix": "D"}, True), + ({"secretRef": {"name": "hf-token-secret"}, "prefix": "HF_"}, False), + ], +) +def test_env_from_is_parsed_and_warned_when_it_may_set_router_env(source, warned): + deployment = with_frontend() + deployment["spec"]["components"][0]["podTemplate"]["spec"]["containers"][0][ + "envFrom" + ] = [source] + assert isinstance( + fc.parse_frontend(deployment, planner_mode="agg"), FrontendStartup + ) + report = evaluate(deployment) + envfrom_warnings = [w for w in report.warnings if "envFrom" in w] + assert bool(envfrom_warnings) == warned + assert not reasons_matching(report, "P4") + + +def test_router_policy_config_is_unparseable(): + error = parse_error( + with_frontend(args=(*KV_ARGS, "--router-policy-config", "/etc/policy.yaml")) + ) + assert "--router-policy-config" in error + + +@pytest.mark.parametrize( + "args,env,match", + [ + ((*KV_ARGS, "--router-kv-overlap-score-credit", "-1"), {}, "invalid"), + ((*KV_ARGS, "--router-temperature", "hot"), {}, "cannot parse"), + ((*KV_ARGS,), {"DYN_ROUTER_TEMPERATURE": "hot"}, "invalid frontend env"), + ( + (*KV_ARGS, "--router-approximate-cache-policy", "lru"), + {}, + "invalid", + ), + ], +) +def test_invalid_router_config_is_unparseable(args, env, match): + error = parse_error(with_frontend(args=args, env=trace_env(**env))) + assert error.startswith("P4:") + assert match in error + + +# ------------------------------------------------------ parser fidelity + + +GOLDEN_CASES = [ + ((*KV_ARGS,), {}), + ((*KV_ARGS, "--router-temperature", "0.5", "--router-replica-sync"), {}), + ( + ("-m", "dynamo.frontend"), + {"DYN_ROUTER_MODE": "kv", "DYN_ROUTER_USE_KV_EVENTS": "0"}, + ), + ((*KV_ARGS, "--no-kv-events", "--router-ttl-secs", "30"), {}), + ( + (*KV_ARGS, "--router-kv-overlap-score-credit", "0"), + {"DYN_ROUTER_TEMPERATURE": "0.1"}, + ), + (("-m", "dynamo.frontend", "--load-aware"), {}), + ((*KV_ARGS,), {"DYN_OVERLAP_SCORE_WEIGHT": "2"}), + ( + (*KV_ARGS,), + {"DYN_ROUTER_KV_OVERLAP_SCORE_WEIGHT": "0", "DYN_OVERLAP_SCORE_WEIGHT": "2"}, + ), + ((*KV_ARGS, "--router-temp", "0.25", "--router-event-threads", "8"), {}), + ( + (*KV_ARGS, "--router-tracking-hash", "keyed-xxh3-v1"), + {"DYN_ROUTER_TRACKING_KEY_FILE": "/k", "DYN_ROUTER_TRACKING_KEY_ID": "e1"}, + ), +] + + +@pytest.mark.parametrize(("args", "env"), GOLDEN_CASES) +def test_matches_real_frontend_parser(monkeypatch, args, env): + """The frontend's own parser, run with its env in os.environ, agrees.""" + + for name in list(os.environ): + if name.startswith("DYN_"): + monkeypatch.delenv(name) + for name, value in env.items(): + monkeypatch.setenv(name, value) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + parser = argparse.ArgumentParser() + FrontendArgGroup().add_arguments(parser) + namespace, _ = parser.parse_known_args(list(args[2:])) + config = FrontendConfig.from_cli_args(namespace) + if config.load_aware: + config.router_mode = "kv" + config.apply_load_aware_preset() + config.apply_conditional_disagg_config() + expected = json.loads( + KvRouterConfig(**{f: getattr(config, f) for f in _KV_ROUTER_FIELDS}).to_json() + ) + + for name in env: + monkeypatch.delenv(name) + startup = parse(with_frontend(args=args, env=trace_env(**env))) + actual = json.loads( + KvRouterConfig.from_json(json.dumps(startup.router_config)).to_json() + ) + assert actual == expected + assert startup.router_mode == fc._router_mode(config.router_mode) + + +def test_router_fields_are_all_covered(): + """Every frontend router field is in the JSON or checked on the flags.""" + + serialized = set( + json.loads( + KvRouterConfig( + router_tracking_hash="keyed-xxh3-v1", + router_tracking_key_file="/k", + router_tracking_key_id="e", + decode_active_request_weight=1.0, + enable_session_prefix_index=True, + shared_cache_multiplier=0.5, + conditional_disagg_enabled=True, + conditional_disagg_policy="isl_or_load", + conditional_disagg_eff_isl_threshold=1, + conditional_disagg_eff_isl_ratio_threshold=0.5, + conditional_disagg_prefill_busy_threshold=0.5, + conditional_disagg_decode_busy_threshold=0.5, + ).to_json() + ) + ) + covered = serialized | set(fc._SERDE_SKIPPED_DEFAULTS) | {"overlap_score_weight"} + assert set(_KV_ROUTER_FIELDS) <= covered + + +# --------------------------------------------------------- P4 premises + + +def test_recipe_dgd_satisfies_every_premise(): + report = evaluate(dgd()) + assert report.p4_ok and report.spec_ok, report.reasons + assert report.reasons == () + assert report.startup is not None + assert not [w for w in report.warnings if w.startswith("P6")] + + +def test_validated_planner_config_satisfies_every_premise(): + config = PlannerConfig( + namespace="test-ns", + environment="kubernetes", + mode="agg", + enable_load_scaling=False, + enable_throughput_scaling=False, + retune=retune_config().model_dump(), + ) + report = evaluate(dgd(), config=config) + assert report.spec_ok, report.reasons + + +def test_parse_error_fails_p4_only(): + deployment = with_frontend( + command=("/bin/sh", "-c"), args=("python3 -m dynamo.frontend",) + ) + report = evaluate(deployment) + assert not report.p4_ok and not report.spec_ok + assert len(report.reasons) == 1 and "shell-wrapped" in report.reasons[0] + assert report.startup is None + + +def test_caller_parse_error_is_reported(): + report = fc.evaluate_premises( + dgd(), + planner_config=planner_config(), + startup=None, + parse_error="P4: injected", + planner_replicas=None, + ) + assert not report.p4_ok + assert report.reasons == ("P4: injected",) + + +@pytest.mark.parametrize( + ("args", "env", "match"), + [ + (("-m", "dynamo.frontend"), {}, "routes with 'round_robin'"), + ( + ("-m", "dynamo.frontend", "--router-mode", "random"), + {}, + "routes with 'random'", + ), + ((*KV_ARGS, "--no-router-kv-events"), {}, "use_kv_events=False"), + ( + (*KV_ARGS, "--router-host-cache-hit-weight", "0.5"), + {}, + "host_cache_hit_weight=0.5", + ), + ( + (*KV_ARGS, "--router-queue-threshold", "0.5"), + {}, + "router_queue_threshold=0.5", + ), + ( + (*KV_ARGS, "--router-kv-overlap-score-credit-decay", "1"), + {}, + "overlap_score_credit_decay", + ), + ( + (*KV_ARGS, "--router-decode-active-request-weight", "1"), + {}, + "decode_active_request_weight", + ), + ( + (*KV_ARGS, "--router-prefill-load-model", "ais"), + {}, + "router_prefill_load_model is 'aic'", + ), + ( + (*KV_ARGS, "--router-conditional-disagg"), + {}, + "conditional disaggregation", + ), + ( + ( + *KV_ARGS, + "--no-router-kv-events", + "--router-approximate-cache-policy", + "lru", + ), + {}, + "router_approximate_cache_policy='lru'", + ), + ((*KV_ARGS, "--router-prefill-policy", "default"), {}, "router_prefill_policy"), + ((*KV_ARGS, "--router-decode-policy", "default"), {}, "router_decode_policy"), + ((*KV_ARGS,), {"DYN_OVERLAP_SCORE_WEIGHT": "2"}, "overlap_score_weight"), + ( + ("-m", "dynamo.frontend"), + {"DYN_ROUTER_LOAD_AWARE": "1"}, + "load-aware preset", + ), + ( + (*KV_ARGS,), + {"DYN_ROUTER_WORKER_SELECTION_POLICY": "mine"}, + "worker-selection", + ), + ( + (*KV_ARGS,), + {"DYN_ROUTER_POSITIONAL_SEARCH_MODE": "x"}, + "DYN_ROUTER_POSITIONAL_SEARCH_MODE", + ), + ( + (*KV_ARGS,), + {"DYN_ROUTER_SESSION_AFFINITY_TTL_SECS": "60"}, + "session_affinity_ttl_secs=60", + ), + ( + (*KV_ARGS, "--active-decode-blocks-threshold", "0.9"), + {}, + "active_decode_blocks_threshold", + ), + ( + (*KV_ARGS, "--active-prefill-tokens-threshold", "100"), + {}, + "active_prefill_tokens_threshold", + ), + ], +) +def test_p4_rejects(args, env, match): + report = evaluate(with_frontend(args=args, env=trace_env(**env))) + assert not report.p4_ok and not report.spec_ok + assert any( + match in reason for reason in reasons_matching(report, "P4:") + ), report.reasons + + +@pytest.mark.parametrize( + ("args", "env"), + [ + # Searched knobs may start anywhere. + ( + ( + *KV_ARGS, + "--router-temperature", + "0.5", + "--router-prefill-load-scale", + "2", + ), + {}, + ), + ((*KV_ARGS, "--router-kv-overlap-score-credit", "0.5"), {}), + # Tolerated fields. + ((*KV_ARGS, "--router-event-threads", "8", "--router-replica-sync"), {}), + ((*KV_ARGS, "--enable-session-prefix-index", "--serve-indexer"), {}), + ( + (*KV_ARGS, "--router-tracking-hash", "keyed-xxh3-v1"), + {"DYN_ROUTER_TRACKING_KEY_FILE": "/k", "DYN_ROUTER_TRACKING_KEY_ID": "e"}, + ), + # Frontend-only and transport settings. + ( + (*KV_ARGS,), + { + "DYN_ROUTER_MIN_INITIAL_WORKERS": "2", + "DYN_ROUTER_ZMQ_ENDPOINTS_PER_SUB": "4", + }, + ), + ((*KV_ARGS, "--router-session-affinity-mode", "soft"), {}), + ], +) +def test_p4_accepts(args, env): + report = evaluate(with_frontend(args=args, env=trace_env(**env))) + assert report.p4_ok, report.reasons + + +def test_p4_unsearched_knob_when_router_domain_is_pinned(): + # A scalar still searches (pins) the knob: it stays in the domain. + config = planner_config( + router={ + "overlap_score_credit": 1.0, + "prefill_load_scale": 1.0, + "temperature": 0.0, + } + ) + report = evaluate( + with_frontend(args=(*KV_ARGS, "--router-temperature", "0.5")), config=config + ) + assert report.p4_ok, report.reasons + + +def test_p4_without_router_search_requires_all_defaults(): + config = planner_config(router=None) + ok = evaluate(dgd(), config=config) + assert ok.p4_ok, ok.reasons + report = evaluate( + with_frontend(args=(*KV_ARGS, "--router-temperature", "0.5")), config=config + ) + assert any("router_temperature=0.5" in r for r in report.reasons) + assert "(none)" in report.reasons[0] + + +def test_p4_zero_startup_credit_rejects_positive_candidates(): + zero = (*KV_ARGS, "--router-kv-overlap-score-credit", "0") + # The default domain collapses the omitted credit to [0.0]. + assert evaluate(with_frontend(args=zero)).p4_ok + config = planner_config( + router={"overlap_score_credit": {"choices": [0.0, 0.5]}, "temperature": 0.0} + ) + report = evaluate(with_frontend(args=zero), config=config) + assert not report.p4_ok + assert "overlap_score_credit > 0" in report.reasons[0] + + +def test_p4_disagg_decode_credit_zero_is_not_a_zero_credit_startup(): + # Plain decode routers always report credit 0; only prefill counts. + config = planner_config(mode="disagg") + report = evaluate( + dgd( + frontend(), + worker("prefill", type_="prefill"), + worker("decode", type_="decode"), + planner(), + ), + config=config, + ) + assert report.p4_ok, report.reasons + assert report.startup is not None + assert set(report.startup.startup_maps) == {"prefill", "decode"} + + +def test_p4_written_load_model_must_match_startup(): + config = planner_config(router={"prefill_load_model": {"type": "ais"}}) + report = evaluate(dgd(), config=config) + assert not report.p4_ok + assert "prefill_load_model" in report.reasons[0] + + +@pytest.mark.parametrize( + ("declared", "match"), + [ + ({}, None), + ({"router_temperature": 0.5}, None), + ({"router_temperature": 0.5, "use_kv_events": True}, None), + ({"router_temperature": 0.4}, "router_temperature: declared 0.4, frontend 0.5"), + ({}, "router_temperature: declared 0.0"), + ({"not_a_field": 1}, "not a KvRouterConfig JSON"), + ], +) +def test_startup_router_config_must_match(declared, match): + config = planner_config(startup_router_config=declared) + args = (*KV_ARGS, "--router-temperature", "0.5") + if declared == {} and match is None: + args = KV_ARGS + report = evaluate(with_frontend(args=args), config=config) + if match is None: + assert report.p4_ok, report.reasons + else: + assert not report.p4_ok + assert any(match in r for r in report.reasons), report.reasons + + +# ------------------------------------------- P4: worker router advertisement + + +SECRET = {"valueFrom": {"secretKeyRef": {"name": "s", "key": "k"}}} + + +def router_worker( + *args: str, + env: list[dict[str, Any]] | None = None, + name: str = "agg", + type_: str = "worker", +) -> dict[str, Any]: + component = worker(name, type_=type_) + main = component["podTemplate"]["spec"]["containers"][0] + main["args"] = [*main["args"], *args] + if env is not None: + main["env"] = env + return component + + +def shell_worker(script: str) -> dict[str, Any]: + component = worker() + component["podTemplate"]["spec"]["containers"][0].update( + command=["/bin/sh", "-c"], args=[script] + ) + return component + + +def minimax_prefill_worker(*router_args: str) -> dict[str, Any]: + """The PrefillWorker of recipes/minimax-m3/vllm/disagg-gb200-agentic.""" + + return { + "name": "PrefillWorker", + "type": "prefill", + "replicas": 3, + "podTemplate": { + "spec": { + "containers": [ + { + "name": "main", + "command": ["python3"], + "args": [ + "-m", + "dynamo.vllm", + *router_args, + "--model", + "$(MODEL_NAME)", + "--served-model-name", + "$(SERVED_MODEL_NAME)", + "--tensor-parallel-size", + "4", + "--speculative-config", + "$(SPECULATIVE_CONFIG)", + "--disaggregation-mode", + "prefill", + "--kv-transfer-config", + "$(KV_TRANSFER_CONFIG)", + "--kv-events-config", + '{"enable_kv_cache_events":true,"publisher":"zmq"}', + ], + "env": [ + { + "name": "KV_TRANSFER_CONFIG", + "value": '{"kv_connector":"NixlConnector",' + '"kv_role":"kv_both"}', + }, + { + "name": "SPECULATIVE_CONFIG", + "valueFrom": { + "configMapKeyRef": { + "name": "minimax-m3-config", + "key": "speculative-config", + } + }, + }, + {"name": "MODEL_NAME", "value": "nvidia/MiniMax-M3-NVFP4"}, + { + "name": "SERVED_MODEL_NAME", + "value": "nvidia/MiniMax-M3-NVFP4", + }, + ], + } + ] + } + }, + } + + +@pytest.mark.parametrize( + ("args", "env", "match"), + [ + (("--router-mode", "kv"), None, "advertises its own router configuration"), + (("--router-mode=round-robin",), None, "router mode 'round-robin'"), + (("--router-m", "kv", "--no-router-kv-events"), None, "router mode 'kv'"), + ((), [{"name": "DYN_ROUTER_MODE", "value": "kv"}], "router mode 'kv'"), + ((), [{"name": "DYN_ROUTER_MODE", "value": ""}], "router mode ''"), + ( + (), + [{"name": "DYN_ROUTER_MODE", **SECRET}], + "DYN_ROUTER_MODE through valueFrom", + ), + ( + (), + [ + {"name": "A_MODE", **SECRET}, + {"name": "DYN_ROUTER_MODE", "value": "$(A_MODE)"}, + ], + "DYN_ROUTER_MODE through valueFrom", + ), + ( + ("--router-mode", "$(A_MODE)"), + [{"name": "A_MODE", **SECRET}], + "router flag that references $(A_MODE)", + ), + ( + ("--router-mode", "$(A_MODE)"), + [{"name": "A_MODE", "value": "kv"}], + "router mode 'kv'", + ), + (("--router-mode", "fastest"), None, "cannot parse"), + ( + (), + [{"name": "DYN_ROUTER_TEMPERATURE", "value": "hot"}], + "invalid router env", + ), + ], +) +def test_p4_rejects_a_worker_that_advertises_a_router_config(args, env, match): + report = evaluate(dgd(frontend(), router_worker(*args, env=env), planner())) + assert not report.p4_ok and not report.spec_ok + assert report.startup is not None # the frontend itself parses + rejected = reasons_matching(report, "P4: worker component agg ") + assert any(match in reason for reason in rejected), report.reasons + assert all("on the frontend only" in reason for reason in rejected) + + +@pytest.mark.parametrize( + ("args", "env"), + [ + # Router flags and env without a mode advertise nothing. + (("--no-router-kv-events", "--router-temperature", "0.5"), None), + ((), [{"name": "DYN_ROUTER_TEMPERATURE", "value": "0.5"}]), + ((), [{"name": "DYN_ROUTER_MODE_HINT", "value": "kv"}]), + # Unreadable env in a non-mode argument. + (("--router-temperature", "$(A_SECRET)"), [{"name": "A_SECRET", **SECRET}]), + ( + ("--speculative-config", "$(A_SECRET)", "$(A_SECRET)"), + [{"name": "A_SECRET", **SECRET}], + ), + ], +) +def test_p4_accepts_workers_that_advertise_nothing(args, env): + report = evaluate(dgd(frontend(), router_worker(*args, env=env), planner())) + assert report.p4_ok and report.spec_ok, report.reasons + + +def test_p4_rejects_graph_env_router_mode_reaching_the_workers(): + # recipes/solar-open2-250b: a graph-level DYN_ROUTER_MODE reaches the + # frontend and every worker. + deployment = dgd( + frontend(args=("-m", "dynamo.frontend")), + worker("prefill", type_="prefill"), + worker("decode", type_="decode"), + planner(), + env=[{"name": "DYN_ROUTER_MODE", "value": "kv"}], + ) + report = evaluate(deployment, config=planner_config("disagg")) + assert report.startup is not None and report.startup.router_mode == "kv_router" + assert not report.p4_ok + assert [r.split(" advertises")[0] for r in reasons_matching(report, "P4:")] == [ + "P4: worker component prefill", + "P4: worker component decode", + ] + + +def test_p4_checks_the_real_recipe_prefill_worker(): + config = planner_config("disagg", model="nvidia/MiniMax-M3-NVFP4") + advertising = dgd( + frontend(), + minimax_prefill_worker( + "--router-mode", "kv", "--no-router-track-output-blocks" + ), + worker("DecodeWorker", type_="decode", model="nvidia/MiniMax-M3-NVFP4"), + planner(), + ) + report = evaluate(advertising, config=config) + assert not report.p4_ok + assert reasons_matching( + report, + "P4: worker component PrefillWorker advertises its own router " + "configuration (router mode 'kv'", + ) + + # Without the router flags the worker inherits the frontend's routing; + # its unreadable --speculative-config value does not matter, and its + # $(MODEL_NAME) / $(SERVED_MODEL_NAME) expand. + inheriting = dgd( + frontend(), + minimax_prefill_worker(), + worker("DecodeWorker", type_="decode", model="nvidia/MiniMax-M3-NVFP4"), + planner(), + ) + report = evaluate(inheriting, config=config) + assert report.p4_ok and report.spec_ok, report.reasons + assert not [w for w in report.warnings if w.startswith("P7:")], report.warnings + assert fc.served_model_name(inheriting) == "nvidia/MiniMax-M3-NVFP4" + + +@pytest.mark.parametrize( + ("script", "match"), + [ + (f"python3 -m dynamo.vllm --model {MODEL} --router-mode kv", "router mode"), + ( + f"export DYN_ROUTER_MODE=kv && exec python3 -m dynamo.vllm --model {MODEL}", + "sets DYN_ROUTER_MODE in its shell command", + ), + ( + f"DYN_ROUTER_MODE=kv python3 -m dynamo.vllm --model {MODEL}", + "sets DYN_ROUTER_MODE in its shell command", + ), + ('python3 -m dynamo.vllm --router-mode "$MODE"', "cannot parse"), + (f"python3 -m dynamo.vllm --model {MODEL} --no-router-kv-events", None), + ], +) +def test_p4_shell_wrapped_worker(script, match): + report = evaluate(dgd(frontend(), shell_worker(script), planner())) + rejected = reasons_matching(report, "P4: worker component agg ") + if match is None: + assert report.p4_ok, report.reasons + else: + assert not report.p4_ok + assert any(match in reason for reason in rejected), report.reasons + + +def test_p4_rejects_a_v1alpha1_worker_router_mode(): + deployment = { + "apiVersion": "nvidia.com/v1alpha1", + "kind": "DynamoGraphDeployment", + "metadata": {"name": DGD, "namespace": NAMESPACE}, + "spec": { + "services": { + "Frontend": { + "componentType": "frontend", + "envs": trace_env(), + "volumeMounts": [{"name": CLAIM, "mountPoint": "/trace"}], + "extraPodSpec": { + "mainContainer": {"command": ["python3"], "args": list(KV_ARGS)} + }, + }, + "VllmWorker": { + "componentType": "worker", + "envs": [{"name": "DYN_ROUTER_MODE", "value": "kv"}], + "extraPodSpec": { + "mainContainer": { + "command": ["python3", "-m", "dynamo.vllm"], + "args": ["--model", MODEL], + } + }, + }, + "Planner": {"componentType": "planner", "replicas": 1}, + }, + }, + } + report = evaluate(deployment) + assert not report.p4_ok + assert reasons_matching(report, "P4: worker component VllmWorker advertises") + + +def test_worker_check_ignores_the_planner_env(monkeypatch): + monkeypatch.setenv("DYN_ROUTER_MODE", "kv") + fc._worker_router_cached.cache_clear() + try: + report = evaluate(dgd()) + finally: + fc._worker_router_cached.cache_clear() + assert report.p4_ok, report.reasons + assert os.environ["DYN_ROUTER_MODE"] == "kv" + + +WORKER_GOLDEN_CASES = [ + ((), {}), + (("--router-mode", "kv"), {}), + (("--router-m", "random"), {}), + (("--router-mode=power-of-two",), {}), + ((), {"DYN_ROUTER_MODE": "kv"}), + (("--router-mode", "least-loaded"), {"DYN_ROUTER_MODE": "kv"}), + (("--no-router-kv-events", "--router-temperature", "0.5"), {}), + ((), {"DYN_ROUTER_TEMPERATURE": "0.5"}), +] + + +@pytest.mark.parametrize(("args", "env"), WORKER_GOLDEN_CASES) +def test_worker_router_mode_matches_the_real_worker_parser(monkeypatch, args, env): + """Backends parse their left-over argv with parse_worker_router_config.""" + + from dynamo.common.configuration.groups.router_args import ( + parse_worker_router_config, + ) + + for name in list(os.environ): + if name.startswith("DYN_"): + monkeypatch.delenv(name) + for name, value in env.items(): + monkeypatch.setenv(name, value) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + config, _ = parse_worker_router_config([f"--model={MODEL}", *args]) + for name in env: + monkeypatch.delenv(name) + + worker_env = [{"name": name, "value": value} for name, value in env.items()] + report = evaluate(dgd(frontend(), router_worker(*args, env=worker_env), planner())) + rejected = reasons_matching(report, "P4: worker component agg ") + if config.router_mode is None: + assert rejected == [], rejected + else: + assert any( + f"router mode {config.router_mode!r}" in reason for reason in rejected + ), rejected + + +def test_worker_env_from_is_warned_once(): + deployment = dgd(frontend(), worker("a"), worker("b"), planner()) + for component in deployment["spec"]["components"]: + component["podTemplate"]["spec"]["containers"][0]["envFrom"] = [ + {"secretRef": {"name": "hf-token-secret"}} + ] + report = evaluate(deployment) + assert report.p4_ok and report.spec_ok, report.reasons + env_from = sorted(w for w in report.warnings if "envFrom" in w) + assert env_from == [ + "P4: the frontend container uses envFrom secretRef 'hf-token-secret'; " + "the planner cannot read it and assumes it sets no router env (P14 " + "still checks the routers' policy parameters)", + "P4: worker components a, b use envFrom; the planner cannot read it and " + "assumes it sets no router env (a DYN_ROUTER_MODE there would make the " + "worker advertise its own router configuration)", + ] + + +# ---------------------------------------------------------- P6-P10 + + +def test_p6_warns_on_unknown_model(): + report = evaluate(dgd(), config=planner_config(model="acme/private-model")) + assert report.spec_ok + assert any(w.startswith("P6:") for w in report.warnings) + + +def test_p7_served_models(): + two = dgd(frontend(), worker("a"), worker("b", model="meta/llama"), planner()) + report = evaluate(two) + assert reasons_matching(report, "P7:") == [ + f"P7: the DGD serves 2 models ({MODEL}, meta/llama); live retune needs one" + ] + assert fc.served_model_name(two) is None + assert fc.served_model_name(dgd()) == MODEL + + +def test_p7_served_model_prefers_current_status_and_warns_when_unknown(): + unknown = worker("b") + unknown["podTemplate"]["spec"]["containers"][0]["args"] = ["--model", "$(MODEL)"] + deployment = dgd(frontend(), worker("a"), unknown, planner()) + report = evaluate(deployment) + assert report.spec_ok + assert any("served model of b" in w for w in report.warnings) + + deployment["metadata"]["generation"] = 3 + deployment["status"] = { + "observedGeneration": 3, + "components": {"b": {"servedModelName": "other/model"}}, + } + assert fc.served_model_names(deployment) == ({"a": MODEL, "b": "other/model"}, ()) + deployment["status"]["observedGeneration"] = 2 + assert fc.served_model_names(deployment)[1] == ("b",) + + +def test_p7_shell_wrapped_worker_model(): + shell = worker("a") + shell["podTemplate"]["spec"]["containers"][0].update( + command=["/bin/sh", "-c"], + args=[ + f"python3 -m dynamo.sglang --model-path {MODEL} --served-model-name qwen" + ], + ) + assert fc.served_model_names(dgd(frontend(), shell)) == ({"a": "qwen"}, ()) + + +def p7_model_warnings(report: PremiseReport) -> list[str]: + return [w for w in report.warnings if w.startswith("P7: retune.dgdr.model")] + + +def test_p7_warns_when_dgdr_model_is_not_what_the_workers_serve(): + deployment = dgd(frontend(), worker(model="meta-llama/Llama-3.1-8B"), planner()) + report = evaluate(deployment) + assert report.p4_ok and report.spec_ok, report.reasons + assert p7_model_warnings(report) == [ + f"P7: retune.dgdr.model {MODEL!r} is not among the workers' " + "--model/--model-path/--served-model-name values " + "(meta-llama/Llama-3.1-8B); every DGDR Run simulates retune.dgdr.model, " + "so make sure it is the model the DGD serves" + ] + assert not p7_model_warnings(evaluate(dgd())) + + +@pytest.mark.parametrize( + "args", + [ + # An alias: the HF id is still among the values. + ["--model", "/models/qwen", "--served-model-name", MODEL], + ["--model-path", MODEL, "--served-model-name", "qwen"], + ["--model", "/models/qwen", f"--served-model-name=qwen,{MODEL}"], + # Expanded against the worker's env. + ["--model", "$(MODEL_ID)"], + ], +) +def test_p7_dgdr_model_matches_any_worker_model_value(args): + component = worker() + component["podTemplate"]["spec"]["containers"][0].update( + args=args, env=[{"name": "MODEL_ID", "value": MODEL}] + ) + report = evaluate(dgd(frontend(), component, planner())) + assert not p7_model_warnings(report), report.warnings + + +def test_p7_dgdr_model_counts_the_current_served_model_status(): + component = worker() + component["podTemplate"]["spec"]["containers"][0]["args"] = ["--model", "$(X)"] + deployment = dgd(frontend(), component, planner()) + deployment["metadata"]["generation"] = 1 + deployment["status"] = { + "observedGeneration": 1, + "components": {"agg": {"servedModelName": "other/model"}}, + } + assert p7_model_warnings(evaluate(deployment)) + deployment["status"]["components"]["agg"]["servedModelName"] = MODEL + assert not p7_model_warnings(evaluate(deployment)) + # Nothing known: only the undetermined-model warning. + del deployment["status"] + report = evaluate(deployment) + assert not p7_model_warnings(report) + assert any("served model of agg" in w for w in report.warnings) + + +@pytest.mark.parametrize( + ("extra", "match"), + [ + ({"name": "Epp", "type": "epp"}, "EPP component"), + ( + worker("side", frontendSidecar="sidecar-frontend"), + "side run a frontendSidecar", + ), + ], +) +def test_p7_gaie_layouts_with_a_frontend(extra, match): + report = evaluate(dgd(frontend(), worker(), planner(), extra)) + assert report.p4_ok + assert any(match in r for r in reasons_matching(report, "P7:")), report.reasons + + +def test_p7_no_frontend(): + report = evaluate(dgd(worker(frontendSidecar="sidecar-frontend"), planner())) + assert not report.p4_ok + assert all(r.startswith("P7:") for r in report.reasons) + assert len(report.reasons) == 1 + + +@pytest.mark.parametrize( + ("args", "env"), + [ + ((*KV_ARGS, "--tls-cert-path", "/c.pem", "--tls-key-path", "/k.pem"), {}), + ((*KV_ARGS,), {"DYN_TLS_CERT_PATH": "/c.pem", "DYN_TLS_KEY_PATH": "/k.pem"}), + ], +) +def test_p7_tls(args, env): + report = evaluate(with_frontend(args=args, env=trace_env(**env))) + assert report.p4_ok + assert reasons_matching(report, "P7:") == [ + "P7: the frontend serves TLS (--tls-cert-path / DYN_TLS_CERT_PATH)" + ] + + +@pytest.mark.parametrize( + ("replicas", "args", "ok"), + [ + (1, KV_ARGS, True), + (0, KV_ARGS, True), + (2, KV_ARGS, False), + (2, (*KV_ARGS, "--router-replica-sync"), True), + ], +) +def test_p8_frontend_replicas(replicas, args, ok): + report = evaluate(dgd(frontend(replicas=replicas, args=args), worker(), planner())) + assert bool(reasons_matching(report, "P8:")) is not ok + assert report.p4_ok + + +def test_p9_scaling_adapter_and_planner_replicas(): + adapter = dgd(frontend(), worker(scalingAdapter={}), planner()) + assert reasons_matching(evaluate(adapter), "P9:") == [ + "P9: worker component agg has a scalingAdapter; its replicas could change during a window" + ] + two_planners = dgd(frontend(), worker(), planner(replicas=2)) + assert "2 replicas" in reasons_matching(evaluate(two_planners), "P9:")[0] + # The caller's own replica count wins over the DGD. + assert not reasons_matching(evaluate(two_planners, planner_replicas=1), "P9:") + assert reasons_matching(evaluate(dgd(), planner_replicas=3), "P9:") + no_planner = evaluate(dgd(frontend(), worker())) + assert no_planner.spec_ok + assert any(w.startswith("P9:") for w in no_planner.warnings) + + +def test_p9_v1alpha1_scaling_adapter_needs_enabled(): + def deployment(adapter): + return { + "metadata": {"name": DGD}, + "spec": { + "services": { + "Frontend": {"componentType": "frontend"}, + "w": {"componentType": "worker", "scalingAdapter": adapter}, + } + }, + } + + disabled = fc._components(deployment({})) + enabled = fc._components(deployment({"enabled": True})) + assert not fc._scaling_adapter(disabled[1]) + assert fc._scaling_adapter(enabled[1]) + + +P10_CASES = [ + ({"DYN_REQUEST_TRACE": None}, None, "DYN_REQUEST_TRACE=1"), + ({"DYN_REQUEST_TRACE": "0"}, None, "DYN_REQUEST_TRACE=1"), + ({"DYN_REQUEST_TRACE": "true"}, None, None), + ({"DYN_REQUEST_TRACE_SINKS": None}, None, "DYN_REQUEST_TRACE_SINKS=file"), + ({"DYN_REQUEST_TRACE_SINKS": "stderr"}, None, "DYN_REQUEST_TRACE_SINKS=file"), + ({"DYN_REQUEST_TRACE_SINKS": "jsonl_gz"}, None, "DYN_REQUEST_TRACE_SINKS=file"), + ({"DYN_REQUEST_TRACE_SINKS": "file,stderr"}, None, None), + ({"DYN_REQUEST_TRACE_FILE_FORMAT": None}, None, None), + ({"DYN_REQUEST_TRACE_FILE_FORMAT": "jsonl"}, None, "DYN_REQUEST_TRACE_FILE_FORMAT"), + ( + {"DYN_REQUEST_TRACE_RECORDS": None}, + None, + "DYN_REQUEST_TRACE_RECORDS=request_end", + ), + ( + {"DYN_REQUEST_TRACE_RECORDS": "request_end,tool"}, + None, + "DYN_REQUEST_TRACE_RECORDS", + ), + ({"DYN_REQUEST_TRACE_RECORDS": " Request_End "}, None, None), + ( + {"DYN_REQUEST_TRACE_FILE_PATH": None}, + None, + "must set DYN_REQUEST_TRACE_FILE_PATH", + ), + ( + {"DYN_REQUEST_TRACE_FILE_PATH": f"/trace/segments/{DGD}/$(DYN_POD_NAME)/req"}, + None, + None, + ), + # The reconciler prunes only the top level of the pod's directory. + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/a/req" + }, + None, + "no further '/'", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req-$(DYN_POD_NAME)" + }, + None, + None, + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/" + }, + None, + "file prefix", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(POD_NAME)/req" + }, + None, + "file prefix", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/trace/other/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" + }, + None, + "file prefix", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": "/data/segments/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" + }, + None, + "file prefix", + ), + ( + { + "DYN_REQUEST_TRACE_FILE_PATH": { + "valueFrom": {"configMapKeyRef": {"name": "c", "key": "p"}} + } + }, + None, + "must be a literal value", + ), + ({"DYN_POD_NAME": None}, None, "DYN_POD_NAME from fieldRef"), + ({"DYN_POD_NAME": "fixed"}, None, "DYN_POD_NAME from fieldRef"), + ( + {"DYN_POD_NAME": {"valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}}}, + None, + "DYN_POD_NAME from fieldRef", + ), + ({"DYN_PARENT_DGD_K8S_NAME": "other"}, None, "overrides DYN_PARENT_DGD_K8S_NAME"), + ({}, {"volumes": []}, "no volume for claim"), + ( + {}, + {"volumes": [{"name": "traces", "persistentVolumeClaim": {"claimName": "x"}}]}, + "no volume for claim", + ), + ({}, {"mounts": []}, "does not mount claim"), + ( + {}, + {"mounts": [{"name": "traces", "mountPath": "/trace", "readOnly": True}]}, + "read-only or through a subPath", + ), + ( + {}, + {"mounts": [{"name": "traces", "mountPath": "/trace", "subPath": "segments"}]}, + "read-only or through a subPath", + ), + ({}, {"mounts": [{"name": "traces", "mountPath": "/trace/"}]}, None), + ( + {}, + { + "mounts": [ + {"name": "traces", "mountPath": "/ro", "readOnly": True}, + {"name": "traces", "mountPath": "/trace"}, + ] + }, + None, + ), +] + + +@pytest.mark.parametrize(("env", "frontend_kwargs", "match"), P10_CASES) +def test_p10_trace_recipe(env, frontend_kwargs, match): + deployment = dgd( + frontend(env=trace_env(**env), **(frontend_kwargs or {})), worker(), planner() + ) + report = evaluate(deployment) + assert report.p4_ok, report.reasons + p10 = reasons_matching(report, "P10:") + if match is None: + assert p10 == [], p10 + assert report.spec_ok + else: + assert any(match in r for r in p10), p10 + assert not report.spec_ok + + +def test_p10_custom_segments_dir_and_graph_env(): + path = "/trace/rt/segs/$(DYN_PARENT_DGD_K8S_NAME)/$(DYN_POD_NAME)/req" + deployment = dgd( + frontend(env=trace_env(DYN_REQUEST_TRACE_FILE_PATH=None)), + worker(), + planner(), + env=[{"name": "DYN_REQUEST_TRACE_FILE_PATH", "value": path}], + ) + report = evaluate(deployment, config=planner_config(segments_dir="/rt/segs/")) + assert report.spec_ok, report.reasons + + +def test_p10_extra_sinks_warn(): + report = evaluate( + with_frontend(env=trace_env(DYN_REQUEST_TRACE_SINKS="file,stderr")) + ) + assert report.spec_ok + assert any("also enables stderr" in w for w in report.warnings) + + +def test_p10_is_checked_when_parsing_fails(): + deployment = with_frontend( + command=("/bin/sh", "-c"), + args=("python3 -m dynamo.frontend",), + env=trace_env(DYN_REQUEST_TRACE=None), + ) + report = evaluate(deployment) + assert reasons_matching(report, "P4:") + assert reasons_matching(report, "P10:") + + +def test_evaluate_requires_dgdr(): + config = PlannerConfig.model_construct(mode="agg", retune=None) + with pytest.raises(ValueError, match="retune.dgdr"): + fc.evaluate_premises( + dgd(), + planner_config=config, + startup=None, + parse_error=None, + planner_replicas=None, + ) + + +def test_malformed_dgd_never_raises(): + odd_frontend = { + "name": "f", + "type": "frontend", + "replicas": "many", + "podTemplate": "not-a-template", + } + for deployment in ( + {}, + {"spec": None}, + {"spec": {"components": [None, 3]}}, + {"spec": {"components": [odd_frontend]}}, + {"spec": {"components": [frontend(volumes=["x"], mounts=[None])]}}, + ): + report = evaluate(deployment) + assert not report.spec_ok + + +def test_unexpected_parser_failure_fails_closed(monkeypatch, caplog): + def broken(env): + raise RuntimeError("frontend arg groups changed") + + monkeypatch.setattr(fc, "_frontend_parser", broken) + fc._parse_cached.cache_clear() + try: + error = parse_error(dgd()) + finally: + fc._parse_cached.cache_clear() + assert error.startswith("P4: the planner failed to parse") + assert "RuntimeError: frontend arg groups changed" in error + assert any(r.levelname == "ERROR" for r in caplog.records) + + +# ------------------------------------------------------------- helpers + + +def test_worker_types_and_canonical_json(): + assert fc.worker_types("disagg") == ("prefill", "decode") + assert fc.worker_types("agg") == ("aggregated",) + assert fc.canonical_json({"b": 1, "a": [1.5]}) == '{"a":[1.5],"b":1}' + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("plain", "plain"), + ("$(A)/x", "1/x"), + ("$$(A)", "$(A)"), + ("$(MISSING)", "$(MISSING)"), + ("$(A", "$(A"), + ("$", "$"), + ("a$$b", "a$b"), + ("$(SECRET)", None), + ], +) +def test_kubernetes_expansion(value, expected): + assert fc._expand(value, {"A": "1"}, {"SECRET"}) == expected + + +def test_parse_is_cached_per_spec(): + fc._parse_cached.cache_clear() + parse(dgd()) + parse(dgd()) + info = fc._parse_cached.cache_info() + assert info.hits >= 1 and info.misses == 1 + parse(with_frontend(args=(*KV_ARGS, "--router-temperature", "0.5"))) + assert fc._parse_cached.cache_info().misses == 2 diff --git a/components/src/dynamo/replay/api.py b/components/src/dynamo/replay/api.py index 0ac237c70b99..5003a1fe2ea1 100644 --- a/components/src/dynamo/replay/api.py +++ b/components/src/dynamo/replay/api.py @@ -155,6 +155,14 @@ def _retune_inputs( AISimulate setting. """ + retune = config.retune + if retune is not None and retune.executor != "local": + # Replay runs local sweeps only; executor 'dgdr' needs a live + # Planner, so do not silently replay it as a local sweep. + raise ValueError( + f"planner.retune executor {retune.executor!r} is not supported in " + "replay; replay retune runs executor 'local'" + ) if trace_format != "dynamo": raise ValueError( "planner.retune replays a Dynamo request trace: it requires " diff --git a/components/src/dynamo/replay/tests/test_retune_replay.py b/components/src/dynamo/replay/tests/test_retune_replay.py index 651ff1627ef0..16f328a3036c 100644 --- a/components/src/dynamo/replay/tests/test_retune_replay.py +++ b/components/src/dynamo/replay/tests/test_retune_replay.py @@ -578,6 +578,38 @@ def test_retune_rejects_an_engine_shape_that_differs_from_the_planner_mode( ) +@pytest.mark.unit +def test_retune_rejects_the_dgdr_executor(tmp_path): + # Replay runs local sweeps only; executor 'dgdr' needs a live Planner. + preset = dict.fromkeys( + ("tensor", "pipeline", "attention_data", "moe_tensor", "moe_expert"), 1 + ) + worker = { + "parallelism": {"preset": [preset]}, + "scheduler": {"max_batched_tokens": 8192, "max_sequences": 256}, + } + engine = {"mode": "aggregated", "workers": {"aggregated": worker}} + dgdr = {"claim_name": "traces", "model": "m", "sku": "h200_sxm", "engine": engine} + config = PlannerConfig.model_validate( + _guard_config() + | { + "retune": { + "interval_seconds": 21600, + "timeout_seconds": 20400, + "router": "default", + "executor": "dgdr", + "dgdr": dgdr, + } + } + ) + trace = _write_trace( + tmp_path / "trace.jsonl", count=4, gap_ms=100, blocks=1, output_tokens=2 + ) + + with pytest.raises(ValueError, match="executor 'dgdr' is not supported in replay"): + api._retune_inputs(config, **_inputs_kwargs(trace)) + + @pytest.mark.unit def test_synthetic_replay_rejects_retune(): with pytest.raises( From 73c0e5889c54271a4df4d571137b98c33454c51d Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 30 Oct 2026 15:59:18 -0700 Subject: [PATCH 076/100] feat(planner): cut live retune windows and reconcile the trace claim Add the two trace-claim pieces of live retune: - retune/live/window_cut.py cuts one retune window from the frontends' gzip request-trace segments on the shared RWX claim into the window file a DGDR Run replays, with the selection, censoring and de-duplication of retune.window.read_trace_window, bounded memory (window_too_large), corrupt-member handling (trace_corrupt) and an atomic output, in a spawn child killed at the deadline or on cancel. - retune/live/reconciler.py garbage-collects what live retune leaves on the claim and in the namespace (old frontend segments, window and temp files, and DGDR Requests beyond retune.dgdr.keep_requests), and is the SweepRegistry a sweep uses to protect its window and Requests. Both are libraries that only the live retune supervisor calls; nothing in this layer wires them into a Planner, which still rejects retune at startup. Third layer of the live retune stack split from #17562. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/retune/live/reconciler.py | 919 +++++++++++ .../dynamo/planner/retune/live/window_cut.py | 1403 +++++++++++++++++ .../tests/unit/test_live_retune_reconciler.py | 1031 ++++++++++++ .../tests/unit/test_live_retune_window_cut.py | 1253 +++++++++++++++ 4 files changed, 4606 insertions(+) create mode 100644 components/src/dynamo/planner/retune/live/reconciler.py create mode 100644 components/src/dynamo/planner/retune/live/window_cut.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py diff --git a/components/src/dynamo/planner/retune/live/reconciler.py b/components/src/dynamo/planner/retune/live/reconciler.py new file mode 100644 index 000000000000..7184900bdd78 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/reconciler.py @@ -0,0 +1,919 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Garbage collection of live retune files and DGDR Requests (design §4.4.5). + +The supervisor runs ``Reconciler.run_once`` only while this process leads and +is initialized. It deletes, with ``fs_root`` the planner's mount of the trace +claim, ``T``/``G`` the retune interval/grace and ``timeout`` the sweep +timeout:: + + ////.NNNNNN.jsonl.gz frontend segments + ///.jsonl.gz window files + ///.tmp-- cut temp files + +- Segments, only when the frontend pod set is known and nonempty: never the + highest-numbered segment of a live pod; other segments of a live pod, and + the whole directory of a pod that no longer exists, once every file is + older than ``T + G + timeout + visibility margin``. A live pod's segments + are the files directly in its directory: P10 requires the sink's file + prefix to sit directly under ``$(DYN_POD_NAME)/``. A pod directory that + cannot be read is skipped for the round; the other pods are still + collected. +- Window files, never while protected: ``window_end_ms`` in + ``[last_window_end_ms, due_window_end_ms]`` with ``now < D_k``, or in use + by this process's sweep (``protect``). Otherwise deleted when at least one + Request references the file and every referencing Request's current Run + has ``Completed=True`` and no Job pod left, or when no Request references + it and it is older than ``T + timeout``. +- Temp files older than ``cut_budget_seconds`` plus a margin, unless their + window is in use by this process's sweep. +- Requests of this DGD name, never one this process's sweep protects: a + terminal Request outside the newest ``keep_requests`` windows, a + non-terminal Request whose window deadline has passed, a Request in the + delete queue, and a Request of another DGD uid (the DGD was recreated). + +``run_once(should_continue=...)`` asks the caller before every delete or +unlink and ends the round as soon as it says no (leadership or the term was +lost while the round ran on its worker thread). ``set_dgd`` follows the DGD +identity when the DGD is recreated under the same name. + +``Reconciler`` is also the ``SweepRegistry`` the DGDR sweep uses to protect +its window and Requests and to queue deletions; those methods are thread-safe +(the sweep runs on the controller's executor thread). Protection is +reference-counted per window: each ``protect`` call takes one reference, +each ``release`` drops one, and the window with the union of every name +given to it stays protected while any reference is held. Two sweeps of the +same window (a rebuilt controller relaunching it while the old one winds +down) therefore cannot drop each other's protection. ``run_once`` blocks on +the filesystem and the Kubernetes API: call it with ``asyncio.to_thread``. +""" + +from __future__ import annotations + +import logging +import os +import re +import shutil +import threading +import time +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Optional + +from dynamo.planner.config.planner_config import DgdrExecutorConfig, RetuneConfig +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD, + LABEL_DGD_UID, + LABEL_WINDOW, + DgdRef, + KubeError, + PodInfo, + RetuneKube, + RetuneMetricsSink, +) + +logger = logging.getLogger(__name__) + +# Allowance for RWX attribute caching and the frontend's flush interval on +# top of T + G + timeout before a segment counts as unused. +DEFAULT_VISIBILITY_MARGIN_S = 600.0 +# Allowance on top of cut_budget_seconds before a temp file counts as +# abandoned (the cut subprocess is killed at its deadline). +DEFAULT_TEMP_MARGIN_S = 300.0 + +WINDOW_SUFFIX = ".jsonl.gz" +TEMP_PREFIX = ".tmp-" +# Label the Job controller puts on a Job's pods (batchv1.JobNameLabel). +JOB_NAME_LABEL = "batch.kubernetes.io/job-name" +_FINISHED_POD_PHASES = frozenset({"Succeeded", "Failed"}) +# Annotation the DGDR v1beta2 Request controller puts on each Run: the +# Request generation it was created for. +RUN_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" +RUN_COMPLETED_CONDITION = "Completed" +REQUEST_ACCEPTED_CONDITION = "Accepted" +REQUEST_INVALID_SPEC_REASON = "InvalidSpec" + +_WINDOW_RE = re.compile(r"^(\d+)" + re.escape(WINDOW_SUFFIX) + r"$") +_TEMP_RE = re.compile(r"^" + re.escape(TEMP_PREFIX) + r"(\d+)-") +# A Rust gzip JSONL sink segment: ..jsonl.gz +_SEGMENT_RE = re.compile(r"^(.*)\.(\d{6,})\.jsonl\.gz$") + + +def window_file_name(window_end_ms: int) -> str: + return f"{int(window_end_ms)}{WINDOW_SUFFIX}" + + +def window_relpath(cfg: DgdrExecutorConfig, dgd_name: str, window_end_ms: int) -> str: + """Claim-relative path of a window file (the Request's trace path).""" + + return "/".join( + (cfg.windows_dir.strip("/"), dgd_name, window_file_name(window_end_ms)) + ) + + +@dataclass(frozen=True) +class ReconcileReport: + """What one ``run_once`` round did.""" + + deleted_segments: int = 0 + deleted_pod_dirs: tuple[str, ...] = () + deleted_windows: tuple[int, ...] = () + deleted_temp_files: int = 0 + deleted_requests: tuple[str, ...] = () + # Ready frontend pods without a segment directory (trace_sink_missing). + sink_missing: tuple[str, ...] = () + errors: dict[str, int] = field(default_factory=dict) + # The round ended early: ``should_continue`` said no, or the DGD + # identity changed (``set_dgd``) while it ran. + stopped: bool = False + + +@dataclass +class _RequestView: + name: str + uid: str + dgd_uid: Optional[str] + window_end_ms: Optional[int] + trace_file: Optional[str] # window file name if it reads one of ours + obj: dict[str, Any] + + +@dataclass +class _Protection: + """One window's protection: references held and the names protected.""" + + count: int = 0 + names: set[str] = field(default_factory=set) + + +@dataclass +class _Round: + """Mutable bookkeeping of one ``run_once`` call. Counts accumulate as + deletions happen, so a phase that fails half way still reports them.""" + + now: float + dgd: DgdRef + segments_root: str + windows_root: str + should_continue: Optional[Callable[[], bool]] + errors: dict[str, int] = field(default_factory=dict) + stopped: bool = False + deleted_segments: int = 0 + deleted_dirs: list[str] = field(default_factory=list) + deleted_windows: list[int] = field(default_factory=list) + deleted_temp: int = 0 + deleted_requests: list[str] = field(default_factory=list) + sink_missing: list[str] = field(default_factory=list) + + def error(self, kind: str) -> None: + self.errors[kind] = self.errors.get(kind, 0) + 1 + + +class Reconciler: + """Deletes what live retune no longer needs; see the module docstring.""" + + def __init__( + self, + kube: RetuneKube, + dgd: DgdRef, + cfg: DgdrExecutorConfig, + retune: RetuneConfig, + fs_root: str, + metrics: RetuneMetricsSink, + now: Callable[[], float] = time.time, + *, + visibility_margin_s: float = DEFAULT_VISIBILITY_MARGIN_S, + temp_margin_s: float = DEFAULT_TEMP_MARGIN_S, + kube_timeout_s: float = 10.0, + ) -> None: + self._kube = kube + self._cfg = cfg + self._fs_root = fs_root + self._metrics = metrics + self._now = now + self._timeout_s = float(kube_timeout_s) + interval_s = float(retune.interval_seconds) + sweep_timeout_s = float(retune.effective_timeout_seconds) + self._segment_max_age_s = ( + interval_s + + float(retune.grace_seconds) + + sweep_timeout_s + + float(visibility_margin_s) + ) + self._window_max_age_s = interval_s + sweep_timeout_s + self._temp_max_age_s = float(cfg.cut_budget_seconds) + float(temp_margin_s) + + self._lock = threading.Lock() + # Guarded by _lock (set_dgd may run while a round is on its thread). + self._dgd = dgd + self._segments_root = self._root(cfg.segments_dir, dgd) + self._windows_root = self._root(cfg.windows_dir, dgd) + # window_end_ms -> references and Request names of running sweeps. + self._protected: dict[int, _Protection] = {} + self._delete_queue: set[str] = set() + # (request uid, generation) of Requests known terminal / finished; + # neither can revert for one generation. + self._terminal: set[tuple[str, int]] = set() + self._finished: set[tuple[str, int]] = set() + self._sink_reported: set[str] = set() + + def _root(self, directory: str, dgd: DgdRef) -> str: + return os.path.join(self._fs_root, directory.strip("/"), dgd.name) + + # ------------------------------------------------------------- identity + + def set_dgd(self, dgd: DgdRef) -> None: + """Follow the DGD identity (it was recreated under the same name). + + Requests labelled with the new uid become this DGD's; those of the + old uid are then deleted as ``other_dgd_uid``. The Request-state + caches are cleared. A round running on another thread ends at its + next delete, since it classified Requests by the old identity. + """ + + with self._lock: + if dgd != self._dgd: + logger.info( + "retune reconciler: DGD %s uid %s -> %s %s", + self._dgd.name, + self._dgd.uid, + dgd.name, + dgd.uid, + ) + self._dgd = dgd + self._segments_root = self._root(self._cfg.segments_dir, dgd) + self._windows_root = self._root(self._cfg.windows_dir, dgd) + self._terminal.clear() + self._finished.clear() + + @property + def dgd(self) -> DgdRef: + with self._lock: + return self._dgd + + # --------------------------------------------------------- SweepRegistry + + def protect(self, window_end_ms: int, request_names: Iterable[str]) -> None: + """Take one reference on the window and add ``request_names`` to + its protected names; each call needs a matching ``release``.""" + + names = set(request_names) + with self._lock: + entry = self._protected.setdefault(int(window_end_ms), _Protection()) + entry.count += 1 + entry.names.update(names) + + def release(self, window_end_ms: int) -> None: + """Drop one reference; the window and its names are unprotected + once no reference is left. Releasing an unknown window is a no-op.""" + + with self._lock: + window = int(window_end_ms) + entry = self._protected.get(window) + if entry is None: + return + entry.count -= 1 + if entry.count <= 0: + del self._protected[window] + + def enqueue_delete(self, request_names: Iterable[str]) -> None: + names = set(request_names) + with self._lock: + self._delete_queue.update(names) + + # ---------------------------------------------------------- inspection + + @property + def segments_root(self) -> str: + with self._lock: + return self._segments_root + + @property + def windows_root(self) -> str: + with self._lock: + return self._windows_root + + def protected_windows(self) -> frozenset[int]: + with self._lock: + return frozenset(self._protected) + + def protected_requests(self) -> frozenset[str]: + with self._lock: + return frozenset( + name for entry in self._protected.values() for name in entry.names + ) + + def pending_deletes(self) -> frozenset[str]: + with self._lock: + return frozenset(self._delete_queue) + + # ---------------------------------------------------------------- round + + def run_once( + self, + *, + last_window_end_ms: Optional[int], + due_window_end_ms: Optional[int], + deadline_for: Callable[[int], float], + frontend_pods: Optional[Sequence[PodInfo]], + should_continue: Optional[Callable[[], bool]] = None, + ) -> ReconcileReport: + """One GC round. ``deadline_for(window_end_ms)`` is that window's + sweep deadline ``D_k`` in epoch seconds; ``frontend_pods`` is the + actuator's listing (None when it failed). ``should_continue`` is + called before every delete or unlink; once it returns False (or + raises) the round deletes nothing more and returns.""" + + now = float(self._now()) + with self._lock: + state = _Round( + now=now, + dgd=self._dgd, + segments_root=self._segments_root, + windows_root=self._windows_root, + should_continue=should_continue, + ) + + def in_window_range(window_end_ms: int) -> bool: + """The design's protected range: undecided and before D_k.""" + + if last_window_end_ms is not None and window_end_ms < last_window_end_ms: + return False + if due_window_end_ms is not None and window_end_ms > due_window_end_ms: + return False + return now < deadline_for(window_end_ms) + + try: + self._collect_segments(state, frontend_pods) + except Exception: # keep the sink check, Request and window GC running + logger.exception("retune reconciler: segment GC failed") + state.error("segments") + if not state.stopped: + try: + state.sink_missing = self._report_sinks(state, frontend_pods) + except Exception: + logger.exception("retune reconciler: trace sink check failed") + state.error("sinks") + + requests: Optional[list[_RequestView]] = None + if not state.stopped: + try: + requests = self._list_requests(state.dgd) + except KubeError as exc: + logger.warning("retune reconciler: listing Requests failed: %s", exc) + state.error("list_requests") + except Exception: + logger.exception("retune reconciler: listing Requests failed") + state.error("list_requests") + try: + self._collect_requests(state, requests, in_window_range, deadline_for) + except Exception: + logger.exception("retune reconciler: Request GC failed") + state.error("requests") + + if not state.stopped: + try: + remaining = ( + None + if requests is None + else [ + view + for view in requests + if view.name not in state.deleted_requests + ] + ) + self._collect_windows(state, remaining, in_window_range) + except Exception: + logger.exception("retune reconciler: window GC failed") + state.error("windows") + + report = ReconcileReport( + deleted_segments=state.deleted_segments, + deleted_pod_dirs=tuple(sorted(state.deleted_dirs)), + deleted_windows=tuple(sorted(state.deleted_windows)), + deleted_temp_files=state.deleted_temp, + deleted_requests=tuple(sorted(state.deleted_requests)), + sink_missing=tuple(sorted(state.sink_missing)), + errors=state.errors, + stopped=state.stopped, + ) + if ( + report.deleted_segments + or report.deleted_pod_dirs + or report.deleted_windows + or report.deleted_temp_files + or report.deleted_requests + ): + logger.info( + "retune reconciler: deleted %d segments, pod dirs %s, windows %s, " + "%d temp files, Requests %s", + report.deleted_segments, + list(report.deleted_pod_dirs), + list(report.deleted_windows), + report.deleted_temp_files, + list(report.deleted_requests), + ) + return report + + def _proceed(self, state: _Round) -> bool: + """Whether the round may delete one more thing; asked right before + every delete or unlink. A no is final for the round.""" + + if state.stopped: + return False + with self._lock: + current = self._dgd + if current != state.dgd: + logger.info( + "retune reconciler: the DGD identity changed during the round; " + "stopping it" + ) + state.stopped = True + return False + if state.should_continue is None: + return True + try: + proceed = bool(state.should_continue()) + except Exception: + logger.exception("retune reconciler: should_continue failed") + proceed = False + if not proceed: + logger.info("retune reconciler: told to stop; ending the round early") + state.stopped = True + return proceed + + # ------------------------------------------------------------- segments + + def _collect_segments( + self, state: _Round, frontend_pods: Optional[Sequence[PodInfo]] + ) -> None: + if not frontend_pods: + # Unknown or empty pod set: every directory would look orphaned. + return + live = {pod.name for pod in frontend_pods} + cutoff = state.now - self._segment_max_age_s + for entry in _scandir(state.segments_root): + if state.stopped: + return + # One pod directory that cannot be read (another uid, umask 077) + # must not starve the pods after it in scandir order. + try: + if not entry.is_dir(follow_symlinks=False): + continue + if entry.name in live: + self._collect_pod_segments(state, entry.path, cutoff) + continue + if _newest_mtime(entry.path) >= cutoff: + continue + if not self._proceed(state): + return + shutil.rmtree(entry.path) + state.deleted_dirs.append(entry.name) + except FileNotFoundError: + continue + except OSError as exc: + logger.warning( + "retune reconciler: segment GC of %s failed: %s", entry.path, exc + ) + state.error("segments") + + def _collect_pod_segments( + self, state: _Round, directory: str, cutoff: float + ) -> None: + # Top level only: P10 puts the sink's file prefix directly under the + # pod directory, so there is no subdirectory to walk. + segments: dict[str, list[tuple[int, os.DirEntry[str]]]] = {} + for entry in _scandir(directory): + match = _SEGMENT_RE.match(entry.name) + if match is None or not entry.is_file(follow_symlinks=False): + continue + segments.setdefault(match.group(1), []).append((int(match.group(2)), entry)) + for entries in segments.values(): + entries.sort(key=lambda item: item[0]) + # The highest-numbered segment is the one the frontend appends to. + for _, entry in entries[:-1]: + try: + if entry.stat(follow_symlinks=False).st_mtime >= cutoff: + continue + if not self._proceed(state): + return + os.unlink(entry.path) + state.deleted_segments += 1 + except FileNotFoundError: + continue + except OSError as exc: + logger.warning( + "retune reconciler: removing %s failed: %s", entry.path, exc + ) + state.error("segments") + + def _report_sinks( + self, state: _Round, frontend_pods: Optional[Sequence[PodInfo]] + ) -> list[str]: + if frontend_pods is None: + return [] + missing: list[str] = [] + current: set[str] = set() + for pod in frontend_pods: + if not pod.ready or pod.terminating: + continue + current.add(pod.name) + absent = not os.path.isdir(os.path.join(state.segments_root, pod.name)) + if absent: + missing.append(pod.name) + self._metrics.trace_sink_missing(pod.name, absent) + for name in self._sink_reported - current: + self._metrics.trace_sink_missing(name, False) + self._sink_reported = current + if missing: + logger.warning( + "retune reconciler: Ready frontends without a trace directory " + "under %s: %s", + state.segments_root, + missing, + ) + return missing + + # -------------------------------------------------------------- windows + + def _collect_windows( + self, + state: _Round, + requests: Optional[list[_RequestView]], + in_window_range: Callable[[int], bool], + ) -> None: + protected = self.protected_windows() + references: dict[str, list[_RequestView]] = {} + for view in requests or (): + if view.trace_file is not None: + references.setdefault(view.trace_file, []).append(view) + + for entry in _scandir(state.windows_root): + if state.stopped: + return + try: + self._collect_window_entry( + state, entry, requests, references, protected, in_window_range + ) + except FileNotFoundError: + continue + except OSError as exc: + logger.warning( + "retune reconciler: window GC of %s failed: %s", entry.path, exc + ) + state.error("windows") + + def _collect_window_entry( + self, + state: _Round, + entry: os.DirEntry[str], + requests: Optional[list[_RequestView]], + references: Mapping[str, list[_RequestView]], + protected: frozenset[int], + in_window_range: Callable[[int], bool], + ) -> None: + if not entry.is_file(follow_symlinks=False): + return + if entry.name.startswith(TEMP_PREFIX): + temp = _TEMP_RE.match(entry.name) + window = int(temp.group(1)) if temp is not None else None + # A temp file of a window this process's sweep is cutting + # belongs to the running cut subprocess. + if window is not None and window in self.protected_windows(): + return + if self._older_than(entry, state.now, self._temp_max_age_s): + if self._unlink(state, entry.path): + state.deleted_temp += 1 + return + match = _WINDOW_RE.match(entry.name) + if match is None or requests is None: + # Unknown file, or references unknown: keep it. + return + window = int(match.group(1)) + if window in protected or in_window_range(window): + return + referrers = references.get(entry.name, []) + if referrers: + if not all(self._finished_with(state, view) for view in referrers): + return + elif not self._older_than(entry, state.now, self._window_max_age_s): + return + # The sweep may have protected it since the snapshot above. + if window in self.protected_windows(): + return + if self._unlink(state, entry.path): + state.deleted_windows.append(window) + + # ------------------------------------------------------------- requests + + def _list_requests(self, dgd: DgdRef) -> list[_RequestView]: + items = self._kube.list_requests( + f"{LABEL_DGD}={dgd.name}", timeout_s=self._timeout_s + ) + views = [] + prefix = self._cfg.windows_dir.strip("/") + "/" + dgd.name + "/" + for obj in items: + metadata = obj.get("metadata") or {} + labels = metadata.get("labels") or {} + pvc = _dig(obj, "spec", "workload", "trace", "source", "pvc") or {} + path = pvc.get("path") + trace_file = None + if ( + pvc.get("claimName") == self._cfg.claim_name + and isinstance(path, str) + and path.startswith(prefix) + and "/" not in path[len(prefix) :] + ): + trace_file = path[len(prefix) :] + views.append( + _RequestView( + name=str(metadata.get("name", "")), + uid=str(metadata.get("uid", "")), + dgd_uid=labels.get(LABEL_DGD_UID), + window_end_ms=_int_or_none(labels.get(LABEL_WINDOW)), + trace_file=trace_file, + obj=obj, + ) + ) + return [view for view in views if view.name] + + def _collect_requests( + self, + state: _Round, + requests: Optional[list[_RequestView]], + in_window_range: Callable[[int], bool], + deadline_for: Callable[[int], float], + ) -> None: + now = state.now + dgd_uid = state.dgd.uid + reasons: dict[str, str] = {name: "queued" for name in self.pending_deletes()} + if requests is not None: + listed = {view.uid for view in requests} + with self._lock: + for cache in (self._terminal, self._finished): + cache.difference_update( + [key for key in cache if key[0] not in listed] + ) + ours = [view for view in requests if view.dgd_uid == dgd_uid] + for view in requests: + if view.dgd_uid is None: + logger.warning( + "retune reconciler: Request %s has no %s label", + view.name, + LABEL_DGD_UID, + ) + elif view.dgd_uid != dgd_uid: + # The DGD was deleted and recreated under the same name. + reasons.setdefault(view.name, "other_dgd_uid") + states = {view.name: self._is_terminal(state, view) for view in ours} + terminal = {name for name, value in states.items() if value is True} + kept_windows = sorted( + { + view.window_end_ms + for view in ours + if view.name in terminal and view.window_end_ms is not None + }, + reverse=True, + )[: self._cfg.keep_requests] + for view in ours: + window = view.window_end_ms + if window is None: + logger.warning( + "retune reconciler: Request %s has no %s label", + view.name, + LABEL_WINDOW, + ) + continue + value = states[view.name] + if value is None: + continue # Run state unknown this round + if value: + if window not in kept_windows and not in_window_range(window): + reasons.setdefault(view.name, "terminal") + elif now >= deadline_for(window): + # Not terminal and past D_k: no sweep can adopt it anymore. + reasons.setdefault(view.name, "deadline_passed") + + for name in sorted(reasons): + if name in self.protected_requests(): + continue + if not self._proceed(state): + return + try: + self._kube.delete_request(name, timeout_s=self._timeout_s) + except KubeError as exc: + logger.warning( + "retune reconciler: deleting Request %s (%s) failed: %s", + name, + reasons[name], + exc, + ) + state.error("delete_request") + continue + logger.info( + "retune reconciler: deleted Request %s (%s)", name, reasons[name] + ) + state.deleted_requests.append(name) + with self._lock: + self._delete_queue.discard(name) + + # ------------------------------------------------- Request / Run state + + def _cached(self, cache: set[tuple[str, int]], key: tuple[str, int]) -> bool: + with self._lock: + return key in cache + + def _remember( + self, state: _Round, cache: set[tuple[str, int]], key: tuple[str, int] + ) -> None: + """Cache a verdict unless ``set_dgd`` cleared the caches meanwhile + (the caches are cleared in place, never replaced).""" + + with self._lock: + if self._dgd == state.dgd: + cache.add(key) + + def _is_terminal(self, state: _Round, view: _RequestView) -> Optional[bool]: + """Whether the Request's current-generation Run completed, or the + operator rejected the current generation (InvalidSpec); None when + the Run could not be read.""" + + generation = _int_or_none(_dig(view.obj, "metadata", "generation")) or 0 + key = (view.uid, generation) + if self._cached(self._terminal, key): + return True + try: + run = self._current_run(view, generation) + except KubeError as exc: + logger.warning( + "retune reconciler: reading the Run of Request %s failed: %s", + view.name, + exc, + ) + state.error("get_run") + return None + terminal = ( + _condition_true(run, RUN_COMPLETED_CONDITION) + if run is not None + else _invalid_spec(view.obj, generation) + ) + if terminal: + self._remember(state, self._terminal, key) + return terminal + + def _finished_with(self, state: _Round, view: _RequestView) -> bool: + """The Request no longer reads its window file: terminal, no active + Run, and no pod of the current Run's Job left.""" + + generation = _int_or_none(_dig(view.obj, "metadata", "generation")) or 0 + key = (view.uid, generation) + if self._cached(self._finished, key): + return True + if _dig(view.obj, "status", "activeRunRef", "name"): + return False + try: + run = self._current_run(view, generation) + if run is None: + finished = _invalid_spec(view.obj, generation) + elif not _condition_true(run, RUN_COMPLETED_CONDITION): + finished = False + else: + job = _dig(run, "status", "jobRef", "name") + # A Succeeded or Failed pod no longer reads the trace; one + # with no phase reported is treated as still running. + finished = job is None or not any( + pod.phase not in _FINISHED_POD_PHASES + for pod in self._kube.list_pods( + f"{JOB_NAME_LABEL}={job}", timeout_s=self._timeout_s + ) + ) + except KubeError as exc: + logger.warning( + "retune reconciler: reading Run state of Request %s failed: %s", + view.name, + exc, + ) + return False + if finished: + self._remember(state, self._finished, key) + return finished + + def _current_run( + self, view: _RequestView, generation: int + ) -> Optional[dict[str, Any]]: + """The Run created for the Request's current generation, if any. + + The Run's generation annotation identifies it, whatever its name; + the Request status names the active and the retained terminal Runs. + """ + + status = view.obj.get("status") or {} + names = [] + active = (status.get("activeRunRef") or {}).get("name") + if active: + names.append(active) + names.extend( + ref.get("name") + for ref in status.get("recentRunRefs") or () + if ref.get("name") + ) + for name in names: + run = self._kube.get_run(name, timeout_s=self._timeout_s) + if run is None: + continue + annotations = _dig(run, "metadata", "annotations") or {} + if _int_or_none(annotations.get(RUN_GENERATION_ANNOTATION)) == generation: + return run + return None + + # ---------------------------------------------------------- filesystem + + @staticmethod + def _older_than(entry: os.DirEntry[str], now: float, age_s: float) -> bool: + try: + return entry.stat(follow_symlinks=False).st_mtime < now - age_s + except FileNotFoundError: + return False + + def _unlink(self, state: _Round, path: str) -> bool: + if not self._proceed(state): + return False + try: + os.unlink(path) + return True + except FileNotFoundError: + return False + except OSError as exc: + logger.warning("retune reconciler: removing %s failed: %s", path, exc) + state.error("windows") + return False + + +# ---------------------------------------------------------------- helpers + + +def _scandir(path: str) -> list[os.DirEntry[str]]: + """Entries of ``path``; none when it is missing. Other ``OSError``s + (e.g. ``PermissionError``) propagate to the caller's per-entry handler.""" + + try: + with os.scandir(path) as entries: + return list(entries) + except (FileNotFoundError, NotADirectoryError): + return [] + + +def _newest_mtime(path: str) -> float: + """Newest mtime of ``path`` and everything under it.""" + + newest = os.lstat(path).st_mtime + for root, dirs, files in os.walk(path): + for name in dirs + files: + try: + newest = max(newest, os.lstat(os.path.join(root, name)).st_mtime) + except FileNotFoundError: + continue + return newest + + +def _dig(obj: Any, *keys: str) -> Any: + for key in keys: + if not isinstance(obj, Mapping): + return None + obj = obj.get(key) + return obj + + +def _int_or_none(value: Any) -> Optional[int]: + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _conditions(obj: Optional[Mapping[str, Any]]) -> list[Mapping[str, Any]]: + conditions = _dig(obj, "status", "conditions") + return [c for c in conditions or () if isinstance(c, Mapping)] + + +def _condition_true(obj: Optional[Mapping[str, Any]], kind: str) -> bool: + return any( + c.get("type") == kind and c.get("status") == "True" for c in _conditions(obj) + ) + + +def _invalid_spec(request: Mapping[str, Any], generation: int) -> bool: + """The operator rejected this generation; only a new one changes that.""" + + for condition in _conditions(request): + if ( + condition.get("type") == REQUEST_ACCEPTED_CONDITION + and condition.get("status") == "False" + and condition.get("reason") == REQUEST_INVALID_SPEC_REASON + and _int_or_none(condition.get("observedGeneration")) == generation + ): + return True + return False diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py new file mode 100644 index 000000000000..54e22b1921df --- /dev/null +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -0,0 +1,1403 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Cut a retune window file from the frontends' live request-trace segments. + +The frontends append ``dynamo.request.trace.v1`` rows to rotating +``//req.NNNNNN.jsonl.gz`` segments on a shared RWX claim, +one gzip member per flush (``lib/llm/src/telemetry/jsonl_gz.rs``). +``cut_window`` turns the ``request_end`` rows that arrived in +``[start_ms, end_ms)`` and completed by ``as_of_ms`` into the bare-row window +file a DGDR Run replays. Selection, censoring, de-duplication, the single +block-size rule and ``row_counts`` are those of +``retune.window.read_trace_window``, and the rows are the same bytes; only +the output is gzip-compressed and the memory is bounded. + +* **Members.** Each gzip member is decoded on its own and checked against its + CRC32 and ISIZE before any of its lines is read. A member that fails (cut + short by a segment still being written, a zero-filled hole a lagging RWX + client shows, a flipped bit) is dropped whole and the reader resynchronises + on the next gzip header, so one run of bad bytes may hold many members. + Runs of bad bytes above ``MAX_DROPPED_MEMBER_RATIO`` of the members, or + bad bytes above that ratio of the gzip bytes, fail the cut + (``trace_corrupt``); a run that reaches the end of a writer's newest + segment is its in-flight flush (or a crashed writer's last one) and is + counted but does not count toward either ratio. +* **Pass 1** parses every row as the replay reader does (a row it would + reject, in a member whose CRC is good, fails the cut as ``trace_corrupt``) + and keeps only columns for each selected row (~45 B: arrival, completion, + a 128-bit digest of ``request_id``, where its line is, block size, flags). + Duplicates are removed by digest, keeping the first in file order, + whenever the columns outgrow ``max_rows`` by a slack, so ``max_rows`` + bounds memory and is checked on de-duplicated rows. +* **Pass 2** re-reads only the members that hold kept rows, in output order + (arrival, completion, then ``request_id`` within equal pairs), re-parses + each line with its hashes and streams ``_window_record`` rows to gzip, + compressed and written by a thread. Decoded lines wait in a bounded cache + that evicts the member needed furthest in the future. A member or row + that no longer matches pass 1 fails the cut (the segment changed). +* **Atomic output.** The file is written to ``.tmp--`` + next to ``out_path``, fsynced, made 0600 and hard-linked into place; an + existing ``out_path`` is reused (another process cut the same window). +* **Child process.** ``cut_window`` runs the cut in a ``spawn`` child, like + the replay sweep's workers, and kills it at the deadline or on cancel, so + a hung RWX read or a runaway cut never holds the sweep thread. +""" + +from __future__ import annotations + +import errno +import gzip +import hashlib +import json +import logging +import math +import multiprocessing +import os +import queue +import re +import secrets +import stat +import struct +import sys +import threading +import time +import zlib +from array import array +from collections import OrderedDict +from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from multiprocessing.connection import Connection, wait +from operator import attrgetter +from pathlib import Path +from typing import Any, BinaryIO, Literal, Optional, Protocol, Union, get_args + +import numpy as np + +from dynamo.planner.offline.trace_data import _reject_duplicate_copies +from dynamo.planner.retune.live.contracts import WindowCutResult +from dynamo.planner.retune.window import ( + _dumps, + _new_counts, + _parse_request_end, + _request_end_event, + _RequestRow, + _window_record, +) + +logger = logging.getLogger(__name__) + +StrPath = Union[str, "os.PathLike[str]"] + + +class _ByteSink(Protocol): + def write(self, data: bytes, /) -> Any: + ... + + +WindowCutFailure = Literal[ + "window_cut_timeout", # the deadline passed; the child was killed + "window_too_large", # more than max_rows de-duplicated rows + "trace_corrupt", # too many bad gzip members, or a malformed row + "window_cut_failed", # I/O error, mixed block sizes, child died, ... + "cancelled", # the caller's cancel event was set; the child was killed +] +WINDOW_CUT_FAILURES: tuple[str, ...] = get_args(WindowCutFailure) + +# Runs of bad bytes (other than in-flight tails) over all members, or their +# bytes over all gzip bytes, above which the cut fails as trace_corrupt. A run +# counts once however many members it wiped out (a zero-filled hole); its +# bytes measure how much of the trace it took. +MAX_DROPPED_MEMBER_RATIO = 0.01 +# Window files are written fast rather than small: on hash-chain rows level +# 1 is ~5x faster than level 6 for ~8% more bytes, and the cut budget pays +# for compression. +OUTPUT_COMPRESSLEVEL = 1 +# Temporary window files: ``.tmp--`` next to out_path. +TEMP_PREFIX = ".tmp-" + +_TEMP_NAME = re.compile(r"^\.tmp-(\d+)-([0-9A-Za-z]+)$") +_SEGMENT_INDEX = re.compile(r"\.(\d+)\.jsonl(?:\.gz)?$") +_GZIP_MAGIC = b"\x1f\x8b\x08" # ID1, ID2, CM=deflate +_GZIP_WBITS = 16 + zlib.MAX_WBITS # one gzip member, header and trailer +_READ_CHUNK = 1 << 20 +_DECODE_STEP = 1 << 16 +_COMPACT_AT = 1 << 20 # consumed reader bytes before the buffer is compacted +_PLAIN_BLOCK = 1 << 20 # bytes of whole lines per plain-JSONL pseudo-member +# A trace file the reconciler deleted under the walk (ESTALE: on NFS). +_VANISHED = frozenset({errno.ENOENT, errno.ESTALE}) +# Pass-1 columns may hold this fraction of max_rows (at least the minimum) +# beyond max_rows before duplicates are removed. +_DEDUP_SLACK_RATIO = 0.25 +_DEDUP_SLACK_MIN = 1024 +_PASS2_CACHE_BYTES = 256 << 20 +_PASS2_BLOCK_ROWS = 1 << 16 +_LINE_OVERHEAD = 64 # bytes object and dict slot per cached line +_OPEN_FILES = 32 +_WRITE_BUFFER = 1 << 20 +_WRITE_QUEUE = 4 # batches waiting for the writer thread +_FLAG_AGENT_CONTEXT = 1 +_FLAG_NO_TOTAL_TIME = 2 +_POLL_INTERVAL_S = 0.25 +_EXIT_GRACE_S = 1.0 +_JOIN_TIMEOUT_S = 5.0 +_CHILD_NICENESS = 10 +_CHILD_SWITCH_INTERVAL_S = 0.0005 +_MAX_DROP_EXAMPLES = 5 + + +class WindowCutError(Exception): + """A window cut failed; ``reason`` is one of ``WINDOW_CUT_FAILURES``.""" + + def __init__(self, reason: WindowCutFailure, message: str) -> None: + if reason not in WINDOW_CUT_FAILURES: + raise ValueError(f"unknown window cut failure reason {reason!r}") + super().__init__(message) + self.reason: WindowCutFailure = reason + self.message = message + + def __reduce__(self) -> tuple[Any, ...]: + return (type(self), (self.reason, self.message)) + + +def temp_file_name(window_end_ms: float, nonce: Optional[str] = None) -> str: + """The name of a window's temporary file (``.tmp--``).""" + + if nonce is None: + nonce = secrets.token_hex(8) + elif not re.fullmatch(r"[0-9A-Za-z]+", nonce): + raise ValueError(f"nonce must be alphanumeric, got {nonce!r}") + return f"{TEMP_PREFIX}{int(round(window_end_ms))}-{nonce}" + + +def temp_file_window_end(name: str) -> Optional[int]: + """The window end of a ``temp_file_name`` name; ``None`` for any other name.""" + + match = _TEMP_NAME.match(name) + return int(match.group(1)) if match else None + + +def segment_index(name: str) -> Optional[int]: + """The ``NNNNNN`` of a ``.NNNNNN.jsonl[.gz]`` segment name, else ``None``.""" + + match = _SEGMENT_INDEX.search(name) + return int(match.group(1)) if match else None + + +# ------------------------------------------------------------- entry points + + +def cut_window( + segments_root: StrPath, + start_ms: float, + end_ms: float, + as_of_ms: float, + out_path: StrPath, + *, + max_rows: int, + deadline_s: float, + cancel: Optional[threading.Event] = None, +) -> WindowCutResult: + """Cut a window file in a spawned child process (``contracts.WindowCutter``). + + ``deadline_s`` is absolute wall-clock time (``time.time()`` seconds), e.g. + ``min(now + cut_budget_seconds, D_k)``. The child is killed at the + deadline (``window_cut_timeout``) or as soon as ``cancel`` is set + (``cancelled``; any object with ``is_set()`` works, so a caller can pass + one that watches several events). Every failure raises ``WindowCutError``; + invalid arguments raise ``ValueError`` before a child is started. + """ + + _validate(start_ms, end_ms, as_of_ms, out_path, max_rows) + if not math.isfinite(deadline_s): + raise ValueError("deadline_s must be finite") + deadline = time.monotonic() + (deadline_s - time.time()) + out_path = os.fspath(out_path) + temp_path = os.path.join( + os.path.dirname(os.path.abspath(out_path)), temp_file_name(end_ms) + ) + started = time.monotonic() + result = _run_in_child( + cut_window_inprocess, + (os.fspath(segments_root), start_ms, end_ms, as_of_ms, out_path), + {"max_rows": max_rows, "temp_path": temp_path}, + deadline=deadline, + cancel=cancel, + cleanup=(temp_path,), + name=f"retune-window-cut-{int(round(end_ms))}", + ) + logger.info( + "retune window [%s, %s) as of %s: %d requests in %s (%s) in %.1fs; " + "dropped %d gzip members (%d bytes)", + start_ms, + end_ms, + as_of_ms, + result.num_requests, + result.path, + "reused" if result.reused else "written", + time.monotonic() - started, + result.dropped_members, + result.dropped_bytes, + ) + return result + + +def cut_window_inprocess( + segments_root: StrPath, + start_ms: float, + end_ms: float, + as_of_ms: float, + out_path: StrPath, + *, + max_rows: int, + temp_path: Optional[StrPath] = None, +) -> WindowCutResult: + """Cut a window file in this process; the body of ``cut_window``'s child. + + An empty window writes no file (``path=None``). ``temp_path`` names the + temporary file (it must be in ``out_path``'s directory); by default a + fresh ``temp_file_name`` there. Failures raise ``WindowCutError``. + """ + + _validate(start_ms, end_ms, as_of_ms, out_path, max_rows) + out_path = os.fspath(out_path) + directory = os.path.dirname(os.path.abspath(out_path)) + if temp_path is None: + temp_path = os.path.join(directory, temp_file_name(end_ms)) + temp_path = os.fspath(temp_path) + if os.path.dirname(os.path.abspath(temp_path)) != directory: + # os.link needs one filesystem; the reconciler looks for it there. + raise ValueError(f"temp_path {temp_path!r} is not next to {out_path!r}") + try: + files = _segment_files(os.fspath(segments_root)) + selection = _Selection(start_ms, end_ms, as_of_ms, max_rows) + items = _trace_items(files) + try: + selection.scan(items) + finally: + items.close() + selection.finish() + + def result(path: Optional[str], reused: bool = False) -> WindowCutResult: + return WindowCutResult( + path=path, + num_requests=selection.num_rows, + trace_block_size=selection.block_size, + row_counts=dict(selection.counts), + dropped_members=selection.dropped_members, + dropped_bytes=selection.dropped_bytes, + reused=reused, + ) + + if selection.num_rows == 0: + return result(None) + if os.path.lexists(out_path): + # Another process (an earlier leader, an adopted Request's cut) + # already published this window; DGDR Requests name it by path. + return result(out_path, reused=True) + paths = [path for path, _ in files] + created = _publish( + out_path, temp_path, lambda sink: _write_rows(selection, paths, sink) + ) + return result(out_path, reused=not created) + except OSError as error: + raise WindowCutError( + "window_cut_failed", f"{type(error).__name__}: {error}" + ) from error + + +def _validate( + start_ms: float, + end_ms: float, + as_of_ms: float, + out_path: StrPath, + max_rows: int, +) -> None: + for name, value in ( + ("start_ms", start_ms), + ("end_ms", end_ms), + ("as_of_ms", as_of_ms), + ): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{name} must be a number, got {value!r}") + if not math.isfinite(value): + raise ValueError(f"{name} must be finite") + if not start_ms < end_ms: + raise ValueError("start_ms must be before end_ms") + if as_of_ms < end_ms: + raise ValueError("as_of_ms must not be before end_ms") + if isinstance(max_rows, bool) or not isinstance(max_rows, int) or max_rows < 1: + raise ValueError(f"max_rows must be a positive integer, got {max_rows!r}") + if not os.fspath(out_path).endswith(".gz"): + raise ValueError(f"window file must be gzip (.gz): {os.fspath(out_path)!r}") + + +# ------------------------------------------------------------ child process + + +def _run_in_child( + function: Callable[..., Any], + args: Sequence[Any], + kwargs: Mapping[str, Any], + *, + deadline: float, + cancel: Optional[threading.Event], + cleanup: Sequence[str] = (), + name: str = "retune-window-cut", +) -> Any: + """``function(*args, **kwargs)`` in a ``spawn`` child; its return value. + + ``deadline`` is ``time.monotonic()`` time. The child is killed at the + deadline, on cancel, or once it has answered; ``cleanup`` paths are + removed unless it answered with a result (a killed child leaves its + temporary file behind). A ``WindowCutError`` raised by ``function`` is + re-raised with its reason; any other failure, the child's death included, + is ``window_cut_failed``. + """ + + _check_interrupted(deadline, cancel) + context = multiprocessing.get_context("spawn") + receiver, sender = context.Pipe(duplex=False) + process = context.Process( + target=_child_main, + args=(sender, function, tuple(args), dict(kwargs)), + name=name, + daemon=True, + ) + outcome: Optional[tuple[Any, ...]] = None + try: + try: + process.start() + except Exception as error: + raise WindowCutError( + "window_cut_failed", f"cannot start the window cut process: {error}" + ) from error + finally: + sender.close() + while outcome is None: + _check_interrupted(deadline, cancel) + timeout = max(0.0, min(deadline - time.monotonic(), _POLL_INTERVAL_S)) + if not wait([receiver, process.sentinel], timeout): + continue + try: + outcome = receiver.recv() + except (EOFError, OSError): + process.join(_JOIN_TIMEOUT_S) + raise WindowCutError( + "window_cut_failed", + f"window cut process exited with code {process.exitcode} " + "before returning a result", + ) from None + finally: + _stop(process, grace_s=_EXIT_GRACE_S if outcome is not None else 0.0) + receiver.close() + if outcome is None or outcome[0] != "ok": + for path in cleanup: + _unlink_quietly(path) + assert outcome is not None # the loop only ends with one + if outcome[0] == "ok": + return outcome[1] + _, reason, message = outcome + raise WindowCutError(reason, message) + + +def _check_interrupted(deadline: float, cancel: Optional[threading.Event]) -> None: + if cancel is not None and cancel.is_set(): + raise WindowCutError("cancelled", "window cut cancelled") + if time.monotonic() >= deadline: + raise WindowCutError("window_cut_timeout", "window cut exceeded its deadline") + + +def _stop(process: multiprocessing.process.BaseProcess, *, grace_s: float) -> None: + """Let ``process`` exit for ``grace_s``, then kill it and reap it. + + A child blocked in uninterruptible I/O (a hard-mounted RWX volume) does + not die on SIGKILL until the I/O returns; it is left behind rather than + holding the sweep thread. SIGKILL takes effect before it returns to user + space, so it cannot publish a window afterwards. + """ + + if process.pid is None: + return + if grace_s > 0: + process.join(grace_s) + if process.is_alive(): + process.kill() + process.join(_JOIN_TIMEOUT_S) + if process.is_alive(): + logger.error( + "retune window cut process %s (pid %s) did not exit after SIGKILL; " + "leaving it behind", + process.name, + process.pid, + ) + + +def _child_main( + conn: Connection, + function: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> None: + """Child entry point: answer ``("ok", value)`` or ``("error", reason, message)``.""" + + try: + os.nice(_CHILD_NICENESS) + except OSError: + pass + # The writer thread takes the GIL back after every zlib block; at the + # default 5 ms interval it would wait behind parsing (pass 2 ~30% slower). + sys.setswitchinterval(_CHILD_SWITCH_INTERVAL_S) + try: + outcome: tuple[Any, ...] = ("ok", function(*args, **kwargs)) + except WindowCutError as error: + outcome = ("error", error.reason, error.message) + except BaseException as error: # noqa: BLE001 - every failure is reported + outcome = ("error", "window_cut_failed", f"{type(error).__name__}: {error}") + try: + conn.send(outcome) + except Exception as error: + conn.send(("error", "window_cut_failed", f"unsendable result: {error}")) + finally: + conn.close() + + +# ------------------------------------------------------------ gzip members + + +@dataclass(frozen=True) +class _Member: + """A gzip member that decoded and matched its CRC32 and ISIZE.""" + + offset: int + end: int + payload: bytes + + +@dataclass(frozen=True) +class _Garbage: + """A maximal run of bytes ``[offset, end)`` that holds no valid member.""" + + offset: int + end: int + at_eof: bool + + +class _GzipMemberReader: + """Iterate a gzip file as validated members and runs of bad bytes. + + Every byte from ``offset`` to the end of the file is in exactly one item, + in file order. A member is yielded only after zlib reached its end and + its trailer's CRC32 and ISIZE match the decompressed bytes (zlib checks + them too; the explicit check does not depend on how zlib was built). + Anything else - a member cut short by the end of the file, a corrupt + one, zeros - starts a ``_Garbage`` run that ends at the next gzip header + that decodes, or at the end of the file. Memory holds one member's + compressed and decompressed bytes plus a read chunk. + """ + + def __init__(self, source: BinaryIO, offset: int = 0) -> None: + if offset: + source.seek(offset) + self._source = source + self._buf = bytearray() + self._pos = 0 # current position in _buf + self._base = offset # file offset of _buf[0] + self._eof = False + self._pending: Optional[_Member] = None + + def __iter__(self) -> _GzipMemberReader: + return self + + def __next__(self) -> Union[_Member, _Garbage]: + if self._pending is not None: + pending, self._pending = self._pending, None + return pending + if self._pos >= len(self._buf) and not self._fill(): + raise StopIteration + start = self._offset() + member = self._decode() + if member is not None: + return member + while self._skip_to_magic(): + member = self._decode() + if member is not None: + self._pending = member + return _Garbage(start, member.offset, at_eof=False) + return _Garbage(start, self._offset(), at_eof=True) + + def _offset(self) -> int: + return self._base + self._pos + + def _fill(self) -> bool: + if self._eof: + return False + chunk = self._source.read(_READ_CHUNK) + if not chunk: + self._eof = True + return False + self._buf += chunk + return True + + def _consume(self, count: int) -> None: + self._pos += count + if self._pos >= _COMPACT_AT and 2 * self._pos >= len(self._buf): + del self._buf[: self._pos] + self._base += self._pos + self._pos = 0 + + def _decode(self) -> Optional[_Member]: + """Decode and consume the member at the current position. + + Returns ``None``, consuming nothing, when no valid member starts here. + """ + + decoder = zlib.decompressobj(wbits=_GZIP_WBITS) + parts: list[bytes] = [] + index = self._pos + while True: + if index >= len(self._buf) and not self._fill(): + return None # cut short by the end of the file + piece = self._buf[index : index + _DECODE_STEP] + try: + parts.append(decoder.decompress(piece)) + except zlib.error: + return None + if decoder.eof: + end = index + len(piece) - len(decoder.unused_data) + payload = b"".join(parts) + if not _trailer_matches(self._buf[end - 8 : end], payload): + return None + member = _Member(self._offset(), self._base + end, payload) + self._consume(end - self._pos) + return member + index += len(piece) + + def _skip_to_magic(self) -> bool: + """Consume at least one byte, then up to the next gzip magic. + + ``False`` (everything consumed) when no magic is left in the file. + """ + + self._consume(1) + keep = len(_GZIP_MAGIC) - 1 # a magic may straddle two reads + while True: + found = self._buf.find(_GZIP_MAGIC, self._pos) + if found >= 0: + self._consume(found - self._pos) + return True + self._consume(max(0, len(self._buf) - keep - self._pos)) + if not self._fill(): + self._consume(len(self._buf) - self._pos) + return False + + +def _trailer_matches(trailer: Union[bytes, bytearray], payload: bytes) -> bool: + crc, size = struct.unpack(" Optional[bytes]: + """The payload of ``data`` when it is exactly one valid gzip member.""" + + decoder = zlib.decompressobj(wbits=_GZIP_WBITS) + try: + payload = decoder.decompress(data) + except zlib.error: + return None + if not decoder.eof or decoder.unused_data or len(data) < 8: + return None + return payload if _trailer_matches(data[-8:], payload) else None + + +# ---------------------------------------------------------------- pass 1 + + +def _segment_files(root: str) -> list[tuple[str, bool]]: + """Trace files under ``root`` in replay-reader order, and which are newest. + + The files and their order are those of ``resolve_trace_files([root])``: + sorted ``*.jsonl`` and ``*.jsonl.gz`` regular files, and a file present + both compressed and not is an error. A directory or file under ``root`` + that vanishes during the walk (ENOENT, or ESTALE on NFS: the reconciler + deletes a dead pod's segments) is skipped; any other error listing or + stat-ing one (EACCES, EIO, ETIMEDOUT on a soft mount, ...) fails the cut + as ``window_cut_failed`` rather than silently losing a pod's rows, as + does any error on ``root`` itself. Each file comes with whether it is the + newest segment (highest ``NNNNNN``) of its directory, the one its writer + may still be appending to. + """ + + try: + root_mode = os.stat(root).st_mode + except OSError as error: + if error.errno in _VANISHED: + raise WindowCutError( + "window_cut_failed", f"trace segments root {root} does not exist" + ) from error + raise _unreadable(root, error) from error + if stat.S_ISDIR(root_mode): + + def walk_error(error: OSError) -> None: + if error.filename != root and error.errno in _VANISHED: + logger.info( + "retune window cut: trace directory %s vanished (%s); " + "skipping it", + error.filename, + error, + ) + return + raise _unreadable(error.filename, error) from error + + found: list[Path] = [] + for directory, _, names in os.walk(root, onerror=walk_error): + for name in names: + if name.endswith(".jsonl") or name.endswith(".jsonl.gz"): + candidate = os.path.join(directory, name) + if _is_regular_file(candidate): + found.append(Path(candidate)) + try: + paths = _reject_duplicate_copies(sorted(found)) + except ValueError as error: + raise WindowCutError("window_cut_failed", str(error)) from error + else: + paths = [Path(root)] + newest: dict[Path, tuple[int, str]] = {} + for path in paths: + index = segment_index(path.name) + key = (-1 if index is None else index, path.name) + if path.parent not in newest or key > newest[path.parent]: + newest[path.parent] = key + return [(str(path), newest[path.parent][1] == path.name) for path in paths] + + +def _is_regular_file(path: str) -> bool: + """Whether ``path`` is a regular file (following symlinks, like ``is_file``). + + ``False`` when it vanished; any other stat error fails the cut. + """ + + try: + return stat.S_ISREG(os.stat(path).st_mode) + except OSError as error: + if error.errno in _VANISHED: + return False + raise _unreadable(path, error) from error + + +def _unreadable(path: Any, error: OSError) -> WindowCutError: + return WindowCutError( + "window_cut_failed", + f"cannot read trace segments at {path}: {type(error).__name__}: {error}", + ) + + +@dataclass(frozen=True) +class _PlainBlock: + """Whole lines ``[offset, end)`` of a plain JSONL file. + + ``dropped`` counts the bytes of an unterminated last line that does not + parse (still being written); they are not in ``payload``. + """ + + offset: int + end: int + payload: bytes + dropped: int = 0 + + +# (file index, path, newest, item); an OSError item means the file vanished. +_TraceItem = tuple[int, str, bool, Union[_Member, _Garbage, _PlainBlock, OSError]] + + +def _trace_items( + files: Sequence[tuple[str, bool]], +) -> Generator[_TraceItem, None, None]: + """Every member, bad byte run and plain block of ``files``, in file order. + + A file that vanishes (ENOENT, or ESTALE on NFS) before or while it is + read yields its error and the walk goes on; other I/O errors raise. + """ + + for file_index, (path, newest) in enumerate(files): + try: + with open(path, "rb") as source: + items: Iterator[Union[_Member, _Garbage, _PlainBlock]] = ( + _GzipMemberReader(source) + if path.endswith(".gz") + else _plain_blocks(source) + ) + for item in items: + yield file_index, path, newest, item + except OSError as error: + if error.errno not in _VANISHED: + raise + yield file_index, path, newest, error + + +def _plain_blocks(source: BinaryIO) -> Iterator[_PlainBlock]: + """A plain JSONL file as blocks of about ``_PLAIN_BLOCK`` bytes of lines. + + Reading stops at the first block that does not end in a newline: its + last line was the end of the file when read. A writer may finish that + line before the next read, which would then start inside it. + """ + + offset = 0 + while True: + lines = source.readlines(_PLAIN_BLOCK) + if not lines: + return + block = b"".join(lines) + if block.endswith(b"\n"): + yield _PlainBlock(offset, offset + len(block), block) + offset += len(block) + continue + # Only the file's last line can be unterminated. One that does not + # parse is still being written; one that does is a row, as the + # replay reader reads it. + payload, dropped = block, 0 + cut = block.rfind(b"\n") + 1 + tail = block[cut:] + if tail.strip() and not _is_json(tail): + payload, dropped = block[:cut], len(tail) + yield _PlainBlock(offset, offset + len(payload), payload, dropped) + return + + +class _Selection: + """Pass 1: the selected rows of every trace file, as columns. + + Row ``i`` is ``arrival[i]``, ``completion[i]``, ``rid[16*i:16*i+16]`` + (BLAKE2b of ``request_id``), line ``line[i]`` of member + ``members[member[i]]``, ``block[i]`` and ``flags[i]``. Rows are in file + order, the order the replay reader sees them, so de-duplication keeps the + same (first) row it does. + """ + + def __init__( + self, start_ms: float, end_ms: float, as_of_ms: float, max_rows: int + ) -> None: + self.start_ms = start_ms + self.end_ms = end_ms + self.as_of_ms = as_of_ms + self.max_rows = max_rows + self._dedup_at = max_rows + max( + _DEDUP_SLACK_MIN, int(max_rows * _DEDUP_SLACK_RATIO) + ) + self.counts = _new_counts() + # (file index, offset, end, is gzip) of each member with a selected + # row; a plain JSONL file is cut into pseudo-members of whole lines. + self.members: list[tuple[int, int, int, bool]] = [] + self.arrival = array("q") + self.completion = array("q") + self.rid = bytearray() + self.member = array("I") + self.line = array("I") + self.block = array("q") + self.flags = array("B") + self.duplicates = 0 + self.valid_members = 0 + # Runs of bad bytes (``_Garbage``), each counted once however many + # members it held, and their bytes (plus plain-JSONL partial tails). + self.dropped_members = 0 + self.dropped_bytes = 0 + # The runs, and their bytes, that count toward MAX_DROPPED_MEMBER_RATIO. + self.corrupt_members = 0 + self.corrupt_bytes = 0 + # Bytes of every gzip member and run of bad bytes. + self.gzip_bytes = 0 + self.block_size: Optional[int] = None + self._drop_examples: list[str] = [] + + @property + def num_rows(self) -> int: + return len(self.arrival) + + def digest(self, row: int) -> bytes: + return bytes(self.rid[16 * row : 16 * row + 16]) + + def scan(self, items: Iterable[_TraceItem]) -> None: + """Consume ``_trace_items`` (in file order).""" + + for file_index, path, newest, item in items: + if isinstance(item, _Member): + self.valid_members += 1 + self.gzip_bytes += item.end - item.offset + self._scan_payload( + item.payload, file_index, path, item.offset, item.end, True + ) + elif isinstance(item, _PlainBlock): + self.dropped_bytes += item.dropped + if item.payload: + self._scan_payload( + item.payload, file_index, path, item.offset, item.end, False + ) + elif isinstance(item, _Garbage): + self._dropped(path, item, newest=newest) + else: + # The reconciler deletes only segments far older than any + # window being cut; a row taken from this one fails pass 2. + logger.warning( + "retune window cut: trace file %s vanished (%s); skipping it", + path, + item, + ) + + def finish(self) -> None: + """De-duplicate, then apply the corruption, size and block-size rules.""" + + self._dedup() + self.counts["duplicate"] = self.duplicates + total = self.valid_members + self.dropped_members + # A run counts as one member: its bytes catch a hole that wiped out + # many members. + if ( + self.corrupt_members > MAX_DROPPED_MEMBER_RATIO * total + or self.corrupt_bytes > MAX_DROPPED_MEMBER_RATIO * self.gzip_bytes + ): + raise WindowCutError( + "trace_corrupt", + f"{self.corrupt_members} of {total} gzip members of the trace " + f"failed validation ({self.corrupt_bytes} of {self.gzip_bytes} " + f"bytes; limit {MAX_DROPPED_MEMBER_RATIO:.0%} of either); " + f"dropped byte runs: {', '.join(self._drop_examples)}", + ) + if self.num_rows > self.max_rows: + raise _too_large(self.max_rows) + if not self.num_rows: + return + blocks = np.unique(np.frombuffer(self.block, dtype=self.block.typecode)) + if len(blocks) > 1: + raise WindowCutError( + "window_cut_failed", + "trace window mixes trace_block_size values " + f"{blocks.tolist()}; replay needs one block size", + ) + self.block_size = int(blocks[0]) + flags: np.ndarray = np.frombuffer(self.flags, dtype=self.flags.typecode) + self.counts["agent_context_stripped"] = int( + np.count_nonzero(flags & _FLAG_AGENT_CONTEXT) + ) + self.counts["censor_unavailable"] = int( + np.count_nonzero(flags & _FLAG_NO_TOTAL_TIME) + ) + + def _dropped(self, path: str, item: _Garbage, *, newest: bool) -> None: + size = item.end - item.offset + self.dropped_members += 1 + self.dropped_bytes += size + self.gzip_bytes += size + if item.at_eof and newest: + # The flush its writer is appending, or a crashed writer's last + # one: at most one batch, written after the cut began. + logger.debug( + "retune window cut: %d trailing bytes of %s hold no complete " + "gzip member (in-flight write)", + size, + path, + ) + return + self.corrupt_members += 1 + self.corrupt_bytes += size + if len(self._drop_examples) < _MAX_DROP_EXAMPLES: + self._drop_examples.append(f"{path}@{item.offset}+{size}") + logger.warning( + "retune window cut: dropped %d bytes at %s offset %d that hold no " + "valid gzip member", + size, + path, + item.offset, + ) + + def _scan_payload( + self, + payload: bytes, + file_index: int, + path: str, + offset: int, + end: int, + is_gzip: bool, + ) -> None: + lines = payload.split(b"\n") + if not lines[-1]: + lines.pop() + member_id = -1 + start_ms, end_ms, as_of_ms = self.start_ms, self.end_ms, self.as_of_ms + counts = self.counts + for index, raw in enumerate(lines): + if not raw.strip(): + continue + location = f"{path}@{offset}:{index + 1}" + event = _event(raw, location, "trace_corrupt") + if event is None: + continue + row = _request_row(event, location, "trace_corrupt") + if row is None: + counts["missing_output_tokens"] += 1 + continue + if not start_ms <= row.arrival_ms < end_ms: + continue + if row.completion_ms > as_of_ms: + counts["censored"] += 1 + continue + if member_id < 0: + member_id = len(self.members) + self.members.append((file_index, offset, end, is_gzip)) + self._append(row, member_id, index, location) + + def _append( + self, row: _RequestRow, member_id: int, index: int, location: str + ) -> None: + try: + self.arrival.append(row.arrival_ms) + self.completion.append(row.completion_ms) + self.block.append(row.trace_block_size) + except OverflowError as error: + raise WindowCutError( + "trace_corrupt", f"trace row value out of range at {location}" + ) from error + self.rid += _digest(row.request_id) + self.member.append(member_id) + self.line.append(index) + self.flags.append( + (_FLAG_AGENT_CONTEXT if row.had_agent_context else 0) + | (_FLAG_NO_TOTAL_TIME if row.total_time_ms is None else 0) + ) + if len(self.arrival) > self._dedup_at: + # Duplicates only shrink the window, and rows already kept stay + # kept: past max_rows after de-duplication, the window is too large. + self._dedup() + if len(self.arrival) > self.max_rows: + raise _too_large(self.max_rows) + + def _dedup(self) -> None: + count = len(self.arrival) + if count < 2: + return + keep = _first_occurrences(self.rid, count) + kept = int(np.count_nonzero(keep)) + if kept == count: + return + self.duplicates += count - kept + self.arrival = _compress(self.arrival, keep) + self.completion = _compress(self.completion, keep) + self.member = _compress(self.member, keep) + self.line = _compress(self.line, keep) + self.block = _compress(self.block, keep) + self.flags = _compress(self.flags, keep) + self.rid = bytearray( + np.frombuffer(self.rid, dtype=np.uint8).reshape(count, 16)[keep].tobytes() + ) + + +def _first_occurrences(rid: bytearray, count: int) -> np.ndarray: + """Mask of the rows whose 128-bit digest has not occurred before them.""" + + words: np.ndarray = np.frombuffer(rid, dtype=np.uint64).reshape(count, 2) + high, low = words[:, 0], words[:, 1] + order = np.lexsort((np.arange(count), low, high)) + high, low = high[order], low[order] + repeat: np.ndarray = np.zeros(count, dtype=bool) + repeat[1:] = (high[1:] == high[:-1]) & (low[1:] == low[:-1]) + keep: np.ndarray = np.ones(count, dtype=bool) + keep[order[repeat]] = False + return keep + + +def _compress(column: array, keep: np.ndarray) -> array: + kept = array(column.typecode) + kept.frombytes(np.frombuffer(column, dtype=column.typecode)[keep].tobytes()) + return kept + + +def _too_large(max_rows: int) -> WindowCutError: + return WindowCutError( + "window_too_large", + f"the window has more than {max_rows} requests (max_window_rows)", + ) + + +def _digest(request_id: str) -> bytes: + # UTF-8 with surrogatepass is injective on str, lone surrogates included. + return hashlib.blake2b( + request_id.encode("utf-8", "surrogatepass"), digest_size=16 + ).digest() + + +def _is_json(raw: bytes) -> bool: + try: + json.loads(raw) + except ValueError: + return False + return True + + +def _event( + raw: bytes, location: str, failure: WindowCutFailure +) -> Optional[dict[str, Any]]: + """The ``request_end`` event of a trace line, as the replay reader reads it.""" + + try: + record = json.loads(raw) + except ValueError as error: # JSONDecodeError, UnicodeDecodeError + raise WindowCutError(failure, f"invalid JSON at {location}: {error}") from error + if not isinstance(record, dict): + raise WindowCutError(failure, f"trace row at {location} must be a JSON object") + try: + return _request_end_event(record, location) + except ValueError as error: + raise WindowCutError(failure, str(error)) from error + + +def _request_row( + event: dict[str, Any], location: str, failure: WindowCutFailure +) -> Optional[_RequestRow]: + try: + return _parse_request_end(event, location, need_hashes=True) + except ValueError as error: + raise WindowCutError(failure, str(error)) from error + + +# ---------------------------------------------------------------- pass 2 + + +def _publish(out_path: str, temp_path: str, write: Callable[[_ByteSink], None]) -> bool: + """Write ``out_path`` atomically through ``temp_path``. + + Returns ``False`` when ``out_path`` already existed: it is kept and the + new file is discarded. ``temp_path`` is removed in every case. + """ + + directory = os.path.dirname(os.path.abspath(out_path)) + os.makedirs(directory, exist_ok=True) + try: + descriptor = os.open( + temp_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_CLOEXEC", 0), + 0o600, + ) + with open(descriptor, "wb") as raw: + # No name and no mtime in the header: equal cuts are equal bytes. + with gzip.GzipFile( + filename="", + mode="wb", + fileobj=raw, + compresslevel=OUTPUT_COMPRESSLEVEL, + mtime=0, + ) as sink: + write(sink) + raw.flush() + os.fsync(raw.fileno()) + # Only the owner reads the window. The Run Job reads it as the planner + # image user (uid 1000), which the Planner also runs as by default. + os.chmod(temp_path, 0o600) + try: + os.link(temp_path, out_path) + created = True + except FileExistsError: + created = False + finally: + _unlink_quietly(temp_path) + _fsync_directory(directory) + return created + + +def _fsync_directory(directory: str) -> None: + try: + descriptor = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + except OSError as error: + logger.debug("retune window cut: cannot open %s to fsync: %s", directory, error) + return + try: + os.fsync(descriptor) + except OSError as error: + # Some network filesystems reject fsync on a directory; their LINK + # is synchronous at the server anyway. + logger.debug("retune window cut: fsync of %s failed: %s", directory, error) + finally: + os.close(descriptor) + + +def _unlink_quietly(path: str) -> None: + try: + os.unlink(path) + except FileNotFoundError: + pass + except OSError as error: + logger.warning("retune window cut: cannot remove %s: %s", path, error) + + +def _write_rows(selection: _Selection, paths: Sequence[str], sink: _ByteSink) -> None: + """Pass 2: write the kept rows to ``sink`` in window-file order. + + The order is ``read_trace_window``'s: (arrival, completion, request_id). + The columns order rows by the first two; rows with an equal pair are + collected and ordered by their ``request_id`` before they are written. + """ + + count = selection.num_rows + arrival: np.ndarray = np.frombuffer( + selection.arrival, dtype=selection.arrival.typecode + ) + completion: np.ndarray = np.frombuffer( + selection.completion, dtype=selection.completion.typecode + ) + members: np.ndarray = np.frombuffer( + selection.member, dtype=selection.member.typecode + ) + lines: np.ndarray = np.frombuffer(selection.line, dtype=selection.line.typecode) + order = np.lexsort((completion, arrival)) + position: np.ndarray = np.empty(count, dtype=np.int64) + position[order] = np.arange(count, dtype=np.int64) + # Each member's rows, in the order they are written. + by_member = np.lexsort((position, members)) + bounds = np.asarray( + np.searchsorted(members[by_member], np.arange(len(selection.members) + 1)) + ) + cache = _MemberLines( + paths, + selection.members, + bounds, + position[by_member], + lines[by_member], + capacity=_PASS2_CACHE_BYTES, + ) + del position, by_member + group: list[_RequestRow] = [] + group_key: Optional[tuple[int, int]] = None + writer = _LineWriter(sink) + try: + for first in range(0, count, _PASS2_BLOCK_ROWS): + block = order[first : first + _PASS2_BLOCK_ROWS] + for row, arrival_ms, completion_ms, member_id, line_index in zip( + block.tolist(), + arrival[block].tolist(), + completion[block].tolist(), + members[block].tolist(), + lines[block].tolist(), + ): + if (arrival_ms, completion_ms) != group_key: + _write_group(group, writer) + group = [] + group_key = (arrival_ms, completion_ms) + raw = cache.take(member_id, line_index) + location = cache.location(member_id, line_index) + event = _event(raw, location, "window_cut_failed") + parsed = ( + None + if event is None + else _request_row(event, location, "window_cut_failed") + ) + if ( + parsed is None + or parsed.arrival_ms != arrival_ms + or parsed.completion_ms != completion_ms + or _digest(parsed.request_id) != selection.digest(row) + ): + raise _changed(f"the row at {location} differs") + group.append(parsed) + _write_group(group, writer) + writer.close() + except BaseException: + writer.abort() + raise + finally: + cache.close() + + +def _write_group(group: list[_RequestRow], writer: _LineWriter) -> None: + if len(group) > 1: + group.sort(key=attrgetter("request_id")) + for row in group: + writer.write(_dumps(_window_record(row))) + + +def _changed(detail: str) -> WindowCutError: + return WindowCutError( + "window_cut_failed", f"trace segment changed between passes: {detail}" + ) + + +class _LineWriter: + """Batches window lines and writes them to the gzip sink in a thread. + + zlib and file writes release the GIL, so compressing and writing one + batch overlaps parsing the next; at most ``_WRITE_QUEUE`` batches wait. + ``close`` writes what is left and re-raises the thread's error; + ``abort`` stops it without writing more. + """ + + def __init__(self, sink: _ByteSink) -> None: + self._sink = sink + self._parts: list[str] = [] + self._size = 0 + self._queue: queue.Queue[Optional[bytes]] = queue.Queue(maxsize=_WRITE_QUEUE) + self._error: Optional[BaseException] = None + self._aborted = False + self._stopped = False + self._thread = threading.Thread( + target=self._run, name="retune-window-write", daemon=True + ) + self._thread.start() + + def write(self, line: str) -> None: + self._parts.append(line) + self._parts.append("\n") + self._size += len(line) + 1 + if self._size >= _WRITE_BUFFER: + self._send() + + def close(self) -> None: + self._send() + self._stop() + if self._error is not None: + raise self._error + + def abort(self) -> None: + self._aborted = True + self._stop() + + def _send(self) -> None: + if self._error is not None: + raise self._error + if self._parts: + self._queue.put("".join(self._parts).encode("utf-8")) + self._parts = [] + self._size = 0 + + def _stop(self) -> None: + if not self._stopped: + self._stopped = True + self._queue.put(None) + self._thread.join() + + def _run(self) -> None: + while True: + data = self._queue.get() + if data is None: + return + if self._error is not None or self._aborted: + continue + try: + self._sink.write(data) + except BaseException as error: # noqa: BLE001 - re-raised by close() + self._error = error + + +class _MemberLines: + """The lines pass 2 still has to write, decoded member by member. + + Rows are taken in output order. A member is read and decoded when one of + its rows is first needed and keeps only its lines still to be written; it + is dropped when its last one is taken. Above ``capacity`` bytes, the + members whose next row is written last are evicted (Belady) and decoded + again when needed, so memory stays bounded whatever the request durations. + + ``bounds[m]:bounds[m+1]`` indexes member ``m``'s rows in ``positions`` + (their output positions, ascending) and ``lines`` (their line indices). + """ + + def __init__( + self, + paths: Sequence[str], + members: Sequence[tuple[int, int, int, bool]], + bounds: np.ndarray, + positions: np.ndarray, + lines: np.ndarray, + *, + capacity: int, + ) -> None: + self._paths = paths + self._members = members + self._cursor: np.ndarray = bounds[:-1].astype(np.int64) + self._ends = bounds[1:] + self._positions = positions + self._lines = lines + self._capacity = capacity + self._loaded: dict[int, dict[int, bytes]] = {} + self._sizes: dict[int, int] = {} + self._bytes = 0 + self._files: OrderedDict[int, BinaryIO] = OrderedDict() + self.loads = 0 + + def location(self, member_id: int, line_index: int) -> str: + file_index, offset, _, _ = self._members[member_id] + return f"{self._paths[file_index]}@{offset}:{line_index + 1}" + + def take(self, member_id: int, line_index: int) -> bytes: + lines = self._loaded.get(member_id) + if lines is None: + lines = self._load(member_id) + raw = lines.pop(line_index, None) + if raw is None: + raise WindowCutError( + "window_cut_failed", + f"window cut lost track of the row at " + f"{self.location(member_id, line_index)}", + ) + self._cursor[member_id] += 1 + size = len(raw) + _LINE_OVERHEAD + self._sizes[member_id] -= size + self._bytes -= size + if not lines: + del self._loaded[member_id] + del self._sizes[member_id] + elif self._bytes > self._capacity: + self._evict() + return raw + + def close(self) -> None: + for source in self._files.values(): + source.close() + self._files.clear() + + def _load(self, member_id: int) -> dict[int, bytes]: + file_index, offset, end, is_gzip = self._members[member_id] + data = self._read(file_index, offset, end) + payload = _decode_exact_member(data) if is_gzip else data + if payload is None: + raise _changed( + f"the gzip member at {self._paths[file_index]}@{offset} " + "no longer decodes" + ) + pieces = payload.split(b"\n") + wanted = self._lines[self._cursor[member_id] : self._ends[member_id]].tolist() + if max(wanted) >= len(pieces): + raise _changed( + f"the member at {self._paths[file_index]}@{offset} has fewer lines" + ) + lines = {index: pieces[index] for index in wanted} + size = sum(map(len, lines.values())) + _LINE_OVERHEAD * len(lines) + self._loaded[member_id] = lines + self._sizes[member_id] = size + self._bytes += size + self.loads += 1 + return lines + + def _read(self, file_index: int, offset: int, end: int) -> bytes: + source = self._files.pop(file_index, None) + if source is None: + if len(self._files) >= _OPEN_FILES: + _, oldest = self._files.popitem(last=False) + oldest.close() + source = open(self._paths[file_index], "rb") + self._files[file_index] = source + source.seek(offset) + data = source.read(end - offset) + if len(data) != end - offset: + raise _changed(f"{self._paths[file_index]} is shorter than when scanned") + return data + + def _evict(self) -> None: + target = self._capacity * 3 // 4 + furthest_first = sorted( + self._loaded, + key=lambda member_id: int(self._positions[self._cursor[member_id]]), + reverse=True, + ) + for member_id in furthest_first: + if self._bytes <= target: + return + del self._loaded[member_id] + self._bytes -= self._sizes.pop(member_id) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py new file mode 100644 index 000000000000..a86953ff1516 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py @@ -0,0 +1,1031 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the live retune reconciler (segment, window, temp-file and +DGDR Request garbage collection) and its ``SweepRegistry``.""" + +from __future__ import annotations + +import os +import stat +import threading +from collections.abc import Callable +from pathlib import Path +from typing import Any, Optional + +import pytest + +from dynamo.planner.config.planner_config import DgdrExecutorConfig, RetuneConfig +from dynamo.planner.retune.live.contracts import ( + LABEL_DGD, + LABEL_DGD_UID, + LABEL_WINDOW, + DgdRef, + KubeError, + PodInfo, +) +from dynamo.planner.retune.live.reconciler import ( + JOB_NAME_LABEL, + RUN_GENERATION_ANNOTATION, + Reconciler, + window_file_name, + window_relpath, +) + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +DGD = DgdRef(name="dgd", namespace="ns", uid="uid-now", dynamo_namespace="dyn-ns") +NOW = 1_800_000_000.0 +INTERVAL = 3600.0 +GRACE = 120.0 +TIMEOUT = 1800.0 +MARGIN = 600.0 +SEGMENT_AGE = INTERVAL + GRACE + TIMEOUT + MARGIN # 6120 +WINDOW_AGE = INTERVAL + TIMEOUT # 5400 +TEMP_AGE = 900.0 + 300.0 # cut_budget + temp margin +OLD = 100_000.0 # older than every threshold + + +def window_ms(k: int) -> int: + """End of the k-th window before NOW's current one, in epoch ms.""" + + return int((NOW - k * INTERVAL) * 1000) + + +def deadline_for(window_end_ms: int) -> float: + return window_end_ms / 1000 + GRACE + TIMEOUT + + +# ------------------------------------------------------------------ fakes + + +class FakeKube: + namespace = DGD.namespace + + def __init__(self) -> None: + self.requests: dict[str, dict[str, Any]] = {} + self.runs: dict[str, dict[str, Any]] = {} + self.job_pods: dict[str, list[PodInfo]] = {} + self.deleted: list[str] = [] + self.selectors: list[str] = [] + self.get_run_calls: list[str] = [] + self.fail_list = False + self.fail_get_run = False + self.fail_delete: set[str] = set() + self.on_delete: Optional[Callable[[str], None]] = None + + def list_requests(self, label_selector: str, *, timeout_s: float = 10.0): + self.selectors.append(label_selector) + if self.fail_list: + raise KubeError("list failed") + return [dict(obj) for obj in self.requests.values()] + + def delete_request( + self, name: str, *, propagation: str = "Background", timeout_s: float = 10.0 + ): + if name in self.fail_delete: + raise KubeError("delete failed") + if self.on_delete is not None: + self.on_delete(name) + self.deleted.append(name) + self.requests.pop(name, None) + + def get_run(self, name: str, *, timeout_s: float = 10.0): + self.get_run_calls.append(name) + if self.fail_get_run: + raise KubeError("get failed") + return self.runs.get(name) + + def list_pods(self, label_selector: str, *, timeout_s: float = 10.0): + key, _, value = label_selector.partition("=") + assert key == JOB_NAME_LABEL + return list(self.job_pods.get(value, [])) + + # -- builders + + def add_request( + self, + name: str, + window: int, + *, + dgd_uid: Optional[str] = DGD.uid, + state: str = "completed", # completed | running | creating | invalid + job_pods: int = 0, + job_pod_phase: Optional[str] = None, + claim: str = "traces", + path: Optional[str] = None, + generation: int = 1, + ) -> None: + labels = {LABEL_DGD: DGD.name, LABEL_WINDOW: str(window)} + if dgd_uid is not None: + labels[LABEL_DGD_UID] = dgd_uid + status: dict[str, Any] = {"conditions": []} + run_name = f"{name}-{generation}" + job = f"{run_name}-job" + if state in ("completed", "running"): + completed = state == "completed" + self.runs[run_name] = { + "metadata": { + "name": run_name, + "annotations": {RUN_GENERATION_ANNOTATION: str(generation)}, + }, + "status": { + "jobRef": {"name": job}, + "conditions": [ + { + "type": "Completed", + "status": "True" if completed else "False", + "reason": "SearchSucceeded" if completed else "JobRunning", + } + ], + }, + } + if completed: + status["recentRunRefs"] = [{"name": run_name}] + else: + status["activeRunRef"] = {"name": run_name} + self.job_pods[job] = [ + PodInfo( + name=f"{job}-{i}", + uid=str(i), + ip=None, + ready=False, + terminating=False, + phase=job_pod_phase, + ) + for i in range(job_pods) + ] + accepted = ("True", "RunCreated") + elif state == "invalid": + accepted = ("False", "InvalidSpec") + else: + accepted = ("False", "RunCreating") + status["activeRunRef"] = {"name": run_name} + status["conditions"].append( + { + "type": "Accepted", + "status": accepted[0], + "reason": accepted[1], + "observedGeneration": generation, + } + ) + self.requests[name] = { + "metadata": { + "name": name, + "uid": f"{name}-uid", + "generation": generation, + "labels": labels, + }, + "spec": { + "workload": { + "trace": { + "format": "dynamo", + "source": { + "pvc": { + "claimName": claim, + "path": path or f"windows/dgd/{window}.jsonl.gz", + } + }, + } + } + }, + "status": status, + } + + def finish_job(self, name: str) -> None: + for run in self.runs.values(): + if run["metadata"]["name"].startswith(name): + self.job_pods[run["status"]["jobRef"]["name"]] = [] + + +class RecordingMetrics: + def __init__(self) -> None: + self.sinks: list[tuple[str, bool]] = [] + self.fail_sinks = False + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + if self.fail_sinks: + raise RuntimeError("metrics backend failed") + self.sinks.append((pod, missing)) + + def __getattr__(self, name: str): + return lambda *args, **kwargs: None + + +def make( + tmp_path: Path, *, keep_requests: int = 2 +) -> tuple[Reconciler, FakeKube, RecordingMetrics]: + cfg = DgdrExecutorConfig( + claim_name="traces", + model="Qwen/Qwen3-0.6B", + sku="h200_sxm", + engine={}, + cut_budget_seconds=900, + keep_requests=keep_requests, + ) + retune = RetuneConfig( + interval_seconds=INTERVAL, + grace_seconds=GRACE, + timeout_seconds=TIMEOUT, + executor="dgdr", + dgdr=cfg, + ) + kube = FakeKube() + metrics = RecordingMetrics() + reconciler = Reconciler( + kube, + DGD, + cfg, + retune, + str(tmp_path), + metrics, + now=lambda: NOW, + visibility_margin_s=MARGIN, + ) + return reconciler, kube, metrics + + +def touch(path: Path, age_s: float, content: bytes = b"x") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + os.utime(path, (NOW - age_s, NOW - age_s)) + return path + + +def age_dir(path: Path, age_s: float) -> None: + os.utime(path, (NOW - age_s, NOW - age_s)) + + +def frontend(name: str, *, ready: bool = True, terminating: bool = False) -> PodInfo: + return PodInfo( + name=name, + uid=f"{name}-uid", + ip="10.0.0.1", + ready=ready, + terminating=terminating, + ) + + +def run_once( + reconciler: Reconciler, + *, + last: Optional[int] = None, + due: Optional[int] = None, + pods: Optional[list[PodInfo]] = None, + should_continue: Optional[Callable[[], bool]] = None, +): + return reconciler.run_once( + last_window_end_ms=last, + due_window_end_ms=due, + deadline_for=deadline_for, + frontend_pods=pods, + should_continue=should_continue, + ) + + +# ----------------------------------------------------------------- segments + + +def test_segments_of_a_live_pod_keep_the_newest_and_recent_ones(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + pod_dir = Path(reconciler.segments_root) / "fe-0" + old0 = touch(pod_dir / "req.000000.jsonl.gz", OLD) + recent = touch(pod_dir / "req.000001.jsonl.gz", SEGMENT_AGE - 60) + old2 = touch(pod_dir / "req.000002.jsonl.gz", OLD) + newest = touch(pod_dir / "req.000010.jsonl.gz", OLD) # numeric, not lexical + other = touch(pod_dir / "notes.txt", OLD) + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_segments == 2 + assert not old0.exists() and not old2.exists() + assert recent.exists() and newest.exists() and other.exists() + assert report.errors == {} + + +def test_directory_of_a_gone_pod_goes_only_when_everything_is_old( + tmp_path: Path, +) -> None: + reconciler, _, _ = make(tmp_path) + root = Path(reconciler.segments_root) + touch(root / "gone-old" / "req.000000.jsonl.gz", OLD) + touch(root / "gone-old" / "req.000001.jsonl.gz", SEGMENT_AGE + 1) + age_dir(root / "gone-old", OLD) + touch(root / "gone-recent" / "req.000000.jsonl.gz", OLD) + touch(root / "gone-recent" / "req.000001.jsonl.gz", 10) + age_dir(root / "gone-recent", OLD) + touch(root / "fe-0" / "req.000000.jsonl.gz", 10) + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_pod_dirs == ("gone-old",) + assert not (root / "gone-old").exists() + assert (root / "gone-recent").exists() + assert (root / "fe-0").exists() + + +@pytest.mark.parametrize("pods", [None, []]) +def test_no_segment_is_deleted_without_a_known_pod_set(tmp_path: Path, pods) -> None: + reconciler, _, _ = make(tmp_path) + root = Path(reconciler.segments_root) + seg = touch(root / "fe-0" / "req.000000.jsonl.gz", OLD) + touch(root / "fe-0" / "req.000001.jsonl.gz", OLD) + age_dir(root / "fe-0", OLD) + + report = run_once(reconciler, pods=pods) + + assert report.deleted_segments == 0 and report.deleted_pod_dirs == () + assert seg.exists() + + +def test_trace_sink_missing_is_reported_and_cleared(tmp_path: Path) -> None: + reconciler, _, metrics = make(tmp_path) + root = Path(reconciler.segments_root) + touch(root / "fe-0" / "req.000000.jsonl.gz", 0) + pods = [ + frontend("fe-0"), + frontend("fe-1"), + frontend("starting", ready=False), + frontend("leaving", terminating=True), + ] + + report = run_once(reconciler, pods=pods) + assert report.sink_missing == ("fe-1",) + assert sorted(metrics.sinks) == [("fe-0", False), ("fe-1", True)] + + metrics.sinks.clear() + (root / "fe-1").mkdir() + report = run_once(reconciler, pods=pods) + assert report.sink_missing == () + assert ("fe-1", False) in metrics.sinks + + # A pod that leaves has its gauge cleared. + metrics.sinks.clear() + run_once(reconciler, pods=[frontend("fe-0")]) + assert sorted(metrics.sinks) == [("fe-0", False), ("fe-1", False)] + + # An unknown pod set reports nothing. + metrics.sinks.clear() + assert run_once(reconciler, pods=None).sink_missing == () + assert metrics.sinks == [] + + +@pytest.mark.skipif( + hasattr(os, "geteuid") and os.geteuid() == 0, + reason="root reads a directory whatever its mode", +) +def test_an_unreadable_pod_directory_does_not_stop_the_others(tmp_path: Path) -> None: + reconciler, kube, metrics = make(tmp_path, keep_requests=0) + root = Path(reconciler.segments_root) + names = [f"fe-{i}" for i in range(6)] + for name in names: + touch(root / name / "req.000000.jsonl.gz", OLD) + touch(root / name / "req.000001.jsonl.gz", OLD) + touch(root / name / "req.000002.jsonl.gz", OLD) + kube.add_request("rt-old", window_ms(6), state="completed") + # Created by a frontend running as another uid, or with umask 077. + locked = root / "fe-3" + locked.chmod(0) + try: + report = run_once( + reconciler, due=window_ms(0), pods=[frontend(name) for name in names] + ) + finally: + locked.chmod(stat.S_IRWXU) + + # Every other pod lost its two old segments, whatever the scandir order. + assert report.deleted_segments == 2 * (len(names) - 1) + assert len(list(locked.iterdir())) == 3 + assert report.errors == {"segments": 1} + # The sink check and the Request GC still ran. + assert sorted(metrics.sinks) == [(name, False) for name in names] + assert report.deleted_requests == ("rt-old",) + + +def test_a_failing_sink_check_keeps_the_segment_counts(tmp_path: Path) -> None: + reconciler, _, metrics = make(tmp_path) + pod_dir = Path(reconciler.segments_root) / "fe-0" + touch(pod_dir / "req.000000.jsonl.gz", OLD) + touch(pod_dir / "req.000001.jsonl.gz", OLD) + metrics.fail_sinks = True + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_segments == 1 + assert report.errors == {"sinks": 1} + + +def test_only_the_top_level_segments_of_a_live_pod_are_collected( + tmp_path: Path, +) -> None: + # P10 requires the sink's file prefix directly under the pod directory, + # so the reconciler does not walk subdirectories of a live pod. + reconciler, _, _ = make(tmp_path) + pod_dir = Path(reconciler.segments_root) / "fe-0" + nested = [touch(pod_dir / "sub" / f"req.00000{i}.jsonl.gz", OLD) for i in range(3)] + top = [touch(pod_dir / f"req.00000{i}.jsonl.gz", OLD) for i in range(3)] + + report = run_once(reconciler, pods=[frontend("fe-0")]) + + assert report.deleted_segments == 2 + assert [path.exists() for path in top] == [False, False, True] + assert all(path.exists() for path in nested) + + +# ------------------------------------------------------------------ windows + + +def window_path(reconciler: Reconciler, window: int) -> Path: + return Path(reconciler.windows_root) / window_file_name(window) + + +def test_protected_windows_are_never_deleted(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + last, due = window_ms(2), int((NOW - 60) * 1000) + assert NOW >= deadline_for(last) and NOW < deadline_for(due) + last_path = touch(window_path(reconciler, last), OLD) + due_path = touch(window_path(reconciler, due), OLD) + stale = window_ms(5) + stale_path = touch(window_path(reconciler, stale), OLD) + registry_window = window_ms(7) + registry_path = touch(window_path(reconciler, registry_window), OLD) + reconciler.protect(registry_window, []) + + report = run_once(reconciler, last=last, due=due, pods=[frontend("fe-0")]) + + assert due_path.exists() # undecided and before its deadline + assert not last_path.exists() # in the range, but past its deadline + assert registry_path.exists() # this process's sweep uses it + assert not stale_path.exists() + assert report.deleted_windows == tuple(sorted((last, stale))) + + reconciler.release(registry_window) + report = run_once(reconciler, last=last, due=due) + assert report.deleted_windows == (registry_window,) + assert due_path.exists() + + +def test_range_protection_uses_the_window_deadline(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + # Two undecided windows, both before their deadline. + young = int((NOW - 60) * 1000) + older = int((NOW - 1000) * 1000) + paths = [touch(window_path(reconciler, w), OLD) for w in (young, older)] + + run_once(reconciler, last=older, due=young) + assert all(path.exists() for path in paths) + + # Without a persisted last window, every window before the due one with + # a live deadline is protected too. + run_once(reconciler, last=None, due=young) + assert all(path.exists() for path in paths) + + +def test_referenced_window_goes_once_every_referrer_is_finished(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + # A window outside the protected range whose deadline has not passed, + # so only the referrer rule decides. + window = int((NOW - 60) * 1000) + path = touch(window_path(reconciler, window), 10) # young: no fallback + kube.add_request("rt-a", window, state="completed") + kube.add_request("rt-b", window, state="running", job_pods=1) + + def round_(): + return run_once(reconciler, last=window_ms(0), due=window_ms(0)) + + round_() + assert path.exists() # rt-b's Run is still active + + # Completed, but the Job pod is still there. + kube.add_request("rt-b", window, state="completed", job_pods=1) + round_() + assert path.exists() + + kube.finish_job("rt-b") + report = round_() + assert report.deleted_windows == (window,) + assert not path.exists() + assert kube.deleted == [] # the Requests stay (keep_requests) + + +@pytest.mark.parametrize( + ("phase", "deleted"), + [("Succeeded", True), ("Failed", True), ("Running", False), (None, False)], +) +def test_finished_job_pods_of_a_completed_run_release_its_window( + tmp_path: Path, phase: Optional[str], deleted: bool +) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = int((NOW - 60) * 1000) + path = touch(window_path(reconciler, window), 10) # young: no fallback + kube.add_request("rt-a", window, state="completed", job_pods=1, job_pod_phase=phase) + + report = run_once(reconciler, last=window_ms(0), due=window_ms(0)) + assert (report.deleted_windows == (window,)) == deleted + assert path.exists() != deleted + + +def test_invalid_spec_request_does_not_hold_its_window(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = window_ms(4) + path = touch(window_path(reconciler, window), 10) + kube.add_request("rt-a", window, state="invalid") + + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_windows == (window,) + assert not path.exists() + + +def test_request_waiting_for_its_run_holds_its_window(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = window_ms(4) + path = touch(window_path(reconciler, window), OLD) + kube.add_request("rt-a", window, state="creating") + assert path.exists() + # The deadline has passed, so the Request itself is deleted; the window + # file then has no referrer and is old. + report = run_once(reconciler, due=window_ms(0)) + assert kube.deleted == ["rt-a"] + assert report.deleted_windows == (window,) + assert not path.exists() + + reconciler2, kube2, _ = make(tmp_path / "other", keep_requests=10) + young_window = int((NOW - 60) * 1000) + young_path = touch(window_path(reconciler2, young_window), OLD) + kube2.add_request("rt-b", young_window, state="creating") + report = run_once(reconciler2, due=None, last=None) + assert young_path.exists() + assert kube2.deleted == [] + + +def test_unreferenced_window_needs_the_fallback_age(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + young = touch(window_path(reconciler, window_ms(5)), WINDOW_AGE - 60) + old = touch(window_path(reconciler, window_ms(6)), WINDOW_AGE + 60) + unknown = touch(Path(reconciler.windows_root) / "README", OLD) + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_windows == (window_ms(6),) + assert young.exists() and not old.exists() and unknown.exists() + + +def test_windows_are_kept_when_requests_cannot_be_listed(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path) + path = touch(window_path(reconciler, window_ms(6)), OLD) + temp = touch(Path(reconciler.windows_root) / f".tmp-{window_ms(6)}-abc", OLD) + kube.fail_list = True + + report = run_once(reconciler, due=window_ms(0)) + + assert path.exists() + assert not temp.exists() # temp files do not depend on references + assert report.errors == {"list_requests": 1} + + +def test_a_request_of_another_claim_does_not_reference_the_window( + tmp_path: Path, +) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = window_ms(6) + path = touch(window_path(reconciler, window), OLD) + kube.add_request("rt-a", window, state="running", claim="other-claim") + kube.add_request( + "rt-b", window, state="running", path=f"windows/other-dgd/{window}.jsonl.gz" + ) + # Both Requests are past their deadline and get deleted anyway; the + # window had no referrer and is old. + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_windows == (window,) + assert not path.exists() + + +def test_temp_files(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + root = Path(reconciler.windows_root) + protected = window_ms(1) + reconciler.protect(protected, []) + old = touch(root / f".tmp-{window_ms(3)}-a1", TEMP_AGE + 1) + young = touch(root / f".tmp-{window_ms(3)}-b2", TEMP_AGE - 1) + in_use = touch(root / f".tmp-{protected}-c3", OLD) + odd = touch(root / ".tmp-garbage", TEMP_AGE + 1) + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_temp_files == 2 + assert not old.exists() and not odd.exists() + assert young.exists() and in_use.exists() + + +# ----------------------------------------------------------------- requests + + +def test_terminal_requests_outside_the_newest_kept_windows_go(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=2) + windows = [window_ms(k) for k in (5, 4, 3)] + for i, window in enumerate(windows): + kube.add_request(f"rt-{i}-a", window, state="completed") + kube.add_request(f"rt-{i}-b", window, state="invalid") + + report = run_once(reconciler, due=window_ms(0)) + + assert kube.selectors == [f"{LABEL_DGD}=dgd"] + assert report.deleted_requests == ("rt-0-a", "rt-0-b") + assert set(kube.requests) == {"rt-1-a", "rt-1-b", "rt-2-a", "rt-2-b"} + + +def test_keep_zero_still_keeps_requests_of_an_undecided_window(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + due = int((NOW - 60) * 1000) + kube.add_request("rt-due", due, state="completed") + kube.add_request("rt-old", window_ms(4), state="completed") + + report = run_once(reconciler, last=window_ms(4), due=due) + + # rt-due's window is undecided and before its deadline: the next sweep + # adopts its Succeeded Request instead of recreating it. + assert report.deleted_requests == ("rt-old",) + + +def test_non_terminal_request_goes_after_its_deadline(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + current = int((NOW - 60) * 1000) + kube.add_request("rt-live", current, state="running", job_pods=1) + kube.add_request("rt-late", window_ms(2), state="running", job_pods=1) + + report = run_once(reconciler, due=current) + + assert report.deleted_requests == ("rt-late",) + + +def test_requests_of_a_recreated_dgd_are_deleted(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + current = int((NOW - 60) * 1000) + kube.add_request("rt-old-uid", current, dgd_uid="uid-before", state="running") + kube.add_request("rt-no-uid", current, dgd_uid=None, state="running") + kube.add_request("rt-ours", current, state="running") + + report = run_once(reconciler, due=current) + + assert report.deleted_requests == ("rt-old-uid",) + + +def test_delete_queue(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + current = int((NOW - 60) * 1000) + kube.add_request("rt-a", current, state="running") + kube.add_request("rt-b", current, state="running") + reconciler.enqueue_delete(["rt-a", "rt-b", "rt-missing"]) + kube.fail_delete = {"rt-b"} + + report = run_once(reconciler, due=current) + + assert report.deleted_requests == ("rt-a", "rt-missing") + assert report.errors == {"delete_request": 1} + assert reconciler.pending_deletes() == frozenset({"rt-b"}) + + kube.fail_delete = set() + report = run_once(reconciler, due=current) + assert report.deleted_requests == ("rt-b",) + assert reconciler.pending_deletes() == frozenset() + + +def test_the_delete_queue_is_processed_when_listing_fails(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path) + reconciler.enqueue_delete(["rt-a"]) + kube.fail_list = True + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_requests == ("rt-a",) + + +def test_protected_requests_are_never_deleted(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + old = window_ms(6) + kube.add_request("rt-terminal", old, state="completed") + kube.add_request("rt-late", old, state="running") + kube.add_request("rt-other-uid", old, dgd_uid="uid-before", state="running") + reconciler.protect(old, ["rt-terminal", "rt-late", "rt-other-uid"]) + reconciler.enqueue_delete(["rt-late"]) + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_requests == () + assert kube.deleted == [] + assert reconciler.pending_deletes() == frozenset({"rt-late"}) + assert reconciler.protected_requests() == frozenset( + {"rt-terminal", "rt-late", "rt-other-uid"} + ) + + reconciler.release(old) + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_requests == ("rt-late", "rt-other-uid", "rt-terminal") + + +def test_unreadable_run_state_keeps_the_request(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + kube.add_request("rt-a", window_ms(6), state="completed") + window = touch(window_path(reconciler, window_ms(6)), 10) + kube.fail_get_run = True + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_requests == () + assert report.deleted_windows == () + assert window.exists() + assert report.errors == {"get_run": 1} + + +def test_terminal_and_finished_verdicts_are_cached(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + window = window_ms(6) + kube.add_request("rt-a", window, state="completed") + touch(window_path(reconciler, window), 10) + + run_once(reconciler, due=window_ms(0)) + calls = len(kube.get_run_calls) + assert calls > 0 + touch(window_path(reconciler, window), 10) # recreated: still finished + report = run_once(reconciler, due=window_ms(0)) + assert len(kube.get_run_calls) == calls + assert report.deleted_windows == (window,) + + +def test_deleted_requests_stop_referencing_their_window(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + old_window, young_window = window_ms(6), window_ms(7) + old = touch(window_path(reconciler, old_window), WINDOW_AGE + 60) + young = touch(window_path(reconciler, young_window), WINDOW_AGE - 60) + kube.add_request("rt-old", old_window, state="running") + kube.add_request("rt-young", young_window, state="running") + + report = run_once(reconciler, due=window_ms(0)) + + assert report.deleted_requests == ("rt-old", "rt-young") + assert report.deleted_windows == (old_window,) + assert not old.exists() and young.exists() + + +# ------------------------------------------------------------ early stop + + +def one_of_each(tmp_path: Path) -> tuple[Reconciler, FakeKube]: + """Six deletable things: 2 segments, a pod dir, a temp file, a Request + and the window file it referenced.""" + + reconciler, kube, _ = make(tmp_path, keep_requests=0) + root = Path(reconciler.segments_root) + for i in range(3): + touch(root / "fe-0" / f"req.00000{i}.jsonl.gz", OLD) + touch(root / "gone" / "req.000000.jsonl.gz", OLD) + age_dir(root / "gone", OLD) + touch(Path(reconciler.windows_root) / f".tmp-{window_ms(9)}-a1", OLD) + touch(window_path(reconciler, window_ms(6)), OLD) + kube.add_request("rt-a", window_ms(6), state="completed") + return reconciler, kube + + +def deletions(report) -> int: + return ( + report.deleted_segments + + len(report.deleted_pod_dirs) + + len(report.deleted_windows) + + report.deleted_temp_files + + len(report.deleted_requests) + ) + + +def test_should_continue_is_asked_before_every_delete(tmp_path: Path) -> None: + reconciler, _ = one_of_each(tmp_path) + calls: list[int] = [] + + def should_continue() -> bool: + calls.append(1) + return True + + report = run_once( + reconciler, + due=window_ms(0), + pods=[frontend("fe-0")], + should_continue=should_continue, + ) + + assert deletions(report) == 6 + assert len(calls) == 6 + assert not report.stopped + + +def test_a_no_from_should_continue_ends_the_round(tmp_path: Path) -> None: + reconciler, kube = one_of_each(tmp_path) + answers = [True, True] + + def should_continue() -> bool: + return answers.pop(0) if answers else False + + report = run_once( + reconciler, + due=window_ms(0), + pods=[frontend("fe-0")], + should_continue=should_continue, + ) + + # Two segment deletions, then nothing: no Request was even listed. + assert deletions(report) == 2 + assert report.stopped + assert kube.selectors == [] and kube.deleted == [] + assert window_path(reconciler, window_ms(6)).exists() + + report = run_once(reconciler, due=window_ms(0), pods=[frontend("fe-0")]) + assert deletions(report) == 4 + assert not report.stopped + + +def test_a_failing_should_continue_ends_the_round(tmp_path: Path) -> None: + reconciler, kube = one_of_each(tmp_path) + + def should_continue() -> bool: + raise RuntimeError("lease state unknown") + + report = run_once( + reconciler, + due=window_ms(0), + pods=[frontend("fe-0")], + should_continue=should_continue, + ) + + assert deletions(report) == 0 + assert report.stopped + assert kube.deleted == [] + + +# --------------------------------------------------------------- set_dgd + + +RECREATED = DgdRef( + name=DGD.name, namespace=DGD.namespace, uid="uid-new", dynamo_namespace="dyn-ns" +) + + +def test_set_dgd_follows_a_recreated_dgd(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=2) + current = int((NOW - 60) * 1000) + # The new DGD's Requests: one kept as a recent terminal window, one + # still running before its deadline. + kube.add_request("new-w1-done", window_ms(1), dgd_uid="uid-new") + kube.add_request("new-w0-running", current, dgd_uid="uid-new", state="running") + kube.add_request("old-w2", window_ms(2), dgd_uid=DGD.uid, state="running") + + reconciler.set_dgd(RECREATED) + report = run_once(reconciler, due=current) + + assert reconciler.dgd == RECREATED + assert report.deleted_requests == ("old-w2",) + assert set(kube.requests) == {"new-w1-done", "new-w0-running"} + + +def test_set_dgd_clears_the_request_state_caches(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + kube.add_request("rt-a", window_ms(3), state="completed") + + run_once(reconciler, due=window_ms(0)) + calls = len(kube.get_run_calls) + assert calls > 0 + run_once(reconciler, due=window_ms(0)) + assert len(kube.get_run_calls) == calls # cached terminal verdict + + reconciler.set_dgd(DGD) + run_once(reconciler, due=window_ms(0)) + assert len(kube.get_run_calls) > calls + + +def test_set_dgd_during_a_round_stops_it(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path) + reconciler.enqueue_delete(["rt-a", "rt-b", "rt-c"]) + kube.on_delete = lambda name: reconciler.set_dgd(RECREATED) + + report = run_once(reconciler, due=window_ms(0)) + + # It classified Requests by the old uid: nothing after the change. + assert report.deleted_requests == ("rt-a",) + assert report.stopped + assert reconciler.pending_deletes() == frozenset({"rt-b", "rt-c"}) + + +def test_set_dgd_with_a_new_name_moves_the_roots(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + renamed = DgdRef(name="other", namespace="ns", uid="u2", dynamo_namespace="d") + + reconciler.set_dgd(renamed) + + assert reconciler.segments_root == str(tmp_path / "segments" / "other") + assert reconciler.windows_root == str(tmp_path / "windows" / "other") + + +# ------------------------------------------------------------------ registry + + +def test_registry_is_reference_counted_per_window(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + reconciler.protect(5, []) + reconciler.protect(5, ["rt-a"]) + reconciler.protect(5, ["rt-b"]) + reconciler.protect(6, ["rt-c"]) + assert reconciler.protected_windows() == frozenset({5, 6}) + assert reconciler.protected_requests() == frozenset({"rt-a", "rt-b", "rt-c"}) + + # Every protect call holds one reference; the names stay the union of + # everything given to the window while any reference is left. + reconciler.release(5) + reconciler.release(5) + assert reconciler.protected_windows() == frozenset({5, 6}) + assert reconciler.protected_requests() == frozenset({"rt-a", "rt-b", "rt-c"}) + reconciler.release(5) + assert reconciler.protected_windows() == frozenset({6}) + assert reconciler.protected_requests() == frozenset({"rt-c"}) + + reconciler.release(5) # already released: no-op, never negative + reconciler.release(99) # unknown: no-op + assert reconciler.protected_windows() == frozenset({6}) + reconciler.protect(5, ["rt-d"]) + assert reconciler.protected_requests() == frozenset({"rt-c", "rt-d"}) + reconciler.release(5) + assert reconciler.protected_windows() == frozenset({6}) + assert reconciler.protected_requests() == frozenset({"rt-c"}) + + +def test_an_old_sweeps_release_keeps_a_relaunched_sweeps_protection( + tmp_path: Path, +) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + window = window_ms(6) + path = touch(window_path(reconciler, window), OLD) + kube.add_request("rt-old", window, state="running") + kube.add_request("rt-new", window, state="running") + # A rebuilt controller relaunches the window while the aborted sweep + # of the previous generation is still winding down. + reconciler.protect(window, ["rt-old"]) + reconciler.protect(window, ["rt-new"]) + reconciler.release(window) # the aborted sweep's finally + + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_requests == () + assert report.deleted_windows == () + assert path.exists() + + reconciler.release(window) # the relaunched sweep ends + report = run_once(reconciler, due=window_ms(0)) + assert report.deleted_requests == ("rt-new", "rt-old") + assert report.deleted_windows == (window,) + + +def test_registry_is_thread_safe_during_rounds(tmp_path: Path) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=0) + for k in range(10): + kube.add_request(f"rt-{k}", window_ms(20 + k), state="completed") + errors: list[BaseException] = [] + stop = threading.Event() + + def churn(offset: int) -> None: + try: + i = 0 + while not stop.is_set(): + window = 1_000 * offset + i % 50 + reconciler.protect(window, [f"x-{offset}-{i}"]) + reconciler.enqueue_delete([]) + reconciler.release(window) + i += 1 + except BaseException as exc: # surfaced below + errors.append(exc) + + threads = [threading.Thread(target=churn, args=(n,)) for n in range(4)] + for thread in threads: + thread.start() + try: + for _ in range(20): + run_once(reconciler, due=window_ms(0)) + finally: + stop.set() + for thread in threads: + thread.join() + assert errors == [] + assert reconciler.protected_windows() == frozenset() + assert kube.requests == {} + + +# ------------------------------------------------------------------ helpers + + +def test_window_relpath_matches_the_reconciler_layout(tmp_path: Path) -> None: + reconciler, _, _ = make(tmp_path) + cfg = DgdrExecutorConfig( + claim_name="traces", model="m", sku="s", engine={}, windows_dir="/w/x/" + ) + assert window_relpath(cfg, "dgd", 1234) == "w/x/dgd/1234.jsonl.gz" + assert reconciler.windows_root == str(tmp_path / "windows" / "dgd") + assert reconciler.segments_root == str(tmp_path / "segments" / "dgd") diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py new file mode 100644 index 000000000000..6587cd9afede --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -0,0 +1,1253 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the live retune window cutter. + +The golden tests compare the cut window, decompressed, byte for byte with +``read_trace_window`` over the same clean segments. A few tests spawn the +real child process; they carry a timeout. +""" + +import errno +import gzip +import io +import json +import os +import pickle +import random +import stat +import threading +import time + +import pytest + +from dynamo.planner.retune.live import window_cut +from dynamo.planner.retune.live.contracts import WindowCutResult +from dynamo.planner.retune.live.window_cut import ( + MAX_DROPPED_MEMBER_RATIO, + WINDOW_CUT_FAILURES, + WindowCutError, + _decode_exact_member, + _Garbage, + _GzipMemberReader, + _Member, + _run_in_child, + _segment_files, + cut_window, + cut_window_inprocess, + segment_index, + temp_file_name, + temp_file_window_end, +) +from dynamo.planner.retune.window import ROW_COUNT_KEYS, read_trace_window + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +SCHEMA = "dynamo.request.trace.v1" +START, END, AS_OF = 5_000, 15_000, 15_500 +_UNSET = object() + + +# ------------------------------------------------------------------ helpers + + +def _row( + request_id, + received_ms=_UNSET, + *, + total_ms=10.0, + event_ms=None, + isl=128, + osl=8, + block_size=64, + model="m", + agent_context=None, + output_tokens=_UNSET, +): + request = {"request_id": request_id} + if model is not None: + request["model"] = model + if received_ms is not _UNSET and received_ms is not None: + request["request_received_ms"] = received_ms + if total_ms is not None: + request["total_time_ms"] = total_ms + if output_tokens is not None: + request["output_tokens"] = osl if output_tokens is _UNSET else output_tokens + request["replay"] = { + "trace_block_size": block_size, + "input_length": isl, + "input_sequence_hashes": [len(request_id), 7, 11_400_714_819_323_198_485], + } + if event_ms is None: + base = received_ms if isinstance(received_ms, int) else 0 + event_ms = base + (round(total_ms) if total_ms is not None else 0) + record = { + "schema": SCHEMA, + "event_type": "request_end", + "event_time_unix_ms": event_ms, + "request": request, + } + if agent_context is not None: + record["agent_context"] = agent_context + return record + + +def _envelope(record, timestamp=0): + # The frontend's gzip sink wraps every record like this. + return {"timestamp": timestamp, "event": record} + + +def _lines(records): + return b"".join(json.dumps(record).encode("utf-8") + b"\n" for record in records) + + +def _member(records): + return gzip.compress(_lines(records)) + + +def _write(path, *chunks): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"".join(chunks)) + return path + + +def _out(tmp_path, name="w.jsonl.gz"): + return tmp_path / "windows" / "dgd" / name + + +def _cut(root, tmp_path, start=START, end=END, as_of=AS_OF, *, max_rows=10**6): + out = _out(tmp_path) + return cut_window_inprocess(root, start, end, as_of, out, max_rows=max_rows), out + + +def _golden(root, tmp_path, start=START, end=END, as_of=AS_OF): + out = tmp_path / "golden" / "window.jsonl" + window = read_trace_window([str(root)], start, end, as_of, out) + return window, out.read_bytes() + + +def _assert_golden(result, out, window, golden_bytes): + assert isinstance(result, WindowCutResult) + assert result.num_requests == window.num_requests + assert result.trace_block_size == window.trace_block_size + assert result.row_counts == window.row_counts + assert set(result.row_counts) == set(ROW_COUNT_KEYS) + if window.num_requests: + assert result.path == str(out) + assert gzip.decompress(out.read_bytes()) == golden_bytes + else: + assert result.path is None + assert not out.exists() + + +def _temp_files(directory): + if not directory.exists(): + return [] + return [name for name in os.listdir(directory) if name.startswith(".tmp-")] + + +def _random_trace(root, seed, *, pods=3, segments=3, members=5, plain_pod=False): + """Segments with every row shape ``read_trace_window`` handles. + + Arrivals and durations are coarse so (arrival, completion) pairs tie + across files; some request IDs repeat (first in file order wins), some + are not ASCII. + """ + + rng = random.Random(seed) + issued = [] + serial = 0 + for pod in range(pods): + for segment in range(segments): + chunks = [] + for _ in range(members): + records = [] + for _ in range(rng.randint(0, 14)): + roll = rng.random() + if roll < 0.04: + records.append( + { + "schema": SCHEMA, + "event_type": rng.choice(["tool_start", "tool_end"]), + "event_time_unix_ms": rng.randrange(20_000), + "agent_context": {"session_id": "s"}, + } + ) + continue + if issued and roll < 0.16: + request_id = rng.choice(issued) + else: + serial += 1 + prefix = rng.choice(["req-", "req-", "réq-", "請求-", "Req-"]) + request_id = f"{prefix}{serial % 97}-{serial}" + issued.append(request_id) + received = rng.randrange(0, 400) * 50 + total = rng.choice([10.0, 10.5, 49.5, 99.4, 600.0, 1_200.25]) + kwargs = {} + if rng.random() < 0.08: + kwargs["received_ms"] = None + kwargs["event_ms"] = received + round(total) + else: + kwargs["received_ms"] = received + if rng.random() < 0.05: + kwargs["total_ms"] = None + kwargs["event_ms"] = received + rng.randrange(0, 900) + else: + kwargs["total_ms"] = total + if rng.random() < 0.03: + kwargs["output_tokens"] = None + if rng.random() < 0.05: + kwargs["agent_context"] = {"session_id": f"s{serial}"} + if rng.random() < 0.1: + kwargs["model"] = None + received_arg = kwargs.pop("received_ms") + record = _row( + request_id, + received_arg, + isl=rng.randrange(1, 4096), + osl=rng.randrange(1, 512), + **kwargs, + ) + if rng.random() < 0.5: + record = _envelope(record, timestamp=rng.randrange(10**6)) + records.append(record) + payload = _lines(records) + if rng.random() < 0.1: + payload += b"\n \n" # blank lines are skipped + chunks.append(payload) + directory = root / f"frontend-{pod}" + if plain_pod and pod == 0: + _write(directory / f"req.{segment:06d}.jsonl", *chunks) + else: + _write( + directory / f"req.{segment:06d}.jsonl.gz", + *(gzip.compress(chunk) for chunk in chunks), + ) + return root + + +def _items(data): + return list(_GzipMemberReader(io.BytesIO(data))) + + +# ----------------------------------------------------------- gzip members + + +@pytest.fixture(params=["default", "tiny", "small"]) +def reader_chunks(request, monkeypatch): + """Run reader tests with reads and decode steps that split headers.""" + + sizes = {"default": None, "tiny": (7, 5, 16), "small": (64, 3, 128)}[request.param] + if sizes is not None: + read_chunk, decode_step, compact_at = sizes + monkeypatch.setattr(window_cut, "_READ_CHUNK", read_chunk) + monkeypatch.setattr(window_cut, "_DECODE_STEP", decode_step) + monkeypatch.setattr(window_cut, "_COMPACT_AT", compact_at) + return request.param + + +def _spans(*members): + offsets = [0] + for member in members: + offsets.append(offsets[-1] + len(member)) + return offsets + + +def test_clean_multi_member_file(reader_chunks): + payloads = [b'{"a":1}\n', b"", b'{"b":2}\n{"c":3}\n', b"x" * 5000 + b"\n"] + members = [gzip.compress(payload) for payload in payloads] + offsets = _spans(*members) + + items = _items(b"".join(members)) + + assert items == [ + _Member(offsets[i], offsets[i + 1], payloads[i]) for i in range(len(payloads)) + ] + + +@pytest.mark.parametrize("keep", ["header", "middle", "trailer"]) +def test_truncated_last_member_is_dropped(reader_chunks, keep): + first, last = gzip.compress(b"a\n" * 50), gzip.compress(b"b\n" * 50) + cut = {"header": 5, "middle": len(last) // 2, "trailer": len(last) - 3}[keep] + + items = _items(first + last[:cut]) + + assert items == [ + _Member(0, len(first), b"a\n" * 50), + _Garbage(len(first), len(first) + cut, at_eof=True), + ] + + +def test_partial_member_followed_by_complete_members(reader_chunks): + first = gzip.compress(b"first\n" * 40) + full = gzip.compress(b"".join(b"partial %d\n" % i for i in range(400))) + partial = full[: len(full) // 2] + third = gzip.compress(b"third\n") + fourth = gzip.compress(b"fourth\n") + offsets = _spans(first, partial, third, fourth) + + items = _items(first + partial + third + fourth) + + assert items == [ + _Member(offsets[0], offsets[1], b"first\n" * 40), + _Garbage(offsets[1], offsets[2], at_eof=False), + _Member(offsets[2], offsets[3], b"third\n"), + _Member(offsets[3], offsets[4], b"fourth\n"), + ] + + +@pytest.mark.parametrize("where", ["crc", "isize", "data"]) +def test_flipped_member_is_dropped_and_reader_resyncs(reader_chunks, where): + first = gzip.compress(b"first\n") + middle = bytearray(gzip.compress(b"middle row\n" * 30)) + index = {"crc": len(middle) - 8, "isize": len(middle) - 2, "data": 14}[where] + middle[index] ^= 0x40 + last = gzip.compress(b"last\n") + offsets = _spans(first, middle, last) + + items = _items(first + bytes(middle) + last) + + assert items == [ + _Member(offsets[0], offsets[1], b"first\n"), + _Garbage(offsets[1], offsets[2], at_eof=False), + _Member(offsets[2], offsets[3], b"last\n"), + ] + + +def test_zero_filled_hole_between_members(reader_chunks): + first, last = gzip.compress(b"first\n"), gzip.compress(b"last\n") + hole = bytes(4096) + + items = _items(first + hole + last) + + assert items == [ + _Member(0, len(first), b"first\n"), + _Garbage(len(first), len(first) + len(hole), at_eof=False), + _Member(len(first) + len(hole), len(first) + len(hole) + len(last), b"last\n"), + ] + + +def test_false_gzip_magic_inside_garbage_is_one_run(reader_chunks): + first, last = gzip.compress(b"first\n"), gzip.compress(b"last\n") + junk = b"\x1f\x8b\x08\x00junk" * 20 + b"\x1f\x8b" + + items = _items(first + junk + last) + + assert items == [ + _Member(0, len(first), b"first\n"), + _Garbage(len(first), len(first) + len(junk), at_eof=False), + _Member(len(first) + len(junk), len(first) + len(junk) + len(last), b"last\n"), + ] + + +def test_garbage_only_and_empty_files(reader_chunks): + assert _items(b"") == [] + assert _items(b"plain text, not gzip\n") == [_Garbage(0, 21, at_eof=True)] + member = gzip.compress(b"a\n") + assert _items(member + bytes(10)) == [ + _Member(0, len(member), b"a\n"), + _Garbage(len(member), len(member) + 10, at_eof=True), + ] + + +def test_reader_starts_at_an_offset(): + first, second = gzip.compress(b"first\n"), gzip.compress(b"second\n") + source = io.BytesIO(first + second) + + items = list(_GzipMemberReader(source, len(first))) + + assert items == [_Member(len(first), len(first) + len(second), b"second\n")] + + +def test_decode_exact_member(): + member = gzip.compress(b"row\n") + assert _decode_exact_member(member) == b"row\n" + assert _decode_exact_member(member + b"x") is None + assert _decode_exact_member(member[:-1]) is None + flipped = bytearray(member) + flipped[-6] ^= 1 + assert _decode_exact_member(bytes(flipped)) is None + assert _decode_exact_member(b"") is None + + +# ------------------------------------------------------------ golden cuts + + +@pytest.mark.parametrize("seed", [0, 1, 2]) +def test_cut_matches_read_trace_window(tmp_path, seed): + root = _random_trace(tmp_path / "segments" / "dgd", seed) + window, golden_bytes = _golden(root, tmp_path) + assert window.num_requests > 50 + assert window.row_counts["duplicate"] > 0 + assert window.row_counts["censored"] > 0 + + result, out = _cut(root, tmp_path) + + _assert_golden(result, out, window, golden_bytes) + assert result.dropped_members == 0 + assert result.dropped_bytes == 0 + assert not result.reused + assert stat.S_IMODE(out.stat().st_mode) == 0o600 + assert _temp_files(out.parent) == [] + + +def test_cut_matches_golden_with_plain_jsonl_files(tmp_path): + root = _random_trace(tmp_path / "segments" / "dgd", 3, plain_pod=True) + window, golden_bytes = _golden(root, tmp_path) + + result, out = _cut(root, tmp_path) + + _assert_golden(result, out, window, golden_bytes) + + +def test_cut_matches_golden_under_eviction_and_small_reads(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments" / "dgd", 4, members=8) + window, golden_bytes = _golden(root, tmp_path) + monkeypatch.setattr(window_cut, "_PASS2_CACHE_BYTES", 1) + monkeypatch.setattr(window_cut, "_PASS2_BLOCK_ROWS", 7) + monkeypatch.setattr(window_cut, "_READ_CHUNK", 97) + monkeypatch.setattr(window_cut, "_DECODE_STEP", 13) + monkeypatch.setattr(window_cut, "_COMPACT_AT", 64) + monkeypatch.setattr(window_cut, "_WRITE_BUFFER", 10) + monkeypatch.setattr(window_cut, "_OPEN_FILES", 2) + loads = [] + original_load = window_cut._MemberLines._load + + def counting_load(self, member_id): + loads.append(member_id) + return original_load(self, member_id) + + monkeypatch.setattr(window_cut._MemberLines, "_load", counting_load) + + result, out = _cut(root, tmp_path) + + _assert_golden(result, out, window, golden_bytes) + # Every member was evicted after each row and decoded again. + assert len(loads) == window.num_requests + assert len(set(loads)) < len(loads) + + +def test_dedup_compaction_keeps_first_rows_and_bounds_rows(tmp_path, monkeypatch): + rows = [_row(f"r{i}", START + 10 * i) for i in range(10)] + # Many repeats of the same IDs, in later files: the first row wins. + repeats = [_row(f"r{i % 10}", START + 5 + i, isl=999) for i in range(40)] + root = tmp_path / "segments" + _write(root / "a" / "req.000000.jsonl.gz", _member(rows)) + _write(root / "b" / "req.000000.jsonl.gz", *(_member([row]) for row in repeats)) + window, golden_bytes = _golden(root, tmp_path) + assert window.num_requests == 10 + monkeypatch.setattr(window_cut, "_DEDUP_SLACK_MIN", 1) + monkeypatch.setattr(window_cut, "_DEDUP_SLACK_RATIO", 0.0) + compactions = [] + original_dedup = window_cut._Selection._dedup + + def counting_dedup(self): + compactions.append(self.num_rows) + original_dedup(self) + + monkeypatch.setattr(window_cut._Selection, "_dedup", counting_dedup) + + result, out = _cut(root, tmp_path, max_rows=10) + + _assert_golden(result, out, window, golden_bytes) + assert result.row_counts["duplicate"] == 40 + assert len(compactions) > 2 # during pass 1, not only at the end + + +def test_too_many_rows_is_window_too_large(tmp_path, monkeypatch): + root = tmp_path / "segments" + rows = [_row(f"r{i}", START + i) for i in range(30)] + _write(root / "a" / "req.000000.jsonl.gz", _member(rows)) + monkeypatch.setattr(window_cut, "_DEDUP_SLACK_MIN", 1) + monkeypatch.setattr(window_cut, "_DEDUP_SLACK_RATIO", 0.0) + + result, _ = _cut(root, tmp_path, max_rows=30) + assert result.num_requests == 30 + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path, max_rows=29) + assert caught.value.reason == "window_too_large" + + +def test_empty_window_writes_no_file(tmp_path): + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("early", START - 100), _row("late", END + 1)]), + _member([_row("no-output", START + 1, output_tokens=None)]), + ) + window, golden_bytes = _golden(root, tmp_path) + + result, out = _cut(root, tmp_path) + + assert golden_bytes == b"" + _assert_golden(result, out, window, golden_bytes) + assert result.row_counts["missing_output_tokens"] == 1 + assert not out.parent.exists() + + +def test_empty_segments_root_is_an_empty_window(tmp_path): + root = tmp_path / "segments" + (root / "frontend-0").mkdir(parents=True) + + result, out = _cut(root, tmp_path) + + assert result.path is None + assert result.num_requests == 0 + assert result.trace_block_size is None + + +def test_missing_segments_root_fails(tmp_path): + with pytest.raises(WindowCutError) as caught: + _cut(tmp_path / "nowhere", tmp_path) + assert caught.value.reason == "window_cut_failed" + assert "does not exist" in str(caught.value) + + +def test_mixed_block_sizes_fail(tmp_path): + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("a", START, block_size=64), _row("b", START, block_size=16)]), + ) + with pytest.raises(ValueError, match="mixes trace_block_size"): + _golden(root, tmp_path) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "window_cut_failed" + assert "mixes trace_block_size values [16, 64]" in str(caught.value) + + +@pytest.mark.parametrize( + "line, message", + [ + (b"{not json", "invalid JSON"), + (b"[1, 2]", "must be a JSON object"), + (b'{"schema": "other", "event_type": "request_end"}', "expected"), + ( + json.dumps( + { + "schema": SCHEMA, + "event_type": "request_end", + "event_time_unix_ms": 1, + "request": { + "request_id": "x", + "output_tokens": 1, + "replay": {"input_length": 1, "trace_block_size": 64}, + }, + } + ).encode(), + "input_sequence_hashes", + ), + ], +) +def test_malformed_row_in_a_valid_member_is_trace_corrupt(tmp_path, line, message): + # The member's CRC is good: the writer wrote this, as replay would see it. + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("good", START)]), + gzip.compress(line + b"\n"), + ) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "trace_corrupt" + assert message in str(caught.value) + + +def test_out_of_range_value_is_trace_corrupt(tmp_path): + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("huge", START, block_size=2**70)]), + ) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "trace_corrupt" + + +# --------------------------------------------------- dropped gzip members + + +def _tiny_members(prefix, count, first_ms=START): + return [ + _member([_envelope(_row(f"{prefix}{i}", first_ms + i), timestamp=i)]) + for i in range(count) + ] + + +def _cut_and_twin(tmp_path, build): + """Cut ``build(bad=True)`` and the golden of ``build(bad=False)``. + + ``build(root, bad)`` writes the segments with or without the bad bytes; + the rows of a dropped member never reach the window. + """ + + clean = tmp_path / "clean" + build(clean, bad=False) + window, golden_bytes = _golden(clean, tmp_path) + damaged = tmp_path / "damaged" + build(damaged, bad=True) + result, out = _cut(damaged, tmp_path) + _assert_golden(result, out, window, golden_bytes) + return result + + +def test_in_flight_tail_of_the_newest_segment_is_not_corruption(tmp_path): + lost = _member([_row("in-flight", START + 900)]) + + def build(root, bad): + _write(root / "a" / "req.000000.jsonl.gz", *_tiny_members("old", 3)) + members = _tiny_members("new", 2, START + 100) + _write( + root / "a" / "req.000001.jsonl.gz", + *members, + *([lost[: len(lost) // 2]] if bad else []), + ) + + result = _cut_and_twin(tmp_path, build) + + # 1 of 6 members is far above the ratio, but it is the writer's tail. + assert result.dropped_members == 1 + assert result.dropped_bytes == len(lost) // 2 + + +def test_truncated_tail_of_an_older_segment_is_corruption(tmp_path): + root = tmp_path / "segments" + lost = _member([_row("lost", START + 900)]) + _write(root / "a" / "req.000000.jsonl.gz", *_tiny_members("old", 3), lost[:20]) + _write(root / "a" / "req.000001.jsonl.gz", *_tiny_members("new", 3)) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "trace_corrupt" + assert "1 of 7 gzip members" in str(caught.value) + assert "req.000000.jsonl.gz@" in str(caught.value) + + +@pytest.mark.parametrize("damage", ["partial", "crc", "hole"]) +def test_damaged_members_within_the_ratio_are_dropped(tmp_path, damage): + # 1 bad member among 100 is exactly the limit: the cut goes on. Its bytes + # are at most one member's, below the limit too. + count = int(1 / MAX_DROPPED_MEMBER_RATIO) - 1 + bad_row = _member([_row("bad", START + 50)]) + if damage == "partial": + damaged = bad_row[: len(bad_row) - 12] + elif damage == "crc": + flipped = bytearray(bad_row) + flipped[-7] ^= 0x10 + damaged = bytes(flipped) + else: + damaged = bytes(len(bad_row)) + assert len(damaged) <= min(map(len, _tiny_members("m", count))) + + def build(root, bad): + members = _tiny_members("m", count) + if bad: + members.insert(count // 2, damaged) + _write(root / "a" / "req.000000.jsonl.gz", *members) + + result = _cut_and_twin(tmp_path, build) + + assert result.dropped_members == 1 + assert result.dropped_bytes == len(damaged) + + +def test_damaged_members_above_the_ratio_are_trace_corrupt(tmp_path): + root = tmp_path / "segments" + members = _tiny_members("m", 98) + members.insert(10, bytes(64)) + members.insert(50, bytes(64)) + _write(root / "a" / "req.000000.jsonl.gz", *members) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "trace_corrupt" + assert "2 of 100 gzip members" in str(caught.value) + + +@pytest.mark.parametrize("newest", [True, False]) +def test_hole_spanning_many_members_is_trace_corrupt(tmp_path, newest): + # A zero-filled hole over 20 of 200 members is one run of bad bytes: 1 of + # 181 members is within the ratio, but 10% of the bytes is not. + root = tmp_path / "segments" + members = _tiny_members("m", 200) + hole = sum(map(len, members[50:70])) + data = b"".join([*members[:50], bytes(hole), *members[70:]]) + _write(root / "a" / "req.000000.jsonl.gz", data) + if not newest: + _write(root / "a" / "req.000001.jsonl.gz", *_tiny_members("next", 3)) + total_members = 181 + (0 if newest else 3) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + + assert caught.value.reason == "trace_corrupt" + message = str(caught.value) + assert f"1 of {total_members} gzip members" in message + assert f"({hole} of " in message + assert f"req.000000.jsonl.gz@{sum(map(len, members[:50]))}+{hole}" in message + + +def test_hole_within_the_byte_ratio_is_dropped(tmp_path): + # A hole over 1 of 200 members: 1 run, under 1% of the bytes. + members = _tiny_members("m", 200) + + def build(root, bad): + kept = [*members[:50], *members[51:]] + if bad: + kept.insert(50, bytes(len(members[50]))) + _write(root / "a" / "req.000000.jsonl.gz", *kept) + + result = _cut_and_twin(tmp_path, build) + + assert result.dropped_members == 1 + assert result.dropped_bytes == len(members[50]) + + +@pytest.mark.parametrize("plain_block", [None, 64]) +def test_plain_jsonl_unterminated_tail(tmp_path, monkeypatch, plain_block): + if plain_block is not None: + # Blocks of one line each: the tail is in a later block. + monkeypatch.setattr(window_cut, "_PLAIN_BLOCK", plain_block) + good = [_row("a", START), _row("b", START + 1)] + + def build(root, bad): + _write( + root / "a" / "req.000000.jsonl", + _lines(good), + *([b'{"schema": "dynamo.req'] if bad else []), + ) + + result = _cut_and_twin(tmp_path, build) + assert result.dropped_bytes == len(b'{"schema": "dynamo.req') + assert result.dropped_members == 0 + + # An unterminated last line that parses is a row, as replay reads it. + root = tmp_path / "parses" + _write( + root / "a" / "req.000000.jsonl", + _lines(good) + json.dumps(_row("c", START + 2)).encode(), + ) + window, golden_bytes = _golden(root, tmp_path) + assert window.num_requests == 3 + out = _out(tmp_path, "parses.jsonl.gz") + result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10) + _assert_golden(result, out, window, golden_bytes) + + +@pytest.mark.parametrize("tail", ["partial", "parses"]) +def test_plain_jsonl_stops_at_the_unterminated_tail(tmp_path, tail): + # The writer finishes the last line (and writes another) right after the + # first read: reading on would start inside that line. + path = tmp_path / "req.000000.jsonl" + first = _lines([_row("a", START)]) + if tail == "partial": + line = _lines([_row("b", START + 1)]) + written, rest = line[:20], line[20:] + else: + written, rest = json.dumps(_row("b", START + 1)).encode(), b"\n" + _write(path, first, written) + + class Appending: + def __init__(self, source): + self.source = source + self.reads = 0 + + def readlines(self, hint): + lines = self.source.readlines(hint) + self.reads += 1 + if self.reads == 1: + with open(path, "ab") as writer: + writer.write(rest + _lines([_row("c", START + 2)])) + return lines + + with open(path, "rb") as source: + reader = Appending(source) + blocks = list(window_cut._plain_blocks(reader)) + + assert reader.reads == 1 + assert len(blocks) == 1 + if tail == "partial": + assert blocks[0].payload == first + assert blocks[0].dropped == len(written) + else: + assert blocks[0].payload == first + written + assert blocks[0].dropped == 0 + assert blocks[0].end == len(blocks[0].payload) + selection = window_cut._Selection(START, END, AS_OF, 10) + selection.scan((0, str(path), True, block) for block in blocks) + assert selection.num_rows == (1 if tail == "partial" else 2) + + +def test_segment_files_order_and_newest(tmp_path): + root = tmp_path / "segments" + for name in ( + "b/req.000999.jsonl.gz", + "b/req.1000000.jsonl.gz", + "a/req.000000.jsonl.gz", + "a/req.000001.jsonl", + "a/notes.txt", + "c/single.jsonl.gz", + ): + _write(root / name, b"") + + files = _segment_files(str(root)) + + assert files == [ + (str(root / "a/req.000000.jsonl.gz"), False), + (str(root / "a/req.000001.jsonl"), True), + (str(root / "b/req.000999.jsonl.gz"), False), + (str(root / "b/req.1000000.jsonl.gz"), True), + (str(root / "c/single.jsonl.gz"), True), + ] + + +def test_compressed_and_plain_copies_of_one_segment_fail(tmp_path): + root = tmp_path / "segments" + _write(root / "a" / "req.000000.jsonl.gz", _member([_row("a", START)])) + _write(root / "a" / "req.000000.jsonl", _lines([_row("a", START)])) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "window_cut_failed" + + +def test_vanished_file_is_skipped(tmp_path, monkeypatch): + root = tmp_path / "segments" + _write(root / "a" / "req.000000.jsonl.gz", _member([_row("a", START)])) + listed = _segment_files(str(root)) + gone = str(root / "a" / "req.000001.jsonl.gz") + monkeypatch.setattr(window_cut, "_segment_files", lambda _: [*listed, (gone, True)]) + + result, _ = _cut(root, tmp_path) + + assert result.num_requests == 1 + + +def _two_pods(root): + for pod in ("a", "b"): + _write( + root / pod / "req.000000.jsonl.gz", + _member([_row(f"{pod}-{i}", START + i) for i in range(5)]), + ) + return root + + +def _skip_as_root(): + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root ignores directory permissions") + + +@pytest.mark.parametrize("mode", [0o000, 0o600]) +def test_unreadable_pod_directory_fails(tmp_path, mode): + # 0o000 cannot be listed; 0o600 can, but its files cannot be stat-ed. + # Either way that pod's rows would be missing from the window. + _skip_as_root() + root = _two_pods(tmp_path / "segments") + pod = root / "b" + pod.chmod(mode) + try: + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + finally: + pod.chmod(0o755) + + assert caught.value.reason == "window_cut_failed" + assert str(pod) in str(caught.value) + assert "PermissionError" in str(caught.value) + + +def test_unreadable_segments_root_fails(tmp_path): + # Not an empty window: the cut cannot see any segment. + _skip_as_root() + root = _two_pods(tmp_path / "segments") + root.chmod(0o000) + try: + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + finally: + root.chmod(0o755) + + assert caught.value.reason == "window_cut_failed" + assert str(root) in str(caught.value) + + +@pytest.mark.parametrize("where", ["pod-list", "file-stat", "root-list", "root-stat"]) +@pytest.mark.parametrize( + "code", [errno.ENOENT, errno.ESTALE, errno.EIO, errno.ETIMEDOUT, errno.EACCES] +) +def test_segment_walk_skips_only_vanished_entries(tmp_path, monkeypatch, where, code): + root = _two_pods(tmp_path / "segments") + target = { + "pod-list": root / "b", + "file-stat": root / "b" / "req.000000.jsonl.gz", + "root-list": root, + "root-stat": root, + }[where] + name = "scandir" if where.endswith("list") else "stat" + real = getattr(os, name) + + def failing(path, *args, **kwargs): + if os.fspath(path) == str(target): + raise OSError(code, os.strerror(code), os.fspath(path)) + return real(path, *args, **kwargs) + + monkeypatch.setattr(os, name, failing) + vanished = code in (errno.ENOENT, errno.ESTALE) + + if vanished and not where.startswith("root"): + # The reconciler deleted pod b's segments under the walk. + assert _segment_files(str(root)) == [ + (str(root / "a" / "req.000000.jsonl.gz"), True) + ] + return + with pytest.raises(WindowCutError) as caught: + _segment_files(str(root)) + assert caught.value.reason == "window_cut_failed" + if vanished and where == "root-stat": + assert "does not exist" in str(caught.value) + else: + assert str(target) in str(caught.value) + assert os.strerror(code) in str(caught.value) + + +# -------------------------------------------------------- atomic output + + +def test_existing_window_file_is_reused(tmp_path): + root = _random_trace(tmp_path / "segments", 5) + window, _ = _golden(root, tmp_path) + out = _out(tmp_path) + _write(out, b"published by another process") + + result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + + assert result.reused + assert result.path == str(out) + assert result.num_requests == window.num_requests + assert out.read_bytes() == b"published by another process" + assert _temp_files(out.parent) == [] + + +def test_link_race_reuses_the_winner(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments", 6) + out = _out(tmp_path) + real_link = os.link + linked = [] + + def racing_link(source, target): + linked.append(os.path.basename(source)) + _write(out, b"winner") + return real_link(source, target) + + monkeypatch.setattr(os, "link", racing_link) + + result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + + assert result.reused + assert out.read_bytes() == b"winner" + assert temp_file_window_end(linked[0]) == END + assert _temp_files(out.parent) == [] + + +def test_link_enoent_is_window_cut_failed(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments", 7) + out = _out(tmp_path) + + def vanished_link(source, target): + raise FileNotFoundError(2, "No such file or directory", source) + + monkeypatch.setattr(os, "link", vanished_link) + + with pytest.raises(WindowCutError) as caught: + cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + assert caught.value.reason == "window_cut_failed" + assert not out.exists() + assert _temp_files(out.parent) == [] + + +def test_output_is_deterministic_and_0600_under_any_umask(tmp_path): + root = _random_trace(tmp_path / "segments", 8) + first, second = _out(tmp_path, "a.jsonl.gz"), _out(tmp_path, "b.jsonl.gz") + previous = os.umask(0o077) + try: + cut_window_inprocess(root, START, END, AS_OF, first, max_rows=10**6) + cut_window_inprocess(root, START, END, AS_OF, second, max_rows=10**6) + finally: + os.umask(previous) + + assert first.read_bytes() == second.read_bytes() + assert stat.S_IMODE(first.stat().st_mode) == 0o600 + with gzip.open(first, "rb") as source: + assert source.read(1) + # No file name or mtime in the gzip header. + header = first.read_bytes()[:10] + assert header[3] & 0x08 == 0 + assert header[4:8] == b"\x00\x00\x00\x00" + + +@pytest.mark.parametrize("change", ["rewritten", "truncated", "deleted"]) +def test_segment_changed_between_passes_fails(tmp_path, monkeypatch, change): + root = tmp_path / "segments" + segment = _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("a", START), _row("b", START + 1)]), + _member([_row("c", START + 2)]), + ) + original_write_rows = window_cut._write_rows + + def mutate_then_write(selection, paths, sink): + if change == "rewritten": + segment.write_bytes( + _member([_row("x", START), _row("y", START + 1)]) + + _member([_row("z", START + 2)]) + ) + elif change == "truncated": + segment.write_bytes(segment.read_bytes()[:40]) + else: + segment.unlink() + return original_write_rows(selection, paths, sink) + + monkeypatch.setattr(window_cut, "_write_rows", mutate_then_write) + out = _out(tmp_path) + + with pytest.raises(WindowCutError) as caught: + cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10) + assert caught.value.reason == "window_cut_failed" + assert not out.exists() + assert _temp_files(out.parent) == [] + + +def test_line_writer_writes_in_order_and_reraises_sink_errors(monkeypatch): + monkeypatch.setattr(window_cut, "_WRITE_BUFFER", 8) + sink = io.BytesIO() + writer = window_cut._LineWriter(sink) + for index in range(100): + writer.write(f"row {index}") + writer.close() + assert sink.getvalue() == b"".join(b"row %d\n" % i for i in range(100)) + + class FullDisk(io.BytesIO): + def write(self, data): + raise OSError(28, "No space left on device") + + writer = window_cut._LineWriter(FullDisk()) + with pytest.raises(OSError, match="No space left"): + for index in range(100): + writer.write(f"row {index}") + writer.close() + writer.abort() # idempotent after a failure + + aborted = io.BytesIO() + writer = window_cut._LineWriter(aborted) + writer.write("x" * 100) + writer.abort() + writer.abort() + assert writer._thread.is_alive() is False + + +def test_write_failure_is_window_cut_failed_and_leaves_no_files(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments", 10) + out = _out(tmp_path) + + def failing_write(selection, paths, sink): + raise OSError(28, "No space left on device") + + monkeypatch.setattr(window_cut, "_write_rows", failing_write) + + with pytest.raises(WindowCutError) as caught: + cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + assert caught.value.reason == "window_cut_failed" + assert "No space left" in str(caught.value) + assert not out.exists() + assert _temp_files(out.parent) == [] + + +# ------------------------------------------------- arguments and helpers + + +@pytest.mark.parametrize( + "start, end, as_of, out, max_rows", + [ + (10, 10, 20, "w.jsonl.gz", 1), + (10, 20, 19, "w.jsonl.gz", 1), + (10, 20, 20, "w.jsonl", 1), + (10, 20, 20, "w.jsonl.gz", 0), + (10, 20, 20, "w.jsonl.gz", True), + (float("nan"), 20, 20, "w.jsonl.gz", 1), + ("10", 20, 20, "w.jsonl.gz", 1), + ], +) +def test_invalid_arguments_raise_value_error( + tmp_path, start, end, as_of, out, max_rows +): + with pytest.raises(ValueError): + cut_window_inprocess( + tmp_path, start, end, as_of, tmp_path / out, max_rows=max_rows + ) + with pytest.raises(ValueError): + cut_window( + tmp_path, + start, + end, + as_of, + tmp_path / out, + max_rows=max_rows, + deadline_s=time.time() + 60, + ) + + +def test_temp_path_must_be_next_to_out_path(tmp_path): + with pytest.raises(ValueError, match="not next to"): + cut_window_inprocess( + tmp_path, + START, + END, + AS_OF, + tmp_path / "a" / "w.jsonl.gz", + max_rows=1, + temp_path=tmp_path / "b" / temp_file_name(END), + ) + + +def test_temp_and_segment_names(): + name = temp_file_name(1_700_000_000_000.0, "ab12") + assert name == ".tmp-1700000000000-ab12" + assert temp_file_window_end(name) == 1_700_000_000_000 + assert temp_file_window_end(temp_file_name(5)) == 5 + assert temp_file_name(5) != temp_file_name(5) + assert temp_file_window_end("1700000000000.jsonl.gz") is None + assert temp_file_window_end(".tmp-12-") is None + with pytest.raises(ValueError): + temp_file_name(5, "../x") + assert segment_index("req.000042.jsonl.gz") == 42 + assert segment_index("req.1000000.jsonl") == 1_000_000 + assert segment_index("req.jsonl.gz") is None + + +def test_window_cut_error_reason_and_pickle(): + error = WindowCutError("trace_corrupt", "bad member") + copy = pickle.loads(pickle.dumps(error)) + assert (copy.reason, str(copy)) == ("trace_corrupt", "bad member") + assert set(WINDOW_CUT_FAILURES) == { + "window_cut_timeout", + "window_too_large", + "trace_corrupt", + "window_cut_failed", + "cancelled", + } + with pytest.raises(ValueError): + WindowCutError("nope", "x") + + +# ------------------------------------------------------- child process + + +def test_interrupted_before_spawn_does_not_start_a_child(tmp_path, monkeypatch): + def no_spawn(*_): + raise AssertionError("a child was started") + + monkeypatch.setattr(window_cut.multiprocessing, "get_context", no_spawn) + out = _out(tmp_path) + + with pytest.raises(WindowCutError) as caught: + cut_window( + tmp_path, START, END, AS_OF, out, max_rows=1, deadline_s=time.time() - 1 + ) + assert caught.value.reason == "window_cut_timeout" + + cancel = threading.Event() + cancel.set() + with pytest.raises(WindowCutError) as caught: + cut_window( + tmp_path, + START, + END, + AS_OF, + out, + max_rows=1, + deadline_s=time.time() + 60, + cancel=cancel, + ) + assert caught.value.reason == "cancelled" + + +@pytest.mark.timeout(120) +def test_cut_window_in_child_matches_golden(tmp_path): + root = _random_trace(tmp_path / "segments" / "dgd", 9) + window, golden_bytes = _golden(root, tmp_path) + out = _out(tmp_path) + + result = cut_window( + root, START, END, AS_OF, out, max_rows=10**6, deadline_s=time.time() + 100 + ) + + _assert_golden(result, out, window, golden_bytes) + assert _temp_files(out.parent) == [] + + +@pytest.mark.timeout(120) +def test_cut_window_in_child_reports_the_failure_reason(tmp_path): + root = tmp_path / "segments" + _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("a", START), _row("b", START + 1)]), + ) + + with pytest.raises(WindowCutError) as caught: + cut_window( + root, + START, + END, + AS_OF, + _out(tmp_path), + max_rows=1, + deadline_s=time.time() + 100, + ) + assert caught.value.reason == "window_too_large" + assert "more than 1 requests" in str(caught.value) + + +@pytest.mark.timeout(120) +def test_child_that_dies_is_window_cut_failed(): + with pytest.raises(WindowCutError) as caught: + _run_in_child(os._exit, (3,), {}, deadline=time.monotonic() + 100, cancel=None) + assert caught.value.reason == "window_cut_failed" + assert "exited with code 3" in str(caught.value) + + +@pytest.mark.timeout(60) +@pytest.mark.parametrize("interrupt", ["cancel", "deadline"]) +def test_interrupt_kills_child_and_removes_its_temp_file(tmp_path, interrupt): + temp = tmp_path / temp_file_name(END) + temp.write_bytes(b"partial window") + cancel = threading.Event() + timer = threading.Timer(0.5, cancel.set) + deadline = time.monotonic() + (100 if interrupt == "cancel" else 0.5) + started = time.monotonic() + timer.start() + try: + with pytest.raises(WindowCutError) as caught: + _run_in_child( + time.sleep, + (600,), + {}, + deadline=deadline, + cancel=cancel if interrupt == "cancel" else None, + cleanup=(str(temp),), + ) + finally: + timer.cancel() + timer.join() + expected = "cancelled" if interrupt == "cancel" else "window_cut_timeout" + assert caught.value.reason == expected + assert time.monotonic() - started < 20 + assert not temp.exists() From 2dce2a546947245c5e13d025375f8569209ba1c8 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Fri, 30 Oct 2026 16:04:33 -0700 Subject: [PATCH 077/100] feat(planner): run a live retune sweep through DGDR v1beta2 Requests Add the live sweep executor and its Kubernetes client: - retune/live/dgdr.py: run_dgdr_sweep, the live sweep_fn of RetuneController. It shards the router grid into balanced Cartesian Requests of at most 100 points plus an off-grid baseline shard and a common-random-numbers probe, adopts the Requests a previous leader created for the window, cuts the window file, creates one Request per remaining shard, verifies each Run's input, maps and scores the Candidates on retune.objective, and checks the CRN probe. It never raises and never deletes: removals go to the SweepRegistry queue, and shutdown or lost leadership return at once with every Request kept for the next leader. - retune/live/kube.py: contracts.RetuneKube over the kubernetes client, with request timeouts, Kube* errors that carry the HTTP status, and Leases read without model deserialization so they round-trip unchanged. The executor is not wired into a Planner yet: a live Planner still rejects retune at startup, and replay still rejects executor 'dgdr'. Fourth layer of the live retune stack split from #17562. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../src/dynamo/planner/retune/live/dgdr.py | 1926 +++++++++++++++++ .../src/dynamo/planner/retune/live/kube.py | 397 ++++ .../tests/unit/test_live_retune_dgdr.py | 1884 ++++++++++++++++ .../tests/unit/test_live_retune_kube.py | 530 +++++ 4 files changed, 4737 insertions(+) create mode 100644 components/src/dynamo/planner/retune/live/dgdr.py create mode 100644 components/src/dynamo/planner/retune/live/kube.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py create mode 100644 components/src/dynamo/planner/tests/unit/test_live_retune_kube.py diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py new file mode 100644 index 000000000000..f7761f6b5621 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -0,0 +1,1926 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Live retune sweep executor: one window through DGDR v1beta2 Requests. + +``run_dgdr_sweep`` is the live ``sweep_fn`` of ``RetuneController`` (design +§4.5). It runs on the controller's single sweep thread and never raises: + +1. Shard the router grid (``plan_shards``): balanced Cartesian sub-grids of + at most 100 points (a Request's ``maxCandidates`` bound), one + single-point shard for an off-grid baseline or startup point, and a CRN + probe that evaluates one fixed grid point a second time. +2. Adopt the Requests a previous leader created for this window, matched by + ``(dgd-uid, window, shard, spec hash)``; others of the window are queued + for deletion. A window whose every shard is adopted as Succeeded is + harvested without cutting it again. An adopted attempt whose Run failed + or ran another input is replaced by attempt+1 while the shard still + fits before the deadline. +3. Cut the window file (a subprocess, ``context.cut_window``), then create + one Request per remaining shard with Strict field validation and check + that the API server kept every submitted field. +4. Wait for each Request's Run, check that ``Run.status.aisimulate.input`` + is what the Request asked for, and read the Run's Candidates. +5. Map each Candidate's router adapter parameters back to live names, + require identical non-router parameters across the window, score each + with ``score_summary`` on ``retune.objective``, and check the CRN probe. + +The executor never deletes anything itself: Requests to remove go to the +``SweepRegistry`` deletion queue, which the reconciler drains while this +process leads. Shutdown (``cancel``) and loss of leadership (``abort``) +return at once with ``failure_reason`` ``shutdown``/``lost_leadership`` and +leave every Request in place for the next leader to adopt. Every mutating +call first checks ``abort`` and ``context.is_leader()``. + +A shard that holds the baseline, or the probe, is critical: when it fails +the window fails. Any other failed shard only leaves its points unscored. +""" + +from __future__ import annotations + +import copy +import functools +import hashlib +import itertools +import json +import logging +import math +import os +import re +import threading +import time +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Literal, NoReturn, Optional + +from dynamo.planner.config.planner_config import DgdrExecutorConfig, RetuneConfig +from dynamo.planner.core.types import RetuneParams +from dynamo.planner.retune.live.contracts import ( + DGDR_GROUP, + DGDR_ROUTER_KNOBS, + DGDR_VERSION, + LABEL_ATTEMPT, + LABEL_DGD, + LABEL_DGD_UID, + LABEL_PREFIX, + LABEL_SHARD, + LABEL_SPEC, + LABEL_WINDOW, + RUN_UID_LABEL, + DgdRef, + KubeAlreadyExists, + KubeError, + KubeInvalid, + KubeNotFound, + RetuneKube, + SweepRegistry, + WindowCutResult, + WindowCutter, +) +from dynamo.planner.retune.sweep import ( + SweepRequest, + SweepResult, + SweepStatus, + score_summary, +) + +logger = logging.getLogger(__name__) + +ShardKind = Literal["grid", "point", "probe"] + +# A Request publishes at most recommendation.maxCandidates (<= 100) points. +MAX_SHARD_POINTS = 100 +# Seconds a new Request may go without an Accepted condition before the +# window fails as operator_not_reconciling. +ACCEPT_TIMEOUT_S = 60.0 +# Relative score difference the CRN probe tolerates. +CRN_REL_TOLERANCE = 1e-9 +# Oldest Dynamo runtime whose trace replay runs with common random numbers. +# None: the version is not checked and the CRN probe alone decides. +CRN_MIN_VERSION: Optional[str] = None +# Annotation holding the cut window's row counts, so a window adopted as +# Succeeded reports them without cutting the window again. +ROWS_ANNOTATION = LABEL_PREFIX + "rows" +# Operator constants (deploy/operator at #17483). +REQUEST_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" +TRACE_MOUNT_PATH = "/var/run/aisimulate/trace" +ROUTER_ADAPTER = "dynamo.router" +_ROLES = ("aggregated", "prefill", "decode") +_TEMPERATURE = DGDR_ROUTER_KNOBS["temperature"] +# spec.objective.metric -> AISimulate optimization.target (operator lowering). +_METRIC_TARGETS = { + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", +} +# Statuses a retry cannot fix (bad request, auth, missing API). +_PERMANENT_STATUSES = frozenset({400, 401, 403, 404, 405, 422}) +# How the API server words a rejection by an admission webhook. +_ADMISSION_DENIED = re.compile(r'admission webhook "[^"]*" denied the request') +_K8S_TIMEOUT_S = 10.0 +_SNAP_TOLERANCE = 1e-9 +_MIN_K8S_TIMEOUT_S = 1.0 +_ABSENT = object() + + +# ------------------------------------------------------------------ shards + + +@dataclass(frozen=True) +class Shard: + """One Request's slice of the window's candidates. + + ``choices`` maps each live router knob to the values the shard spans; + ``points`` is their Cartesian product (live names). A ``point`` or + ``probe`` shard has one value per knob. + """ + + id: str + kind: ShardKind + choices: dict[str, tuple[float, ...]] + points: tuple[dict[str, float], ...] + + @property + def size(self) -> int: + return len(self.points) + + def contains(self, point: Mapping[str, float]) -> bool: + return all(point.get(key) in values for key, values in self.choices.items()) + + +def plan_shards( + router_domain: Mapping[str, Sequence[float]], + *, + baseline: Mapping[str, float], + startup: Mapping[str, float], + max_shards: int, + probe: bool = True, + parallelism: int | None = None, +) -> list[Shard]: + """Split ``router_domain`` (live knob -> values) into Request shards. + + The grid becomes at most ``max_shards`` Cartesian sub-grids of at most + ``MAX_SHARD_POINTS`` points: each knob's sorted values are cut into + contiguous, near-equal chunks. The split minimizes the largest shard's + wall time, then the shard count: with the Run ``parallelism`` the time + is its evaluation rounds (``1 + ceil((n - 1) / parallelism)``, as in + ``need_seconds``), so a shard that would not shorten the window is not + added; without it, the largest shard's size. ``baseline`` and ``startup`` + (live knob -> value) each get a single-point shard when off the grid. + With ``probe``, a ``probe`` shard repeats one fixed grid point (the + middle value of each knob, a positive temperature when the grid has + one) to check common random numbers across Runs. Raises ``ValueError`` + when no split fits or a point does not cover the domain's knobs. + """ + + if not router_domain: + raise ValueError("router domain is empty; there is nothing to shard") + if max_shards < 1: + raise ValueError(f"max_shards must be at least 1, got {max_shards}") + keys = tuple(sorted(router_domain)) + axes: dict[str, tuple[float, ...]] = {} + for key in keys: + values = tuple(sorted({float(value) for value in router_domain[key]})) + if not values: + raise ValueError(f"router domain {key!r} has no values") + if not all(math.isfinite(value) for value in values): + raise ValueError(f"router domain {key!r} has a non-finite value") + axes[key] = values + for name, point in (("baseline", baseline), ("startup", startup)): + if set(point) != set(keys): + raise ValueError( + f"{name} router point keys {sorted(point)} do not match the " + f"router domain {list(keys)}" + ) + + if parallelism is not None and parallelism < 1: + raise ValueError(f"parallelism must be at least 1, got {parallelism}") + sizes = [len(axes[key]) for key in keys] + best: tuple[float, int, int, tuple[int, ...]] | None = None + for splits in _splits(sizes, max_shards): + largest = math.prod( + math.ceil(size / split) for size, split in zip(sizes, splits) + ) + # An oversized shard can never be chosen over one that fits. + cost = ( + math.inf + if largest > MAX_SHARD_POINTS + else largest + if parallelism is None + else _rounds(largest, parallelism) + ) + candidate = (cost, math.prod(splits), largest, splits) + if best is None or candidate < best: + best = candidate + assert best is not None # all-ones always fits max_shards >= 1 + _, _, largest, splits = best + if largest > MAX_SHARD_POINTS: + raise ValueError( + f"the router grid of {math.prod(sizes)} points cannot be split into " + f"at most {max_shards} Cartesian shards of at most " + f"{MAX_SHARD_POINTS} points (the smallest largest shard has " + f"{largest}); raise retune.dgdr.run.max_shards or shrink the domain" + ) + + shards: list[Shard] = [] + chunked = [_chunks(axes[key], split) for key, split in zip(keys, splits)] + for index, combo in enumerate(itertools.product(*chunked)): + choices = dict(zip(keys, combo)) + shards.append( + Shard( + id=f"grid-{index}", + kind="grid", + choices=choices, + points=_product(choices), + ) + ) + + def in_grid(point: Mapping[str, float]) -> bool: + return all(float(point[key]) in axes[key] for key in keys) + + def single(shard_id: str, kind: ShardKind, point: Mapping[str, float]) -> Shard: + values = {key: float(point[key]) for key in keys} + return Shard( + id=shard_id, + kind=kind, + choices={key: (value,) for key, value in values.items()}, + points=(values,), + ) + + baseline_point = {key: float(baseline[key]) for key in keys} + startup_point = {key: float(startup[key]) for key in keys} + if not in_grid(baseline_point): + shards.append(single("point-baseline", "point", baseline_point)) + if not in_grid(startup_point) and startup_point != baseline_point: + shards.append(single("point-startup", "point", startup_point)) + if probe: + shards.append(single("probe", "probe", _probe_point(axes))) + return shards + + +def _rounds(points: int, parallelism: int) -> int: + """Evaluation rounds of a Run: the first candidate alone, then batches.""" + + return 1 + math.ceil((points - 1) / parallelism) + + +def _splits(sizes: Sequence[int], max_shards: int) -> Iterable[tuple[int, ...]]: + """Every per-knob split count whose product is at most ``max_shards``.""" + + def walk(index: int, budget: int) -> Iterable[tuple[int, ...]]: + if index == len(sizes): + yield () + return + for split in range(1, min(sizes[index], budget) + 1): + for rest in walk(index + 1, budget // split): + yield (split, *rest) + + return walk(0, max_shards) + + +def _chunks(values: tuple[float, ...], count: int) -> list[tuple[float, ...]]: + """``values`` cut into ``count`` contiguous chunks whose sizes differ by <= 1.""" + + base, extra = divmod(len(values), count) + chunks: list[tuple[float, ...]] = [] + start = 0 + for index in range(count): + end = start + base + (1 if index < extra else 0) + chunks.append(values[start:end]) + start = end + return chunks + + +def _product(choices: Mapping[str, tuple[float, ...]]) -> tuple[dict[str, float], ...]: + keys = sorted(choices) + return tuple( + dict(zip(keys, values)) + for values in itertools.product(*(choices[key] for key in keys)) + ) + + +def _probe_point(axes: Mapping[str, tuple[float, ...]]) -> dict[str, float]: + """The fixed CRN probe point: each knob's middle value; a positive + temperature when the grid has one, so the probe exercises sampling.""" + + point: dict[str, float] = {} + for key, values in axes.items(): + pool = values + if key == _TEMPERATURE: + positive = tuple(value for value in values if value > 0) + pool = positive or values + point[key] = pool[len(pool) // 2] + return point + + +def need_seconds(shard: Shard, cfg: DgdrExecutorConfig, *, needs_cut: bool) -> float: + """Wall time a shard needs from now to finish (design §4.1). + + ``(cut_budget if needs_cut) + job_startup + (1 + ceil((n - 1) / + parallelism)) * candidate_timeout``: AISimulate evaluates the first + candidate alone, then ``parallelism`` at a time. + """ + + rounds = _rounds(shard.size, cfg.run.parallelism) + return ( + (cfg.cut_budget_seconds if needs_cut else 0.0) + + cfg.job_startup_seconds + + rounds * cfg.run.candidate_timeout_seconds + ) + + +# ----------------------------------------------------------------- Request + + +def spec_hash(spec: Mapping[str, Any]) -> str: + """First 16 hex digits of the SHA-256 of the canonical JSON of ``spec``.""" + + canonical = json.dumps(spec, sort_keys=True, separators=(",", ":"), allow_nan=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16] + + +def window_claim_path(cfg: DgdrExecutorConfig, dgd: DgdRef, window_end_ms: int) -> str: + """Claim-relative path of a window file: ``//.jsonl.gz``.""" + + return f"{cfg.windows_dir.strip('/')}/{dgd.name}/{int(window_end_ms)}.jsonl.gz" + + +def request_name( + dgd: DgdRef, window_end_ms: int, shard_id: str, digest: str, attempt: int +) -> str: + """``-rt-``.""" + + key = f"{dgd.uid}|{int(window_end_ms)}|{shard_id}|{digest}|{attempt}" + suffix = hashlib.sha256(key.encode("utf-8")).hexdigest()[:10] + prefix = dgd.name[:40].rstrip("-.") or "dgd" + return f"{prefix}-rt-{suffix}" + + +def _label_value(value: str) -> str: + """``value`` when it is a valid label value, else a hashed 63-char form.""" + + if len(value) <= 63 and re.fullmatch( + r"([A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?)?", value + ): + return value + digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:10] + head = re.sub(r"[^-A-Za-z0-9_.]", "-", value)[:52].strip("-_.") + return f"{head}-{digest}" if head else digest + + +def build_request( + *, + dgd: DgdRef, + cfg: DgdrExecutorConfig, + retune: RetuneConfig, + sla_ttft_ms: float, + sla_itl_ms: float, + window_end_ms: int, + shard: Shard, + spec_replicas: Mapping[str, int], + backend: str, + attempt: int, +) -> dict: + """The DGDR v1beta2 Request that replays one shard of a window (§4.5.2). + + The engine is ``cfg.engine`` with every leaf a scalar and each used + role's ``parallelism.preset[0].replicas`` set from ``spec_replicas`` + (unused roles are deleted with ``null``); the GPU budget is the sum of + replicas times GPUs per worker (tensor x pipeline x attention_data). + Router knobs use their AISimulate public names: ``{choices: [...]}`` + for a grid shard, a scalar for a point or probe shard. Raises + ``ValueError`` for an engine or shard the Request cannot express. + """ + + engine = _pinned_engine(cfg.engine, spec_replicas, backend) + size = shard.size + if not 1 <= size <= MAX_SHARD_POINTS: + raise ValueError(f"shard {shard.id} has {size} points; 1..100 fit a Request") + spec: dict[str, Any] = { + "model": {"name": cfg.model}, + "backends": [backend], + "hardware": { + "gpu": {"sku": cfg.sku, "budget": _gpu_budget(engine, spec_replicas)} + }, + "workload": { + "trace": { + "format": "dynamo", + "source": { + "pvc": { + "claimName": cfg.claim_name, + "path": window_claim_path(cfg, dgd, window_end_ms), + } + }, + } + }, + "objective": { + "mode": "Optimize", + "metric": retune.objective, + "sla": {"ttftMs": float(sla_ttft_ms), "itlMs": float(sla_itl_ms)}, + }, + "recommendation": {"maxCandidates": size}, + "aisimulate": { + "inputOverrides": { + "engine": engine, + "router": _router_section(shard), + "optimizer": { + "algorithm": "random", + "max_trials": 2 * size, + "parallelism": cfg.run.parallelism, + "candidate_timeout_seconds": cfg.run.candidate_timeout_seconds, + }, + "optimization": {"strict_sla": False}, + "dgdr_run": {"generator": "direct"}, + } + }, + } + digest = spec_hash(spec) + return { + "apiVersion": f"{DGDR_GROUP}/{DGDR_VERSION}", + "kind": "DynamoGraphDeploymentRequest", + "metadata": { + "name": request_name(dgd, window_end_ms, shard.id, digest, attempt), + "namespace": dgd.namespace, + "labels": { + LABEL_DGD: _label_value(dgd.name), + LABEL_DGD_UID: dgd.uid, + LABEL_WINDOW: str(int(window_end_ms)), + LABEL_SHARD: shard.id, + LABEL_SPEC: digest, + LABEL_ATTEMPT: str(attempt), + }, + "ownerReferences": [ + { + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeployment", + "name": dgd.name, + "uid": dgd.uid, + "blockOwnerDeletion": False, + } + ], + }, + "spec": spec, + } + + +def _pinned_engine( + engine: Mapping[str, Any], spec_replicas: Mapping[str, int], backend: str +) -> dict[str, Any]: + pinned = copy.deepcopy(dict(engine)) + roles = set(spec_replicas) + if roles not in ({"aggregated"}, {"prefill", "decode"}): + raise ValueError( + "spec_replicas must name 'aggregated', or 'prefill' and 'decode'; " + f"got {sorted(roles)}" + ) + written = pinned.get("backend") + if written is None: + pinned["backend"] = backend + elif written != backend: + raise ValueError( + f"retune.dgdr.engine.backend {written!r} differs from the deployed " + f"backend {backend!r}" + ) + workers = pinned.get("workers") + if not isinstance(workers, dict): + raise ValueError("retune.dgdr.engine.workers must be a mapping") + for role in _ROLES: + if role not in roles: + workers[role] = None + continue + worker = workers.get(role) + preset = ( + worker.get("parallelism", {}).get("preset") + if isinstance(worker, dict) and isinstance(worker.get("parallelism"), dict) + else None + ) + if ( + not isinstance(preset, list) + or len(preset) != 1 + or not isinstance(preset[0], dict) + ): + raise ValueError( + f"retune.dgdr.engine.workers.{role}.parallelism.preset must be a " + "list of exactly one entry" + ) + replicas = int(spec_replicas[role]) + if replicas < 1: + raise ValueError( + f"{role} has {replicas} spec replicas; at least 1 is needed" + ) + preset[0]["replicas"] = replicas + return pinned + + +def _gpu_budget(engine: Mapping[str, Any], spec_replicas: Mapping[str, int]) -> int: + budget = 0 + for role, replicas in spec_replicas.items(): + entry = engine["workers"][role]["parallelism"]["preset"][0] + per_worker = 1 + for knob in ("tensor", "pipeline", "attention_data"): + value = entry.get(knob, 1) + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError( + f"retune.dgdr.engine.workers.{role}.parallelism.preset[0]." + f"{knob} must be a positive integer, got {value!r}" + ) + per_worker *= value + budget += int(replicas) * per_worker + return budget + + +def _router_section(shard: Shard) -> dict[str, Any]: + live_knobs = set(DGDR_ROUTER_KNOBS.values()) + if set(shard.choices) != live_knobs: + raise ValueError( + f"shard {shard.id} router knobs {sorted(shard.choices)} must be exactly " + f"{sorted(live_knobs)}, the knobs a DGDR Request can express" + ) + section: dict[str, Any] = { + "policy": "kv_router", + "prefill_load_model": {"type": "none"}, + } + for public, live in DGDR_ROUTER_KNOBS.items(): + values = [float(value) for value in shard.choices[live]] + section[public] = {"choices": values} if shard.kind == "grid" else values[0] + return section + + +# --------------------------------------------------------- input read-back + + +def merge_patch(target: Any, patch: Any) -> Any: + """RFC 7386 JSON merge patch (the operator's lowering); never mutates.""" + + if not isinstance(patch, Mapping): + return copy.deepcopy(patch) + result = dict(target) if isinstance(target, Mapping) else {} + for key, value in patch.items(): + if value is None: + result.pop(key, None) + else: + result[key] = merge_patch(result.get(key), value) + return result + + +def _get(obj: Any, path: Sequence[str], default: Any = None) -> Any: + for key in path: + if not isinstance(obj, Mapping) or key not in obj: + return default + obj = obj[key] + return obj + + +def _same_json(left: Any, right: Any) -> bool: + """JSON equality: numbers by value (2 == 2.0), booleans never equal numbers.""" + + if isinstance(left, bool) or isinstance(right, bool): + return isinstance(left, bool) and isinstance(right, bool) and left == right + if isinstance(left, (int, float)) and isinstance(right, (int, float)): + return float(left) == float(right) + if isinstance(left, Mapping) and isinstance(right, Mapping): + return set(left) == set(right) and all( + _same_json(left[key], right[key]) for key in left + ) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all( + _same_json(a, b) for a, b in zip(left, right) + ) + return type(left) is type(right) and left == right + + +def missing_fields(stored: Any, submitted: Any, path: str = "") -> list[str]: + """Submitted leaves the stored object lacks or changed (pruned or rejected). + + ``stored`` may hold more (admission defaults); a submitted ``null`` only + requires the stored value to be absent or ``null``. + """ + + if isinstance(submitted, Mapping): + if not isinstance(stored, Mapping): + return [path or "."] + problems: list[str] = [] + for key, value in submitted.items(): + child = f"{path}.{key}" if path else key + if value is None: + if stored.get(key) is not None: + problems.append(child) + continue + if key not in stored: + problems.append(child) + continue + problems.extend(missing_fields(stored[key], value, child)) + return problems + return [] if _same_json(stored, submitted) else [path or "."] + + +def input_mismatches( + spec: Mapping[str, Any], effective: Mapping[str, Any] +) -> list[str]: + """How ``Run.status.aisimulate.input`` differs from what ``spec`` asks for. + + The expected input follows the operator's lowering at #17483: the + public role underlay, ``spec.model``/``backends``/``hardware`` into + ``engine``, then ``inputOverrides`` as a merge patch. ``engine``, + ``router`` and ``optimizer`` must match exactly; the objective, SLA, + budget, trace source and the direct generator are checked by value. + """ + + overrides = _get(spec, ("aisimulate", "inputOverrides"), {}) or {} + typed_engine: dict[str, Any] = { + "workers": {role: {} for role in _ROLES}, + "model": _get(spec, ("model", "name")), + "backend": {"choices": list(spec.get("backends") or [])}, + "hardware": _get(spec, ("hardware", "gpu", "sku")), + } + expected_trace = _get(spec, ("workload", "trace", "source", "pvc", "path"), "") + mounted = TRACE_MOUNT_PATH + ( + "/trace.jsonl.gz" if str(expected_trace).endswith(".gz") else "/trace.jsonl" + ) + sla = _get(spec, ("objective", "sla"), {}) or {} + checks: list[tuple[tuple[str, ...], Any]] = [ + (("engine",), merge_patch(typed_engine, overrides.get("engine") or {})), + (("router",), merge_patch({}, overrides.get("router") or {})), + (("optimizer",), merge_patch({}, overrides.get("optimizer") or {})), + ( + ("optimization", "target"), + _METRIC_TARGETS.get(_get(spec, ("objective", "metric"), ""), _ABSENT), + ), + ( + ("optimization", "strict_sla"), + _get(overrides, ("optimization", "strict_sla")), + ), + ( + ("optimization", "constraints", "max_candidate_gpus"), + _get(spec, ("hardware", "gpu", "budget")), + ), + (("evaluation", "sla", "ttft_ms"), sla.get("ttftMs", _ABSENT)), + (("evaluation", "sla", "itl_ms"), sla.get("itlMs", _ABSENT)), + (("evaluation", "sla", "e2e_ms"), sla.get("e2eMs", _ABSENT)), + ( + ("traffic", "source"), + {"type": "trace", "format": "dynamo", "paths": [mounted]}, + ), + (("traffic", "load"), {"type": "trace_timestamps"}), + (("dgdr_run", "generator"), _get(overrides, ("dgdr_run", "generator"))), + ( + ("dgdr_run", "max_candidates"), + _get(spec, ("recommendation", "maxCandidates")), + ), + ] + # Normalize like the API server (JSON numbers, no tuples). + effective = json.loads(json.dumps(effective)) + problems: list[str] = [] + for path, expected in checks: + expected = expected if expected is _ABSENT else json.loads(json.dumps(expected)) + actual = _get(effective, path, _ABSENT) + if expected is _ABSENT: + if actual is not _ABSENT and actual is not None: + problems.append(f"{'.'.join(path)}: expected unset, got {actual!r}") + elif actual is _ABSENT or not _same_json(actual, expected): + shown = "unset" if actual is _ABSENT else repr(actual) + problems.append(f"{'.'.join(path)}: expected {expected!r}, got {shown}") + return problems + + +def _version_tuple(text: str | None) -> tuple[int, ...] | None: + if not text: + return None + found = re.match(r"^v?(\d+)\.(\d+)(?:\.(\d+))?", text.strip()) + if found is None: + return None + return tuple(int(part) for part in found.groups(default="0")) + + +def crn_unsupported( + runtime_version: str | None, runtime_image: str | None, minimum: str | None +) -> bool: + """Whether a parseable runtime version is below ``minimum`` (None: unknown).""" + + floor = _version_tuple(minimum) + if floor is None: + return False + version = _version_tuple(runtime_version) + if version is None and runtime_image: + # registry/repo:tag@digest -> tag + last = runtime_image.split("@", 1)[0].rsplit("/", 1)[-1] + if ":" in last: + version = _version_tuple(last.split(":", 1)[1]) + return version is not None and version < floor + + +# ---------------------------------------------------------------- executor + + +@dataclass(frozen=True) +class DgdrContext: + """Everything ``run_dgdr_sweep`` needs besides the ``SweepRequest``. + + ``backend`` is the deployed backend (``vllm``, ``sglang``, ``trtllm``). + ``is_leader`` is checked before every mutating call. ``now``/``sleep`` + are wall-clock (``deadline_s`` is epoch seconds). ``crn_min_version`` is + the oldest runtime that replays traces with common random numbers. + """ + + dgd: DgdRef + cfg: DgdrExecutorConfig + retune: RetuneConfig + backend: str + kube: RetuneKube + registry: SweepRegistry + cut_window: WindowCutter + is_leader: Callable[[], bool] + now: Callable[[], float] = time.time + sleep: Callable[[float], None] = time.sleep + poll_interval_s: float = 5.0 + crn_min_version: Optional[str] = CRN_MIN_VERSION + + +def run_dgdr_sweep( + request: SweepRequest, + *, + cancel: threading.Event, + abort: Optional[threading.Event], + context: DgdrContext, +) -> SweepResult: + """Sweep one window through DGDR v1beta2 Requests; never raises. + + Scores are keyed by ``candidate.key()`` for every candidate of + ``request`` (live router names, no planner part); a candidate no Run + published scores ``None``. See the module docstring for the protocol. + """ + + started = context.now() + try: + return _DgdrSweep(request, cancel, abort, context, started).run() + except Exception as error: # defensive: the executor must never raise + logger.exception("retune DGDR sweep of window %s failed", request.window_index) + return SweepResult( + status="failed", + wall_s=max(0.0, context.now() - started), + error=f"{type(error).__name__}: {error}", + failure_reason="internal_error", + details={}, + ) + + +class _Interrupted(Exception): + """Shutdown or loss of leadership: return without a decision.""" + + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +class _WindowFailed(Exception): + """The window ends without scores (``failed``, ``timeout`` or ``empty``).""" + + def __init__( + self, status: SweepStatus, reason: Optional[str], message: str + ) -> None: + super().__init__(message) + self.status = status + self.reason = reason + self.message = message + + +class _EitherEvent(threading.Event): + """An event that reads as set once any of its sources is set. + + Hands the window cutter one ``cancel`` for both shutdown and abort. + """ + + def __init__(self, *sources: Optional[threading.Event]) -> None: + super().__init__() + self._sources = [source for source in sources if source is not None] + + def is_set(self) -> bool: + return super().is_set() or any(source.is_set() for source in self._sources) + + isSet = is_set + + def wait(self, timeout: Optional[float] = None) -> bool: + end = None if timeout is None else time.monotonic() + timeout + while not self.is_set(): + remaining = None if end is None else end - time.monotonic() + if remaining is not None and remaining <= 0: + return False + super().wait(0.05 if remaining is None else min(0.05, remaining)) + return True + + +ShardState = Literal["create", "unknown", "wait", "done", "missing"] + + +@dataclass +class _ShardRun: + """Progress of one shard's Request within this sweep.""" + + shard: Shard + critical: bool + spec: dict + digest: str + attempt: int = 1 + state: ShardState = "create" + name: Optional[str] = None + adopted: bool = False + # This sweep sent a create for the current attempt that may have landed; + # False for an attempt a previous leader created. + created: bool = False + reason: Optional[str] = None + message: Optional[str] = None + run_name: Optional[str] = None + run_uid: Optional[str] = None + checked_run_uid: Optional[str] = None + unaccepted_since: Optional[float] = None + pending_since: Optional[float] = None + run_pending: bool = False + rows: Optional[dict] = None + published: int = 0 + + def require_name(self) -> str: + """The current attempt's Request name. + + It is set once a create is sent or a Request is adopted, so a shard in + the ``unknown`` or ``wait`` state, where it is read, always has it. + """ + + if self.name is None: + raise RuntimeError( + f"shard {self.shard.id} has no Request name in state {self.state}" + ) + return self.name + + def summary(self) -> dict[str, Any]: + return { + "kind": self.shard.kind, + "points": self.shard.size, + "critical": self.critical, + "request": self.name, + "attempt": self.attempt, + "adopted": self.adopted, + "run": self.run_name, + "state": self.state, + "reason": self.reason, + "message": self.message, + "published": self.published, + "run_pending": self.run_pending, + } + + +@dataclass +class _Scored: + score: Optional[float] + entry: dict[str, Any] = field(default_factory=dict) + + +class _DgdrSweep: + def __init__( + self, + request: SweepRequest, + cancel: threading.Event, + abort: Optional[threading.Event], + context: DgdrContext, + started: float, + ) -> None: + self.request = request + self.cancel = cancel + self.abort = abort + self.ctx = context + self.cfg = context.cfg + self.started = started + deadline = getattr(request, "deadline_s", None) + self.deadline = ( + float(deadline) if deadline is not None else started + request.timeout_s + ) + self.window_end_ms = int(round(request.window_end_ms)) + # Registry references this sweep took; each is released at the end. + self._protections = 0 + self.claim_path = window_claim_path(self.cfg, context.dgd, self.window_end_ms) + self.local_path = os.path.join(self.cfg.mount_path, self.claim_path) + self.shards: dict[str, _ShardRun] = {} + self.cut_done = False + self.num_requests = 0 + self.row_counts: dict[str, int] = {} + self.details: dict[str, Any] = { + "window": {"path": self.claim_path, "cut": False}, + "deadline_s": self.deadline, + "generation": getattr(request, "generation", 0), + "replicas_verified": getattr(request, "replicas_verified", True), + } + self.k8s_errors = 0 + self.runtime: dict[str, Any] = {} + + # ------------------------------------------------------------ driver + + def run(self) -> SweepResult: + try: + self._prepare() + self._check_interrupt() + self._adopt() + self._create_all() + self._poll() + return self._harvest() + except _Interrupted as stop: + logger.info( + "retune window %s sweep stopped (%s); its Requests stay for the " + "next leader", + self.request.window_index, + stop.reason, + ) + return self._result( + "failed", + failure_reason=stop.reason, + error=f"sweep stopped: {stop.reason}", + ) + except _WindowFailed as failed: + self._abandon_in_flight() + log = logger.info if failed.status == "empty" else logger.warning + log( + "retune window %s %s (%s): %s", + self.request.window_index, + failed.status, + failed.reason, + failed.message, + ) + return self._result( + failed.status, + failure_reason=failed.reason, + error=failed.message if failed.status != "empty" else None, + ) + finally: + # One release per protect: the registry counts references per + # window, and another generation's sweep may hold its own. + try: + for _ in range(self._protections): + self.ctx.registry.release(self.window_end_ms) + except Exception: # pragma: no cover - registry is in-process + logger.exception("retune registry release failed") + self._protections = 0 + + def _protect(self, names: list[str]) -> None: + self.ctx.registry.protect(self.window_end_ms, names) + self._protections += 1 + + def _result( + self, + status: SweepStatus, + *, + failure_reason: Optional[str] = None, + error: Optional[str] = None, + scores: Optional[dict[str, Optional[float]]] = None, + metrics: Optional[dict[str, dict[str, Any]]] = None, + ) -> SweepResult: + details = dict(self.details) + details["shards"] = {key: run.summary() for key, run in self.shards.items()} + details["k8s_errors"] = self.k8s_errors + details.update(self.runtime) + return SweepResult( + status=status, + num_requests=self.num_requests, + row_counts=dict(self.row_counts), + scores=scores or {}, + metrics=metrics or {}, + wall_s=max(0.0, self._now() - self.started), + error=error, + failure_reason=failure_reason, + details=details, + ) + + # ----------------------------------------------------------- helpers + + def _now(self) -> float: + return self.ctx.now() + + def _timeout(self) -> float: + return max(_MIN_K8S_TIMEOUT_S, min(_K8S_TIMEOUT_S, self.deadline - self._now())) + + def _check_interrupt(self) -> None: + if self.cancel.is_set(): + raise _Interrupted("shutdown") + if self.abort is not None and self.abort.is_set(): + raise _Interrupted("lost_leadership") + + def _check_mutation(self) -> None: + self._check_interrupt() + if not self.ctx.is_leader(): + raise _Interrupted("lost_leadership") + + def _fail( + self, reason: Optional[str], message: str, status: SweepStatus = "failed" + ) -> NoReturn: + raise _WindowFailed(status, reason, message) + + def _retrying(self, what: str, call: Callable[[], Any]) -> Any: + """``call`` until it succeeds; a permanent error or the deadline fails.""" + + while True: + self._check_interrupt() + try: + return call() + except KubeError as error: + self.k8s_errors += 1 + if _permanent(error): + self._fail("k8s_error", f"{what}: {error}") + if self._now() + self.ctx.poll_interval_s >= self.deadline: + self._fail("k8s_error", f"{what}: {error}", "timeout") + logger.warning("retune %s failed, retrying: %s", what, error) + self.ctx.sleep(self.ctx.poll_interval_s) + + def _read_failed(self, what: str, error: KubeError) -> None: + """A GET failed: a permanent error fails the window; else poll again.""" + + self.k8s_errors += 1 + if _permanent(error): + self._fail("k8s_error", f"{what}: {error}") + logger.warning("retune %s failed: %s", what, error) + + def _enqueue_delete(self, names: Iterable[Optional[str]]) -> None: + present = [name for name in names if name] + if present: + self.ctx.registry.enqueue_delete(present) + + def _abandon_in_flight(self) -> None: + """Queue this window's unfinished Requests for deletion.""" + + self._enqueue_delete( + run.name + for run in self.shards.values() + if run.state in ("wait", "unknown") and run.name + ) + + # ------------------------------------------------------------- setup + + def _prepare(self) -> None: + request = self.request + domain = getattr(request, "router_domain", None) + if not domain: + self._fail("invalid_request", "the sweep request has no router domain") + if any(candidate.planner for candidate in request.candidates): + self._fail( + "invalid_request", + "a DGDR sweep searches router parameters only (Scenario 1)", + ) + keys = set(domain) + startup = self._startup_point(domain) + try: + shards = plan_shards( + domain, + baseline=request.baseline.router, + startup=startup, + max_shards=self.cfg.run.max_shards, + parallelism=self.cfg.run.parallelism, + ) + except (KeyError, ValueError) as error: + self._fail("invalid_domain", str(error)) + replicas = getattr(request, "spec_replicas", None) or request.replicas + self.details["replicas"] = dict(replicas) + probe = next((shard for shard in shards if shard.kind == "probe"), None) + for shard in shards: + try: + body = build_request( + dgd=self.ctx.dgd, + cfg=self.cfg, + retune=self.ctx.retune, + sla_ttft_ms=request.sla_ttft_ms, + sla_itl_ms=request.sla_itl_ms, + window_end_ms=self.window_end_ms, + shard=shard, + spec_replicas=replicas, + backend=self.ctx.backend, + attempt=1, + ) + except (KeyError, TypeError, ValueError) as error: + self._fail("invalid_engine", f"shard {shard.id}: {error}") + # The baseline must be scored, and the probe's grid twin is what + # the probe is compared with. + critical = ( + shard.kind == "probe" + or shard.contains(request.baseline.router) + or (probe is not None and shard.contains(probe.points[0])) + ) + self.shards[shard.id] = _ShardRun( + shard=shard, + critical=critical, + spec=body["spec"], + digest=body["metadata"]["labels"][LABEL_SPEC], + ) + uncovered = [ + candidate.router + for candidate in request.candidates + if set(candidate.router) != keys + or not any( + run.shard.kind != "probe" and run.shard.contains(candidate.router) + for run in self.shards.values() + ) + ] + if uncovered: + self.details["uncovered_candidates"] = uncovered[:10] + + def _startup_point(self, domain: Mapping[str, Sequence[float]]) -> dict[str, float]: + """The off-grid candidate other than the baseline, else the baseline. + + The controller's candidates are the grid plus the startup point and + the baseline, so an off-grid candidate is one of those two. + """ + + baseline = self.request.baseline + for candidate in self.request.candidates: + router = candidate.router + if candidate == baseline or set(router) != set(domain): + continue + if not all(float(router[key]) in domain[key] for key in domain): + return dict(router) + return dict(baseline.router) + + def _build(self, run: _ShardRun) -> dict: + body = build_request( + dgd=self.ctx.dgd, + cfg=self.cfg, + retune=self.ctx.retune, + sla_ttft_ms=self.request.sla_ttft_ms, + sla_itl_ms=self.request.sla_itl_ms, + window_end_ms=self.window_end_ms, + shard=run.shard, + spec_replicas=self.details["replicas"], + backend=self.ctx.backend, + attempt=run.attempt, + ) + rows = {"num_requests": self.num_requests, "row_counts": self.row_counts} + body["metadata"]["annotations"] = { + ROWS_ANNOTATION: json.dumps(rows, sort_keys=True, separators=(",", ":")) + } + return body + + # ------------------------------------------------------------- adopt + + def _adopt(self) -> None: + """Adopt the latest attempt of each shard; queue the rest for deletion. + + Each adopted attempt is observed once here: a Succeeded one needs no + cut, and a failed one is replaced while there is time (``_run_failed``). + """ + + selector = ( + f"{LABEL_DGD_UID}={self.ctx.dgd.uid},{LABEL_WINDOW}={self.window_end_ms}" + ) + existing = self._retrying( + "list window Requests", + lambda: self.ctx.kube.list_requests(selector, timeout_s=self._timeout()), + ) + stale: list[Optional[str]] = [] + latest: dict[str, dict] = {} + for obj in existing: + metadata = obj.get("metadata") or {} + labels = metadata.get("labels") or {} + name = metadata.get("name") + run = self.shards.get(labels.get(LABEL_SHARD, "")) + if run is None or labels.get(LABEL_SPEC) != run.digest: + stale.append(name) + continue + attempt = _int(labels.get(LABEL_ATTEMPT), 0) + if metadata.get("deletionTimestamp"): + # Being deleted: never adopt it, and never reuse its attempt. + run.attempt = max(run.attempt, attempt + 1) + continue + current = latest.get(run.shard.id) + if current is None or attempt > _int( + current["metadata"]["labels"].get(LABEL_ATTEMPT), 0 + ): + if current is not None: + stale.append(current["metadata"]["name"]) + latest[run.shard.id] = obj + else: + stale.append(name) + self._enqueue_delete(stale) + for shard_id, obj in latest.items(): + run = self.shards[shard_id] + metadata = obj["metadata"] + run.name = metadata["name"] + run.attempt = max( + run.attempt, _int(metadata["labels"].get(LABEL_ATTEMPT), 1) + ) + run.adopted = True + run.state = "wait" + run.rows = _rows(metadata.get("annotations")) + logger.info( + "retune window %s adopts Request %s for shard %s", + self.request.window_index, + run.name, + shard_id, + ) + if latest: + self._protect([obj["metadata"]["name"] for obj in latest.values()]) + # Classify the adopted Requests: a window whose shards all + # succeeded is harvested without cutting it again. + for run in self.shards.values(): + if run.state == "wait": + self._observe(run) + + # ------------------------------------------------------------ create + + def _create_all(self) -> None: + pending = [run for run in self._ordered() if run.state == "create"] + if not pending: + rows = next( + (run.rows for run in self.shards.values() if run.rows is not None), + None, + ) + if rows is not None: + self.num_requests = _int(rows.get("num_requests"), 0) + self.row_counts = dict(rows.get("row_counts") or {}) + return + # Gate every shard before the cut, so a critical shard that cannot + # finish fails the window without spending the cut budget. + for run in pending: + self._too_late(run) + for run in pending: + if run.state == "create": + self._create(run) + + def _ordered(self) -> list[_ShardRun]: + """Critical shards first, so they get the time that is left.""" + + return sorted(self.shards.values(), key=lambda run: not run.critical) + + def _too_late(self, run: _ShardRun) -> bool: + """Apply the launch gate; True when the shard was given up.""" + + need = need_seconds(run.shard, self.cfg, needs_cut=not self.cut_done) + left = self.deadline - self._now() + if left >= need: + return False + message = ( + f"shard {run.shard.id} needs {need:.0f}s but {max(left, 0.0):.0f}s are " + "left before the deadline" + ) + if run.critical: + self._fail("launch_too_late", message, "timeout") + run.state = "missing" + run.reason = "launch_too_late" + run.message = message + logger.warning("retune window %s: %s", self.request.window_index, message) + return True + + def _create(self, run: _ShardRun) -> None: + self._check_mutation() + if self._too_late(run): + return + if not self.cut_done: + self._cut() + if self._too_late(run): + return + self._check_mutation() + body = self._build(run) + name = body["metadata"]["name"] + run.name = name + run.rows = None + self._protect([name]) + try: + created = self.ctx.kube.create_request(body, timeout_s=self._timeout()) + except KubeAlreadyExists: + self._adopt_existing(run) + return + except KubeError as error: + self.k8s_errors += 1 + if _admission_denied(error): + self._fail("request_rejected", f"Request {name} rejected: {error}") + if isinstance(error, KubeInvalid): + self._fail("schema_unsupported", f"Request {name} rejected: {error}") + if _permanent(error): + self._fail("k8s_error", f"create Request {name}: {error}") + # The create may have landed; look it up on the next poll. + run.created = True + run.state = "unknown" + run.message = f"create: {error}" + logger.warning("retune create of Request %s failed: %s", name, error) + return + run.created = True + dropped = missing_fields(created.get("spec"), body["spec"], "spec") + labels = (created.get("metadata") or {}).get("labels") or {} + dropped += [ + f"metadata.labels.{key}" + for key, value in body["metadata"]["labels"].items() + if labels.get(key) != value + ] + if dropped: + self._enqueue_delete([name]) + run.state = "missing" + self._fail( + "schema_unsupported", + f"the API server dropped or changed {sorted(dropped)} of Request {name}", + ) + run.state = "wait" + run.unaccepted_since = self._now() + logger.info( + "retune window %s created Request %s (shard %s, %d points, attempt %d)", + self.request.window_index, + name, + run.shard.id, + run.shard.size, + run.attempt, + ) + + def _adopt_existing(self, run: _ShardRun) -> None: + """The name exists (an earlier create landed): adopt it if it is ours.""" + + existing = self._retrying( + f"get Request {run.name}", + lambda: self.ctx.kube.get_request( + run.require_name(), timeout_s=self._timeout() + ), + ) + if existing is None: + # Deleted in between; create it again on the next poll. + run.state = "create" + return + labels = (existing.get("metadata") or {}).get("labels") or {} + if ( + labels.get(LABEL_SPEC) == run.digest + and labels.get(LABEL_DGD_UID) == self.ctx.dgd.uid + and labels.get(LABEL_SHARD) == run.shard.id + ): + run.state = "wait" + run.adopted = True + run.unaccepted_since = self._now() + return + self._shard_failed( + run, "request_conflict", f"Request {run.name} exists with other labels" + ) + + # ------------------------------------------------------------ window + + def _cut(self) -> None: + """Cut the window file; an empty window ends the sweep as ``empty``.""" + + self._check_interrupt() + segments_root = ( + self.request.trace_paths[0] + if self.request.trace_paths + else os.path.join( + self.cfg.mount_path, self.cfg.segments_dir, self.ctx.dgd.name + ) + ) + budget = min(self.cfg.cut_budget_seconds, self.deadline - self._now()) + if budget <= 0: + self._fail("launch_too_late", "no time left to cut the window", "timeout") + # WindowCutter takes the deadline as absolute wall-clock seconds. + cut_deadline_s = self._now() + budget + cancel = _EitherEvent(self.cancel, self.abort) + try: + os.makedirs(os.path.dirname(self.local_path), exist_ok=True) + result: WindowCutResult = self.ctx.cut_window( + segments_root, + int(round(self.request.window_start_ms)), + self.window_end_ms, + int(round(self.request.as_of_ms)), + self.local_path, + max_rows=self.cfg.max_window_rows, + deadline_s=cut_deadline_s, + cancel=cancel, + ) + except Exception as error: + # A cut cancelled by shutdown or abort reports which one it was. + self._check_interrupt() + reason = getattr(error, "reason", None) + if reason not in ( + "window_cut_timeout", + "window_too_large", + "trace_corrupt", + "window_cut_failed", + ): + reason = "window_cut_failed" + self._fail(reason, f"window cut: {type(error).__name__}: {error}") + self.cut_done = True + self.num_requests = int(result.num_requests) + self.row_counts = dict(result.row_counts) + self.details["window"].update( + cut=True, + num_requests=result.num_requests, + reused=result.reused, + dropped_members=result.dropped_members, + dropped_bytes=result.dropped_bytes, + trace_block_size=result.trace_block_size, + ) + if result.path is None or result.num_requests == 0: + self._fail(None, "empty window", "empty") + + # -------------------------------------------------------------- wait + + def _poll(self) -> None: + while True: + self._check_interrupt() + active = [ + run + for run in self._ordered() + if run.state in ("create", "unknown", "wait") + ] + if not active: + return + if self._now() >= self.deadline: + pending = [run.shard.id for run in active if run.run_pending] + self._fail( + "run_pending" if pending else "timeout", + "the deadline passed with shards " + f"{sorted(run.shard.id for run in active)} unfinished" + + (f" (Runs pending: {sorted(pending)})" if pending else ""), + "timeout", + ) + for run in active: + self._check_interrupt() + if run.state == "create": + self._create(run) + elif run.state == "unknown": + self._resolve_unknown(run) + else: + self._observe(run) + if any( + run.state in ("create", "unknown", "wait") + for run in self.shards.values() + ): + left = self.deadline - self._now() + self.ctx.sleep(max(0.0, min(self.ctx.poll_interval_s, left))) + + def _resolve_unknown(self, run: _ShardRun) -> None: + """After a create whose outcome is unknown: adopt it or create again.""" + + try: + existing = self.ctx.kube.get_request( + run.require_name(), timeout_s=self._timeout() + ) + except KubeError as error: + self._read_failed(f"get Request {run.name}", error) + return + if existing is None: + run.state = "create" + return + self._adopt_existing(run) + + def _shard_failed(self, run: _ShardRun, reason: str, message: str) -> None: + """A shard ended without scores; its Request (if any) is kept for audit. + + A critical shard fails the window; any other leaves its points unscored. + """ + + run.state = "missing" + run.reason = reason + run.message = message + if run.critical: + self._fail(reason, f"shard {run.shard.id}: {message}") + logger.warning( + "retune window %s shard %s %s: %s; its points are unscored", + self.request.window_index, + run.shard.id, + reason, + message, + ) + + def _observe(self, run: _ShardRun) -> None: + """One look at a shard's Request and Run (the §4.5.3 state machine).""" + + kube = self.ctx.kube + try: + obj = kube.get_request(run.require_name(), timeout_s=self._timeout()) + except KubeError as error: + self._read_failed(f"get Request {run.name}", error) + return + if obj is None or (obj.get("metadata") or {}).get("deletionTimestamp"): + self._shard_failed(run, "request_deleted", f"Request {run.name} is gone") + return + if run.rows is None: + run.rows = _rows((obj.get("metadata") or {}).get("annotations")) + generation = _int((obj.get("metadata") or {}).get("generation"), 1) + accepted = _condition(obj, "Accepted") + if accepted is None or _int(accepted.get("observedGeneration"), 0) < generation: + if run.unaccepted_since is None: + run.unaccepted_since = self._now() + elif self._now() - run.unaccepted_since >= ACCEPT_TIMEOUT_S: + self._fail( + "operator_not_reconciling", + f"Request {run.name} has no Accepted condition after " + f"{ACCEPT_TIMEOUT_S:.0f}s", + ) + return + run.unaccepted_since = None + if accepted.get("status") != "True": + if accepted.get("reason") == "InvalidSpec": + self._fail( + "invalid_spec", + f"Request {run.name}: {accepted.get('message', '')}", + ) + # RunCreating, RunNameConflict, CancelingSupersededRun: wait. + return + try: + found = self._find_run(obj) + except KubeError as error: + self._read_failed(f"get the Run of Request {run.name}", error) + return + if found is None: + return # created, not yet visible + run.run_name = found["metadata"]["name"] + run_uid = found["metadata"].get("uid") + status = found.get("status") or {} + completed = _condition(found, "Completed") or {} + terminal = completed.get("status") == "True" + succeeded = terminal and completed.get("reason") == "SearchSucceeded" + outcome = ( + f"Run {run.run_name} {completed.get('reason')}: " + f"{completed.get('message', '')}" + ) + if not terminal: + self._track_pending(run, status) + if run.checked_run_uid != run_uid: + effective = _get(status, ("aisimulate", "input")) + if not isinstance(effective, Mapping): + if not terminal: + return # the input is recorded together with jobRef + if not succeeded: + self._run_failed(run, outcome) + return + self._input_mismatch(run, ["status.aisimulate.input is missing"]) + return + if not self._check_input(run, effective): + return + run.checked_run_uid = run_uid + if succeeded: + run.state = "done" + run.run_uid = run_uid + elif terminal: + self._run_failed(run, outcome) + + def _track_pending(self, run: _ShardRun, status: Mapping[str, Any]) -> None: + """Flag a live Run that has had no Job for a long time (``run_pending``). + + The operator writes ``status.aisimulate.input`` and ``jobRef`` in one + status patch once the Job exists; until then the Run has no status + at all, so this runs before the input is read back. + """ + + if status.get("jobRef") or status.get("startTime"): + run.pending_since = None + run.run_pending = False + return + now = self._now() + if run.pending_since is None: + run.pending_since = now + elif now - run.pending_since >= max( + self.cfg.job_startup_seconds, self.ctx.poll_interval_s + ): + if not run.run_pending: + logger.warning( + "retune Run %s has no Job after %.0fs (run_pending)", + run.run_name, + now - run.pending_since, + ) + run.run_pending = True + + def _run_failed(self, run: _ShardRun, outcome: str) -> None: + """A Run ended without success. + + An attempt a previous leader created is replaced while there is time + (design §4.5.3 adopts only a live or Succeeded attempt); otherwise + the shard fails. + """ + + if run.adopted and not run.created: + name = run.name + if self._replace_attempt(run, outcome): + logger.warning( + "retune window %s replaces adopted Request %s of shard %s: %s", + self.request.window_index, + name, + run.shard.id, + outcome, + ) + return + self._shard_failed(run, "run_failed", outcome) + + def _find_run(self, obj: Mapping[str, Any]) -> Optional[dict]: + """The Run of the Request's current generation, by the operator's refs.""" + + metadata = obj.get("metadata") or {} + status = obj.get("status") or {} + generation = _int(metadata.get("generation"), 1) + names: list[str] = [] + active = (status.get("activeRunRef") or {}).get("name") + if active: + names.append(active) + names += [ + ref.get("name") + for ref in status.get("recentRunRefs") or () + if ref.get("name") + ] + names.append(f"{metadata.get('name')}-{generation}") + uid = metadata.get("uid") + for name in dict.fromkeys(names): + found = self.ctx.kube.get_run(name, timeout_s=self._timeout()) + if found is None: + continue + run_meta = found.get("metadata") or {} + annotations = run_meta.get("annotations") or {} + owners = run_meta.get("ownerReferences") or () + if annotations.get(REQUEST_GENERATION_ANNOTATION) == str( + generation + ) and any(owner.get("uid") == uid for owner in owners): + return found + return None + + def _check_input(self, run: _ShardRun, effective: Mapping[str, Any]) -> bool: + """Read back the Run's lowered input; False when the shard moved on.""" + + dgdr_run = effective.get("dgdr_run") or {} + image = dgdr_run.get("runtime_image") + version = dgdr_run.get("runtime_version_override") + if image or version: + self.runtime = {"runtime_image": image, "runtime_version": version} + problems = input_mismatches(run.spec, effective) + if problems: + self._input_mismatch(run, problems) + return False + if crn_unsupported(version, image, self.ctx.crn_min_version): + self._fail( + "crn_unsupported", + f"Run {run.run_name} runtime {version or image} predates CRN " + f"trace replay ({self.ctx.crn_min_version})", + ) + return True + + def _input_mismatch(self, run: _ShardRun, problems: list[str]) -> None: + """An adopted attempt is replaced while there is time; else it fails.""" + + message = f"Run {run.run_name} input differs: {'; '.join(problems[:5])}" + logger.warning("retune window %s: %s", self.request.window_index, message) + if run.adopted and self._replace_attempt(run, message): + return + self._enqueue_delete([run.name]) + run.state = "missing" + if run.critical: + self._fail("input_mismatch", message) + run.reason = "input_mismatch" + run.message = message + + def _replace_attempt(self, run: _ShardRun, message: str) -> bool: + """Queue the attempt for deletion and create attempt+1, if it fits. + + False (nothing changed) when ``D_k - now < need(shard)``. + """ + + need = need_seconds(run.shard, self.cfg, needs_cut=not self.cut_done) + if self.deadline - self._now() < need: + return False + self._enqueue_delete([run.name]) + run.attempt += 1 + run.state = "create" + run.adopted = False + run.created = False + run.name = None + run.run_name = None + run.checked_run_uid = None + run.unaccepted_since = None + run.pending_since = None + run.run_pending = False + run.message = message + return True + + # ----------------------------------------------------------- harvest + + def _harvest(self) -> SweepResult: + request = self.request + domain_keys = set(getattr(request, "router_domain", None) or {}) + results: dict[str, _Scored] = {} + probe_results: dict[str, _Scored] = {} + engine_signature: Optional[str] = None + signature_source: Optional[str] = None + unmapped: list[str] = [] + for run in self._ordered(): + if run.state != "done": + continue + selector = f"{RUN_UID_LABEL}={run.run_uid}" + candidates = self._retrying( + f"list Candidates of Run {run.run_name}", + functools.partial(self._list_candidates, selector), + ) + run.published = len(candidates) + for candidate in sorted( + candidates, key=lambda obj: (obj.get("metadata") or {}).get("name", "") + ): + name = (candidate.get("metadata") or {}).get("name", "") + parameters = _get(candidate, ("spec", "parameters")) + try: + point, signature = _map_parameters(parameters, run.shard) + except _Unmapped as error: + if error.fatal: + self._fail("engine_not_pinned", f"Candidate {name}: {error}") + unmapped.append(f"{name}: {error}") + continue + if engine_signature is None: + engine_signature, signature_source = signature, name + elif signature != engine_signature: + self._fail( + "engine_not_pinned", + f"Candidates {signature_source} and {name} differ in " + "non-router parameters", + ) + key = RetuneParams(router=point).key() + target = probe_results if run.shard.kind == "probe" else results + if key in target: + continue + metrics = _get(candidate, ("status", "metrics"), {}) or {} + entry: dict[str, Any] = { + "params": RetuneParams(router=point).to_dict(), + "shard": run.shard.id, + "candidate": name, + } + try: + score, info = score_summary( + metrics, request.window_s, request.objective + ) + except ValueError as error: + entry["error"] = str(error) + target[key] = _Scored(None, entry) + continue + entry.update(info) + target[key] = _Scored(score, entry) + if unmapped: + self.details["unmapped_candidates"] = unmapped[:10] + self._check_probe(results, probe_results) + + scores: dict[str, Optional[float]] = {} + metrics_out: dict[str, dict[str, Any]] = {} + missing = 0 + for candidate in request.candidates: + key = candidate.key() + found = results.get(key) + if found is not None and set(candidate.router) == domain_keys: + scores[key] = found.score + metrics_out[key] = found.entry + else: + scores[key] = None + metrics_out[key] = { + "params": candidate.to_dict(), + "error": self._missing_reason(candidate.router), + } + if scores[key] is None: + missing += 1 + self.details["missing_candidates"] = missing + logger.info( + "retune window %s harvested %d of %d candidates from %d Requests", + request.window_index, + len(scores) - missing, + len(scores), + sum(1 for run in self.shards.values() if run.state == "done"), + ) + return self._result("ok", scores=scores, metrics=metrics_out) + + def _list_candidates(self, selector: str) -> list[dict]: + return self.ctx.kube.list_candidates(selector, timeout_s=self._timeout()) + + def _missing_reason(self, point: Mapping[str, float]) -> str: + for run in self.shards.values(): + if run.shard.kind != "probe" and run.shard.contains(point): + if run.state == "missing": + return f"shard {run.shard.id} {run.reason}: {run.message}" + return f"not published by Run {run.run_name} (infeasible or failed)" + return "not covered by any shard" + + def _check_probe( + self, results: Mapping[str, _Scored], probe_results: Mapping[str, _Scored] + ) -> None: + probe = self.shards.get("probe") + if probe is None: + return + point = probe.shard.points[0] + key = RetuneParams(router=point).key() + grid = results.get(key) + repeat = probe_results.get(key) + grid_score = None if grid is None else grid.score + probe_score = None if repeat is None else repeat.score + report: dict[str, Any] = { + "point": point, + "grid_score": grid_score, + "probe_score": probe_score, + } + self.details["crn_probe"] = report + if grid_score is None or probe_score is None: + self._fail( + "crn_mismatch", + f"CRN probe point {point} is unscored (grid {grid_score}, " + f"probe {probe_score})", + ) + scale = max(abs(grid_score), abs(probe_score)) + error = 0.0 if scale == 0 else abs(grid_score - probe_score) / scale + report["relative_error"] = error + if not error <= CRN_REL_TOLERANCE: + self._fail( + "crn_mismatch", + f"CRN probe point {point} scored {probe_score!r}, its grid " + f"evaluation {grid_score!r} (relative error {error:.3g})", + ) + + +class _Unmapped(Exception): + """A Candidate that maps to no point; ``fatal`` when the engine drifted.""" + + def __init__(self, message: str, *, fatal: bool = False) -> None: + super().__init__(message) + self.fatal = fatal + + +def _map_parameters(parameters: Any, shard: Shard) -> tuple[dict[str, float], str]: + """A Candidate's live router point and its non-router signature. + + The signature is the canonical JSON of the parameters without the three + searched knobs; every Candidate of a window must share it. Raises + ``_Unmapped`` for parameters that name no point of ``shard``, fatally + for a router other than the pinned ``kv_router`` without a load model. + """ + + if not isinstance(parameters, Mapping): + raise _Unmapped("spec.parameters is not an object") + adapters = parameters.get("adapters") + if not isinstance(adapters, Mapping): + raise _Unmapped("spec.parameters.adapters is missing") + router = adapters.get(ROUTER_ADAPTER) + if not isinstance(router, Mapping): + raise _Unmapped(f"spec.parameters.adapters[{ROUTER_ADAPTER!r}] is missing") + load_model = router.get("prefill_load_model") + load_type = ( + load_model.get("type") if isinstance(load_model, Mapping) else load_model + ) + if router.get("policy") != "kv_router" or load_type != "none": + raise _Unmapped( + f"router policy {router.get('policy')!r} with prefill load model " + f"{load_type!r}; the Request pinned kv_router and none", + fatal=True, + ) + point: dict[str, float] = {} + for public, live in DGDR_ROUTER_KNOBS.items(): + value = router.get(public) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise _Unmapped(f"router knob {public} is {value!r}") + snapped = _snap(float(value), shard.choices.get(live, ())) + if snapped is None: + raise _Unmapped( + f"router knob {public}={value!r} is not one of the shard's values" + ) + point[live] = snapped + rest = dict(parameters) + rest["adapters"] = { + **adapters, + ROUTER_ADAPTER: { + key: value for key, value in router.items() if key not in DGDR_ROUTER_KNOBS + }, + } + signature = json.dumps(rest, sort_keys=True, separators=(",", ":"), default=str) + return point, signature + + +def _snap(value: float, values: Sequence[float]) -> Optional[float]: + """The domain value ``value`` denotes, allowing for float round-trips.""" + + for candidate in values: + if abs(value - candidate) <= _SNAP_TOLERANCE * max(1.0, abs(candidate)): + return candidate + return None + + +def _condition(obj: Mapping[str, Any], kind: str) -> Optional[dict]: + for condition in _get(obj, ("status", "conditions"), ()) or (): + if isinstance(condition, Mapping) and condition.get("type") == kind: + return dict(condition) + return None + + +def _permanent(error: KubeError) -> bool: + """A Kubernetes error a retry cannot fix (bad request, auth, missing API).""" + + return getattr(error, "status", None) in _PERMANENT_STATUSES or isinstance( + error, (KubeInvalid, KubeNotFound) + ) + + +def _admission_denied(error: KubeError) -> bool: + """A permanent rejection whose message comes from an admission webhook.""" + + return _permanent(error) and _ADMISSION_DENIED.search(str(error)) is not None + + +def _int(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _rows(annotations: Any) -> Optional[dict]: + if not isinstance(annotations, Mapping) or ROWS_ANNOTATION not in annotations: + return None + try: + rows = json.loads(annotations[ROWS_ANNOTATION]) + except (TypeError, ValueError): + return None + return rows if isinstance(rows, dict) else None diff --git a/components/src/dynamo/planner/retune/live/kube.py b/components/src/dynamo/planner/retune/live/kube.py new file mode 100644 index 000000000000..359d3aa92f98 --- /dev/null +++ b/components/src/dynamo/planner/retune/live/kube.py @@ -0,0 +1,397 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``contracts.RetuneKube`` over the official ``kubernetes`` client. + +Every call is synchronous (callers on the event loop use +``asyncio.to_thread``), carries ``_request_timeout=(connect, read)`` and +raises the ``contracts.Kube*`` errors instead of client exceptions. The +error carries the HTTP status as ``error.status`` (``None`` for a network +error or timeout), so a caller can tell 403 from the rest. + +Custom resources (the DGD at v1beta1, DGDR v1beta2 Requests, Runs and +Candidates) come back as the API server's JSON. Leases are read without the +client's model deserialization (``_preload_content=False``) so a Lease dict +round-trips unchanged: deserializing turns ``renewTime`` into a ``datetime`` +whose ``isoformat`` drops a zero fraction, which a later replace would send +back in a form the ``MicroTime`` parser rejects. +""" + +from __future__ import annotations + +import json +import logging +import math +from collections.abc import Callable, Mapping +from typing import Any, Literal, Optional, TypeVar + +from kubernetes import client +from kubernetes.client import ApiException + +from dynamo.planner.retune.live.contracts import ( + DGDR_CANDIDATES, + DGDR_GROUP, + DGDR_REQUESTS, + DGDR_RUNS, + DGDR_VERSION, + KubeAlreadyExists, + KubeConflict, + KubeError, + KubeInvalid, + KubeNotFound, + PodInfo, +) + +logger = logging.getLogger(__name__) + +DGD_VERSION = "v1beta1" +DGD_PLURAL = "dynamographdeployments" +# The frontend's container and HTTP port names (deploy/operator/internal/consts). +MAIN_CONTAINER_NAME = "main" +HTTP_PORT_NAME = "http" +FIELD_MANAGER = "dynamo-planner-retune" + +_T = TypeVar("_T") +_Operation = Literal["read", "create", "update", "delete"] + + +def request_timeout(timeout_s: float) -> tuple[int, int]: + """The client's ``(connect, read)`` timeout for ``timeout_s``, at least 1 s each.""" + + if not math.isfinite(timeout_s) or timeout_s <= 0: + seconds = 1 + else: + seconds = max(1, int(math.ceil(timeout_s))) + return (seconds, seconds) + + +def _kube_error(error: Exception, operation: _Operation, what: str) -> KubeError: + """Map a client exception to the contracts error for ``operation``.""" + + if isinstance(error, ApiException): + status = error.status + detail = _api_message(error) + message = f"{operation} {what}: HTTP {status}: {detail}" + mapped: KubeError + if status == 404: + mapped = KubeNotFound(message) + elif status == 409 and operation == "create": + mapped = KubeAlreadyExists(message) + elif status == 409: + mapped = KubeConflict(message) + elif status in (400, 422): + mapped = KubeInvalid(message) + else: + mapped = KubeError(message) + mapped.status = status # type: ignore[attr-defined] + return mapped + mapped = KubeError(f"{operation} {what}: {type(error).__name__}: {error}") + mapped.status = None # type: ignore[attr-defined] + return mapped + + +def _api_message(error: ApiException) -> str: + """The ``Status.message`` of an API error body, else its reason.""" + + body = error.body + if isinstance(body, bytes): + body = body.decode("utf-8", errors="replace") + if isinstance(body, str) and body: + try: + parsed = json.loads(body) + except ValueError: + return body[:500] + if isinstance(parsed, dict) and parsed.get("message"): + return str(parsed["message"])[:500] + return str(error.reason or "") + + +def _pod_info(pod: Any) -> PodInfo: + """``PodInfo`` of a ``V1Pod``.""" + + metadata = pod.metadata + status = pod.status + ready = any( + condition.type == "Ready" and condition.status == "True" + for condition in (status.conditions if status is not None else None) or () + ) + return PodInfo( + name=metadata.name, + uid=metadata.uid, + ip=status.pod_ip if status is not None else None, + ready=ready, + terminating=metadata.deletion_timestamp is not None, + labels=dict(metadata.labels or {}), + http_port=_http_port(pod.spec), + phase=status.phase if status is not None else None, + ) + + +def _http_port(spec: Any) -> Optional[int]: + """The ``http`` container port of the main container, else of any container.""" + + containers = list((spec.containers if spec is not None else None) or ()) + containers.sort(key=lambda container: container.name != MAIN_CONTAINER_NAME) + for container in containers: + for port in container.ports or (): + if port.name == HTTP_PORT_NAME: + return int(port.container_port) + return None + + +def _decode(response: Any) -> dict: + """The JSON body of a ``_preload_content=False`` response.""" + + data = response.data + if isinstance(data, bytes): + data = data.decode("utf-8") + return json.loads(data) if data else {} + + +class KubernetesRetuneKube: + """``contracts.RetuneKube`` for one namespace. + + ``api_client`` defaults to a new ``ApiClient`` on the default + configuration, which ``KubernetesAPI`` loads (in-cluster or kubeconfig); + ``from_kube_api`` reuses that connector's client instead. + """ + + def __init__(self, namespace: str, api_client: Optional[client.ApiClient] = None): + if not namespace: + raise ValueError("namespace must be set") + self.namespace = namespace + self.api_client = api_client if api_client is not None else client.ApiClient() + self._custom = client.CustomObjectsApi(self.api_client) + self._core = client.CoreV1Api(self.api_client) + self._coordination = client.CoordinationV1Api(self.api_client) + + @classmethod + def from_kube_api(cls, kube_api: Any) -> KubernetesRetuneKube: + """Share a ``KubernetesAPI``'s client and namespace.""" + + return cls( + kube_api.current_namespace, + api_client=kube_api.custom_api.api_client, + ) + + # ------------------------------------------------------------ helpers + + def _call(self, operation: _Operation, what: str, function: Callable[[], _T]) -> _T: + try: + return function() + except KubeError: + raise + except Exception as error: # ApiException, urllib3 and socket errors + raise _kube_error(error, operation, what) from error + + def _get_custom( + self, version: str, plural: str, name: str, timeout_s: float + ) -> Optional[dict]: + try: + return self._call( + "read", + f"{plural}/{name}", + lambda: self._custom.get_namespaced_custom_object( + group=DGDR_GROUP, + version=version, + namespace=self.namespace, + plural=plural, + name=name, + _request_timeout=request_timeout(timeout_s), + ), + ) + except KubeNotFound: + return None + + def _list_custom( + self, version: str, plural: str, label_selector: str, timeout_s: float + ) -> list[dict]: + result = self._call( + "read", + f"{plural}?labelSelector={label_selector}", + lambda: self._custom.list_namespaced_custom_object( + group=DGDR_GROUP, + version=version, + namespace=self.namespace, + plural=plural, + label_selector=label_selector, + _request_timeout=request_timeout(timeout_s), + ), + ) + return list(result.get("items") or []) + + # ----------------------------------------------------------------- DGD + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + found = self._get_custom(DGD_VERSION, DGD_PLURAL, name, timeout_s) + if found is None: + error = KubeNotFound( + f"DynamoGraphDeployment {self.namespace}/{name} not found" + ) + error.status = 404 # type: ignore[attr-defined] + raise error + return found + + # ---------------------------------------------------------------- pods + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + result = self._call( + "read", + f"pods?labelSelector={label_selector}", + lambda: self._core.list_namespaced_pod( + namespace=self.namespace, + label_selector=label_selector, + _request_timeout=request_timeout(timeout_s), + ), + ) + return [_pod_info(pod) for pod in result.items or ()] + + # ---------------------------------------------------- DGDR v1beta2 API + + def create_request( + self, + body: dict, + *, + dry_run: bool = False, + timeout_s: float = 10.0, + ) -> dict: + kwargs: dict[str, Any] = { + "field_validation": "Strict", + "field_manager": FIELD_MANAGER, + "_request_timeout": request_timeout(timeout_s), + } + if dry_run: + kwargs["dry_run"] = "All" + name = (body.get("metadata") or {}).get("name", "") + return self._call( + "create", + f"{DGDR_REQUESTS}/{name}", + lambda: self._custom.create_namespaced_custom_object( + group=DGDR_GROUP, + version=DGDR_VERSION, + namespace=self.namespace, + plural=DGDR_REQUESTS, + body=body, + **kwargs, + ), + ) + + def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + return self._get_custom(DGDR_VERSION, DGDR_REQUESTS, name, timeout_s) + + def list_requests( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + return self._list_custom(DGDR_VERSION, DGDR_REQUESTS, label_selector, timeout_s) + + def delete_request( + self, + name: str, + *, + propagation: Literal["Background", "Foreground"] = "Background", + timeout_s: float = 10.0, + ) -> None: + try: + self._call( + "delete", + f"{DGDR_REQUESTS}/{name}", + lambda: self._custom.delete_namespaced_custom_object( + group=DGDR_GROUP, + version=DGDR_VERSION, + namespace=self.namespace, + plural=DGDR_REQUESTS, + name=name, + propagation_policy=propagation, + _request_timeout=request_timeout(timeout_s), + ), + ) + except KubeNotFound: + pass + + def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + return self._get_custom(DGDR_VERSION, DGDR_RUNS, name, timeout_s) + + def list_candidates( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + return self._list_custom( + DGDR_VERSION, DGDR_CANDIDATES, label_selector, timeout_s + ) + + # -------------------------------------------------------------- Leases + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + try: + return self._call( + "read", + f"leases/{name}", + lambda: _decode( + self._coordination.read_namespaced_lease( + name=name, + namespace=self.namespace, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + except KubeNotFound: + return None + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + name = (body.get("metadata") or {}).get("name", "") + return self._call( + "create", + f"leases/{name}", + lambda: _decode( + self._coordination.create_namespaced_lease( + namespace=self.namespace, + body=_lease_body(body), + field_manager=FIELD_MANAGER, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + return self._call( + "update", + f"leases/{name}", + lambda: _decode( + self._coordination.replace_namespaced_lease( + name=name, + namespace=self.namespace, + body=_lease_body(body), + field_manager=FIELD_MANAGER, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + """Missing Leases are not an error.""" + + try: + self._call( + "delete", + f"leases/{name}", + lambda: self._coordination.delete_namespaced_lease( + name=name, + namespace=self.namespace, + _request_timeout=request_timeout(timeout_s), + ), + ) + except KubeNotFound: + pass + + +def _lease_body(body: Mapping[str, Any]) -> dict: + """A Lease body with its type fields set (the API server requires them).""" + + lease = dict(body) + lease.setdefault("apiVersion", "coordination.k8s.io/v1") + lease.setdefault("kind", "Lease") + return lease diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py new file mode 100644 index 000000000000..3fb2c4df72bb --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -0,0 +1,1884 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: E402 +# Optional-dependency preflight must run before the executor imports the sweep. + +"""Unit tests for the live retune DGDR executor (``retune.live.dgdr``). + +Each sweep runs against ``FakeRetuneKube`` with ``FakeDgdrOperator`` +advancing every Request one phase per ``sleep`` of a fake clock, so the +Request -> Run -> Candidate lifecycle, adoption, failures and timeouts are +all deterministic. +""" + +from __future__ import annotations + +import itertools +import math +import threading +from collections.abc import Callable, Mapping +from typing import Any, Optional + +import pytest + +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from dynamo.planner.config.planner_config import ( + DgdrExecutorConfig, + RetuneConfig, + RetuneEngineSpec, +) +from dynamo.planner.core.types import RetuneParams +from dynamo.planner.retune.live import dgdr +from dynamo.planner.retune.live.contracts import ( + LABEL_ATTEMPT, + LABEL_DGD_UID, + LABEL_SHARD, + LABEL_SPEC, + LABEL_WINDOW, + NON_DECISION_FAILURES, + DgdRef, + KubeAlreadyExists, + KubeError, + KubeInvalid, + KubeNotFound, + WindowCutResult, +) +from dynamo.planner.retune.live.dgdr import ( + DgdrContext, + Shard, + build_request, + crn_unsupported, + input_mismatches, + missing_fields, + need_seconds, + plan_shards, + request_name, + run_dgdr_sweep, + spec_hash, +) +from dynamo.planner.retune.sweep import SweepRequest, score_summary +from dynamo.planner.tests.live_retune_fakes import ( + FakeDgdrOperator, + FakeRetuneKube, + default_metrics, + lower_request_input, +) + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +CREDIT = "overlap_score_credit" +SCALE = "prefill_load_scale" +TEMP = "router_temperature" +LIVE_TO_PUBLIC = {CREDIT: CREDIT, SCALE: SCALE, TEMP: "temperature"} +DEFAULT_DOMAIN = { + CREDIT: (0.0, 0.5, 1.0), + SCALE: (0.0, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0), + TEMP: (0.0, 0.2, 0.5, 1.0), +} +SMALL_DOMAIN = {CREDIT: (0.5, 1.0), SCALE: (1.0, 2.0), TEMP: (0.0, 0.5)} +WINDOW_END_MS = 1_793_000_000_000 +INTERVAL_S = 3600.0 +GRACE_S = 120.0 +TIMEOUT_S = 3000.0 +LAUNCH_S = WINDOW_END_MS / 1000.0 + GRACE_S +POLL_S = 5.0 +DGD = DgdRef(name="llm", namespace="test-ns", uid="dgd-uid-1", dynamo_namespace="dyn") +AGG_ENGINE = { + "mode": "aggregated", + "backend": "vllm", + "context_length": 32768, + "workers": { + "aggregated": { + "parallelism": { + "preset": [ + { + "tensor": 2, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, + } + ] + } + }, + "prefill": None, + "decode": None, + }, +} +DISAGG_ENGINE = { + "mode": "disaggregated", + "backend": "vllm", + "workers": { + "aggregated": None, + "prefill": { + "parallelism": { + "preset": [ + { + "tensor": 1, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, + } + ] + } + }, + "decode": { + "parallelism": { + "preset": [ + { + "tensor": 2, + "pipeline": 1, + "attention_data": 2, + "moe_tensor": 1, + "moe_expert": 1, + } + ] + } + }, + }, +} + + +# ------------------------------------------------------------------ helpers + + +def make_cfg(**overrides: Any) -> DgdrExecutorConfig: + values: dict[str, Any] = { + "claim_name": "request-traces", + "model": "Qwen/Qwen3-32B", + "sku": "h200_sxm", + "engine": AGG_ENGINE, + "cut_budget_seconds": 100.0, + "job_startup_seconds": 60.0, + "run": {"parallelism": 8, "candidate_timeout_seconds": 300.0, "max_shards": 3}, + } + values.update(overrides) + return DgdrExecutorConfig.model_validate(values) + + +def make_retune(cfg: DgdrExecutorConfig, **overrides: Any) -> RetuneConfig: + values: dict[str, Any] = { + "interval_seconds": INTERVAL_S, + "grace_seconds": GRACE_S, + "timeout_seconds": TIMEOUT_S, + "router": "default", + "executor": "dgdr", + "dgdr": cfg, + "objective": "GoodputPerGPU", + } + values.update(overrides) + return RetuneConfig.model_validate(values) + + +def grid(domain: Mapping[str, tuple[float, ...]]) -> list[dict[str, float]]: + keys = sorted(domain) + return [ + dict(zip(keys, values)) + for values in itertools.product(*(domain[key] for key in keys)) + ] + + +def make_request( + domain: Mapping[str, tuple[float, ...]] = SMALL_DOMAIN, + *, + baseline: Optional[Mapping[str, float]] = None, + startup: Optional[Mapping[str, float]] = None, + deadline_s: Optional[float] = LAUNCH_S + TIMEOUT_S, + spec_replicas: Optional[dict[str, int]] = None, + objective: str = "GoodputPerGPU", + router_domain: Any = "same", + planner: Optional[dict[str, Any]] = None, + trace_paths: tuple[str, ...] = ("/trace/segments/llm",), +) -> SweepRequest: + points = grid(domain) + baseline = dict(baseline or points[-1]) + startup = dict(startup or baseline) + candidates = {RetuneParams(router=point) for point in points} + candidates.add(RetuneParams(router=startup)) + base = RetuneParams(router=baseline, planner=planner or {}) + candidates.add(base) + replicas = spec_replicas or {"aggregated": 3} + disagg = "prefill" in replicas + return SweepRequest( + window_index=7, + window_start_ms=WINDOW_END_MS - INTERVAL_S * 1000.0, + window_end_ms=float(WINDOW_END_MS), + history_start_ms=WINDOW_END_MS - INTERVAL_S * 1000.0, + as_of_ms=WINDOW_END_MS + GRACE_S * 1000.0, + trace_paths=trace_paths, + arrival_speedup_ratio=1.0, + engine=RetuneEngineSpec( + router_mode="kv_router", + router_config={}, + prefill_engine_args={} if disagg else None, + decode_engine_args={} if disagg else None, + ), + candidates=tuple(sorted(candidates, key=RetuneParams.key)), + baseline=base, + replicas=dict(replicas), + base_planner=None, + predictor_candidates=None, + objective=objective, # type: ignore[arg-type] + sla_ttft_ms=500.0, + sla_itl_ms=30.0, + max_parallel=1, + timeout_s=TIMEOUT_S, + router_domain=( + {key: tuple(values) for key, values in domain.items()} + if router_domain == "same" + else router_domain + ), + deadline_s=deadline_s, + spec_replicas=dict(replicas), + generation=3, + ) + + +def public(point: Mapping[str, float]) -> dict[str, float]: + return {LIVE_TO_PUBLIC[key]: value for key, value in point.items()} + + +def expected_score(point: Mapping[str, float], request: SweepRequest) -> float: + score, _ = score_summary( + default_metrics(public(point)), request.window_s, request.objective + ) + return score + + +class FakeClock: + """Wall clock for the executor; ``sleep`` advances it and runs hooks.""" + + def __init__(self, start: float = LAUNCH_S) -> None: + self.now = start + self.sleeps: list[float] = [] + self.hooks: list[Callable[[], None]] = [] + + def time(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += max(seconds, 0.001) + for hook in list(self.hooks): + hook() + + +class FakeRegistry: + def __init__(self) -> None: + self.protected: dict[int, set[str]] = {} + self.released: list[int] = [] + self.references: dict[int, int] = {} + self.deleted: list[str] = [] + self.lock = threading.Lock() + + def protect(self, window_end_ms: int, request_names) -> None: + with self.lock: + self.protected.setdefault(window_end_ms, set()).update(request_names) + self.references[window_end_ms] = self.references.get(window_end_ms, 0) + 1 + + def release(self, window_end_ms: int) -> None: + with self.lock: + self.released.append(window_end_ms) + self.references[window_end_ms] = self.references.get(window_end_ms, 0) - 1 + + def balanced(self, window_end_ms: int) -> bool: + """Every reference the sweep took on the window was released.""" + return ( + window_end_ms in self.released + and self.references.get(window_end_ms, 0) == 0 + ) + + def enqueue_delete(self, request_names) -> None: + with self.lock: + self.deleted.extend(request_names) + + +class FakeCutter: + """Records cuts; returns ``num_requests`` rows or raises ``error``.""" + + def __init__( + self, + *, + num_requests: int = 1234, + error: Optional[Exception] = None, + on_cut: Optional[Callable[[], None]] = None, + ) -> None: + self.num_requests = num_requests + self.error = error + self.on_cut = on_cut + self.calls: list[dict[str, Any]] = [] + + def __call__( + self, + segments_root, + start_ms, + end_ms, + as_of_ms, + out_path, + *, + max_rows, + deadline_s, + cancel=None, + ) -> WindowCutResult: + self.calls.append( + { + "segments_root": segments_root, + "start_ms": start_ms, + "end_ms": end_ms, + "as_of_ms": as_of_ms, + "out_path": out_path, + "max_rows": max_rows, + "deadline_s": deadline_s, + "cancel": cancel, + } + ) + if self.on_cut is not None: + self.on_cut() + if self.error is not None: + raise self.error + if self.num_requests == 0: + return WindowCutResult( + path=None, num_requests=0, trace_block_size=None, row_counts={} + ) + return WindowCutResult( + path=out_path, + num_requests=self.num_requests, + trace_block_size=64, + row_counts={"selected": self.num_requests}, + dropped_members=1, + dropped_bytes=10, + ) + + +class CutError(Exception): + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +class Harness: + """A fake cluster, operator, clock, registry and cutter for one window.""" + + def __init__( + self, + tmp_path, + *, + cfg: Optional[DgdrExecutorConfig] = None, + operator: bool = True, + **retune_overrides: Any, + ) -> None: + self.clock = FakeClock() + self.kube = FakeRetuneKube("test-ns", now=self.clock.time) + self.cfg = cfg or make_cfg(mount_path=str(tmp_path / "trace")) + self.retune = make_retune(self.cfg, **retune_overrides) + self.registry = FakeRegistry() + self.cutter = FakeCutter() + self.leader = True + self.cancel = threading.Event() + self.abort = threading.Event() + self.operator = FakeDgdrOperator(self.kube) + if operator: + self.clock.hooks.append(self.operator.step) + + def context(self, **overrides: Any) -> DgdrContext: + values: dict[str, Any] = { + "dgd": DGD, + "cfg": self.cfg, + "retune": self.retune, + "backend": "vllm", + "kube": self.kube, + "registry": self.registry, + "cut_window": self.cutter, + "is_leader": lambda: self.leader, + "now": self.clock.time, + "sleep": self.clock.sleep, + "poll_interval_s": POLL_S, + } + values.update(overrides) + return DgdrContext(**values) + + def sweep(self, request: SweepRequest, **overrides: Any): + return run_dgdr_sweep( + request, + cancel=self.cancel, + abort=self.abort, + context=self.context(**overrides), + ) + + def created(self) -> list[dict]: + return [ + call.target + for call in self.kube.calls_of("create_request") + if not call.kwargs.get("dry_run") + ] + + def request_of_shard(self, shard_id: str) -> dict: + found = [ + obj + for obj in self.kube.requests.values() + if obj["metadata"]["labels"][LABEL_SHARD] == shard_id + ] + assert len(found) == 1, (shard_id, [o["metadata"]["name"] for o in found]) + return found[0] + + +def assert_scores(result, request: SweepRequest, missing=()) -> None: + missing_keys = {RetuneParams(router=point).key() for point in missing} + assert set(result.scores) == {candidate.key() for candidate in request.candidates} + for candidate in request.candidates: + key = candidate.key() + if key in missing_keys: + assert result.scores[key] is None, key + assert "error" in result.metrics[key] + else: + assert result.scores[key] == pytest.approx( + expected_score(candidate.router, request), rel=1e-12 + ), key + assert result.metrics[key]["params"] == candidate.to_dict() + + +def kube_error(cls: type[KubeError], status: Optional[int], message: str) -> KubeError: + """A ``Kube*`` error as ``kube.py`` raises it, with its HTTP ``status``.""" + + error = cls(f"HTTP {status}: {message}") + error.status = status # type: ignore[attr-defined] + return error + + +def lose_leadership_after_start(h: Harness, request: SweepRequest) -> None: + """A first leader creates the window's Requests, the operator starts their + Runs, and the leader loses leadership; the next sweep is another leader.""" + + h.operator.max_phase = "started" + h.clock.hooks.append(lambda: h.abort.set() if len(h.clock.sleeps) >= 2 else None) + assert h.sweep(request).failure_reason == "lost_leadership" + h.abort = threading.Event() + h.operator.max_phase = None + h.clock.hooks = [h.operator.step] + + +# ------------------------------------------------------------------ shards + + +def test_plan_shards_default_grid_is_three_balanced_shards_and_a_probe(): + points = grid(DEFAULT_DOMAIN) + baseline = points[0] + shards = plan_shards( + DEFAULT_DOMAIN, baseline=baseline, startup=baseline, max_shards=3 + ) + grid_shards = [shard for shard in shards if shard.kind == "grid"] + assert [shard.size for shard in grid_shards] == [36, 36, 36] + assert [shard.id for shard in shards] == ["grid-0", "grid-1", "grid-2", "probe"] + covered = [point for shard in grid_shards for point in shard.points] + assert sorted(map(str, covered)) == sorted(map(str, points)) + for shard in grid_shards: + # Each shard is the Cartesian product of its choices. + assert shard.size == math.prod(len(v) for v in shard.choices.values()) + probe = shards[-1] + # The middle value of each knob, a positive temperature. + assert probe.points == ({CREDIT: 0.5, SCALE: 2.0, TEMP: 0.5},) + assert probe.choices == {CREDIT: (0.5,), SCALE: (2.0,), TEMP: (0.5,)} + + +def test_plan_shards_adds_off_grid_baseline_and_startup_points(): + off_baseline = {CREDIT: 0.75, SCALE: 1.0, TEMP: 0.0} + off_startup = {CREDIT: 1.0, SCALE: 3.0, TEMP: 0.0} + shards = plan_shards( + SMALL_DOMAIN, + baseline=off_baseline, + startup=off_startup, + max_shards=3, + parallelism=8, + ) + by_id = {shard.id: shard for shard in shards} + assert set(by_id) == {"grid-0", "point-baseline", "point-startup", "probe"} + assert by_id["point-baseline"].points == (off_baseline,) + assert by_id["point-baseline"].kind == "point" + assert by_id["point-startup"].points == (off_startup,) + # The same off-grid point once. + shards = plan_shards( + SMALL_DOMAIN, + baseline=off_baseline, + startup=off_baseline, + max_shards=3, + parallelism=8, + ) + assert [shard.id for shard in shards] == ["grid-0", "point-baseline", "probe"] + # An on-grid startup needs no shard; no probe when disabled. + shards = plan_shards( + SMALL_DOMAIN, + baseline=grid(SMALL_DOMAIN)[0], + startup=grid(SMALL_DOMAIN)[1], + max_shards=3, + probe=False, + parallelism=8, + ) + assert [shard.id for shard in shards] == ["grid-0"] + + +def test_plan_shards_splits_uneven_dimension_and_rejects_impossible_grids(): + domain = {CREDIT: tuple(float(v) for v in range(250))} + zero = {CREDIT: 0.0} + shards = plan_shards(domain, baseline=zero, startup=zero, max_shards=3, probe=False) + assert [shard.size for shard in shards] == [84, 83, 83] + assert shards[0].choices[CREDIT][0] == 0.0 + assert shards[-1].choices[CREDIT][-1] == 249.0 + with pytest.raises(ValueError, match="cannot be split"): + plan_shards(domain, baseline=zero, startup=zero, max_shards=2) + with pytest.raises(ValueError, match="cannot be split"): + plan_shards( + DEFAULT_DOMAIN, + baseline=grid(DEFAULT_DOMAIN)[0], + startup=grid(DEFAULT_DOMAIN)[0], + max_shards=1, + ) + point = grid(SMALL_DOMAIN)[0] + + def sizes(**kwargs): + shards = plan_shards( + SMALL_DOMAIN, baseline=point, startup=point, probe=False, **kwargs + ) + return [shard.size for shard in shards] + + # Without the Run parallelism the largest shard is minimized. + assert sizes(max_shards=16) == [1] * 8 + assert sizes(max_shards=3) == [4, 4] + # With it, a shard is added only when it saves an evaluation round + # (AISimulate runs the first candidate alone, so single points save one). + assert sizes(max_shards=3, parallelism=8) == [8] + assert sizes(max_shards=16, parallelism=8) == [1] * 8 + assert sizes(max_shards=3, parallelism=1) == [4, 4] + # The default grid is three shards of 36 either way. + default = plan_shards( + DEFAULT_DOMAIN, + baseline=grid(DEFAULT_DOMAIN)[0], + startup=grid(DEFAULT_DOMAIN)[0], + max_shards=3, + parallelism=8, + ) + assert [shard.size for shard in default] == [36, 36, 36, 1] + + +def test_plan_shards_validates_inputs_and_probe_falls_back_to_zero_temperature(): + point = {CREDIT: 0.5, SCALE: 1.0} + with pytest.raises(ValueError, match="do not match"): + plan_shards(SMALL_DOMAIN, baseline=point, startup=point, max_shards=3) + with pytest.raises(ValueError, match="empty"): + plan_shards({}, baseline={}, startup={}, max_shards=3) + domain = {CREDIT: (0.0, 1.0), SCALE: (1.0,), TEMP: (0.0,)} + shards = plan_shards( + domain, baseline=grid(domain)[0], startup=grid(domain)[0], max_shards=3 + ) + assert shards[-1].points == ({CREDIT: 1.0, SCALE: 1.0, TEMP: 0.0},) + + +def test_need_seconds_follows_the_design_formula(): + cfg = make_cfg( + cut_budget_seconds=900.0, + job_startup_seconds=300.0, + run={"parallelism": 8, "candidate_timeout_seconds": 1800.0}, + ) + shard = plan_shards( + DEFAULT_DOMAIN, + baseline=grid(DEFAULT_DOMAIN)[0], + startup=grid(DEFAULT_DOMAIN)[0], + max_shards=3, + )[0] + assert shard.size == 36 + # 1 + ceil(35 / 8) = 6 rounds. + assert need_seconds(shard, cfg, needs_cut=True) == 900 + 300 + 6 * 1800 + assert need_seconds(shard, cfg, needs_cut=False) == 300 + 6 * 1800 + single = Shard( + id="probe", kind="probe", choices={CREDIT: (1.0,)}, points=({CREDIT: 1.0},) + ) + assert need_seconds(single, cfg, needs_cut=False) == 300 + 1800 + + +# ---------------------------------------------------------------- Request + + +def test_spec_hash_is_canonical(): + assert spec_hash({"a": 1, "b": [1, 2]}) == spec_hash({"b": [1, 2], "a": 1}) + assert spec_hash({"a": 1}) != spec_hash({"a": 2}) + assert len(spec_hash({})) == 16 + int(spec_hash({}), 16) + + +def _shards(domain=SMALL_DOMAIN, **kwargs): + point = grid(domain)[0] + return plan_shards( + domain, baseline=point, startup=point, max_shards=3, parallelism=8, **kwargs + ) + + +def test_build_request_aggregated_grid_shard(): + cfg = make_cfg() + retune = make_retune(cfg) + shard = _shards()[0] + body = build_request( + dgd=DGD, + cfg=cfg, + retune=retune, + sla_ttft_ms=500, + sla_itl_ms=30, + window_end_ms=WINDOW_END_MS, + shard=shard, + spec_replicas={"aggregated": 3}, + backend="vllm", + attempt=1, + ) + spec = body["spec"] + assert body["apiVersion"] == "nvidia.com/v1beta2" + assert body["kind"] == "DynamoGraphDeploymentRequest" + assert spec["model"] == {"name": "Qwen/Qwen3-32B"} + assert spec["backends"] == ["vllm"] + # 3 replicas x (tensor 2 x pipeline 1 x attention_data 1). + assert spec["hardware"] == {"gpu": {"sku": "h200_sxm", "budget": 6}} + assert spec["workload"] == { + "trace": { + "format": "dynamo", + "source": { + "pvc": { + "claimName": "request-traces", + "path": f"windows/llm/{WINDOW_END_MS}.jsonl.gz", + } + }, + } + } + assert spec["objective"] == { + "mode": "Optimize", + "metric": "GoodputPerGPU", + "sla": {"ttftMs": 500.0, "itlMs": 30.0}, + } + assert spec["recommendation"] == {"maxCandidates": 8} + overrides = spec["aisimulate"]["inputOverrides"] + workers = overrides["engine"]["workers"] + assert workers["aggregated"]["parallelism"]["preset"][0]["replicas"] == 3 + assert workers["prefill"] is None and workers["decode"] is None + assert overrides["engine"]["backend"] == "vllm" + assert overrides["router"] == { + "policy": "kv_router", + "prefill_load_model": {"type": "none"}, + "overlap_score_credit": {"choices": [0.5, 1.0]}, + "prefill_load_scale": {"choices": [1.0, 2.0]}, + "temperature": {"choices": [0.0, 0.5]}, + } + assert overrides["optimizer"] == { + "algorithm": "random", + "max_trials": 16, + "parallelism": 8, + "candidate_timeout_seconds": 300.0, + } + assert overrides["optimization"] == {"strict_sla": False} + assert overrides["dgdr_run"] == {"generator": "direct"} + # The config's engine is not mutated. + assert ( + "replicas" + not in AGG_ENGINE["workers"]["aggregated"]["parallelism"]["preset"][0] + ) + + metadata = body["metadata"] + digest = spec_hash(spec) + assert metadata["labels"] == { + "nvidia.com/planner-retune-dgd": "llm", + LABEL_DGD_UID: "dgd-uid-1", + LABEL_WINDOW: str(WINDOW_END_MS), + LABEL_SHARD: "grid-0", + LABEL_SPEC: digest, + LABEL_ATTEMPT: "1", + } + assert metadata["ownerReferences"] == [ + { + "apiVersion": "nvidia.com/v1beta1", + "kind": "DynamoGraphDeployment", + "name": "llm", + "uid": "dgd-uid-1", + "blockOwnerDeletion": False, + } + ] + assert metadata["name"] == request_name(DGD, WINDOW_END_MS, "grid-0", digest, 1) + assert metadata["name"].startswith("llm-rt-") and len(metadata["name"]) == 17 + assert request_name(DGD, WINDOW_END_MS, "grid-0", digest, 2) != metadata["name"] + + +def test_build_request_point_shard_uses_scalars_and_disagg_budget(): + cfg = make_cfg(engine=DISAGG_ENGINE) + retune = make_retune(cfg, objective="TTFT") + probe = _shards()[-1] + body = build_request( + dgd=DGD, + cfg=cfg, + retune=retune, + sla_ttft_ms=200, + sla_itl_ms=20, + window_end_ms=WINDOW_END_MS, + shard=probe, + spec_replicas={"prefill": 2, "decode": 4}, + backend="vllm", + attempt=3, + ) + spec = body["spec"] + # 2 x 1 + 4 x (2 x 1 x 2). + assert spec["hardware"]["gpu"]["budget"] == 18 + assert spec["objective"]["metric"] == "TTFT" + overrides = spec["aisimulate"]["inputOverrides"] + workers = overrides["engine"]["workers"] + assert workers["aggregated"] is None + assert workers["prefill"]["parallelism"]["preset"][0]["replicas"] == 2 + assert workers["decode"]["parallelism"]["preset"][0]["replicas"] == 4 + assert overrides["router"]["temperature"] == 0.5 + assert overrides["router"]["overlap_score_credit"] == 1.0 + assert overrides["router"]["prefill_load_scale"] == 2.0 + assert overrides["optimizer"]["max_trials"] == 2 + assert spec["recommendation"]["maxCandidates"] == 1 + assert body["metadata"]["labels"][LABEL_ATTEMPT] == "3" + + +@pytest.mark.parametrize( + "engine, replicas, backend, match", + [ + ({**AGG_ENGINE, "backend": "sglang"}, {"aggregated": 1}, "vllm", "differs"), + ( + {**AGG_ENGINE, "workers": {"aggregated": {}}}, + {"aggregated": 1}, + "vllm", + "preset", + ), + (AGG_ENGINE, {"prefill": 1}, "vllm", "spec_replicas"), + (AGG_ENGINE, {"aggregated": 0}, "vllm", "at least 1"), + ], +) +def test_build_request_rejects_engines_it_cannot_pin(engine, replicas, backend, match): + cfg = make_cfg(engine=engine) + with pytest.raises(ValueError, match=match): + build_request( + dgd=DGD, + cfg=cfg, + retune=make_retune(cfg), + sla_ttft_ms=1, + sla_itl_ms=1, + window_end_ms=WINDOW_END_MS, + shard=_shards()[0], + spec_replicas=replicas, + backend=backend, + attempt=1, + ) + + +def test_build_request_requires_the_three_dgdr_knobs(): + cfg = make_cfg() + shard = Shard( + id="grid-0", kind="grid", choices={CREDIT: (1.0,)}, points=({CREDIT: 1.0},) + ) + with pytest.raises(ValueError, match="exactly"): + build_request( + dgd=DGD, + cfg=cfg, + retune=make_retune(cfg), + sla_ttft_ms=1, + sla_itl_ms=1, + window_end_ms=WINDOW_END_MS, + shard=shard, + spec_replicas={"aggregated": 1}, + backend="vllm", + attempt=1, + ) + + +def _body(shard_index=0, **kwargs): + cfg = make_cfg(**kwargs) + return build_request( + dgd=DGD, + cfg=cfg, + retune=make_retune(cfg), + sla_ttft_ms=500, + sla_itl_ms=30, + window_end_ms=WINDOW_END_MS, + shard=_shards()[shard_index], + spec_replicas={"aggregated": 3}, + backend="vllm", + attempt=1, + ) + + +def test_input_read_back_accepts_the_operator_lowering_and_finds_drift(): + body = _body() + body["metadata"]["namespace"] = "test-ns" + effective = lower_request_input(body, run_name="r-1") + assert input_mismatches(body["spec"], effective) == [] + + drifted = lower_request_input(body, run_name="r-1") + drifted["engine"]["workers"]["aggregated"]["parallelism"]["preset"][0][ + "replicas" + ] = 1 + drifted["engine"]["workers"]["prefill"] = {} + drifted["router"]["temperature"] = {"choices": [0.0]} + drifted["evaluation"]["sla"]["e2e_ms"] = 10.0 + drifted["traffic"]["load"]["speedup"] = 2.0 + drifted["optimization"]["strict_sla"] = True + problems = input_mismatches(body["spec"], drifted) + joined = "\n".join(problems) + for path in ( + "engine:", + "router:", + "evaluation.sla.e2e_ms", + "traffic.load", + "optimization.strict_sla", + ): + assert path in joined, (path, problems) + + stripped = lower_request_input(body, run_name="r-1") + del stripped["dgdr_run"]["generator"] + del stripped["optimization"]["constraints"] + problems = input_mismatches(body["spec"], stripped) + assert any(p.startswith("dgdr_run.generator") for p in problems) + assert any("max_candidate_gpus" in p for p in problems) + + +def test_missing_fields_reports_pruned_leaves_and_allows_defaults(): + submitted = {"a": {"b": 1, "c": [1, 2], "gone": None}, "d": True} + stored = {"a": {"b": 1.0, "c": [1, 2], "extra": 5}, "d": True, "default": 1} + assert missing_fields(stored, submitted) == [] + assert missing_fields({"a": {"b": 1}}, submitted, "spec") == ["spec.a.c", "spec.d"] + assert missing_fields( + {"a": {"b": 1, "c": [1, 2], "gone": 3}, "d": 1}, submitted + ) == [ + "a.gone", + "d", + ] + + +def test_crn_unsupported_compares_parseable_versions_only(): + assert not crn_unsupported("1.7.0", None, None) + assert crn_unsupported("1.7.0", None, "1.8.0") + assert not crn_unsupported("1.8.0", None, "1.8.0") + assert not crn_unsupported("1.9.1rc0", None, "1.8.0") + assert crn_unsupported(None, "nvcr.io/nvidia/planner:1.7.2@sha256:ab", "1.8") + assert not crn_unsupported("main", "image-without-tag", "1.8.0") + + +# ------------------------------------------------------------ the sweep + + +def test_fresh_window_creates_cuts_harvests_and_scores(tmp_path): + h = Harness(tmp_path) + request = make_request() + result = h.sweep(request) + + assert result.status == "ok", (result.error, result.failure_reason) + assert result.failure_reason is None + assert result.num_requests == 1234 + assert result.row_counts == {"selected": 1234} + assert_scores(result, request) + # One grid shard (8 points) and the probe. + created = h.created() + assert [obj["metadata"]["labels"][LABEL_SHARD] for obj in created] == [ + "grid-0", + "probe", + ] + assert all( + call.kwargs["timeout_s"] <= 10.0 for call in h.kube.calls_of("create_request") + ) + # The window was cut once, into the claim path the Requests name. + (cut,) = h.cutter.calls + assert cut["out_path"] == str( + tmp_path / "trace" / f"windows/llm/{WINDOW_END_MS}.jsonl.gz" + ) + assert cut["segments_root"] == "/trace/segments/llm" + assert cut["start_ms"] == WINDOW_END_MS - 3_600_000 + assert cut["end_ms"] == WINDOW_END_MS + assert cut["as_of_ms"] == WINDOW_END_MS + 120_000 + # Absolute wall-clock seconds: now plus the cut budget. + assert cut["deadline_s"] == LAUNCH_S + 100.0 + assert cut["max_rows"] == h.cfg.max_window_rows + assert (tmp_path / "trace" / "windows" / "llm").is_dir() + # Requests were protected, nothing deleted, the window released. + assert h.registry.protected[WINDOW_END_MS] == { + o["metadata"]["name"] for o in created + } + assert h.registry.deleted == [] + assert h.registry.balanced(WINDOW_END_MS) + details = result.details + assert details["crn_probe"]["relative_error"] == 0.0 + assert details["runtime_image"].endswith(":1.8.0") + assert details["runtime_version"] == "1.8.0" + assert details["missing_candidates"] == 0 + assert details["shards"]["grid-0"]["critical"] is True + assert details["window"]["dropped_members"] == 1 + assert details["generation"] == 3 + assert result.wall_s > 0 + + +def test_default_grid_with_off_grid_startup_runs_five_requests(tmp_path): + h = Harness(tmp_path) + startup = {CREDIT: 1.0, SCALE: 1.5, TEMP: 0.0} + request = make_request(DEFAULT_DOMAIN, baseline=startup, startup=startup) + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + shard_ids = sorted(obj["metadata"]["labels"][LABEL_SHARD] for obj in h.created()) + assert shard_ids == ["grid-0", "grid-1", "grid-2", "point-baseline", "probe"] + # The probe point's grid twin is critical; the baseline shard is too. + critical = {k for k, v in result.details["shards"].items() if v["critical"]} + assert critical == {"grid-1", "point-baseline", "probe"} + + +def test_disaggregated_window(tmp_path): + h = Harness(tmp_path, cfg=make_cfg(engine=DISAGG_ENGINE, mount_path=str(tmp_path))) + request = make_request(spec_replicas={"prefill": 1, "decode": 2}) + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + spec = h.created()[0]["spec"] + assert spec["hardware"]["gpu"]["budget"] == 1 + 2 * 4 + + +def test_window_adopted_as_succeeded_is_not_cut_again(tmp_path): + h = Harness(tmp_path) + request = make_request() + first = h.sweep(request) + assert first.status == "ok" + creates = len(h.kube.calls_of("create_request")) + + # A new leader (fresh cutter) sweeps the same window. + h.cutter = FakeCutter(num_requests=1) + second = h.sweep(request) + assert second.status == "ok", second.error + assert h.cutter.calls == [] + assert len(h.kube.calls_of("create_request")) == creates + assert second.scores == first.scores + # Row counts come from the Request annotation of the cut. + assert second.num_requests == 1234 + assert second.row_counts == {"selected": 1234} + assert all(shard["adopted"] for shard in second.details["shards"].values()) + + +def test_lost_leadership_leaves_requests_for_the_next_leader(tmp_path): + h = Harness(tmp_path) + h.operator.max_phase = "started" + h.clock.hooks.append(lambda: h.abort.set() if len(h.clock.sleeps) >= 2 else None) + request = make_request() + result = h.sweep(request) + assert result.status == "failed" + assert result.failure_reason == "lost_leadership" + assert result.failure_reason in NON_DECISION_FAILURES + assert h.registry.deleted == [] + assert h.registry.balanced(WINDOW_END_MS) + names = set(h.kube.requests) + assert len(names) == 2 + + # The next leader adopts the in-flight Requests without cutting. + h.abort = threading.Event() + h.operator.max_phase = None + h.clock.hooks = [h.operator.step] + h.cutter = FakeCutter() + second = h.sweep(request) + assert second.status == "ok", second.error + assert set(h.kube.requests) == names + assert h.cutter.calls == [] + assert_scores(second, request) + + +def test_shutdown_returns_at_once_without_deleting(tmp_path): + h = Harness(tmp_path) + h.operator.max_phase = "accepted" + h.clock.hooks.append(h.cancel.set) + result = h.sweep(make_request()) + assert result.failure_reason == "shutdown" + assert result.status == "failed" + assert h.registry.deleted == [] + assert len(h.kube.requests) == 2 + + +def test_not_leader_creates_nothing(tmp_path): + h = Harness(tmp_path) + h.leader = False + result = h.sweep(make_request()) + assert result.failure_reason == "lost_leadership" + assert h.created() == [] + assert h.cutter.calls == [] + + +def test_abort_during_cut_reports_lost_leadership(tmp_path): + h = Harness(tmp_path) + h.cutter = FakeCutter(on_cut=h.abort.set, error=CutError("cancelled")) + result = h.sweep(make_request()) + assert result.failure_reason == "lost_leadership" + assert h.created() == [] + # The cutter's cancel event reflects abort. + assert h.cutter.calls[0]["cancel"].is_set() + + +def test_stale_spec_and_older_attempts_are_queued_for_deletion(tmp_path): + h = Harness(tmp_path) + request = make_request() + h.sweep(request) + grid_request = h.request_of_shard("grid-0") + name = grid_request["metadata"]["name"] + + # A stale shard spec and an older attempt of the same spec. + stale = { + "metadata": { + "name": "llm-rt-stale", + "labels": { + **grid_request["metadata"]["labels"], + LABEL_SPEC: "0" * 16, + }, + }, + "spec": {}, + } + older = { + "metadata": { + "name": "llm-rt-older", + "labels": {**grid_request["metadata"]["labels"], LABEL_ATTEMPT: "0"}, + }, + "spec": {}, + } + unknown_shard = { + "metadata": { + "name": "llm-rt-unknown", + "labels": {**grid_request["metadata"]["labels"], LABEL_SHARD: "grid-9"}, + }, + "spec": {}, + } + for obj in (stale, older, unknown_shard): + h.kube.create_request(obj) + other_window = { + "metadata": { + "name": "llm-rt-other", + "labels": {**grid_request["metadata"]["labels"], LABEL_WINDOW: "1"}, + }, + "spec": {}, + } + h.kube.create_request(other_window) + h.registry.deleted.clear() + result = h.sweep(request) + assert result.status == "ok", result.error + assert sorted(h.registry.deleted) == [ + "llm-rt-older", + "llm-rt-stale", + "llm-rt-unknown", + ] + assert result.details["shards"]["grid-0"]["request"] == name + + +def test_create_that_lands_but_times_out_is_adopted(tmp_path): + h = Harness(tmp_path) + h.kube.fail("create_request", KubeError("timeout"), landed=True) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert len(h.kube.requests) == 2 + assert result.details["k8s_errors"] == 1 + assert_scores(result, request) + + +def test_create_already_exists_with_same_spec_is_adopted(tmp_path): + h = Harness(tmp_path) + h.kube.fail("create_request", KubeAlreadyExists("exists"), landed=True) + result = h.sweep(make_request()) + assert result.status == "ok", result.error + assert len(h.kube.requests) == 2 + + +def test_create_that_fails_is_retried(tmp_path): + h = Harness(tmp_path) + h.kube.fail("create_request", KubeError("503"), times=2) + result = h.sweep(make_request()) + assert result.status == "ok", result.error + assert len(h.kube.requests) == 2 + assert result.details["k8s_errors"] == 2 + + +@pytest.mark.parametrize( + "error", + [ + kube_error(KubeError, 403, "requests is forbidden: cannot create"), + kube_error(KubeNotFound, 404, "the server could not find the resource"), + kube_error(KubeError, 405, "method not allowed"), + KubeNotFound("no such API"), + ], + ids=["403", "404", "405", "not-found"], +) +def test_permanent_create_error_fails_fast(tmp_path, error): + h = Harness(tmp_path) + h.kube.fail("create_request", error, times=1000) + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + assert result.status == "failed" + assert result.failure_reason == "k8s_error" + assert str(error) in result.error + # No create -> unknown -> GET -> create loop until the launch gate trips. + assert len(h.created()) == 1 + assert h.clock.sleeps == [] + assert result.details["k8s_errors"] == 1 + assert h.kube.requests == {} + + +@pytest.mark.parametrize( + "cls, status", [(KubeInvalid, 422), (KubeInvalid, 400), (KubeError, 403)] +) +def test_admission_webhook_rejection_is_request_rejected(tmp_path, cls, status): + h = Harness(tmp_path) + message = ( + 'admission webhook "vdynamographdeploymentrequest-v1beta2.kb.io" denied ' + 'the request: DynamoGraphDeploymentRequest.nvidia.com "x" is invalid: ' + 'spec.hardware.gpu.sku: Invalid value: "h200_sxm": no node offers it' + ) + h.kube.fail("create_request", kube_error(cls, status, message)) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "request_rejected" + assert "no node offers it" in result.error + assert len(h.created()) == 1 + + +def test_webhook_unreachable_is_retried(tmp_path): + h = Harness(tmp_path) + error = kube_error( + KubeError, + 500, + 'Internal error occurred: failed calling webhook "x.kb.io": connection refused', + ) + h.kube.fail("create_request", error, times=2) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert result.details["k8s_errors"] == 2 + assert_scores(result, request) + + +@pytest.mark.parametrize( + "error", + [kube_error(KubeError, 403, "requests is forbidden: cannot get"), KubeNotFound()], + ids=["403", "not-found"], +) +def test_permanent_get_error_fails_fast(tmp_path, error): + h = Harness(tmp_path) + # The create's outcome is unknown, so the next poll looks the Request up. + h.kube.fail("create_request", KubeError("timeout")) + h.kube.fail("get_request", error, times=1000) + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + assert result.status == "failed" + assert result.failure_reason == "k8s_error" + assert "get Request" in result.error + assert h.clock.now <= LAUNCH_S + POLL_S + + +def test_transient_get_errors_are_retried(tmp_path): + h = Harness(tmp_path) + h.kube.fail("get_request", KubeError("timeout"), times=3) + h.kube.fail("get_run", kube_error(KubeError, 503, "unavailable"), times=2) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert result.details["k8s_errors"] == 5 + assert_scores(result, request) + + +def test_name_taken_by_a_foreign_request_fails_a_critical_shard(tmp_path): + h = Harness(tmp_path) + request = make_request() + # Pre-create a foreign object under the grid shard's name. + body = build_request( + dgd=DGD, + cfg=h.cfg, + retune=h.retune, + sla_ttft_ms=500, + sla_itl_ms=30, + window_end_ms=WINDOW_END_MS, + shard=_shards()[0], + spec_replicas={"aggregated": 3}, + backend="vllm", + attempt=1, + ) + h.kube.requests[body["metadata"]["name"]] = { + "metadata": {"name": body["metadata"]["name"], "labels": {"foreign": "yes"}}, + "spec": {}, + } + result = h.sweep(request) + assert result.status == "failed" + assert result.failure_reason == "request_conflict" + assert body["metadata"]["name"] not in h.registry.deleted + + +def test_strict_rejection_is_schema_unsupported(tmp_path): + h = Harness(tmp_path) + h.kube.unknown_fields = {"spec.workload.trace"} + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "schema_unsupported" + assert h.kube.requests == {} + + +def test_silently_pruned_field_is_schema_unsupported(tmp_path): + h = Harness(tmp_path) + h.kube.pruned_fields = {"spec.workload.trace.source"} + result = h.sweep(make_request()) + assert result.failure_reason == "schema_unsupported" + assert "spec.workload.trace.source" in result.error + # The pruned Request is queued for deletion. + assert h.registry.deleted == list(h.kube.requests) + + +def test_invalid_spec_fails_the_window_and_abandons_in_flight(tmp_path): + h = Harness(tmp_path, operator=False) + + def reject(kube, created): + if created["metadata"]["labels"][LABEL_SHARD] == "probe": + kube.set_request_condition( + created["metadata"]["name"], + status="False", + reason="InvalidSpec", + message="bad engine", + ) + + h.kube.on_request_created.append(reject) + result = h.sweep(make_request()) + assert result.failure_reason == "invalid_spec" + assert "bad engine" in result.error + assert sorted(h.registry.deleted) == sorted(h.kube.requests) + + +def test_operator_not_reconciling_after_sixty_seconds(tmp_path): + h = Harness(tmp_path, operator=False) + result = h.sweep(make_request()) + assert result.failure_reason == "operator_not_reconciling" + assert h.clock.now - LAUNCH_S >= dgdr.ACCEPT_TIMEOUT_S + assert h.clock.now - LAUNCH_S < dgdr.ACCEPT_TIMEOUT_S + 2 * POLL_S + + +def test_run_name_conflict_keeps_waiting(tmp_path): + h = Harness(tmp_path, operator=False) + phases = {"steps": 0} + + def operator(): + phases["steps"] += 1 + for name in list(h.kube.requests): + if phases["steps"] < 30: + h.kube.set_request_condition( + name, status="False", reason="RunNameConflict" + ) + if phases["steps"] >= 30: + h.operator.step() + + h.clock.hooks.append(operator) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert h.clock.now - LAUNCH_S > dgdr.ACCEPT_TIMEOUT_S + assert_scores(result, request) + + +def test_run_pending_until_the_deadline_times_out(tmp_path): + h = Harness(tmp_path) + # The operator never creates the Jobs. Like the real operator, it then + # writes no Run status at all: the input arrives together with jobRef. + h.operator.max_phase = "accepted" + request = make_request(deadline_s=LAUNCH_S + 900.0) + result = h.sweep(request) + assert [h.kube.run_of(name)["status"] for name in h.kube.requests] == [{}, {}] + assert result.status == "timeout" + assert result.failure_reason == "run_pending" + assert "Runs pending" in result.error + assert h.clock.now >= LAUNCH_S + 900.0 + assert sorted(h.registry.deleted) == sorted(h.kube.requests) + assert all(shard["run_pending"] for shard in result.details["shards"].values()) + + +def test_run_pending_clears_once_the_job_exists(tmp_path, caplog): + h = Harness(tmp_path) + h.operator.max_phase = "accepted" + + def create_jobs_late(): + if h.clock.now - LAUNCH_S >= 3 * h.cfg.job_startup_seconds: + h.operator.max_phase = None + + h.clock.hooks.insert(0, create_jobs_late) + request = make_request() + with caplog.at_level("WARNING", logger=dgdr.__name__): + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + assert "(run_pending)" in caplog.text + assert not any(shard["run_pending"] for shard in result.details["shards"].values()) + + +def test_job_created_but_not_started_is_not_run_pending(tmp_path): + h = Harness(tmp_path, operator=False) + + def create_jobs(): + for name in list(h.kube.requests): + if h.kube.run_of(name) is None: + h.kube.create_job(h.kube.accept_request(name)["metadata"]["name"]) + + h.clock.hooks.append(create_jobs) + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + for name in h.kube.requests: + status = h.kube.run_of(name)["status"] + assert status["aisimulate"]["input"] and status["jobRef"] + assert "startTime" not in status + assert result.status == "timeout" + assert result.failure_reason == "timeout" + assert not any(shard["run_pending"] for shard in result.details["shards"].values()) + + +def test_running_job_past_the_deadline_times_out(tmp_path): + h = Harness(tmp_path) + h.operator.max_phase = "started" + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + assert result.status == "timeout" + assert result.failure_reason == "timeout" + assert sorted(h.registry.deleted) == sorted(h.kube.requests) + + +def test_launch_too_late_for_a_critical_shard(tmp_path): + h = Harness(tmp_path) + # The 8-point grid shard needs 100 + 60 + 2 * 300 = 760 s. + result = h.sweep(make_request(deadline_s=LAUNCH_S + 700.0)) + assert result.status == "timeout" + assert result.failure_reason == "launch_too_late" + assert h.cutter.calls == [] + assert h.created() == [] + + +def test_launch_too_late_for_an_ordinary_shard_leaves_its_points_missing(tmp_path): + cfg = make_cfg( + mount_path=str(tmp_path), + run={"parallelism": 1, "candidate_timeout_seconds": 300.0, "max_shards": 3}, + ) + h = Harness(tmp_path, cfg=cfg) + domain = { + CREDIT: tuple(float(v) for v in range(10)), + SCALE: (1.0,), + TEMP: (0.0, 0.5), + } + baseline = {CREDIT: 2.5, SCALE: 1.0, TEMP: 0.0} # off the grid + shards = plan_shards( + domain, baseline=baseline, startup=baseline, max_shards=3, parallelism=1 + ) + sizes = {shard.id: shard.size for shard in shards} + # Ten credits in chunks of 4, 3 and 3; the probe (credit 5) is in grid-1. + assert sizes == { + "grid-0": 8, + "grid-1": 6, + "grid-2": 6, + "point-baseline": 1, + "probe": 1, + } + assert shards[-1].points[0][CREDIT] == 5.0 + need6 = need_seconds(shards[1], cfg, needs_cut=True) + need8 = need_seconds(shards[0], cfg, needs_cut=True) + assert need6 < need8 + request = make_request(domain, baseline=baseline, deadline_s=LAUNCH_S + need6 + 1) + result = h.sweep(request) + assert result.status == "ok", result.error + assert result.details["shards"]["grid-0"]["reason"] == "launch_too_late" + assert result.details["shards"]["grid-0"]["request"] is None + assert result.details["shards"]["grid-2"]["state"] == "done" + assert_scores(result, request, missing=shards[0].points) + assert ( + "launch_too_late" + in result.metrics[RetuneParams(router=shards[0].points[0]).key()]["error"] + ) + + +def test_failed_run_of_an_ordinary_shard_leaves_its_points_missing(tmp_path): + h = Harness(tmp_path) + startup = {CREDIT: 1.0, SCALE: 1.5, TEMP: 0.0} + request = make_request(baseline=grid(SMALL_DOMAIN)[0], startup=startup) + + def fail_startup(kube, created): + if created["metadata"]["labels"][LABEL_SHARD] == "point-startup": + h.operator.fail_runs[created["metadata"]["name"]] = "SearchFailed" + + h.kube.on_request_created.append(fail_startup) + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request, missing=[startup]) + shard = result.details["shards"]["point-startup"] + assert shard["reason"] == "run_failed" + assert "SearchFailed" in shard["message"] + # A terminal Request is kept for audit. + assert h.registry.deleted == [] + + +def test_failed_run_of_the_baseline_shard_fails_the_window(tmp_path): + h = Harness(tmp_path) + + def fail_grid(kube, created): + if created["metadata"]["labels"][LABEL_SHARD] == "grid-0": + h.operator.fail_runs[created["metadata"]["name"]] = "JobFailed" + + h.kube.on_request_created.append(fail_grid) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "run_failed" + assert "JobFailed" in result.error + probe = h.request_of_shard("probe")["metadata"]["name"] + grid0 = h.request_of_shard("grid-0")["metadata"]["name"] + # The probe may already be done (kept) or in flight (deleted); the failed + # baseline shard is kept for audit. + assert grid0 not in h.registry.deleted + assert set(h.registry.deleted) <= {probe} + + +def test_input_mismatch_of_a_created_critical_shard_fails(tmp_path): + h = Harness(tmp_path) + + def skew(name, effective): + effective["engine"]["context_length"] = 1024 + + h.operator.mutate_input = skew + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "input_mismatch" + assert "engine" in result.error + assert set(h.registry.deleted) == set(h.kube.requests) + + +def test_adopted_attempt_with_input_mismatch_is_replaced(tmp_path): + h = Harness(tmp_path) + request = make_request() + skewed: set[str] = set() + + def skew(name, effective): + if not skewed or name in skewed: + skewed.add(name) + effective["router"]["prefill_load_model"] = {"type": "ais"} + + h.operator.mutate_input = skew + # The first leader creates the Requests and loses leadership before the + # operator accepts them. + h.clock.hooks = [lambda: h.abort.set() if len(h.clock.sleeps) >= 2 else None] + assert h.sweep(request).failure_reason == "lost_leadership" + + h.abort = threading.Event() + h.clock.hooks = [h.operator.step] + result = h.sweep(request) + (bad,) = skewed + assert result.status == "ok", result.error + assert_scores(result, request) + assert bad in h.registry.deleted + replaced = [s for s in result.details["shards"].values() if s["attempt"] == 2] + assert len(replaced) == 1 + # The replacement needed the window file, so the window was cut. + assert len(h.cutter.calls) == 2 + + +def test_adopted_attempt_whose_run_failed_is_replaced(tmp_path): + h = Harness(tmp_path) + request = make_request() + lose_leadership_after_start(h, request) + old = h.request_of_shard("grid-0")["metadata"]["name"] + probe = h.request_of_shard("probe")["metadata"]["name"] + # While no leader runs, the baseline shard's Job fails. + h.kube.complete_run( + h.kube.run_of(old)["metadata"]["name"], reason="JobFailed", message="evicted" + ) + creates = len(h.created()) + + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + shard = result.details["shards"]["grid-0"] + assert shard["attempt"] == 2 + assert not shard["adopted"] + assert shard["request"] not in (old, None) + assert "JobFailed" in shard["message"] + assert result.details["shards"]["probe"]["request"] == probe + assert h.registry.deleted == [old] + assert len(h.created()) == creates + 1 + # The new attempt needed the window file, so the window was cut again. + assert len(h.cutter.calls) == 2 + + +def test_adopted_failed_run_without_time_for_a_new_attempt_fails(tmp_path): + h = Harness(tmp_path) + # The grid shard needs 100 + 60 + 2 * 300 = 760 s with a cut. + request = make_request(deadline_s=LAUNCH_S + 800.0) + lose_leadership_after_start(h, request) + old = h.request_of_shard("grid-0")["metadata"]["name"] + h.kube.complete_run( + h.kube.run_of(old)["metadata"]["name"], reason="JobFailed", message="evicted" + ) + h.clock.now = LAUNCH_S + 100.0 # 700 s are left + creates = len(h.created()) + + result = h.sweep(request) + assert result.status == "failed" + assert result.failure_reason == "run_failed" + assert "JobFailed" in result.error + assert len(h.created()) == creates + assert result.details["shards"]["grid-0"]["attempt"] == 1 + # The failed attempt is kept for audit. + assert old not in h.registry.deleted + + +def test_adopted_failed_run_of_an_ordinary_shard_is_replaced(tmp_path): + h = Harness(tmp_path) + startup = {CREDIT: 1.0, SCALE: 1.5, TEMP: 0.0} + request = make_request(baseline=grid(SMALL_DOMAIN)[0], startup=startup) + lose_leadership_after_start(h, request) + old = h.request_of_shard("point-startup")["metadata"]["name"] + h.kube.complete_run( + h.kube.run_of(old)["metadata"]["name"], + reason="TraceSourceUnavailable", + message="claim is gone", + ) + + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + assert result.details["shards"]["point-startup"]["attempt"] == 2 + assert h.registry.deleted == [old] + + +def test_failed_run_of_a_request_this_sweep_created_is_not_replaced(tmp_path): + h = Harness(tmp_path) + + def grid0(body) -> bool: + return body["metadata"]["labels"][LABEL_SHARD] == "grid-0" + + # The create lands but reports an error, so the sweep adopts its own + # Request by name; its failed Run is not retried. + h.kube.fail("create_request", KubeError("timeout"), landed=True, match=grid0) + + def fail_grid(kube, created): + if grid0(created): + h.operator.fail_runs[created["metadata"]["name"]] = "JobFailed" + + h.kube.on_request_created.append(fail_grid) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "run_failed" + shard = result.details["shards"]["grid-0"] + assert shard["adopted"] and shard["attempt"] == 1 + assert len([b for b in h.created() if grid0(b)]) == 1 + + +def test_input_mismatch_of_an_ordinary_shard_leaves_points_missing(tmp_path): + h = Harness(tmp_path) + startup = {CREDIT: 1.0, SCALE: 1.5, TEMP: 0.0} + request = make_request(baseline=grid(SMALL_DOMAIN)[0], startup=startup) + + def skew(name, effective): + if h.kube.requests[name]["metadata"]["labels"][LABEL_SHARD] == "point-startup": + effective["optimizer"]["parallelism"] = 1 + + h.operator.mutate_input = skew + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request, missing=[startup]) + assert result.details["shards"]["point-startup"]["reason"] == "input_mismatch" + + +def test_missing_run_input_on_a_succeeded_run_is_a_mismatch(tmp_path): + h = Harness(tmp_path, operator=False) + + def operator(): + for name in list(h.kube.requests): + if h.kube.run_of(name) is None: + h.kube.accept_request(name) + run = h.kube.run_of(name)["metadata"]["name"] + h.kube.start_run(run, record_input=False) + h.kube.complete_run(run) + + h.clock.hooks.append(operator) + result = h.sweep(make_request()) + assert result.failure_reason == "input_mismatch" + assert "status.aisimulate.input is missing" in result.error + + +def test_crn_unsupported_runtime_fails_the_window(tmp_path): + h = Harness(tmp_path) + h.operator.dynamo_version = "1.7.0" + result = h.sweep(make_request(), crn_min_version="1.8.0") + assert result.failure_reason == "crn_unsupported" + assert result.details["runtime_version"] == "1.7.0" + + +def test_crn_probe_mismatch_fails_the_window(tmp_path): + def evaluate(request, point): + metrics = default_metrics(point) + if request["metadata"]["labels"][LABEL_SHARD] == "probe": + metrics["goodput_output_throughput_tok_s"] *= 1.0 + 1e-6 + return metrics + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(make_request()) + assert result.failure_reason == "crn_mismatch" + assert result.details["crn_probe"]["relative_error"] == pytest.approx( + 1e-6, rel=1e-3 + ) + + +def test_crn_probe_within_tolerance_passes(tmp_path): + def evaluate(request, point): + metrics = default_metrics(point) + if request["metadata"]["labels"][LABEL_SHARD] == "probe": + metrics["goodput_output_throughput_tok_s"] *= 1.0 + 1e-13 + return metrics + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(make_request()) + assert result.status == "ok", result.error + + +def test_unscored_probe_is_a_crn_mismatch(tmp_path): + probe_point = {CREDIT: 1.0, SCALE: 2.0, TEMP: 0.5} + + def evaluate(request, point): + if request["metadata"]["labels"][LABEL_SHARD] == "probe": + return None + return default_metrics(point) + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(make_request()) + assert result.failure_reason == "crn_mismatch" + assert result.details["crn_probe"]["point"] == probe_point + + +def test_unpublished_and_unscorable_points_are_none(tmp_path): + request = make_request() + dropped = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.0} + broken = {CREDIT: 0.5, SCALE: 2.0, TEMP: 0.0} + + def evaluate(request_obj, point): + if point == public(dropped): + return None + metrics = default_metrics(point) + if point == public(broken): + del metrics["gpu_hours"] + return metrics + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request, missing=[dropped, broken]) + assert "gpu_hours" in result.metrics[RetuneParams(router=broken).key()]["error"] + assert ( + "not published" in result.metrics[RetuneParams(router=dropped).key()]["error"] + ) + assert result.details["missing_candidates"] == 2 + + +def test_scores_follow_the_retune_objective(tmp_path): + h = Harness(tmp_path, objective="TTFT") + request = make_request(objective="TTFT") + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request) + assert h.created()[0]["spec"]["objective"]["metric"] == "TTFT" + + +def test_engine_drift_between_candidates_is_engine_not_pinned(tmp_path): + h = Harness(tmp_path) + + def drift(name, parameters): + if h.kube.requests[name]["metadata"]["labels"][LABEL_SHARD] == "probe": + parameters["aggregated_parallelism"]["tensor"] = 4 + + h.operator.mutate_parameters = drift + result = h.sweep(make_request()) + assert result.failure_reason == "engine_not_pinned" + + +def test_router_policy_drift_is_engine_not_pinned(tmp_path): + h = Harness(tmp_path) + + def drift(name, parameters): + parameters["adapters"]["dynamo.router"]["prefill_load_model"] = {"type": "ais"} + + h.operator.mutate_parameters = drift + result = h.sweep(make_request()) + assert result.failure_reason == "engine_not_pinned" + + +def test_candidate_with_an_unknown_point_is_ignored(tmp_path): + h = Harness(tmp_path) + request = make_request() + odd = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.0} + + def drift(name, parameters): + router = parameters["adapters"]["dynamo.router"] + if ( + router["overlap_score_credit"], + router["prefill_load_scale"], + router["temperature"], + ) == (0.5, 1.0, 0.0): + router["prefill_load_scale"] = 7.0 + + h.operator.mutate_parameters = drift + result = h.sweep(request) + assert result.status == "ok", result.error + assert_scores(result, request, missing=[odd]) + assert "not one of the shard's values" in result.details["unmapped_candidates"][0] + + +def test_unscored_baseline_is_reported_for_the_controller(tmp_path): + request = make_request(baseline=grid(SMALL_DOMAIN)[0]) + baseline = request.baseline.router + + def evaluate(request_obj, point): + return None if point == public(baseline) else default_metrics(point) + + h = Harness(tmp_path) + h.operator.evaluate = evaluate + result = h.sweep(request) + # Status ok: _decide records the window failed for its missing baseline. + assert result.status == "ok" + assert result.scores[request.baseline.key()] is None + + +def test_empty_window_creates_nothing(tmp_path): + h = Harness(tmp_path) + h.cutter = FakeCutter(num_requests=0) + result = h.sweep(make_request()) + assert result.status == "empty" + assert result.failure_reason is None + assert result.error is None + assert h.created() == [] + + +@pytest.mark.parametrize( + "error, reason", + [ + (CutError("window_too_large"), "window_too_large"), + (CutError("trace_corrupt"), "trace_corrupt"), + (CutError("window_cut_timeout"), "window_cut_timeout"), + (CutError("something_else"), "window_cut_failed"), + (OSError("ENOENT"), "window_cut_failed"), + ], +) +def test_cut_failures_fail_the_window(tmp_path, error, reason): + h = Harness(tmp_path) + h.cutter = FakeCutter(error=error) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == reason + assert h.created() == [] + + +def test_transient_list_failures_are_retried(tmp_path): + h = Harness(tmp_path) + h.kube.fail("list_requests", KubeError("timeout"), times=2) + h.kube.fail("list_candidates", KubeError("timeout")) + request = make_request() + result = h.sweep(request) + assert result.status == "ok", result.error + assert result.details["k8s_errors"] == 3 + assert_scores(result, request) + + +def test_permanent_list_failure_fails_the_window(tmp_path): + h = Harness(tmp_path) + error = KubeError("forbidden") + error.status = 403 # type: ignore[attr-defined] + h.kube.fail("list_requests", error) + result = h.sweep(make_request()) + assert result.failure_reason == "k8s_error" + + +def test_unexpected_exception_never_escapes(tmp_path): + h = Harness(tmp_path) + h.kube.fail("list_requests", RuntimeError("boom")) + result = h.sweep(make_request()) + assert result.status == "failed" + assert result.failure_reason == "internal_error" + assert "boom" in result.error + + +@pytest.mark.parametrize( + "kwargs, reason", + [ + ({"router_domain": None}, "invalid_request"), + ({"planner": {"load_scaling_down_sensitivity": 50}}, "invalid_request"), + ( + {"router_domain": {CREDIT: tuple(float(v) for v in range(500))}}, + "invalid_domain", + ), + ], +) +def test_invalid_requests_fail_cleanly(tmp_path, kwargs, reason): + h = Harness(tmp_path) + if "router_domain" in kwargs and kwargs["router_domain"] is not None: + # A domain whose keys do not match the candidates' router keys. + request = make_request(router_domain=kwargs["router_domain"]) + else: + request = make_request(**kwargs) + result = h.sweep(request) + assert result.status == "failed" + assert result.failure_reason == reason + assert h.created() == [] + + +def test_invalid_engine_fails_cleanly(tmp_path): + h = Harness( + tmp_path, + cfg=make_cfg( + engine={**AGG_ENGINE, "backend": "sglang"}, mount_path=str(tmp_path) + ), + ) + result = h.sweep(make_request()) + assert result.failure_reason == "invalid_engine" + + +def test_deadline_defaults_to_the_request_timeout(tmp_path): + h = Harness(tmp_path) + h.operator.max_phase = "started" + result = h.sweep(make_request(deadline_s=None)) + assert result.status == "timeout" + assert result.details["deadline_s"] == LAUNCH_S + TIMEOUT_S + assert h.clock.now >= LAUNCH_S + TIMEOUT_S + + +def test_sweep_with_the_real_window_cutter(tmp_path): + """The executor and ``window_cut.cut_window`` agree on paths and deadline.""" + + window_cut = pytest.importorskip("dynamo.planner.retune.live.window_cut") + import gzip + import json + import time + + from dynamo.planner.tests.retune_sweep_helpers import trace_rows + + pod_dir = tmp_path / "trace" / "segments" / "llm" / "frontend-0" + pod_dir.mkdir(parents=True) + arrivals = [WINDOW_END_MS - 1000 * i for i in range(1, 6)] + rows = trace_rows(arrivals) + trace_rows([WINDOW_END_MS + 5], prefix="late") + with gzip.open(pod_dir / "req.000001.jsonl.gz", "wt", encoding="utf-8") as out: + out.writelines(json.dumps(row) + "\n" for row in rows) + + h = Harness(tmp_path) + clock = h.clock + + def real_cut(*args, deadline_s, **kwargs): + # The fake clock runs in the past; keep the same budget in real time. + return window_cut.cut_window( + *args, deadline_s=time.time() + (deadline_s - clock.now), **kwargs + ) + + request = make_request(trace_paths=(str(tmp_path / "trace" / "segments" / "llm"),)) + result = h.sweep(request, cut_window=real_cut) + assert result.status == "ok", result.error + assert result.num_requests == 5 + window_file = tmp_path / "trace" / "windows" / "llm" / f"{WINDOW_END_MS}.jsonl.gz" + with gzip.open(window_file, "rt", encoding="utf-8") as lines: + assert len(lines.readlines()) == 5 + claim_path = h.created()[0]["spec"]["workload"]["trace"]["source"]["pvc"]["path"] + assert tmp_path / "trace" / claim_path == window_file + assert_scores(result, request) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py new file mode 100644 index 000000000000..6fe507868881 --- /dev/null +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py @@ -0,0 +1,530 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for ``KubernetesRetuneKube`` and the shared ``FakeRetuneKube``. + +The client tests run the real ``kubernetes`` API classes and replace only +the REST transport, so they check the URLs, query parameters, timeouts and +bodies the client library actually sends, and the error mapping. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from typing import Any, Optional + +import pytest +from kubernetes import client +from kubernetes.client import ApiException + +from dynamo.planner.retune.live.contracts import ( + KubeAlreadyExists, + KubeConflict, + KubeError, + KubeInvalid, + KubeNotFound, + PodInfo, + RetuneKube, +) +from dynamo.planner.retune.live.kube import KubernetesRetuneKube, request_timeout +from dynamo.planner.tests.live_retune_fakes import ( + FakeRetuneKube, + lower_request_input, + match_labels, +) + +pytestmark = [ + pytest.mark.gpu_0, + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.planner, +] + +NS = "test-ns" + + +class FakeResponse: + def __init__(self, body: Any, status: int = 200) -> None: + self.status = status + self.reason = "OK" + self.data = body if isinstance(body, (str, bytes)) else json.dumps(body) + + def getheaders(self) -> dict: + return {"content-type": "application/json"} + + def getheader(self, name: str, default: Optional[str] = None) -> Optional[str]: + return self.getheaders().get(name.lower(), default) + + +class Transport: + """Stands in for ``RESTClientObject.request``: records and replies.""" + + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + self.replies: list[Any] = [] + + def reply(self, value: Any) -> None: + self.replies.append(value) + + def __call__( + self, + method, + url, + query_params=None, + headers=None, + post_params=None, + body=None, + _preload_content=True, + _request_timeout=None, + ): + self.calls.append( + { + "method": method, + "url": url, + "query": dict(query_params or []), + "body": body, + "preload": _preload_content, + "timeout": _request_timeout, + } + ) + value = self.replies.pop(0) if self.replies else {} + if isinstance(value, Exception): + raise value + if isinstance(value, FakeResponse): + return value + return FakeResponse(value) + + +def api_error(status: int, message: str = "nope") -> ApiException: + error = ApiException(status=status, reason="Error") + error.body = json.dumps({"kind": "Status", "message": message}) + return error + + +@pytest.fixture +def kube(): + api_client = client.ApiClient(client.Configuration()) + transport = Transport() + api_client.rest_client.request = transport + retune_kube = KubernetesRetuneKube(NS, api_client=api_client) + yield retune_kube, transport + api_client.close() + + +def path(call: dict) -> str: + return call["url"].split("://", 1)[-1].split("/", 1)[-1] + + +# ---------------------------------------------------------------- client + + +def test_request_timeout_is_a_tuple_of_whole_seconds(): + assert request_timeout(10.0) == (10, 10) + assert request_timeout(2.5) == (3, 3) + assert request_timeout(0.2) == (1, 1) + assert request_timeout(0.0) == (1, 1) + assert request_timeout(float("nan")) == (1, 1) + + +def test_satisfies_the_protocol(kube): + retune_kube, _ = kube + assert isinstance(retune_kube, KubernetesRetuneKube) + _: RetuneKube = retune_kube + assert retune_kube.namespace == NS + with pytest.raises(ValueError): + KubernetesRetuneKube("") + + +def test_get_dgd(kube): + retune_kube, transport = kube + dgd = {"metadata": {"name": "llm", "uid": "u1"}, "spec": {"services": {}}} + transport.reply(dgd) + assert retune_kube.get_dgd("llm", timeout_s=4) == dgd + (call,) = transport.calls + assert call["method"] == "GET" + assert ( + path(call) + == f"apis/nvidia.com/v1beta1/namespaces/{NS}/dynamographdeployments/llm" + ) + assert call["timeout"] == (4, 4) + + transport.reply(api_error(404)) + with pytest.raises(KubeNotFound) as raised: + retune_kube.get_dgd("gone") + assert raised.value.status == 404 + + +def test_create_request_is_strict_and_dry_run_on_request(kube): + retune_kube, transport = kube + body = {"metadata": {"name": "r1"}, "spec": {"model": {"name": "m"}}} + transport.reply({**body, "metadata": {"name": "r1", "uid": "x"}}) + created = retune_kube.create_request(body, timeout_s=2.5) + assert created["metadata"]["uid"] == "x" + call = transport.calls[-1] + assert call["method"] == "POST" + assert ( + path(call) + == f"apis/nvidia.com/v1beta2/namespaces/{NS}/dynamographdeploymentrequests" + ) + assert call["query"]["fieldValidation"] == "Strict" + assert call["query"]["fieldManager"] == "dynamo-planner-retune" + assert "dryRun" not in call["query"] + assert call["body"] == body + assert call["timeout"] == (3, 3) + + transport.reply(body) + retune_kube.create_request(body, dry_run=True) + assert transport.calls[-1]["query"]["dryRun"] == "All" + assert transport.calls[-1]["query"]["fieldValidation"] == "Strict" + + +@pytest.mark.parametrize( + "status, error_type", + [ + (409, KubeAlreadyExists), + (422, KubeInvalid), + (400, KubeInvalid), + (403, KubeError), + (500, KubeError), + ], +) +def test_create_request_maps_api_errors(kube, status, error_type): + retune_kube, transport = kube + transport.reply(api_error(status, "strict decoding error: unknown field")) + with pytest.raises(error_type) as raised: + retune_kube.create_request({"metadata": {"name": "r1"}}) + assert raised.value.status == status + assert f"HTTP {status}" in str(raised.value) + assert "unknown field" in str(raised.value) + if error_type is KubeError: + assert type(raised.value) is KubeError + + +def test_network_errors_become_kube_errors(kube): + retune_kube, transport = kube + # Imported here: the marker-report hook collects tests without urllib3. + from urllib3.exceptions import ReadTimeoutError + + transport.reply(ReadTimeoutError(None, "url", "timed out")) + with pytest.raises(KubeError) as raised: + retune_kube.get_request("r1") + assert raised.value.status is None + assert "ReadTimeoutError" in str(raised.value) + + +def test_requests_runs_and_candidates(kube): + retune_kube, transport = kube + transport.reply({"metadata": {"name": "r1"}}) + assert retune_kube.get_request("r1") == {"metadata": {"name": "r1"}} + transport.reply(api_error(404)) + assert retune_kube.get_request("missing") is None + + transport.reply({"items": [{"metadata": {"name": "r1"}}]}) + assert retune_kube.list_requests("a=b,c=d") == [{"metadata": {"name": "r1"}}] + call = transport.calls[-1] + assert ( + path(call) + == f"apis/nvidia.com/v1beta2/namespaces/{NS}/dynamographdeploymentrequests" + ) + assert call["query"]["labelSelector"] == "a=b,c=d" + + transport.reply({"metadata": {"name": "r1-1"}}) + assert retune_kube.get_run("r1-1") == {"metadata": {"name": "r1-1"}} + assert path(transport.calls[-1]).endswith("dynamographdeploymentruns/r1-1") + transport.reply(api_error(404)) + assert retune_kube.get_run("gone") is None + + transport.reply({"items": []}) + assert retune_kube.list_candidates("nvidia.com/dgdr-run-uid=u") == [] + call = transport.calls[-1] + assert path(call).endswith("dynamographdeploymentcandidates") + assert call["query"]["labelSelector"] == "nvidia.com/dgdr-run-uid=u" + + transport.reply({}) + retune_kube.delete_request("r1", propagation="Foreground", timeout_s=7) + call = transport.calls[-1] + assert call["method"] == "DELETE" + assert path(call).endswith("dynamographdeploymentrequests/r1") + assert call["query"]["propagationPolicy"] == "Foreground" + assert call["timeout"] == (7, 7) + transport.reply(api_error(404)) + retune_kube.delete_request("gone") # missing is not an error + transport.reply(api_error(500)) + with pytest.raises(KubeError): + retune_kube.delete_request("r1") + + +def _pod( + name: str, + *, + ready: bool = True, + deleting: bool = False, + containers: Optional[list] = None, + ip: Optional[str] = "10.0.0.5", +) -> dict: + return { + "metadata": { + "name": name, + "uid": f"uid-{name}", + "labels": {"nvidia.com/dynamo-component-type": "frontend"}, + **({"deletionTimestamp": "2026-10-28T00:00:00Z"} if deleting else {}), + }, + "spec": { + "containers": containers + if containers is not None + else [ + {"name": "sidecar", "ports": [{"name": "http", "containerPort": 9999}]}, + { + "name": "main", + "ports": [ + {"name": "metrics", "containerPort": 9090}, + {"name": "http", "containerPort": 8000}, + ], + }, + ] + }, + "status": { + "podIP": ip, + "conditions": [ + {"type": "PodScheduled", "status": "True"}, + {"type": "Ready", "status": "True" if ready else "False"}, + ], + }, + } + + +def test_list_pods_builds_pod_info(kube): + retune_kube, transport = kube + transport.reply( + { + "kind": "PodList", + "apiVersion": "v1", + "metadata": {}, + "items": [ + _pod("a"), + _pod("b", ready=False, deleting=True, ip=None), + _pod( + "c", + containers=[ + { + "name": "other", + "ports": [{"name": "http", "containerPort": 8080}], + } + ], + ), + _pod("d", containers=[{"name": "main"}]), + ], + } + ) + pods = retune_kube.list_pods("x=y", timeout_s=3) + call = transport.calls[-1] + assert path(call) == f"api/v1/namespaces/{NS}/pods" + assert call["query"]["labelSelector"] == "x=y" + assert call["timeout"] == (3, 3) + assert pods[0] == PodInfo( + name="a", + uid="uid-a", + ip="10.0.0.5", + ready=True, + terminating=False, + labels={"nvidia.com/dynamo-component-type": "frontend"}, + http_port=8000, # the main container's port wins + ) + assert (pods[1].ready, pods[1].terminating, pods[1].ip) == (False, True, None) + assert pods[2].http_port == 8080 + assert pods[3].http_port is None + + +def test_leases_round_trip_raw_json(kube): + retune_kube, transport = kube + lease = { + "apiVersion": "coordination.k8s.io/v1", + "kind": "Lease", + "metadata": {"name": "llm-planner-retune", "resourceVersion": "5"}, + "spec": { + "holderIdentity": "pod/abc", + "leaseDurationSeconds": 60, + "renewTime": "2026-10-28T12:00:00.000000Z", + }, + } + transport.reply(FakeResponse(json.dumps(lease).encode())) + # MicroTime strings come back verbatim, not re-serialized datetimes. + assert retune_kube.get_lease("llm-planner-retune") == lease + call = transport.calls[-1] + assert call["preload"] is False + assert ( + path(call) + == f"apis/coordination.k8s.io/v1/namespaces/{NS}/leases/llm-planner-retune" + ) + transport.reply(api_error(404)) + assert retune_kube.get_lease("gone") is None + + body = {"metadata": {"name": "l2"}, "spec": {"holderIdentity": "me"}} + transport.reply({**body, "metadata": {"name": "l2", "resourceVersion": "1"}}) + created = retune_kube.create_lease(body) + assert created["metadata"]["resourceVersion"] == "1" + call = transport.calls[-1] + assert call["method"] == "POST" + assert call["body"]["apiVersion"] == "coordination.k8s.io/v1" + assert call["body"]["kind"] == "Lease" + assert "apiVersion" not in body # the caller's dict is not mutated + transport.reply(api_error(409)) + with pytest.raises(KubeAlreadyExists): + retune_kube.create_lease(body) + + transport.reply(lease) + retune_kube.replace_lease("llm-planner-retune", lease, timeout_s=2) + call = transport.calls[-1] + assert call["method"] == "PUT" + assert call["body"]["metadata"]["resourceVersion"] == "5" + assert call["timeout"] == (2, 2) + transport.reply(api_error(409, "the object has been modified")) + with pytest.raises(KubeConflict) as raised: + retune_kube.replace_lease("llm-planner-retune", lease) + assert raised.value.status == 409 + transport.reply(api_error(404)) + with pytest.raises(KubeNotFound): + retune_kube.replace_lease("gone", lease) + + transport.reply({"kind": "Status", "status": "Success"}) + retune_kube.delete_lease("l2") + assert transport.calls[-1]["method"] == "DELETE" + transport.reply(api_error(404)) + retune_kube.delete_lease("l2") + + +def test_from_kube_api_shares_the_connector_client(): + api_client = client.ApiClient(client.Configuration()) + kube_api = SimpleNamespace( + current_namespace="ns2", + custom_api=client.CustomObjectsApi(api_client), + ) + retune_kube = KubernetesRetuneKube.from_kube_api(kube_api) + assert retune_kube.namespace == "ns2" + assert retune_kube.api_client is api_client + api_client.close() + + +# ------------------------------------------------------------------ fake + + +def test_fake_label_selectors(): + labels = {"a": "1", "b": "2"} + assert match_labels(labels, "") + assert match_labels(labels, "a=1,b==2") + assert match_labels(labels, "a,!c") + assert not match_labels(labels, "a!=1") + assert not match_labels(labels, "c") + assert not match_labels(labels, "!a") + with pytest.raises(ValueError): + match_labels(labels, "a in (1,2)") + + +def test_fake_lease_compare_and_swap_and_faults(): + kube = FakeRetuneKube(NS, now=lambda: 1_793_000_000.0) + created = kube.create_lease( + {"metadata": {"name": "l"}, "spec": {"holderIdentity": ""}} + ) + with pytest.raises(KubeAlreadyExists): + kube.create_lease({"metadata": {"name": "l"}}) + stale = dict(created, spec={"holderIdentity": "me"}) + updated = kube.replace_lease("l", stale) + assert updated["spec"]["holderIdentity"] == "me" + assert ( + updated["metadata"]["resourceVersion"] != created["metadata"]["resourceVersion"] + ) + with pytest.raises(KubeConflict): + kube.replace_lease("l", stale) # resourceVersion is stale now + with pytest.raises(KubeNotFound): + kube.replace_lease("missing", stale) + + # A write that times out but lands. + kube.fail("replace_lease", KubeError("timeout"), landed=True) + with pytest.raises(KubeError): + kube.replace_lease("l", dict(updated, spec={"holderIdentity": "other"})) + assert kube.get_lease("l")["spec"]["holderIdentity"] == "other" + # A write that fails before landing. + current = kube.get_lease("l") + kube.fail("replace_lease", KubeError("503")) + with pytest.raises(KubeError): + kube.replace_lease("l", dict(current, spec={"holderIdentity": "x"})) + assert kube.get_lease("l")["spec"]["holderIdentity"] == "other" + kube.delete_lease("l") + kube.delete_lease("l") + assert kube.get_lease("l") is None + assert [call.method for call in kube.calls].count("replace_lease") == 5 + + +def test_fake_requests_cascade_and_copies(): + kube = FakeRetuneKube(NS) + body = {"metadata": {"name": "r", "labels": {"w": "1"}}, "spec": {"x": 1}} + stored = kube.create_request(body) + stored["spec"]["x"] = 2 # copies never alias the store + assert kube.get_request("r")["spec"]["x"] == 1 + assert kube.create_request({"metadata": {"name": "dry"}}, dry_run=True) + assert kube.get_request("dry") is None + run = kube.accept_request("r") + kube.start_run(run["metadata"]["name"]) + kube.complete_run(run["metadata"]["name"], candidates=[({"p": 1}, {"m": 2})]) + request = kube.get_request("r") + assert request["status"]["recentRunRefs"] == [{"name": "r-1"}] + assert "activeRunRef" not in request["status"] + assert ( + len(kube.list_candidates(f"nvidia.com/dgdr-run-uid={run['metadata']['uid']}")) + == 1 + ) + assert kube.list_requests("w=1")[0]["metadata"]["name"] == "r" + kube.delete_request("r", propagation="Foreground") + assert kube.runs == {} and kube.candidates == {} + assert kube.deleted_requests == [("r", "Foreground")] + + +def test_fake_lowering_applies_overrides_and_operator_fields(): + request = { + "metadata": {"name": "r", "namespace": NS}, + "spec": { + "model": {"name": "m"}, + "backends": ["vllm"], + "hardware": {"gpu": {"sku": "h200_sxm", "budget": 4}}, + "workload": { + "trace": { + "format": "dynamo", + "source": {"pvc": {"claimName": "c", "path": "w/1.jsonl.gz"}}, + } + }, + "objective": { + "mode": "Optimize", + "metric": "GoodputPerGPU", + "sla": {"ttftMs": 1.0}, + }, + "recommendation": {"maxCandidates": 3}, + "aisimulate": { + "inputOverrides": { + "engine": {"backend": "vllm", "workers": {"prefill": None}}, + "dgdr_run": {"generator": "direct"}, + } + }, + }, + } + lowered = lower_request_input(request, run_name="r-1") + assert lowered["engine"]["backend"] == "vllm" + assert "prefill" not in lowered["engine"]["workers"] + assert lowered["engine"]["workers"]["decode"] == {} + assert lowered["optimization"] == { + "constraints": {"max_candidate_gpus": 4}, + "target": "goodput_per_gpu", + "strict_sla": True, + } + assert lowered["traffic"] == { + "source": { + "type": "trace", + "format": "dynamo", + "paths": ["/var/run/aisimulate/trace/trace.jsonl.gz"], + }, + "load": {"type": "trace_timestamps"}, + } + assert lowered["optimizer"] == {"max_trials": 320.0} + assert lowered["dgdr_run"]["generator"] == "direct" + assert lowered["dgdr_run"]["max_candidates"] == 3 + assert lowered["dgdr_run"]["namespace"] == NS From fd45a8592be81a07589051e8c93b914fb1418bb5 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 08:26:41 -0800 Subject: [PATCH 078/100] fix(replay): address review on trace replay common random numbers Fold the trace replay determinism check into test_trace_runner_preserves_current_replay_arguments with an agg and a disagg row, and drop the separate test file. Cut the runner comment to the invariant, and update replay/api.py: _deterministic now applies to every trace replay through the Sweeper's Dynamo runner, not only to retune sweeps. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/replay/api.py | 7 +- components/src/dynamo/replay/simulation.py | 10 +- .../dynamo/replay/tests/test_simulation.py | 35 ++++- .../replay/tests/test_simulation_crn.py | 140 ------------------ 4 files changed, 39 insertions(+), 153 deletions(-) delete mode 100644 components/src/dynamo/replay/tests/test_simulation_crn.py diff --git a/components/src/dynamo/replay/api.py b/components/src/dynamo/replay/api.py index 0ac237c70b99..baf1a285538a 100644 --- a/components/src/dynamo/replay/api.py +++ b/components/src/dynamo/replay/api.py @@ -97,7 +97,7 @@ class _TraceReplayOptions(_CommonReplayOptions, total=False): trace_num_prefix_groups: int report_jsonl_path: str | os.PathLike[str] | None max_sim_time_ms: float | None - # Internal to retune sweeps; not part of the public API. + # Internal; not part of the public API. _planner_warmup_observations: "Sequence[TrafficObservation] | None" _deterministic: bool @@ -420,8 +420,9 @@ def run_trace_replay( # Internal: a retune sweep's inner planner is warmed with the history it # already bucketed instead of its load_predictor_warmup_trace. _planner_warmup_observations=None, - # Internal: a retune sweep replays the baseline and every candidate with - # the canonical router seed and ordinal request ids (common random numbers). + # Internal: replay with the canonical router seed and ordinal request ids + # (common random numbers). Retune sweeps and every trace replay through + # the Sweeper's Dynamo runner set it. _deterministic=False, ) -> ReplayReport | dict[str, Any]: """Run trace replay. diff --git a/components/src/dynamo/replay/simulation.py b/components/src/dynamo/replay/simulation.py index ca04ea4b9874..251e033199b4 100644 --- a/components/src/dynamo/replay/simulation.py +++ b/components/src/dynamo/replay/simulation.py @@ -409,13 +409,9 @@ def _run_trace( # embedded value instead of imposing the synthetic-workload default. if trace_format not in {"weka", "dynamo"} and trace_block_size is None: trace_block_size = self.trace_block_size - # Common random numbers: every candidate of a trace sweep replays the - # same trace with ordinal request ids and the canonical router seed, - # so candidates differ only by their configuration and a score does - # not depend on the other candidates, the Run or the parallelism. - # Live planner retune compares candidates scored by different DGDR - # Runs of one window on that basis. The synthetic path has no such - # option. + # Trace replays use ordinal request ids and the canonical router seed, + # so a candidate's score depends only on its configuration and the + # trace. if deployment.deployment_mode == "agg": return run_trace_replay( trace_files=trace_files, diff --git a/components/src/dynamo/replay/tests/test_simulation.py b/components/src/dynamo/replay/tests/test_simulation.py index 4c7c7b3b7cea..23725992c6d5 100644 --- a/components/src/dynamo/replay/tests/test_simulation.py +++ b/components/src/dynamo/replay/tests/test_simulation.py @@ -97,7 +97,35 @@ def _agg_deployment() -> BackendDeploymentSpec: ) -def test_trace_runner_preserves_current_replay_arguments(monkeypatch) -> None: +def _disagg_deployment() -> BackendDeploymentSpec: + return BackendDeploymentSpec( + deployment_mode="disagg", + backend="vllm", + backend_version="0.11.0", + prefill_engine_args={ + "worker_type": "prefill", + "timing_model": {"type": "polynomial"}, + }, + decode_engine_args={ + "worker_type": "decode", + "timing_model": {"type": "polynomial"}, + }, + num_prefill_workers=1, + num_decode_workers=2, + ) + + +@pytest.mark.parametrize( + ("deployment", "workers"), + [ + (_agg_deployment(), {"num_workers": 3}), + (_disagg_deployment(), {"num_prefill_workers": 1, "num_decode_workers": 2}), + ], + ids=["agg", "disagg"], +) +def test_trace_runner_preserves_current_replay_arguments( + monkeypatch, deployment, workers +) -> None: seen = {} def fake_run_trace_replay(**kwargs): @@ -113,7 +141,7 @@ def fake_run_trace_replay(**kwargs): monkeypatch.setattr(simulation, "KvRouterConfig", _FakeRouterConfig) monkeypatch.setattr(simulation, "run_trace_replay", fake_run_trace_replay) spec = ReplaySpec( - backend_deployment=_agg_deployment(), + backend_deployment=deployment, workload={ "trace_path": "tiny.jsonl", "trace_format": "dynamo", @@ -159,7 +187,8 @@ def fake_run_trace_replay(**kwargs): assert seen["trace_files"] == "tiny.jsonl" assert seen["trace_format"] == "dynamo" - assert seen["num_workers"] == 3 + assert {name: seen[name] for name in workers} == workers + assert seen["_deterministic"] is True assert seen["router_mode"] == "kv_router" assert seen["planner_config"] == {"mode": "agg"} assert seen["arrival_speedup_ratio"] == 2.0 diff --git a/components/src/dynamo/replay/tests/test_simulation_crn.py b/components/src/dynamo/replay/tests/test_simulation_crn.py deleted file mode 100644 index ea45aeabdf05..000000000000 --- a/components/src/dynamo/replay/tests/test_simulation_crn.py +++ /dev/null @@ -1,140 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: E402 -# Optional-dependency preflight must run before the simulation imports. - -"""Common random numbers in the Sweeper replay runner. - -Trace replays (what a DGDR trace Run evaluates) are deterministic, so live -planner retune can compare candidates scored by different Runs of one -window; synthetic replays have no such option and must not receive it. -""" - -from __future__ import annotations - -import inspect - -import pytest - -pytest.importorskip( - "aisimulate.sweeper", - reason="AI Simulate is an optional Dynamo simulation dependency", -) - -from aisimulate.sweeper.replay import BackendDeploymentSpec, ReplaySpec - -from dynamo.replay import ReplayReport -from dynamo.replay import api as replay_api -from dynamo.replay import simulation - -pytestmark = [ - pytest.mark.pre_merge, - pytest.mark.unit, - pytest.mark.gpu_0, - pytest.mark.planner, -] - -_REPORT = ReplayReport( - summary={"completed_requests": 1, "output_throughput_tok_s": 1.0}, - per_request=None, - coverage={}, - planner=None, -) - - -def _deployment(mode: str) -> BackendDeploymentSpec: - if mode == "agg": - return BackendDeploymentSpec( - deployment_mode="agg", - backend="vllm", - backend_version="0.11.0", - agg_engine_args={"timing_model": {"type": "polynomial"}}, - num_workers=2, - ) - return BackendDeploymentSpec( - deployment_mode="disagg", - backend="vllm", - backend_version="0.11.0", - prefill_engine_args={ - "worker_type": "prefill", - "timing_model": {"type": "polynomial"}, - }, - decode_engine_args={ - "worker_type": "decode", - "timing_model": {"type": "polynomial"}, - }, - num_prefill_workers=1, - num_decode_workers=2, - ) - - -def _strict_fake(real, seen: dict): - """A replay entry point that binds its kwargs to ``real``'s signature.""" - - signature = inspect.signature(real) - - def fake(*args, **kwargs): - signature.bind(*args, **kwargs) - seen.update(kwargs) - return _REPORT - - return fake - - -@pytest.mark.parametrize("mode", ["agg", "disagg"]) -@pytest.mark.parametrize( - "workload", - [ - {"trace_path": "window.jsonl.gz", "trace_format": "dynamo"}, - {"trace_paths": ["a.jsonl", "b.jsonl"], "trace_format": "mooncake"}, - ], -) -def test_trace_replays_are_deterministic(monkeypatch, mode, workload): - seen: dict = {} - monkeypatch.setattr( - simulation, - "run_trace_replay", - _strict_fake(replay_api.run_trace_replay, seen), - ) - spec = ReplaySpec( - backend_deployment=_deployment(mode), - workload=workload, - goal={"target": "throughput"}, - ) - - simulation.DynamoReplayRunnerFactory().create(0).run(spec) - - assert seen["_deterministic"] is True - - -@pytest.mark.parametrize("mode", ["agg", "disagg"]) -def test_synthetic_replays_do_not_receive_the_flag(monkeypatch, mode): - # The synthetic entry point has no such parameter; the strict fake fails - # if the flag leaks into the shared kwargs. - assert ( - "_deterministic" - not in inspect.signature(replay_api.run_synthetic_trace_replay).parameters - ) - seen: dict = {} - monkeypatch.setattr( - simulation, - "run_synthetic_trace_replay", - _strict_fake(replay_api.run_synthetic_trace_replay, seen), - ) - spec = ReplaySpec( - backend_deployment=_deployment(mode), - workload={ - "trace_path": None, - "isl": 512, - "osl": 128, - "num_request_ratio": 10.0, - "concurrency": None, - "request_rate": 20.0, - }, - goal={"target": "throughput"}, - ) - - simulation.DynamoReplayRunnerFactory().create(0).run(spec) - - assert seen # the synthetic path ran - assert "_deterministic" not in seen From 1f3bc6b1aa40e1ae1f5609f1de815d6646c1de2e Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 08:42:14 -0800 Subject: [PATCH 079/100] fix(planner): address review on the live retune configuration and premises - Parse the frontend's and workers' router flags through a public parser builder: ArgEnv in dynamo.common.configuration.utils carries another process's env, the router, KV router and frontend argument groups take it, and frontend_args.build_frontend_parser(env) returns the parser and its env var -> dest map. Every env default comes from that env, notices are skipped and nothing reads the local filesystem, so frontend_config no longer copies functions with overridden globals or touches router_args' private action classes. - Keep one DGDR shard planner: balanced_shard_parts takes the Run parallelism and minimizes evaluation rounds first, dgdr_rounds is public, and P13 plans with the same call the executor will use. - evaluate_premises(dgd, planner_config=...) derives the startup and the parse error itself and reads the planner replicas from the DGD. - Drop the unused trace_incomplete_threshold, the v1alpha1 DGD layout and the graph spec.envs fallback, and envFrom from the parse cache key. - P6 treats only ImportError as "aisimulate absent"; any other failure is logged and reported as a P6 warning. - Move the function-body imports to the module top, remove references to the design document, and move live_retune_fakes.py to the DGDR sweep layer, which uses it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../configuration/groups/kv_router_args.py | 47 +- .../configuration/groups/router_args.py | 64 +- .../src/dynamo/common/configuration/utils.py | 37 +- .../common/tests/configuration/test_utils.py | 30 + .../src/dynamo/frontend/frontend_args.py | 101 +- .../tests/test_frontend_parser_builder.py | 59 + .../dynamo/planner/config/planner_config.py | 91 +- .../dynamo/planner/retune/live/contracts.py | 7 +- .../planner/retune/live/frontend_config.py | 319 +---- .../dynamo/planner/tests/live_retune_fakes.py | 1098 ----------------- .../tests/unit/test_live_retune_config.py | 43 +- .../unit/test_live_retune_frontend_config.py | 170 +-- 12 files changed, 505 insertions(+), 1561 deletions(-) create mode 100644 components/src/dynamo/frontend/tests/test_frontend_parser_builder.py delete mode 100644 components/src/dynamo/planner/tests/live_retune_fakes.py diff --git a/components/src/dynamo/common/configuration/groups/kv_router_args.py b/components/src/dynamo/common/configuration/groups/kv_router_args.py index 4e96deff1b8b..a1a2178182d0 100644 --- a/components/src/dynamo/common/configuration/groups/kv_router_args.py +++ b/components/src/dynamo/common/configuration/groups/kv_router_args.py @@ -24,6 +24,7 @@ from dynamo.common.configuration.arg_group import ArgGroup from dynamo.common.configuration.config_base import ConfigBase from dynamo.common.configuration.utils import ( + ArgEnv, Deprecated, add_argument, add_negatable_bool_argument, @@ -297,9 +298,17 @@ def kv_router_kwargs(self) -> dict: class KvRouterArgGroup(ArgGroup): - """CLI arguments for the shared KvRouterConfig parameters.""" + """CLI arguments for the shared KvRouterConfig parameters. + + Args: + env: Another process's env (see ``ArgEnv``). + """ + + def __init__(self, *, env: Optional[ArgEnv] = None) -> None: + self.env = env def add_arguments(self, parser) -> None: + environ = os.environ if self.env is None else self.env.environ g = parser.add_argument_group("KV Router Options") add_negatable_bool_argument( @@ -320,6 +329,7 @@ def add_arguments(self, parser) -> None: "and prefill-token load tracking, and disables remote/shared cache indexers. " "The builtin policy does not request cache inputs in this mode." ), + env=self.env, ) add_argument( g, @@ -334,6 +344,7 @@ def add_arguments(self, parser) -> None: ), arg_type=float, dest="overlap_score_credit", + env=self.env, ) add_argument( g, @@ -349,6 +360,7 @@ def add_arguments(self, parser) -> None: ), arg_type=float, dest="overlap_score_credit_decay", + env=self.env, ) add_argument( g, @@ -356,7 +368,7 @@ def add_arguments(self, parser) -> None: obsolete_flag="--kv-overlap-score-weight", env_var=( "DYN_ROUTER_KV_OVERLAP_SCORE_WEIGHT" - if "DYN_ROUTER_KV_OVERLAP_SCORE_WEIGHT" in os.environ + if "DYN_ROUTER_KV_OVERLAP_SCORE_WEIGHT" in environ else "DYN_OVERLAP_SCORE_WEIGHT" ), dest="overlap_score_weight", @@ -364,6 +376,7 @@ def add_arguments(self, parser) -> None: deprecated=_policy_parameter("prefill_load_scale"), default=None, help=argparse.SUPPRESS, + env=self.env, ) add_argument( g, @@ -377,6 +390,7 @@ def add_arguments(self, parser) -> None: ), arg_type=float, dest="prefill_load_scale", + env=self.env, ) add_argument( g, @@ -392,6 +406,7 @@ def add_arguments(self, parser) -> None: ), arg_type=float, dest="decode_active_request_weight", + env=self.env, ) add_argument( g, @@ -406,6 +421,7 @@ def add_arguments(self, parser) -> None: ), arg_type=float, dest="host_cache_hit_weight", + env=self.env, ) add_argument( g, @@ -419,6 +435,7 @@ def add_arguments(self, parser) -> None: ), arg_type=float, dest="disk_cache_hit_weight", + env=self.env, ) add_argument( g, @@ -431,6 +448,7 @@ def add_arguments(self, parser) -> None: "Higher values promote more randomness, and 0 falls back to deterministic." ), arg_type=float, + env=self.env, ) add_negatable_bool_argument( g, @@ -444,6 +462,7 @@ def add_arguments(self, parser) -> None: ), dest="use_kv_events", obsolete_flag="--kv-events", + env=self.env, ) add_negatable_bool_argument( g, @@ -454,6 +473,7 @@ def add_arguments(self, parser) -> None: "KV Router: Enable best-effort active-sequence synchronization through " "the Runtime event plane." ), + env=self.env, ) add_negatable_bool_argument( g, @@ -466,6 +486,7 @@ def add_arguments(self, parser) -> None: "By default, active blocks are tracked for load balancing." ), obsolete_flag="--track-active-blocks", + env=self.env, ) add_negatable_bool_argument( g, @@ -480,6 +501,7 @@ def add_arguments(self, parser) -> None: "suffix; shared prompt blocks retain full weight." ), obsolete_flag="--track-output-blocks", + env=self.env, ) add_negatable_bool_argument( g, @@ -493,6 +515,7 @@ def add_arguments(self, parser) -> None: "(when KV cache reuse is not expected)." ), obsolete_flag="--assume-kv-reuse", + env=self.env, ) add_negatable_bool_argument( g, @@ -505,6 +528,7 @@ def add_arguments(self, parser) -> None: "Use --no-router-track-prefill-tokens to ignore prompt tokens in router " "prefill-token load, queue pressure, and active_prefill_tokens metrics." ), + env=self.env, ) add_argument( g, @@ -513,6 +537,7 @@ def add_arguments(self, parser) -> None: default="public-xxh3-v1", choices=["public-xxh3-v1", "keyed-xxh3-v1"], help="KV Router: Hash function for router-derived active-sequence identities.", + env=self.env, ) add_argument( g, @@ -520,6 +545,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_ROUTER_TRACKING_KEY_FILE", default=None, help="KV Router: File containing the 32-byte provider tracking key.", + env=self.env, ) add_argument( g, @@ -527,6 +553,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_ROUTER_TRACKING_KEY_ID", default=None, help="KV Router: Provider-managed tracking-key epoch identifier.", + env=self.env, ) add_argument( g, @@ -540,6 +567,7 @@ def add_arguments(self, parser) -> None: "'none' keeps static prompt load accounting. " "'ais' decays the oldest active prefill request using AISimulate-predicted duration." ), + env=self.env, ) add_argument( g, @@ -551,6 +579,7 @@ def add_arguments(self, parser) -> None: "Only used when --no-router-kv-events is set." ), arg_type=float, + env=self.env, ) add_argument( g, @@ -564,6 +593,7 @@ def add_arguments(self, parser) -> None: "the existing time-based behavior; 'lru' uses each worker rank's " "advertised KV capacity. Side indexers remain TTL-only." ), + env=self.env, ) add_argument( g, @@ -588,6 +618,7 @@ def add_arguments(self, parser) -> None: "explicitly for predictable semantics, or use a smaller threshold." ), arg_type=nullable_float, + env=self.env, ) add_argument( g, @@ -602,6 +633,7 @@ def add_arguments(self, parser) -> None: "router_queue_threshold is set." ), arg_type=str, + env=self.env, ) add_negatable_bool_argument( g, @@ -616,6 +648,7 @@ def add_arguments(self, parser) -> None: "for each request." ), dest="conditional_disagg_enabled", + env=self.env, ) add_argument( g, @@ -631,6 +664,7 @@ def add_arguments(self, parser) -> None: arg_type=_conditional_disagg_config_arg, dest="conditional_disagg_config", metavar="JSON", + env=self.env, ) add_argument( g, @@ -647,6 +681,7 @@ def add_arguments(self, parser) -> None: "Use 'default' for Dynamo's built-in worker selector." ), arg_type=str, + env=self.env, ) add_argument( g, @@ -663,6 +698,7 @@ def add_arguments(self, parser) -> None: "Use 'default' for Dynamo's built-in worker selector." ), arg_type=str, + env=self.env, ) add_argument( g, @@ -675,6 +711,7 @@ def add_arguments(self, parser) -> None: "approximate routing when --no-router-kv-events is set." ), arg_type=int, + env=self.env, ) add_argument( g, @@ -692,6 +729,7 @@ def add_arguments(self, parser) -> None: ), arg_type=str, choices=["fcfs", "wspt"], + env=self.env, ) add_negatable_bool_argument( g, @@ -703,6 +741,7 @@ def add_arguments(self, parser) -> None: "component via the request plane instead of maintaining a local radix tree." ), dest="use_remote_indexer", + env=self.env, ) add_negatable_bool_argument( g, @@ -715,6 +754,7 @@ def add_arguments(self, parser) -> None: "index for routing decisions yet." ), dest="enable_session_prefix_index", + env=self.env, ) add_argument( g, @@ -730,6 +770,7 @@ def add_arguments(self, parser) -> None: "The default policy uses 0.5 when shared cache is enabled." ), arg_type=float, + env=self.env, ) add_argument( g, @@ -744,6 +785,7 @@ def add_arguments(self, parser) -> None: ), arg_type=str, choices=["none", "hicache"], + env=self.env, ) add_argument( g, @@ -756,4 +798,5 @@ def add_arguments(self, parser) -> None: "Independent of --router-ttl-secs, which covers pure approximate mode." ), arg_type=float, + env=self.env, ) diff --git a/components/src/dynamo/common/configuration/groups/router_args.py b/components/src/dynamo/common/configuration/groups/router_args.py index 728c76ea05fa..243fc36153e0 100644 --- a/components/src/dynamo/common/configuration/groups/router_args.py +++ b/components/src/dynamo/common/configuration/groups/router_args.py @@ -24,7 +24,12 @@ KvRouterArgGroup, KvRouterConfigBase, ) -from dynamo.common.configuration.utils import add_argument, nullable_float, nullable_int +from dynamo.common.configuration.utils import ( + ArgEnv, + add_argument, + nullable_float, + nullable_int, +) if TYPE_CHECKING: from dynamo.llm import RouterConfig @@ -64,7 +69,14 @@ ) -class _IgnoredAdmissionControlAction(argparse.Action): +class _IgnoredAction(argparse.Action): + """Store nothing, so the namespace never carries the value.""" + + def __call__(self, parser, namespace, values, option_string=None): + return None + + +class _IgnoredAdmissionControlAction(_IgnoredAction): """Warn and store nothing, so the namespace never carries the value.""" def __call__(self, parser, namespace, values, option_string=None): @@ -165,6 +177,8 @@ class RouterArgGroup(ArgGroup): consumes. ``--router-min-initial-workers`` gates frontend startup and is not carried on the model card, so a worker registering it would ship a flag that does nothing. + env: Another process's env (see ``ArgEnv``); the removed and + deprecated flags then parse without their notices. """ def __init__( @@ -172,15 +186,18 @@ def __init__( *, default_router_mode: Optional[str], include_frontend_only: bool, + env: Optional[ArgEnv] = None, ) -> None: self.default_router_mode = default_router_mode self.include_frontend_only = include_frontend_only + self.env = env def add_arguments(self, parser) -> None: - if "DYN_ADMISSION_CONTROL" in os.environ: - logger.warning(_ADMISSION_CONTROL_REMOVAL_WARNING) - if "DYN_ENFORCE_DISAGG" in os.environ: - logger.warning(_ENFORCE_DISAGG_DEPRECATION, "DYN_ENFORCE_DISAGG") + if self.env is None: + if "DYN_ADMISSION_CONTROL" in os.environ: + logger.warning(_ADMISSION_CONTROL_REMOVAL_WARNING) + if "DYN_ENFORCE_DISAGG" in os.environ: + logger.warning(_ENFORCE_DISAGG_DEPRECATION, "DYN_ENFORCE_DISAGG") g = parser.add_argument_group("Router Options") @@ -195,7 +212,11 @@ def add_arguments(self, parser) -> None: g.add_argument( "--admission-control", choices=("token-capacity", "none"), - action=_IgnoredAdmissionControlAction, + action=( + _IgnoredAdmissionControlAction + if self.env is None + else _IgnoredAction + ), default=argparse.SUPPRESS, help=argparse.SUPPRESS, ) @@ -210,7 +231,12 @@ def add_arguments(self, parser) -> None: "readiness are determined from registered worker types." ), arg_type=None, - action=_DeprecatedEnforceDisaggAction, + action=( + _DeprecatedEnforceDisaggAction + if self.env is None + else argparse.BooleanOptionalAction + ), + env=self.env, ) add_argument( g, @@ -225,6 +251,7 @@ def add_arguments(self, parser) -> None: ), arg_type=int, dest="min_initial_workers", + env=self.env, ) add_argument( @@ -249,6 +276,7 @@ def add_arguments(self, parser) -> None: "least-loaded", "device-aware-weighted", ], + env=self.env, ) add_argument( g, @@ -263,6 +291,7 @@ def add_arguments(self, parser) -> None: ), arg_type=int, dest="session_affinity_ttl_secs", + env=self.env, ) add_argument( g, @@ -276,6 +305,7 @@ def add_arguments(self, parser) -> None: ), choices=("hard", "soft"), dest="session_affinity_mode", + env=self.env, ) add_argument( g, @@ -288,6 +318,7 @@ def add_arguments(self, parser) -> None: "Unset by default; pass 'None' to disable it." ), arg_type=nullable_float, + env=self.env, ) add_argument( g, @@ -303,6 +334,7 @@ def add_arguments(self, parser) -> None: "with --active-prefill-tokens-threshold-frac." ), arg_type=nullable_int, + env=self.env, ) add_argument( g, @@ -316,6 +348,7 @@ def add_arguments(self, parser) -> None: "OR logic with --active-prefill-tokens-threshold." ), arg_type=nullable_float, + env=self.env, ) @@ -344,16 +377,19 @@ class WorkerRouterConfig(RouterConfigBase, KvRouterConfigBase): enforce_disagg: bool = False -def add_worker_router_arguments(parser: argparse.ArgumentParser) -> None: +def add_worker_router_arguments( + parser: argparse.ArgumentParser, *, env: Optional[ArgEnv] = None +) -> None: """Register the worker-side router flags. ``--router-mode`` defaults to ``None``: a worker that omits it advertises - nothing and inherits the frontend's configuration. + nothing and inherits the frontend's configuration. ``env`` is another + process's env (see ``ArgEnv``). """ - RouterArgGroup(default_router_mode=None, include_frontend_only=False).add_arguments( - parser - ) - KvRouterArgGroup().add_arguments(parser) + RouterArgGroup( + default_router_mode=None, include_frontend_only=False, env=env + ).add_arguments(parser) + KvRouterArgGroup(env=env).add_arguments(parser) def parse_worker_router_config( diff --git a/components/src/dynamo/common/configuration/utils.py b/components/src/dynamo/common/configuration/utils.py index e3e090b83fe0..8a441be1d47f 100644 --- a/components/src/dynamo/common/configuration/utils.py +++ b/components/src/dynamo/common/configuration/utils.py @@ -7,7 +7,8 @@ import os import re import warnings -from dataclasses import dataclass +from collections.abc import Mapping +from dataclasses import dataclass, field from typing import Any, Callable, Optional, TypeVar, Union T = TypeVar("T") @@ -30,6 +31,23 @@ def warn(self, name: str) -> None: warnings.warn(self.message(name), FutureWarning, stacklevel=3) +@dataclass +class ArgEnv: + """The env of another process, for registering that process's arguments. + + Argument groups read their env defaults from ``os.environ`` and warn + about deprecated or removed settings. A process that parses another + process's configuration (the planner reads its frontend's from the + DynamoGraphDeployment spec) passes an ``ArgEnv`` instead: env defaults + come from ``environ`` only, deprecation and removal notices are skipped + because they belong to the process that runs with that env, and + ``env_dests`` records the env var -> dest of every argument registered. + """ + + environ: Mapping[str, str] + env_dests: dict[str, str] = field(default_factory=dict) + + def parse_bool(value: str) -> bool: """Parse Dynamo's truthy and falsy configuration values.""" normalized = value.strip().lower() @@ -64,6 +82,8 @@ def env_or_default( env_var: str, default: T, value_type: Optional[Union[type, Callable[..., Any]]] = None, + *, + env: Optional[ArgEnv] = None, ) -> T: """ Get value from environment variable or return default. @@ -76,11 +96,12 @@ def env_or_default( value_type: If provided, use this type to convert the env value. If None, the type is taken from type(default). Use value_type when default is None but you still want the env value coerced (e.g. env_or_default("DYN_FOO", None, value_type=int)). + env: Read ``env.environ`` instead of ``os.environ``. Returns: Environment variable value (type-converted) or default """ - value = os.environ.get(env_var) + value = (os.environ if env is None else env.environ).get(env_var) if value is None: return default @@ -129,6 +150,7 @@ def add_argument( arg_type: Optional[Union[type, Callable[..., Any]]] = str, env_value_type: Optional[Union[type, Callable[..., Any]]] = None, deprecated: Optional[Deprecated] = None, + env: Optional[ArgEnv] = None, **kwargs: Any, ) -> None: """ @@ -147,6 +169,7 @@ def add_argument( arg_type: Type for the argument (default: str) env_value_type: Optional parser used only for the environment value deprecated: Warn on explicit CLI/env use and show the migration in help. + env: Another process's env (see ``ArgEnv``); skips ``deprecated``. """ arg_dest = _get_dest_name(flag_name, kwargs.get("dest")) value_type_for_env = env_value_type @@ -154,7 +177,12 @@ def add_argument( value_type_for_env = arg_type if isinstance(default, list) and (arg_type is None or arg_type is str): value_type_for_env = None - default_with_env = env_or_default(env_var, default, value_type=value_type_for_env) + default_with_env = env_or_default( + env_var, default, value_type=value_type_for_env, env=env + ) + if env is not None: + env.env_dests[env_var] = arg_dest + deprecated = None names = [flag_name] @@ -210,6 +238,7 @@ def add_negatable_bool_argument( obsolete_flag: Optional[str] = None, env_value_type: Optional[Callable[..., bool]] = None, deprecated: Optional[Deprecated] = None, + env: Optional[ArgEnv] = None, ) -> None: """ Add negatable boolean flag (--foo / --no-foo). @@ -223,6 +252,7 @@ def add_negatable_bool_argument( dest: Optional destination name for the parsed value obsolete_flag: Optional obsolete/legacy flag (for help msg only, must start with '--') env_value_type: Optional strict parser for the environment value + env: Another process's env (see ``ArgEnv``) """ add_argument( parser, @@ -236,6 +266,7 @@ def add_negatable_bool_argument( env_value_type=env_value_type, action=argparse.BooleanOptionalAction, deprecated=deprecated, + env=env, ) diff --git a/components/src/dynamo/common/tests/configuration/test_utils.py b/components/src/dynamo/common/tests/configuration/test_utils.py index 5c25dbf2da24..32ec739efca5 100644 --- a/components/src/dynamo/common/tests/configuration/test_utils.py +++ b/components/src/dynamo/common/tests/configuration/test_utils.py @@ -8,6 +8,7 @@ import pytest from dynamo.common.configuration.utils import ( + ArgEnv, Deprecated, add_argument, add_negatable_bool_argument, @@ -391,6 +392,35 @@ def test_deprecated_argument_env_and_cli_precedence(monkeypatch): assert parser.parse_args(["--old", "4"]).old == 4 +def test_arg_env_replaces_os_environ_and_skips_deprecation(monkeypatch, recwarn): + monkeypatch.setenv("TEST_OLD", "3") + env = ArgEnv(environ={"TEST_FLAG": "false", "TEST_OLD": "5"}) + parser = argparse.ArgumentParser() + add_argument( + parser, + flag_name="--old", + env_var="TEST_OLD", + default=1, + help="Old", + arg_type=int, + deprecated=Deprecated("--new", remove_in="v2.0"), + env=env, + ) + add_negatable_bool_argument( + parser, + flag_name="--flag", + env_var="TEST_FLAG", + default=True, + help="Flag", + env=env, + ) + assert vars(parser.parse_args([])) == {"old": 5, "flag": False} + assert parser.parse_args(["--old", "4"]).old == 4 + assert env.env_dests == {"TEST_OLD": "old", "TEST_FLAG": "flag"} + assert env_or_default("TEST_OLD", 0, env=ArgEnv(environ={})) == 0 + assert not recwarn + + def test_deprecated_hidden_argument_is_quiet_unless_used(monkeypatch, recwarn): monkeypatch.delenv("TEST_OLD", raising=False) parser = argparse.ArgumentParser() diff --git a/components/src/dynamo/frontend/frontend_args.py b/components/src/dynamo/frontend/frontend_args.py index 51ea3e032e76..d73e0ddf65ba 100644 --- a/components/src/dynamo/frontend/frontend_args.py +++ b/components/src/dynamo/frontend/frontend_args.py @@ -4,6 +4,7 @@ import argparse import os import pathlib +from collections.abc import Mapping from typing import Any, Dict, Optional from dynamo.common.config_dump import register_encoder @@ -22,6 +23,7 @@ RouterConfigBase, ) from dynamo.common.configuration.utils import ( + ArgEnv, add_argument, add_negatable_bool_argument, env_or_default, @@ -366,7 +368,17 @@ def _preprocess_for_encode_config(config: FrontendConfig) -> Dict[str, Any]: class FrontendArgGroup(ArgGroup): - """Frontend configuration parameters.""" + """Frontend configuration parameters. + + Args: + env: Another process's env (see ``ArgEnv``). Nothing then reads the + local filesystem: ``--model-path`` is not checked, and the + AISimulate perf-model arguments, whose values name files, are not + registered. + """ + + def __init__(self, *, env: Optional[ArgEnv] = None) -> None: + self.env = env def add_arguments(self, parser) -> None: parser.add_argument( @@ -381,7 +393,7 @@ def add_arguments(self, parser) -> None: "--interactive", dest="interactive", action=argparse.BooleanOptionalAction, - default=env_or_default("DYN_INTERACTIVE", False), + default=env_or_default("DYN_INTERACTIVE", False, env=self.env), help="Interactive text chat.\nenv var: DYN_INTERACTIVE", ) @@ -394,6 +406,7 @@ def add_arguments(self, parser) -> None: "Dynamo namespace for model discovery scoping. Use for exact namespace matching. " "If --namespace-prefix is also specified, prefix takes precedence." ), + env=self.env, ) add_argument( @@ -403,6 +416,7 @@ def add_arguments(self, parser) -> None: default=None, help="KV cache block size (u32).", arg_type=int, + env=self.env, ) add_argument( @@ -411,6 +425,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_HTTP_HOST", default="0.0.0.0", help="HTTP host for the engine (str).", + env=self.env, ) add_argument( g, @@ -419,6 +434,7 @@ def add_arguments(self, parser) -> None: default=8000, help="HTTP port for the engine (u16).", arg_type=int, + env=self.env, ) add_argument( g, @@ -426,6 +442,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_BATCH_GATEWAY_URL", default=None, help="Internal Batch Gateway URL for proxying /v1/files and /v1/batches.", + env=self.env, ) add_negatable_bool_argument( g, @@ -434,6 +451,7 @@ def add_arguments(self, parser) -> None: default=False, help="Serve this frontend's local KV indexers over the request plane.", dest="serve_indexer", + env=self.env, ) add_argument( g, @@ -442,6 +460,7 @@ def add_arguments(self, parser) -> None: default=None, help="TLS certificate path, PEM format.", arg_type=pathlib.Path, + env=self.env, ) add_argument( g, @@ -450,6 +469,7 @@ def add_arguments(self, parser) -> None: default=None, help="TLS certificate key path, PEM format.", arg_type=pathlib.Path, + env=self.env, ) add_argument( g, @@ -458,6 +478,7 @@ def add_arguments(self, parser) -> None: default=None, help="Client CA certificate path for mutual TLS, PEM format.", arg_type=pathlib.Path, + env=self.env, ) add_argument( @@ -466,6 +487,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_TCP_TLS_CERT_PATH", default=None, help="Path to PEM certificate for the TCP server.", + env=self.env, ) add_argument( @@ -474,6 +496,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_TCP_TLS_KEY_PATH", default=None, help="Path to PEM private key for the TCP server certificate.", + env=self.env, ) add_argument( @@ -482,6 +505,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_TCP_TLS_CA_CERT_PATH", default=None, help="Path to PEM CA certificate used to verify the TCP peer's certificate.", + env=self.env, ) add_argument( @@ -490,6 +514,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_TCP_TLS_CLIENT_CERT_PATH", default=None, help="Path to PEM client certificate presented to the TCP server for mTLS.", + env=self.env, ) add_argument( @@ -498,6 +523,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_TCP_TLS_CLIENT_KEY_PATH", default=None, help="Path to PEM private key for the TCP client certificate (mTLS).", + env=self.env, ) add_argument( @@ -507,6 +533,7 @@ def add_arguments(self, parser) -> None: default=None, help="Path to PEM CA certificate the TCP server uses to verify client " "certificates. When set, clients must present a trusted certificate (mTLS enforced).", + env=self.env, ) add_argument( @@ -515,6 +542,7 @@ def add_arguments(self, parser) -> None: env_var="NATS_TLS_CA_CERT_PATH", default=None, help="Path to PEM CA certificate for verifying the NATS server.", + env=self.env, ) add_negatable_bool_argument( @@ -523,6 +551,7 @@ def add_arguments(self, parser) -> None: env_var="NATS_TLS_INSECURE", default=False, help="Disable NATS TLS certificate verification. For local development only.", + env=self.env, ) add_argument( @@ -531,6 +560,7 @@ def add_arguments(self, parser) -> None: env_var="NATS_TLS_CLIENT_CERT_PATH", default=None, help="Path to PEM client certificate presented to the NATS server for mTLS.", + env=self.env, ) add_argument( @@ -539,16 +569,20 @@ def add_arguments(self, parser) -> None: env_var="NATS_TLS_CLIENT_KEY_PATH", default=None, help="Path to PEM private key for the NATS client certificate (mTLS).", + env=self.env, ) # Router options (shared with dynamo.router) RouterArgGroup( - default_router_mode="round-robin", include_frontend_only=True + default_router_mode="round-robin", + include_frontend_only=True, + env=self.env, ).add_arguments(parser) # KV router options (shared with dynamo.router) - KvRouterArgGroup().add_arguments(parser) - AisPerfArgGroup().add_arguments(parser) + KvRouterArgGroup(env=self.env).add_arguments(parser) + if self.env is None: + AisPerfArgGroup().add_arguments(parser) add_argument( g, @@ -560,6 +594,7 @@ def add_arguments(self, parser) -> None: "namespaces starting with this prefix (e.g., 'ns' matches 'ns', 'ns-abc123', " "'ns-def456'). Takes precedence over --namespace if both are specified." ), + env=self.env, ) add_argument( @@ -575,6 +610,7 @@ def add_arguments(self, parser) -> None: "session; unfinished audio is replayed to a compatible warm worker." ), arg_type=int, + env=self.env, ) add_argument( @@ -589,6 +625,7 @@ def add_arguments(self, parser) -> None: "Default: no limit." ), arg_type=int, + env=self.env, ) add_argument( @@ -598,6 +635,7 @@ def add_arguments(self, parser) -> None: default=None, help="Model name as a string (e.g., 'Llama-3.2-1B-Instruct')", arg_type=validate_model_name, + env=self.env, ) add_argument( g, @@ -605,7 +643,8 @@ def add_arguments(self, parser) -> None: env_var="DYN_MODEL_PATH", default=None, help="Path to model directory on disk (e.g., /tmp/model_cache/llama3.2_1B/)", - arg_type=validate_model_path, + arg_type=validate_model_path if self.env is None else str, + env=self.env, ) add_argument( g, @@ -616,6 +655,7 @@ def add_arguments(self, parser) -> None: "Prefix for Dynamo frontend metrics. If unset, uses DYN_METRICS_PREFIX env var " "or 'dynamo_frontend'." ), + env=self.env, ) add_negatable_bool_argument( g, @@ -623,6 +663,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_KSERVE_GRPC_SERVER", default=False, help="Start KServe gRPC server.", + env=self.env, ) add_argument( g, @@ -634,6 +675,7 @@ def add_arguments(self, parser) -> None: "Defaults to 8788." ), arg_type=int, + env=self.env, ) add_argument( @@ -642,6 +684,7 @@ def add_arguments(self, parser) -> None: env_var="DYN_DUMP_CONFIG_TO", default=None, help="Dump config to the specified file path.", + env=self.env, ) add_argument( @@ -657,6 +700,7 @@ def add_arguments(self, parser) -> None: "path. May be repeated. DYN_FRONTEND_ROUTE_EXTENSIONS accepts " "whitespace-separated values." ), + env=self.env, ) add_argument( @@ -670,6 +714,7 @@ def add_arguments(self, parser) -> None: "File uses root dir from env var DYN_FILE_KV or defaults to $TMPDIR/dynamo_store_kv." ), choices=["kubernetes", "etcd", "file", "mem"], + env=self.env, ) add_argument( g, @@ -681,6 +726,7 @@ def add_arguments(self, parser) -> None: "'tcp' is fastest [nats|tcp]" ), choices=["nats", "tcp"], + env=self.env, ) add_argument( g, @@ -689,6 +735,7 @@ def add_arguments(self, parser) -> None: default="tcp", help="Select the response transport. Frontend and workers must match.", choices=["tcp", "quic", "velo"], + env=self.env, ) add_argument( g, @@ -699,6 +746,7 @@ def add_arguments(self, parser) -> None: "defaults to 'zmq' for all discovery backends. Set to 'nats' to use a " "NATS-based event plane.", choices=["nats", "zmq"], + env=self.env, ) add_negatable_bool_argument( g, @@ -709,6 +757,7 @@ def add_arguments(self, parser) -> None: "[EXPERIMENTAL] Enable Anthropic Messages API endpoint (/v1/messages). " "This feature is experimental and may change." ), + env=self.env, ) add_negatable_bool_argument( g, @@ -719,6 +768,7 @@ def add_arguments(self, parser) -> None: "[EXPERIMENTAL] Enable typed decision APIs (/v1/decisions and " "/v1/systemone) for qualified aggregate SGLang models." ), + env=self.env, ) add_negatable_bool_argument( g, @@ -729,6 +779,7 @@ def add_arguments(self, parser) -> None: "Strip the Claude Code billing preamble (x-anthropic-billing-header) " "from the system prompt. Saves tokens and improves prompt caching." ), + env=self.env, ) add_negatable_bool_argument( g, @@ -741,6 +792,7 @@ def add_arguments(self, parser) -> None: "for each complete tool call before finish_reason arrives. " "Can be combined with --enable-streaming-reasoning-dispatch." ), + env=self.env, ) add_negatable_bool_argument( g, @@ -753,6 +805,7 @@ def add_arguments(self, parser) -> None: "with the complete reasoning block once thinking ends. " "Can be combined with --enable-streaming-tool-dispatch." ), + env=self.env, ) add_argument( g, @@ -763,6 +816,7 @@ def add_arguments(self, parser) -> None: "OpenAI-compatible response field used for emitted reasoning content." ), choices=["reasoning_content", "reasoning"], + env=self.env, ) # NOTE: This flag also exists in DynamoRuntimeArgGroup (runtime_args.py). # Both definitions are needed: runtime_args controls the Rust-native @@ -779,6 +833,7 @@ def add_arguments(self, parser) -> None: "tool_choice='none'. Prevents models from generating raw XML " "tool calls in the content field." ), + env=self.env, ) add_argument( g, @@ -793,6 +848,7 @@ def add_arguments(self, parser) -> None: "parsing, and reasoning parsing." ), choices=["dynamo", "vllm", "sglang"], + env=self.env, ) add_negatable_bool_argument( @@ -806,13 +862,16 @@ def add_arguments(self, parser) -> None: "Logs per-function timing, request concurrency, and hot-path section durations. " "Supported with '--dyn-chat-processor vllm' and '--dyn-chat-processor sglang'." ), + env=self.env, ) g.add_argument( "--dyn-preprocess-workers", dest="preprocess_workers", type=int, - default=env_or_default("DYN_PREPROCESS_WORKERS", None, value_type=int), + default=env_or_default( + "DYN_PREPROCESS_WORKERS", None, value_type=int, env=self.env + ), help=( "[EXPERIMENTAL] SGLang preprocessing worker processes per model. " "When unset, selects up to 2 workers per model from visible CPU capacity, " @@ -837,6 +896,7 @@ def add_arguments(self, parser) -> None: "Has no effect on TikToken models." ), choices=["default", "fastokens", "basetenkenizer"], + env=self.env, ) add_negatable_bool_argument( @@ -854,6 +914,7 @@ def add_arguments(self, parser) -> None: "continues running." ), env_value_type=parse_bool, + env=self.env, ) add_negatable_bool_argument( @@ -865,4 +926,28 @@ def add_arguments(self, parser) -> None: "Trust remote code when loading the tokenizer. Required for models " "that ship custom tokenizer code (e.g. Qwen, Falcon)." ), - ) + env=self.env, + ) + + +def build_frontend_parser( + env: Mapping[str, str], + parser: Optional[argparse.ArgumentParser] = None, +) -> tuple[argparse.ArgumentParser, dict[str, str]]: + """The argument parser of a frontend that runs with ``env``. + + For a process that parses another frontend's command line (the planner + reads one from the DynamoGraphDeployment spec): env defaults come from + ``env`` only, never ``os.environ``, and nothing logs, warns or reads the + local filesystem (see ``FrontendArgGroup``). Registers the arguments on + ``parser`` when given, else on a new ``ArgumentParser(add_help=False)``. + Returns the parser and the env var -> dest map of every argument with an + env var. Raises what the frontend raises at startup on invalid env + (``ValueError``, ``TypeError`` or ``argparse.ArgumentTypeError``). + """ + + arg_env = ArgEnv(environ=dict(env)) + if parser is None: + parser = argparse.ArgumentParser(add_help=False) + FrontendArgGroup(env=arg_env).add_arguments(parser) + return parser, arg_env.env_dests diff --git a/components/src/dynamo/frontend/tests/test_frontend_parser_builder.py b/components/src/dynamo/frontend/tests/test_frontend_parser_builder.py new file mode 100644 index 000000000000..b095423b9765 --- /dev/null +++ b/components/src/dynamo/frontend/tests/test_frontend_parser_builder.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``build_frontend_parser``: another frontend's parser, against an explicit env.""" + +import argparse +import logging +import os + +import pytest + +from dynamo.frontend.frontend_args import build_frontend_parser + +pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0] + + +def test_reads_only_the_given_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DYN_ROUTER_MODE", "kv") + monkeypatch.setenv("DYN_ROUTER_TEMPERATURE", "0.9") + before = dict(os.environ) + + parser, env_dests = build_frontend_parser({"DYN_ROUTER_MODE": "random"}) + namespace = parser.parse_args([]) + + assert namespace.router_mode == "random" + assert namespace.router_temperature == 0.0 + assert env_dests["DYN_ROUTER_MODE"] == "router_mode" + assert env_dests["DYN_ROUTER_KV_OVERLAP_SCORE_CREDIT"] == "overlap_score_credit" + assert dict(os.environ) == before + + +def test_parses_without_notices_or_local_files( + caplog: pytest.LogCaptureFixture, recwarn: pytest.WarningsRecorder +) -> None: + env = {"DYN_ADMISSION_CONTROL": "none", "DYN_ENFORCE_DISAGG": "1"} + parser = argparse.ArgumentParser(add_help=False) + with caplog.at_level(logging.WARNING): + assert build_frontend_parser(env, parser)[0] is parser + namespace, unknown = parser.parse_known_args( + [ + "--router-temperature", + "0.5", + "--admission-control", + "none", + "--enforce-disagg", + "--model-path", + "/not/a/local/dir", + "--ais-perf-config", + "/not/a/local/file.yaml", + ] + ) + + assert namespace.router_temperature == 0.5 + assert namespace.model_path == "/not/a/local/dir" + assert not hasattr(namespace, "admission_control") + # The AISimulate perf-model flags name local files and are not registered. + assert unknown == ["--ais-perf-config", "/not/a/local/file.yaml"] + assert not caplog.records + assert not recwarn diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index fcdcdedbc8d6..2b2065b25226 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -978,15 +978,6 @@ class DgdrExecutorConfig(BaseModel): allow_inf_nan=False, description="Period of the router actuator and reconciler rounds.", ) - trace_incomplete_threshold: float = Field( - default=0.05, - ge=0, - le=1, - description=( - "With a Prometheus endpoint, a window whose request count is below " - "the frontend's by more than this fraction fails (trace_incomplete)." - ), - ) # Live retune (``retune.executor: dgdr``) limits. A sweep must end this long @@ -1018,42 +1009,66 @@ class DgdrExecutorConfig(BaseModel): _DGDR_SCHEDULER_KNOBS = ("max_batched_tokens", "max_sequences") +def dgdr_rounds(points: int, parallelism: int) -> int: + """Evaluation rounds of a DGDR Run that scores ``points`` candidates. + + The AISimulate optimizer evaluates the first candidate alone, then + ``parallelism`` candidates at a time. + """ + + if points < 1: + raise ValueError(f"a DGDR shard has at least one point, got {points}") + if parallelism < 1: + raise ValueError(f"parallelism must be at least 1, got {parallelism}") + return 1 + math.ceil((points - 1) / parallelism) + + def dgdr_need_seconds( points: int, dgdr: DgdrExecutorConfig, *, needs_cut: bool ) -> float: """Wall-clock seconds one DGDR shard of ``points`` router points needs. ``cut_budget`` (when the window file must still be cut) + Run Job - start-up + one round per ``run.parallelism`` candidates, plus one: the - AISimulate optimizer evaluates its first candidate alone. ``parallelism`` - is the configured value; Job resources may lower it, so this is - optimistic. + start-up + ``dgdr_rounds(points, run.parallelism)`` rounds of + ``candidate_timeout``. ``parallelism`` is the configured value; Job + resources may lower it, so this is optimistic. """ - if points < 1: - raise ValueError(f"a DGDR shard has at least one point, got {points}") - rounds = 1 + math.ceil((points - 1) / dgdr.run.parallelism) + rounds = dgdr_rounds(points, dgdr.run.parallelism) cut = dgdr.cut_budget_seconds if needs_cut else 0.0 return cut + dgdr.job_startup_seconds + rounds * dgdr.run.candidate_timeout_seconds -def balanced_shard_parts(sizes: Sequence[int], max_shards: int) -> tuple[int, ...]: - """How many chunks to cut each grid dimension into for DGDR sharding. - - ``sizes`` are the value counts of the router grid dimensions. Cutting - dimension ``i`` into ``parts[i]`` contiguous chunks of at most - ``ceil(sizes[i] / parts[i])`` values gives ``prod(parts)`` Cartesian - shards, the largest with ``largest_shard_points(sizes, parts)`` points. - Returns the parts with ``prod(parts) <= max_shards`` that minimize the - largest shard, then the shard count, then lexicographically, so every - caller derives the same split: the default 3 x 9 x 4 grid with - ``max_shards=3`` gives ``(1, 3, 1)``, three shards of 36 points. +def balanced_shard_parts( + sizes: Sequence[int], max_shards: int, *, parallelism: int | None = None +) -> tuple[int, ...]: + """How many chunks to cut each router grid dimension into for DGDR sharding. + + ``sizes`` are the value counts of the grid dimensions. Cutting dimension + ``i`` into ``parts[i]`` contiguous chunks whose sizes differ by at most + one gives ``prod(parts)`` Cartesian shards, the largest with + ``largest_shard_points(sizes, parts)`` points. Of the parts with + ``prod(parts) <= max_shards``, returns the one that minimizes, in order: + + 1. whether the largest shard exceeds ``DGDR_MAX_SHARD_POINTS``, and when + every split does, its points (so the caller reports the smallest); + 2. the largest shard's wall time: its ``dgdr_rounds`` with + ``parallelism``, its points without; + 3. the shard count, the largest shard's points, then the parts. + + P13 calls it with ``parallelism=run.parallelism``; the live executor + must plan each window's shards with the same call, so that both see the + same largest shard. The default 3 x 9 x 4 grid with ``max_shards=3`` + gives ``(1, 3, 1)``, three shards of 36 points, with or without + parallelism. """ if max_shards < 1: raise ValueError(f"max_shards must be at least 1, got {max_shards}") if any(size < 1 for size in sizes): raise ValueError(f"every grid dimension needs a value, got sizes {sizes}") + if parallelism is not None and parallelism < 1: + raise ValueError(f"parallelism must be at least 1, got {parallelism}") def splits(index: int, budget: int): if index == len(sizes): @@ -1063,14 +1078,14 @@ def splits(index: int, budget: int): for rest in splits(index + 1, budget // part): yield (part, *rest) - return min( - splits(0, max_shards), - key=lambda parts: ( - largest_shard_points(sizes, parts), - math.prod(parts), - parts, - ), - ) + def rank(parts: tuple[int, ...]) -> tuple[Any, ...]: + largest = largest_shard_points(sizes, parts) + if largest > DGDR_MAX_SHARD_POINTS: + return (True, largest, math.prod(parts), parts) + cost = largest if parallelism is None else dgdr_rounds(largest, parallelism) + return (False, cost, math.prod(parts), largest, parts) + + return min(splits(0, max_shards), key=rank) def largest_shard_points(sizes: Sequence[int], parts: Sequence[int]) -> int: @@ -2528,7 +2543,9 @@ def _validate_live_retune_budget( # _validate_live_retune rejects both first. raise ValueError("retune.executor 'dgdr' requires dgdr and timeout_seconds") sizes = [len(values) for values in router_domain.values()] - parts = balanced_shard_parts(sizes, dgdr.run.max_shards) + parts = balanced_shard_parts( + sizes, dgdr.run.max_shards, parallelism=dgdr.run.parallelism + ) points = largest_shard_points(sizes, parts) if points > DGDR_MAX_SHARD_POINTS: raise ValueError( @@ -2540,7 +2557,7 @@ def _validate_live_retune_budget( ) need = dgdr_need_seconds(points, dgdr, needs_cut=True) if need > retune.timeout_seconds: - rounds = 1 + math.ceil((points - 1) / dgdr.run.parallelism) + rounds = dgdr_rounds(points, dgdr.run.parallelism) raise ValueError( f"the largest retune grid shard ({points} points) needs " f"{need:g}s: cut_budget_seconds {dgdr.cut_budget_seconds:g} + " diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py index cfa462a592b1..173a6bf159ba 100644 --- a/components/src/dynamo/planner/retune/live/contracts.py +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -59,7 +59,8 @@ # SweepResult.failure_reason values that mean "this process stopped working # on the window", not "the window failed": no decision is recorded and the -# controller rewinds so the window can be adopted (§4.7.5). +# controller rewinds so the window can be relaunched or adopted by the next +# leader. NON_DECISION_FAILURES = frozenset({"shutdown", "lost_leadership"}) GateReason = Literal["not_leader", "not_ready", "already_decided"] @@ -305,8 +306,8 @@ class ActuationTarget: ``observe``: no writes; latch initials and check parity only. ``include_not_ready`` also targets NotReady, non-terminating pods. ``learn_unknown`` lets the actuator reset a router whose initial was - never latched (revision > 0) once to learn it; that is active mode only - (§4.8). With False (advisory) such a router is never written: rollback + never latched (revision > 0) once to learn it; that is active mode only. + With False (advisory) such a router is never written: rollback leaves it alone and does not wait for it to converge. """ diff --git a/components/src/dynamo/planner/retune/live/frontend_config.py b/components/src/dynamo/planner/retune/live/frontend_config.py index 76485dbb7f10..fb7d74863f9c 100644 --- a/components/src/dynamo/planner/retune/live/frontend_config.py +++ b/components/src/dynamo/planner/retune/live/frontend_config.py @@ -8,20 +8,20 @@ argv and env say. The two agree only when the frontend leaves every unsearched routing field at its default (premise P4), so the planner reads the frontend's startup from the DynamoGraphDeployment spec, never from a -running pod (design §4.3): +running pod: - ``find_frontend`` picks the one standalone frontend component. - ``parse_frontend`` rebuilds the frontend's effective argv (the operator's ``command``/``args`` defaulting) and env (operator-injected, graph ``env`` and the main container's, with Kubernetes ``$(VAR)`` expansion), then - parses them with the frontend's own argument groups. Those groups read - ``os.environ`` while registering their arguments; here they run as copies - of their functions whose ``os`` sees only the frontend's env, so the - planner process's environment is neither read nor modified. A - shell-wrapped command, a ``--router-policy-config`` file, router env set - through ``valueFrom`` and an argument whose unreadable ``$(VAR)`` could - set a router flag make the frontend unparseable. ``envFrom`` sources are - assumed to set no router env; ``evaluate_premises`` warns about them. + parses them with the frontend's own parser (``build_frontend_parser``), + which reads env defaults from that env alone, so the planner process's + environment is neither read nor modified. A shell-wrapped command, a + ``--router-policy-config`` file, router env set through ``valueFrom`` and + an argument whose unreadable ``$(VAR)`` could set a router flag make the + frontend unparseable. ``envFrom`` sources are assumed to set no router + env; ``evaluate_premises`` warns about them, and P14 compares the + routers' policy parameters with the parsed startup. - ``evaluate_premises`` checks P4 and P7-P10 and warns on P6 and P7; P14 is the router actuator's. P4 also parses every worker component's router flags and env the same way: a worker that resolves a router mode @@ -38,11 +38,9 @@ import argparse import json import logging -import os import posixpath import re import shlex -import types from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from functools import lru_cache @@ -52,8 +50,18 @@ _KV_ROUTER_FIELDS, KvRouterConfigBase, ) -from dynamo.common.configuration.groups.router_args import RouterConfigBase +from dynamo.common.configuration.groups.router_args import ( + RouterConfigBase, + add_worker_router_arguments, +) +from dynamo.common.configuration.utils import ArgEnv +from dynamo.frontend.frontend_args import build_frontend_parser from dynamo.llm import KvRouterConfig +from dynamo.planner.retune.domains import ( + OVERLAP_SCORE_CREDIT, + expand_router_domain, + grid_points, +) from dynamo.planner.retune.live.contracts import FrontendStartup, PremiseReport if TYPE_CHECKING: @@ -79,7 +87,7 @@ ENV_PARENT_DGD_NAMESPACE = "DYN_PARENT_DGD_K8S_NAMESPACE" _OPERATOR_FIELD_REF_ENV = ("POD_NAME", "POD_NAMESPACE", "POD_UID") -# The trace recipe of design §4.4.1 (P10). +# The frontend's request trace recipe (P10). ENV_POD_NAME = "DYN_POD_NAME" ENV_TRACE = "DYN_REQUEST_TRACE" ENV_TRACE_SINKS = "DYN_REQUEST_TRACE_SINKS" @@ -170,11 +178,6 @@ _OPAQUE = "\x00opaque:" _OPAQUE_REPR = repr(_OPAQUE)[1:-1] -# Receives the frontend argument groups' own log lines (deprecation and -# removed-flag notices), which would otherwise repeat on every parse. -_SILENT_LOGGER = logging.getLogger(f"{__name__}.frontend_args") -_SILENT_LOGGER.disabled = True - class FrontendParseError(ValueError): """The frontend's router configuration cannot be derived from the spec.""" @@ -198,10 +201,8 @@ def canonical_json(value: Any) -> str: def find_frontend(dgd: Mapping[str, Any]) -> tuple[str, dict[str, Any]]: """Return the name and spec of the DGD's one standalone frontend component. - v1beta1 lists components in ``spec.components``; a v1alpha1 - ``spec.services`` map is converted to the same shape (``type``, - ``replicas``, ``podTemplate.spec`` with a ``main`` container), so callers - read one layout. Raises ``FrontendParseError`` ("P7: ...") unless there + Reads the v1beta1 ``spec.components`` list (the planner reads every DGD + as v1beta1). Raises ``FrontendParseError`` ("P7: ...") unless there is exactly one standalone frontend, naming an EPP or ``frontendSidecar`` layout when that is why. """ @@ -288,23 +289,17 @@ def parse_frontend( def evaluate_premises( - dgd: Mapping[str, Any], - *, - planner_config: PlannerConfig, - startup: FrontendStartup | None, - parse_error: str | None, - planner_replicas: int | None, + dgd: Mapping[str, Any], *, planner_config: PlannerConfig ) -> PremiseReport: """Evaluate the spec premises P4 and P7-P10, and the P6 and P7 warnings. - ``startup``/``parse_error`` are ``parse_frontend``'s result for ``dgd``; - what a ``FrontendStartup`` cannot carry (serde-skipped flags, session - affinity, unread router env, TLS, the trace recipe) is re-derived from - ``dgd`` (cached, so cheap). P4 also fails for a worker component that - advertises its own router configuration. ``planner_replicas``, when - given, replaces the planner component replicas read from the DGD (P9). - A malformed DGD never raises; every problem becomes a reason. Raises - ``ValueError`` only when ``planner_config.retune.dgdr`` is unset. + Parses the frontend as ``parse_frontend`` does (cached, so cheap) and + reports its startup; P4 also covers what a ``FrontendStartup`` cannot + carry (serde-skipped flags, session affinity, unread router env) and + fails for a worker component that advertises its own router + configuration. A malformed DGD never raises; every problem becomes a + reason. Raises ``ValueError`` only when ``planner_config.retune.dgdr`` is + unset. """ retune = planner_config.retune @@ -313,6 +308,7 @@ def evaluate_premises( dgdr = retune.dgdr mode = str(planner_config.mode) analysis = _analyze(dgd, mode) + startup = analysis.startup components = _components(dgd) p4: list[str] = [] others: list[str] = [] @@ -321,7 +317,7 @@ def evaluate_premises( # P4: a parsed KV router whose unsearched fields are the defaults and # whose startup passes the retune controller's constructor checks. if startup is None: - p4.append(parse_error or analysis.error or "P4: the frontend was not parsed") + p4.append(analysis.error or "P4: the frontend configuration cannot be parsed") else: p4.extend(analysis.p4_violations) p4.extend(_startup_errors(startup, retune, mode)) @@ -394,7 +390,7 @@ def evaluate_premises( f"P9: worker component {component.get('name')} has a " "scalingAdapter; its replicas could change during a window" ) - others.extend(_planner_replica_errors(components, planner_replicas, warnings)) + others.extend(_planner_replica_errors(components, warnings)) # P10: the frontend records request_end traces onto the claim. if analysis.component is not None and analysis.env is not None: @@ -483,7 +479,6 @@ def _analyze(dgd: Mapping[str, Any], planner_mode: str) -> _Analysis: "argv": list(argv), "values": dict(env.values), "opaque": sorted(env.opaque), - "env_from": [dict(source) for source in env.env_from], } ), worker_types(planner_mode), @@ -530,7 +525,6 @@ def _parse_cached(key: str, wts: tuple[str, ...]) -> _Parsed: inputs["argv"], inputs["values"], frozenset(inputs["opaque"]), - inputs["env_from"], wts, ) except FrontendParseError as error: @@ -549,7 +543,6 @@ def _parse( argv: Sequence[str], values: Mapping[str, str], opaque: frozenset[str], - env_from: Sequence[Mapping[str, Any]], wts: tuple[str, ...], ) -> _Parsed: args = _frontend_args(argv) @@ -570,10 +563,6 @@ def relevant(name: str) -> bool: "(or references such a value); the planner cannot read it, so set " "it with a literal value" ) - # envFrom sources (typically an HF token secret) are assumed not to set - # router env; evaluate_premises warns, and P14 compares the routers' - # policy parameters with the parsed startup. - del env_from # An unreadable $(VAR) becomes a marker the parser carries as a value: a # non-router flag keeps it unconverted, a router flag fails the parse, @@ -995,173 +984,46 @@ def _check_value(self, action, value): # type: ignore[no-untyped-def] return super()._check_value(action, value) -class _OsWithEnv: - """``os`` with ``environ`` replaced by the frontend's env.""" - - def __init__(self, environ: Mapping[str, str]) -> None: - self.environ = dict(environ) - - def __getattr__(self, name: str) -> Any: - return getattr(os, name) - - -class _QuietIgnore(argparse.Action): - """Accept and drop a removed flag, without the frontend's warning.""" - - def __call__(self, parser, namespace, values, option_string=None) -> None: # type: ignore[no-untyped-def] - return None - - -class _NoArguments: - """Replaces an argument group none of whose flags matter here.""" - - def __init__(self, *args: Any, **kwargs: Any) -> None: - pass - - def add_arguments(self, parser: Any) -> None: - return None - - -def _rebind( - function: Callable[..., Any], overrides: Mapping[str, Any] -) -> Callable[..., Any]: - """A copy of ``function`` whose module globals are overlaid with ``overrides``. - - The module is untouched, so other threads keep running the original. - """ - - rebound = types.FunctionType( - function.__code__, - {**function.__globals__, **overrides}, - function.__name__, - function.__defaults__, - function.__closure__, - ) - rebound.__kwdefaults__ = function.__kwdefaults__ - return rebound - - def _frontend_parser( env: Mapping[str, str], ) -> tuple[argparse.ArgumentParser, dict[str, str]]: """The frontend's argument parser with every env default read from ``env``. - Runs ``FrontendArgGroup.add_arguments`` and the router groups it adds as - copies whose ``os``, ``env_or_default``, ``add_argument`` and - ``add_negatable_bool_argument`` see only ``env``. Deprecation warnings, - the AISimulate perf-model flags (whose type reads files) and the - ``--model-path`` directory check are dropped. Returns the parser and the - env name -> dest map of everything it registered. + ``build_frontend_parser`` registers the frontend's arguments without + reading the planner's env or filesystem and without the frontend's + notices. Returns the parser and the env name -> dest map of everything + it registered. """ - from dynamo.common.configuration.groups import kv_router_args, router_args - from dynamo.frontend import frontend_args - - overrides, bound, registered = _env_bound(env) - add_frontend_arguments = _rebind( - frontend_args.FrontendArgGroup.add_arguments, - { - **overrides, - "RouterArgGroup": bound(router_args.RouterArgGroup), - "KvRouterArgGroup": bound(kv_router_args.KvRouterArgGroup), - "AisPerfArgGroup": _NoArguments, - "validate_model_path": str, - }, - ) - parser = _ArgsParser() try: - add_frontend_arguments(frontend_args.FrontendArgGroup(), parser) + return build_frontend_parser(env, _ArgsParser()) except (ValueError, TypeError, argparse.ArgumentTypeError) as error: # The frontend itself fails to start on such env. raise FrontendParseError(f"P4: invalid frontend env: {error}") from None - # The removed flags only log when used; keep their arity, drop the noise. - for action in parser._actions: - if isinstance(action, router_args._DeprecatedEnforceDisaggAction): - action.__class__ = argparse.BooleanOptionalAction # type: ignore[assignment] - elif isinstance(action, router_args._IgnoredAdmissionControlAction): - action.__class__ = _QuietIgnore # type: ignore[assignment] - return parser, registered - def _worker_router_parser( env: Mapping[str, str], ) -> tuple[argparse.ArgumentParser, dict[str, str]]: """A worker's router-advertisement parser with env defaults from ``env``. - Runs ``router_args.add_worker_router_arguments``, which every backend - (vLLM, SGLang, TensorRT-LLM, mocker) uses, the way ``_frontend_parser`` - runs the frontend's groups. Only ``router_mode`` is relevant: it decides - whether the worker advertises a router configuration at all. + Registers ``router_args.add_worker_router_arguments``, which every + backend (vLLM, SGLang, TensorRT-LLM, mocker) uses, against ``env``. Only + ``router_mode`` is relevant: it decides whether the worker advertises a + router configuration at all. """ - from dynamo.common.configuration.groups import kv_router_args, router_args - - overrides, bound, registered = _env_bound(env) - add_worker_arguments = _rebind( - router_args.add_worker_router_arguments, - { - "RouterArgGroup": bound(router_args.RouterArgGroup), - "KvRouterArgGroup": bound(kv_router_args.KvRouterArgGroup), - }, - ) + arg_env = ArgEnv(environ=dict(env)) parser = _ArgsParser( relevant=frozenset({_ROUTER_MODE_DEST}), describe=_worker_parse_error_message, ) try: - add_worker_arguments(parser) + add_worker_router_arguments(parser, env=arg_env) except (ValueError, TypeError, argparse.ArgumentTypeError) as error: # The worker itself fails to start on such env. raise FrontendParseError(f"has invalid router env ({error})") from None - return parser, registered - - -def _env_bound( - env: Mapping[str, str], -) -> tuple[dict[str, Any], Callable[[type], type], dict[str, str]]: - """Globals that make an argument group read only ``env``. - - Returns the overrides for ``_rebind``, a function that subclasses an - ``ArgGroup`` with its ``add_arguments`` rebound to them, and the env - name -> dest map the bound groups fill while registering. - """ - - from dynamo.common.configuration import utils - - fake_os = _OsWithEnv(env) - registered: dict[str, str] = {} - env_or_default = _rebind(utils.env_or_default, {"os": fake_os}) - base_add = _rebind( - utils.add_argument, {"os": fake_os, "env_or_default": env_or_default} - ) - - def add_argument(parser: Any, **kwargs: Any) -> None: - kwargs.pop("deprecated", None) - flag = str(kwargs["flag_name"]) - registered[str(kwargs["env_var"])] = str( - kwargs.get("dest") or flag.lstrip("-").replace("-", "_") - ) - base_add(parser, **kwargs) - - overrides = { - "os": fake_os, - "env_or_default": env_or_default, - "add_argument": add_argument, - "add_negatable_bool_argument": _rebind( - utils.add_negatable_bool_argument, {"add_argument": add_argument} - ), - "logger": _SILENT_LOGGER, - } - - def bound(group: Any) -> type: - return type( - group.__name__, - (group,), - {"add_arguments": _rebind(group.add_arguments, overrides)}, - ) - - return overrides, bound, registered + return parser, arg_env.env_dests # --------------------------------------------------------- premise checks @@ -1347,8 +1209,6 @@ def _worker_argv(component: Mapping[str, Any], env: _Env) -> tuple[list[str], bo def _router_domain_keys(retune: RetuneConfig) -> list[str]: if retune.router is None: return [] - from dynamo.planner.retune.domains import expand_router_domain - return list(expand_router_domain(retune)) @@ -1363,12 +1223,6 @@ def _controller_errors( if retune.router is None: return [] - from dynamo.planner.retune.domains import ( - OVERLAP_SCORE_CREDIT, - expand_router_domain, - grid_points, - ) - startup_json = canonical_json(dict(router_config)) try: startup_maps = _policy_maps(startup_json, wts) @@ -1443,16 +1297,8 @@ def _startup_config_mismatch( def _planner_replica_errors( - components: Sequence[Mapping[str, Any]], - planner_replicas: Optional[int], - warnings: list[str], + components: Sequence[Mapping[str, Any]], warnings: list[str] ) -> list[str]: - if planner_replicas is not None: - if planner_replicas != 1: - return [ - f"P9: the planner has {planner_replicas} replicas; live retune needs 1" - ] - return [] planners = [c for c in components if _type(c) == PLANNER_TYPE] if not planners: warnings.append( @@ -1474,7 +1320,7 @@ def _trace_recipe_errors( segments_dir: str, warnings: list[str], ) -> list[str]: - """P10: the frontend writes request_end segments onto the claim (§4.4.1).""" + """P10: the frontend writes request_end segments onto the claim.""" assert analysis.spec is not None and analysis.env is not None entries = analysis.env.entries @@ -1590,11 +1436,23 @@ def _join_dir(mount: str, *parts: str) -> str: return "/".join([mount.rstrip("/"), *(part for part in parts if part)]) + "/" +@lru_cache(maxsize=8) def _model_warning(model: str) -> Optional[str]: + """P6 for ``model``; cached, so a broken install logs once per model.""" + try: + # Optional dependency of a warn-only premise: without it, P6 is not + # checked. from aisimulate_core.sdk.common import DefaultHFModels - except Exception: # noqa: BLE001 - optional dependency of a warn-only premise + except ImportError: return None + except Exception as error: # noqa: BLE001 - evaluate_premises never raises + logger.exception("live retune could not load AISimulate's default HF models") + return ( + f"P6: cannot check retune.dgdr.model {model!r} against AISimulate's " + f"default HF models ({type(error).__name__}: {error}); each DGDR Run " + "must be able to download it" + ) if model in DefaultHFModels: return None return ( @@ -1607,65 +1465,14 @@ def _model_warning(model: str) -> Optional[str]: def _components(dgd: Mapping[str, Any]) -> list[Mapping[str, Any]]: - spec = dgd.get("spec") or {} - components = spec.get("components") + components = (dgd.get("spec") or {}).get("components") if isinstance(components, list): return [c for c in components if isinstance(c, Mapping)] - services = spec.get("services") - if isinstance(services, Mapping): - return [ - _from_v1alpha1(str(name), service) - for name, service in services.items() - if isinstance(service, Mapping) - ] return [] -def _from_v1alpha1(name: str, service: Mapping[str, Any]) -> dict[str, Any]: - """A v1alpha1 service in the v1beta1 component shape this module reads.""" - - extra = service.get("extraPodSpec") or {} - main = dict(extra.get("mainContainer") or {}) - main["name"] = MAIN_CONTAINER - env: dict[str, Any] = {} - for entry in [*(service.get("envs") or []), *(main.get("env") or [])]: - if isinstance(entry, Mapping) and entry.get("name"): - env[str(entry["name"])] = entry - main["env"] = list(env.values()) - env_from = list(main.get("envFrom") or []) - if service.get("envFromSecret"): - env_from.append({"secretRef": {"name": service["envFromSecret"]}}) - main["envFrom"] = env_from - # v1alpha1 volumeMounts name a PVC and mount it at mountPoint. - mounts = list(main.get("volumeMounts") or []) - volumes = list(extra.get("volumes") or []) - for mount in service.get("volumeMounts") or []: - if isinstance(mount, Mapping) and mount.get("name"): - claim = str(mount["name"]) - mounts.append({"name": claim, "mountPath": mount.get("mountPoint")}) - volumes.append( - {"name": claim, "persistentVolumeClaim": {"claimName": claim}} - ) - main["volumeMounts"] = mounts - component: dict[str, Any] = { - "name": name, - "type": service.get("componentType") or "", - "podTemplate": {"spec": {"containers": [main], "volumes": volumes}}, - } - for key in ("replicas", "roles", "modelRef"): - if service.get(key) is not None: - component[key] = service[key] - if service.get("frontendSidecar"): - component["frontendSidecar"] = "sidecar-frontend" - # v1alpha1 needs enabled: true; v1beta1 opts in with any object. - if (service.get("scalingAdapter") or {}).get("enabled"): - component["scalingAdapter"] = {} - return component - - def _graph_env(dgd: Mapping[str, Any]) -> list[Any]: - spec = dgd.get("spec") or {} - return list(spec.get("env") or spec.get("envs") or []) + return list((dgd.get("spec") or {}).get("env") or []) def _pod_spec(spec: Mapping[str, Any]) -> Mapping[str, Any]: diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py deleted file mode 100644 index 0429ec30f493..000000000000 --- a/components/src/dynamo/planner/tests/live_retune_fakes.py +++ /dev/null @@ -1,1098 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""In-memory fakes of the Kubernetes side of live retune. - -``FakeRetuneKube`` implements ``contracts.RetuneKube`` over plain dicts with -the API-server behaviour the live modules depend on: ``resourceVersion`` -compare-and-swap on Lease replace, ``AlreadyExists`` on create, Strict field -validation and silent pruning on Request create, label selectors, cascading -Request deletion, and injectable faults (including writes that time out but -land). ``FakeDgdrOperator`` drives the Requests in a ``FakeRetuneKube`` -through the DGDR v1beta2 lifecycle (Accepted -> Run -> Completed with -Candidates) the way the operator at #17483 does, lowering each Request into -``Run.status.aisimulate.input`` with the same merge-patch rules. - -This is a plain module, not a conftest: tests import what they need. -""" - -from __future__ import annotations - -import copy -import functools -import itertools -import json -import re -import threading -import time -import uuid -from collections import deque -from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass, field, replace -from typing import Any, Literal, Optional - -from dynamo.planner.retune.live.contracts import ( - DGDR_ROUTER_KNOBS, - POD_LABEL_COMPONENT_TYPE, - POD_LABEL_DGD_NAME, - RUN_UID_LABEL, - KubeAlreadyExists, - KubeConflict, - KubeError, - KubeInvalid, - KubeNotFound, - PodInfo, -) - -# Operator constants (deploy/operator/internal/controller at #17483). -REQUEST_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" -CANDIDATE_ID_LABEL = "nvidia.com/dgdr-candidate-id" -TRACE_MOUNT_PATH = "/var/run/aisimulate/trace" -DEFAULT_RUNTIME_IMAGE = "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.8.0" -DEFAULT_DYNAMO_VERSION = "1.8.0" -ROUTER_ADAPTER = "dynamo.router" - -_METRIC_TARGETS = { - "Throughput": "throughput", - "ThroughputPerGPU": "throughput_per_gpu", - "ThroughputPerUser": "throughput_per_user", - "TTFT": "ttft", - "E2ELatency": "e2e_latency", - "Goodput": "goodput", - "GoodputPerGPU": "goodput_per_gpu", -} - -KubeMethod = Literal[ - "get_dgd", - "list_pods", - "create_request", - "get_request", - "list_requests", - "delete_request", - "get_run", - "list_candidates", - "get_lease", - "create_lease", - "replace_lease", - "delete_lease", -] - - -# ------------------------------------------------------------------ helpers - - -def merge_patch(target: Any, patch: Any) -> Any: - """RFC 7386 JSON merge patch; returns a new value, never mutates.""" - - if not isinstance(patch, Mapping): - return copy.deepcopy(patch) - result = dict(target) if isinstance(target, Mapping) else {} - for key, value in patch.items(): - if value is None: - result.pop(key, None) - else: - result[key] = merge_patch(result.get(key), value) - return result - - -def nested_get(obj: Any, *path: str, default: Any = None) -> Any: - for key in path: - if not isinstance(obj, Mapping) or key not in obj: - return default - obj = obj[key] - return obj - - -def _nested_set(obj: dict, path: tuple[str, ...], value: Any) -> None: - for key in path[:-1]: - child = obj.get(key) - if not isinstance(child, dict): - child = {} - obj[key] = child - obj = child - obj[path[-1]] = value - - -def _nested_pop(obj: Any, path: tuple[str, ...]) -> bool: - for key in path[:-1]: - if not isinstance(obj, dict) or key not in obj: - return False - obj = obj[key] - if isinstance(obj, dict) and path[-1] in obj: - del obj[path[-1]] - return True - return False - - -def _split(path: str) -> tuple[str, ...]: - return tuple(path.split(".")) - - -def rfc3339(epoch_s: float) -> str: - return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch_s)) - - -_SELECTOR_TERM = re.compile( - r"^\s*(!?)\s*([A-Za-z0-9._/-]+)\s*(?:(==|=|!=)\s*(\S*))?\s*$" -) - - -def match_labels(labels: Mapping[str, str] | None, selector: str) -> bool: - """Equality-based label selector: ``k=v``, ``k==v``, ``k!=v``, ``k``, ``!k``.""" - - labels = labels or {} - if not selector.strip(): - return True - for term in selector.split(","): - found = _SELECTOR_TERM.match(term) - if found is None: - raise ValueError(f"unsupported label selector term {term!r}") - negate, key, op, value = found.groups() - if op is None: - present = key in labels - if present == bool(negate): - return False - elif negate: - raise ValueError(f"unsupported label selector term {term!r}") - elif op in ("=", "=="): - if labels.get(key) != value: - return False - elif labels.get(key) == value: # != - return False - return True - - -def lower_request_input( - request: Mapping[str, Any], - *, - run_name: str, - runtime_image: str = DEFAULT_RUNTIME_IMAGE, - dynamo_version: str = DEFAULT_DYNAMO_VERSION, - gpus_per_node: int = 8, -) -> dict[str, Any]: - """``Run.status.aisimulate.input`` for a Request, as the operator lowers it. - - Mirrors ``aisimulate.LowerWithUnderlay`` and ``dgdrRunInput`` at #17483: - the public role underlay, the ``dgdr_run`` static underlay, the typed - underlay, the derived defaults, ``inputOverrides`` as the final merge - patch, then the operator-owned ``dgdr_run`` fields and the trace source. - """ - - spec = request.get("spec") or {} - max_candidates = int(nested_get(spec, "recommendation", "maxCandidates", default=1)) - underlay: dict[str, Any] = { - "engine": {"workers": {"aggregated": {}, "prefill": {}, "decode": {}}} - } - underlay = merge_patch( - underlay, {"dgdr_run": {"name": run_name, "max_candidates": max_candidates}} - ) - typed: dict[str, Any] = {} - model = nested_get(spec, "model", "name") - if model is not None: - _nested_set(typed, ("engine", "model"), model) - if spec.get("backends"): - _nested_set(typed, ("engine", "backend", "choices"), list(spec["backends"])) - sku = nested_get(spec, "hardware", "gpu", "sku") - if sku: - _nested_set(typed, ("engine", "hardware"), sku) - budget = nested_get(spec, "hardware", "gpu", "budget") - if budget is not None: - _nested_set( - typed, ("optimization", "constraints", "max_candidate_gpus"), int(budget) - ) - trace = nested_get(spec, "workload", "trace") - trace_source = None - if trace is not None: - path = nested_get(trace, "source", "pvc", "path", default="") - mounted = TRACE_MOUNT_PATH + ( - "/trace.jsonl.gz" if path.endswith(".gz") else "/trace.jsonl" - ) - trace_source = { - "type": "trace", - "format": trace.get("format"), - "paths": [mounted], - } - _nested_set(typed, ("traffic", "source"), dict(trace_source)) - _nested_set(typed, ("traffic", "load", "type"), "trace_timestamps") - speedup = nested_get(trace, "replay", "arrivalSpeedupRatio") - if speedup is not None: - _nested_set(typed, ("traffic", "load", "speedup"), speedup) - objective = spec.get("objective") or {} - if objective.get("metric") is not None: - _nested_set( - typed, ("optimization", "target"), _METRIC_TARGETS[objective["metric"]] - ) - sla = objective.get("sla") or {} - for api_key, input_key in ( - ("ttftMs", "ttft_ms"), - ("itlMs", "itl_ms"), - ("e2eMs", "e2e_ms"), - ): - if sla.get(api_key) is not None: - _nested_set(typed, ("evaluation", "sla", input_key), sla[api_key]) - underlay = merge_patch(underlay, typed) - if "traffic" in underlay: - source_type = nested_get(underlay, "traffic", "source", "type") - if source_type != "trace" and nested_get(underlay, "traffic", "stop") is None: - _nested_set(underlay, ("traffic", "stop", "requests"), 1000.0) - if nested_get(underlay, "optimizer", "max_trials") is None: - _nested_set(underlay, ("optimizer", "max_trials"), 320.0) - if nested_get(underlay, "evaluation", "sla") and ( - nested_get(underlay, "optimization", "strict_sla") is None - ): - _nested_set(underlay, ("optimization", "strict_sla"), True) - overrides = nested_get(spec, "aisimulate", "inputOverrides", default={}) or {} - effective = merge_patch(underlay, overrides) - reserved = { - "name": run_name, - "max_candidates": max_candidates, - "snapshot_dir": "/var/run/aisimulate/output", - "namespace": nested_get(request, "metadata", "namespace"), - "runtime_image": runtime_image, - "runtime_version_override": dynamo_version, - "derive_runtime_images": True, - "num_gpus_per_node": gpus_per_node, - } - for key, value in reserved.items(): - _nested_set(effective, ("dgdr_run", key), value) - if trace_source is not None: - traffic = effective.setdefault("traffic", {}) - traffic.setdefault("source", {}).update(trace_source) - # Round-trip through JSON like the API server does. - return json.loads(json.dumps(effective)) - - -def router_points(effective_input: Mapping[str, Any]) -> list[dict[str, float]]: - """The router points (public knob names) an input's router section spans.""" - - router = effective_input.get("router") or {} - axes: list[tuple[str, list[float]]] = [] - for public in DGDR_ROUTER_KNOBS: - value = router.get(public) - if isinstance(value, Mapping): - axes.append((public, [float(v) for v in value["choices"]])) - elif value is not None: - axes.append((public, [float(value)])) - return [ - dict(zip([name for name, _ in axes], values)) - for values in itertools.product(*(values for _, values in axes)) - ] - - -def engine_parameters(effective_input: Mapping[str, Any]) -> dict[str, Any]: - """The non-router parameters every Candidate of a pinned engine shares.""" - - engine = effective_input.get("engine") or {} - workers = engine.get("workers") or {} - params: dict[str, Any] = { - "backend": engine.get("backend"), - "deployment_mode": "disagg" if engine.get("mode") == "disaggregated" else "agg", - "model": engine.get("model"), - "hardware": engine.get("hardware"), - } - for role, worker in sorted(workers.items()): - if isinstance(worker, Mapping): - preset = nested_get(worker, "parallelism", "preset", default=[{}]) - params[f"{role}_parallelism"] = dict(preset[0]) if preset else {} - return json.loads(json.dumps(params)) - - -def candidate_parameters( - effective_input: Mapping[str, Any], point_public: Mapping[str, float] -) -> dict[str, Any]: - """``Candidate.spec.parameters`` of one evaluated router point.""" - - router = effective_input.get("router") or {} - return { - **engine_parameters(effective_input), - "adapters": { - ROUTER_ADAPTER: { - "policy": router.get("policy", "kv_router"), - "prefill_load_model": dict( - router.get("prefill_load_model") or {"type": "none"} - ), - **{name: float(value) for name, value in point_public.items()}, - } - }, - } - - -def default_metrics(point_public: Mapping[str, float]) -> dict[str, float]: - """Deterministic replay summary metrics of a router point (for tests).""" - - credit = point_public.get("overlap_score_credit", 1.0) - scale = point_public.get("prefill_load_scale", 1.0) - temperature = point_public.get("temperature", 0.0) - goodput = 1000.0 + 100.0 * credit - 3.0 * abs(scale - 2.0) - 10.0 * temperature - return { - "goodput_output_throughput_tok_s": goodput, - "output_throughput_tok_s": goodput * 1.1, - "gpu_hours": 2.0, - "duration_ms": 3_600_000.0, - "mean_ttft_ms": 100.0 + scale, - "num_ttft_samples": 1000.0, - "mean_e2e_latency_ms": 1000.0 + scale, - "num_e2e_latency_samples": 1000.0, - "mean_output_token_throughput_per_user": 50.0 + credit, - } - - -# ------------------------------------------------------------------- faults - - -@dataclass -class Fault: - """One injected failure of a ``FakeRetuneKube`` method. - - ``landed=True`` performs the call and then raises (a write that timed - out but reached the API server). ``match`` limits the fault to calls - whose first positional argument (a name, selector or body) satisfies it. - """ - - error: Exception - times: int = 1 - landed: bool = False - match: Optional[Callable[[Any], bool]] = None - - -@dataclass -class KubeCall: - method: str - target: Any - kwargs: dict[str, Any] = field(default_factory=dict) - - -# --------------------------------------------------------------------- kube - - -class FakeRetuneKube: - """In-memory ``contracts.RetuneKube`` (thread-safe). - - Stores DGDs, pods (``PodInfo``), Requests, Runs, Candidates and Leases - as plain dicts. Objects come back as deep copies, so callers can never - alias the store. ``now`` stamps ``creationTimestamp``. - - Request create honours ``field_validation='Strict'``: a path listed in - ``unknown_fields`` (dotted, e.g. ``spec.workload.trace``) is rejected - with ``KubeInvalid``; a path in ``pruned_fields`` is dropped silently - (an API server that ignores the validation directive). ``reject_create`` - rejects every create with ``KubeInvalid``. ``on_request_created`` hooks - run after each stored (non dry-run) create. - """ - - def __init__( - self, - namespace: str = "test-ns", - *, - now: Callable[[], float] = time.time, - ) -> None: - self.namespace = namespace - self.now = now - self.dgds: dict[str, dict] = {} - self.pods: dict[str, PodInfo] = {} - self.requests: dict[str, dict] = {} - self.runs: dict[str, dict] = {} - # Each Run's lowered input, recorded in its status by ``create_job``. - self.run_inputs: dict[str, dict] = {} - self.candidates: dict[str, dict] = {} - self.leases: dict[str, dict] = {} - self.calls: list[KubeCall] = [] - self.deleted_requests: list[tuple[str, str]] = [] - self.unknown_fields: set[str] = set() - self.pruned_fields: set[str] = set() - self.reject_create: Optional[str] = None - self.on_request_created: list[Callable[[FakeRetuneKube, dict], None]] = [] - self.cascade_deletes = True - self._faults: dict[str, deque[Fault]] = {} - self._rv = itertools.count(1) - self._lock = threading.RLock() - - # ------------------------------------------------------------ faults - - def fail( - self, - method: KubeMethod, - error: Exception | None = None, - *, - times: int = 1, - landed: bool = False, - match: Optional[Callable[[Any], bool]] = None, - ) -> None: - """Make the next ``times`` matching calls of ``method`` raise ``error``.""" - - with self._lock: - self._faults.setdefault(method, deque()).append( - Fault( - error=error or KubeError(f"injected {method} failure"), - times=times, - landed=landed, - match=match, - ) - ) - - def clear_faults(self) -> None: - with self._lock: - self._faults.clear() - - def calls_of(self, method: str) -> list[KubeCall]: - with self._lock: - return [call for call in self.calls if call.method == method] - - def _call( - self, - method: str, - target: Any, - kwargs: dict[str, Any], - operation: Callable[[], Any], - ) -> Any: - with self._lock: - self.calls.append(KubeCall(method, copy.deepcopy(target), dict(kwargs))) - fault = self._take_fault(method, target) - if fault is not None and not fault.landed: - raise fault.error - result = operation() - if fault is not None: - raise fault.error - return result - - def _take_fault(self, method: str, target: Any) -> Optional[Fault]: - queue = self._faults.get(method) - if not queue: - return None - for fault in list(queue): - if fault.match is None or fault.match(target): - fault.times -= 1 - if fault.times <= 0: - queue.remove(fault) - return fault - return None - - def _stamp(self, obj: dict, *, kind: str, api_version: str) -> dict: - metadata = obj.setdefault("metadata", {}) - obj.setdefault("apiVersion", api_version) - obj.setdefault("kind", kind) - metadata.setdefault("namespace", self.namespace) - metadata.setdefault("uid", str(uuid.uuid4())) - metadata.setdefault("creationTimestamp", rfc3339(self.now())) - metadata.setdefault("generation", 1) - metadata["resourceVersion"] = str(next(self._rv)) - return obj - - def bump(self, obj: dict) -> None: - """Give a stored object a new ``resourceVersion``.""" - - obj["metadata"]["resourceVersion"] = str(next(self._rv)) - - # --------------------------------------------------------------- DGD - - def put_dgd(self, dgd: dict) -> dict: - with self._lock: - stored = copy.deepcopy(dgd) - self._stamp( - stored, kind="DynamoGraphDeployment", api_version="nvidia.com/v1beta1" - ) - self.dgds[stored["metadata"]["name"]] = stored - return copy.deepcopy(stored) - - def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: - def operation() -> dict: - if name not in self.dgds: - raise KubeNotFound(f"DynamoGraphDeployment {name!r} not found") - return copy.deepcopy(self.dgds[name]) - - return self._call("get_dgd", name, {"timeout_s": timeout_s}, operation) - - # -------------------------------------------------------------- pods - - def add_pod( - self, - name: str, - *, - dgd: str | None = None, - component_type: str = "frontend", - ready: bool = True, - terminating: bool = False, - ip: str | None = None, - http_port: int | None = 8000, - labels: Mapping[str, str] | None = None, - ) -> PodInfo: - all_labels = dict(labels or {}) - if dgd is not None: - all_labels.setdefault(POD_LABEL_DGD_NAME, dgd) - if component_type: - all_labels.setdefault(POD_LABEL_COMPONENT_TYPE, component_type) - pod = PodInfo( - name=name, - uid=str(uuid.uuid4()), - ip=ip if ip is not None else f"10.0.0.{len(self.pods) + 1}", - ready=ready, - terminating=terminating, - labels=all_labels, - http_port=http_port, - ) - with self._lock: - self.pods[name] = pod - return pod - - def update_pod(self, name: str, **changes: Any) -> PodInfo: - with self._lock: - pod = replace(self.pods[name], **changes) - self.pods[name] = pod - return pod - - def remove_pod(self, name: str) -> None: - with self._lock: - self.pods.pop(name, None) - - def list_pods( - self, label_selector: str, *, timeout_s: float = 10.0 - ) -> list[PodInfo]: - def operation() -> list[PodInfo]: - return [ - pod - for pod in self.pods.values() - if match_labels(pod.labels, label_selector) - ] - - return self._call( - "list_pods", label_selector, {"timeout_s": timeout_s}, operation - ) - - # ----------------------------------------------------------- Requests - - def create_request( - self, - body: dict, - *, - dry_run: bool = False, - timeout_s: float = 10.0, - ) -> dict: - def operation() -> dict: - if self.reject_create is not None: - raise KubeInvalid(self.reject_create) - stored = copy.deepcopy(body) - for path in sorted(self.unknown_fields): - if nested_get(stored, *_split(path)) is not None: - raise KubeInvalid(f'strict decoding error: unknown field "{path}"') - for path in self.pruned_fields: - _nested_pop(stored, _split(path)) - name = nested_get(stored, "metadata", "name") - if not name: - raise KubeInvalid("metadata.name is required") - if name in self.requests: - raise KubeAlreadyExists( - f"dynamographdeploymentrequests {name!r} already exists" - ) - self._stamp( - stored, - kind="DynamoGraphDeploymentRequest", - api_version="nvidia.com/v1beta2", - ) - if dry_run: - return stored - self.requests[name] = stored - for hook in list(self.on_request_created): - hook(self, copy.deepcopy(stored)) - return copy.deepcopy(stored) - - return self._call( - "create_request", - body, - {"dry_run": dry_run, "timeout_s": timeout_s}, - operation, - ) - - def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - def operation() -> Optional[dict]: - found = self.requests.get(name) - return None if found is None else copy.deepcopy(found) - - return self._call("get_request", name, {"timeout_s": timeout_s}, operation) - - def list_requests( - self, label_selector: str, *, timeout_s: float = 10.0 - ) -> list[dict]: - def operation() -> list[dict]: - return [ - copy.deepcopy(obj) - for obj in self.requests.values() - if match_labels(obj["metadata"].get("labels"), label_selector) - ] - - return self._call( - "list_requests", label_selector, {"timeout_s": timeout_s}, operation - ) - - def delete_request( - self, - name: str, - *, - propagation: Literal["Background", "Foreground"] = "Background", - timeout_s: float = 10.0, - ) -> None: - def operation() -> None: - request = self.requests.pop(name, None) - if request is None: - return - self.deleted_requests.append((name, propagation)) - if not self.cascade_deletes: - return - request_uid = request["metadata"]["uid"] - for run_name, run in list(self.runs.items()): - owners = run["metadata"].get("ownerReferences") or [] - if any(owner.get("uid") == request_uid for owner in owners): - self._delete_run(run_name) - - return self._call( - "delete_request", - name, - {"propagation": propagation, "timeout_s": timeout_s}, - operation, - ) - - # --------------------------------------------------- Runs, Candidates - - def _delete_run(self, run_name: str) -> None: - run = self.runs.pop(run_name, None) - self.run_inputs.pop(run_name, None) - if run is None: - return - run_uid = run["metadata"]["uid"] - for candidate_name, candidate in list(self.candidates.items()): - labels = candidate["metadata"].get("labels") or {} - if labels.get(RUN_UID_LABEL) == run_uid: - del self.candidates[candidate_name] - - def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - def operation() -> Optional[dict]: - found = self.runs.get(name) - return None if found is None else copy.deepcopy(found) - - return self._call("get_run", name, {"timeout_s": timeout_s}, operation) - - def list_candidates( - self, label_selector: str, *, timeout_s: float = 10.0 - ) -> list[dict]: - def operation() -> list[dict]: - return [ - copy.deepcopy(obj) - for obj in self.candidates.values() - if match_labels(obj["metadata"].get("labels"), label_selector) - ] - - return self._call( - "list_candidates", label_selector, {"timeout_s": timeout_s}, operation - ) - - # ------------------------------------------------------------- Leases - - def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - def operation() -> Optional[dict]: - found = self.leases.get(name) - return None if found is None else copy.deepcopy(found) - - return self._call("get_lease", name, {"timeout_s": timeout_s}, operation) - - def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: - def operation() -> dict: - stored = copy.deepcopy(body) - name = nested_get(stored, "metadata", "name") - if not name: - raise KubeInvalid("metadata.name is required") - if name in self.leases: - raise KubeAlreadyExists(f"leases {name!r} already exists") - stored["metadata"].pop("resourceVersion", None) - self._stamp(stored, kind="Lease", api_version="coordination.k8s.io/v1") - self.leases[name] = stored - return copy.deepcopy(stored) - - return self._call("create_lease", body, {"timeout_s": timeout_s}, operation) - - def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: - def operation() -> dict: - current = self.leases.get(name) - if current is None: - raise KubeNotFound(f"leases {name!r} not found") - sent = nested_get(body, "metadata", "resourceVersion") - if sent != current["metadata"]["resourceVersion"]: - raise KubeConflict( - f"leases {name!r}: resourceVersion {sent!r} is stale " - f"(current {current['metadata']['resourceVersion']!r})" - ) - stored = copy.deepcopy(body) - metadata = stored.setdefault("metadata", {}) - for key in ("uid", "creationTimestamp", "namespace", "generation"): - metadata[key] = current["metadata"].get(key) - metadata["name"] = name - stored.setdefault("apiVersion", current.get("apiVersion")) - stored.setdefault("kind", current.get("kind")) - self.bump(stored) - self.leases[name] = stored - return copy.deepcopy(stored) - - return self._call( - "replace_lease", (name, body), {"timeout_s": timeout_s}, operation - ) - - def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: - def operation() -> None: - self.leases.pop(name, None) - - return self._call("delete_lease", name, {"timeout_s": timeout_s}, operation) - - # ------------------------------------------------- operator simulation - - def set_request_condition( - self, - name: str, - *, - status: str, - reason: str, - message: str = "", - observed_generation: int | None = None, - ) -> None: - """Set the Request's ``Accepted`` condition.""" - - with self._lock: - request = self.requests[name] - generation = request["metadata"].get("generation", 1) - conditions = [ - condition - for condition in request.setdefault("status", {}).get("conditions", []) - if condition.get("type") != "Accepted" - ] - conditions.append( - { - "type": "Accepted", - "status": status, - "reason": reason, - "message": message, - "observedGeneration": ( - generation - if observed_generation is None - else observed_generation - ), - "lastTransitionTime": rfc3339(self.now()), - } - ) - request["status"]["conditions"] = conditions - self.bump(request) - - def accept_request( - self, - name: str, - *, - runtime_image: str = DEFAULT_RUNTIME_IMAGE, - dynamo_version: str = DEFAULT_DYNAMO_VERSION, - mutate_input: Optional[Callable[[dict], None]] = None, - ) -> dict: - """Create the Request's Run (``-``) and mark it Accepted. - - As with the operator, the new Run has no status yet: ``create_job`` - later records its lowered input (kept in ``run_inputs`` until then) - together with ``jobRef``. - """ - - with self._lock: - request = self.requests[name] - generation = int(request["metadata"].get("generation", 1)) - run_name = f"{name}-{generation}" - effective = lower_request_input( - request, - run_name=run_name, - runtime_image=runtime_image, - dynamo_version=dynamo_version, - ) - if mutate_input is not None: - mutate_input(effective) - run: dict[str, Any] = { - "metadata": { - "name": run_name, - "annotations": {REQUEST_GENERATION_ANNOTATION: str(generation)}, - "ownerReferences": [ - { - "apiVersion": "nvidia.com/v1beta2", - "kind": "DynamoGraphDeploymentRequest", - "name": name, - "uid": request["metadata"]["uid"], - "controller": True, - "blockOwnerDeletion": True, - } - ], - }, - "spec": copy.deepcopy( - {k: v for k, v in request.get("spec", {}).items() if k != "v1beta1"} - ), - "status": {}, - } - self._stamp( - run, kind="DynamoGraphDeploymentRun", api_version="nvidia.com/v1beta2" - ) - self.runs[run_name] = run - self.run_inputs[run_name] = effective - status = request.setdefault("status", {}) - status["observedGeneration"] = generation - status["activeRunRef"] = {"name": run_name} - self.set_request_condition( - name, - status="True", - reason="RunCreated", - message=f"Run {run_name!r} exists for generation {generation}", - ) - return copy.deepcopy(run) - - def create_job(self, run_name: str, *, record_input: bool = True) -> None: - """Create the Run's Job, as the operator's ``reconcileRunStatus`` does. - - One status write records ``status.aisimulate.input`` and ``jobRef`` - with ``Completed=False`` (``JobPending``). ``record_input=False`` - leaves the input out (an operator that does not record it). - """ - - with self._lock: - run = self.runs[run_name] - status = run.setdefault("status", {}) - if record_input: - status["aisimulate"] = { - "input": copy.deepcopy(self.run_inputs[run_name]) - } - status["jobRef"] = {"name": f"{run_name}-job", "uid": str(uuid.uuid4())} - status["phase"] = "Pending" - self._set_completed( - run, "False", "JobPending", "AISimulate Job is waiting to start" - ) - - def start_run(self, run_name: str, *, record_input: bool = True) -> None: - """Start the Run's Job (creating it first, see ``create_job``).""" - - with self._lock: - run = self.runs[run_name] - if not run["status"].get("jobRef"): - self.create_job(run_name, record_input=record_input) - status = run["status"] - status["startTime"] = rfc3339(self.now()) - status["phase"] = "Running" - self._set_completed(run, "False", "JobRunning", "AISimulate Job is running") - - def add_candidate( - self, - run_name: str, - parameters: Mapping[str, Any], - metrics: Mapping[str, Any], - ) -> dict: - with self._lock: - run = self.runs[run_name] - canonical = json.dumps(parameters, sort_keys=True, separators=(",", ":")) - candidate_id = ( - "evaluated-point-" + uuid.uuid5(uuid.NAMESPACE_OID, canonical).hex[:12] - ) - candidate = { - "metadata": { - "name": f"{run_name}-{candidate_id}", - "labels": { - RUN_UID_LABEL: run["metadata"]["uid"], - CANDIDATE_ID_LABEL: candidate_id, - }, - "ownerReferences": [ - { - "apiVersion": "nvidia.com/v1beta2", - "kind": "DynamoGraphDeploymentRun", - "name": run_name, - "uid": run["metadata"]["uid"], - "controller": True, - } - ], - }, - "spec": {"parameters": json.loads(canonical)}, - "status": {"metrics": json.loads(json.dumps(dict(metrics)))}, - } - self._stamp( - candidate, - kind="DynamoGraphDeploymentCandidate", - api_version="nvidia.com/v1beta2", - ) - self.candidates[candidate["metadata"]["name"]] = candidate - return copy.deepcopy(candidate) - - def complete_run( - self, - run_name: str, - *, - reason: str = "SearchSucceeded", - message: str = "", - candidates: Iterable[tuple[Mapping[str, Any], Mapping[str, Any]]] = (), - ) -> None: - """Publish ``candidates`` (parameters, metrics) and complete the Run.""" - - with self._lock: - run = self.runs[run_name] - refs = [ - { - "name": self.add_candidate(run_name, params, metrics)["metadata"][ - "name" - ] - } - for params, metrics in candidates - ] - status = run["status"] - status["candidateRefs"] = refs - status["phase"] = "Succeeded" if reason == "SearchSucceeded" else "Failed" - status["completionTime"] = rfc3339(self.now()) - self._set_completed(run, "True", reason, message) - owner = (run["metadata"].get("ownerReferences") or [{}])[0] - request = self.requests.get(owner.get("name", "")) - if request is not None: - request_status = request.setdefault("status", {}) - active = request_status.get("activeRunRef") or {} - if active.get("name") == run_name: - request_status.pop("activeRunRef", None) - recent = [ - ref - for ref in request_status.get("recentRunRefs", []) - if ref.get("name") != run_name - ] - request_status["recentRunRefs"] = [{"name": run_name}, *recent] - self.bump(request) - - def _set_completed(self, run: dict, status: str, reason: str, message: str) -> None: - conditions = [ - condition - for condition in run["status"].get("conditions", []) - if condition.get("type") != "Completed" - ] - conditions.append( - { - "type": "Completed", - "status": status, - "reason": reason, - "message": message, - "lastTransitionTime": rfc3339(self.now()), - } - ) - run["status"]["conditions"] = conditions - self.bump(run) - - def run_of(self, request_name: str) -> Optional[dict]: - """The stored Run a Request owns, if any (the newest).""" - - with self._lock: - request = self.requests.get(request_name) - if request is None: - return None - owned = [ - run - for run in self.runs.values() - if any( - owner.get("uid") == request["metadata"]["uid"] - for owner in run["metadata"].get("ownerReferences") or [] - ) - ] - return copy.deepcopy(owned[-1]) if owned else None - - -Evaluator = Callable[[dict, Mapping[str, float]], Optional[Mapping[str, Any]]] - - -class FakeDgdrOperator: - """Drives every Request of a ``FakeRetuneKube`` one phase per ``step()``. - - Phases: accept (a Run with no status, as before its Job exists), start - (the lowered input, jobRef and startTime; the operator records the - input together with jobRef), complete (one Candidate per router point - the input spans, metrics from ``evaluate(request, point_public)``; - ``None`` leaves the point unpublished, as AISimulate drops an - infeasible point). ``hold`` keeps - named Requests in their current phase; ``fail_runs`` maps a Request - name to the ``Completed`` reason its Run ends with. ``mutate_input`` - edits a Run's lowered input and ``mutate_parameters`` a Candidate's - parameters (both get the Request name), to simulate version skew. - ``max_phase`` stops every Request at ``accepted`` or ``started``. - """ - - def __init__( - self, - kube: FakeRetuneKube, - *, - evaluate: Optional[Evaluator] = None, - runtime_image: str = DEFAULT_RUNTIME_IMAGE, - dynamo_version: str = DEFAULT_DYNAMO_VERSION, - mutate_input: Optional[Callable[[str, dict], None]] = None, - mutate_parameters: Optional[Callable[[str, dict], None]] = None, - ) -> None: - self.kube = kube - self.evaluate: Evaluator = evaluate or ( - lambda _request, point: default_metrics(point) - ) - self.runtime_image = runtime_image - self.dynamo_version = dynamo_version - self.mutate_input = mutate_input - self.mutate_parameters = mutate_parameters - self.max_phase: Optional[Literal["accepted", "started"]] = None - self.hold: set[str] = set() - self.fail_runs: dict[str, str] = {} - self.steps = 0 - - def step(self) -> int: - """Advance every Request one phase; returns how many moved.""" - - moved = 0 - with self.kube._lock: - self.steps += 1 - for name in list(self.kube.requests): - if name in self.hold: - continue - run = self.kube.run_of(name) - if run is None: - hook = self.mutate_input - self.kube.accept_request( - name, - runtime_image=self.runtime_image, - dynamo_version=self.dynamo_version, - mutate_input=( - None if hook is None else functools.partial(hook, name) - ), - ) - moved += 1 - continue - run_name = run["metadata"]["name"] - completed = next( - ( - condition - for condition in run["status"].get("conditions", []) - if condition.get("type") == "Completed" - ), - None, - ) - if completed is not None and completed.get("status") == "True": - continue - if self.max_phase == "accepted": - continue - if not run["status"].get("startTime"): - self.kube.start_run(run_name) - moved += 1 - continue - if self.max_phase == "started": - continue - reason = self.fail_runs.get(name) - if reason is not None: - self.kube.complete_run(run_name, reason=reason, message="injected") - moved += 1 - continue - effective = run["status"]["aisimulate"]["input"] - request = self.kube.requests[name] - published = [] - for point in router_points(effective): - metrics = self.evaluate(copy.deepcopy(request), point) - if metrics is not None: - parameters = candidate_parameters(effective, point) - if self.mutate_parameters is not None: - self.mutate_parameters(name, parameters) - published.append((parameters, metrics)) - self.kube.complete_run(run_name, candidates=published) - moved += 1 - return moved - - def run_to_completion(self, max_steps: int = 10) -> None: - for _ in range(max_steps): - if not self.step(): - return diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py index 7ce2be19ba81..35e37715a33f 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py @@ -31,6 +31,7 @@ PlannerConfig, balanced_shard_parts, dgdr_need_seconds, + dgdr_rounds, largest_shard_points, ) from dynamo.planner.retune.domains import expand_router_domain @@ -75,7 +76,7 @@ def _worker(**overrides): "context_length": "max", "workers": {"prefill": _worker(), "decode": _worker()}, } -# Design §4.1 defaults: T = 6 h, timeout = T - 20 min. +# The live retune defaults: T = 6 h, timeout = T - 20 min. INTERVAL = 21600.0 TIMEOUT = 20400.0 # The default router domain is 3 x 9 x 4 = 108 points; with max_shards 3 the @@ -566,9 +567,8 @@ def test_balanced_shard_parts(sizes, max_shards, parts, largest): def test_balanced_shard_parts_minimizes_the_largest_shard_exhaustively(): - for sizes in ((3, 9, 4), (5, 7), (2, 3, 5), (12,), (1, 1, 30)): + for sizes in ((3, 9, 4), (5, 7), (2, 3, 5), (12,), (1, 1, 30), (250,)): for max_shards in range(1, 17): - parts = balanced_shard_parts(sizes, max_shards) best = min( largest_shard_points(sizes, (a, b, c)[: len(sizes)]) for a in range(1, max_shards + 1) @@ -577,16 +577,45 @@ def test_balanced_shard_parts_minimizes_the_largest_shard_exhaustively(): if math.prod((a, b, c)[: len(sizes)]) <= max_shards and all(p <= n for p, n in zip((a, b, c)[: len(sizes)], sizes)) ) + parts = balanced_shard_parts(sizes, max_shards) assert largest_shard_points(sizes, parts) == best + # With the Run parallelism, the split takes the fewest rounds, so + # its largest shard fits and finishes exactly when the smallest + # largest shard does: P13 and the executor agree. + for parallelism in (1, 8, 36): + parts = balanced_shard_parts(sizes, max_shards, parallelism=parallelism) + largest = largest_shard_points(sizes, parts) + if best > DGDR_MAX_SHARD_POINTS: + assert largest == best + else: + assert largest <= DGDR_MAX_SHARD_POINTS + assert dgdr_rounds(largest, parallelism) == dgdr_rounds( + best, parallelism + ) + + +@pytest.mark.parametrize( + ("sizes", "max_shards", "parallelism", "parts"), + [ + # A shard is added only when it saves an evaluation round; the first + # candidate runs alone, so single points save one. + ((2, 2, 2), 3, 8, (1, 1, 1)), + ((2, 2, 2), 16, 8, (2, 2, 2)), + ((2, 2, 2), 3, 1, (1, 1, 2)), + ((3, 9, 4), 3, 8, (1, 3, 1)), + ], +) +def test_balanced_shard_parts_with_parallelism(sizes, max_shards, parallelism, parts): + assert balanced_shard_parts(sizes, max_shards, parallelism=parallelism) == parts @pytest.mark.parametrize( - ("sizes", "max_shards"), - [((3, 0), 3), ((3,), 0)], + ("sizes", "max_shards", "parallelism"), + [((3, 0), 3, None), ((3,), 0, None), ((3,), 3, 0)], ) -def test_balanced_shard_parts_rejects_bad_input(sizes, max_shards): +def test_balanced_shard_parts_rejects_bad_input(sizes, max_shards, parallelism): with pytest.raises(ValueError): - balanced_shard_parts(sizes, max_shards) + balanced_shard_parts(sizes, max_shards, parallelism=parallelism) def test_default_router_grid_splits_into_three_shards_of_36(): diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py index ce64e08551eb..3faf3db93f2c 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py @@ -5,10 +5,10 @@ """Frontend startup parsing and the spec premises of live retune. -Covers ``find_frontend`` (v1beta1 and v1alpha1 layouts), ``parse_frontend`` -(the operator's argv defaulting, env merging and ``$(VAR)`` expansion, the -frontend's own argument groups run against an explicit env, a golden -comparison with the real frontend parser, disaggregated maps) and +Covers ``find_frontend``, ``parse_frontend`` (the operator's argv +defaulting, env merging and ``$(VAR)`` expansion, the frontend's own parser +built against an explicit env, a golden comparison with the real frontend +parser, disaggregated maps) and ``evaluate_premises`` (P4, P6-P10) on table-driven DGD fixtures, including P4 on workers that advertise their own router configuration (compared with the real worker parser) and the P7 check of ``retune.dgdr.model``. @@ -20,6 +20,8 @@ import copy import json import os +import sys +import types import warnings from typing import Any @@ -93,7 +95,7 @@ def trace_env(**overrides: Any) -> list[dict[str, Any]]: - """The recipe env of design §4.4.1; an override of None drops the entry.""" + """The P10 trace recipe env; an override of None drops the entry.""" entries: dict[str, dict[str, Any]] = { "DYN_POD_NAME": { @@ -260,20 +262,9 @@ def parse_error(deployment: dict[str, Any], mode: str = "agg") -> str: def evaluate( - deployment: dict[str, Any], - *, - config: PlannerConfig | None = None, - planner_replicas: int | None = None, + deployment: dict[str, Any], *, config: PlannerConfig | None = None ) -> PremiseReport: - config = config or planner_config() - result = fc.parse_frontend(deployment, planner_mode=str(config.mode)) - return fc.evaluate_premises( - deployment, - planner_config=config, - startup=result if isinstance(result, FrontendStartup) else None, - parse_error=result if isinstance(result, str) else None, - planner_replicas=planner_replicas, - ) + return fc.evaluate_premises(deployment, planner_config=config or planner_config()) def reasons_matching(report: PremiseReport, prefix: str) -> list[str]: @@ -293,47 +284,6 @@ def test_find_frontend_v1beta1(): assert spec["type"] == "frontend" -def test_find_frontend_v1alpha1_services_map(): - deployment = { - "apiVersion": "nvidia.com/v1alpha1", - "kind": "DynamoGraphDeployment", - "metadata": {"name": DGD, "namespace": NAMESPACE}, - "spec": { - "envs": [{"name": "DYN_ROUTER_TEMPERATURE", "value": "0.5"}], - "services": { - "Frontend": { - "componentType": "frontend", - "replicas": 1, - "envs": trace_env(), - "volumeMounts": [{"name": CLAIM, "mountPoint": "/trace"}], - "extraPodSpec": { - "mainContainer": { - "command": ["python3"], - "args": list(KV_ARGS), - } - }, - }, - "VllmWorker": { - "componentType": "worker", - "replicas": 2, - "extraPodSpec": {"mainContainer": {"args": [f"--model {MODEL}"]}}, - }, - "Planner": {"componentType": "planner", "replicas": 1}, - }, - }, - } - name, spec = fc.find_frontend(deployment) - assert name == "Frontend" - assert spec["podTemplate"]["spec"]["containers"][0]["args"] == list(KV_ARGS) - - startup = parse(deployment) - assert startup.router_mode == "kv_router" - assert startup.router_config == {"router_temperature": 0.5} - report = evaluate(deployment) - assert report.spec_ok, report.reasons - assert fc.served_model_name(deployment) == MODEL - - @pytest.mark.parametrize( ("components", "match"), [ @@ -880,18 +830,6 @@ def test_parse_error_fails_p4_only(): assert report.startup is None -def test_caller_parse_error_is_reported(): - report = fc.evaluate_premises( - dgd(), - planner_config=planner_config(), - startup=None, - parse_error="P4: injected", - planner_replicas=None, - ) - assert not report.p4_ok - assert report.reasons == ("P4: injected",) - - @pytest.mark.parametrize( ("args", "env", "match"), [ @@ -1343,40 +1281,6 @@ def test_p4_shell_wrapped_worker(script, match): assert any(match in reason for reason in rejected), report.reasons -def test_p4_rejects_a_v1alpha1_worker_router_mode(): - deployment = { - "apiVersion": "nvidia.com/v1alpha1", - "kind": "DynamoGraphDeployment", - "metadata": {"name": DGD, "namespace": NAMESPACE}, - "spec": { - "services": { - "Frontend": { - "componentType": "frontend", - "envs": trace_env(), - "volumeMounts": [{"name": CLAIM, "mountPoint": "/trace"}], - "extraPodSpec": { - "mainContainer": {"command": ["python3"], "args": list(KV_ARGS)} - }, - }, - "VllmWorker": { - "componentType": "worker", - "envs": [{"name": "DYN_ROUTER_MODE", "value": "kv"}], - "extraPodSpec": { - "mainContainer": { - "command": ["python3", "-m", "dynamo.vllm"], - "args": ["--model", MODEL], - } - }, - }, - "Planner": {"componentType": "planner", "replicas": 1}, - }, - }, - } - report = evaluate(deployment) - assert not report.p4_ok - assert reasons_matching(report, "P4: worker component VllmWorker advertises") - - def test_worker_check_ignores_the_planner_env(monkeypatch): monkeypatch.setenv("DYN_ROUTER_MODE", "kv") fc._worker_router_cached.cache_clear() @@ -1458,6 +1362,33 @@ def test_p6_warns_on_unknown_model(): assert any(w.startswith("P6:") for w in report.warnings) +@pytest.fixture +def uncached_model_warning(): + fc._model_warning.cache_clear() + yield + fc._model_warning.cache_clear() + + +@pytest.mark.usefixtures("uncached_model_warning") +def test_p6_without_aisimulate_is_not_checked(monkeypatch): + monkeypatch.setitem(sys.modules, "aisimulate_core.sdk.common", None) + assert fc._model_warning("acme/private-model") is None + + +@pytest.mark.usefixtures("uncached_model_warning") +def test_p6_broken_aisimulate_warns(monkeypatch, caplog): + def broken(name: str) -> Any: + raise RuntimeError("broken install") + + module = types.ModuleType("aisimulate_core.sdk.common") + module.__getattr__ = broken # type: ignore[method-assign] + monkeypatch.setitem(sys.modules, "aisimulate_core.sdk.common", module) + warning = fc._model_warning(MODEL) + assert warning is not None and warning.startswith("P6: cannot check") + assert "RuntimeError: broken install" in warning + assert any(r.levelname == "ERROR" for r in caplog.records) + + def test_p7_served_models(): two = dgd(frontend(), worker("a"), worker("b", model="meta/llama"), planner()) report = evaluate(two) @@ -1613,32 +1544,11 @@ def test_p9_scaling_adapter_and_planner_replicas(): ] two_planners = dgd(frontend(), worker(), planner(replicas=2)) assert "2 replicas" in reasons_matching(evaluate(two_planners), "P9:")[0] - # The caller's own replica count wins over the DGD. - assert not reasons_matching(evaluate(two_planners, planner_replicas=1), "P9:") - assert reasons_matching(evaluate(dgd(), planner_replicas=3), "P9:") no_planner = evaluate(dgd(frontend(), worker())) assert no_planner.spec_ok assert any(w.startswith("P9:") for w in no_planner.warnings) -def test_p9_v1alpha1_scaling_adapter_needs_enabled(): - def deployment(adapter): - return { - "metadata": {"name": DGD}, - "spec": { - "services": { - "Frontend": {"componentType": "frontend"}, - "w": {"componentType": "worker", "scalingAdapter": adapter}, - } - }, - } - - disabled = fc._components(deployment({})) - enabled = fc._components(deployment({"enabled": True})) - assert not fc._scaling_adapter(disabled[1]) - assert fc._scaling_adapter(enabled[1]) - - P10_CASES = [ ({"DYN_REQUEST_TRACE": None}, None, "DYN_REQUEST_TRACE=1"), ({"DYN_REQUEST_TRACE": "0"}, None, "DYN_REQUEST_TRACE=1"), @@ -1811,13 +1721,7 @@ def test_p10_is_checked_when_parsing_fails(): def test_evaluate_requires_dgdr(): config = PlannerConfig.model_construct(mode="agg", retune=None) with pytest.raises(ValueError, match="retune.dgdr"): - fc.evaluate_premises( - dgd(), - planner_config=config, - startup=None, - parse_error=None, - planner_replicas=None, - ) + fc.evaluate_premises(dgd(), planner_config=config) def test_malformed_dgd_never_raises(): From 2d27a644cd4be71ad7c19379a050e75a5e82dca0 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 08:42:38 -0800 Subject: [PATCH 080/100] fix(planner): address review on the live retune window cut and reconciler Window cut: - Check for a published window before reading any segment. The window file's gzip header comment now holds the counts of the cut that wrote it, so a later leader adopting the window gets those counts back and can no longer fail it on segments that changed or broke since; a link race reports the winner's counts. - Skip segments last modified more than 300 s before the window starts: rows are appended when requests complete, after they arrive. - Read only .jsonl.gz segments and drop the plain JSONL reader; the live sink writes gzip and the reconciler collects only those segments. - Compress and write synchronously; drop the writer thread and the switch-interval tweak, keep the child's lower CPU priority. Reconciler: - Ask should_continue before listing Requests and before each uncached Run-state read, not only before deletes. - Log a missing trace directory or Request label when it starts rather than every round; the gauge tracks the condition. - Read the Request trace source from spec.workload.trace.source.persistentVolumeClaim.{claimName,subPath} through one helper, trace_pvc_source. - State the protected-window and sink-layout rules inline instead of citing the design document. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/retune/live/reconciler.py | 106 +++-- .../dynamo/planner/retune/live/window_cut.py | 447 +++++++++--------- .../tests/unit/test_live_retune_reconciler.py | 85 +++- .../tests/unit/test_live_retune_window_cut.py | 238 ++++------ 4 files changed, 477 insertions(+), 399 deletions(-) diff --git a/components/src/dynamo/planner/retune/live/reconciler.py b/components/src/dynamo/planner/retune/live/reconciler.py index 7184900bdd78..767cea640033 100644 --- a/components/src/dynamo/planner/retune/live/reconciler.py +++ b/components/src/dynamo/planner/retune/live/reconciler.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Garbage collection of live retune files and DGDR Requests (design §4.4.5). +"""Garbage collection of live retune files and DGDR Requests. The supervisor runs ``Reconciler.run_once`` only while this process leads and is initialized. It deletes, with ``fs_root`` the planner's mount of the trace @@ -16,8 +16,9 @@ highest-numbered segment of a live pod; other segments of a live pod, and the whole directory of a pod that no longer exists, once every file is older than ``T + G + timeout + visibility margin``. A live pod's segments - are the files directly in its directory: P10 requires the sink's file - prefix to sit directly under ``$(DYN_POD_NAME)/``. A pod directory that + are the files directly in its directory: the frontend premise check + requires the sink's file prefix to sit directly under + ``$(DYN_POD_NAME)/``. A pod directory that cannot be read is skipped for the round; the other pods are still collected. - Window files, never while protected: ``window_end_ms`` in @@ -33,10 +34,13 @@ non-terminal Request whose window deadline has passed, a Request in the delete queue, and a Request of another DGD uid (the DGD was recreated). -``run_once(should_continue=...)`` asks the caller before every delete or -unlink and ends the round as soon as it says no (leadership or the term was -lost while the round ran on its worker thread). ``set_dgd`` follows the DGD -identity when the DGD is recreated under the same name. +``run_once(should_continue=...)`` asks the caller before every delete, +unlink or Kubernetes read and ends the round as soon as it says no +(leadership or the term was lost, or the round's budget passed, while it +ran on its worker thread). ``set_dgd`` follows the DGD identity when the DGD +is recreated under the same name. Conditions that persist across rounds +(a frontend without a trace directory, an unlabelled Request) are logged +when they start, not every round. ``Reconciler`` is also the ``SweepRegistry`` the DGDR sweep uses to protect its window and Requests and to queue deletions; those methods are thread-safe @@ -112,6 +116,24 @@ def window_relpath(cfg: DgdrExecutorConfig, dgd_name: str, window_end_ms: int) - ) +def trace_pvc_source(spec: Any) -> tuple[Optional[str], Optional[str]]: + """``(claimName, subPath)`` of a DGDR spec's trace claim. + + The fields are ``workload.trace.source.persistentVolumeClaim.claimName`` + and ``.subPath`` (the claim-relative file the Run replays); each is + ``None`` when absent or not a string. + """ + + pvc = _dig(spec, "workload", "trace", "source", "persistentVolumeClaim") + if not isinstance(pvc, Mapping): + return None, None + claim, sub_path = pvc.get("claimName"), pvc.get("subPath") + return ( + claim if isinstance(claim, str) else None, + sub_path if isinstance(sub_path, str) else None, + ) + + @dataclass(frozen=True) class ReconcileReport: """What one ``run_once`` round did.""" @@ -216,7 +238,12 @@ def __init__( # neither can revert for one generation. self._terminal: set[tuple[str, int]] = set() self._finished: set[tuple[str, int]] = set() + # Owned by the round (one runs at a time): the pods whose + # trace_sink_missing gauge is set, those last reported missing, and + # the (Request uid, label) pairs already reported missing. self._sink_reported: set[str] = set() + self._sink_missing: set[str] = set() + self._unlabelled: set[tuple[str, str]] = set() def _root(self, directory: str, dgd: DgdRef) -> str: return os.path.join(self._fs_root, directory.strip("/"), dgd.name) @@ -322,8 +349,9 @@ def run_once( """One GC round. ``deadline_for(window_end_ms)`` is that window's sweep deadline ``D_k`` in epoch seconds; ``frontend_pods`` is the actuator's listing (None when it failed). ``should_continue`` is - called before every delete or unlink; once it returns False (or - raises) the round deletes nothing more and returns.""" + called before every delete, unlink or Kubernetes read; once it + returns False (or raises) the round deletes and reads nothing more + and returns.""" now = float(self._now()) with self._lock: @@ -336,7 +364,8 @@ def run_once( ) def in_window_range(window_end_ms: int) -> bool: - """The design's protected range: undecided and before D_k.""" + """Undecided (from the last decided window to the due one) and + before its deadline D_k: a sweep may still cut or adopt it.""" if last_window_end_ms is not None and window_end_ms < last_window_end_ms: return False @@ -357,7 +386,7 @@ def in_window_range(window_end_ms: int) -> bool: state.error("sinks") requests: Optional[list[_RequestView]] = None - if not state.stopped: + if self._proceed(state): try: requests = self._list_requests(state.dgd) except KubeError as exc: @@ -417,8 +446,9 @@ def in_window_range(window_end_ms: int) -> bool: return report def _proceed(self, state: _Round) -> bool: - """Whether the round may delete one more thing; asked right before - every delete or unlink. A no is final for the round.""" + """Whether the round may delete or read one more thing; asked right + before every delete, unlink or Kubernetes read. A no is final for + the round.""" if state.stopped: return False @@ -481,8 +511,9 @@ def _collect_segments( def _collect_pod_segments( self, state: _Round, directory: str, cutoff: float ) -> None: - # Top level only: P10 puts the sink's file prefix directly under the - # pod directory, so there is no subdirectory to walk. + # Top level only: the frontend premise check puts the sink's file + # prefix directly under the pod directory, so there is no + # subdirectory to walk. segments: dict[str, list[tuple[int, os.DirEntry[str]]]] = {} for entry in _scandir(directory): match = _SEGMENT_RE.match(entry.name) @@ -526,12 +557,15 @@ def _report_sinks( for name in self._sink_reported - current: self._metrics.trace_sink_missing(name, False) self._sink_reported = current - if missing: + # The gauge tracks the condition; the log marks where it starts. + newly = sorted(set(missing) - self._sink_missing) + self._sink_missing = set(missing) + if newly: logger.warning( "retune reconciler: Ready frontends without a trace directory " "under %s: %s", state.segments_root, - missing, + newly, ) return missing @@ -616,12 +650,11 @@ def _list_requests(self, dgd: DgdRef) -> list[_RequestView]: for obj in items: metadata = obj.get("metadata") or {} labels = metadata.get("labels") or {} - pvc = _dig(obj, "spec", "workload", "trace", "source", "pvc") or {} - path = pvc.get("path") + claim, path = trace_pvc_source(obj.get("spec")) trace_file = None if ( - pvc.get("claimName") == self._cfg.claim_name - and isinstance(path, str) + claim == self._cfg.claim_name + and path is not None and path.startswith(prefix) and "/" not in path[len(prefix) :] ): @@ -655,17 +688,18 @@ def _collect_requests( cache.difference_update( [key for key in cache if key[0] not in listed] ) + self._unlabelled.difference_update( + [key for key in self._unlabelled if key[0] not in listed] + ) ours = [view for view in requests if view.dgd_uid == dgd_uid] for view in requests: if view.dgd_uid is None: - logger.warning( - "retune reconciler: Request %s has no %s label", - view.name, - LABEL_DGD_UID, - ) + self._report_unlabelled(view, LABEL_DGD_UID) elif view.dgd_uid != dgd_uid: # The DGD was deleted and recreated under the same name. reasons.setdefault(view.name, "other_dgd_uid") + # Each uncached verdict reads Runs; _is_terminal asks + # should_continue first, so a stopped round reads nothing more. states = {view.name: self._is_terminal(state, view) for view in ours} terminal = {name for name, value in states.items() if value is True} kept_windows = sorted( @@ -679,11 +713,7 @@ def _collect_requests( for view in ours: window = view.window_end_ms if window is None: - logger.warning( - "retune reconciler: Request %s has no %s label", - view.name, - LABEL_WINDOW, - ) + self._report_unlabelled(view, LABEL_WINDOW) continue value = states[view.name] if value is None: @@ -718,6 +748,16 @@ def _collect_requests( with self._lock: self._delete_queue.discard(name) + def _report_unlabelled(self, view: _RequestView, label: str) -> None: + """Warn once per Request about a missing (or unparsable) label.""" + + key = (view.uid, label) + if key not in self._unlabelled: + self._unlabelled.add(key) + logger.warning( + "retune reconciler: Request %s has no %s label", view.name, label + ) + # ------------------------------------------------- Request / Run state def _cached(self, cache: set[tuple[str, int]], key: tuple[str, int]) -> bool: @@ -743,6 +783,8 @@ def _is_terminal(self, state: _Round, view: _RequestView) -> Optional[bool]: key = (view.uid, generation) if self._cached(self._terminal, key): return True + if not self._proceed(state): + return None try: run = self._current_run(view, generation) except KubeError as exc: @@ -772,6 +814,8 @@ def _finished_with(self, state: _Round, view: _RequestView) -> bool: return True if _dig(view.obj, "status", "activeRunRef", "name"): return False + if not self._proceed(state): + return False try: run = self._current_run(view, generation) if run is None: diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py index 54e22b1921df..96d485a6775c 100644 --- a/components/src/dynamo/planner/retune/live/window_cut.py +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -10,9 +10,18 @@ ``[start_ms, end_ms)`` and completed by ``as_of_ms`` into the bare-row window file a DGDR Run replays. Selection, censoring, de-duplication, the single block-size rule and ``row_counts`` are those of -``retune.window.read_trace_window``, and the rows are the same bytes; only -the output is gzip-compressed and the memory is bounded. - +``retune.window.read_trace_window`` over the segments the cut reads, and the +rows are the same bytes; only the output is gzip-compressed and the memory +is bounded. Only ``.jsonl.gz`` segments are read: the frontend's live trace +sink must write gzip, and the reconciler collects only those segments. + +* **Segments.** A row is appended to its segment when the request + completes, after it arrived, so a segment last modified before + ``start_ms`` holds no row of the window and is not read; the + ``SEGMENT_MTIME_MARGIN_S`` margin covers clock skew between the frontends, + which stamp arrivals, and the file server, which stamps modification + times. The reconciler keeps segments well past the window, so this skips + most of what it retains. * **Members.** Each gzip member is decoded on its own and checked against its CRC32 and ISIZE before any of its lines is read. A member that fails (cut short by a segment still being written, a zero-filled hole a lagging RWX @@ -23,22 +32,25 @@ (``trace_corrupt``); a run that reaches the end of a writer's newest segment is its in-flight flush (or a crashed writer's last one) and is counted but does not count toward either ratio. -* **Pass 1** parses every row as the replay reader does (a row it would - reject, in a member whose CRC is good, fails the cut as ``trace_corrupt``) - and keeps only columns for each selected row (~45 B: arrival, completion, - a 128-bit digest of ``request_id``, where its line is, block size, flags). - Duplicates are removed by digest, keeping the first in file order, - whenever the columns outgrow ``max_rows`` by a slack, so ``max_rows`` - bounds memory and is checked on de-duplicated rows. +* **Pass 1** parses every row of the segments it reads as the replay reader + does (a row it would reject, in a member whose CRC is good, fails the cut + as ``trace_corrupt``) and keeps only columns for each selected row (~45 B: + arrival, completion, a 128-bit digest of ``request_id``, where its line + is, block size, flags). Duplicates are removed by digest, keeping the + first in file order, whenever the columns outgrow ``max_rows`` by a slack, + so ``max_rows`` bounds memory and is checked on de-duplicated rows. * **Pass 2** re-reads only the members that hold kept rows, in output order (arrival, completion, then ``request_id`` within equal pairs), re-parses - each line with its hashes and streams ``_window_record`` rows to gzip, - compressed and written by a thread. Decoded lines wait in a bounded cache - that evicts the member needed furthest in the future. A member or row - that no longer matches pass 1 fails the cut (the segment changed). -* **Atomic output.** The file is written to ``.tmp--`` - next to ``out_path``, fsynced, made 0600 and hard-linked into place; an - existing ``out_path`` is reused (another process cut the same window). + each line with its hashes and streams ``_window_record`` rows to gzip. + Decoded lines wait in a bounded cache that evicts the member needed + furthest in the future. A member or row that no longer matches pass 1 + fails the cut (the segment changed). +* **Atomic output.** The file is one gzip member whose header comment holds + the cut's counts as JSON. It is written to ``.tmp--`` + next to ``out_path``, fsynced, made 0600 and hard-linked into place. An + existing ``out_path`` (another process cut the same window) is checked + before any segment is read and reused with the counts in its header, so a + later leader adopting a published window never cuts it again. * **Child process.** ``cut_window`` runs the cut in a ``spawn`` child, like the replay sweep's workers, and kills it at the deadline or on cancel, so a hung RWX read or a runaway cut never holds the sweep thread. @@ -47,25 +59,22 @@ from __future__ import annotations import errno -import gzip import hashlib import json import logging import math import multiprocessing import os -import queue import re import secrets import stat import struct -import sys import threading import time import zlib from array import array from collections import OrderedDict -from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence +from collections.abc import Callable, Generator, Iterable, Mapping, Sequence from dataclasses import dataclass from multiprocessing.connection import Connection, wait from operator import attrgetter @@ -74,7 +83,6 @@ import numpy as np -from dynamo.planner.offline.trace_data import _reject_duplicate_copies from dynamo.planner.retune.live.contracts import WindowCutResult from dynamo.planner.retune.window import ( _dumps, @@ -109,21 +117,28 @@ def write(self, data: bytes, /) -> Any: # counts once however many members it wiped out (a zero-filled hole); its # bytes measure how much of the trace it took. MAX_DROPPED_MEMBER_RATIO = 0.01 +# Segments last modified more than this before start_ms are not read. +SEGMENT_MTIME_MARGIN_S = 300.0 # Window files are written fast rather than small: on hash-chain rows level # 1 is ~5x faster than level 6 for ~8% more bytes, and the cut budget pays # for compression. OUTPUT_COMPRESSLEVEL = 1 # Temporary window files: ``.tmp--`` next to out_path. TEMP_PREFIX = ".tmp-" +SEGMENT_SUFFIX = ".jsonl.gz" _TEMP_NAME = re.compile(r"^\.tmp-(\d+)-([0-9A-Za-z]+)$") -_SEGMENT_INDEX = re.compile(r"\.(\d+)\.jsonl(?:\.gz)?$") +_SEGMENT_INDEX = re.compile(r"\.(\d+)\.jsonl\.gz$") _GZIP_MAGIC = b"\x1f\x8b\x08" # ID1, ID2, CM=deflate _GZIP_WBITS = 16 + zlib.MAX_WBITS # one gzip member, header and trailer +# The header of a window file: FLG=FCOMMENT, MTIME=0 (equal cuts are equal +# bytes), XFL=0, OS=unknown. The zero-terminated comment that follows holds +# the cut's counts. +_WINDOW_HEADER = _GZIP_MAGIC + b"\x10" + bytes(4) + b"\x00\xff" +_WINDOW_HEADER_READ = 1 << 16 _READ_CHUNK = 1 << 20 _DECODE_STEP = 1 << 16 _COMPACT_AT = 1 << 20 # consumed reader bytes before the buffer is compacted -_PLAIN_BLOCK = 1 << 20 # bytes of whole lines per plain-JSONL pseudo-member # A trace file the reconciler deleted under the walk (ESTALE: on NFS). _VANISHED = frozenset({errno.ENOENT, errno.ESTALE}) # Pass-1 columns may hold this fraction of max_rows (at least the minimum) @@ -135,14 +150,12 @@ def write(self, data: bytes, /) -> Any: _LINE_OVERHEAD = 64 # bytes object and dict slot per cached line _OPEN_FILES = 32 _WRITE_BUFFER = 1 << 20 -_WRITE_QUEUE = 4 # batches waiting for the writer thread _FLAG_AGENT_CONTEXT = 1 _FLAG_NO_TOTAL_TIME = 2 _POLL_INTERVAL_S = 0.25 _EXIT_GRACE_S = 1.0 _JOIN_TIMEOUT_S = 5.0 _CHILD_NICENESS = 10 -_CHILD_SWITCH_INTERVAL_S = 0.0005 _MAX_DROP_EXAMPLES = 5 @@ -178,7 +191,7 @@ def temp_file_window_end(name: str) -> Optional[int]: def segment_index(name: str) -> Optional[int]: - """The ``NNNNNN`` of a ``.NNNNNN.jsonl[.gz]`` segment name, else ``None``.""" + """The ``NNNNNN`` of a ``.NNNNNN.jsonl.gz`` segment name, else ``None``.""" match = _SEGMENT_INDEX.search(name) return int(match.group(1)) if match else None @@ -254,9 +267,11 @@ def cut_window_inprocess( ) -> WindowCutResult: """Cut a window file in this process; the body of ``cut_window``'s child. - An empty window writes no file (``path=None``). ``temp_path`` names the - temporary file (it must be in ``out_path``'s directory); by default a - fresh ``temp_file_name`` there. Failures raise ``WindowCutError``. + An empty window writes no file (``path=None``). When ``out_path`` already + exists, no segment is read: the result carries the counts in its header + (``reused=True``, nothing dropped). ``temp_path`` names the temporary + file (it must be in ``out_path``'s directory); by default a fresh + ``temp_file_name`` there. Failures raise ``WindowCutError``. """ _validate(start_ms, end_ms, as_of_ms, out_path, max_rows) @@ -269,37 +284,37 @@ def cut_window_inprocess( # os.link needs one filesystem; the reconciler looks for it there. raise ValueError(f"temp_path {temp_path!r} is not next to {out_path!r}") try: + if os.path.lexists(out_path): + # Another process (an earlier leader, an adopted Request's cut) + # already published this window; DGDR Requests name it by path. + return _published(out_path) files = _segment_files(os.fspath(segments_root)) selection = _Selection(start_ms, end_ms, as_of_ms, max_rows) - items = _trace_items(files) + items = _trace_items(files, start_ms / 1000.0 - SEGMENT_MTIME_MARGIN_S) try: selection.scan(items) finally: items.close() selection.finish() - - def result(path: Optional[str], reused: bool = False) -> WindowCutResult: - return WindowCutResult( - path=path, - num_requests=selection.num_rows, - trace_block_size=selection.block_size, - row_counts=dict(selection.counts), - dropped_members=selection.dropped_members, - dropped_bytes=selection.dropped_bytes, - reused=reused, - ) - - if selection.num_rows == 0: - return result(None) - if os.path.lexists(out_path): - # Another process (an earlier leader, an adopted Request's cut) - # already published this window; DGDR Requests name it by path. - return result(out_path, reused=True) + result = WindowCutResult( + path=out_path if selection.num_rows else None, + num_requests=selection.num_rows, + trace_block_size=selection.block_size, + row_counts=dict(selection.counts), + dropped_members=selection.dropped_members, + dropped_bytes=selection.dropped_bytes, + ) + if result.path is None: + return result paths = [path for path, _ in files] created = _publish( - out_path, temp_path, lambda sink: _write_rows(selection, paths, sink) + out_path, + temp_path, + _counts_comment(result), + lambda sink: _write_rows(selection, paths, sink), ) - return result(out_path, reused=not created) + # Another process published it first: report the counts it holds. + return result if created else _published(out_path) except OSError as error: raise WindowCutError( "window_cut_failed", f"{type(error).__name__}: {error}" @@ -441,13 +456,11 @@ def _child_main( ) -> None: """Child entry point: answer ``("ok", value)`` or ``("error", reason, message)``.""" + # The cut shares the planner pod's CPU; the planner's own loop comes first. try: os.nice(_CHILD_NICENESS) except OSError: pass - # The writer thread takes the GIL back after every zlib block; at the - # default 5 ms interval it would wait behind parsing (pass 2 ~30% slower). - sys.setswitchinterval(_CHILD_SWITCH_INTERVAL_S) try: outcome: tuple[Any, ...] = ("ok", function(*args, **kwargs)) except WindowCutError as error: @@ -614,18 +627,18 @@ def _decode_exact_member(data: bytes) -> Optional[bytes]: def _segment_files(root: str) -> list[tuple[str, bool]]: - """Trace files under ``root`` in replay-reader order, and which are newest. - - The files and their order are those of ``resolve_trace_files([root])``: - sorted ``*.jsonl`` and ``*.jsonl.gz`` regular files, and a file present - both compressed and not is an error. A directory or file under ``root`` - that vanishes during the walk (ENOENT, or ESTALE on NFS: the reconciler - deletes a dead pod's segments) is skipped; any other error listing or - stat-ing one (EACCES, EIO, ETIMEDOUT on a soft mount, ...) fails the cut - as ``window_cut_failed`` rather than silently losing a pod's rows, as - does any error on ``root`` itself. Each file comes with whether it is the - newest segment (highest ``NNNNNN``) of its directory, the one its writer - may still be appending to. + """Trace segments under ``root`` in replay-reader order, and which are newest. + + The files are the sorted ``*.jsonl.gz`` regular files under ``root``, in + the order ``resolve_trace_files([root])`` reads them; plain ``*.jsonl`` + files are ignored (logged), as the reconciler ignores them. A directory + or file under ``root`` that vanishes during the walk (ENOENT, or ESTALE + on NFS: the reconciler deletes a dead pod's segments) is skipped; any + other error listing or stat-ing one (EACCES, EIO, ETIMEDOUT on a soft + mount, ...) fails the cut as ``window_cut_failed`` rather than silently + losing a pod's rows, as does any error on ``root`` itself. Each file + comes with whether it is the newest segment (highest ``NNNNNN``) of its + directory, the one its writer may still be appending to. """ try: @@ -650,18 +663,30 @@ def walk_error(error: OSError) -> None: raise _unreadable(error.filename, error) from error found: list[Path] = [] + plain: list[str] = [] for directory, _, names in os.walk(root, onerror=walk_error): for name in names: - if name.endswith(".jsonl") or name.endswith(".jsonl.gz"): - candidate = os.path.join(directory, name) - if _is_regular_file(candidate): - found.append(Path(candidate)) - try: - paths = _reject_duplicate_copies(sorted(found)) - except ValueError as error: - raise WindowCutError("window_cut_failed", str(error)) from error - else: + candidate = os.path.join(directory, name) + if name.endswith(".jsonl"): + plain.append(candidate) + elif name.endswith(SEGMENT_SUFFIX) and _is_regular_file(candidate): + found.append(Path(candidate)) + if plain: + logger.warning( + "retune window cut: ignoring %d plain JSONL files under %s (the " + "live trace sink must write %s segments), e.g. %s", + len(plain), + root, + SEGMENT_SUFFIX, + min(plain), + ) + paths = sorted(found) + elif root.endswith(SEGMENT_SUFFIX): paths = [Path(root)] + else: + raise WindowCutError( + "window_cut_failed", f"trace segment {root} is not a {SEGMENT_SUFFIX} file" + ) newest: dict[Path, tuple[int, str]] = {} for path in paths: index = segment_index(path.name) @@ -692,42 +717,33 @@ def _unreadable(path: Any, error: OSError) -> WindowCutError: ) -@dataclass(frozen=True) -class _PlainBlock: - """Whole lines ``[offset, end)`` of a plain JSONL file. - - ``dropped`` counts the bytes of an unterminated last line that does not - parse (still being written); they are not in ``payload``. - """ - - offset: int - end: int - payload: bytes - dropped: int = 0 - - # (file index, path, newest, item); an OSError item means the file vanished. -_TraceItem = tuple[int, str, bool, Union[_Member, _Garbage, _PlainBlock, OSError]] +_TraceItem = tuple[int, str, bool, Union[_Member, _Garbage, OSError]] def _trace_items( - files: Sequence[tuple[str, bool]], + files: Sequence[tuple[str, bool]], not_before_s: float ) -> Generator[_TraceItem, None, None]: - """Every member, bad byte run and plain block of ``files``, in file order. + """Every member and bad byte run of ``files``, in file order. - A file that vanishes (ENOENT, or ESTALE on NFS) before or while it is - read yields its error and the walk goes on; other I/O errors raise. + A file last modified before ``not_before_s`` (epoch seconds) is skipped. + Its modification time is read once it is open: opening a file on NFS + revalidates its cached attributes (close-to-open consistency). A file + that vanishes (ENOENT, or ESTALE on NFS) before or while it is read + yields its error and the walk goes on; other I/O errors raise. """ for file_index, (path, newest) in enumerate(files): try: with open(path, "rb") as source: - items: Iterator[Union[_Member, _Garbage, _PlainBlock]] = ( - _GzipMemberReader(source) - if path.endswith(".gz") - else _plain_blocks(source) - ) - for item in items: + if os.fstat(source.fileno()).st_mtime < not_before_s: + logger.debug( + "retune window cut: %s was last written before the " + "window; skipping it", + path, + ) + continue + for item in _GzipMemberReader(source): yield file_index, path, newest, item except OSError as error: if error.errno not in _VANISHED: @@ -735,38 +751,8 @@ def _trace_items( yield file_index, path, newest, error -def _plain_blocks(source: BinaryIO) -> Iterator[_PlainBlock]: - """A plain JSONL file as blocks of about ``_PLAIN_BLOCK`` bytes of lines. - - Reading stops at the first block that does not end in a newline: its - last line was the end of the file when read. A writer may finish that - line before the next read, which would then start inside it. - """ - - offset = 0 - while True: - lines = source.readlines(_PLAIN_BLOCK) - if not lines: - return - block = b"".join(lines) - if block.endswith(b"\n"): - yield _PlainBlock(offset, offset + len(block), block) - offset += len(block) - continue - # Only the file's last line can be unterminated. One that does not - # parse is still being written; one that does is a row, as the - # replay reader reads it. - payload, dropped = block, 0 - cut = block.rfind(b"\n") + 1 - tail = block[cut:] - if tail.strip() and not _is_json(tail): - payload, dropped = block[:cut], len(tail) - yield _PlainBlock(offset, offset + len(payload), payload, dropped) - return - - class _Selection: - """Pass 1: the selected rows of every trace file, as columns. + """Pass 1: the selected rows of every trace file read, as columns. Row ``i`` is ``arrival[i]``, ``completion[i]``, ``rid[16*i:16*i+16]`` (BLAKE2b of ``request_id``), line ``line[i]`` of member @@ -786,9 +772,8 @@ def __init__( _DEDUP_SLACK_MIN, int(max_rows * _DEDUP_SLACK_RATIO) ) self.counts = _new_counts() - # (file index, offset, end, is gzip) of each member with a selected - # row; a plain JSONL file is cut into pseudo-members of whole lines. - self.members: list[tuple[int, int, int, bool]] = [] + # (file index, offset, end) of each member with a selected row. + self.members: list[tuple[int, int, int]] = [] self.arrival = array("q") self.completion = array("q") self.rid = bytearray() @@ -799,7 +784,7 @@ def __init__( self.duplicates = 0 self.valid_members = 0 # Runs of bad bytes (``_Garbage``), each counted once however many - # members it held, and their bytes (plus plain-JSONL partial tails). + # members it held, and their bytes. self.dropped_members = 0 self.dropped_bytes = 0 # The runs, and their bytes, that count toward MAX_DROPPED_MEMBER_RATIO. @@ -825,14 +810,8 @@ def scan(self, items: Iterable[_TraceItem]) -> None: self.valid_members += 1 self.gzip_bytes += item.end - item.offset self._scan_payload( - item.payload, file_index, path, item.offset, item.end, True + item.payload, file_index, path, item.offset, item.end ) - elif isinstance(item, _PlainBlock): - self.dropped_bytes += item.dropped - if item.payload: - self._scan_payload( - item.payload, file_index, path, item.offset, item.end, False - ) elif isinstance(item, _Garbage): self._dropped(path, item, newest=newest) else: @@ -917,7 +896,6 @@ def _scan_payload( path: str, offset: int, end: int, - is_gzip: bool, ) -> None: lines = payload.split(b"\n") if not lines[-1]: @@ -943,7 +921,7 @@ def _scan_payload( continue if member_id < 0: member_id = len(self.members) - self.members.append((file_index, offset, end, is_gzip)) + self.members.append((file_index, offset, end)) self._append(row, member_id, index, location) def _append( @@ -1025,14 +1003,6 @@ def _digest(request_id: str) -> bytes: ).digest() -def _is_json(raw: bytes) -> bool: - try: - json.loads(raw) - except ValueError: - return False - return True - - def _event( raw: bytes, location: str, failure: WindowCutFailure ) -> Optional[dict[str, Any]]: @@ -1062,11 +1032,18 @@ def _request_row( # ---------------------------------------------------------------- pass 2 -def _publish(out_path: str, temp_path: str, write: Callable[[_ByteSink], None]) -> bool: +def _publish( + out_path: str, + temp_path: str, + comment: bytes, + write: Callable[[_ByteSink], None], +) -> bool: """Write ``out_path`` atomically through ``temp_path``. - Returns ``False`` when ``out_path`` already existed: it is kept and the - new file is discarded. ``temp_path`` is removed in every case. + The file is one gzip member with ``comment`` in its header and what + ``write`` writes as its data. Returns ``False`` when ``out_path`` already + existed: it is kept and the new file is discarded. ``temp_path`` is + removed in every case. """ directory = os.path.dirname(os.path.abspath(out_path)) @@ -1078,15 +1055,9 @@ def _publish(out_path: str, temp_path: str, write: Callable[[_ByteSink], None]) 0o600, ) with open(descriptor, "wb") as raw: - # No name and no mtime in the header: equal cuts are equal bytes. - with gzip.GzipFile( - filename="", - mode="wb", - fileobj=raw, - compresslevel=OUTPUT_COMPRESSLEVEL, - mtime=0, - ) as sink: - write(sink) + sink = _GzipWriter(raw, comment) + write(sink) + sink.close() raw.flush() os.fsync(raw.fileno()) # Only the owner reads the window. The Run Job reads it as the planner @@ -1128,6 +1099,89 @@ def _unlink_quietly(path: str) -> None: logger.warning("retune window cut: cannot remove %s: %s", path, error) +class _GzipWriter: + """One gzip member: ``_WINDOW_HEADER``, the zero-terminated ``comment``, + the deflated bytes of every ``write`` and the CRC32/ISIZE trailer. + + ``gzip.GzipFile`` cannot write a header comment. + """ + + def __init__(self, raw: BinaryIO, comment: bytes) -> None: + raw.write(_WINDOW_HEADER + comment + b"\x00") + self._raw = raw + self._deflate = zlib.compressobj( + OUTPUT_COMPRESSLEVEL, zlib.DEFLATED, -zlib.MAX_WBITS + ) + self._crc = 0 + self._size = 0 + + def write(self, data: bytes) -> None: + self._crc = zlib.crc32(data, self._crc) + self._size += len(data) + self._raw.write(self._deflate.compress(data)) + + def close(self) -> None: + self._raw.write(self._deflate.flush()) + self._raw.write(struct.pack(" bytes: + """The header comment of a window file: the counts of its cut, as ASCII + JSON (which never holds the NUL that ends a gzip comment).""" + + counts = { + "num_requests": result.num_requests, + "trace_block_size": result.trace_block_size, + "row_counts": result.row_counts, + } + return json.dumps(counts, sort_keys=True, separators=(",", ":")).encode("ascii") + + +def _published(path: str) -> WindowCutResult: + """The result of the window file already at ``path``, from its header. + + Nothing was dropped by this call: it read no segment. A file without + the counts of a cut in its header fails as ``window_cut_failed``. + """ + + with open(path, "rb") as source: + header = source.read(_WINDOW_HEADER_READ) + start = len(_WINDOW_HEADER) + end = header.find(b"\x00", start) + try: + if not header.startswith(_WINDOW_HEADER) or end < 0: + raise ValueError("no header comment") + counts = json.loads(header[start:end]) + num_requests = counts["num_requests"] + block_size = counts["trace_block_size"] + row_counts = counts["row_counts"] + if not ( + _is_count(num_requests) + and num_requests > 0 + and _is_count(block_size) + and isinstance(row_counts, dict) + and all(_is_count(value) for value in row_counts.values()) + ): + raise ValueError(f"malformed counts {counts!r}") + except (ValueError, TypeError, KeyError) as error: + raise WindowCutError( + "window_cut_failed", + f"window file {path} exists but its gzip header holds no cut " + f"counts: {error}", + ) from error + return WindowCutResult( + path=path, + num_requests=num_requests, + trace_block_size=block_size, + row_counts=row_counts, + reused=True, + ) + + +def _is_count(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + def _write_rows(selection: _Selection, paths: Sequence[str], sink: _ByteSink) -> None: """Pass 2: write the kept rows to ``sink`` in window-file order. @@ -1198,10 +1252,7 @@ def _write_rows(selection: _Selection, paths: Sequence[str], sink: _ByteSink) -> raise _changed(f"the row at {location} differs") group.append(parsed) _write_group(group, writer) - writer.close() - except BaseException: - writer.abort() - raise + writer.flush() finally: cache.close() @@ -1220,70 +1271,26 @@ def _changed(detail: str) -> WindowCutError: class _LineWriter: - """Batches window lines and writes them to the gzip sink in a thread. - - zlib and file writes release the GIL, so compressing and writing one - batch overlaps parsing the next; at most ``_WRITE_QUEUE`` batches wait. - ``close`` writes what is left and re-raises the thread's error; - ``abort`` stops it without writing more. - """ + """Joins window lines into writes of about ``_WRITE_BUFFER`` bytes.""" def __init__(self, sink: _ByteSink) -> None: self._sink = sink self._parts: list[str] = [] self._size = 0 - self._queue: queue.Queue[Optional[bytes]] = queue.Queue(maxsize=_WRITE_QUEUE) - self._error: Optional[BaseException] = None - self._aborted = False - self._stopped = False - self._thread = threading.Thread( - target=self._run, name="retune-window-write", daemon=True - ) - self._thread.start() def write(self, line: str) -> None: self._parts.append(line) self._parts.append("\n") self._size += len(line) + 1 if self._size >= _WRITE_BUFFER: - self._send() + self.flush() - def close(self) -> None: - self._send() - self._stop() - if self._error is not None: - raise self._error - - def abort(self) -> None: - self._aborted = True - self._stop() - - def _send(self) -> None: - if self._error is not None: - raise self._error + def flush(self) -> None: if self._parts: - self._queue.put("".join(self._parts).encode("utf-8")) + self._sink.write("".join(self._parts).encode("utf-8")) self._parts = [] self._size = 0 - def _stop(self) -> None: - if not self._stopped: - self._stopped = True - self._queue.put(None) - self._thread.join() - - def _run(self) -> None: - while True: - data = self._queue.get() - if data is None: - return - if self._error is not None or self._aborted: - continue - try: - self._sink.write(data) - except BaseException as error: # noqa: BLE001 - re-raised by close() - self._error = error - class _MemberLines: """The lines pass 2 still has to write, decoded member by member. @@ -1301,7 +1308,7 @@ class _MemberLines: def __init__( self, paths: Sequence[str], - members: Sequence[tuple[int, int, int, bool]], + members: Sequence[tuple[int, int, int]], bounds: np.ndarray, positions: np.ndarray, lines: np.ndarray, @@ -1322,7 +1329,7 @@ def __init__( self.loads = 0 def location(self, member_id: int, line_index: int) -> str: - file_index, offset, _, _ = self._members[member_id] + file_index, offset, _ = self._members[member_id] return f"{self._paths[file_index]}@{offset}:{line_index + 1}" def take(self, member_id: int, line_index: int) -> bytes: @@ -1353,9 +1360,9 @@ def close(self) -> None: self._files.clear() def _load(self, member_id: int) -> dict[int, bytes]: - file_index, offset, end, is_gzip = self._members[member_id] + file_index, offset, end = self._members[member_id] data = self._read(file_index, offset, end) - payload = _decode_exact_member(data) if is_gzip else data + payload = _decode_exact_member(data) if payload is None: raise _changed( f"the gzip member at {self._paths[file_index]}@{offset} " diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py index a86953ff1516..0706330640a2 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py @@ -6,7 +6,9 @@ from __future__ import annotations +import logging import os +import shutil import stat import threading from collections.abc import Callable @@ -118,7 +120,7 @@ def add_request( job_pods: int = 0, job_pod_phase: Optional[str] = None, claim: str = "traces", - path: Optional[str] = None, + sub_path: Optional[str] = None, generation: int = 1, ) -> None: labels = {LABEL_DGD: DGD.name, LABEL_WINDOW: str(window)} @@ -184,11 +186,10 @@ def add_request( "spec": { "workload": { "trace": { - "format": "dynamo", "source": { - "pvc": { + "persistentVolumeClaim": { "claimName": claim, - "path": path or f"windows/dgd/{window}.jsonl.gz", + "subPath": sub_path or f"windows/dgd/{window}.jsonl.gz", } }, } @@ -423,8 +424,9 @@ def test_a_failing_sink_check_keeps_the_segment_counts(tmp_path: Path) -> None: def test_only_the_top_level_segments_of_a_live_pod_are_collected( tmp_path: Path, ) -> None: - # P10 requires the sink's file prefix directly under the pod directory, - # so the reconciler does not walk subdirectories of a live pod. + # The frontend premise check requires the sink's file prefix directly + # under the pod directory, so the reconciler does not walk + # subdirectories of a live pod. reconciler, _, _ = make(tmp_path) pod_dir = Path(reconciler.segments_root) / "fe-0" nested = [touch(pod_dir / "sub" / f"req.00000{i}.jsonl.gz", OLD) for i in range(3)] @@ -596,7 +598,10 @@ def test_a_request_of_another_claim_does_not_reference_the_window( path = touch(window_path(reconciler, window), OLD) kube.add_request("rt-a", window, state="running", claim="other-claim") kube.add_request( - "rt-b", window, state="running", path=f"windows/other-dgd/{window}.jsonl.gz" + "rt-b", + window, + state="running", + sub_path=f"windows/other-dgd/{window}.jsonl.gz", ) # Both Requests are past their deadline and get deleted anyway; the # window had no referrer and is old. @@ -801,7 +806,9 @@ def deletions(report) -> int: ) -def test_should_continue_is_asked_before_every_delete(tmp_path: Path) -> None: +def test_should_continue_is_asked_before_every_delete_and_read( + tmp_path: Path, +) -> None: reconciler, _ = one_of_each(tmp_path) calls: list[int] = [] @@ -817,10 +824,35 @@ def should_continue() -> bool: ) assert deletions(report) == 6 - assert len(calls) == 6 + # Six deletes, the Request listing and rt-a's Run read. + assert len(calls) == 8 assert not report.stopped +@pytest.mark.parametrize(("allowed", "run_reads"), [(2, 1), (4, 3)]) +def test_a_stopped_round_reads_no_more_run_state( + tmp_path: Path, allowed: int, run_reads: int +) -> None: + # Each read can wait for its timeout: once the budget has passed or + # leadership is lost, neither the Request verdicts (allowed=2: the + # listing, then one Run read) nor the window referrers (allowed=4) + # read anything more. + reconciler, kube, _ = make(tmp_path, keep_requests=10) + for k in (4, 5, 6): + kube.add_request(f"rt-{k}", window_ms(k), state="completed") + touch(window_path(reconciler, window_ms(k)), 10) + answers = [True] * allowed + + def should_continue() -> bool: + return answers.pop(0) if answers else False + + report = run_once(reconciler, due=window_ms(0), should_continue=should_continue) + + assert report.stopped + assert len(kube.get_run_calls) == run_reads + assert kube.deleted == [] and report.deleted_windows == () + + def test_a_no_from_should_continue_ends_the_round(tmp_path: Path) -> None: reconciler, kube = one_of_each(tmp_path) answers = [True, True] @@ -864,6 +896,41 @@ def should_continue() -> bool: assert kube.deleted == [] +def test_persistent_conditions_are_logged_when_they_start( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + current = int((NOW - 60) * 1000) + kube.add_request("rt-no-uid", current, dgd_uid=None, state="running") + kube.add_request("rt-no-window", current, state="running") + del kube.requests["rt-no-window"]["metadata"]["labels"][LABEL_WINDOW] + pods = [frontend("fe-0")] + sink = Path(reconciler.segments_root) / "fe-0" + + def warnings() -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING + ] + + with caplog.at_level(logging.WARNING, logger=Reconciler.__module__): + for _ in range(3): + run_once(reconciler, due=current, pods=pods) + assert len(warnings()) == 3 # fe-0's sink, one per missing label + sink.mkdir(parents=True) + run_once(reconciler, due=current, pods=pods) + shutil.rmtree(sink) # missing again: a new occurrence + run_once(reconciler, due=current, pods=pods) + run_once(reconciler, due=current, pods=pods) + + messages = warnings() + assert len(messages) == 4 + assert sum("without a trace directory" in m for m in messages) == 2 + assert sum(LABEL_DGD_UID in m for m in messages) == 1 + assert sum(LABEL_WINDOW in m for m in messages) == 1 + + # --------------------------------------------------------------- set_dgd diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py index 6587cd9afede..a0fcebb78c79 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -15,9 +15,11 @@ import os import pickle import random +import shutil import stat import threading import time +from dataclasses import replace import pytest @@ -25,6 +27,7 @@ from dynamo.planner.retune.live.contracts import WindowCutResult from dynamo.planner.retune.live.window_cut import ( MAX_DROPPED_MEMBER_RATIO, + SEGMENT_MTIME_MARGIN_S, WINDOW_CUT_FAILURES, WindowCutError, _decode_exact_member, @@ -151,7 +154,7 @@ def _temp_files(directory): return [name for name in os.listdir(directory) if name.startswith(".tmp-")] -def _random_trace(root, seed, *, pods=3, segments=3, members=5, plain_pod=False): +def _random_trace(root, seed, *, pods=3, segments=3, members=5): """Segments with every row shape ``read_trace_window`` handles. Arrivals and durations are coarse so (arrival, completion) pairs tie @@ -220,14 +223,10 @@ def _random_trace(root, seed, *, pods=3, segments=3, members=5, plain_pod=False) if rng.random() < 0.1: payload += b"\n \n" # blank lines are skipped chunks.append(payload) - directory = root / f"frontend-{pod}" - if plain_pod and pod == 0: - _write(directory / f"req.{segment:06d}.jsonl", *chunks) - else: - _write( - directory / f"req.{segment:06d}.jsonl.gz", - *(gzip.compress(chunk) for chunk in chunks), - ) + _write( + root / f"frontend-{pod}" / f"req.{segment:06d}.jsonl.gz", + *(gzip.compress(chunk) for chunk in chunks), + ) return root @@ -396,9 +395,12 @@ def test_cut_matches_read_trace_window(tmp_path, seed): assert _temp_files(out.parent) == [] -def test_cut_matches_golden_with_plain_jsonl_files(tmp_path): - root = _random_trace(tmp_path / "segments" / "dgd", 3, plain_pod=True) +def test_plain_jsonl_files_are_ignored(tmp_path): + # The reconciler never collects them, and the live sink writes gzip. + root = _random_trace(tmp_path / "segments" / "dgd", 3) window, golden_bytes = _golden(root, tmp_path) + for pod in ("frontend-0", "frontend-9"): + _write(root / pod / "req.000000.jsonl", _lines([_row("plain", START + 1)])) result, out = _cut(root, tmp_path) @@ -466,8 +468,9 @@ def test_too_many_rows_is_window_too_large(tmp_path, monkeypatch): monkeypatch.setattr(window_cut, "_DEDUP_SLACK_MIN", 1) monkeypatch.setattr(window_cut, "_DEDUP_SLACK_RATIO", 0.0) - result, _ = _cut(root, tmp_path, max_rows=30) + result, out = _cut(root, tmp_path, max_rows=30) assert result.num_requests == 30 + out.unlink() with pytest.raises(WindowCutError) as caught: _cut(root, tmp_path, max_rows=29) assert caught.value.reason == "window_too_large" @@ -712,88 +715,13 @@ def build(root, bad): assert result.dropped_bytes == len(members[50]) -@pytest.mark.parametrize("plain_block", [None, 64]) -def test_plain_jsonl_unterminated_tail(tmp_path, monkeypatch, plain_block): - if plain_block is not None: - # Blocks of one line each: the tail is in a later block. - monkeypatch.setattr(window_cut, "_PLAIN_BLOCK", plain_block) - good = [_row("a", START), _row("b", START + 1)] - - def build(root, bad): - _write( - root / "a" / "req.000000.jsonl", - _lines(good), - *([b'{"schema": "dynamo.req'] if bad else []), - ) - - result = _cut_and_twin(tmp_path, build) - assert result.dropped_bytes == len(b'{"schema": "dynamo.req') - assert result.dropped_members == 0 - - # An unterminated last line that parses is a row, as replay reads it. - root = tmp_path / "parses" - _write( - root / "a" / "req.000000.jsonl", - _lines(good) + json.dumps(_row("c", START + 2)).encode(), - ) - window, golden_bytes = _golden(root, tmp_path) - assert window.num_requests == 3 - out = _out(tmp_path, "parses.jsonl.gz") - result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10) - _assert_golden(result, out, window, golden_bytes) - - -@pytest.mark.parametrize("tail", ["partial", "parses"]) -def test_plain_jsonl_stops_at_the_unterminated_tail(tmp_path, tail): - # The writer finishes the last line (and writes another) right after the - # first read: reading on would start inside that line. - path = tmp_path / "req.000000.jsonl" - first = _lines([_row("a", START)]) - if tail == "partial": - line = _lines([_row("b", START + 1)]) - written, rest = line[:20], line[20:] - else: - written, rest = json.dumps(_row("b", START + 1)).encode(), b"\n" - _write(path, first, written) - - class Appending: - def __init__(self, source): - self.source = source - self.reads = 0 - - def readlines(self, hint): - lines = self.source.readlines(hint) - self.reads += 1 - if self.reads == 1: - with open(path, "ab") as writer: - writer.write(rest + _lines([_row("c", START + 2)])) - return lines - - with open(path, "rb") as source: - reader = Appending(source) - blocks = list(window_cut._plain_blocks(reader)) - - assert reader.reads == 1 - assert len(blocks) == 1 - if tail == "partial": - assert blocks[0].payload == first - assert blocks[0].dropped == len(written) - else: - assert blocks[0].payload == first + written - assert blocks[0].dropped == 0 - assert blocks[0].end == len(blocks[0].payload) - selection = window_cut._Selection(START, END, AS_OF, 10) - selection.scan((0, str(path), True, block) for block in blocks) - assert selection.num_rows == (1 if tail == "partial" else 2) - - def test_segment_files_order_and_newest(tmp_path): root = tmp_path / "segments" for name in ( "b/req.000999.jsonl.gz", "b/req.1000000.jsonl.gz", "a/req.000000.jsonl.gz", - "a/req.000001.jsonl", + "a/req.000001.jsonl", # plain JSONL: ignored "a/notes.txt", "c/single.jsonl.gz", ): @@ -802,22 +730,54 @@ def test_segment_files_order_and_newest(tmp_path): files = _segment_files(str(root)) assert files == [ - (str(root / "a/req.000000.jsonl.gz"), False), - (str(root / "a/req.000001.jsonl"), True), + (str(root / "a/req.000000.jsonl.gz"), True), (str(root / "b/req.000999.jsonl.gz"), False), (str(root / "b/req.1000000.jsonl.gz"), True), (str(root / "c/single.jsonl.gz"), True), ] -def test_compressed_and_plain_copies_of_one_segment_fail(tmp_path): - root = tmp_path / "segments" - _write(root / "a" / "req.000000.jsonl.gz", _member([_row("a", START)])) - _write(root / "a" / "req.000000.jsonl", _lines([_row("a", START)])) +def test_a_segments_root_that_is_not_a_gzip_segment_fails(tmp_path): + plain = _write(tmp_path / "req.000000.jsonl", _lines([_row("a", START)])) with pytest.raises(WindowCutError) as caught: - _cut(root, tmp_path) + _cut(plain, tmp_path) assert caught.value.reason == "window_cut_failed" + single = _write(tmp_path / "req.000000.jsonl.gz", _member([_row("a", START)])) + assert _cut(single, tmp_path)[0].num_requests == 1 + + +def test_segments_last_written_before_the_window_are_not_read(tmp_path): + # A row is written when its request completes, after it arrived: a + # segment last modified before the window started holds none of its + # rows. This one does (it cannot in production) to show it is not read, + # and its hole would fail the cut as trace_corrupt if it were. + start, end = 1_700_000_000_000, 1_700_000_010_000 + root = tmp_path / "segments" + old = _write( + root / "a" / "req.000000.jsonl.gz", + _member([_row("old-0", start + 1)]), + bytes(64), + _member([_row("old-1", start + 2)]), + ) + _write(root / "a" / "req.000001.jsonl.gz", _member([_row("new", start + 3)])) + out = _out(tmp_path) + + def cut(): + return cut_window_inprocess(root, start, end, end + 500, out, max_rows=10) + + skipped = start / 1000 - SEGMENT_MTIME_MARGIN_S - 1 + os.utime(old, (skipped, skipped)) + result = cut() + assert result.num_requests == 1 + assert result.dropped_members == 0 + out.unlink() + + within_margin = start / 1000 - SEGMENT_MTIME_MARGIN_S + 1 + os.utime(old, (within_margin, within_margin)) + with pytest.raises(WindowCutError) as caught: + cut() + assert caught.value.reason == "trace_corrupt" def test_vanished_file_is_skipped(tmp_path, monkeypatch): @@ -922,38 +882,65 @@ def failing(path, *args, **kwargs): # -------------------------------------------------------- atomic output -def test_existing_window_file_is_reused(tmp_path): +def _reused(result, path): + return replace( + result, path=str(path), reused=True, dropped_members=0, dropped_bytes=0 + ) + + +def test_published_window_is_reused_without_reading_the_trace(tmp_path): root = _random_trace(tmp_path / "segments", 5) - window, _ = _golden(root, tmp_path) - out = _out(tmp_path) - _write(out, b"published by another process") + first, out = _cut(root, tmp_path) + published = out.read_bytes() + # A later leader adopting the window must not fail it on segments that + # changed, broke or were collected since it was published. + shutil.rmtree(root) - result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + result, _ = _cut(root, tmp_path) - assert result.reused - assert result.path == str(out) - assert result.num_requests == window.num_requests - assert out.read_bytes() == b"published by another process" + assert result == _reused(first, out) + assert out.read_bytes() == published assert _temp_files(out.parent) == [] -def test_link_race_reuses_the_winner(tmp_path, monkeypatch): +def test_a_window_file_without_cut_counts_fails(tmp_path): + root = _random_trace(tmp_path / "segments", 5) + out = _out(tmp_path) + _write(out, gzip.compress(_lines([_row("a", START)]))) + + with pytest.raises(WindowCutError) as caught: + cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + assert caught.value.reason == "window_cut_failed" + assert "holds no cut counts" in str(caught.value) + + +def test_link_race_reports_the_winners_counts(tmp_path, monkeypatch): root = _random_trace(tmp_path / "segments", 6) + winner_out = _out(tmp_path, "winner.jsonl.gz") + winner = cut_window_inprocess( + _random_trace(tmp_path / "other", 7), + START, + END, + AS_OF, + winner_out, + max_rows=10**6, + ) + assert winner.num_requests != _golden(root, tmp_path)[0].num_requests out = _out(tmp_path) real_link = os.link linked = [] def racing_link(source, target): linked.append(os.path.basename(source)) - _write(out, b"winner") + real_link(winner_out, target) # another process published first return real_link(source, target) monkeypatch.setattr(os, "link", racing_link) result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) - assert result.reused - assert out.read_bytes() == b"winner" + assert result == _reused(winner, out) + assert out.read_bytes() == winner_out.read_bytes() assert temp_file_window_end(linked[0]) == END assert _temp_files(out.parent) == [] @@ -988,9 +975,9 @@ def test_output_is_deterministic_and_0600_under_any_umask(tmp_path): assert stat.S_IMODE(first.stat().st_mode) == 0o600 with gzip.open(first, "rb") as source: assert source.read(1) - # No file name or mtime in the gzip header. + # A comment (the counts) but no file name or mtime in the gzip header. header = first.read_bytes()[:10] - assert header[3] & 0x08 == 0 + assert header[3] == 0x10 assert header[4:8] == b"\x00\x00\x00\x00" @@ -1026,34 +1013,6 @@ def mutate_then_write(selection, paths, sink): assert _temp_files(out.parent) == [] -def test_line_writer_writes_in_order_and_reraises_sink_errors(monkeypatch): - monkeypatch.setattr(window_cut, "_WRITE_BUFFER", 8) - sink = io.BytesIO() - writer = window_cut._LineWriter(sink) - for index in range(100): - writer.write(f"row {index}") - writer.close() - assert sink.getvalue() == b"".join(b"row %d\n" % i for i in range(100)) - - class FullDisk(io.BytesIO): - def write(self, data): - raise OSError(28, "No space left on device") - - writer = window_cut._LineWriter(FullDisk()) - with pytest.raises(OSError, match="No space left"): - for index in range(100): - writer.write(f"row {index}") - writer.close() - writer.abort() # idempotent after a failure - - aborted = io.BytesIO() - writer = window_cut._LineWriter(aborted) - writer.write("x" * 100) - writer.abort() - writer.abort() - assert writer._thread.is_alive() is False - - def test_write_failure_is_window_cut_failed_and_leaves_no_files(tmp_path, monkeypatch): root = _random_trace(tmp_path / "segments", 10) out = _out(tmp_path) @@ -1129,7 +1088,8 @@ def test_temp_and_segment_names(): with pytest.raises(ValueError): temp_file_name(5, "../x") assert segment_index("req.000042.jsonl.gz") == 42 - assert segment_index("req.1000000.jsonl") == 1_000_000 + assert segment_index("req.1000000.jsonl.gz") == 1_000_000 + assert segment_index("req.000042.jsonl") is None assert segment_index("req.jsonl.gz") is None From c0556969ca0a5cf1d362bcc59a0f7cc9cef13bfe Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 09:14:24 -0800 Subject: [PATCH 081/100] fix(planner): address review on the live retune DGDR sweep - Requests name the trace as workload.trace.source.persistentVolumeClaim {claimName, subPath} and leave format to its CRD default (Dynamo); the input read-back finds the trace file with reconciler.trace_pvc_source. - The Run input read-back is the create path's subset check: the input may carry new operator defaults under engine, router, optimizer or the trace source, but must hold every value the Request asks for, and a null (a deleted worker role, an unset SLA or replay speedup) must stay unset. inputOverrides win over the typed trace source; nothing restores it. - Shard planning and the launch gate call planner_config's balanced_shard_parts, largest_shard_points and dgdr_need_seconds instead of a second copy. - Drop the CRN runtime version floor, which nothing set; the Run's runtime image and version are still recorded in the details. - An open Run's Completed reason and message (ContainerWaiting, TraceSourceMissing, JobRunning) become the shard's message and are part of the timeout and run_pending errors. - The wait between polls blocks on the cancel and abort events, so either ends it at once; a small stop flag with is_set() replaces the polling Event subclass handed to the window cutter, whose cancel parameter is typed as any object with is_set() (contracts.SupportsIsSet). - Read typed SweepRequest fields directly; KubernetesRetuneKube requires its api_client; drop the tests of the fakes. - Re-add tests/live_retune_fakes.py (moved here from the configuration PR), lowering the new trace source like the operator. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/retune/live/contracts.py | 9 +- .../src/dynamo/planner/retune/live/dgdr.py | 477 ++++--- .../src/dynamo/planner/retune/live/kube.py | 12 +- .../dynamo/planner/retune/live/window_cut.py | 9 +- .../dynamo/planner/tests/live_retune_fakes.py | 1119 +++++++++++++++++ .../tests/unit/test_live_retune_dgdr.py | 165 ++- .../tests/unit/test_live_retune_kube.py | 135 +- 7 files changed, 1475 insertions(+), 451 deletions(-) create mode 100644 components/src/dynamo/planner/tests/live_retune_fakes.py diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py index 173a6bf159ba..fd3d0347a45a 100644 --- a/components/src/dynamo/planner/retune/live/contracts.py +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -375,6 +375,13 @@ class WindowCutResult: reused: bool = False # the final file already existed (EEXIST) +class SupportsIsSet(Protocol): + """A cancellation flag: a ``threading.Event`` or any object with ``is_set()``.""" + + def is_set(self) -> bool: + ... + + class WindowCutter(Protocol): def __call__( self, @@ -386,7 +393,7 @@ def __call__( *, max_rows: int, deadline_s: float, - cancel: Optional[threading.Event] = None, + cancel: Optional[SupportsIsSet] = None, ) -> WindowCutResult: ... diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py index f7761f6b5621..ca4de57d66f4 100644 --- a/components/src/dynamo/planner/retune/live/dgdr.py +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -3,8 +3,8 @@ """Live retune sweep executor: one window through DGDR v1beta2 Requests. -``run_dgdr_sweep`` is the live ``sweep_fn`` of ``RetuneController`` (design -§4.5). It runs on the controller's single sweep thread and never raises: +``run_dgdr_sweep`` is the live ``sweep_fn`` of ``RetuneController``. It +runs on the controller's single sweep thread and never raises: 1. Shard the router grid (``plan_shards``): balanced Cartesian sub-grids of at most 100 points (a Request's ``maxCandidates`` bound), one @@ -29,8 +29,9 @@ ``SweepRegistry`` deletion queue, which the reconciler drains while this process leads. Shutdown (``cancel``) and loss of leadership (``abort``) return at once with ``failure_reason`` ``shutdown``/``lost_leadership`` and -leave every Request in place for the next leader to adopt. Every mutating -call first checks ``abort`` and ``context.is_leader()``. +leave every Request in place for the next leader to adopt: the wait between +polls ends as soon as either is set. Every mutating call first checks +``abort`` and ``context.is_leader()``. A shard that holds the baseline, or the probe, is critical: when it fails the window fails. Any other failed shard only leaves its points unscored. @@ -53,7 +54,14 @@ from dataclasses import dataclass, field from typing import Any, Literal, NoReturn, Optional -from dynamo.planner.config.planner_config import DgdrExecutorConfig, RetuneConfig +from dynamo.planner.config.planner_config import ( + DGDR_MAX_SHARD_POINTS, + DgdrExecutorConfig, + RetuneConfig, + balanced_shard_parts, + dgdr_need_seconds, + largest_shard_points, +) from dynamo.planner.core.types import RetuneParams from dynamo.planner.retune.live.contracts import ( DGDR_GROUP, @@ -77,6 +85,7 @@ WindowCutResult, WindowCutter, ) +from dynamo.planner.retune.live.reconciler import trace_pvc_source from dynamo.planner.retune.sweep import ( SweepRequest, SweepResult, @@ -88,16 +97,11 @@ ShardKind = Literal["grid", "point", "probe"] -# A Request publishes at most recommendation.maxCandidates (<= 100) points. -MAX_SHARD_POINTS = 100 # Seconds a new Request may go without an Accepted condition before the # window fails as operator_not_reconciling. ACCEPT_TIMEOUT_S = 60.0 # Relative score difference the CRN probe tolerates. CRN_REL_TOLERANCE = 1e-9 -# Oldest Dynamo runtime whose trace replay runs with common random numbers. -# None: the version is not checked and the CRN probe alone decides. -CRN_MIN_VERSION: Optional[str] = None # Annotation holding the cut window's row counts, so a window adopted as # Succeeded reports them without cutting the window again. ROWS_ANNOTATION = LABEL_PREFIX + "rows" @@ -164,13 +168,14 @@ def plan_shards( """Split ``router_domain`` (live knob -> values) into Request shards. The grid becomes at most ``max_shards`` Cartesian sub-grids of at most - ``MAX_SHARD_POINTS`` points: each knob's sorted values are cut into - contiguous, near-equal chunks. The split minimizes the largest shard's + ``DGDR_MAX_SHARD_POINTS`` points: each knob's sorted values are cut into + contiguous, near-equal chunks by ``balanced_shard_parts``, the split + the configuration check (P13) uses. It minimizes the largest shard's wall time, then the shard count: with the Run ``parallelism`` the time - is its evaluation rounds (``1 + ceil((n - 1) / parallelism)``, as in - ``need_seconds``), so a shard that would not shorten the window is not - added; without it, the largest shard's size. ``baseline`` and ``startup`` - (live knob -> value) each get a single-point shard when off the grid. + is its evaluation rounds (``dgdr_rounds``), so a shard that would not + shorten the window is not added; without it, the largest shard's size. + ``baseline`` and ``startup`` (live knob -> value) each get a + single-point shard when off the grid. With ``probe``, a ``probe`` shard repeats one fixed grid point (the middle value of each knob, a positive temperature when the grid has one) to check common random numbers across Runs. Raises ``ValueError`` @@ -197,37 +202,19 @@ def plan_shards( f"router domain {list(keys)}" ) - if parallelism is not None and parallelism < 1: - raise ValueError(f"parallelism must be at least 1, got {parallelism}") sizes = [len(axes[key]) for key in keys] - best: tuple[float, int, int, tuple[int, ...]] | None = None - for splits in _splits(sizes, max_shards): - largest = math.prod( - math.ceil(size / split) for size, split in zip(sizes, splits) - ) - # An oversized shard can never be chosen over one that fits. - cost = ( - math.inf - if largest > MAX_SHARD_POINTS - else largest - if parallelism is None - else _rounds(largest, parallelism) - ) - candidate = (cost, math.prod(splits), largest, splits) - if best is None or candidate < best: - best = candidate - assert best is not None # all-ones always fits max_shards >= 1 - _, _, largest, splits = best - if largest > MAX_SHARD_POINTS: + parts = balanced_shard_parts(sizes, max_shards, parallelism=parallelism) + largest = largest_shard_points(sizes, parts) + if largest > DGDR_MAX_SHARD_POINTS: raise ValueError( f"the router grid of {math.prod(sizes)} points cannot be split into " f"at most {max_shards} Cartesian shards of at most " - f"{MAX_SHARD_POINTS} points (the smallest largest shard has " + f"{DGDR_MAX_SHARD_POINTS} points (the smallest largest shard has " f"{largest}); raise retune.dgdr.run.max_shards or shrink the domain" ) shards: list[Shard] = [] - chunked = [_chunks(axes[key], split) for key, split in zip(keys, splits)] + chunked = [_chunks(axes[key], part) for key, part in zip(keys, parts)] for index, combo in enumerate(itertools.product(*chunked)): choices = dict(zip(keys, combo)) shards.append( @@ -262,26 +249,6 @@ def single(shard_id: str, kind: ShardKind, point: Mapping[str, float]) -> Shard: return shards -def _rounds(points: int, parallelism: int) -> int: - """Evaluation rounds of a Run: the first candidate alone, then batches.""" - - return 1 + math.ceil((points - 1) / parallelism) - - -def _splits(sizes: Sequence[int], max_shards: int) -> Iterable[tuple[int, ...]]: - """Every per-knob split count whose product is at most ``max_shards``.""" - - def walk(index: int, budget: int) -> Iterable[tuple[int, ...]]: - if index == len(sizes): - yield () - return - for split in range(1, min(sizes[index], budget) + 1): - for rest in walk(index + 1, budget // split): - yield (split, *rest) - - return walk(0, max_shards) - - def _chunks(values: tuple[float, ...], count: int) -> list[tuple[float, ...]]: """``values`` cut into ``count`` contiguous chunks whose sizes differ by <= 1.""" @@ -317,22 +284,6 @@ def _probe_point(axes: Mapping[str, tuple[float, ...]]) -> dict[str, float]: return point -def need_seconds(shard: Shard, cfg: DgdrExecutorConfig, *, needs_cut: bool) -> float: - """Wall time a shard needs from now to finish (design §4.1). - - ``(cut_budget if needs_cut) + job_startup + (1 + ceil((n - 1) / - parallelism)) * candidate_timeout``: AISimulate evaluates the first - candidate alone, then ``parallelism`` at a time. - """ - - rounds = _rounds(shard.size, cfg.run.parallelism) - return ( - (cfg.cut_budget_seconds if needs_cut else 0.0) - + cfg.job_startup_seconds - + rounds * cfg.run.candidate_timeout_seconds - ) - - # ----------------------------------------------------------------- Request @@ -385,7 +336,7 @@ def build_request( backend: str, attempt: int, ) -> dict: - """The DGDR v1beta2 Request that replays one shard of a window (§4.5.2). + """The DGDR v1beta2 Request that replays one shard of a window. The engine is ``cfg.engine`` with every leaf a scalar and each used role's ``parallelism.preset[0].replicas`` set from ``spec_replicas`` @@ -398,21 +349,24 @@ def build_request( engine = _pinned_engine(cfg.engine, spec_replicas, backend) size = shard.size - if not 1 <= size <= MAX_SHARD_POINTS: - raise ValueError(f"shard {shard.id} has {size} points; 1..100 fit a Request") + if not 1 <= size <= DGDR_MAX_SHARD_POINTS: + raise ValueError( + f"shard {shard.id} has {size} points; " + f"1..{DGDR_MAX_SHARD_POINTS} fit a Request" + ) spec: dict[str, Any] = { "model": {"name": cfg.model}, "backends": [backend], "hardware": { "gpu": {"sku": cfg.sku, "budget": _gpu_budget(engine, spec_replicas)} }, + # The trace format is left to its CRD default, Dynamo. "workload": { "trace": { - "format": "dynamo", "source": { - "pvc": { + "persistentVolumeClaim": { "claimName": cfg.claim_name, - "path": window_claim_path(cfg, dgd, window_end_ms), + "subPath": window_claim_path(cfg, dgd, window_end_ms), } }, } @@ -553,20 +507,6 @@ def _router_section(shard: Shard) -> dict[str, Any]: # --------------------------------------------------------- input read-back -def merge_patch(target: Any, patch: Any) -> Any: - """RFC 7386 JSON merge patch (the operator's lowering); never mutates.""" - - if not isinstance(patch, Mapping): - return copy.deepcopy(patch) - result = dict(target) if isinstance(target, Mapping) else {} - for key, value in patch.items(): - if value is None: - result.pop(key, None) - else: - result[key] = merge_patch(result.get(key), value) - return result - - def _get(obj: Any, path: Sequence[str], default: Any = None) -> Any: for key in path: if not isinstance(obj, Mapping) or key not in obj: @@ -576,20 +516,13 @@ def _get(obj: Any, path: Sequence[str], default: Any = None) -> Any: def _same_json(left: Any, right: Any) -> bool: - """JSON equality: numbers by value (2 == 2.0), booleans never equal numbers.""" + """JSON equality of leaves: numbers by value (2 == 2.0), booleans never + equal numbers.""" if isinstance(left, bool) or isinstance(right, bool): return isinstance(left, bool) and isinstance(right, bool) and left == right if isinstance(left, (int, float)) and isinstance(right, (int, float)): return float(left) == float(right) - if isinstance(left, Mapping) and isinstance(right, Mapping): - return set(left) == set(right) and all( - _same_json(left[key], right[key]) for key in left - ) - if isinstance(left, list) and isinstance(right, list): - return len(left) == len(right) and all( - _same_json(a, b) for a, b in zip(left, right) - ) return type(left) is type(right) and left == right @@ -600,22 +533,57 @@ def missing_fields(stored: Any, submitted: Any, path: str = "") -> list[str]: requires the stored value to be absent or ``null``. """ + prefix = (path,) if path else () + return [ + ".".join(where) or "." + for where, _, _ in _differences(stored, submitted, prefix) + ] + + +def _differences( + stored: Any, submitted: Any, path: tuple[str, ...] +) -> list[tuple[tuple[str, ...], Any, Any]]: + """``(path, submitted, stored)`` of every submitted value ``stored`` lacks. + + A mapping may hold more keys than submitted, at any depth, including + the mappings of a list of the submitted length. A submitted ``null`` + requires the stored value to be absent or ``null``; ``_ABSENT`` stands + for an absent stored value. + """ + + if submitted is None: + if stored is None or stored is _ABSENT: + return [] + return [(path, None, stored)] if isinstance(submitted, Mapping): if not isinstance(stored, Mapping): - return [path or "."] - problems: list[str] = [] + return [(path, submitted, stored)] + found: list[tuple[tuple[str, ...], Any, Any]] = [] for key, value in submitted.items(): - child = f"{path}.{key}" if path else key - if value is None: - if stored.get(key) is not None: - problems.append(child) - continue - if key not in stored: - problems.append(child) - continue - problems.extend(missing_fields(stored[key], value, child)) - return problems - return [] if _same_json(stored, submitted) else [path or "."] + found += _differences(stored.get(key, _ABSENT), value, (*path, str(key))) + return found + if ( + isinstance(submitted, list) + and isinstance(stored, list) + and len(stored) == len(submitted) + ): + found = [] + for index, (left, right) in enumerate(zip(stored, submitted)): + found += _differences(left, right, (*path, str(index))) + return found + return [] if _same_json(stored, submitted) else [(path, submitted, stored)] + + +def _overlay(target: Any, patch: Any) -> Any: + """``patch`` merged over ``target`` like a merge patch, but a ``null`` in + ``patch`` stays (as "must be unset") instead of deleting the key.""" + + if not isinstance(patch, Mapping): + return copy.deepcopy(patch) + result = dict(target) if isinstance(target, Mapping) else {} + for key, value in patch.items(): + result[key] = _overlay(result.get(key), value) + return result def input_mismatches( @@ -623,94 +591,63 @@ def input_mismatches( ) -> list[str]: """How ``Run.status.aisimulate.input`` differs from what ``spec`` asks for. - The expected input follows the operator's lowering at #17483: the - public role underlay, ``spec.model``/``backends``/``hardware`` into - ``engine``, then ``inputOverrides`` as a merge patch. ``engine``, - ``router`` and ``optimizer`` must match exactly; the objective, SLA, - budget, trace source and the direct generator are checked by value. + The expected input is what the operator lowers ``spec``'s typed fields + to (model, backends, GPU SKU and budget, objective, SLA, the trace file + on its read-only mount, ``maxCandidates``) with ``inputOverrides`` + merged over them. ``inputOverrides`` wins: nothing restores the typed + trace source after it. The check is the create path's subset check + (``missing_fields``), so the input may carry more, such as a new + operator default, but must hold every expected value; an expected + ``null`` (a worker role the overrides delete, an SLA or a replay + speedup the Request leaves unset) must be absent or ``null``. """ - overrides = _get(spec, ("aisimulate", "inputOverrides"), {}) or {} - typed_engine: dict[str, Any] = { - "workers": {role: {} for role in _ROLES}, - "model": _get(spec, ("model", "name")), - "backend": {"choices": list(spec.get("backends") or [])}, - "hardware": _get(spec, ("hardware", "gpu", "sku")), - } - expected_trace = _get(spec, ("workload", "trace", "source", "pvc", "path"), "") + _, sub_path = trace_pvc_source(spec) mounted = TRACE_MOUNT_PATH + ( - "/trace.jsonl.gz" if str(expected_trace).endswith(".gz") else "/trace.jsonl" + "/trace.jsonl.gz" if (sub_path or "").endswith(".gz") else "/trace.jsonl" ) + backends = list(spec.get("backends") or []) sla = _get(spec, ("objective", "sla"), {}) or {} - checks: list[tuple[tuple[str, ...], Any]] = [ - (("engine",), merge_patch(typed_engine, overrides.get("engine") or {})), - (("router",), merge_patch({}, overrides.get("router") or {})), - (("optimizer",), merge_patch({}, overrides.get("optimizer") or {})), - ( - ("optimization", "target"), - _METRIC_TARGETS.get(_get(spec, ("objective", "metric"), ""), _ABSENT), - ), - ( - ("optimization", "strict_sla"), - _get(overrides, ("optimization", "strict_sla")), - ), - ( - ("optimization", "constraints", "max_candidate_gpus"), - _get(spec, ("hardware", "gpu", "budget")), - ), - (("evaluation", "sla", "ttft_ms"), sla.get("ttftMs", _ABSENT)), - (("evaluation", "sla", "itl_ms"), sla.get("itlMs", _ABSENT)), - (("evaluation", "sla", "e2e_ms"), sla.get("e2eMs", _ABSENT)), - ( - ("traffic", "source"), - {"type": "trace", "format": "dynamo", "paths": [mounted]}, - ), - (("traffic", "load"), {"type": "trace_timestamps"}), - (("dgdr_run", "generator"), _get(overrides, ("dgdr_run", "generator"))), - ( - ("dgdr_run", "max_candidates"), - _get(spec, ("recommendation", "maxCandidates")), - ), - ] + typed = { + "engine": { + "model": _get(spec, ("model", "name")), + "backend": {"choices": backends} if backends else None, + "hardware": _get(spec, ("hardware", "gpu", "sku")), + }, + "optimization": { + "target": _METRIC_TARGETS.get(_get(spec, ("objective", "metric"), "")), + "constraints": { + "max_candidate_gpus": _get(spec, ("hardware", "gpu", "budget")) + }, + }, + "evaluation": { + "sla": { + "ttft_ms": sla.get("ttftMs"), + "itl_ms": sla.get("itlMs"), + "e2e_ms": sla.get("e2eMs"), + } + }, + "traffic": { + "source": {"type": "trace", "format": "dynamo", "paths": [mounted]}, + "load": { + "type": "trace_timestamps", + "speedup": _get( + spec, ("workload", "trace", "replay", "arrivalSpeedupRatio") + ), + }, + }, + "dgdr_run": {"max_candidates": _get(spec, ("recommendation", "maxCandidates"))}, + } + overrides = _get(spec, ("aisimulate", "inputOverrides"), {}) or {} # Normalize like the API server (JSON numbers, no tuples). - effective = json.loads(json.dumps(effective)) - problems: list[str] = [] - for path, expected in checks: - expected = expected if expected is _ABSENT else json.loads(json.dumps(expected)) - actual = _get(effective, path, _ABSENT) - if expected is _ABSENT: - if actual is not _ABSENT and actual is not None: - problems.append(f"{'.'.join(path)}: expected unset, got {actual!r}") - elif actual is _ABSENT or not _same_json(actual, expected): - shown = "unset" if actual is _ABSENT else repr(actual) - problems.append(f"{'.'.join(path)}: expected {expected!r}, got {shown}") - return problems - - -def _version_tuple(text: str | None) -> tuple[int, ...] | None: - if not text: - return None - found = re.match(r"^v?(\d+)\.(\d+)(?:\.(\d+))?", text.strip()) - if found is None: - return None - return tuple(int(part) for part in found.groups(default="0")) - - -def crn_unsupported( - runtime_version: str | None, runtime_image: str | None, minimum: str | None -) -> bool: - """Whether a parseable runtime version is below ``minimum`` (None: unknown).""" - - floor = _version_tuple(minimum) - if floor is None: - return False - version = _version_tuple(runtime_version) - if version is None and runtime_image: - # registry/repo:tag@digest -> tag - last = runtime_image.split("@", 1)[0].rsplit("/", 1)[-1] - if ":" in last: - version = _version_tuple(last.split(":", 1)[1]) - return version is not None and version < floor + expected = json.loads(json.dumps(_overlay(typed, overrides))) + actual = json.loads(json.dumps(effective)) + return [ + f"{'.'.join(path)}: expected " + f"{'unset' if want is None else repr(want)}, got " + f"{'unset' if got is _ABSENT else repr(got)}" + for path, want, got in _differences(actual, expected, ()) + ] # ---------------------------------------------------------------- executor @@ -721,9 +658,10 @@ class DgdrContext: """Everything ``run_dgdr_sweep`` needs besides the ``SweepRequest``. ``backend`` is the deployed backend (``vllm``, ``sglang``, ``trtllm``). - ``is_leader`` is checked before every mutating call. ``now``/``sleep`` - are wall-clock (``deadline_s`` is epoch seconds). ``crn_min_version`` is - the oldest runtime that replays traces with common random numbers. + ``is_leader`` is checked before every mutating call. ``now`` is the + wall clock (``deadline_s`` is epoch seconds). Between polls the sweep + waits on ``cancel`` and ``abort``, so either ends the wait at once; + ``sleep``, when set, replaces that wait (a test's simulated clock). """ dgd: DgdRef @@ -735,9 +673,8 @@ class DgdrContext: cut_window: WindowCutter is_leader: Callable[[], bool] now: Callable[[], float] = time.time - sleep: Callable[[float], None] = time.sleep + sleep: Optional[Callable[[float], None]] = None poll_interval_s: float = 5.0 - crn_min_version: Optional[str] = CRN_MIN_VERSION def run_dgdr_sweep( @@ -788,29 +725,45 @@ def __init__( self.message = message -class _EitherEvent(threading.Event): - """An event that reads as set once any of its sources is set. +class _Stop: + """Shutdown (``cancel``) or loss of leadership (``abort``), as one flag. - Hands the window cutter one ``cancel`` for both shutdown and abort. + The window cutter only asks ``is_set()``. ``wait`` is the sweep's pause + between polls and returns as soon as either event is set. """ - def __init__(self, *sources: Optional[threading.Event]) -> None: - super().__init__() - self._sources = [source for source in sources if source is not None] + def __init__( + self, cancel: threading.Event, abort: Optional[threading.Event] + ) -> None: + self._events = tuple(event for event in (cancel, abort) if event is not None) def is_set(self) -> bool: - return super().is_set() or any(source.is_set() for source in self._sources) + return any(event.is_set() for event in self._events) - isSet = is_set + def wait(self, timeout: float) -> bool: + """Block up to ``timeout`` seconds; True once either event is set. - def wait(self, timeout: Optional[float] = None) -> bool: - end = None if timeout is None else time.monotonic() + timeout - while not self.is_set(): - remaining = None if end is None else end - time.monotonic() - if remaining is not None and remaining <= 0: - return False - super().wait(0.05 if remaining is None else min(0.05, remaining)) - return True + A ``threading.Event`` is waited on alone, so with two events each + gets a helper thread that waits on it for the same ``timeout`` and + wakes this call; no thread outlives the timeout. + """ + + if len(self._events) == 1: + return self._events[0].wait(timeout) + if self.is_set(): + return True + woken = threading.Event() + + def watch(event: threading.Event) -> None: + if event.wait(timeout): + woken.set() + + for event in self._events: + threading.Thread( + target=watch, args=(event,), name="retune-dgdr-wait", daemon=True + ).start() + woken.wait(timeout) + return self.is_set() ShardState = Literal["create", "unknown", "wait", "done", "missing"] @@ -833,6 +786,9 @@ class _ShardRun: created: bool = False reason: Optional[str] = None message: Optional[str] = None + # The open Run's Completed reason and message (JobRunning, or why it + # waits: ContainerWaiting, TraceSourceMissing); None once it ends. + run_status: Optional[str] = None run_name: Optional[str] = None run_uid: Optional[str] = None checked_run_uid: Optional[str] = None @@ -855,6 +811,12 @@ def require_name(self) -> str: ) return self.name + def status(self) -> Optional[str]: + """The open Run's status, else why the shard ended or its last + attempt was replaced.""" + + return self.run_status or self.message + def summary(self) -> dict[str, Any]: return { "kind": self.shard.kind, @@ -866,7 +828,7 @@ def summary(self) -> dict[str, Any]: "run": self.run_name, "state": self.state, "reason": self.reason, - "message": self.message, + "message": self.status(), "published": self.published, "run_pending": self.run_pending, } @@ -890,12 +852,14 @@ def __init__( self.request = request self.cancel = cancel self.abort = abort + self.stop = _Stop(cancel, abort) self.ctx = context self.cfg = context.cfg self.started = started - deadline = getattr(request, "deadline_s", None) self.deadline = ( - float(deadline) if deadline is not None else started + request.timeout_s + float(request.deadline_s) + if request.deadline_s is not None + else started + request.timeout_s ) self.window_end_ms = int(round(request.window_end_ms)) # Registry references this sweep took; each is released at the end. @@ -909,8 +873,8 @@ def __init__( self.details: dict[str, Any] = { "window": {"path": self.claim_path, "cut": False}, "deadline_s": self.deadline, - "generation": getattr(request, "generation", 0), - "replicas_verified": getattr(request, "replicas_verified", True), + "generation": request.generation, + "replicas_verified": request.replicas_verified, } self.k8s_errors = 0 self.runtime: dict[str, Any] = {} @@ -999,6 +963,14 @@ def _now(self) -> float: def _timeout(self) -> float: return max(_MIN_K8S_TIMEOUT_S, min(_K8S_TIMEOUT_S, self.deadline - self._now())) + def _pause(self, seconds: float) -> None: + """Wait between polls; shutdown or abort ends the wait at once.""" + + if self.ctx.sleep is not None: + self.ctx.sleep(seconds) + else: + self.stop.wait(seconds) + def _check_interrupt(self) -> None: if self.cancel.is_set(): raise _Interrupted("shutdown") @@ -1029,7 +1001,7 @@ def _retrying(self, what: str, call: Callable[[], Any]) -> Any: if self._now() + self.ctx.poll_interval_s >= self.deadline: self._fail("k8s_error", f"{what}: {error}", "timeout") logger.warning("retune %s failed, retrying: %s", what, error) - self.ctx.sleep(self.ctx.poll_interval_s) + self._pause(self.ctx.poll_interval_s) def _read_failed(self, what: str, error: KubeError) -> None: """A GET failed: a permanent error fails the window; else poll again.""" @@ -1057,7 +1029,7 @@ def _abandon_in_flight(self) -> None: def _prepare(self) -> None: request = self.request - domain = getattr(request, "router_domain", None) + domain = request.router_domain if not domain: self._fail("invalid_request", "the sweep request has no router domain") if any(candidate.planner for candidate in request.candidates): @@ -1077,7 +1049,7 @@ def _prepare(self) -> None: ) except (KeyError, ValueError) as error: self._fail("invalid_domain", str(error)) - replicas = getattr(request, "spec_replicas", None) or request.replicas + replicas = request.spec_replicas or request.replicas self.details["replicas"] = dict(replicas) probe = next((shard for shard in shards if shard.kind == "probe"), None) for shard in shards: @@ -1250,7 +1222,7 @@ def _ordered(self) -> list[_ShardRun]: def _too_late(self, run: _ShardRun) -> bool: """Apply the launch gate; True when the shard was given up.""" - need = need_seconds(run.shard, self.cfg, needs_cut=not self.cut_done) + need = dgdr_need_seconds(run.shard.size, self.cfg, needs_cut=not self.cut_done) left = self.deadline - self._now() if left >= need: return False @@ -1370,7 +1342,6 @@ def _cut(self) -> None: self._fail("launch_too_late", "no time left to cut the window", "timeout") # WindowCutter takes the deadline as absolute wall-clock seconds. cut_deadline_s = self._now() + budget - cancel = _EitherEvent(self.cancel, self.abort) try: os.makedirs(os.path.dirname(self.local_path), exist_ok=True) result: WindowCutResult = self.ctx.cut_window( @@ -1381,7 +1352,7 @@ def _cut(self) -> None: self.local_path, max_rows=self.cfg.max_window_rows, deadline_s=cut_deadline_s, - cancel=cancel, + cancel=self.stop, ) except Exception as error: # A cut cancelled by shutdown or abort reports which one it was. @@ -1423,11 +1394,17 @@ def _poll(self) -> None: return if self._now() >= self.deadline: pending = [run.shard.id for run in active if run.run_pending] + last = [ + f"{run.shard.id}: {run.status()}" + for run in sorted(active, key=lambda run: run.shard.id) + if run.status() + ] self._fail( "run_pending" if pending else "timeout", "the deadline passed with shards " f"{sorted(run.shard.id for run in active)} unfinished" - + (f" (Runs pending: {sorted(pending)})" if pending else ""), + + (f" (Runs pending: {sorted(pending)})" if pending else "") + + (f"; {'; '.join(last)}" if last else ""), "timeout", ) for run in active: @@ -1443,7 +1420,7 @@ def _poll(self) -> None: for run in self.shards.values() ): left = self.deadline - self._now() - self.ctx.sleep(max(0.0, min(self.ctx.poll_interval_s, left))) + self._pause(max(0.0, min(self.ctx.poll_interval_s, left))) def _resolve_unknown(self, run: _ShardRun) -> None: """After a create whose outcome is unknown: adopt it or create again.""" @@ -1480,7 +1457,13 @@ def _shard_failed(self, run: _ShardRun, reason: str, message: str) -> None: ) def _observe(self, run: _ShardRun) -> None: - """One look at a shard's Request and Run (the §4.5.3 state machine).""" + """One look at a shard's Request and Run. + + The Request must be Accepted for its current generation (within + ``ACCEPT_TIMEOUT_S``); its Run's input is read back once per Run + UID; a Succeeded Run makes the shard ``done``, a failed one goes to + ``_run_failed``, and a live one is waited on. + """ kube = self.ctx.kube try: @@ -1531,6 +1514,7 @@ def _observe(self, run: _ShardRun) -> None: f"Run {run.run_name} {completed.get('reason')}: " f"{completed.get('message', '')}" ) + run.run_status = outcome if not terminal and completed.get("reason") else None if not terminal: self._track_pending(run, status) if run.checked_run_uid != run_uid: @@ -1556,8 +1540,10 @@ def _track_pending(self, run: _ShardRun, status: Mapping[str, Any]) -> None: """Flag a live Run that has had no Job for a long time (``run_pending``). The operator writes ``status.aisimulate.input`` and ``jobRef`` in one - status patch once the Job exists; until then the Run has no status - at all, so this runs before the input is read back. + status patch once the Job exists; until then the Run has no status, + or only a ``Completed=False`` condition that says why it waits (such + as ``TraceSourceMissing``), so this runs before the input is read + back. """ if status.get("jobRef") or status.get("startTime"): @@ -1572,17 +1558,18 @@ def _track_pending(self, run: _ShardRun, status: Mapping[str, Any]) -> None: ): if not run.run_pending: logger.warning( - "retune Run %s has no Job after %.0fs (run_pending)", + "retune Run %s has no Job after %.0fs (run_pending): %s", run.run_name, now - run.pending_since, + run.run_status or "no status yet", ) run.run_pending = True def _run_failed(self, run: _ShardRun, outcome: str) -> None: """A Run ended without success. - An attempt a previous leader created is replaced while there is time - (design §4.5.3 adopts only a live or Succeeded attempt); otherwise + An attempt a previous leader created is replaced while there is time, + since only a live or Succeeded attempt is worth adopting; otherwise the shard fails. """ @@ -1636,17 +1623,12 @@ def _check_input(self, run: _ShardRun, effective: Mapping[str, Any]) -> bool: image = dgdr_run.get("runtime_image") version = dgdr_run.get("runtime_version_override") if image or version: + # For audit: the runtime that replayed the window. self.runtime = {"runtime_image": image, "runtime_version": version} problems = input_mismatches(run.spec, effective) if problems: self._input_mismatch(run, problems) return False - if crn_unsupported(version, image, self.ctx.crn_min_version): - self._fail( - "crn_unsupported", - f"Run {run.run_name} runtime {version or image} predates CRN " - f"trace replay ({self.ctx.crn_min_version})", - ) return True def _input_mismatch(self, run: _ShardRun, problems: list[str]) -> None: @@ -1669,7 +1651,7 @@ def _replace_attempt(self, run: _ShardRun, message: str) -> bool: False (nothing changed) when ``D_k - now < need(shard)``. """ - need = need_seconds(run.shard, self.cfg, needs_cut=not self.cut_done) + need = dgdr_need_seconds(run.shard.size, self.cfg, needs_cut=not self.cut_done) if self.deadline - self._now() < need: return False self._enqueue_delete([run.name]) @@ -1679,6 +1661,7 @@ def _replace_attempt(self, run: _ShardRun, message: str) -> bool: run.created = False run.name = None run.run_name = None + run.run_status = None run.checked_run_uid = None run.unaccepted_since = None run.pending_since = None @@ -1690,7 +1673,7 @@ def _replace_attempt(self, run: _ShardRun, message: str) -> bool: def _harvest(self) -> SweepResult: request = self.request - domain_keys = set(getattr(request, "router_domain", None) or {}) + domain_keys = set(request.router_domain or {}) results: dict[str, _Scored] = {} probe_results: dict[str, _Scored] = {} engine_signature: Optional[str] = None diff --git a/components/src/dynamo/planner/retune/live/kube.py b/components/src/dynamo/planner/retune/live/kube.py index 359d3aa92f98..74f82c93d28e 100644 --- a/components/src/dynamo/planner/retune/live/kube.py +++ b/components/src/dynamo/planner/retune/live/kube.py @@ -149,18 +149,18 @@ def _decode(response: Any) -> dict: class KubernetesRetuneKube: - """``contracts.RetuneKube`` for one namespace. + """``contracts.RetuneKube`` for one namespace over ``api_client``. - ``api_client`` defaults to a new ``ApiClient`` on the default - configuration, which ``KubernetesAPI`` loads (in-cluster or kubeconfig); - ``from_kube_api`` reuses that connector's client instead. + The client must carry a loaded configuration (in-cluster or kubeconfig): + a bare ``ApiClient()`` targets localhost. ``from_kube_api`` reuses the + planner connector's client, which ``KubernetesAPI`` configured. """ - def __init__(self, namespace: str, api_client: Optional[client.ApiClient] = None): + def __init__(self, namespace: str, api_client: client.ApiClient): if not namespace: raise ValueError("namespace must be set") self.namespace = namespace - self.api_client = api_client if api_client is not None else client.ApiClient() + self.api_client = api_client self._custom = client.CustomObjectsApi(self.api_client) self._core = client.CoreV1Api(self.api_client) self._coordination = client.CoordinationV1Api(self.api_client) diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py index 96d485a6775c..6a548d52a6bc 100644 --- a/components/src/dynamo/planner/retune/live/window_cut.py +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -69,7 +69,6 @@ import secrets import stat import struct -import threading import time import zlib from array import array @@ -83,7 +82,7 @@ import numpy as np -from dynamo.planner.retune.live.contracts import WindowCutResult +from dynamo.planner.retune.live.contracts import SupportsIsSet, WindowCutResult from dynamo.planner.retune.window import ( _dumps, _new_counts, @@ -209,7 +208,7 @@ def cut_window( *, max_rows: int, deadline_s: float, - cancel: Optional[threading.Event] = None, + cancel: Optional[SupportsIsSet] = None, ) -> WindowCutResult: """Cut a window file in a spawned child process (``contracts.WindowCutter``). @@ -356,7 +355,7 @@ def _run_in_child( kwargs: Mapping[str, Any], *, deadline: float, - cancel: Optional[threading.Event], + cancel: Optional[SupportsIsSet], cleanup: Sequence[str] = (), name: str = "retune-window-cut", ) -> Any: @@ -416,7 +415,7 @@ def _run_in_child( raise WindowCutError(reason, message) -def _check_interrupted(deadline: float, cancel: Optional[threading.Event]) -> None: +def _check_interrupted(deadline: float, cancel: Optional[SupportsIsSet]) -> None: if cancel is not None and cancel.is_set(): raise WindowCutError("cancelled", "window cut cancelled") if time.monotonic() >= deadline: diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py new file mode 100644 index 000000000000..b1cd13784d0d --- /dev/null +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -0,0 +1,1119 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""In-memory fakes of the Kubernetes side of live retune. + +``FakeRetuneKube`` implements ``contracts.RetuneKube`` over plain dicts with +the API-server behaviour the live modules depend on: ``resourceVersion`` +compare-and-swap on Lease replace, ``AlreadyExists`` on create, Strict field +validation and silent pruning on Request create, label selectors, cascading +Request deletion, and injectable faults (including writes that time out but +land). ``FakeDgdrOperator`` drives the Requests in a ``FakeRetuneKube`` +through the DGDR v1beta2 lifecycle (Accepted -> Run -> Completed with +Candidates) the way the operator at #17483 does, lowering each Request into +``Run.status.aisimulate.input`` with the same merge-patch rules. + +This is a plain module, not a conftest: tests import what they need. +""" + +from __future__ import annotations + +import copy +import functools +import itertools +import json +import re +import threading +import time +import uuid +from collections import deque +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field, replace +from typing import Any, Literal, Optional + +from dynamo.planner.retune.live.contracts import ( + DGDR_ROUTER_KNOBS, + POD_LABEL_COMPONENT_TYPE, + POD_LABEL_DGD_NAME, + RUN_UID_LABEL, + KubeAlreadyExists, + KubeConflict, + KubeError, + KubeInvalid, + KubeNotFound, + PodInfo, +) + +# Operator constants (deploy/operator/internal/controller at #17483). +REQUEST_GENERATION_ANNOTATION = "nvidia.com/dgdr-request-generation" +CANDIDATE_ID_LABEL = "nvidia.com/dgdr-candidate-id" +TRACE_MOUNT_PATH = "/var/run/aisimulate/trace" +DEFAULT_RUNTIME_IMAGE = "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.8.0" +DEFAULT_DYNAMO_VERSION = "1.8.0" +ROUTER_ADAPTER = "dynamo.router" +# spec.workload.trace.format: the CRD default, and each API value's +# AISimulate traffic.source.format spelling. +DEFAULT_TRACE_FORMAT = "Dynamo" +_TRACE_FORMATS = {"Dynamo": "dynamo"} + +_METRIC_TARGETS = { + "Throughput": "throughput", + "ThroughputPerGPU": "throughput_per_gpu", + "ThroughputPerUser": "throughput_per_user", + "TTFT": "ttft", + "E2ELatency": "e2e_latency", + "Goodput": "goodput", + "GoodputPerGPU": "goodput_per_gpu", +} + +KubeMethod = Literal[ + "get_dgd", + "list_pods", + "create_request", + "get_request", + "list_requests", + "delete_request", + "get_run", + "list_candidates", + "get_lease", + "create_lease", + "replace_lease", + "delete_lease", +] + + +# ------------------------------------------------------------------ helpers + + +def merge_patch(target: Any, patch: Any) -> Any: + """RFC 7386 JSON merge patch; returns a new value, never mutates.""" + + if not isinstance(patch, Mapping): + return copy.deepcopy(patch) + result = dict(target) if isinstance(target, Mapping) else {} + for key, value in patch.items(): + if value is None: + result.pop(key, None) + else: + result[key] = merge_patch(result.get(key), value) + return result + + +def nested_get(obj: Any, *path: str, default: Any = None) -> Any: + for key in path: + if not isinstance(obj, Mapping) or key not in obj: + return default + obj = obj[key] + return obj + + +def _nested_set(obj: dict, path: tuple[str, ...], value: Any) -> None: + for key in path[:-1]: + child = obj.get(key) + if not isinstance(child, dict): + child = {} + obj[key] = child + obj = child + obj[path[-1]] = value + + +def _nested_pop(obj: Any, path: tuple[str, ...]) -> bool: + for key in path[:-1]: + if not isinstance(obj, dict) or key not in obj: + return False + obj = obj[key] + if isinstance(obj, dict) and path[-1] in obj: + del obj[path[-1]] + return True + return False + + +def _split(path: str) -> tuple[str, ...]: + return tuple(path.split(".")) + + +def rfc3339(epoch_s: float) -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch_s)) + + +_SELECTOR_TERM = re.compile( + r"^\s*(!?)\s*([A-Za-z0-9._/-]+)\s*(?:(==|=|!=)\s*(\S*))?\s*$" +) + + +def match_labels(labels: Mapping[str, str] | None, selector: str) -> bool: + """Equality-based label selector: ``k=v``, ``k==v``, ``k!=v``, ``k``, ``!k``.""" + + labels = labels or {} + if not selector.strip(): + return True + for term in selector.split(","): + found = _SELECTOR_TERM.match(term) + if found is None: + raise ValueError(f"unsupported label selector term {term!r}") + negate, key, op, value = found.groups() + if op is None: + present = key in labels + if present == bool(negate): + return False + elif negate: + raise ValueError(f"unsupported label selector term {term!r}") + elif op in ("=", "=="): + if labels.get(key) != value: + return False + elif labels.get(key) == value: # != + return False + return True + + +def lower_request_input( + request: Mapping[str, Any], + *, + run_name: str, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + gpus_per_node: int = 8, +) -> dict[str, Any]: + """``Run.status.aisimulate.input`` for a Request, as the operator lowers it. + + Mirrors ``aisimulate.LowerWithUnderlay`` and ``dgdrRunInput`` at #17483: + the public role underlay, the ``dgdr_run`` static underlay, the typed + underlay, the derived defaults, ``inputOverrides`` as the final merge + patch, then the operator-owned ``dgdr_run`` fields. ``inputOverrides`` + wins over the typed trace source: nothing restores it afterwards. + """ + + spec = request.get("spec") or {} + max_candidates = int(nested_get(spec, "recommendation", "maxCandidates", default=1)) + underlay: dict[str, Any] = { + "engine": {"workers": {"aggregated": {}, "prefill": {}, "decode": {}}} + } + underlay = merge_patch( + underlay, {"dgdr_run": {"name": run_name, "max_candidates": max_candidates}} + ) + typed: dict[str, Any] = {} + model = nested_get(spec, "model", "name") + if model is not None: + _nested_set(typed, ("engine", "model"), model) + if spec.get("backends"): + _nested_set(typed, ("engine", "backend", "choices"), list(spec["backends"])) + sku = nested_get(spec, "hardware", "gpu", "sku") + if sku: + _nested_set(typed, ("engine", "hardware"), sku) + budget = nested_get(spec, "hardware", "gpu", "budget") + if budget is not None: + _nested_set( + typed, ("optimization", "constraints", "max_candidate_gpus"), int(budget) + ) + trace = nested_get(spec, "workload", "trace") + if trace is not None: + sub_path = nested_get( + trace, "source", "persistentVolumeClaim", "subPath", default="" + ) + mounted = TRACE_MOUNT_PATH + ( + "/trace.jsonl.gz" if sub_path.endswith(".gz") else "/trace.jsonl" + ) + api_format = trace.get("format", DEFAULT_TRACE_FORMAT) + if api_format not in _TRACE_FORMATS: + raise ValueError(f"unsupported trace format {api_format!r}") + trace_source = { + "type": "trace", + "format": _TRACE_FORMATS[api_format], + "paths": [mounted], + } + _nested_set(typed, ("traffic", "source"), trace_source) + _nested_set(typed, ("traffic", "load", "type"), "trace_timestamps") + speedup = nested_get(trace, "replay", "arrivalSpeedupRatio") + if speedup is not None: + _nested_set(typed, ("traffic", "load", "speedup"), speedup) + objective = spec.get("objective") or {} + if objective.get("metric") is not None: + _nested_set( + typed, ("optimization", "target"), _METRIC_TARGETS[objective["metric"]] + ) + sla = objective.get("sla") or {} + for api_key, input_key in ( + ("ttftMs", "ttft_ms"), + ("itlMs", "itl_ms"), + ("e2eMs", "e2e_ms"), + ): + if sla.get(api_key) is not None: + _nested_set(typed, ("evaluation", "sla", input_key), sla[api_key]) + underlay = merge_patch(underlay, typed) + if "traffic" in underlay: + source_type = nested_get(underlay, "traffic", "source", "type") + if source_type != "trace" and nested_get(underlay, "traffic", "stop") is None: + _nested_set(underlay, ("traffic", "stop", "requests"), 1000.0) + if nested_get(underlay, "optimizer", "max_trials") is None: + _nested_set(underlay, ("optimizer", "max_trials"), 320.0) + if nested_get(underlay, "evaluation", "sla") and ( + nested_get(underlay, "optimization", "strict_sla") is None + ): + _nested_set(underlay, ("optimization", "strict_sla"), True) + overrides = nested_get(spec, "aisimulate", "inputOverrides", default={}) or {} + effective = merge_patch(underlay, overrides) + reserved = { + "name": run_name, + "max_candidates": max_candidates, + "snapshot_dir": "/var/run/aisimulate/output", + "namespace": nested_get(request, "metadata", "namespace"), + "runtime_image": runtime_image, + "runtime_version_override": dynamo_version, + "derive_runtime_images": True, + "num_gpus_per_node": gpus_per_node, + } + for key, value in reserved.items(): + _nested_set(effective, ("dgdr_run", key), value) + # Round-trip through JSON like the API server does. + return json.loads(json.dumps(effective)) + + +def router_points(effective_input: Mapping[str, Any]) -> list[dict[str, float]]: + """The router points (public knob names) an input's router section spans.""" + + router = effective_input.get("router") or {} + axes: list[tuple[str, list[float]]] = [] + for public in DGDR_ROUTER_KNOBS: + value = router.get(public) + if isinstance(value, Mapping): + axes.append((public, [float(v) for v in value["choices"]])) + elif value is not None: + axes.append((public, [float(value)])) + return [ + dict(zip([name for name, _ in axes], values)) + for values in itertools.product(*(values for _, values in axes)) + ] + + +def engine_parameters(effective_input: Mapping[str, Any]) -> dict[str, Any]: + """The non-router parameters every Candidate of a pinned engine shares.""" + + engine = effective_input.get("engine") or {} + workers = engine.get("workers") or {} + params: dict[str, Any] = { + "backend": engine.get("backend"), + "deployment_mode": "disagg" if engine.get("mode") == "disaggregated" else "agg", + "model": engine.get("model"), + "hardware": engine.get("hardware"), + } + for role, worker in sorted(workers.items()): + if isinstance(worker, Mapping): + preset = nested_get(worker, "parallelism", "preset", default=[{}]) + params[f"{role}_parallelism"] = dict(preset[0]) if preset else {} + return json.loads(json.dumps(params)) + + +def candidate_parameters( + effective_input: Mapping[str, Any], point_public: Mapping[str, float] +) -> dict[str, Any]: + """``Candidate.spec.parameters`` of one evaluated router point.""" + + router = effective_input.get("router") or {} + return { + **engine_parameters(effective_input), + "adapters": { + ROUTER_ADAPTER: { + "policy": router.get("policy", "kv_router"), + "prefill_load_model": dict( + router.get("prefill_load_model") or {"type": "none"} + ), + **{name: float(value) for name, value in point_public.items()}, + } + }, + } + + +def default_metrics(point_public: Mapping[str, float]) -> dict[str, float]: + """Deterministic replay summary metrics of a router point (for tests).""" + + credit = point_public.get("overlap_score_credit", 1.0) + scale = point_public.get("prefill_load_scale", 1.0) + temperature = point_public.get("temperature", 0.0) + goodput = 1000.0 + 100.0 * credit - 3.0 * abs(scale - 2.0) - 10.0 * temperature + return { + "goodput_output_throughput_tok_s": goodput, + "output_throughput_tok_s": goodput * 1.1, + "gpu_hours": 2.0, + "duration_ms": 3_600_000.0, + "mean_ttft_ms": 100.0 + scale, + "num_ttft_samples": 1000.0, + "mean_e2e_latency_ms": 1000.0 + scale, + "num_e2e_latency_samples": 1000.0, + "mean_output_token_throughput_per_user": 50.0 + credit, + } + + +# ------------------------------------------------------------------- faults + + +@dataclass +class Fault: + """One injected failure of a ``FakeRetuneKube`` method. + + ``landed=True`` performs the call and then raises (a write that timed + out but reached the API server). ``match`` limits the fault to calls + whose first positional argument (a name, selector or body) satisfies it. + """ + + error: Exception + times: int = 1 + landed: bool = False + match: Optional[Callable[[Any], bool]] = None + + +@dataclass +class KubeCall: + method: str + target: Any + kwargs: dict[str, Any] = field(default_factory=dict) + + +# --------------------------------------------------------------------- kube + + +class FakeRetuneKube: + """In-memory ``contracts.RetuneKube`` (thread-safe). + + Stores DGDs, pods (``PodInfo``), Requests, Runs, Candidates and Leases + as plain dicts. Objects come back as deep copies, so callers can never + alias the store. ``now`` stamps ``creationTimestamp``. + + Request create honours ``field_validation='Strict'``: a path listed in + ``unknown_fields`` (dotted, e.g. ``spec.workload.trace``) is rejected + with ``KubeInvalid``; a path in ``pruned_fields`` is dropped silently + (an API server that ignores the validation directive). Like the CRD + schema, create defaults ``spec.workload.trace.format`` to ``Dynamo``. + ``reject_create`` rejects every create with ``KubeInvalid``. + ``on_request_created`` hooks run after each stored (non dry-run) create. + """ + + def __init__( + self, + namespace: str = "test-ns", + *, + now: Callable[[], float] = time.time, + ) -> None: + self.namespace = namespace + self.now = now + self.dgds: dict[str, dict] = {} + self.pods: dict[str, PodInfo] = {} + self.requests: dict[str, dict] = {} + self.runs: dict[str, dict] = {} + # Each Run's lowered input, recorded in its status by ``create_job``. + self.run_inputs: dict[str, dict] = {} + self.candidates: dict[str, dict] = {} + self.leases: dict[str, dict] = {} + self.calls: list[KubeCall] = [] + self.deleted_requests: list[tuple[str, str]] = [] + self.unknown_fields: set[str] = set() + self.pruned_fields: set[str] = set() + self.reject_create: Optional[str] = None + self.on_request_created: list[Callable[[FakeRetuneKube, dict], None]] = [] + self.cascade_deletes = True + self._faults: dict[str, deque[Fault]] = {} + self._rv = itertools.count(1) + self._lock = threading.RLock() + + # ------------------------------------------------------------ faults + + def fail( + self, + method: KubeMethod, + error: Exception | None = None, + *, + times: int = 1, + landed: bool = False, + match: Optional[Callable[[Any], bool]] = None, + ) -> None: + """Make the next ``times`` matching calls of ``method`` raise ``error``.""" + + with self._lock: + self._faults.setdefault(method, deque()).append( + Fault( + error=error or KubeError(f"injected {method} failure"), + times=times, + landed=landed, + match=match, + ) + ) + + def clear_faults(self) -> None: + with self._lock: + self._faults.clear() + + def calls_of(self, method: str) -> list[KubeCall]: + with self._lock: + return [call for call in self.calls if call.method == method] + + def _call( + self, + method: str, + target: Any, + kwargs: dict[str, Any], + operation: Callable[[], Any], + ) -> Any: + with self._lock: + self.calls.append(KubeCall(method, copy.deepcopy(target), dict(kwargs))) + fault = self._take_fault(method, target) + if fault is not None and not fault.landed: + raise fault.error + result = operation() + if fault is not None: + raise fault.error + return result + + def _take_fault(self, method: str, target: Any) -> Optional[Fault]: + queue = self._faults.get(method) + if not queue: + return None + for fault in list(queue): + if fault.match is None or fault.match(target): + fault.times -= 1 + if fault.times <= 0: + queue.remove(fault) + return fault + return None + + def _stamp(self, obj: dict, *, kind: str, api_version: str) -> dict: + metadata = obj.setdefault("metadata", {}) + obj.setdefault("apiVersion", api_version) + obj.setdefault("kind", kind) + metadata.setdefault("namespace", self.namespace) + metadata.setdefault("uid", str(uuid.uuid4())) + metadata.setdefault("creationTimestamp", rfc3339(self.now())) + metadata.setdefault("generation", 1) + metadata["resourceVersion"] = str(next(self._rv)) + return obj + + def bump(self, obj: dict) -> None: + """Give a stored object a new ``resourceVersion``.""" + + obj["metadata"]["resourceVersion"] = str(next(self._rv)) + + # --------------------------------------------------------------- DGD + + def put_dgd(self, dgd: dict) -> dict: + with self._lock: + stored = copy.deepcopy(dgd) + self._stamp( + stored, kind="DynamoGraphDeployment", api_version="nvidia.com/v1beta1" + ) + self.dgds[stored["metadata"]["name"]] = stored + return copy.deepcopy(stored) + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + if name not in self.dgds: + raise KubeNotFound(f"DynamoGraphDeployment {name!r} not found") + return copy.deepcopy(self.dgds[name]) + + return self._call("get_dgd", name, {"timeout_s": timeout_s}, operation) + + # -------------------------------------------------------------- pods + + def add_pod( + self, + name: str, + *, + dgd: str | None = None, + component_type: str = "frontend", + ready: bool = True, + terminating: bool = False, + ip: str | None = None, + http_port: int | None = 8000, + labels: Mapping[str, str] | None = None, + ) -> PodInfo: + all_labels = dict(labels or {}) + if dgd is not None: + all_labels.setdefault(POD_LABEL_DGD_NAME, dgd) + if component_type: + all_labels.setdefault(POD_LABEL_COMPONENT_TYPE, component_type) + pod = PodInfo( + name=name, + uid=str(uuid.uuid4()), + ip=ip if ip is not None else f"10.0.0.{len(self.pods) + 1}", + ready=ready, + terminating=terminating, + labels=all_labels, + http_port=http_port, + ) + with self._lock: + self.pods[name] = pod + return pod + + def update_pod(self, name: str, **changes: Any) -> PodInfo: + with self._lock: + pod = replace(self.pods[name], **changes) + self.pods[name] = pod + return pod + + def remove_pod(self, name: str) -> None: + with self._lock: + self.pods.pop(name, None) + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + def operation() -> list[PodInfo]: + return [ + pod + for pod in self.pods.values() + if match_labels(pod.labels, label_selector) + ] + + return self._call( + "list_pods", label_selector, {"timeout_s": timeout_s}, operation + ) + + # ----------------------------------------------------------- Requests + + def create_request( + self, + body: dict, + *, + dry_run: bool = False, + timeout_s: float = 10.0, + ) -> dict: + def operation() -> dict: + if self.reject_create is not None: + raise KubeInvalid(self.reject_create) + stored = copy.deepcopy(body) + for path in sorted(self.unknown_fields): + if nested_get(stored, *_split(path)) is not None: + raise KubeInvalid(f'strict decoding error: unknown field "{path}"') + for path in self.pruned_fields: + _nested_pop(stored, _split(path)) + trace = nested_get(stored, "spec", "workload", "trace") + if isinstance(trace, dict): + trace.setdefault("format", DEFAULT_TRACE_FORMAT) + name = nested_get(stored, "metadata", "name") + if not name: + raise KubeInvalid("metadata.name is required") + if name in self.requests: + raise KubeAlreadyExists( + f"dynamographdeploymentrequests {name!r} already exists" + ) + self._stamp( + stored, + kind="DynamoGraphDeploymentRequest", + api_version="nvidia.com/v1beta2", + ) + if dry_run: + return stored + self.requests[name] = stored + for hook in list(self.on_request_created): + hook(self, copy.deepcopy(stored)) + return copy.deepcopy(stored) + + return self._call( + "create_request", + body, + {"dry_run": dry_run, "timeout_s": timeout_s}, + operation, + ) + + def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.requests.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_request", name, {"timeout_s": timeout_s}, operation) + + def list_requests( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + def operation() -> list[dict]: + return [ + copy.deepcopy(obj) + for obj in self.requests.values() + if match_labels(obj["metadata"].get("labels"), label_selector) + ] + + return self._call( + "list_requests", label_selector, {"timeout_s": timeout_s}, operation + ) + + def delete_request( + self, + name: str, + *, + propagation: Literal["Background", "Foreground"] = "Background", + timeout_s: float = 10.0, + ) -> None: + def operation() -> None: + request = self.requests.pop(name, None) + if request is None: + return + self.deleted_requests.append((name, propagation)) + if not self.cascade_deletes: + return + request_uid = request["metadata"]["uid"] + for run_name, run in list(self.runs.items()): + owners = run["metadata"].get("ownerReferences") or [] + if any(owner.get("uid") == request_uid for owner in owners): + self._delete_run(run_name) + + return self._call( + "delete_request", + name, + {"propagation": propagation, "timeout_s": timeout_s}, + operation, + ) + + # --------------------------------------------------- Runs, Candidates + + def _delete_run(self, run_name: str) -> None: + run = self.runs.pop(run_name, None) + self.run_inputs.pop(run_name, None) + if run is None: + return + run_uid = run["metadata"]["uid"] + for candidate_name, candidate in list(self.candidates.items()): + labels = candidate["metadata"].get("labels") or {} + if labels.get(RUN_UID_LABEL) == run_uid: + del self.candidates[candidate_name] + + def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.runs.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_run", name, {"timeout_s": timeout_s}, operation) + + def list_candidates( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + def operation() -> list[dict]: + return [ + copy.deepcopy(obj) + for obj in self.candidates.values() + if match_labels(obj["metadata"].get("labels"), label_selector) + ] + + return self._call( + "list_candidates", label_selector, {"timeout_s": timeout_s}, operation + ) + + # ------------------------------------------------------------- Leases + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.leases.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_lease", name, {"timeout_s": timeout_s}, operation) + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + stored = copy.deepcopy(body) + name = nested_get(stored, "metadata", "name") + if not name: + raise KubeInvalid("metadata.name is required") + if name in self.leases: + raise KubeAlreadyExists(f"leases {name!r} already exists") + stored["metadata"].pop("resourceVersion", None) + self._stamp(stored, kind="Lease", api_version="coordination.k8s.io/v1") + self.leases[name] = stored + return copy.deepcopy(stored) + + return self._call("create_lease", body, {"timeout_s": timeout_s}, operation) + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + current = self.leases.get(name) + if current is None: + raise KubeNotFound(f"leases {name!r} not found") + sent = nested_get(body, "metadata", "resourceVersion") + if sent != current["metadata"]["resourceVersion"]: + raise KubeConflict( + f"leases {name!r}: resourceVersion {sent!r} is stale " + f"(current {current['metadata']['resourceVersion']!r})" + ) + stored = copy.deepcopy(body) + metadata = stored.setdefault("metadata", {}) + for key in ("uid", "creationTimestamp", "namespace", "generation"): + metadata[key] = current["metadata"].get(key) + metadata["name"] = name + stored.setdefault("apiVersion", current.get("apiVersion")) + stored.setdefault("kind", current.get("kind")) + self.bump(stored) + self.leases[name] = stored + return copy.deepcopy(stored) + + return self._call( + "replace_lease", (name, body), {"timeout_s": timeout_s}, operation + ) + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + def operation() -> None: + self.leases.pop(name, None) + + return self._call("delete_lease", name, {"timeout_s": timeout_s}, operation) + + # ------------------------------------------------- operator simulation + + def set_request_condition( + self, + name: str, + *, + status: str, + reason: str, + message: str = "", + observed_generation: int | None = None, + ) -> None: + """Set the Request's ``Accepted`` condition.""" + + with self._lock: + request = self.requests[name] + generation = request["metadata"].get("generation", 1) + conditions = [ + condition + for condition in request.setdefault("status", {}).get("conditions", []) + if condition.get("type") != "Accepted" + ] + conditions.append( + { + "type": "Accepted", + "status": status, + "reason": reason, + "message": message, + "observedGeneration": ( + generation + if observed_generation is None + else observed_generation + ), + "lastTransitionTime": rfc3339(self.now()), + } + ) + request["status"]["conditions"] = conditions + self.bump(request) + + def accept_request( + self, + name: str, + *, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + mutate_input: Optional[Callable[[dict], None]] = None, + ) -> dict: + """Create the Request's Run (``-``) and mark it Accepted. + + As with the operator, the new Run has no status yet: ``create_job`` + later records its lowered input (kept in ``run_inputs`` until then) + together with ``jobRef``. + """ + + with self._lock: + request = self.requests[name] + generation = int(request["metadata"].get("generation", 1)) + run_name = f"{name}-{generation}" + effective = lower_request_input( + request, + run_name=run_name, + runtime_image=runtime_image, + dynamo_version=dynamo_version, + ) + if mutate_input is not None: + mutate_input(effective) + run: dict[str, Any] = { + "metadata": { + "name": run_name, + "annotations": {REQUEST_GENERATION_ANNOTATION: str(generation)}, + "ownerReferences": [ + { + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRequest", + "name": name, + "uid": request["metadata"]["uid"], + "controller": True, + "blockOwnerDeletion": True, + } + ], + }, + "spec": copy.deepcopy( + {k: v for k, v in request.get("spec", {}).items() if k != "v1beta1"} + ), + "status": {}, + } + self._stamp( + run, kind="DynamoGraphDeploymentRun", api_version="nvidia.com/v1beta2" + ) + self.runs[run_name] = run + self.run_inputs[run_name] = effective + status = request.setdefault("status", {}) + status["observedGeneration"] = generation + status["activeRunRef"] = {"name": run_name} + self.set_request_condition( + name, + status="True", + reason="RunCreated", + message=f"Run {run_name!r} exists for generation {generation}", + ) + return copy.deepcopy(run) + + def create_job(self, run_name: str, *, record_input: bool = True) -> None: + """Create the Run's Job, as the operator's ``reconcileRunStatus`` does. + + One status write records ``status.aisimulate.input`` and ``jobRef`` + with ``Completed=False`` (``JobPending``). ``record_input=False`` + leaves the input out (an operator that does not record it). + """ + + with self._lock: + run = self.runs[run_name] + status = run.setdefault("status", {}) + if record_input: + status["aisimulate"] = { + "input": copy.deepcopy(self.run_inputs[run_name]) + } + status["jobRef"] = {"name": f"{run_name}-job", "uid": str(uuid.uuid4())} + status["phase"] = "Pending" + self._set_completed( + run, "False", "JobPending", "AISimulate Job is waiting to start" + ) + + def start_run(self, run_name: str, *, record_input: bool = True) -> None: + """Start the Run's Job (creating it first, see ``create_job``).""" + + with self._lock: + run = self.runs[run_name] + if not run["status"].get("jobRef"): + self.create_job(run_name, record_input=record_input) + status = run["status"] + status["startTime"] = rfc3339(self.now()) + status["phase"] = "Running" + self._set_completed(run, "False", "JobRunning", "AISimulate Job is running") + + def set_run_completed( + self, run_name: str, *, status: str, reason: str, message: str = "" + ) -> None: + """Set the Run's ``Completed`` condition, e.g. ``False`` with + ``ContainerWaiting`` while a Job's container cannot start.""" + + with self._lock: + run = self.runs[run_name] + run.setdefault("status", {}) + self._set_completed(run, status, reason, message) + + def add_candidate( + self, + run_name: str, + parameters: Mapping[str, Any], + metrics: Mapping[str, Any], + ) -> dict: + with self._lock: + run = self.runs[run_name] + canonical = json.dumps(parameters, sort_keys=True, separators=(",", ":")) + candidate_id = ( + "evaluated-point-" + uuid.uuid5(uuid.NAMESPACE_OID, canonical).hex[:12] + ) + candidate = { + "metadata": { + "name": f"{run_name}-{candidate_id}", + "labels": { + RUN_UID_LABEL: run["metadata"]["uid"], + CANDIDATE_ID_LABEL: candidate_id, + }, + "ownerReferences": [ + { + "apiVersion": "nvidia.com/v1beta2", + "kind": "DynamoGraphDeploymentRun", + "name": run_name, + "uid": run["metadata"]["uid"], + "controller": True, + } + ], + }, + "spec": {"parameters": json.loads(canonical)}, + "status": {"metrics": json.loads(json.dumps(dict(metrics)))}, + } + self._stamp( + candidate, + kind="DynamoGraphDeploymentCandidate", + api_version="nvidia.com/v1beta2", + ) + self.candidates[candidate["metadata"]["name"]] = candidate + return copy.deepcopy(candidate) + + def complete_run( + self, + run_name: str, + *, + reason: str = "SearchSucceeded", + message: str = "", + candidates: Iterable[tuple[Mapping[str, Any], Mapping[str, Any]]] = (), + ) -> None: + """Publish ``candidates`` (parameters, metrics) and complete the Run.""" + + with self._lock: + run = self.runs[run_name] + refs = [ + { + "name": self.add_candidate(run_name, params, metrics)["metadata"][ + "name" + ] + } + for params, metrics in candidates + ] + status = run["status"] + status["candidateRefs"] = refs + status["phase"] = "Succeeded" if reason == "SearchSucceeded" else "Failed" + status["completionTime"] = rfc3339(self.now()) + self._set_completed(run, "True", reason, message) + owner = (run["metadata"].get("ownerReferences") or [{}])[0] + request = self.requests.get(owner.get("name", "")) + if request is not None: + request_status = request.setdefault("status", {}) + active = request_status.get("activeRunRef") or {} + if active.get("name") == run_name: + request_status.pop("activeRunRef", None) + recent = [ + ref + for ref in request_status.get("recentRunRefs", []) + if ref.get("name") != run_name + ] + request_status["recentRunRefs"] = [{"name": run_name}, *recent] + self.bump(request) + + def _set_completed(self, run: dict, status: str, reason: str, message: str) -> None: + conditions = [ + condition + for condition in run["status"].get("conditions", []) + if condition.get("type") != "Completed" + ] + conditions.append( + { + "type": "Completed", + "status": status, + "reason": reason, + "message": message, + "lastTransitionTime": rfc3339(self.now()), + } + ) + run["status"]["conditions"] = conditions + self.bump(run) + + def run_of(self, request_name: str) -> Optional[dict]: + """The stored Run a Request owns, if any (the newest).""" + + with self._lock: + request = self.requests.get(request_name) + if request is None: + return None + owned = [ + run + for run in self.runs.values() + if any( + owner.get("uid") == request["metadata"]["uid"] + for owner in run["metadata"].get("ownerReferences") or [] + ) + ] + return copy.deepcopy(owned[-1]) if owned else None + + +Evaluator = Callable[[dict, Mapping[str, float]], Optional[Mapping[str, Any]]] + + +class FakeDgdrOperator: + """Drives every Request of a ``FakeRetuneKube`` one phase per ``step()``. + + Phases: accept (a Run with no status, as before its Job exists), start + (the lowered input, jobRef and startTime; the operator records the + input together with jobRef), complete (one Candidate per router point + the input spans, metrics from ``evaluate(request, point_public)``; + ``None`` leaves the point unpublished, as AISimulate drops an + infeasible point). ``hold`` keeps + named Requests in their current phase; ``fail_runs`` maps a Request + name to the ``Completed`` reason its Run ends with. ``mutate_input`` + edits a Run's lowered input and ``mutate_parameters`` a Candidate's + parameters (both get the Request name), to simulate version skew. + ``max_phase`` stops every Request at ``accepted`` or ``started``. + """ + + def __init__( + self, + kube: FakeRetuneKube, + *, + evaluate: Optional[Evaluator] = None, + runtime_image: str = DEFAULT_RUNTIME_IMAGE, + dynamo_version: str = DEFAULT_DYNAMO_VERSION, + mutate_input: Optional[Callable[[str, dict], None]] = None, + mutate_parameters: Optional[Callable[[str, dict], None]] = None, + ) -> None: + self.kube = kube + self.evaluate: Evaluator = evaluate or ( + lambda _request, point: default_metrics(point) + ) + self.runtime_image = runtime_image + self.dynamo_version = dynamo_version + self.mutate_input = mutate_input + self.mutate_parameters = mutate_parameters + self.max_phase: Optional[Literal["accepted", "started"]] = None + self.hold: set[str] = set() + self.fail_runs: dict[str, str] = {} + self.steps = 0 + + def step(self) -> int: + """Advance every Request one phase; returns how many moved.""" + + moved = 0 + with self.kube._lock: + self.steps += 1 + for name in list(self.kube.requests): + if name in self.hold: + continue + run = self.kube.run_of(name) + if run is None: + hook = self.mutate_input + self.kube.accept_request( + name, + runtime_image=self.runtime_image, + dynamo_version=self.dynamo_version, + mutate_input=( + None if hook is None else functools.partial(hook, name) + ), + ) + moved += 1 + continue + run_name = run["metadata"]["name"] + completed = next( + ( + condition + for condition in run["status"].get("conditions", []) + if condition.get("type") == "Completed" + ), + None, + ) + if completed is not None and completed.get("status") == "True": + continue + if self.max_phase == "accepted": + continue + if not run["status"].get("startTime"): + self.kube.start_run(run_name) + moved += 1 + continue + if self.max_phase == "started": + continue + reason = self.fail_runs.get(name) + if reason is not None: + self.kube.complete_run(run_name, reason=reason, message="injected") + moved += 1 + continue + effective = run["status"]["aisimulate"]["input"] + request = self.kube.requests[name] + published = [] + for point in router_points(effective): + metrics = self.evaluate(copy.deepcopy(request), point) + if metrics is not None: + parameters = candidate_parameters(effective, point) + if self.mutate_parameters is not None: + self.mutate_parameters(name, parameters) + published.append((parameters, metrics)) + self.kube.complete_run(run_name, candidates=published) + moved += 1 + return moved + + def run_to_completion(self, max_steps: int = 10) -> None: + for _ in range(max_steps): + if not self.step(): + return diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py index 3fb2c4df72bb..fec09326633f 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -13,9 +13,12 @@ from __future__ import annotations +import gzip import itertools +import json import math import threading +import time from collections.abc import Callable, Mapping from typing import Any, Optional @@ -30,6 +33,7 @@ DgdrExecutorConfig, RetuneConfig, RetuneEngineSpec, + dgdr_need_seconds, ) from dynamo.planner.core.types import RetuneParams from dynamo.planner.retune.live import dgdr @@ -51,15 +55,14 @@ DgdrContext, Shard, build_request, - crn_unsupported, input_mismatches, missing_fields, - need_seconds, plan_shards, request_name, run_dgdr_sweep, spec_hash, ) +from dynamo.planner.retune.live.reconciler import trace_pvc_source from dynamo.planner.retune.sweep import SweepRequest, score_summary from dynamo.planner.tests.live_retune_fakes import ( FakeDgdrOperator, @@ -582,28 +585,6 @@ def test_plan_shards_validates_inputs_and_probe_falls_back_to_zero_temperature() assert shards[-1].points == ({CREDIT: 1.0, SCALE: 1.0, TEMP: 0.0},) -def test_need_seconds_follows_the_design_formula(): - cfg = make_cfg( - cut_budget_seconds=900.0, - job_startup_seconds=300.0, - run={"parallelism": 8, "candidate_timeout_seconds": 1800.0}, - ) - shard = plan_shards( - DEFAULT_DOMAIN, - baseline=grid(DEFAULT_DOMAIN)[0], - startup=grid(DEFAULT_DOMAIN)[0], - max_shards=3, - )[0] - assert shard.size == 36 - # 1 + ceil(35 / 8) = 6 rounds. - assert need_seconds(shard, cfg, needs_cut=True) == 900 + 300 + 6 * 1800 - assert need_seconds(shard, cfg, needs_cut=False) == 300 + 6 * 1800 - single = Shard( - id="probe", kind="probe", choices={CREDIT: (1.0,)}, points=({CREDIT: 1.0},) - ) - assert need_seconds(single, cfg, needs_cut=False) == 300 + 1800 - - # ---------------------------------------------------------------- Request @@ -644,13 +625,13 @@ def test_build_request_aggregated_grid_shard(): assert spec["backends"] == ["vllm"] # 3 replicas x (tensor 2 x pipeline 1 x attention_data 1). assert spec["hardware"] == {"gpu": {"sku": "h200_sxm", "budget": 6}} + # The trace format is left to the CRD default. assert spec["workload"] == { "trace": { - "format": "dynamo", "source": { - "pvc": { + "persistentVolumeClaim": { "claimName": "request-traces", - "path": f"windows/llm/{WINDOW_END_MS}.jsonl.gz", + "subPath": f"windows/llm/{WINDOW_END_MS}.jsonl.gz", } }, } @@ -821,28 +802,51 @@ def test_input_read_back_accepts_the_operator_lowering_and_finds_drift(): drifted["engine"]["workers"]["aggregated"]["parallelism"]["preset"][0][ "replicas" ] = 1 + # A role the overrides delete must stay deleted. drifted["engine"]["workers"]["prefill"] = {} drifted["router"]["temperature"] = {"choices": [0.0]} drifted["evaluation"]["sla"]["e2e_ms"] = 10.0 drifted["traffic"]["load"]["speedup"] = 2.0 + drifted["traffic"]["source"]["paths"] = ["/elsewhere/trace.jsonl.gz"] drifted["optimization"]["strict_sla"] = True problems = input_mismatches(body["spec"], drifted) - joined = "\n".join(problems) - for path in ( - "engine:", - "router:", + assert sorted(problem.split(":")[0] for problem in problems) == [ + "engine.workers.aggregated.parallelism.preset.0.replicas", + "engine.workers.prefill", "evaluation.sla.e2e_ms", - "traffic.load", "optimization.strict_sla", - ): - assert path in joined, (path, problems) + "router.temperature.choices", + "traffic.load.speedup", + "traffic.source.paths.0", + ] + assert "engine.workers.prefill: expected unset, got {}" in problems stripped = lower_request_input(body, run_name="r-1") del stripped["dgdr_run"]["generator"] del stripped["optimization"]["constraints"] problems = input_mismatches(body["spec"], stripped) - assert any(p.startswith("dgdr_run.generator") for p in problems) - assert any("max_candidate_gpus" in p for p in problems) + assert "dgdr_run.generator: expected 'direct', got unset" in problems + assert any(p.startswith("optimization.constraints:") for p in problems) + + +def test_input_read_back_allows_operator_defaults_and_follows_overrides(): + body = _body() + effective = lower_request_input(body, run_name="r-1") + # Fields the Request does not set, such as a new operator default. + effective["engine"]["kv_cache_dtype"] = "auto" + preset = effective["engine"]["workers"]["aggregated"]["parallelism"]["preset"] + preset[0]["prefill_context"] = 1 + effective["router"]["queue_threshold"] = 0.5 + effective["optimizer"]["seed"] = 0 + effective["traffic"]["source"]["compression"] = "auto" + assert input_mismatches(body["spec"], effective) == [] + + # inputOverrides win over the typed trace source. + overrides = body["spec"]["aisimulate"]["inputOverrides"] + overrides["traffic"] = {"source": {"paths": ["/data/other.jsonl"]}} + effective = lower_request_input(body, run_name="r-1") + assert effective["traffic"]["source"]["paths"] == ["/data/other.jsonl"] + assert input_mismatches(body["spec"], effective) == [] def test_missing_fields_reports_pruned_leaves_and_allows_defaults(): @@ -856,15 +860,10 @@ def test_missing_fields_reports_pruned_leaves_and_allows_defaults(): "a.gone", "d", ] - - -def test_crn_unsupported_compares_parseable_versions_only(): - assert not crn_unsupported("1.7.0", None, None) - assert crn_unsupported("1.7.0", None, "1.8.0") - assert not crn_unsupported("1.8.0", None, "1.8.0") - assert not crn_unsupported("1.9.1rc0", None, "1.8.0") - assert crn_unsupported(None, "nvcr.io/nvidia/planner:1.7.2@sha256:ab", "1.8") - assert not crn_unsupported("main", "image-without-tag", "1.8.0") + # The mappings of a list may hold defaults too; the list keeps its length. + assert missing_fields({"p": [{"a": 1, "b": 2}]}, {"p": [{"a": 1}]}) == [] + assert missing_fields({"p": [{"a": 2}]}, {"p": [{"a": 1}]}) == ["p.0.a"] + assert missing_fields({"p": [{"a": 1}, {"a": 2}]}, {"p": [{"a": 1}]}) == ["p"] # ------------------------------------------------------------ the sweep @@ -1000,6 +999,26 @@ def test_shutdown_returns_at_once_without_deleting(tmp_path): assert len(h.kube.requests) == 2 +@pytest.mark.parametrize( + "event, reason", [("cancel", "shutdown"), ("abort", "lost_leadership")] +) +def test_the_wait_between_polls_ends_when_the_sweep_is_stopped(tmp_path, event, reason): + # Wall-clock time and the sweep's own wait: the poll interval is far + # longer than the test, so only the event can end the wait. + h = Harness(tmp_path, operator=False) + request = make_request(deadline_s=time.time() + TIMEOUT_S) + timer = threading.Timer(0.2, getattr(h, event).set) + timer.start() + started = time.monotonic() + try: + result = h.sweep(request, now=time.time, sleep=None, poll_interval_s=600.0) + finally: + timer.cancel() + assert result.failure_reason == reason + assert time.monotonic() - started < 60.0 + assert len(h.kube.requests) == 2 + + def test_not_leader_creates_nothing(tmp_path): h = Harness(tmp_path) h.leader = False @@ -1332,6 +1351,44 @@ def create_jobs(): assert not any(shard["run_pending"] for shard in result.details["shards"].values()) +@pytest.mark.parametrize( + "job, reason, message, failure", + [ + (True, "ContainerWaiting", "sweeper: ErrImagePull", "timeout"), + ( + False, + "TraceSourceMissing", + 'trace PersistentVolumeClaim "request-traces" does not exist', + "run_pending", + ), + ], +) +def test_a_waiting_run_reports_why_at_the_deadline( + tmp_path, job, reason, message, failure +): + h = Harness(tmp_path, operator=False) + + def operator(): + for name in list(h.kube.requests): + if h.kube.run_of(name) is None: + run = h.kube.accept_request(name)["metadata"]["name"] + if job: + h.kube.start_run(run) + h.kube.set_run_completed( + run, status="False", reason=reason, message=message + ) + + h.clock.hooks.append(operator) + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + assert result.status == "timeout" + assert result.failure_reason == failure + for shard_id in ("grid-0", "probe"): + shard = result.details["shards"][shard_id] + assert shard["state"] == "wait" + assert shard["message"].endswith(f"{reason}: {message}") + assert f"{shard_id}: {shard['message']}" in result.error + + def test_running_job_past_the_deadline_times_out(tmp_path): h = Harness(tmp_path) h.operator.max_phase = "started" @@ -1376,8 +1433,8 @@ def test_launch_too_late_for_an_ordinary_shard_leaves_its_points_missing(tmp_pat "probe": 1, } assert shards[-1].points[0][CREDIT] == 5.0 - need6 = need_seconds(shards[1], cfg, needs_cut=True) - need8 = need_seconds(shards[0], cfg, needs_cut=True) + need6 = dgdr_need_seconds(shards[1].size, cfg, needs_cut=True) + need8 = dgdr_need_seconds(shards[0].size, cfg, needs_cut=True) assert need6 < need8 request = make_request(domain, baseline=baseline, deadline_s=LAUNCH_S + need6 + 1) result = h.sweep(request) @@ -1599,14 +1656,6 @@ def operator(): assert "status.aisimulate.input is missing" in result.error -def test_crn_unsupported_runtime_fails_the_window(tmp_path): - h = Harness(tmp_path) - h.operator.dynamo_version = "1.7.0" - result = h.sweep(make_request(), crn_min_version="1.8.0") - assert result.failure_reason == "crn_unsupported" - assert result.details["runtime_version"] == "1.7.0" - - def test_crn_probe_mismatch_fails_the_window(tmp_path): def evaluate(request, point): metrics = default_metrics(point) @@ -1850,10 +1899,6 @@ def test_sweep_with_the_real_window_cutter(tmp_path): """The executor and ``window_cut.cut_window`` agree on paths and deadline.""" window_cut = pytest.importorskip("dynamo.planner.retune.live.window_cut") - import gzip - import json - import time - from dynamo.planner.tests.retune_sweep_helpers import trace_rows pod_dir = tmp_path / "trace" / "segments" / "llm" / "frontend-0" @@ -1879,6 +1924,6 @@ def real_cut(*args, deadline_s, **kwargs): window_file = tmp_path / "trace" / "windows" / "llm" / f"{WINDOW_END_MS}.jsonl.gz" with gzip.open(window_file, "rt", encoding="utf-8") as lines: assert len(lines.readlines()) == 5 - claim_path = h.created()[0]["spec"]["workload"]["trace"]["source"]["pvc"]["path"] + _, claim_path = trace_pvc_source(h.created()[0]["spec"]) assert tmp_path / "trace" / claim_path == window_file assert_scores(result, request) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py index 6fe507868881..73f074f39084 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for ``KubernetesRetuneKube`` and the shared ``FakeRetuneKube``. +"""Unit tests for ``KubernetesRetuneKube``. -The client tests run the real ``kubernetes`` API classes and replace only +The tests run the real ``kubernetes`` API classes and replace only the REST transport, so they check the URLs, query parameters, timeouts and bodies the client library actually sends, and the error mapping. """ @@ -28,11 +28,6 @@ RetuneKube, ) from dynamo.planner.retune.live.kube import KubernetesRetuneKube, request_timeout -from dynamo.planner.tests.live_retune_fakes import ( - FakeRetuneKube, - lower_request_input, - match_labels, -) pytestmark = [ pytest.mark.gpu_0, @@ -133,7 +128,7 @@ def test_satisfies_the_protocol(kube): _: RetuneKube = retune_kube assert retune_kube.namespace == NS with pytest.raises(ValueError): - KubernetesRetuneKube("") + KubernetesRetuneKube("", api_client=retune_kube.api_client) def test_get_dgd(kube): @@ -404,127 +399,3 @@ def test_from_kube_api_shares_the_connector_client(): assert retune_kube.namespace == "ns2" assert retune_kube.api_client is api_client api_client.close() - - -# ------------------------------------------------------------------ fake - - -def test_fake_label_selectors(): - labels = {"a": "1", "b": "2"} - assert match_labels(labels, "") - assert match_labels(labels, "a=1,b==2") - assert match_labels(labels, "a,!c") - assert not match_labels(labels, "a!=1") - assert not match_labels(labels, "c") - assert not match_labels(labels, "!a") - with pytest.raises(ValueError): - match_labels(labels, "a in (1,2)") - - -def test_fake_lease_compare_and_swap_and_faults(): - kube = FakeRetuneKube(NS, now=lambda: 1_793_000_000.0) - created = kube.create_lease( - {"metadata": {"name": "l"}, "spec": {"holderIdentity": ""}} - ) - with pytest.raises(KubeAlreadyExists): - kube.create_lease({"metadata": {"name": "l"}}) - stale = dict(created, spec={"holderIdentity": "me"}) - updated = kube.replace_lease("l", stale) - assert updated["spec"]["holderIdentity"] == "me" - assert ( - updated["metadata"]["resourceVersion"] != created["metadata"]["resourceVersion"] - ) - with pytest.raises(KubeConflict): - kube.replace_lease("l", stale) # resourceVersion is stale now - with pytest.raises(KubeNotFound): - kube.replace_lease("missing", stale) - - # A write that times out but lands. - kube.fail("replace_lease", KubeError("timeout"), landed=True) - with pytest.raises(KubeError): - kube.replace_lease("l", dict(updated, spec={"holderIdentity": "other"})) - assert kube.get_lease("l")["spec"]["holderIdentity"] == "other" - # A write that fails before landing. - current = kube.get_lease("l") - kube.fail("replace_lease", KubeError("503")) - with pytest.raises(KubeError): - kube.replace_lease("l", dict(current, spec={"holderIdentity": "x"})) - assert kube.get_lease("l")["spec"]["holderIdentity"] == "other" - kube.delete_lease("l") - kube.delete_lease("l") - assert kube.get_lease("l") is None - assert [call.method for call in kube.calls].count("replace_lease") == 5 - - -def test_fake_requests_cascade_and_copies(): - kube = FakeRetuneKube(NS) - body = {"metadata": {"name": "r", "labels": {"w": "1"}}, "spec": {"x": 1}} - stored = kube.create_request(body) - stored["spec"]["x"] = 2 # copies never alias the store - assert kube.get_request("r")["spec"]["x"] == 1 - assert kube.create_request({"metadata": {"name": "dry"}}, dry_run=True) - assert kube.get_request("dry") is None - run = kube.accept_request("r") - kube.start_run(run["metadata"]["name"]) - kube.complete_run(run["metadata"]["name"], candidates=[({"p": 1}, {"m": 2})]) - request = kube.get_request("r") - assert request["status"]["recentRunRefs"] == [{"name": "r-1"}] - assert "activeRunRef" not in request["status"] - assert ( - len(kube.list_candidates(f"nvidia.com/dgdr-run-uid={run['metadata']['uid']}")) - == 1 - ) - assert kube.list_requests("w=1")[0]["metadata"]["name"] == "r" - kube.delete_request("r", propagation="Foreground") - assert kube.runs == {} and kube.candidates == {} - assert kube.deleted_requests == [("r", "Foreground")] - - -def test_fake_lowering_applies_overrides_and_operator_fields(): - request = { - "metadata": {"name": "r", "namespace": NS}, - "spec": { - "model": {"name": "m"}, - "backends": ["vllm"], - "hardware": {"gpu": {"sku": "h200_sxm", "budget": 4}}, - "workload": { - "trace": { - "format": "dynamo", - "source": {"pvc": {"claimName": "c", "path": "w/1.jsonl.gz"}}, - } - }, - "objective": { - "mode": "Optimize", - "metric": "GoodputPerGPU", - "sla": {"ttftMs": 1.0}, - }, - "recommendation": {"maxCandidates": 3}, - "aisimulate": { - "inputOverrides": { - "engine": {"backend": "vllm", "workers": {"prefill": None}}, - "dgdr_run": {"generator": "direct"}, - } - }, - }, - } - lowered = lower_request_input(request, run_name="r-1") - assert lowered["engine"]["backend"] == "vllm" - assert "prefill" not in lowered["engine"]["workers"] - assert lowered["engine"]["workers"]["decode"] == {} - assert lowered["optimization"] == { - "constraints": {"max_candidate_gpus": 4}, - "target": "goodput_per_gpu", - "strict_sla": True, - } - assert lowered["traffic"] == { - "source": { - "type": "trace", - "format": "dynamo", - "paths": ["/var/run/aisimulate/trace/trace.jsonl.gz"], - }, - "load": {"type": "trace_timestamps"}, - } - assert lowered["optimizer"] == {"max_trials": 320.0} - assert lowered["dgdr_run"]["generator"] == "direct" - assert lowered["dgdr_run"]["max_candidates"] == 3 - assert lowered["dgdr_run"]["namespace"] == NS From 12f4389c7f57267e36b7e8706cf10f06152a250b Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 09:14:29 -0800 Subject: [PATCH 082/100] fix(operator): address review on the DGDR trace source API API (v1beta2, unreleased): - Rename spec.workload.trace.source.pvc to persistentVolumeClaim and its path to subPath, the Kubernetes names for the claim volume and the volumeMount subPath, through the CEL rule, structural validation, error paths, lowering, fixtures, docs and CRDs. - Spell the trace format enum Dynamo, make format optional with the CRD default Dynamo, and translate it explicitly to AISimulate's native traffic.source.format dynamo while lowering. Admission no longer requires or checks the format itself; lowering rejects a format it cannot translate. - Mark every serialized v1beta2 field +optional or +required (no schema change for existing fields) and enforce it with kube-api-linter's optionalorrequired rule: `make lint-api` (part of `make lint`) and the operator Dockerfile linter stage run it over api/v1beta2. inputOverrides stay authoritative over a typed trace: - Admission no longer rejects traffic.source.type, format or paths, or a null traffic or traffic.source, when a trace is set. - The Run controller no longer restores the typed source after the merge patch; deletions stay deleted. The typed claim still alone decides the Job's mount and claim dependency. - Document the trace fields as an underlay and the overrides as the final JSON Merge Patch. Controller: - Drop the unused PVC update RBAC verb. - Complete a trace cancellation from the uncached absence confirmation instead of listing Jobs a second time. - Wake only open Runs, or completed Runs whose finalizer the claim still carries, on claim events. - List the namespace's Jobs from the API server once per claim reconcile instead of once per stale finalizer. - Narrow the claimName doc: a deleted Run's finalizer can be released while its garbage-collected Job's pod still stops; PVC protection keeps the claim meanwhile. Tests: - Require the sweeper's CreateContainerConfigError in the symlink clustertest before checking the Run diagnostic. - Add trace_request_lowering.golden.json and the envtest that admits the live-retune planner's shard Request, freezes its Run and lowers it with the Run controller (rewrite with -update); the planner's tests read it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../components/operator/files/role.yaml | 12 +- deploy/operator/.golangci-kal.yml | 38 ++++ deploy/operator/Dockerfile | 5 + deploy/operator/Makefile | 14 +- deploy/operator/api/roundtrip_fuzz_test.go | 4 +- .../dynamographdeploymentcandidate_types.go | 9 +- .../dynamographdeploymentrequest_types.go | 79 ++++--- .../v1beta2/dynamographdeploymentrun_types.go | 22 +- .../api/v1beta2/zz_generated.deepcopy.go | 4 +- ...dia.com_dynamographdeploymentrequests.yaml | 63 +++--- .../nvidia.com_dynamographdeploymentruns.yaml | 63 +++--- deploy/operator/internal/aisimulate/doc.go | 7 +- deploy/operator/internal/aisimulate/input.go | 7 +- .../trace_request_lowering.golden.json | 191 ++++++++++++++++ deploy/operator/internal/aisimulate/trace.go | 44 ++-- .../internal/aisimulate/trace_test.go | 22 +- .../internal/controller/clusterenv_test.go | 6 +- .../dgdr_v1beta2_trace_clustertest_test.go | 52 +++-- .../dgdr_v1beta2_trace_golden_envtest_test.go | 162 ++++++++++++++ ...mographdeploymentrun_v1beta2_controller.go | 50 ++--- ...phdeploymentrun_v1beta2_controller_test.go | 159 +++++++++----- ...ymentrun_v1beta2_trace_claim_controller.go | 92 ++++---- .../dgdr-v1beta2/vllm-trace/input.yaml | 6 +- .../dynamographdeploymentrequest_v1beta2.go | 3 +- .../webhook/defaulting/suite_envtest_test.go | 11 +- ...hdeploymentrequest_carrier_envtest_test.go | 11 +- .../dynamographdeploymentrequest_v1beta2.go | 2 +- ...ploymentrequest_v1beta2_input_overrides.go | 50 +---- ...ploymentrequest_validation_envtest_test.go | 204 ++++++++++-------- .../webhook/validation/shared_helpers.go | 8 +- .../webhook/validation/shared_v1beta2.go | 57 +++-- .../additional-resources/api-reference-k8s.md | 83 +++---- .../dynamo-graph-deployment-request.mdx | 49 +++-- .../kubernetes-api/full-api-reference.mdx | 99 +++++---- 34 files changed, 1144 insertions(+), 544 deletions(-) create mode 100644 deploy/operator/.golangci-kal.yml create mode 100644 deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json create mode 100644 deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 09548c4a7cd8..97258d1d9b3f 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -11,7 +11,6 @@ rules: resources: - configmaps - events - - persistentvolumeclaims - services verbs: - create @@ -29,6 +28,17 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - create + - delete + - get + - list + - patch + - watch - apiGroups: - "" resources: diff --git a/deploy/operator/.golangci-kal.yml b/deploy/operator/.golangci-kal.yml new file mode 100644 index 000000000000..c197eb455f36 --- /dev/null +++ b/deploy/operator/.golangci-kal.yml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Kubernetes API conventions for the v1beta2 API types, checked by kube-api-linter +# (sigs.k8s.io/kube-api-linter). It runs through the golangci-lint build that the +# kube-api-linter module ships, so `make lint-api` uses this configuration alone. + +version: "2" + +run: + timeout: 10m + +linters: + default: none + enable: + - kubeapilinter + settings: + custom: + kubeapilinter: + type: module + description: kube-api-linter checks Kubernetes API conventions. + settings: + linters: + disable: + - "*" + enable: + # Every serialized field carries exactly one of +optional or +required. + - optionalorrequired + exclusions: + generated: strict + rules: + - linters: + - kubeapilinter + path-except: ^api/v1beta2/ + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/deploy/operator/Dockerfile b/deploy/operator/Dockerfile index b903b4c20838..419dac24b0fe 100644 --- a/deploy/operator/Dockerfile +++ b/deploy/operator/Dockerfile @@ -35,6 +35,11 @@ RUN go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.15.0 # Run linting RUN golangci-lint run --timeout=5m +# Check the v1beta2 API types against Kubernetes API conventions. Keep the version in sync with +# KUBE_API_LINTER_VERSION in the Makefile. +RUN go install sigs.k8s.io/kube-api-linter/cmd/golangci-lint-kube-api-linter@v0.0.0-20261012124120-79b557aa8c8d && \ + golangci-lint-kube-api-linter run --timeout=5m -c .golangci-kal.yml ./api/v1beta2/... + # Test stage FROM base AS tester diff --git a/deploy/operator/Makefile b/deploy/operator/Makefile index 0ed10aa45a6d..9ff52910b2c2 100644 --- a/deploy/operator/Makefile +++ b/deploy/operator/Makefile @@ -385,13 +385,17 @@ integration: envtest setup-clustertest ## Run envtests and clustertests, creatin $(MAKE) clustertest .PHONY: lint -lint: golangci-lint ## Run golangci-lint linter & yamllint +lint: golangci-lint lint-api ## Run golangci-lint linter, the Kubernetes API linter & yamllint $(GOLANGCI_LINT) run .PHONY: lint-fix lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes $(GOLANGCI_LINT) run --fix +.PHONY: lint-api +lint-api: kube-api-linter ## Check the v1beta2 API types against Kubernetes API conventions + $(KUBE_API_LINTER) run -c .golangci-kal.yml ./api/v1beta2/... + ##@ Build .PHONY: build @@ -579,6 +583,8 @@ YQ_VERSION ?= v4.45.4 CONTROLLER_TOOLS_VERSION ?= v0.17.3 ENVTEST_VERSION ?= release-0.19 GOLANGCI_LINT_VERSION ?= v2.15.0 +# kube-api-linter publishes no tags; this pseudo-version is its 2026-10-12 main. +KUBE_API_LINTER_VERSION ?= v0.0.0-20261012124120-79b557aa8c8d ## Tool Binaries KUBECTL ?= $(LOCALBIN)/kubectl-$(KUBECTL_VERSION) @@ -588,6 +594,7 @@ YQ ?= $(LOCALBIN)/yq-$(YQ_VERSION) CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION) ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION) GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) +KUBE_API_LINTER = $(LOCALBIN)/golangci-lint-kube-api-linter-$(KUBE_API_LINTER_VERSION) ## ---------- Binary download targets ---------- @@ -651,6 +658,11 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,${GOLANGCI_LINT_VERSION}) +.PHONY: kube-api-linter +kube-api-linter: $(KUBE_API_LINTER) ## Build the golangci-lint that includes kube-api-linter locally if necessary. +$(KUBE_API_LINTER): $(LOCALBIN) + $(call go-install-tool,$(KUBE_API_LINTER),sigs.k8s.io/kube-api-linter/cmd/golangci-lint-kube-api-linter,${KUBE_API_LINTER_VERSION}) + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary (ideally with version) # $2 - package url which can be installed diff --git a/deploy/operator/api/roundtrip_fuzz_test.go b/deploy/operator/api/roundtrip_fuzz_test.go index bee73c544229..9672d002b73b 100644 --- a/deploy/operator/api/roundtrip_fuzz_test.go +++ b/deploy/operator/api/roundtrip_fuzz_test.go @@ -578,9 +578,9 @@ func fuzzBeta2Workload(workload *v1beta2.WorkloadSpec, c randfill.Continue) { } if workload.Trace != nil { workload.Trace.Format = v1beta2.TraceFormatDynamo - workload.Trace.Source.PVC = &v1beta2.TracePVCSource{ + workload.Trace.Source.PersistentVolumeClaim = &v1beta2.TracePVCSource{ ClaimName: fmt.Sprintf("traces-%d", c.Uint32()), - Path: fmt.Sprintf("retune/window-%d.jsonl.gz", c.Uint32()), + SubPath: fmt.Sprintf("retune/window-%d.jsonl.gz", c.Uint32()), } if workload.Trace.Replay != nil && workload.Trace.Replay.ArrivalSpeedupRatio != nil { workload.Trace.Replay.ArrivalSpeedupRatio = oneOfPtr(c, 0.5, 1.0, 4.0) diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentcandidate_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentcandidate_types.go index 9dd1fe007845..62c083350a11 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentcandidate_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentcandidate_types.go @@ -40,6 +40,7 @@ type DynamoGraphDeploymentCandidateSpec struct { // Parameters are the resolved evaluation parameters for this point. // +kubebuilder:pruning:PreserveUnknownFields // +kubebuilder:validation:Type=object + // +required Parameters runtime.RawExtension `json:"parameters"` } @@ -52,10 +53,14 @@ type DynamoGraphDeploymentCandidateSpec struct { // DynamoGraphDeploymentCandidate is one bounded, user-visible search result. type DynamoGraphDeploymentCandidate struct { - metav1.TypeMeta `json:",inline"` + metav1.TypeMeta `json:",inline"` + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - Spec DynamoGraphDeploymentCandidateSpec `json:"spec"` + // +required + Spec DynamoGraphDeploymentCandidateSpec `json:"spec"` + + // +optional Status DynamoGraphDeploymentCandidateStatus `json:"status,omitempty"` } diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go index e51a958f546f..2685f6b3e65a 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrequest_types.go @@ -28,6 +28,7 @@ import ( type ModelSpec struct { // Name identifies the model in the syntax accepted by the selected backend. // +kubebuilder:validation:MinLength=1 + // +required Name string `json:"name"` } @@ -91,12 +92,14 @@ type KVCapacityFractionRange struct { // +kubebuilder:validation:ExclusiveMinimum=true // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=1 + // +required Min float64 `json:"min"` // Max is the upper end of the searched fraction range. // +kubebuilder:validation:ExclusiveMinimum=true // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=1 + // +required Max float64 `json:"max"` } @@ -154,14 +157,15 @@ type StaticWorkloadSpec struct { Load *StaticLoadSpec `json:"load,omitempty"` } -// TraceFormat identifies the decoder of a recorded request trace. -// +kubebuilder:validation:Enum=dynamo +// TraceFormat identifies the decoder of a recorded request trace. Lowering translates it to +// AISimulate's native traffic.source.format. +// +kubebuilder:validation:Enum=Dynamo type TraceFormat string const ( // TraceFormatDynamo is the Dynamo request trace (dynamo.request.trace.v1) that the frontend - // records, as JSON Lines or gzip-compressed JSON Lines. - TraceFormatDynamo TraceFormat = "dynamo" + // records, as JSON Lines or gzip-compressed JSON Lines. It lowers to AISimulate's dynamo format. + TraceFormatDynamo TraceFormat = "Dynamo" ) // TracePVCSource names one trace file on a PersistentVolumeClaim in the Request namespace. @@ -169,31 +173,38 @@ type TracePVCSource struct { // ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. // The Run Job mounts only the trace file from it, read-only, and only into the AISimulate // container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` - // until its execution stops; deleting the claim cancels the Runs that read it. + // until the Run completes or, for a Run deleted before it completes, until no Job of the Run + // is left. Garbage collection deletes the Job of a deleted Run before the Job's pod, so the + // finalizer of a deleted Run can be released while that pod still stops; Kubernetes + // PersistentVolumeClaim protection keeps the claim until no pod uses it. Deleting the claim + // cancels the Runs that read it. + // +required // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` ClaimName string `json:"claimName"` - // Path names one trace file relative to the root of the claim. It is neither a directory - // nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' - // segments or a NUL character, which no file name can hold. The file must be complete before - // the Request is created and must never be rewritten: every Run of the Request reads it when - // its Job starts. + // SubPath names one trace file relative to the root of the claim; the Run Job mounts it as + // the subPath of the claim volume. It is neither a directory nor a glob. It must not start or + // end with '/' and must not contain empty, '.' or '..' segments or a NUL character, which no + // file name can hold. The file must be complete before the Request is created and must never + // be rewritten: every Run of the Request reads it when its Job starts. + // +required // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:MaxLength=1024 - // +kubebuilder:validation:XValidation:rule="!self.startsWith('/') && !self.endsWith('/') && !self.contains('//') && !self.matches('(^|/)[.][.]?(/|$)')",message="path must be relative to the claim root without empty, '.' or '..' segments" - // +kubebuilder:validation:XValidation:rule="!self.contains('\\u0000')",message="path must not contain a NUL character" - Path string `json:"path"` + // +kubebuilder:validation:XValidation:rule="!self.startsWith('/') && !self.endsWith('/') && !self.contains('//') && !self.matches('(^|/)[.][.]?(/|$)')",message="subPath must be relative to the claim root without empty, '.' or '..' segments" + // +kubebuilder:validation:XValidation:rule="!self.contains('\\u0000')",message="subPath must not contain a NUL character" + SubPath string `json:"subPath"` } // TraceSourceSpec identifies exactly one trace artifact. A PersistentVolumeClaim is the only // source in this release. -// +kubebuilder:validation:XValidation:rule="has(self.pvc)",message="pvc is required" +// +kubebuilder:validation:XValidation:rule="has(self.persistentVolumeClaim)",message="persistentVolumeClaim is required" type TraceSourceSpec struct { - // PVC reads the trace from a PersistentVolumeClaim in the Request namespace. + // PersistentVolumeClaim reads the trace from a PersistentVolumeClaim in the Request + // namespace. // +optional - PVC *TracePVCSource `json:"pvc,omitempty"` + PersistentVolumeClaim *TracePVCSource `json:"persistentVolumeClaim,omitempty"` } // TraceReplaySpec configures how AISimulate replays a decoded trace. @@ -208,12 +219,13 @@ type TraceReplaySpec struct { // TraceWorkloadSpec replays a recorded request trace open loop at its recorded arrival times. type TraceWorkloadSpec struct { - // Format identifies the trace decoder. It has no default; only dynamo is admitted in this - // release. - // +kubebuilder:validation:Required + // Format identifies the trace decoder. Defaults to Dynamo. + // +optional + // +kubebuilder:default=Dynamo Format TraceFormat `json:"format,omitempty"` // Source identifies the trace artifact. + // +required Source TraceSourceSpec `json:"source"` // Replay configures how the decoded trace is replayed. @@ -234,8 +246,8 @@ type WorkloadSpec struct { // Trace replays a recorded request trace from a PersistentVolumeClaim in the Request // namespace. Admission supplies no static default when it is set. The Run Job mounts only the // trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, - // or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change - // traffic.source.type, format or paths. + // or trace.jsonl.gz when subPath ends in .gz. The trace lowers to the default traffic.source + // and traffic.load, which spec.aisimulate.inputOverrides can replace or delete. // +optional Trace *TraceWorkloadSpec `json:"trace,omitempty"` } @@ -316,11 +328,16 @@ type AISimulateSpec struct { // runtime_image, runtime_version_override, derive_runtime_images, // num_gpus_per_node, and namespace are operator-owned. Request admission // rejects them, and the Run controller enforces its values while lowering the - // immutable snapshot. When spec.workload.trace is set, traffic and - // traffic.source must remain objects, and the trace owns traffic.source.type, - // format, and paths the same way: Request admission rejects them, and the Run - // controller enforces the typed values. Other traffic overrides are AISimulate - // input, which AISimulate validates when the Run's Job starts. The operator runs + // immutable snapshot. A typed spec.workload.trace is an underlay like the other + // typed fields: it supplies the default traffic.source (type trace, format + // dynamo and the mounted trace file in paths) and the recorded-arrival + // traffic.load. Overrides can replace those values, including + // traffic.source.type, format, and paths, and null deletes the named field or + // parent object; the Run controller does not restore it. The typed + // PersistentVolumeClaim source alone decides the Run Job's trace mount and its + // claim dependency, so overriding a native path adds no volume mount. + // AISimulate validates the resulting input when the Run's Job starts; admission + // does not validate it against the AISimulate schema yet. The operator runs // the AISimulate version from its configured planner image; Requests and Runs // cannot select a different image or version. // +optional @@ -367,7 +384,7 @@ type NativeSearchInput struct { // traffic defaults. Any other static object is completed with its missing defaults, and a // trace workload receives no static default. // spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over - // every lowered traffic field it names, except the traffic.source fields a trace owns. + // every lowered traffic field it names, including the fields a trace lowers. // The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction // search. Lowering independently defaults optimizer.max_trials to 320. // +optional @@ -457,10 +474,14 @@ type DynamoGraphDeploymentRequestStatus struct { // DynamoGraphDeploymentRequest is the Schema for simulation-backed deployment searches. type DynamoGraphDeploymentRequest struct { - metav1.TypeMeta `json:",inline"` + metav1.TypeMeta `json:",inline"` + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - Spec DynamoGraphDeploymentRequestSpec `json:"spec"` + // +required + Spec DynamoGraphDeploymentRequestSpec `json:"spec"` + + // +optional Status DynamoGraphDeploymentRequestStatus `json:"status,omitempty"` } diff --git a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go index 1c363bc39090..fb2cf2dde1bf 100644 --- a/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go +++ b/deploy/operator/api/v1beta2/dynamographdeploymentrun_types.go @@ -49,6 +49,7 @@ const ( // ResourceReference identifies a namespaced resource by name and UID. type ResourceReference struct { // Name is the resource name. + // +required Name string `json:"name"` // UID is the resource UID. @@ -59,51 +60,64 @@ type ResourceReference struct { // RoundProgress reports completed and planned optimizer rounds. type RoundProgress struct { // Completed is the number of finished rounds. + // +required Completed int32 `json:"completed"` // Total is the number of planned rounds. + // +required Total int32 `json:"total"` } // BranchProgress reports progress for one independently optimized AISimulate branch. type BranchProgress struct { // Name is the executor-defined stable branch identifier. + // +required Name string `json:"name"` // Rounds reports optimizer progress for this branch. + // +required Rounds RoundProgress `json:"rounds"` } // EvaluationProgress reports unique candidate outcomes without creating one resource per trial. type EvaluationProgress struct { // Scheduled is the number of unique evaluations accepted for execution. + // +optional Scheduled int64 `json:"scheduled,omitempty"` // Running is the number of evaluations currently executing. + // +optional Running int64 `json:"running,omitempty"` // Feasible is the number of completed evaluations that satisfy all constraints. + // +optional Feasible int64 `json:"feasible,omitempty"` // Infeasible is the number of completed evaluations that violate at least one constraint. + // +optional Infeasible int64 `json:"infeasible,omitempty"` // Failed is the number of evaluations that ended in an execution error. + // +optional Failed int64 `json:"failed,omitempty"` // Unsupported is the number of configurations rejected as unsupported before evaluation. + // +optional Unsupported int64 `json:"unsupported,omitempty"` // CacheHits is the number of evaluations satisfied from a prior result. + // +optional CacheHits int64 `json:"cacheHits,omitempty"` } // CandidateProgress reports bounded Kubernetes publication, not all internal trials. type CandidateProgress struct { // ParetoFront is the number of feasible points on the current Pareto front. + // +optional ParetoFront int64 `json:"paretoFront,omitempty"` // Published is the number of Candidate resources currently referenced by this run. + // +optional Published int32 `json:"published,omitempty"` } @@ -189,10 +203,14 @@ type DynamoGraphDeploymentRunStatus struct { // DynamoGraphDeploymentRun is one immutable accepted DGDR request and its execution status. type DynamoGraphDeploymentRun struct { - metav1.TypeMeta `json:",inline"` + metav1.TypeMeta `json:",inline"` + // +optional metav1.ObjectMeta `json:"metadata,omitempty"` - Spec DynamoGraphDeploymentRunSpec `json:"spec"` + // +required + Spec DynamoGraphDeploymentRunSpec `json:"spec"` + + // +optional Status DynamoGraphDeploymentRunStatus `json:"status,omitempty"` } diff --git a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go index 7167809dd96e..8cf585efa0eb 100644 --- a/deploy/operator/api/v1beta2/zz_generated.deepcopy.go +++ b/deploy/operator/api/v1beta2/zz_generated.deepcopy.go @@ -892,8 +892,8 @@ func (in *TraceReplaySpec) DeepCopy() *TraceReplaySpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TraceSourceSpec) DeepCopyInto(out *TraceSourceSpec) { *out = *in - if in.PVC != nil { - in, out := &in.PVC, &out.PVC + if in.PersistentVolumeClaim != nil { + in, out := &in.PersistentVolumeClaim, &out.PersistentVolumeClaim *out = new(TracePVCSource) **out = **in } diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index f0c3c0ece371..cfb93424ce2f 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -9610,11 +9610,16 @@ spec: runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the - immutable snapshot. When spec.workload.trace is set, traffic and - traffic.source must remain objects, and the trace owns traffic.source.type, - format, and paths the same way: Request admission rejects them, and the Run - controller enforces the typed values. Other traffic overrides are AISimulate - input, which AISimulate validates when the Run's Job starts. The operator runs + immutable snapshot. A typed spec.workload.trace is an underlay like the other + typed fields: it supplies the default traffic.source (type trace, format + dynamo and the mounted trace file in paths) and the recorded-arrival + traffic.load. Overrides can replace those values, including + traffic.source.type, format, and paths, and null deletes the named field or + parent object; the Run controller does not restore it. The typed + PersistentVolumeClaim source alone decides the Run Job's trace mount and its + claim dependency, so overriding a native path adds no volume mount. + AISimulate validates the resulting input when the Run's Job starts; admission + does not validate it against the AISimulate schema yet. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. type: object @@ -9791,7 +9796,7 @@ spec: traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over - every lowered traffic field it names, except the traffic.source fields a trace owns. + every lowered traffic field it names, including the fields a trace lowers. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. properties: @@ -9873,15 +9878,14 @@ spec: Trace replays a recorded request trace from a PersistentVolumeClaim in the Request namespace. Admission supplies no static default when it is set. The Run Job mounts only the trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, - or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change - traffic.source.type, format or paths. + or trace.jsonl.gz when subPath ends in .gz. The trace lowers to the default traffic.source + and traffic.load, which spec.aisimulate.inputOverrides can replace or delete. properties: format: - description: |- - Format identifies the trace decoder. It has no default; only dynamo is admitted in this - release. + default: Dynamo + description: Format identifies the trace decoder. Defaults to Dynamo. enum: - - dynamo + - Dynamo type: string replay: description: Replay configures how the decoded trace is replayed. @@ -9897,44 +9901,49 @@ spec: source: description: Source identifies the trace artifact. properties: - pvc: - description: PVC reads the trace from a PersistentVolumeClaim in the Request namespace. + persistentVolumeClaim: + description: |- + PersistentVolumeClaim reads the trace from a PersistentVolumeClaim in the Request + namespace. properties: claimName: description: |- ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` - until its execution stops; deleting the claim cancels the Runs that read it. + until the Run completes or, for a Run deleted before it completes, until no Job of the Run + is left. Garbage collection deletes the Job of a deleted Run before the Job's pod, so the + finalizer of a deleted Run can be released while that pod still stops; Kubernetes + PersistentVolumeClaim protection keeps the claim until no pod uses it. Deleting the claim + cancels the Runs that read it. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - path: + subPath: description: |- - Path names one trace file relative to the root of the claim. It is neither a directory - nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' - segments or a NUL character, which no file name can hold. The file must be complete before - the Request is created and must never be rewritten: every Run of the Request reads it when - its Job starts. + SubPath names one trace file relative to the root of the claim; the Run Job mounts it as + the subPath of the claim volume. It is neither a directory nor a glob. It must not start or + end with '/' and must not contain empty, '.' or '..' segments or a NUL character, which no + file name can hold. The file must be complete before the Request is created and must never + be rewritten: every Run of the Request reads it when its Job starts. maxLength: 1024 minLength: 1 type: string x-kubernetes-validations: - - message: path must be relative to the claim root without empty, '.' or '..' segments + - message: subPath must be relative to the claim root without empty, '.' or '..' segments rule: '!self.startsWith(''/'') && !self.endsWith(''/'') && !self.contains(''//'') && !self.matches(''(^|/)[.][.]?(/|$)'')' - - message: path must not contain a NUL character + - message: subPath must not contain a NUL character rule: '!self.contains(''\u0000'')' required: - claimName - - path + - subPath type: object type: object x-kubernetes-validations: - - message: pvc is required - rule: has(self.pvc) + - message: persistentVolumeClaim is required + rule: has(self.persistentVolumeClaim) required: - - format - source type: object type: object diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml index c5cc3b66259c..7c4942a688a6 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentruns.yaml @@ -77,11 +77,16 @@ spec: runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the - immutable snapshot. When spec.workload.trace is set, traffic and - traffic.source must remain objects, and the trace owns traffic.source.type, - format, and paths the same way: Request admission rejects them, and the Run - controller enforces the typed values. Other traffic overrides are AISimulate - input, which AISimulate validates when the Run's Job starts. The operator runs + immutable snapshot. A typed spec.workload.trace is an underlay like the other + typed fields: it supplies the default traffic.source (type trace, format + dynamo and the mounted trace file in paths) and the recorded-arrival + traffic.load. Overrides can replace those values, including + traffic.source.type, format, and paths, and null deletes the named field or + parent object; the Run controller does not restore it. The typed + PersistentVolumeClaim source alone decides the Run Job's trace mount and its + claim dependency, so overriding a native path adds no volume mount. + AISimulate validates the resulting input when the Run's Job starts; admission + does not validate it against the AISimulate schema yet. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. type: object @@ -250,7 +255,7 @@ spec: traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over - every lowered traffic field it names, except the traffic.source fields a trace owns. + every lowered traffic field it names, including the fields a trace lowers. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. properties: @@ -332,15 +337,14 @@ spec: Trace replays a recorded request trace from a PersistentVolumeClaim in the Request namespace. Admission supplies no static default when it is set. The Run Job mounts only the trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, - or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change - traffic.source.type, format or paths. + or trace.jsonl.gz when subPath ends in .gz. The trace lowers to the default traffic.source + and traffic.load, which spec.aisimulate.inputOverrides can replace or delete. properties: format: - description: |- - Format identifies the trace decoder. It has no default; only dynamo is admitted in this - release. + default: Dynamo + description: Format identifies the trace decoder. Defaults to Dynamo. enum: - - dynamo + - Dynamo type: string replay: description: Replay configures how the decoded trace is replayed. @@ -356,44 +360,49 @@ spec: source: description: Source identifies the trace artifact. properties: - pvc: - description: PVC reads the trace from a PersistentVolumeClaim in the Request namespace. + persistentVolumeClaim: + description: |- + PersistentVolumeClaim reads the trace from a PersistentVolumeClaim in the Request + namespace. properties: claimName: description: |- ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` - until its execution stops; deleting the claim cancels the Runs that read it. + until the Run completes or, for a Run deleted before it completes, until no Job of the Run + is left. Garbage collection deletes the Job of a deleted Run before the Job's pod, so the + finalizer of a deleted Run can be released while that pod still stops; Kubernetes + PersistentVolumeClaim protection keeps the claim until no pod uses it. Deleting the claim + cancels the Runs that read it. maxLength: 253 minLength: 1 pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string - path: + subPath: description: |- - Path names one trace file relative to the root of the claim. It is neither a directory - nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' - segments or a NUL character, which no file name can hold. The file must be complete before - the Request is created and must never be rewritten: every Run of the Request reads it when - its Job starts. + SubPath names one trace file relative to the root of the claim; the Run Job mounts it as + the subPath of the claim volume. It is neither a directory nor a glob. It must not start or + end with '/' and must not contain empty, '.' or '..' segments or a NUL character, which no + file name can hold. The file must be complete before the Request is created and must never + be rewritten: every Run of the Request reads it when its Job starts. maxLength: 1024 minLength: 1 type: string x-kubernetes-validations: - - message: path must be relative to the claim root without empty, '.' or '..' segments + - message: subPath must be relative to the claim root without empty, '.' or '..' segments rule: '!self.startsWith(''/'') && !self.endsWith(''/'') && !self.contains(''//'') && !self.matches(''(^|/)[.][.]?(/|$)'')' - - message: path must not contain a NUL character + - message: subPath must not contain a NUL character rule: '!self.contains(''\u0000'')' required: - claimName - - path + - subPath type: object type: object x-kubernetes-validations: - - message: pvc is required - rule: has(self.pvc) + - message: persistentVolumeClaim is required + rule: has(self.persistentVolumeClaim) required: - - format - source type: object type: object diff --git a/deploy/operator/internal/aisimulate/doc.go b/deploy/operator/internal/aisimulate/doc.go index 158e0dcd2947..63f8a3fbe335 100644 --- a/deploy/operator/internal/aisimulate/doc.go +++ b/deploy/operator/internal/aisimulate/doc.go @@ -32,10 +32,9 @@ // SLA bound is set. Last, spec.aisimulate.inputOverrides is applied on top as a JSON Merge // Patch (RFC 7386): upstream objects merge recursively, arrays and scalars replace the value, // and null deletes it. The input overrides are authoritative: they win on every path they name, -// including the derived defaults, and are applied once, last. Only the operator-owned paths are -// exempt: Request admission rejects them and the Run controller restores them after this -// lowering. They are the operator-owned dgdr_run fields and, for a trace, traffic.source.type, -// format and paths. +// including the derived defaults and the traffic a typed trace lowers, and are applied once, last. +// Only the operator-owned dgdr_run fields are exempt: Request admission rejects them and the Run +// controller restores them after this lowering. // // The document stays unstructured (map[string]any) because its schema belongs to the // AISimulate release bundled in the operator's Planner image, not to this package. Reads and writes go diff --git a/deploy/operator/internal/aisimulate/input.go b/deploy/operator/internal/aisimulate/input.go index 8010ce0fb96d..ad9a6e41fef4 100644 --- a/deploy/operator/internal/aisimulate/input.go +++ b/deploy/operator/internal/aisimulate/input.go @@ -200,7 +200,12 @@ func (b *builder) setTraceWorkload(workload *v1beta2.WorkloadSpec) { return } trace := workload.Trace - b.set(TraceSource(trace), trafficSource) + source, err := traceSource(trace) + if err != nil { + b.err = err + return + } + b.set(source, trafficSource) b.set("trace_timestamps", trafficLoadType) if replay := trace.Replay; replay != nil && replay.ArrivalSpeedupRatio != nil { b.set(*replay.ArrivalSpeedupRatio, trafficLoadSpeedup) diff --git a/deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json b/deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json new file mode 100644 index 000000000000..d3be2a10bd42 --- /dev/null +++ b/deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json @@ -0,0 +1,191 @@ +{ + "request_spec": { + "model": { + "name": "Qwen/Qwen3-32B" + }, + "backends": [ + "vllm" + ], + "hardware": { + "gpu": { + "sku": "h200_sxm", + "budget": 6 + } + }, + "workload": { + "trace": { + "source": { + "persistentVolumeClaim": { + "claimName": "request-traces", + "subPath": "windows/llm/1793000000000.jsonl.gz" + } + } + } + }, + "objective": { + "mode": "Optimize", + "metric": "GoodputPerGPU", + "sla": { + "ttftMs": 500.0, + "itlMs": 30.0 + } + }, + "recommendation": { + "maxCandidates": 8 + }, + "aisimulate": { + "inputOverrides": { + "engine": { + "mode": "aggregated", + "backend": "vllm", + "context_length": 32768, + "workers": { + "aggregated": { + "parallelism": { + "preset": [ + { + "tensor": 2, + "pipeline": 1, + "attention_data": 1, + "moe_tensor": 1, + "moe_expert": 1, + "replicas": 3 + } + ] + } + }, + "prefill": null, + "decode": null + } + }, + "router": { + "policy": "kv_router", + "prefill_load_model": { + "type": "none" + }, + "overlap_score_credit": { + "choices": [ + 0.5, + 1.0 + ] + }, + "prefill_load_scale": { + "choices": [ + 1.0, + 2.0 + ] + }, + "temperature": { + "choices": [ + 0.0, + 0.5 + ] + } + }, + "optimizer": { + "algorithm": "random", + "max_trials": 16, + "parallelism": 8, + "candidate_timeout_seconds": 300.0 + }, + "optimization": { + "strict_sla": false + }, + "dgdr_run": { + "generator": "direct" + } + } + } + }, + "run_input": { + "dgdr_run": { + "derive_runtime_images": true, + "generator": "direct", + "max_candidates": 8, + "name": "llm-rt-golden-1", + "namespace": "test-ns", + "num_gpus_per_node": 8, + "runtime_image": "example.com/dynamo-planner:1.2.3", + "runtime_version_override": "1.2.3", + "snapshot_dir": "/var/run/aisimulate/output" + }, + "engine": { + "backend": "vllm", + "context_length": 32768, + "hardware": "h200_sxm", + "mode": "aggregated", + "model": "Qwen/Qwen3-32B", + "workers": { + "aggregated": { + "parallelism": { + "preset": [ + { + "attention_data": 1, + "moe_expert": 1, + "moe_tensor": 1, + "pipeline": 1, + "replicas": 3, + "tensor": 2 + } + ] + } + } + } + }, + "evaluation": { + "sla": { + "itl_ms": 30, + "ttft_ms": 500 + } + }, + "optimization": { + "constraints": { + "max_candidate_gpus": 6 + }, + "strict_sla": false, + "target": "goodput_per_gpu" + }, + "optimizer": { + "algorithm": "random", + "candidate_timeout_seconds": 300, + "max_trials": 16, + "parallelism": 8 + }, + "router": { + "overlap_score_credit": { + "choices": [ + 0.5, + 1 + ] + }, + "policy": "kv_router", + "prefill_load_model": { + "type": "none" + }, + "prefill_load_scale": { + "choices": [ + 1, + 2 + ] + }, + "temperature": { + "choices": [ + 0, + 0.5 + ] + } + }, + "traffic": { + "load": { + "type": "trace_timestamps" + }, + "source": { + "format": "dynamo", + "paths": [ + "/var/run/aisimulate/trace/trace.jsonl.gz" + ], + "type": "trace" + } + } + } +} diff --git a/deploy/operator/internal/aisimulate/trace.go b/deploy/operator/internal/aisimulate/trace.go index 275411798ac6..0a98d2808bd2 100644 --- a/deploy/operator/internal/aisimulate/trace.go +++ b/deploy/operator/internal/aisimulate/trace.go @@ -18,6 +18,7 @@ package aisimulate import ( + "fmt" "strings" v1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" @@ -26,24 +27,39 @@ import ( // TraceMountPath is the directory of the Run Job's sweeper container that holds the trace file. const TraceMountPath = "/var/run/aisimulate/trace" -// TracePath is where the Run Job mounts the trace file at claimPath: one fixed file under -// TraceMountPath. The Job mounts only that file, through a subPath of the claim, so the kubelet -// resolves claimPath and refuses a symlink that resolves outside the claim. The name keeps the -// ".gz" suffix, by which AISimulate recognizes gzip-compressed JSON Lines. -func TracePath(claimPath string) string { - if strings.HasSuffix(claimPath, ".gz") { +// TracePath is where the Run Job mounts the trace file at subPath of the claim: one fixed file +// under TraceMountPath. The Job mounts only that file, through a subPath of the claim, so the +// kubelet resolves subPath and refuses a symlink that resolves outside the claim. The name keeps +// the ".gz" suffix, by which AISimulate recognizes gzip-compressed JSON Lines. +func TracePath(subPath string) string { + if strings.HasSuffix(subPath, ".gz") { return TraceMountPath + "/trace.jsonl.gz" } return TraceMountPath + "/trace.jsonl" } -// TraceSource returns the traffic.source object that trace owns: a trace of its format read from -// its one claim file under TraceMountPath. trace must not be nil. A trace without a pvc source has -// no paths; the CRD schema rejects that shape. -func TraceSource(trace *v1beta2.TraceWorkloadSpec) map[string]any { - source := map[string]any{"type": "trace", "format": string(trace.Format)} - if pvc := trace.Source.PVC; pvc != nil { - source["paths"] = []any{TracePath(pvc.Path)} +// traceSource returns the traffic.source object a typed trace lowers to: a trace in AISimulate's +// native format read from its one claim file under TraceMountPath. trace must not be nil. A trace +// without a persistentVolumeClaim source has no paths; the CRD schema rejects that shape. +func traceSource(trace *v1beta2.TraceWorkloadSpec) (map[string]any, error) { + format, err := nativeTraceFormat(trace.Format) + if err != nil { + return nil, err + } + source := map[string]any{"type": "trace", "format": format} + if claim := trace.Source.PersistentVolumeClaim; claim != nil { + source["paths"] = []any{TracePath(claim.SubPath)} + } + return source, nil +} + +// nativeTraceFormat maps each API trace format to AISimulate's traffic.source.format spelling. +// The CRD schema defaults an omitted format, so an empty format is an invalid input here. +func nativeTraceFormat(format v1beta2.TraceFormat) (string, error) { + switch format { + case v1beta2.TraceFormatDynamo: + return "dynamo", nil + default: + return "", fmt.Errorf("spec.workload.trace.format: unsupported trace format %q", format) } - return source } diff --git a/deploy/operator/internal/aisimulate/trace_test.go b/deploy/operator/internal/aisimulate/trace_test.go index b74752dddbfb..7214525bc46b 100644 --- a/deploy/operator/internal/aisimulate/trace_test.go +++ b/deploy/operator/internal/aisimulate/trace_test.go @@ -26,7 +26,7 @@ import ( func withTrace(spec *v1beta2.DynamoGraphDeploymentRequestSpec, speedup *float64) *v1beta2.DynamoGraphDeploymentRequestSpec { spec.Workload = &v1beta2.WorkloadSpec{Trace: &v1beta2.TraceWorkloadSpec{ Format: v1beta2.TraceFormatDynamo, - Source: v1beta2.TraceSourceSpec{PVC: &v1beta2.TracePVCSource{ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz"}}, + Source: v1beta2.TraceSourceSpec{PersistentVolumeClaim: &v1beta2.TracePVCSource{ClaimName: "request-traces", SubPath: "retune/w-1.jsonl.gz"}}, }} if speedup != nil { spec.Workload.Trace.Replay = &v1beta2.TraceReplaySpec{ArrivalSpeedupRatio: speedup} @@ -41,7 +41,7 @@ func TestLowerTraceWorkload(t *testing.T) { wantTraffic string }{ { - name: "recorded arrival times without a request stop", + name: "recorded arrival times in the native dynamo format without a request stop", spec: withTrace(defaultedSpec(), nil), wantTraffic: `{"load":{"type":"trace_timestamps"},` + `"source":{"format":"dynamo","paths":["/var/run/aisimulate/trace/trace.jsonl.gz"],"type":"trace"}}`, @@ -70,14 +70,26 @@ func TestLowerTraceWorkload(t *testing.T) { } } +func TestLowerTraceWorkloadRejectsAFormatWithoutANativeSpelling(t *testing.T) { + for _, format := range []v1beta2.TraceFormat{"", "dynamo", "mooncake"} { + t.Run(string(format), func(t *testing.T) { + spec := withTrace(defaultedSpec(), nil) + spec.Workload.Trace.Format = format + if _, err := Lower(&spec.NativeSearchInput); err == nil { + t.Fatalf("Lower accepted trace format %q", format) + } + }) + } +} + func TestTracePathIsOneFixedFileUnderTheMount(t *testing.T) { - for claimPath, want := range map[string]string{ + for subPath, want := range map[string]string{ "retune/w-1.jsonl.gz": "/var/run/aisimulate/trace/trace.jsonl.gz", "retune/w-1.jsonl": "/var/run/aisimulate/trace/trace.jsonl", "../../etc/passwd": "/var/run/aisimulate/trace/trace.jsonl", } { - if got := TracePath(claimPath); got != want { - t.Errorf("TracePath(%q) = %q, want %q", claimPath, got, want) + if got := TracePath(subPath); got != want { + t.Errorf("TracePath(%q) = %q, want %q", subPath, got, want) } } } diff --git a/deploy/operator/internal/controller/clusterenv_test.go b/deploy/operator/internal/controller/clusterenv_test.go index 9d921162729d..fb7d3dc3ec02 100644 --- a/deploy/operator/internal/controller/clusterenv_test.go +++ b/deploy/operator/internal/controller/clusterenv_test.go @@ -379,10 +379,10 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2ProducesCandidates(t *testing require.NoError(t, yaml.Unmarshal(data, scenario)) require.NotNil(t, scenario.Spec.Workload, "trace scenario input has no workload") require.NotNil(t, scenario.Spec.Workload.Trace, "trace scenario input has no trace workload") - require.NotNil(t, scenario.Spec.Workload.Trace.Source.PVC, "trace scenario input has no trace claim") + require.NotNil(t, scenario.Spec.Workload.Trace.Source.PersistentVolumeClaim, "trace scenario input has no trace claim") trace = clusterTestTraceFile{ - claim: scenario.Spec.Workload.Trace.Source.PVC.ClaimName, - path: scenario.Spec.Workload.Trace.Source.PVC.Path, + claim: scenario.Spec.Workload.Trace.Source.PersistentVolumeClaim.ClaimName, + subPath: scenario.Spec.Workload.Trace.Source.PersistentVolumeClaim.SubPath, } createClusterTestTraceClaim(t, env, trace) traceBefore = writeClusterTestTrace(t, env, trace, traceFixture, plannerImage) diff --git a/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go b/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go index 7575f40d37f2..c0b787b31872 100644 --- a/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go +++ b/deploy/operator/internal/controller/dgdr_v1beta2_trace_clustertest_test.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" "time" @@ -29,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" @@ -44,9 +46,13 @@ const ( clusterTestTraceMode = "640" ) -// clusterTestTraceFile is the claim and claim-relative path a trace Request reads. +// clusterTestCreateContainerConfigError is the kubelet's waiting reason for a container whose +// configuration it cannot apply, such as a subPath that resolves outside its volume. +const clusterTestCreateContainerConfigError = "CreateContainerConfigError" + +// clusterTestTraceFile is the claim and the claim-relative subPath a trace Request reads. type clusterTestTraceFile struct { - claim, path string + claim, subPath string } // clusterTestTraceWriter gzips the fixture to the claim file and gives it the requested owner and mode. @@ -208,7 +214,7 @@ func writeClusterTestTrace(t *testing.T, env *clusterenv.TestEnv, trace clusterT })) runClusterTestTraceJob(t, env, "trace-writer", image, trace.claim, "trace-fixture", true, clusterTestTraceWriter, - "/fixture/trace.jsonl", filepath.Join("/claim", trace.path), + "/fixture/trace.jsonl", filepath.Join("/claim", trace.subPath), fmt.Sprint(clusterTestTraceUID), fmt.Sprint(clusterTestTraceGID), clusterTestTraceMode) return statClusterTestTrace(t, env, trace, "trace-before", image) } @@ -218,7 +224,7 @@ func writeClusterTestTrace(t *testing.T, env *clusterenv.TestEnv, trace clusterT func statClusterTestTrace(t *testing.T, env *clusterenv.TestEnv, trace clusterTestTraceFile, name, image string) string { t.Helper() stat := runClusterTestTraceJob(t, env, name, image, trace.claim, "", false, clusterTestTraceStat, - filepath.Join("/claim", trace.path)) + filepath.Join("/claim", trace.subPath)) require.Regexp(t, fmt.Sprintf(`^%d %d %s [0-9a-f]{64}$`, clusterTestTraceUID, clusterTestTraceGID, clusterTestTraceMode), stat) return stat } @@ -235,7 +241,7 @@ func requireClusterTestTraceInput(t *testing.T, run *nvidiacomv1beta2.DynamoGrap } require.NoError(t, yaml.Unmarshal(run.Status.AISimulate.Input.Raw, &input)) require.Equal(t, map[string]any{ - "type": "trace", "format": "dynamo", "paths": []any{aisimulate.TracePath(trace.path)}, + "type": "trace", "format": "dynamo", "paths": []any{aisimulate.TracePath(trace.subPath)}, }, input.Traffic.Source) require.Equal(t, "trace_timestamps", input.Traffic.Load["type"]) } @@ -250,10 +256,10 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2RefusesATraceSymlinkThatLeave env.BlockReplicaSets() t.Log("Place a symlink at the trace path that points at a file outside the claim") - trace := clusterTestTraceFile{claim: "linked-traces", path: "retune/escape.jsonl.gz"} + trace := clusterTestTraceFile{claim: "linked-traces", subPath: "retune/escape.jsonl.gz"} createClusterTestTraceClaim(t, env, trace) runClusterTestTraceJob(t, env, "trace-linker", plannerImage, trace.claim, "", true, clusterTestTraceLinker, - filepath.Join("/claim", trace.path), "/etc/passwd") + filepath.Join("/claim", trace.subPath), "/etc/passwd") t.Log("Apply a Request that reads the linked trace path through production admission") request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{ @@ -268,8 +274,8 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2RefusesATraceSymlinkThatLeave }}, Workload: &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ Format: nvidiacomv1beta2.TraceFormatDynamo, - Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ - ClaimName: trace.claim, Path: trace.path, + Source: nvidiacomv1beta2.TraceSourceSpec{PersistentVolumeClaim: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: trace.claim, SubPath: trace.subPath, }}, }}, }, @@ -286,12 +292,32 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2RefusesATraceSymlinkThatLeave defer runWatch.Stop() startClusterTestDGDRV1beta2Controllers(t, env, plannerImage, operatorImage) - t.Log("The kubelet refuses the subPath, so the sweeper waits and the open Run reports it verbatim") + // An image pull or other waiting state can come first and recover; only the configuration error + // shows that the kubelet tried the trace mount and refused it. + t.Log("The kubelet refuses the subPath, so the sweeper waits with CreateContainerConfigError") + var sweeperPod types.UID + require.Eventually(t, func() bool { + var pods corev1.PodList + require.NoError(t, env.Client().List(ctx, &pods, client.InNamespace(env.Namespace()))) + for i := range pods.Items { + for _, status := range pods.Items[i].Status.ContainerStatuses { + if status.Name == dgdrRunSweeperContainer && status.State.Waiting != nil && + status.State.Waiting.Reason == clusterTestCreateContainerConfigError { + sweeperPod = pods.Items[i].UID + return true + } + } + } + return false + }, 5*time.Minute, time.Second, "the sweeper did not wait with %s", clusterTestCreateContainerConfigError) + + t.Log("The open Run reports that configuration error of the sweeper") var waiting nvidiacomv1beta2.DynamoGraphDeploymentRun - awaitClusterTestRun(t, runWatch, "the Run did not report its waiting sweeper", func(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { + awaitClusterTestRun(t, runWatch, "the Run did not report its sweeper's configuration error", func(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) bool { waiting = *run.DeepCopy() condition := meta.FindStatusCondition(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) - return condition != nil && condition.Reason == dgdrRunReasonContainerWaiting + return condition != nil && condition.Reason == dgdrRunReasonContainerWaiting && strings.HasPrefix(condition.Message, + fmt.Sprintf("container %q is waiting: %s: ", dgdrRunSweeperContainer, clusterTestCreateContainerConfigError)) }) condition := meta.FindStatusCondition(waiting.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) require.Equal(t, metav1.ConditionFalse, condition.Status) @@ -303,6 +329,7 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2RefusesATraceSymlinkThatLeave require.NoError(t, env.Client().List(ctx, &pods, client.InNamespace(env.Namespace()), client.MatchingLabels{batchv1.JobNameLabel: waiting.Status.JobRef.Name})) require.Len(t, pods.Items, 1) + require.Equal(t, sweeperPod, pods.Items[0].UID) var sweeper *corev1.ContainerStatus for i := range pods.Items[0].Status.ContainerStatuses { if pods.Items[0].Status.ContainerStatuses[i].Name == dgdrRunSweeperContainer { @@ -311,6 +338,7 @@ func TestClusterDynamoGraphDeploymentRequestV1beta2RefusesATraceSymlinkThatLeave } require.NotNil(t, sweeper) require.NotNil(t, sweeper.State.Waiting) + require.Equal(t, clusterTestCreateContainerConfigError, sweeper.State.Waiting.Reason) require.Nil(t, sweeper.State.Running) require.Nil(t, sweeper.LastTerminationState.Terminated) require.Zero(t, sweeper.RestartCount) diff --git a/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go b/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go new file mode 100644 index 000000000000..c8814cfe8e0e --- /dev/null +++ b/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go @@ -0,0 +1,162 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +import ( + "bytes" + "encoding/json" + "flag" + "os" + "path/filepath" + "testing" + + nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// updateTraceRequestLoweringGolden rewrites the golden file instead of comparing with it: +// +// go test ./internal/controller -run '^TestDGDRV1beta2TraceRequestLoweringGolden$' -update +var updateTraceRequestLoweringGolden = flag.Bool("update", false, + "rewrite internal/aisimulate/testdata/trace_request_lowering.golden.json") + +// traceRequestLoweringGoldenPath is read by the live-retune planner's tests, which check that the +// planner builds request_spec and accepts run_input as the effective input of its Run. +const traceRequestLoweringGoldenPath = "../aisimulate/testdata/trace_request_lowering.golden.json" + +// liveRetuneShardRequestSpec is the spec of one grid shard the live-retune planner submits: its +// build_request for an aggregated vLLM deployment with 3 replicas of tensor parallelism 2, the window +// ending at 1793000000000 ms, and a 2x2x2 router grid. It omits workload.trace.format, which the +// CRD schema defaults. Change it together with the planner and rerun with -update. +const liveRetuneShardRequestSpec = `{ + "model": {"name": "Qwen/Qwen3-32B"}, + "backends": ["vllm"], + "hardware": {"gpu": {"sku": "h200_sxm", "budget": 6}}, + "workload": { + "trace": { + "source": { + "persistentVolumeClaim": { + "claimName": "request-traces", + "subPath": "windows/llm/1793000000000.jsonl.gz" + } + } + } + }, + "objective": { + "mode": "Optimize", + "metric": "GoodputPerGPU", + "sla": {"ttftMs": 500.0, "itlMs": 30.0} + }, + "recommendation": {"maxCandidates": 8}, + "aisimulate": { + "inputOverrides": { + "engine": { + "mode": "aggregated", + "backend": "vllm", + "context_length": 32768, + "workers": { + "aggregated": { + "parallelism": { + "preset": [ + {"tensor": 2, "pipeline": 1, "attention_data": 1, "moe_tensor": 1, "moe_expert": 1, "replicas": 3} + ] + } + }, + "prefill": null, + "decode": null + } + }, + "router": { + "policy": "kv_router", + "prefill_load_model": {"type": "none"}, + "overlap_score_credit": {"choices": [0.5, 1.0]}, + "prefill_load_scale": {"choices": [1.0, 2.0]}, + "temperature": {"choices": [0.0, 0.5]} + }, + "optimizer": { + "algorithm": "random", + "max_trials": 16, + "parallelism": 8, + "candidate_timeout_seconds": 300.0 + }, + "optimization": {"strict_sla": false}, + "dgdr_run": {"generator": "direct"} + } + } +}` + +// The live-retune planner submits a trace Request and checks the input its Run executes. Both +// sides share one golden file, so a change to the trace API, admission or lowering that the planner +// does not follow fails here or in the planner's tests. +func TestDGDRV1beta2TraceRequestLoweringGolden(t *testing.T) { + env := sharedEnv.ForTest(t) + c := env.Client() + ctx := t.Context() + + t.Log("Decode the planner's shard Request; every field must belong to the v1beta2 API") + var spec nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec + decoder := json.NewDecoder(bytes.NewReader([]byte(liveRetuneShardRequestSpec))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&spec); err != nil { + t.Fatalf("decode the planner's Request spec: %v", err) + } + + t.Log("Create the Request through admission and the CRD schema, which complete it") + request := &nvidiacomv1beta2.DynamoGraphDeploymentRequest{Name: "llm-rt-golden", Namespace: env.Namespace(), Spec: spec} + if err := c.Create(ctx, request); err != nil { + t.Fatalf("create the Request: %v", err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(request), request); err != nil { + t.Fatalf("get the Request: %v", err) + } + + t.Log("Freeze the admitted Request into its Run as the Request controller does, and store the Run") + run, err := (&DynamoGraphDeploymentRequestV1beta2Reconciler{Client: c}).buildRun(request) + if err != nil { + t.Fatalf("build the Run: %v", err) + } + if err := c.Create(ctx, run); err != nil { + t.Fatalf("create the Run: %v", err) + } + if err := c.Get(ctx, client.ObjectKeyFromObject(run), run); err != nil { + t.Fatalf("get the Run: %v", err) + } + + t.Log("Lower the stored Run as the Run controller does, in a fixed namespace so the golden file is stable") + run.Namespace = "test-ns" + input, err := dgdrRunInput(run, "example.com/dynamo-planner:1.2.3", "1.2.3") + if err != nil { + t.Fatalf("lower the Run: %v", err) + } + got, err := json.MarshalIndent(struct { + RequestSpec json.RawMessage `json:"request_spec"` + RunInput map[string]any `json:"run_input"` + }{RequestSpec: json.RawMessage(liveRetuneShardRequestSpec), RunInput: input}, "", " ") + if err != nil { + t.Fatalf("encode the golden file: %v", err) + } + got = append(got, '\n') + + t.Log("Compare with the golden file, or rewrite it with -update") + if *updateTraceRequestLoweringGolden { + if err := os.MkdirAll(filepath.Dir(traceRequestLoweringGoldenPath), 0o755); err != nil { + t.Fatalf("create the golden directory: %v", err) + } + if err := os.WriteFile(traceRequestLoweringGoldenPath, got, 0o644); err != nil { + t.Fatalf("write the golden file: %v", err) + } + } + want, err := os.ReadFile(traceRequestLoweringGoldenPath) + if err != nil { + t.Fatalf("read the golden file: %v", err) + } + if !bytes.Equal(got, want) { + t.Fatalf("%s is stale; rerun with -update and update the planner if needed\n got: %s\nwant: %s", + traceRequestLoweringGoldenPath, got, want) + } +} diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go index e3dcb415e0db..fcb829431ec2 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller.go @@ -12,7 +12,6 @@ import ( "encoding/json" "errors" "fmt" - "maps" "strconv" "strings" "time" @@ -47,7 +46,7 @@ import ( const ( dgdrRunJobControllerUIDIndex = ".metadata.dgdrRunControllerUID" - dgdrRunTraceClaimIndex = ".spec.workload.trace.source.pvc.claimName" + dgdrRunTraceClaimIndex = ".spec.workload.trace.source.persistentVolumeClaim.claimName" dgdrRunMaxJobNameLength = 63 dgdrRunInputKey = "input.yaml" @@ -129,7 +128,7 @@ type DynamoGraphDeploymentRunV1beta2Reconciler struct { // +kubebuilder:rbac:groups=nvidia.com,resources=dynamographdeploymentcandidates/status,verbs=get;update;patch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch -// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;patch;update +// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list;watch;patch // +kubebuilder:rbac:groups="",resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch @@ -554,6 +553,8 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) cancelForDeletedTraceClaim( jobs []batchv1.Job, source *nvidiacomv1beta2.TracePVCSource, ) (ctrl.Result, error) { + message := fmt.Sprintf("trace PersistentVolumeClaim %q in namespace %q was deleted, which cancels the Run", + source.ClaimName, run.Namespace) if len(jobs) == 0 { live, err := r.uncachedOwnedJobs(ctx, run) if err != nil { @@ -564,11 +565,16 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) cancelForDeletedTraceClaim( return ctrl.Result{RequeueAfter: dgdrRunExecutionCacheRetry}, nil } } + + // The API server has just confirmed that no Job is left, so the Run completes without + // listing the Jobs again. + if len(live) == 0 { + return ctrl.Result{}, r.completeRun(ctx, run, dgdrRunReasonTraceSourceDeleted, message) + } jobs = live } - completed, err := r.stopAndCompleteRun(ctx, run, jobs, "for its deleted trace claim", dgdrRunReasonTraceSourceDeleted, - fmt.Sprintf("trace PersistentVolumeClaim %q in namespace %q was deleted, which cancels the Run", source.ClaimName, run.Namespace)) + completed, err := r.stopAndCompleteRun(ctx, run, jobs, "for its deleted trace claim", dgdrRunReasonTraceSourceDeleted, message) if err != nil || completed { return ctrl.Result{}, err } @@ -635,7 +641,7 @@ func dgdrRunTraceClaim(run *nvidiacomv1beta2.DynamoGraphDeploymentRun) *nvidiaco if run.Spec.Workload == nil || run.Spec.Workload.Trace == nil { return nil } - return run.Spec.Workload.Trace.Source.PVC + return run.Spec.Workload.Trace.Source.PersistentVolumeClaim } // dgdrRunTraceClaimNames indexes a Run by the claim its trace is read from. @@ -651,7 +657,10 @@ func dgdrRunTraceClaimNames(obj client.Object) []string { } // runsForTraceClaim maps a claim event to the Runs in the claim's namespace that read their trace -// from it. +// from it and that the claim can still affect: a Run that has not completed, or a completed Run +// whose finalizer the claim still carries and that has yet to release it. Many Runs can share one +// claim, so waking every completed Run on each finalizer change would reconcile them all again +// for nothing. func (r *DynamoGraphDeploymentRunV1beta2Reconciler) runsForTraceClaim(ctx context.Context, claim client.Object) []reconcile.Request { runs := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} if err := r.List(ctx, runs, client.InNamespace(claim.GetNamespace()), @@ -661,7 +670,12 @@ func (r *DynamoGraphDeploymentRunV1beta2Reconciler) runsForTraceClaim(ctx contex } requests := make([]reconcile.Request, 0, len(runs.Items)) for i := range runs.Items { - requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&runs.Items[i])}) + run := &runs.Items[i] + if meta.IsStatusConditionTrue(run.Status.Conditions, nvidiacomv1beta2.DGDRRunConditionCompleted) && + !controllerutil.ContainsFinalizer(claim, dgdrRunTraceClaimFinalizer(run.UID)) { + continue + } + requests = append(requests, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(run)}) } return requests } @@ -1033,24 +1047,6 @@ func dgdrRunInput( return nil, fmt.Errorf("set operator-owned dgdr_run.%s: %w", key, err) } } - - // The typed trace owns traffic.source.type, format and paths the same way. Restoring them keeps a - // Run created without Request admission reading only the file on its read-only mount. Such a Run - // can also replace traffic or traffic.source with a non-object; that parent is rebuilt rather than - // failing every reconcile. - if run.Spec.Workload != nil && run.Spec.Workload.Trace != nil { - traffic, ok := input["traffic"].(map[string]any) - if !ok { - traffic = map[string]any{} - input["traffic"] = traffic - } - source, ok := traffic["source"].(map[string]any) - if !ok { - source = map[string]any{} - traffic["source"] = source - } - maps.Copy(source, aisimulate.TraceSource(run.Spec.Workload.Trace)) - } return input, nil } @@ -1567,7 +1563,7 @@ func dgdrRunSweeperVolumeMounts(run *nvidiacomv1beta2.DynamoGraphDeploymentRun, } if source := dgdrRunTraceClaim(run); source != nil { mounts = append(mounts, corev1.VolumeMount{ - Name: dgdrRunTraceVolume, MountPath: aisimulate.TracePath(source.Path), SubPath: source.Path, ReadOnly: true, + Name: dgdrRunTraceVolume, MountPath: aisimulate.TracePath(source.SubPath), SubPath: source.SubPath, ReadOnly: true, }) } return mounts diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 21925d744b57..8f5c79523b41 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -1813,8 +1813,8 @@ func dgdrRunWithTraceForTest(t *testing.T, inputOverrides map[string]any) *nvidi run := dgdrRunForTest(t, inputOverrides) run.Spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ Format: nvidiacomv1beta2.TraceFormatDynamo, - Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ - ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz", + Source: nvidiacomv1beta2.TraceSourceSpec{PersistentVolumeClaim: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", SubPath: "retune/w-1.jsonl.gz", }}, }} return run @@ -1898,64 +1898,67 @@ func TestV1beta2RunControllerRequestsNoPodGroupsWithoutATrace(t *testing.T) { } } -func TestDGDRRunInputEnforcesTheTypedTraceSource(t *testing.T) { - t.Log("Lower a Run whose overrides try to replace the trace source and change the load") - run := dgdrRunWithTraceForTest(t, map[string]any{"traffic": map[string]any{ - "source": map[string]any{"type": "synthetic", "format": "mooncake", "paths": []any{"/etc/passwd"}}, - "load": map[string]any{"speedup": 4}, - }}) - input, err := dgdrRunInput(run, "example.com/dynamo-planner:1.2.3", "1.2.3") - if err != nil { - t.Fatalf("lower input: %v", err) - } - - t.Log("The typed trace wins for the source; the load override is kept") - traffic := input["traffic"].(map[string]any) - wantSource := map[string]any{"type": "trace", "format": "dynamo", "paths": []any{"/var/run/aisimulate/trace/trace.jsonl.gz"}} - if !reflect.DeepEqual(traffic["source"], wantSource) { - t.Fatalf("traffic.source = %#v, want %#v", traffic["source"], wantSource) - } - load := traffic["load"].(map[string]any) - if load["type"] != "trace_timestamps" || load["speedup"] != json.Number("4") { - t.Fatalf("traffic.load = %#v, want trace_timestamps with the override speedup", load) - } - if _, found := traffic["stop"]; found { - t.Fatalf("traffic.stop = %#v, want none for a trace", traffic["stop"]) - } -} - -func TestDGDRRunInputRebuildsANonObjectTraceSourceParent(t *testing.T) { - wantSource := map[string]any{"type": "trace", "format": "dynamo", "paths": []any{"/var/run/aisimulate/trace/trace.jsonl.gz"}} +func TestDGDRRunInputLetsOverridesReplaceOrDeleteTheTypedTraceSource(t *testing.T) { + typedSource := map[string]any{"type": "trace", "format": "dynamo", "paths": []any{"/var/run/aisimulate/trace/trace.jsonl.gz"}} tests := []struct { - name string - overrides map[string]any - wantLoad map[string]any + name string + overrides map[string]any + wantTraffic any + wantFound bool }{ { - name: "scalar traffic", - overrides: map[string]any{"traffic": "invalid"}, + name: "no override keeps the typed underlay", + overrides: map[string]any{}, + wantTraffic: map[string]any{"source": typedSource, "load": map[string]any{"type": "trace_timestamps"}}, + wantFound: true, + }, + { + name: "replaced source fields and load are kept", + overrides: map[string]any{"traffic": map[string]any{ + "source": map[string]any{"type": "synthetic", "format": "mooncake", "paths": []any{"/etc/passwd"}}, + "load": map[string]any{"speedup": 4}, + }}, + wantTraffic: map[string]any{ + "source": map[string]any{"type": "synthetic", "format": "mooncake", "paths": []any{"/etc/passwd"}}, + "load": map[string]any{"type": "trace_timestamps", "speedup": json.Number("4")}, + }, + wantFound: true, + }, + { + name: "a null source stays deleted", + overrides: map[string]any{"traffic": map[string]any{"source": nil}}, + wantTraffic: map[string]any{"load": map[string]any{"type": "trace_timestamps"}}, + wantFound: true, }, { - name: "scalar traffic.source", - overrides: map[string]any{"traffic": map[string]any{"source": "invalid"}}, - wantLoad: map[string]any{"type": "trace_timestamps"}, + name: "a null traffic stays deleted", + overrides: map[string]any{"traffic": nil}, + }, + { + name: "a scalar traffic.source replaces the object", + overrides: map[string]any{"traffic": map[string]any{"source": "invalid"}}, + wantTraffic: map[string]any{"source": "invalid", "load": map[string]any{"type": "trace_timestamps"}}, + wantFound: true, + }, + { + name: "a scalar traffic replaces the object", + overrides: map[string]any{"traffic": "invalid"}, + wantTraffic: "invalid", + wantFound: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - t.Log("Lower a Run created without admission whose override replaces a trace source parent") + t.Log("Lower a trace Run whose overrides name the traffic the trace lowers") input, err := dgdrRunInput(dgdrRunWithTraceForTest(t, tt.overrides), "example.com/dynamo-planner:1.2.3", "1.2.3") if err != nil { t.Fatalf("lower input: %v", err) } - t.Log("The parent is an object again and carries the typed trace source") - traffic := input["traffic"].(map[string]any) - if !reflect.DeepEqual(traffic["source"], wantSource) { - t.Fatalf("traffic.source = %#v, want %#v", traffic["source"], wantSource) - } - if load, _ := traffic["load"].(map[string]any); !reflect.DeepEqual(load, tt.wantLoad) { - t.Fatalf("traffic.load = %#v, want %#v", traffic["load"], tt.wantLoad) + t.Log("The overrides win, and what they delete is not restored") + traffic, found := input["traffic"] + if found != tt.wantFound || !reflect.DeepEqual(traffic, tt.wantTraffic) { + t.Fatalf("traffic = %#v (found %v), want %#v (found %v)", traffic, found, tt.wantTraffic, tt.wantFound) } }) } @@ -2967,16 +2970,30 @@ func TestV1beta2RunControllerWakesTheRunsThatReadAChangedTraceClaim(t *testing.T reader := dgdrRunWithTraceForTest(t, map[string]any{}) otherClaim := dgdrRunWithTraceForTest(t, map[string]any{}) otherClaim.Name = "search-2" - otherClaim.Spec.Workload.Trace.Source.PVC.ClaimName = "other-traces" + otherClaim.Spec.Workload.Trace.Source.PersistentVolumeClaim.ClaimName = "other-traces" otherNamespace := dgdrRunWithTraceForTest(t, map[string]any{}) otherNamespace.Namespace = "elsewhere" withoutTrace := dgdrRunForTest(t, map[string]any{}) withoutTrace.Name = "search-3" - f := newDGDRRunFixture(t, reader, otherClaim, otherNamespace, withoutTrace) + completedHolding := dgdrRunWithTraceForTest(t, map[string]any{}) + completedHolding.Name, completedHolding.UID = "search-4", "run-uid-4" + completedReleased := dgdrRunWithTraceForTest(t, map[string]any{}) + completedReleased.Name, completedReleased.UID = "search-5", "run-uid-5" + for _, run := range []*nvidiacomv1beta2.DynamoGraphDeploymentRun{completedHolding, completedReleased} { + meta.SetStatusCondition(&run.Status.Conditions, metav1.Condition{ + Type: nvidiacomv1beta2.DGDRRunConditionCompleted, Status: metav1.ConditionTrue, Reason: dgdrRunReasonSucceeded, + }) + } + f := newDGDRRunFixture(t, reader, otherClaim, otherNamespace, withoutTrace, completedHolding, completedReleased) - t.Log("An event on request-traces in default wakes only the Run that reads it") - got := f.reconciler.runsForTraceClaim(t.Context(), traceClaimForTest()) - want := []reconcile.Request{{NamespacedName: client.ObjectKeyFromObject(reader)}} + t.Log("An event on request-traces in default wakes the open Run that reads it and the completed Run whose finalizer it still carries") + claim := traceClaimForTest() + claim.Finalizers = []string{dgdrRunTraceClaimFinalizer(completedHolding.UID)} + got := f.reconciler.runsForTraceClaim(t.Context(), claim) + want := []reconcile.Request{ + {NamespacedName: client.ObjectKeyFromObject(reader)}, + {NamespacedName: client.ObjectKeyFromObject(completedHolding)}, + } if !reflect.DeepEqual(got, want) { t.Fatalf("requests = %v, want %v", got, want) } @@ -3091,18 +3108,31 @@ func TestV1beta2RunControllerCancelsAJobWhoseJobRefWasNeverRecorded(t *testing.T t.Fatalf("Completed condition = %+v, want none until the stop is confirmed", condition) } - t.Log("Once the API server confirms the Job is gone, the Run is canceled") + t.Log("Once the API server confirms the Job is gone, the Run is canceled without a second uncached Job list") + apiJobLists := 0 + f.reconciler.APIReader = interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*batchv1.JobList); ok { + apiJobLists++ + } + return c.List(ctx, list, opts...) + }, + }) f.reconciler.Client = cached f.reconcile() assertDGDRRunCanceledForTrace(t, f.run(), "") + if apiJobLists != 1 { + t.Fatalf("uncached Job lists = %d, want 1", apiJobLists) + } } // newDGDRRunTraceClaimFixture returns a trace claim reconciler over a fake client holding objects, -// whose cached Job List the returned hide flag can make empty. +// whose cached Job List the returned hide flag can make empty, and the count of the Job Lists its +// uncached reader serves. func newDGDRRunTraceClaimFixture( t *testing.T, objects ...client.Object, -) (*DynamoGraphDeploymentRunTraceClaimReconciler, client.Client, *bool) { +) (*DynamoGraphDeploymentRunTraceClaimReconciler, client.Client, *bool, *int) { t.Helper() f := newDGDRRunFixture(t, objects...) hide := new(bool) @@ -3114,7 +3144,16 @@ func newDGDRRunTraceClaimFixture( return c.List(ctx, list, opts...) }, }) - return &DynamoGraphDeploymentRunTraceClaimReconciler{Client: cached, APIReader: f.client}, f.client, hide + apiJobLists := new(int) + uncached := interceptor.NewClient(f.client.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*batchv1.JobList); ok { + *apiJobLists++ + } + return c.List(ctx, list, opts...) + }, + }) + return &DynamoGraphDeploymentRunTraceClaimReconciler{Client: cached, APIReader: uncached}, f.client, hide, apiJobLists } func TestDGDRRunTraceClaimReconcilerReleasesOnlyRunsThatCanNoLongerExecute(t *testing.T) { @@ -3143,6 +3182,7 @@ func TestDGDRRunTraceClaimReconcilerReleasesOnlyRunsThatCanNoLongerExecute(t *te {name: "a deleted Run whose Job only the API server shows", jobFor: []types.UID{"gone-uid"}, hideJobs: true, uids: []types.UID{"gone-uid"}, want: []string{dgdrRunTraceClaimFinalizer("gone-uid")}}, {name: "a deleted Run whose Job is gone", uids: []types.UID{"gone-uid"}}, + {name: "two deleted Runs whose Jobs are gone", uids: []types.UID{"gone-uid", "other-gone-uid"}}, {name: "a Run being deleted whose Job remains", runs: []*nvidiacomv1beta2.DynamoGraphDeploymentRun{deleting}, jobFor: []types.UID{"deleting-uid"}, uids: []types.UID{"deleting-uid"}, want: []string{dgdrRunTraceClaimFinalizer("deleting-uid")}}, @@ -3175,7 +3215,7 @@ func TestDGDRRunTraceClaimReconcilerReleasesOnlyRunsThatCanNoLongerExecute(t *te }, }) } - reconciler, c, hide := newDGDRRunTraceClaimFixture(t, objects...) + reconciler, c, hide, apiJobLists := newDGDRRunTraceClaimFixture(t, objects...) *hide = tt.hideJobs t.Log("Reconcile the claim") @@ -3192,6 +3232,11 @@ func TestDGDRRunTraceClaimReconcilerReleasesOnlyRunsThatCanNoLongerExecute(t *te if !reflect.DeepEqual(stored.Finalizers, want) { t.Fatalf("finalizers = %v, want %v", stored.Finalizers, want) } + + t.Log("One uncached Job list at most confirms every stale finalizer") + if *apiJobLists > 1 { + t.Fatalf("uncached Job lists = %d, want at most 1", *apiJobLists) + } }) } } @@ -3232,7 +3277,7 @@ func TestDGDRRunTraceClaimReconcilerWatchesOnlyWhatReleasesAClaim(t *testing.T) withoutTrace := dgdrRunForTest(t, map[string]any{}) claimRequest := []reconcile.Request{{Namespace: "default", Name: "request-traces"}} unowned := &batchv1.Job{Name: "nightly", Namespace: "default", Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{Volumes: []corev1.Volume{dgdrRunTraceClaimVolume(run.Spec.Workload.Trace.Source.PVC)}}, + Spec: corev1.PodSpec{Volumes: []corev1.Volume{dgdrRunTraceClaimVolume(run.Spec.Workload.Trace.Source.PersistentVolumeClaim)}}, }}} t.Log("Claims pass the filter only while a Run protects them") diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_trace_claim_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_trace_claim_controller.go index c6db3a4392c6..884bd23a1857 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_trace_claim_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_trace_claim_controller.go @@ -32,6 +32,11 @@ import ( // deleted, by its Request or with its namespace, before it released its claim. The claim keeps a // Run's finalizer while that Run exists and is not being deleted, or while a Job the Run controls // exists. The Run reconciler releases the finalizer of a completed Run itself. +// +// For a deleted Run, no Job does not mean no pod: garbage collection deletes the Run's Job in the +// background, before the Job's pod has stopped, so the finalizer can be released while that pod +// still has the trace file mounted. Kubernetes PersistentVolumeClaim protection, not this +// finalizer, keeps the claim until the pod is gone. type DynamoGraphDeploymentRunTraceClaimReconciler struct { client.Client APIReader client.Reader @@ -45,20 +50,29 @@ func (r *DynamoGraphDeploymentRunTraceClaimReconciler) Reconcile(ctx context.Con return ctrl.Result{}, client.IgnoreNotFound(err) } + // The finalizers of Runs that the cache shows gone and without a Job. + stale, err := r.staleTraceClaimFinalizers(ctx, claim) + if err != nil || len(stale) == 0 { + return ctrl.Result{}, err + } + + // The Job cache can miss a Job a Run created just before it was deleted, and releasing the claim + // cannot be undone, so one uncached list of the namespace's Jobs confirms every stale finalizer. + live := &batchv1.JobList{} + if err := r.APIReader.List(ctx, live, client.InNamespace(claim.Namespace)); err != nil { + return ctrl.Result{}, fmt.Errorf("list the Jobs of the Runs that read trace PersistentVolumeClaim %q from the API: %w", + claim.Name, err) + } + for i := range live.Items { + for _, uid := range dgdrRunJobControllerUIDs(&live.Items[i]) { + delete(stale, dgdrRunTraceClaimFinalizer(types.UID(uid))) + } + } + // Drop the finalizer of every Run that can no longer execute against the claim. original := claim.DeepCopy() - for _, finalizer := range original.Finalizers { - uid, ok := strings.CutPrefix(finalizer, dgdrRunTraceClaimFinalizerPrefix) - if !ok { - continue - } - held, err := r.runHoldsClaim(ctx, claim, types.UID(uid)) - if err != nil { - return ctrl.Result{}, err - } - if !held { - controllerutil.RemoveFinalizer(claim, finalizer) - } + for finalizer := range stale { + controllerutil.RemoveFinalizer(claim, finalizer) } if len(claim.Finalizers) == len(original.Finalizers) { return ctrl.Result{}, nil @@ -69,46 +83,44 @@ func (r *DynamoGraphDeploymentRunTraceClaimReconciler) Reconcile(ctx context.Con return ctrl.Result{}, nil } -// runHoldsClaim reports whether the Run with uid may still execute against claim: the Run exists -// and is not being deleted, or a Job it controls exists. A Run never appears after its finalizer, so -// the cache answers for the Run. The Job cache can miss a Job the Run created just before it was -// deleted, and releasing the claim cannot be undone, so the API server confirms that no such Job is -// left before the claim is released. claim must not be nil. -func (r *DynamoGraphDeploymentRunTraceClaimReconciler) runHoldsClaim( +// staleTraceClaimFinalizers returns the Run finalizers on claim that the cache no longer justifies: +// the Run is gone or being deleted, and the cache shows no Job it controls. A Run never appears after +// its finalizer, so the cache answers for the Run; only the Jobs need the API server's confirmation, +// which the caller makes once for all of them. claim must not be nil. +func (r *DynamoGraphDeploymentRunTraceClaimReconciler) staleTraceClaimFinalizers( ctx context.Context, claim *corev1.PersistentVolumeClaim, - uid types.UID, -) (bool, error) { +) (map[string]struct{}, error) { runs := &nvidiacomv1beta2.DynamoGraphDeploymentRunList{} if err := r.List(ctx, runs, client.InNamespace(claim.Namespace), client.MatchingFields{dgdrRunTraceClaimIndex: claim.Name}); err != nil { - return false, fmt.Errorf("list the Runs that read trace PersistentVolumeClaim %q: %w", claim.Name, err) + return nil, fmt.Errorf("list the Runs that read trace PersistentVolumeClaim %q: %w", claim.Name, err) } + active := make(map[string]struct{}, len(runs.Items)) for i := range runs.Items { - if runs.Items[i].UID == uid && runs.Items[i].DeletionTimestamp == nil { - return true, nil + if runs.Items[i].DeletionTimestamp == nil { + active[dgdrRunTraceClaimFinalizer(runs.Items[i].UID)] = struct{}{} } } - // A Job of the Run, as the cache and then the API server show it. - jobs := &batchv1.JobList{} - if err := r.List(ctx, jobs, client.InNamespace(claim.Namespace), - client.MatchingFields{dgdrRunJobControllerUIDIndex: string(uid)}); err != nil { - return false, fmt.Errorf("list the Jobs of Run %q: %w", uid, err) - } - if len(jobs.Items) > 0 { - return true, nil - } - live := &batchv1.JobList{} - if err := r.APIReader.List(ctx, live, client.InNamespace(claim.Namespace)); err != nil { - return false, fmt.Errorf("list the Jobs of Run %q from the API: %w", uid, err) - } - for i := range live.Items { - if slices.Contains(dgdrRunJobControllerUIDs(&live.Items[i]), string(uid)) { - return true, nil + stale := map[string]struct{}{} + for _, finalizer := range claim.Finalizers { + uid, ok := strings.CutPrefix(finalizer, dgdrRunTraceClaimFinalizerPrefix) + if _, held := active[finalizer]; !ok || held { + continue + } + + // A Job of the Run as the cache shows it. + jobs := &batchv1.JobList{} + if err := r.List(ctx, jobs, client.InNamespace(claim.Namespace), + client.MatchingFields{dgdrRunJobControllerUIDIndex: uid}); err != nil { + return nil, fmt.Errorf("list the Jobs of Run %q: %w", uid, err) + } + if len(jobs.Items) == 0 { + stale[finalizer] = struct{}{} } } - return false, nil + return stale, nil } // dgdrRunProtectsTraceClaim reports whether obj carries the finalizer of a Run. diff --git a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml index 3dc482a2bf68..5fe13fc54714 100644 --- a/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml +++ b/deploy/operator/internal/controller/testdata/dgdr-v1beta2/vllm-trace/input.yaml @@ -14,11 +14,11 @@ spec: - vllm workload: trace: - format: dynamo + # format defaults to Dynamo. source: - pvc: + persistentVolumeClaim: claimName: request-traces - path: retune/w-1.jsonl.gz + subPath: retune/w-1.jsonl.gz replay: arrivalSpeedupRatio: 2.0 objective: diff --git a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go index 18cae8332cd5..6abb2fb1f707 100644 --- a/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/defaulting/dynamographdeploymentrequest_v1beta2.go @@ -212,7 +212,8 @@ func defaultGPUHardwareSpecV1beta2(gpu *nvidiacomv1beta2.GPUHardwareSpec, defaul } func defaultWorkloadSpecV1beta2(workload *nvidiacomv1beta2.WorkloadSpec) { - // A trace is the complete traffic model; it receives no static default and no format default. + // A trace is the complete traffic model and receives no static default. The CRD schema, not + // admission, defaults its format. if workload.Trace != nil { return } diff --git a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go index 6135cc8595e8..8d92e344de86 100644 --- a/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go +++ b/deploy/operator/internal/webhook/defaulting/suite_envtest_test.go @@ -368,12 +368,11 @@ func TestDynamoGraphDeploymentRequestV1beta2DefaultingAdmission(t *testing.T) { check: expectV1beta2ExplicitValues, }, { - name: "v1beta2 trace workload receives no static default", + name: "v1beta2 trace workload receives the format default and no static default", request: v1beta2DefaultingRequest(func(spec *nvidiacomv1beta2.DynamoGraphDeploymentRequestSpec) { spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ - Format: nvidiacomv1beta2.TraceFormatDynamo, - Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ - ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz", + Source: nvidiacomv1beta2.TraceSourceSpec{PersistentVolumeClaim: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", SubPath: "retune/w-1.jsonl.gz", }}, }} }), @@ -588,8 +587,8 @@ func expectV1beta2TraceWithoutStaticDefault(t *testing.T, request *nvidiacomv1be t.Helper() want := &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ Format: nvidiacomv1beta2.TraceFormatDynamo, - Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ - ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz", + Source: nvidiacomv1beta2.TraceSourceSpec{PersistentVolumeClaim: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", SubPath: "retune/w-1.jsonl.gz", }}, }} if !reflect.DeepEqual(request.Spec.Workload, want) { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go index 3d4b1698ae54..5759d4c685dc 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_carrier_envtest_test.go @@ -50,13 +50,13 @@ func TestDynamoGraphDeploymentRequestCarrierObjectsStayWritable(t *testing.T) { t.Run("native v1beta2 trace written through v1beta1", func(t *testing.T) { foreign := carrierRoundTrip(t, beta2DGDRForAdmission(setBeta2AdmissionTrace), betaDGDRForAdmission(nil), "v1beta2") - t.Log("The v1beta1 carrier holds the complete trace and no static default") + t.Log("The v1beta1 carrier holds the complete trace with its defaulted format and no static default") trace, found, err := unstructured.NestedMap(foreign.Object, "spec", "v1beta2", "workload", "trace") if err != nil || !found { t.Fatalf("spec.v1beta2.workload.trace found=%v err=%v", found, err) } - path, _, _ := unstructured.NestedString(trace, "source", "pvc", "path") - if trace["format"] != "dynamo" || path != "retune/w-1.jsonl.gz" { + subPath, _, _ := unstructured.NestedString(trace, "source", "persistentVolumeClaim", "subPath") + if trace["format"] != "Dynamo" || subPath != "retune/w-1.jsonl.gz" { t.Fatalf("spec.v1beta2.workload.trace = %#v", trace) } if _, found, _ := unstructured.NestedFieldNoCopy(foreign.Object, "spec", "v1beta2", "workload", "static"); found { @@ -68,9 +68,8 @@ func TestDynamoGraphDeploymentRequestCarrierObjectsStayWritable(t *testing.T) { func setBeta2AdmissionTrace(request *nvidiacomv1beta2.DynamoGraphDeploymentRequest) { setBeta2AdmissionHardware(request) request.Spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ - Format: nvidiacomv1beta2.TraceFormatDynamo, - Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ - ClaimName: "request-traces", Path: "retune/w-1.jsonl.gz", + Source: nvidiacomv1beta2.TraceSourceSpec{PersistentVolumeClaim: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", SubPath: "retune/w-1.jsonl.gz", }}, }} } diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go index 60bb57cf789f..e5b29131ef65 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2.go @@ -66,7 +66,7 @@ func (v *dynamoGraphDeploymentRequestValidation) validateDynamoGraphDeploymentRe if spec.AISimulate == nil { return allErrs } - return append(allErrs, validateAISimulateSpec(spec.AISimulate, spec.Workload, fldPath.Child("aisimulate"))...) + return append(allErrs, validateAISimulateSpec(spec.AISimulate, fldPath.Child("aisimulate"))...) } // validateHardwareSpecV1beta2 validates hardware. spec and fldPath must not be nil. diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go index 201576e1f9a0..bd955a3e54b5 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_v1beta2_input_overrides.go @@ -23,11 +23,9 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) -// validateAISimulateSpec validates the Request's native AISimulate overrides. workload may be -// nil; a workload with a trace owns the trace source the overrides may not change. +// validateAISimulateSpec validates the Request's native AISimulate overrides. func validateAISimulateSpec( input *nvidiacomv1beta2.AISimulateSpec, - workload *nvidiacomv1beta2.WorkloadSpec, fldPath *field.Path, ) field.ErrorList { // Use the lowering package's decoder as the single interpretation of RawExtension. @@ -37,21 +35,16 @@ func validateAISimulateSpec( return nil } - allErrs := field.ErrorList{} - if workload != nil && workload.Trace != nil { - allErrs = validateTraceOverrides(overrides, fldPath.Child("inputOverrides")) - } - dgdrRunValue, found := overrides["dgdr_run"] if !found { - return allErrs + return nil } dgdrRun, ok := dgdrRunValue.(map[string]any) if !ok { - return append(allErrs, field.Forbidden( + return field.ErrorList{field.Forbidden( fldPath.Child("inputOverrides", "dgdr_run"), "must be an object because the operator owns required execution fields within it", - )) + )} } // Search overrides remain authoritative; only execution values coupled to the @@ -66,6 +59,7 @@ func validateAISimulateSpec( "runtime_version_override", "snapshot_dir", } + allErrs := field.ErrorList{} for _, key := range reserved { if _, found := dgdrRun[key]; found { allErrs = append(allErrs, field.Forbidden( @@ -76,37 +70,3 @@ func validateAISimulateSpec( } return allErrs } - -// validateTraceOverrides rejects the traffic overrides that would replace what a trace workload -// owns: traffic.source.type, format and paths, or a traffic or traffic.source value that is not an -// object and would delete them. Every other traffic override is AISimulate input, which AISimulate -// validates itself. -func validateTraceOverrides(overrides map[string]any, fldPath *field.Path) field.ErrorList { - const owned = "is owned by spec.workload.trace and cannot be overridden" - traffic, found := overrides["traffic"] - if !found { - return nil - } - trafficObject, ok := traffic.(map[string]any) - if !ok { - return field.ErrorList{field.Forbidden(fldPath.Child("traffic"), - "must be an object because spec.workload.trace owns traffic.source")} - } - - // The source fields the trace owns, or a source that is not an object. - source, found := trafficObject["source"] - if !found { - return nil - } - sourceObject, ok := source.(map[string]any) - if !ok { - return field.ErrorList{field.Forbidden(fldPath.Child("traffic", "source"), owned)} - } - allErrs := field.ErrorList{} - for _, key := range []string{"format", "paths", "type"} { - if _, found := sourceObject[key]; found { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("traffic", "source", key), owned)) - } - } - return allErrs -} diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index 922b8aa0a1fb..c895f129a4a2 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -751,7 +751,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { discovered: discoveredH200, wantCELErr: `spec.workload.static.load: Invalid value: exactly one of requestsPerSecond, concurrency, or kvCapacityFraction must be specified`, }, - // v1beta2 recorded-trace workload: the shared schema rules and the trace-owned source. + // v1beta2 recorded-trace workload: the shared schema rules, the format default and overrides. { name: "v1beta2 trace workload is kept and receives no static default", request: v1beta2DGDRForAdmission(func(spec map[string]any) { @@ -763,71 +763,75 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { if _, found := v1beta2Field(t, result, "spec", "workload", "static"); found { t.Fatal("workload.static was defaulted beside a trace") } - if got := v1beta2String(t, result, "spec", "workload", "trace", "source", "pvc", "path"); got != "retune/w-1.jsonl.gz" { - t.Fatalf("workload.trace.source.pvc.path = %q", got) + if got := v1beta2String(t, result, "spec", "workload", "trace", "source", "persistentVolumeClaim", "subPath"); got != "retune/w-1.jsonl.gz" { + t.Fatalf("workload.trace.source.persistentVolumeClaim.subPath = %q", got) } - if got := v1beta2String(t, result, "spec", "workload", "trace", "format"); got != "dynamo" { - t.Fatalf("workload.trace.format = %q, want dynamo", got) + if got := v1beta2String(t, result, "spec", "workload", "trace", "format"); got != "Dynamo" { + t.Fatalf("workload.trace.format = %q, want Dynamo", got) } }, }, { - name: "v1beta2 trace path that leaves the claim is rejected by CEL", + name: "v1beta2 trace subPath that leaves the claim is rejected by CEL", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune/../../w.jsonl")} }), gpuDiscovery: true, discovered: discoveredH200, - wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + wantCELErr: `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, { - name: "v1beta2 absolute trace path is rejected by CEL", + name: "v1beta2 absolute trace subPath is rejected by CEL", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("/w.jsonl")} }), gpuDiscovery: true, discovered: discoveredH200, - wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + wantCELErr: `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, { - name: "v1beta2 trace path with an empty segment is rejected by CEL", + name: "v1beta2 trace subPath with an empty segment is rejected by CEL", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune//w.jsonl")} }), gpuDiscovery: true, discovered: discoveredH200, - wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + wantCELErr: `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, { - name: "v1beta2 trace path with a NUL character is rejected by CEL", + name: "v1beta2 trace subPath with a NUL character is rejected by CEL", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune/w\x00.jsonl")} }), gpuDiscovery: true, discovered: discoveredH200, - wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must not contain a NUL character`, + wantCELErr: `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must not contain a NUL character`, }, { name: "v1beta2 trace claim name must be a DNS subdomain", request: v1beta2DGDRForAdmission(func(spec map[string]any) { trace := v1beta2TraceForAdmission("w.jsonl") - trace["source"] = map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "w.jsonl"}} + trace["source"] = map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "w.jsonl"}} spec["workload"] = map[string]any{"trace": trace} }), gpuDiscovery: true, discovered: discoveredH200, - wantSchemaErr: `spec.workload.trace.source.pvc.claimName: Invalid value: "Bad_Claim": spec.workload.trace.source.pvc.claimName in body should match '^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$'`, + wantSchemaErr: `spec.workload.trace.source.persistentVolumeClaim.claimName: Invalid value: "Bad_Claim": spec.workload.trace.source.persistentVolumeClaim.claimName in body should match '^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$'`, }, { - name: "v1beta2 trace format is required", + name: "v1beta2 omitted trace format defaults to Dynamo", request: v1beta2DGDRForAdmission(func(spec map[string]any) { trace := v1beta2TraceForAdmission("w.jsonl") delete(trace, "format") spec["workload"] = map[string]any{"trace": trace} }), - gpuDiscovery: true, - discovered: discoveredH200, - wantSchemaErr: `spec.workload.trace.format: Required value`, + gpuDiscovery: true, + discovered: discoveredH200, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "workload", "trace", "format"); got != "Dynamo" { + t.Fatalf("workload.trace.format = %q, want the Dynamo default", got) + } + }, }, { name: "v1beta2 trace format admits only dynamo", @@ -838,10 +842,10 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }), gpuDiscovery: true, discovered: discoveredH200, - wantSchemaErr: `spec.workload.trace.format: Unsupported value: "mooncake": supported values: "dynamo"`, + wantSchemaErr: `spec.workload.trace.format: Unsupported value: "mooncake": supported values: "Dynamo"`, }, { - name: "v1beta2 trace source requires a pvc", + name: "v1beta2 trace source requires a persistentVolumeClaim", request: v1beta2DGDRForAdmission(func(spec map[string]any) { trace := v1beta2TraceForAdmission("w.jsonl") trace["source"] = map[string]any{} @@ -849,7 +853,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }), gpuDiscovery: true, discovered: discoveredH200, - wantCELErr: `spec.workload.trace.source: Invalid value: pvc is required`, + wantCELErr: `spec.workload.trace.source: Invalid value: persistentVolumeClaim is required`, }, { name: "v1beta2 static and trace are mutually exclusive", @@ -861,7 +865,20 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { wantCELErr: `spec.workload: Invalid value: static and trace are mutually exclusive`, }, { - name: "v1beta2 input overrides cannot change the trace source", + name: "v1beta2 input overrides may restate the trace source", + request: v1beta2DGDRForAdmission(func(spec map[string]any) { + spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl.gz")} + setV1beta2InputOverrides(spec, map[string]any{"traffic": map[string]any{ + "source": map[string]any{ + "type": "trace", "format": "dynamo", "paths": []any{"/var/run/aisimulate/trace/trace.jsonl.gz"}, + }, + }}) + }), + gpuDiscovery: true, + discovered: discoveredH200, + }, + { + name: "v1beta2 input overrides may replace the trace source", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl")} setV1beta2InputOverrides(spec, map[string]any{"traffic": map[string]any{ @@ -871,16 +888,16 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }), gpuDiscovery: true, discovered: discoveredH200, - wantWebhook: []string{ - "spec.aisimulate.inputOverrides.traffic.source.format: Forbidden: is owned by spec.workload.trace and cannot be overridden", - "spec.aisimulate.inputOverrides.traffic.source.paths: Forbidden: is owned by spec.workload.trace and cannot be overridden", - "spec.aisimulate.inputOverrides.traffic.source.type: Forbidden: is owned by spec.workload.trace and cannot be overridden", + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "aisimulate", "inputOverrides", "traffic", "source", "format"); got != "mooncake" { + t.Fatalf("inputOverrides.traffic.source.format = %q, want the override kept", got) + } }, }, // A v1beta2 trace in a v1beta1 carrier stored without admission: CEL ratchets the unchanged // trace on the next v1beta2 write, so the structural webhook recursion rejects it. { - name: "v1beta2 carrier trace path that leaves the claim", + name: "v1beta2 carrier trace subPath that leaves the claim", oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("../escape.jsonl")}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("../escape.jsonl")} @@ -889,15 +906,15 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ - `spec.workload.trace.source.pvc.path: Invalid value: "../escape.jsonl": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "../escape.jsonl": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, // CEL ratchets the unchanged trace and reports what it tolerated as a warning. wantWarnings: []string{ - `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, }, { - name: "v1beta2 carrier absolute trace path", + name: "v1beta2 carrier absolute trace subPath", oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("/w.jsonl")}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("/w.jsonl")} @@ -906,15 +923,15 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ - `spec.workload.trace.source.pvc.path: Invalid value: "/w.jsonl": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "/w.jsonl": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, // CEL ratchets the unchanged trace and reports what it tolerated as a warning. wantWarnings: []string{ - `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, }, { - name: "v1beta2 carrier trace path with an empty segment", + name: "v1beta2 carrier trace subPath with an empty segment", oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("retune//w.jsonl")}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune//w.jsonl")} @@ -923,15 +940,15 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ - `spec.workload.trace.source.pvc.path: Invalid value: "retune//w.jsonl": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "retune//w.jsonl": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, // CEL ratchets the unchanged trace and reports what it tolerated as a warning. wantWarnings: []string{ - `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, }, { - name: "v1beta2 carrier trace path with a NUL character", + name: "v1beta2 carrier trace subPath with a NUL character", oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("retune/w\x00.jsonl")}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune/w\x00.jsonl")} @@ -940,15 +957,15 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ - "spec.workload.trace.source.pvc.path: Invalid value: \"retune/w\\x00.jsonl\": path must not contain a NUL character", + "spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: \"retune/w\\x00.jsonl\": subPath must not contain a NUL character", }, // CEL ratchets the unchanged trace and reports what it tolerated as a warning. wantWarnings: []string{ - `spec.workload.trace.source.pvc.path: Invalid value: "string": path must not contain a NUL character`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must not contain a NUL character`, }, }, { - name: "v1beta2 carrier trace path longer than the claim path limit", + name: "v1beta2 carrier trace subPath longer than its limit", oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission(strings.Repeat("a", 1025))}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission(strings.Repeat("a", 1025))} @@ -957,70 +974,72 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ - "spec.workload.trace.source.pvc.path: Too long: may not be more than 1024 characters", + "spec.workload.trace.source.persistentVolumeClaim.subPath: Too long: may not be more than 1024 characters", }, }, { name: "v1beta2 carrier trace claim that is not a DNS subdomain", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "w.jsonl"}}}}), + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "w.jsonl"}}}}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "w.jsonl"}}}} + spec["workload"] = map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "w.jsonl"}}}} }), seedWithoutWebhook: true, gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ - `spec.workload.trace.source.pvc.claimName: Invalid value: "Bad_Claim": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, + `spec.workload.trace.source.persistentVolumeClaim.claimName: Invalid value: "Bad_Claim": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, }, }, { name: "v1beta2 carrier trace format that is not admitted", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "mooncake", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}}), + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": map[string]any{"format": "mooncake", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}} + spec["workload"] = map[string]any{"trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}} }), seedWithoutWebhook: true, gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ - `spec.workload.trace.format: Unsupported value: "mooncake": supported values: "dynamo"`, + `spec: Invalid value: "": cannot be lowered to the public AISimulate recommendation input: spec.workload.trace.format: unsupported trace format "mooncake"`, }, }, { - name: "v1beta2 carrier trace without a format", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}}), + name: "v1beta2 carrier trace without a format receives the default on the next write", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": map[string]any{"source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}} + spec["workload"] = map[string]any{"trace": map[string]any{"source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}} }), seedWithoutWebhook: true, gpuDiscovery: true, discovered: discoveredH200, - wantWebhook: []string{ - `spec.workload.trace.format: Required value`, + check: func(t *testing.T, result *unstructured.Unstructured) { + if got := v1beta2String(t, result, "spec", "workload", "trace", "format"); got != "Dynamo" { + t.Fatalf("workload.trace.format = %q, want the Dynamo default", got) + } }, }, { - name: "v1beta2 carrier trace source without a pvc", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{}}}), + name: "v1beta2 carrier trace source without a persistentVolumeClaim", + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{}}}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{}}} + spec["workload"] = map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{}}} }), seedWithoutWebhook: true, gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ - `spec.workload.trace.source.pvc: Required value`, + `spec.workload.trace.source.persistentVolumeClaim: Required value`, }, // CEL ratchets the unchanged trace and reports what it tolerated as a warning. wantWarnings: []string{ - `spec.workload.trace.source: Invalid value: "object": pvc is required`, + `spec.workload.trace.source: Invalid value: "object": persistentVolumeClaim is required`, }, }, { name: "v1beta2 carrier trace beside a static workload", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}}), + oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}}} + spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}} }), seedWithoutWebhook: true, gpuDiscovery: true, @@ -1035,9 +1054,9 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, { name: "v1beta2 carrier trace arrival speedup that is not positive", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}}), + oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": map[string]any{"format": "dynamo", "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": "w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}} + spec["workload"] = map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}} }), seedWithoutWebhook: true, gpuDiscovery: true, @@ -1048,28 +1067,28 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, { name: "v1beta2 carrier trace errors are reported together in field order", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": -1.0}}}), + oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": -1.0}}}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"pvc": map[string]any{"claimName": "Bad_Claim", "path": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": -1.0}}} + spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": -1.0}}} }), seedWithoutWebhook: true, gpuDiscovery: true, discovered: discoveredH200, wantWebhook: []string{ `spec.workload: Forbidden: static and trace are mutually exclusive`, - `spec.workload.trace.format: Unsupported value: "mooncake": supported values: "dynamo"`, - `spec.workload.trace.source.pvc.claimName: Invalid value: "Bad_Claim": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, - `spec.workload.trace.source.pvc.path: Invalid value: "../w.jsonl": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.source.persistentVolumeClaim.claimName: Invalid value: "Bad_Claim": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "../w.jsonl": subPath must be relative to the claim root without empty, '.' or '..' segments`, `spec.workload.trace.replay.arrivalSpeedupRatio: Invalid value: -1: must be greater than 0`, + `spec: Invalid value: "": cannot be lowered to the public AISimulate recommendation input: spec.workload.trace.format: unsupported trace format "mooncake"`, }, // CEL ratchets the unchanged trace and reports what it tolerated as a warning. wantWarnings: []string{ `spec.workload: Invalid value: "object": static and trace are mutually exclusive`, - `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, }, { - name: "v1beta2 a trace leaves traffic overrides it does not own to AISimulate", + name: "v1beta2 a trace accepts load, source and stop overrides", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl")} setV1beta2InputOverrides(spec, map[string]any{"traffic": map[string]any{ @@ -1092,28 +1111,22 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { discovered: discoveredH200, }, { - name: "v1beta2 a null traffic override would delete the trace source", + name: "v1beta2 a null traffic override may delete the traffic a trace lowers", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl")} setV1beta2InputOverrides(spec, map[string]any{"traffic": nil}) }), gpuDiscovery: true, discovered: discoveredH200, - wantWebhook: []string{ - "spec.aisimulate.inputOverrides.traffic: Forbidden: must be an object because spec.workload.trace owns traffic.source", - }, }, { - name: "v1beta2 a null source override would delete the trace source", + name: "v1beta2 a null source override may delete the source a trace lowers", request: v1beta2DGDRForAdmission(func(spec map[string]any) { spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("w.jsonl")} setV1beta2InputOverrides(spec, map[string]any{"traffic": map[string]any{"source": nil}}) }), gpuDiscovery: true, discovered: discoveredH200, - wantWebhook: []string{ - "spec.aisimulate.inputOverrides.traffic.source: Forbidden: is owned by spec.workload.trace and cannot be overridden", - }, }, { name: "v1beta2 update to duplicate backends is rejected", @@ -1248,28 +1261,29 @@ func TestDynamoGraphDeploymentRunSpecIsImmutable(t *testing.T) { } // A Run has no admission webhook, so the trace rules it shares with the Request must hold in its -// own CRD schema: a directly created Run cannot name a file outside its claim. +// own CRD schema: a directly created Run cannot name a file outside its claim, and its omitted +// format receives the same default. func TestDynamoGraphDeploymentRunTraceIsValidatedByTheCRD(t *testing.T) { tests := []struct { name string - tracePath string + subPath string withStatic bool wantCELErr string }{ - {name: "a valid trace is stored", tracePath: "retune/w-1.jsonl.gz"}, + {name: "a valid trace is stored with the default format", subPath: "retune/w-1.jsonl.gz"}, { - name: "a path that leaves the claim is rejected", - tracePath: "../w.jsonl", - wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must be relative to the claim root without empty, '.' or '..' segments`, + name: "a subPath that leaves the claim is rejected", + subPath: "../w.jsonl", + wantCELErr: `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, }, { - name: "a path with a NUL character is rejected", - tracePath: "retune/w\x00.jsonl", - wantCELErr: `spec.workload.trace.source.pvc.path: Invalid value: "string": path must not contain a NUL character`, + name: "a subPath with a NUL character is rejected", + subPath: "retune/w\x00.jsonl", + wantCELErr: `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must not contain a NUL character`, }, { name: "static beside a trace is rejected", - tracePath: "retune/w-1.jsonl.gz", + subPath: "retune/w-1.jsonl.gz", withStatic: true, wantCELErr: `spec.workload: Invalid value: static and trace are mutually exclusive`, }, @@ -1278,9 +1292,8 @@ func TestDynamoGraphDeploymentRunTraceIsValidatedByTheCRD(t *testing.T) { t.Run(tt.name, func(t *testing.T) { run := v1beta2RunForAdmission(v1beta2AdmissionModel, nil) run.Spec.Workload = &nvidiacomv1beta2.WorkloadSpec{Trace: &nvidiacomv1beta2.TraceWorkloadSpec{ - Format: nvidiacomv1beta2.TraceFormatDynamo, - Source: nvidiacomv1beta2.TraceSourceSpec{PVC: &nvidiacomv1beta2.TracePVCSource{ - ClaimName: "request-traces", Path: tt.tracePath, + Source: nvidiacomv1beta2.TraceSourceSpec{PersistentVolumeClaim: &nvidiacomv1beta2.TracePVCSource{ + ClaimName: "request-traces", SubPath: tt.subPath, }}, }} if tt.withStatic { @@ -1288,8 +1301,11 @@ func TestDynamoGraphDeploymentRunTraceIsValidatedByTheCRD(t *testing.T) { } actual := runAdmissionTest(t, admissionTestCase{object: run, wantCELError: tt.wantCELErr}) if tt.wantCELErr == "" { - if got := v1beta2String(t, actual, "spec", "workload", "trace", "source", "pvc", "path"); got != "retune/w-1.jsonl.gz" { - t.Fatalf("stored workload.trace.source.pvc.path = %q, want the trace kept", got) + if got := v1beta2String(t, actual, "spec", "workload", "trace", "source", "persistentVolumeClaim", "subPath"); got != "retune/w-1.jsonl.gz" { + t.Fatalf("stored workload.trace.source.persistentVolumeClaim.subPath = %q, want the trace kept", got) + } + if got := v1beta2String(t, actual, "spec", "workload", "trace", "format"); got != "Dynamo" { + t.Fatalf("stored workload.trace.format = %q, want the Dynamo default", got) } } }) @@ -1555,10 +1571,10 @@ func v1beta1CarrierForAdmission(workload map[string]any) *unstructured.Unstructu }} } -func v1beta2TraceForAdmission(path string) map[string]any { +func v1beta2TraceForAdmission(subPath string) map[string]any { return map[string]any{ - "format": "dynamo", - "source": map[string]any{"pvc": map[string]any{"claimName": "request-traces", "path": path}}, + "format": "Dynamo", + "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": subPath}}, } } diff --git a/deploy/operator/internal/webhook/validation/shared_helpers.go b/deploy/operator/internal/webhook/validation/shared_helpers.go index c131e403c9d7..6b03744b25f2 100644 --- a/deploy/operator/internal/webhook/validation/shared_helpers.go +++ b/deploy/operator/internal/webhook/validation/shared_helpers.go @@ -414,13 +414,13 @@ func groveNativeMinimumPath(component *nvidiacomv1beta1.DynamoComponentDeploymen return fldPath.Child("providerOverride", "value", "spec", "minAvailable") } -// traceClaimPathHasOnlyNamedSegments reports whether claimPath is relative to the claim root and +// traceClaimPathHasOnlyNamedSegments reports whether subPath is relative to the claim root and // every segment names an entry: none is empty, "." or "..". -func traceClaimPathHasOnlyNamedSegments(claimPath string) bool { - if claimPath == "" { +func traceClaimPathHasOnlyNamedSegments(subPath string) bool { + if subPath == "" { return false } - for segment := range strings.SplitSeq(claimPath, "/") { + for segment := range strings.SplitSeq(subPath, "/") { if segment == "" || segment == "." || segment == ".." { return false } diff --git a/deploy/operator/internal/webhook/validation/shared_v1beta2.go b/deploy/operator/internal/webhook/validation/shared_v1beta2.go index 426094a5a965..58205e9cd47d 100644 --- a/deploy/operator/internal/webhook/validation/shared_v1beta2.go +++ b/deploy/operator/internal/webhook/validation/shared_v1beta2.go @@ -26,8 +26,9 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) -// maxTraceClaimPathLength is the CRD maxLength of spec.workload.trace.source.pvc.path. -const maxTraceClaimPathLength = 1024 +// maxTraceClaimSubPathLength is the CRD maxLength of +// spec.workload.trace.source.persistentVolumeClaim.subPath. +const maxTraceClaimSubPathLength = 1024 // validateWorkloadSpec validates workload, a type that Requests and Runs share. The CRD schema and // CEL enforce the same rules on every v1beta2 write; this recursion reaches a Request whose v1beta1 @@ -44,21 +45,11 @@ func validateWorkloadSpec(workload *nvidiacomv1beta2.WorkloadSpec, fldPath *fiel return append(allErrs, validateTraceWorkloadSpec(workload.Trace, fldPath.Child("trace"))...) } -// validateTraceWorkloadSpec validates trace. trace and fldPath must not be nil. +// validateTraceWorkloadSpec validates trace. Its format is not checked here: the CRD schema +// defaults and enumerates it, and lowering rejects a format it cannot translate. trace and fldPath +// must not be nil. func validateTraceWorkloadSpec(trace *nvidiacomv1beta2.TraceWorkloadSpec, fldPath *field.Path) field.ErrorList { - allErrs := field.ErrorList{} - - // The format has no default: AISimulate and the DEP default to mooncake, which is not admitted. - switch trace.Format { - case nvidiacomv1beta2.TraceFormatDynamo: - case "": - allErrs = append(allErrs, field.Required(fldPath.Child("format"), "")) - default: - allErrs = append(allErrs, field.NotSupported(fldPath.Child("format"), trace.Format, - []nvidiacomv1beta2.TraceFormat{nvidiacomv1beta2.TraceFormatDynamo})) - } - - allErrs = append(allErrs, validateTraceSourceSpec(&trace.Source, fldPath.Child("source"))...) + allErrs := validateTraceSourceSpec(&trace.Source, fldPath.Child("source")) if trace.Replay != nil { allErrs = append(allErrs, validateTraceReplaySpec(trace.Replay, fldPath.Child("replay"))...) } @@ -67,31 +58,31 @@ func validateTraceWorkloadSpec(trace *nvidiacomv1beta2.TraceWorkloadSpec, fldPat // validateTraceSourceSpec validates source. source and fldPath must not be nil. func validateTraceSourceSpec(source *nvidiacomv1beta2.TraceSourceSpec, fldPath *field.Path) field.ErrorList { - if source.PVC == nil { - return field.ErrorList{field.Required(fldPath.Child("pvc"), "")} + if source.PersistentVolumeClaim == nil { + return field.ErrorList{field.Required(fldPath.Child("persistentVolumeClaim"), "")} } - return validateTracePVCSource(source.PVC, fldPath.Child("pvc")) + return validateTracePVCSource(source.PersistentVolumeClaim, fldPath.Child("persistentVolumeClaim")) } -// validateTracePVCSource validates pvc: a PersistentVolumeClaim name and one nameable file inside -// it. pvc and fldPath must not be nil. -func validateTracePVCSource(pvc *nvidiacomv1beta2.TracePVCSource, fldPath *field.Path) field.ErrorList { +// validateTracePVCSource validates claim: a PersistentVolumeClaim name and one nameable file inside +// it. claim and fldPath must not be nil. +func validateTracePVCSource(claim *nvidiacomv1beta2.TracePVCSource, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - for _, msg := range validation.IsDNS1123Subdomain(pvc.ClaimName) { - allErrs = append(allErrs, field.Invalid(fldPath.Child("claimName"), pvc.ClaimName, msg)) + for _, msg := range validation.IsDNS1123Subdomain(claim.ClaimName) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("claimName"), claim.ClaimName, msg)) } - // The path names one file below the claim root: relative, with only named segments. - pathPath := fldPath.Child("path") - if utf8.RuneCountInString(pvc.Path) > maxTraceClaimPathLength { - allErrs = append(allErrs, field.TooLongCharacters(pathPath, pvc.Path, maxTraceClaimPathLength)) + // The subPath names one file below the claim root: relative, with only named segments. + subPath := fldPath.Child("subPath") + if utf8.RuneCountInString(claim.SubPath) > maxTraceClaimSubPathLength { + allErrs = append(allErrs, field.TooLongCharacters(subPath, claim.SubPath, maxTraceClaimSubPathLength)) } - if !traceClaimPathHasOnlyNamedSegments(pvc.Path) { - allErrs = append(allErrs, field.Invalid(pathPath, pvc.Path, - "path must be relative to the claim root without empty, '.' or '..' segments")) + if !traceClaimPathHasOnlyNamedSegments(claim.SubPath) { + allErrs = append(allErrs, field.Invalid(subPath, claim.SubPath, + "subPath must be relative to the claim root without empty, '.' or '..' segments")) } - if strings.ContainsRune(pvc.Path, 0) { - allErrs = append(allErrs, field.Invalid(pathPath, pvc.Path, "path must not contain a NUL character")) + if strings.ContainsRune(claim.SubPath, 0) { + allErrs = append(allErrs, field.Invalid(subPath, claim.SubPath, "subPath must not contain a NUL character")) } return allErrs } diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index da18357de679..2903ddf191ca 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -3321,7 +3321,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `inputOverrides` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from
the typed fields. It uses the upstream AISimulate JSON contract, including
snake_case names. Values here take precedence, and null deletes a lowered
field. The dgdr_run member must remain an object, and its operator-owned
transport, identity, and runtime fields name, max_candidates, snapshot_dir,
runtime_image, runtime_version_override, derive_runtime_images,
num_gpus_per_node, and namespace are operator-owned. Request admission
rejects them, and the Run controller enforces its values while lowering the
immutable snapshot. When spec.workload.trace is set, traffic and
traffic.source must remain objects, and the trace owns traffic.source.type,
format, and paths the same way: Request admission rejects them, and the Run
controller enforces the typed values. Other traffic overrides are AISimulate
input, which AISimulate validates when the Run's Job starts. The operator runs
the AISimulate version from its configured planner image; Requests and Runs
cannot select a different image or version. | | Type: object
Optional: \{\}
| +| `inputOverrides` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from
the typed fields. It uses the upstream AISimulate JSON contract, including
snake_case names. Values here take precedence, and null deletes a lowered
field. The dgdr_run member must remain an object, and its operator-owned
transport, identity, and runtime fields name, max_candidates, snapshot_dir,
runtime_image, runtime_version_override, derive_runtime_images,
num_gpus_per_node, and namespace are operator-owned. Request admission
rejects them, and the Run controller enforces its values while lowering the
immutable snapshot. A typed spec.workload.trace is an underlay like the other
typed fields: it supplies the default traffic.source (type trace, format
dynamo and the mounted trace file in paths) and the recorded-arrival
traffic.load. Overrides can replace those values, including
traffic.source.type, format, and paths, and null deletes the named field or
parent object; the Run controller does not restore it. The typed
PersistentVolumeClaim source alone decides the Run Job's trace mount and its
claim dependency, so overriding a native path adds no volume mount.
AISimulate validates the resulting input when the Run's Job starts; admission
does not validate it against the AISimulate schema yet. The operator runs
the AISimulate version from its configured planner image; Requests and Runs
cannot select a different image or version. | | Type: object
Optional: \{\}
| #### Backend @@ -3359,8 +3359,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | Name is the executor-defined stable branch identifier. | | | -| `rounds` _[RoundProgress](#roundprogress)_ | Rounds reports optimizer progress for this branch. | | | +| `name` _string_ | Name is the executor-defined stable branch identifier. | | Required: \{\}
| +| `rounds` _[RoundProgress](#roundprogress)_ | Rounds reports optimizer progress for this branch. | | Required: \{\}
| #### CandidateProgress @@ -3376,8 +3376,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `paretoFront` _integer_ | ParetoFront is the number of feasible points on the current Pareto front. | | | -| `published` _integer_ | Published is the number of Candidate resources currently referenced by this run. | | | +| `paretoFront` _integer_ | ParetoFront is the number of feasible points on the current Pareto front. | | Optional: \{\}
| +| `published` _integer_ | Published is the number of Candidate resources currently referenced by this run. | | Optional: \{\}
| #### DGDRRunPhase @@ -3432,9 +3432,9 @@ DynamoGraphDeploymentCandidate is one bounded, user-visible search result. | --- | --- | --- | --- | | `apiVersion` _string_ | `nvidia.com/v1beta2` | | | | `kind` _string_ | `DynamoGraphDeploymentCandidate` | | | -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | -| `spec` _[DynamoGraphDeploymentCandidateSpec](#dynamographdeploymentcandidatespec)_ | | | | -| `status` _[DynamoGraphDeploymentCandidateStatus](#dynamographdeploymentcandidatestatus)_ | | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| +| `spec` _[DynamoGraphDeploymentCandidateSpec](#dynamographdeploymentcandidatespec)_ | | | Required: \{\}
| +| `status` _[DynamoGraphDeploymentCandidateStatus](#dynamographdeploymentcandidatestatus)_ | | | Optional: \{\}
| #### DynamoGraphDeploymentCandidateSpec @@ -3461,7 +3461,7 @@ _Appears in:_ | `restart` _[Restart](#restart)_ | restart specifies the restart policy for the graph deployment. | | Optional: \{\}
| | `topologyConstraint` _[SpecTopologyConstraint](#spectopologyconstraint)_ | topologyConstraint is the deployment-level topology constraint. When
set, `spec.topologyConstraint.clusterTopologyName` names the ClusterTopology
CR to use. `spec.topologyConstraint.packDomain` is optional at this
level and can be omitted when only components carry constraints.
Components without their own `topologyConstraint` inherit from this value. | | Optional: \{\}
| | `experimental` _[DynamoGraphDeploymentExperimentalSpec](#dynamographdeploymentexperimentalspec)_ | experimental groups graph-level preview features whose API shape and
behavior may change in breaking ways between v1beta1 releases. | | Optional: \{\}
| -| `parameters` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Parameters are the resolved evaluation parameters for this point. | | Type: object
| +| `parameters` _[RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg)_ | Parameters are the resolved evaluation parameters for this point. | | Type: object
Required: \{\}
| #### DynamoGraphDeploymentCandidateStatus @@ -3494,9 +3494,9 @@ DynamoGraphDeploymentRequest is the Schema for simulation-backed deployment sear | --- | --- | --- | --- | | `apiVersion` _string_ | `nvidia.com/v1beta2` | | | | `kind` _string_ | `DynamoGraphDeploymentRequest` | | | -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | -| `spec` _[DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec)_ | | | | -| `status` _[DynamoGraphDeploymentRequestStatus](#v1beta2-dynamographdeploymentrequeststatus)_ | | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| +| `spec` _[DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec)_ | | | Required: \{\}
| +| `status` _[DynamoGraphDeploymentRequestStatus](#v1beta2-dynamographdeploymentrequeststatus)_ | | | Optional: \{\}
| #### v1beta2 DynamoGraphDeploymentRequestNativeSpec @@ -3515,7 +3515,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, except the traffic.source fields a trace owns.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, including the fields a trace lowers.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3557,7 +3557,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, except the traffic.source fields a trace owns.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, including the fields a trace lowers.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3598,9 +3598,9 @@ DynamoGraphDeploymentRun is one immutable accepted DGDR request and its executio | --- | --- | --- | --- | | `apiVersion` _string_ | `nvidia.com/v1beta2` | | | | `kind` _string_ | `DynamoGraphDeploymentRun` | | | -| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | -| `spec` _[DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec)_ | | | | -| `status` _[DynamoGraphDeploymentRunStatus](#dynamographdeploymentrunstatus)_ | | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | Optional: \{\}
| +| `spec` _[DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec)_ | | | Required: \{\}
| +| `status` _[DynamoGraphDeploymentRunStatus](#dynamographdeploymentrunstatus)_ | | | Optional: \{\}
| #### DynamoGraphDeploymentRunSpec @@ -3624,7 +3624,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, except the traffic.source fields a trace owns.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, including the fields a trace lowers.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3667,13 +3667,13 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `scheduled` _integer_ | Scheduled is the number of unique evaluations accepted for execution. | | | -| `running` _integer_ | Running is the number of evaluations currently executing. | | | -| `feasible` _integer_ | Feasible is the number of completed evaluations that satisfy all constraints. | | | -| `infeasible` _integer_ | Infeasible is the number of completed evaluations that violate at least one constraint. | | | -| `failed` _integer_ | Failed is the number of evaluations that ended in an execution error. | | | -| `unsupported` _integer_ | Unsupported is the number of configurations rejected as unsupported before evaluation. | | | -| `cacheHits` _integer_ | CacheHits is the number of evaluations satisfied from a prior result. | | | +| `scheduled` _integer_ | Scheduled is the number of unique evaluations accepted for execution. | | Optional: \{\}
| +| `running` _integer_ | Running is the number of evaluations currently executing. | | Optional: \{\}
| +| `feasible` _integer_ | Feasible is the number of completed evaluations that satisfy all constraints. | | Optional: \{\}
| +| `infeasible` _integer_ | Infeasible is the number of completed evaluations that violate at least one constraint. | | Optional: \{\}
| +| `failed` _integer_ | Failed is the number of evaluations that ended in an execution error. | | Optional: \{\}
| +| `unsupported` _integer_ | Unsupported is the number of configurations rejected as unsupported before evaluation. | | Optional: \{\}
| +| `cacheHits` _integer_ | CacheHits is the number of evaluations satisfied from a prior result. | | Optional: \{\}
| #### GPUHardwareSpec @@ -3727,8 +3727,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `min` _float_ | Min is the lower end of the searched fraction range. | | ExclusiveMinimum: true
Maximum: 1
Minimum: 0
| -| `max` _float_ | Max is the upper end of the searched fraction range. | | ExclusiveMinimum: true
Maximum: 1
Minimum: 0
| +| `min` _float_ | Min is the lower end of the searched fraction range. | | ExclusiveMinimum: true
Maximum: 1
Minimum: 0
Required: \{\}
| +| `max` _float_ | Max is the upper end of the searched fraction range. | | ExclusiveMinimum: true
Maximum: 1
Minimum: 0
Required: \{\}
| #### KVCapacityFractionSpec @@ -3764,7 +3764,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | Name identifies the model in the syntax accepted by the selected backend. | | MinLength: 1
| +| `name` _string_ | Name identifies the model in the syntax accepted by the selected backend. | | MinLength: 1
Required: \{\}
| #### NativeSearchInput @@ -3786,7 +3786,7 @@ _Appears in:_ | `model` _[ModelSpec](#modelspec)_ | Model identifies the model whose deployment configurations are evaluated. | | Optional: \{\}
| | `backends` _[Backend](#backend) array_ | Backends lists one or more inference backends searched by AISimulate and used by generated candidates. | | Enum: [vllm sglang trtllm]
MinItems: 1
Optional: \{\}
| | `hardware` _[HardwareSpec](#v1beta2-hardwarespec)_ | Hardware bounds the accelerator configurations evaluated by the search.
When omitted, admission discovers SKU, budget and node width. An exactly empty object opts
out of operator hardware discovery and typed defaults. Admission completes every nonempty
object with any missing hardware values; explicit values remain authoritative. | | Optional: \{\}
| -| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, except the traffic.source fields a trace owns.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| +| `workload` _[WorkloadSpec](#v1beta2-workloadspec)_ | Workload defines the traffic model used for every candidate evaluation: synthetic static
traffic or a recorded trace.
When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10;
an explicit empty workload receives the same shape. workload.static: \{\} opts out of all
traffic defaults. Any other static object is completed with its missing defaults, and a
trace workload receives no static default.
spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over
every lowered traffic field it names, including the fields a trace lowers.
The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction
search. Lowering independently defaults optimizer.max_trials to 320. | | Optional: \{\}
| | `objective` _[ObjectiveSpec](#objectivespec)_ | Objective defines scalar optimization. When omitted,
admission supplies the object. Its mode defaults to Optimize, its scalar metric defaults
to ThroughputPerGPU, and its absent SLA defaults to TTFT 2000 ms and ITL 30 ms. | | Optional: \{\}
| | `aisimulate` _[AISimulateSpec](#aisimulatespec)_ | AISimulate supplies public recommendation input to the operator's AISimulate execution bundle. | | Optional: \{\}
| | `recommendation` _[RecommendationSpec](#recommendationspec)_ | Recommendation controls bounded projection into DGDC resources. | | Optional: \{\}
| @@ -3898,7 +3898,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `name` _string_ | Name is the resource name. | | | +| `name` _string_ | Name is the resource name. | | Required: \{\}
| | `uid` _[UID](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#uid-types-pkg)_ | UID is the resource UID. | | Optional: \{\}
| @@ -3915,8 +3915,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `completed` _integer_ | Completed is the number of finished rounds. | | | -| `total` _integer_ | Total is the number of planned rounds. | | | +| `completed` _integer_ | Completed is the number of finished rounds. | | Required: \{\}
| +| `total` _integer_ | Total is the number of planned rounds. | | Required: \{\}
| #### StaticLoadSpec @@ -3960,17 +3960,18 @@ _Appears in:_ _Underlying type:_ _string_ -TraceFormat identifies the decoder of a recorded request trace. +TraceFormat identifies the decoder of a recorded request trace. Lowering translates it to +AISimulate's native traffic.source.format. _Validation:_ -- Enum: [dynamo] +- Enum: [Dynamo] _Appears in:_ - [TraceWorkloadSpec](#traceworkloadspec) | Field | Description | | --- | --- | -| `dynamo` | TraceFormatDynamo is the Dynamo request trace (dynamo.request.trace.v1) that the frontend
records, as JSON Lines or gzip-compressed JSON Lines.
| +| `Dynamo` | TraceFormatDynamo is the Dynamo request trace (dynamo.request.trace.v1) that the frontend
records, as JSON Lines or gzip-compressed JSON Lines. It lowers to AISimulate's dynamo format.
| #### TracePVCSource @@ -3986,8 +3987,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `claimName` _string_ | ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace.
The Run Job mounts only the trace file from it, read-only, and only into the AISimulate
container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-`
until its execution stops; deleting the claim cancels the Runs that read it. | | MaxLength: 253
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
| -| `path` _string_ | Path names one trace file relative to the root of the claim. It is neither a directory
nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..'
segments or a NUL character, which no file name can hold. The file must be complete before
the Request is created and must never be rewritten: every Run of the Request reads it when
its Job starts. | | MaxLength: 1024
MinLength: 1
| +| `claimName` _string_ | ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace.
The Run Job mounts only the trace file from it, read-only, and only into the AISimulate
container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-`
until the Run completes or, for a Run deleted before it completes, until no Job of the Run
is left. Garbage collection deletes the Job of a deleted Run before the Job's pod, so the
finalizer of a deleted Run can be released while that pod still stops; Kubernetes
PersistentVolumeClaim protection keeps the claim until no pod uses it. Deleting the claim
cancels the Runs that read it. | | MaxLength: 253
MinLength: 1
Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
Required: \{\}
| +| `subPath` _string_ | SubPath names one trace file relative to the root of the claim; the Run Job mounts it as
the subPath of the claim volume. It is neither a directory nor a glob. It must not start or
end with '/' and must not contain empty, '.' or '..' segments or a NUL character, which no
file name can hold. The file must be complete before the Request is created and must never
be rewritten: every Run of the Request reads it when its Job starts. | | MaxLength: 1024
MinLength: 1
Required: \{\}
| #### TraceReplaySpec @@ -4020,7 +4021,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `pvc` _[TracePVCSource](#tracepvcsource)_ | PVC reads the trace from a PersistentVolumeClaim in the Request namespace. | | Optional: \{\}
| +| `persistentVolumeClaim` _[TracePVCSource](#tracepvcsource)_ | PersistentVolumeClaim reads the trace from a PersistentVolumeClaim in the Request
namespace. | | Optional: \{\}
| #### TraceWorkloadSpec @@ -4036,8 +4037,8 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `format` _[TraceFormat](#traceformat)_ | Format identifies the trace decoder. It has no default; only dynamo is admitted in this
release. | | Enum: [dynamo]
Required: \{\}
| -| `source` _[TraceSourceSpec](#tracesourcespec)_ | Source identifies the trace artifact. | | | +| `format` _[TraceFormat](#traceformat)_ | Format identifies the trace decoder. Defaults to Dynamo. | Dynamo | Enum: [Dynamo]
Optional: \{\}
| +| `source` _[TraceSourceSpec](#tracesourcespec)_ | Source identifies the trace artifact. | | Required: \{\}
| | `replay` _[TraceReplaySpec](#tracereplayspec)_ | Replay configures how the decoded trace is replayed. | | Optional: \{\}
| @@ -4059,7 +4060,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `static` _[StaticWorkloadSpec](#staticworkloadspec)_ | Static defines AISimulate synthetic traffic. When omitted without a trace, admission
supplies the complete static default. An explicit empty object opts out; any nonempty
object is completed with omitted inputTokens, outputTokens and load defaults. | | Optional: \{\}
| -| `trace` _[TraceWorkloadSpec](#traceworkloadspec)_ | Trace replays a recorded request trace from a PersistentVolumeClaim in the Request
namespace. Admission supplies no static default when it is set. The Run Job mounts only the
trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl,
or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change
traffic.source.type, format or paths. | | Optional: \{\}
| +| `trace` _[TraceWorkloadSpec](#traceworkloadspec)_ | Trace replays a recorded request trace from a PersistentVolumeClaim in the Request
namespace. Admission supplies no static default when it is set. The Run Job mounts only the
trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl,
or trace.jsonl.gz when subPath ends in .gz. The trace lowers to the default traffic.source
and traffic.load, which spec.aisimulate.inputOverrides can replace or delete. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index c6251b17ef44..534c33c3f19b 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -1136,11 +1136,11 @@ spec: name: Qwen/Qwen3-0.6B workload: trace: - format: dynamo # required; no default + # format defaults to Dynamo source: - pvc: + persistentVolumeClaim: claimName: request-traces # PersistentVolumeClaim in this namespace - path: windows/2026-10-27T10-00.jsonl.gz + subPath: windows/2026-10-27T10-00.jsonl.gz replay: arrivalSpeedupRatio: 1.0 # optional; 2.0 replays arrivals twice as fast ``` @@ -1152,7 +1152,7 @@ Before you create the request: With `ReadWriteOnce`, a Job scheduled to another node than the writer waits in `ContainerCreating`, and with `ReadWriteOncePod`, a Job waits in `Pending` until the writer's pod is gone. The Run reports `Running` while its Job waits. -- Finish writing the file. `path` names one file relative to the claim root; directories and globs +- Finish writing the file. `subPath` names one file relative to the claim root; directories and globs are not expanded. Every Run of the request reads the file when its Job starts, so do not rewrite it afterwards. To replay several rolled `jsonl_gz` segments, concatenate them into one `.jsonl.gz` file. A relative symlink to another file in the claim works. An absolute symlink does not, @@ -1164,25 +1164,38 @@ Before you create the request: The operator mounts only this file, read-only and only into the AISimulate container, through a `subPath` of the claim at `/var/run/aisimulate/trace/trace.jsonl.gz`, or at -`/var/run/aisimulate/trace/trace.jsonl` when `path` does not end in `.gz`. The kubelet resolves -`path` inside the claim and refuses to mount a file outside it. If the file does not exist when +`/var/run/aisimulate/trace/trace.jsonl` when `subPath` does not end in `.gz`. The kubelet resolves +`subPath` inside the claim and refuses to mount a file outside it. If the file does not exist when the Job starts, the kubelet cannot mount it either, and the AISimulate container waits; see [Trace claim lifecycle](#trace-claim-lifecycle). On a volume type whose node directory the kubelet -can write, such as `hostPath`, the kubelet instead creates an empty directory at `path` on the +can write, such as `hostPath`, the kubelet instead creates an empty directory at `subPath` on the claim, and the Run fails when AISimulate cannot read it. -`format: dynamo` is the only admitted format. It reads the Dynamo request trace +`format` defaults to `Dynamo`, the only supported format. It reads the Dynamo request trace (`dynamo.request.trace.v1`) written by the frontend, as JSON Lines or as gzip-compressed JSON Lines when the file name ends in `.gz`. See the [Request Trace Reference](../observability/request-traces.mdx). -The operator lowers the trace to the AISimulate input `traffic.source` (`type: trace`, `format`, and -`paths` with the mounted file) and `traffic.load.type: trace_timestamps`, with -`traffic.load.speedup` when `arrivalSpeedupRatio` is set. It adds no `traffic.stop.requests`. When -`trace` is set, `spec.aisimulate.inputOverrides` cannot set `traffic.source.type`, -`traffic.source.format`, or `traffic.source.paths`, or replace `traffic` or `traffic.source` with a -non-object value. Admission rejects these overrides, and the Run controller restores the typed -values. Other `traffic` overrides, such as `traffic.load` and `traffic.stop`, go to AISimulate, -which validates the complete input when the Run's Job starts; an input it rejects fails the Run. +The operator lowers the trace to the AISimulate input `traffic.source` (`type: trace`, +`format: dynamo`, and `paths` with the mounted file) and `traffic.load.type: trace_timestamps`, +with `traffic.load.speedup` when `arrivalSpeedupRatio` is set. It adds no `traffic.stop.requests`. +These values are an underlay, like every value lowered from a typed field: +`spec.aisimulate.inputOverrides` is applied last as a JSON Merge Patch and can replace any of them, +including `traffic.source.type`, `traffic.source.format`, and `traffic.source.paths`. A `null` +override deletes the named field or parent object, and the operator does not restore it. For +example, this override deletes the lowered source: + +```yaml +spec: + aisimulate: + inputOverrides: + traffic: + source: null +``` + +The typed `persistentVolumeClaim` source alone decides which claim the Run's Job mounts and which +claim the Run protects. Overriding `traffic.source.paths` adds no volume mount. AISimulate validates +the resulting input when the Run's Job starts, and an input it rejects fails the Run; admission does +not validate it against the AISimulate schema yet. ### Trace claim lifecycle @@ -1195,7 +1208,9 @@ request's `spec`. It copies container status messages into the Run but never int Run that reads the claim adds its own finalizer, so Runs of several requests can share a claim. The operator removes a Run's finalizer once the Run is complete and none of its Jobs is still running, or once the Run and its Jobs are deleted, for example when the request retires old Runs or is - deleted. + deleted. Kubernetes garbage collection deletes the Job of a deleted Run before the Job's pod has + stopped, so the finalizer of a Run deleted before it completed can be removed while that pod still + mounts the file; `kubernetes.io/pvc-protection` keeps the claim until the pod is gone. - **Deleting the claim cancels its Runs.** When the claim has a `deletionTimestamp`, the operator stops the Job of every Run that reads it through foreground deletion, whether or not the Job's pod has started. Once the Job and its pod are gone, the Run fails with reason `TraceSourceDeleted`, and diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 65952d6a1eb0..88c25ce328fb 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -4634,7 +4634,7 @@ planner image. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) -InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered field. The dgdr_run member must remain an object, and its operator-owned transport, identity, and runtime fields name, max_candidates, snapshot_dir, runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the immutable snapshot. When spec.workload.trace is set, traffic and traffic.source must remain objects, and the trace owns traffic.source.type, format, and paths the same way: Request admission rejects them, and the Run controller enforces the typed values. Other traffic overrides are AISimulate input, which AISimulate validates when the Run's Job starts. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. +InputOverrides is an RFC 7386 JSON Merge Patch over the input lowered from the typed fields. It uses the upstream AISimulate JSON contract, including snake_case names. Values here take precedence, and null deletes a lowered field. The dgdr_run member must remain an object, and its operator-owned transport, identity, and runtime fields name, max_candidates, snapshot_dir, runtime_image, runtime_version_override, derive_runtime_images, num_gpus_per_node, and namespace are operator-owned. Request admission rejects them, and the Run controller enforces its values while lowering the immutable snapshot. A typed spec.workload.trace is an underlay like the other typed fields: it supplies the default traffic.source (type trace, format dynamo and the mounted trace file in paths) and the recorded-arrival traffic.load. Overrides can replace those values, including traffic.source.type, format, and paths, and null deletes the named field or parent object; the Run controller does not restore it. The typed PersistentVolumeClaim source alone decides the Run Job's trace mount and its claim dependency, so overriding a native path adds no volume mount. AISimulate validates the resulting input when the Run's Job starts; admission does not validate it against the AISimulate schema yet. The operator runs the AISimulate version from its configured planner image; Requests and Runs cannot select a different image or version. See [RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg). **Validation:** Type: object Optional: \{\} @@ -4663,13 +4663,15 @@ BranchProgress reports progress for one independently optimized AISimulate branc **Kind:** `type` **Appears in:** [DGDRRunProgress](#dgdrrunprogress) - + Name is the executor-defined stable branch identifier. +**Validation:** Required: \{\} - + Rounds reports optimizer progress for this branch. See [RoundProgress](#roundprogress). +**Validation:** Required: \{\} @@ -4682,10 +4684,12 @@ CandidateProgress reports bounded Kubernetes publication, not all internal trial ParetoFront is the number of feasible points on the current Pareto front. +**Validation:** Optional: \{\} Published is the number of Candidate resources currently referenced by this run. +**Validation:** Optional: \{\} @@ -4749,14 +4753,17 @@ DynamoGraphDeploymentCandidate is one bounded, user-visible search result. Refer to Kubernetes API documentation for fields of `metadata`. See [ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta). +**Validation:** Optional: \{\} - + See [DynamoGraphDeploymentCandidateSpec](#dynamographdeploymentcandidatespec). +**Validation:** Required: \{\} See [DynamoGraphDeploymentCandidateStatus](#dynamographdeploymentcandidatestatus). +**Validation:** Optional: \{\} @@ -4824,10 +4831,10 @@ See [DynamoGraphDeploymentExperimentalSpec](#dynamographdeploymentexperimentalsp **Validation:** Optional: \{\} - + Parameters are the resolved evaluation parameters for this point. See [RawExtension](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#rawextension-runtime-pkg). -**Validation:** Type: object +**Validation:** Type: object Required: \{\} @@ -4862,14 +4869,17 @@ DynamoGraphDeploymentRequest is the Schema for simulation-backed deployment sear Refer to Kubernetes API documentation for fields of `metadata`. See [ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta). +**Validation:** Optional: \{\} - + See [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec). +**Validation:** Required: \{\} See [DynamoGraphDeploymentRequestStatus](#v1beta2-dynamographdeploymentrequeststatus). +**Validation:** Optional: \{\} @@ -4899,7 +4909,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. +Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, including the fields a trace lowers. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -4981,7 +4991,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. +Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, including the fields a trace lowers. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5065,14 +5075,17 @@ DynamoGraphDeploymentRun is one immutable accepted DGDR request and its executio Refer to Kubernetes API documentation for fields of `metadata`. See [ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#objectmeta-v1-meta). +**Validation:** Optional: \{\} - + See [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec). +**Validation:** Required: \{\} See [DynamoGraphDeploymentRunStatus](#dynamographdeploymentrunstatus). +**Validation:** Optional: \{\} @@ -5107,7 +5120,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. +Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, including the fields a trace lowers. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5202,30 +5215,37 @@ EvaluationProgress reports unique candidate outcomes without creating one resour Scheduled is the number of unique evaluations accepted for execution. +**Validation:** Optional: \{\} Running is the number of evaluations currently executing. +**Validation:** Optional: \{\} Feasible is the number of completed evaluations that satisfy all constraints. +**Validation:** Optional: \{\} Infeasible is the number of completed evaluations that violate at least one constraint. +**Validation:** Optional: \{\} Failed is the number of evaluations that ended in an execution error. +**Validation:** Optional: \{\} Unsupported is the number of configurations rejected as unsupported before evaluation. +**Validation:** Optional: \{\} CacheHits is the number of evaluations satisfied from a prior result. +**Validation:** Optional: \{\} @@ -5274,14 +5294,14 @@ KVCapacityFractionRange defines the searched fraction of a candidate's KV capaci **Kind:** `type` **Appears in:** [KVCapacityFractionSpec](#kvcapacityfractionspec) - + Min is the lower end of the searched fraction range. -**Validation:** ExclusiveMinimum: true Maximum: 1 Minimum: 0 +**Validation:** ExclusiveMinimum: true Maximum: 1 Minimum: 0 Required: \{\} - + Max is the upper end of the searched fraction range. -**Validation:** ExclusiveMinimum: true Maximum: 1 Minimum: 0 +**Validation:** ExclusiveMinimum: true Maximum: 1 Minimum: 0 Required: \{\} @@ -5311,9 +5331,9 @@ ModelSpec identifies the model whose deployment configurations are evaluated. **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta2-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta2-dynamographdeploymentrequestspec), [DynamoGraphDeploymentRunSpec](#dynamographdeploymentrunspec), [NativeSearchInput](#nativesearchinput) - + Name identifies the model in the syntax accepted by the selected backend. -**Validation:** MinLength: 1 +**Validation:** MinLength: 1 Required: \{\} @@ -5344,7 +5364,7 @@ See [HardwareSpec](#v1beta2-hardwarespec). -Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, except the traffic.source fields a trace owns. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. +Workload defines the traffic model used for every candidate evaluation: synthetic static traffic or a recorded trace. When omitted, admission supplies 1024 input tokens, 128 output tokens and concurrency 10; an explicit empty workload receives the same shape. workload.static: \{\} opts out of all traffic defaults. Any other static object is completed with its missing defaults, and a trace workload receives no static default. spec.aisimulate.inputOverrides is applied as an RFC 7386 JSON Merge Patch and wins over every lowered traffic field it names, including the fields a trace lowers. The default evaluates a fixed concurrency, so it does not create a KV-capacity-fraction search. Lowering independently defaults optimizer.max_trials to 320. See [WorkloadSpec](#v1beta2-workloadspec). **Validation:** Optional: \{\} @@ -5470,8 +5490,9 @@ ResourceReference identifies a namespaced resource by name and UID. **Kind:** `type` **Appears in:** [DynamoGraphDeploymentRunStatus](#dynamographdeploymentrunstatus) - + Name is the resource name. +**Validation:** Required: \{\} @@ -5488,12 +5509,14 @@ RoundProgress reports completed and planned optimizer rounds. **Kind:** `type` **Appears in:** [BranchProgress](#branchprogress) - + Completed is the number of finished rounds. +**Validation:** Required: \{\} - + Total is the number of planned rounds. +**Validation:** Required: \{\} @@ -5547,17 +5570,18 @@ See [StaticLoadSpec](#staticloadspec). -TraceFormat identifies the decoder of a recorded request trace. +TraceFormat identifies the decoder of a recorded request trace. Lowering translates it to +AISimulate's native traffic.source.format. **Kind:** `enum` **Underlying type:** `string` -**Validation:** Enum: [dynamo] +**Validation:** Enum: [Dynamo] **Appears in:** [TraceWorkloadSpec](#traceworkloadspec) **Allowed values** -- dynamo TraceFormatDynamo is the Dynamo request trace (dynamo.request.trace.v1) that the frontend records, as JSON Lines or gzip-compressed JSON Lines. +- Dynamo TraceFormatDynamo is the Dynamo request trace (dynamo.request.trace.v1) that the frontend records, as JSON Lines or gzip-compressed JSON Lines. It lowers to AISimulate's dynamo format. @@ -5567,14 +5591,14 @@ TracePVCSource names one trace file on a PersistentVolumeClaim in the Request na **Kind:** `type` **Appears in:** [TraceSourceSpec](#tracesourcespec) - -ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` until its execution stops; deleting the claim cancels the Runs that read it. -**Validation:** MaxLength: 253 MinLength: 1 Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + +ClaimName is the PersistentVolumeClaim, in the Request namespace, that holds the trace. The Run Job mounts only the trace file from it, read-only, and only into the AISimulate container. Each Run protects the claim with the finalizer `nvidia.com/dgdr-run-` until the Run completes or, for a Run deleted before it completes, until no Job of the Run is left. Garbage collection deletes the Job of a deleted Run before the Job's pod, so the finalizer of a deleted Run can be released while that pod still stops; Kubernetes PersistentVolumeClaim protection keeps the claim until no pod uses it. Deleting the claim cancels the Runs that read it. +**Validation:** MaxLength: 253 MinLength: 1 Pattern: `^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` Required: \{\} - -Path names one trace file relative to the root of the claim. It is neither a directory nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' segments or a NUL character, which no file name can hold. The file must be complete before the Request is created and must never be rewritten: every Run of the Request reads it when its Job starts. -**Validation:** MaxLength: 1024 MinLength: 1 + +SubPath names one trace file relative to the root of the claim; the Run Job mounts it as the subPath of the claim volume. It is neither a directory nor a glob. It must not start or end with '/' and must not contain empty, '.' or '..' segments or a NUL character, which no file name can hold. The file must be complete before the Request is created and must never be rewritten: every Run of the Request reads it when its Job starts. +**Validation:** MaxLength: 1024 MinLength: 1 Required: \{\} @@ -5599,8 +5623,8 @@ source in this release. **Kind:** `type` **Appears in:** [TraceWorkloadSpec](#traceworkloadspec) - -PVC reads the trace from a PersistentVolumeClaim in the Request namespace. + +PersistentVolumeClaim reads the trace from a PersistentVolumeClaim in the Request namespace. See [TracePVCSource](#tracepvcsource). **Validation:** Optional: \{\} @@ -5613,15 +5637,16 @@ TraceWorkloadSpec replays a recorded request trace open loop at its recorded arr **Kind:** `type` **Appears in:** [WorkloadSpec](#v1beta2-workloadspec) - -Format identifies the trace decoder. It has no default; only dynamo is admitted in this release. + +Format identifies the trace decoder. Defaults to Dynamo. See [TraceFormat](#traceformat). -**Validation:** Enum: [dynamo] Required: \{\} +**Validation:** Enum: [Dynamo] Optional: \{\} - + Source identifies the trace artifact. See [TraceSourceSpec](#tracesourcespec). +**Validation:** Required: \{\} @@ -5646,7 +5671,7 @@ See [StaticWorkloadSpec](#staticworkloadspec). -Trace replays a recorded request trace from a PersistentVolumeClaim in the Request namespace. Admission supplies no static default when it is set. The Run Job mounts only the trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, or trace.jsonl.gz when the path ends in .gz, and spec.aisimulate.inputOverrides cannot change traffic.source.type, format or paths. +Trace replays a recorded request trace from a PersistentVolumeClaim in the Request namespace. Admission supplies no static default when it is set. The Run Job mounts only the trace file, read-only through a subPath of the claim, at /var/run/aisimulate/trace/trace.jsonl, or trace.jsonl.gz when subPath ends in .gz. The trace lowers to the default traffic.source and traffic.load, which spec.aisimulate.inputOverrides can replace or delete. See [TraceWorkloadSpec](#traceworkloadspec). **Validation:** Optional: \{\} From 444ab6d169ff79215522874474d7379eafcec769 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 09:19:17 -0800 Subject: [PATCH 083/100] test(planner): check the DGDR sweep against the operator's lowering golden The operator's TestDGDRV1beta2TraceRequestLoweringGolden writes deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json: the Request spec of one aggregated grid shard and the Run input that dgdrRunInput lowers it to. The new test checks that build_request returns request_spec, that input_mismatches accepts run_input, and that a restored prefill role or a different trace mount path is still reported. It also checks live_retune_fakes.lower_request_input against run_input, whose docstrings now name the golden as the oracle. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/tests/live_retune_fakes.py | 9 ++- .../tests/unit/test_live_retune_dgdr.py | 57 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py index b1cd13784d0d..09a1cbe4aa83 100644 --- a/components/src/dynamo/planner/tests/live_retune_fakes.py +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -11,7 +11,11 @@ land). ``FakeDgdrOperator`` drives the Requests in a ``FakeRetuneKube`` through the DGDR v1beta2 lifecycle (Accepted -> Run -> Completed with Candidates) the way the operator at #17483 does, lowering each Request into -``Run.status.aisimulate.input`` with the same merge-patch rules. +``Run.status.aisimulate.input`` with the same merge-patch rules. The +operator's golden +``deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json`` +is the oracle for that lowering; ``test_live_retune_dgdr`` checks +``lower_request_input`` against it. This is a plain module, not a conftest: tests import what they need. """ @@ -180,7 +184,8 @@ def lower_request_input( the public role underlay, the ``dgdr_run`` static underlay, the typed underlay, the derived defaults, ``inputOverrides`` as the final merge patch, then the operator-owned ``dgdr_run`` fields. ``inputOverrides`` - wins over the typed trace source: nothing restores it afterwards. + wins over the typed trace source: nothing restores it afterwards. For + the golden's ``request_spec`` it returns the golden's ``run_input``. """ spec = request.get("spec") or {} diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py index fec09326633f..c977c9535127 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -20,6 +20,7 @@ import threading import time from collections.abc import Callable, Mapping +from pathlib import Path from typing import Any, Optional import pytest @@ -849,6 +850,62 @@ def test_input_read_back_allows_operator_defaults_and_follows_overrides(): assert input_mismatches(body["spec"], effective) == [] +# Written by the operator's TestDGDRV1beta2TraceRequestLoweringGolden +# (deploy/operator/internal/controller): request_spec is the Request this +# executor builds for _body(), run_input what dgdrRunInput lowers it to. +LOWERING_GOLDEN = Path( + "deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json" +) + + +def _lowering_golden() -> dict[str, Any]: + for parent in Path(__file__).resolve().parents: + candidate = parent / LOWERING_GOLDEN + if candidate.is_file(): + return json.loads(candidate.read_text(encoding="utf-8")) + raise AssertionError(f"{LOWERING_GOLDEN} not found above {Path(__file__)}") + + +def test_request_and_run_input_match_the_operator_golden(): + golden = _lowering_golden() + body = _body() + assert json.loads(json.dumps(body["spec"])) == golden["request_spec"] + run_input = golden["run_input"] + assert input_mismatches(body["spec"], run_input) == [] + # The fake operator lowers the Request like the real one. + request = {"metadata": {"namespace": "test-ns"}, "spec": golden["request_spec"]} + assert ( + lower_request_input( + request, + run_name=run_input["dgdr_run"]["name"], + runtime_image=run_input["dgdr_run"]["runtime_image"], + dynamo_version=run_input["dgdr_run"]["runtime_version_override"], + gpus_per_node=run_input["dgdr_run"]["num_gpus_per_node"], + ) + == run_input + ) + + restored = json.loads(json.dumps(run_input)) + restored["engine"]["workers"]["prefill"] = {} + assert input_mismatches(body["spec"], restored) == [ + "engine.workers.prefill: expected unset, got {}" + ] + moved = json.loads(json.dumps(run_input)) + moved["traffic"]["source"]["paths"] = ["/var/run/aisimulate/trace/trace.jsonl"] + assert input_mismatches(body["spec"], moved) == [ + "traffic.source.paths.0: expected '/var/run/aisimulate/trace/trace.jsonl.gz'," + " got '/var/run/aisimulate/trace/trace.jsonl'" + ] + # An uncompressed window is mounted as trace.jsonl. + uncompressed = json.loads(json.dumps(body["spec"])) + claim = uncompressed["workload"]["trace"]["source"]["persistentVolumeClaim"] + claim["subPath"] = claim["subPath"].removesuffix(".gz") + assert input_mismatches(uncompressed, run_input) == [ + "traffic.source.paths.0: expected '/var/run/aisimulate/trace/trace.jsonl'," + " got '/var/run/aisimulate/trace/trace.jsonl.gz'" + ] + + def test_missing_fields_reports_pruned_leaves_and_allows_defaults(): submitted = {"a": {"b": 1, "c": [1, 2], "gone": None}, "d": True} stored = {"a": {"b": 1.0, "c": [1, 2], "extra": 5}, "d": True, "default": 1} From dadae47d83d5660ea1782589601c146cc4b1a70c Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 09:24:10 -0800 Subject: [PATCH 084/100] fix(planner): address review on enabling live retune through DGDR v1beta2 - Persist a router's latched initial before moving it off it: the actuator holds a PATCH until the passed (persisted) initials hold the router's initial, and the supervisor re-runs the round once the new initials are written, so a SIGTERM between write and state write leaves no router an advisory or clean-up rollback cannot restore. - Run the legacy clean-up check of a planner without retune in the background; it reads the two Leases with the light kube/state modules, loads the live retune stack only when one exists, and logs 403 at debug. Install the SIGTERM handler whenever that check (or live retune) runs. - One single-thread SweepExecutor owned by the engine adapter is shared by every controller generation; drop the predecessor/running_future chain. The supervisor drops its abort list and in-flight counter and, stopping before the engine, waits for the sweep thread (retune_sweeps_done). - Drop the no-op last_decided_k rewind from replace_live_retune. - Guard decommission deletes with the leadership/epoch check. - Reject a state of an unknown format version without overwriting it; reject non-finite numbers and a bad updated_ms as corrupt. - Deep-copy RetuneDecision details; explicit raises instead of asserts. - bootstrap: module-top imports, narrow excepts, explicit config checks, evaluate_premises(dgd, planner_config=...) per the premises API change. - Helm: gate the live retune Planner RBAC on dgdrV1Beta2.plannerLiveRetune (default false) and trim unused verbs (no watch; Runs get, Candidates list). - Docs: select clean-up Requests by DGD uid, the new Helm value, drop trace_incomplete_threshold; state rules inline instead of design sections. - Tests: the interleaving harness runs the real RetuneController and engine adapter with a stub sweep runner; the actuator fake no longer re-derives the router_policy server rules, and redundant tests are trimmed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/planner/__main__.py | 10 +- components/src/dynamo/planner/core/base.py | 177 ++- components/src/dynamo/planner/core/types.py | 6 +- .../plugins/orchestrator/engine_adapter.py | 82 +- .../src/dynamo/planner/retune/controller.py | 141 +- .../dynamo/planner/retune/live/actuator.py | 9 +- .../dynamo/planner/retune/live/bootstrap.py | 224 ++- .../src/dynamo/planner/retune/live/metrics.py | 3 +- .../src/dynamo/planner/retune/live/state.py | 77 +- .../dynamo/planner/retune/live/supervisor.py | 288 ++-- .../tests/integration/test_live_retune_e2e.py | 14 +- .../src/dynamo/planner/tests/retune_stubs.py | 29 +- .../tests/unit/test_live_retune_actuator.py | 637 +++----- .../tests/unit/test_live_retune_adapter.py | 163 +- .../unit/test_live_retune_controller_hooks.py | 219 +-- .../tests/unit/test_live_retune_metrics.py | 5 +- .../tests/unit/test_live_retune_state.py | 59 +- .../tests/unit/test_live_retune_supervisor.py | 1312 ++++++++++------- deploy/helm/charts/platform/README.md | 1 + .../operator/templates/planner.yaml | 39 +- .../platform/components/operator/values.yaml | 5 + .../platform/tests/planner_rbac_test.yaml | 135 +- deploy/helm/charts/platform/values.yaml | 5 + .../auto-deployment/planner-live-retune.md | 32 +- .../components/planner-configuration.mdx | 3 - 25 files changed, 1908 insertions(+), 1767 deletions(-) diff --git a/components/src/dynamo/planner/__main__.py b/components/src/dynamo/planner/__main__.py index 800685f8220d..bbb7c268e35b 100644 --- a/components/src/dynamo/planner/__main__.py +++ b/components/src/dynamo/planner/__main__.py @@ -53,7 +53,7 @@ async def generate(request: RequestType): generate, # type: ignore[arg-type] health_check_payload={"text": "health"}, ) - if config.retune is None: + if not planner.may_hold_retune_lease: await asyncio.gather(serve, planner.run()) return await _run_until_sigterm(runtime, serve, planner.run()) @@ -64,10 +64,10 @@ async def _run_until_sigterm( ) -> None: """Serve and run the planner; SIGTERM stops the planner cleanly. - Live retune holds a leadership Lease and may have a sweep in flight. - Without a handler SIGTERM kills the process before ``run()``'s - ``finally`` closes the retune controller and releases the Lease, so the - next leader would wait out the Lease. The handler cancels only the run + Live retune, and the clean-up of a removed one, hold a leadership Lease, + and live retune may have a sweep in flight. Without a handler SIGTERM + kills the process before ``run()``'s ``finally`` stops the sweep and + releases the Lease, so the next leader would wait out the Lease. The handler cancels only the run task; once its shutdown finished, the runtime is shut down to stop serving the endpoint and the call returns normally. """ diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index cc7ddd93ed0a..ea11f47bbb6d 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -17,13 +17,14 @@ import time from collections.abc import AsyncIterator, Iterable from contextlib import asynccontextmanager -from typing import TYPE_CHECKING, Optional, Protocol +from typing import TYPE_CHECKING, Any, Optional, Protocol import aiohttp.web from prometheus_client import start_http_server from dynamo.planner.config.defaults import SubComponentType, TargetReplica from dynamo.planner.config.planner_config import PlannerConfig +from dynamo.planner.connectors.kubernetes import KubernetesConnector from dynamo.planner.control_api import ( _MinimumEndpointUnavailableError, _MinimumEndpointValidationError, @@ -57,6 +58,9 @@ runtime_candidate_errors, runtime_configuration_errors, ) +from dynamo.planner.retune.live.contracts import KubeError +from dynamo.planner.retune.live.kube import KubernetesRetuneKube +from dynamo.planner.retune.live.state import find_retune_leases from dynamo.runtime import DistributedRuntime if TYPE_CHECKING: @@ -68,6 +72,24 @@ logger = logging.getLogger(__name__) +def _live_retune_bootstrap() -> Any: + """The live retune stack, loaded only by a planner that runs it. + + It pulls in the DGDR sweep executor and the replay stack behind it; + a planner with retune config loads it at startup, one without only + when a removed live retune left its Leases behind. + """ + + return importlib.import_module("dynamo.planner.retune.live.bootstrap") + + +def _log_legacy_retune_failure(task: asyncio.Task[None]) -> None: + if not task.cancelled() and task.exception() is not None: + logger.error( + "Live retune legacy clean-up check failed", exc_info=task.exception() + ) + + class LiveRetuneHandle(Protocol): """What ``retune.live.bootstrap.build_live_retune`` returns to the base. @@ -75,7 +97,9 @@ class LiveRetuneHandle(Protocol): does not satisfy the retune premises (the engine then starts with retune disabled). ``start`` runs after the engine is bootstrapped and may call back ``replace_live_retune``; ``on_effects`` sees every tick's effects - (it must be cheap and never block); ``stop`` releases leadership. + (it must be cheap and never block); ``stop`` runs before the engine shuts + down: it aborts the sweep, waits for the sweep thread and releases + leadership. """ def retune_inputs(self) -> Optional[RetuneInputs]: @@ -95,11 +119,12 @@ def on_effects(self, effects: PlannerEffects) -> None: class RetuneCleanupHandle(Protocol): - """What ``retune.live.bootstrap.build_legacy_cleanup`` returns (§4.11). + """What ``retune.live.bootstrap.build_legacy_cleanup`` returns. - A planner without retune config rolls back what a removed live retune - deployed, then deletes its Requests and Leases; ``start`` runs after the - engine is bootstrapped and ``stop`` releases leadership. + A planner without retune config whose DGD still has the Leases of a + removed live retune rolls back the routers that retune deployed, then + deletes its Requests and Leases; ``start`` begins that clean-up and + ``stop`` releases leadership. """ async def start(self, host: NativePlannerBase) -> None: @@ -110,9 +135,9 @@ async def stop(self) -> None: _CONFIG_LOCK_TIMEOUT_SECONDS = 10.0 -# Building the legacy retune clean-up reads two Leases and the DGD; startup -# never waits longer than this for it. -_LEGACY_RETUNE_CLEANUP_TIMEOUT_SECONDS = 30.0 +# The background check for a removed live retune reads two Leases, then, +# only if one exists, the DGD; each Kubernetes call is bounded by this. +_LEGACY_RETUNE_CALL_TIMEOUT_SECONDS = 10.0 # A tick waits briefly for the common fast submission path, then leaves the # task running. It is deliberately not cancelled: a remote request may have # been accepted even when its response has not arrived, so retrying could @@ -229,7 +254,9 @@ def __init__( self._last_worker_counts: Optional[WorkerCounts] = None # Live retune (``retune.executor: dgdr``): built in _async_init. self._live_retune: Optional[LiveRetuneHandle] = None - # §4.11 legacy clean-up of a removed live retune (no retune config). + # Without retune config, a Kubernetes planner checks in the background + # whether a removed live retune left its Leases, and cleans up after it. + self._legacy_retune_task: Optional[asyncio.Task[None]] = None self._legacy_retune_cleanup: Optional[RetuneCleanupHandle] = None async def _async_init(self) -> None: @@ -244,15 +271,13 @@ async def _async_init(self) -> None: # Before the first _ensure_engine (regression bootstrap builds # the engine): the engine takes the live retune inputs/hooks. self._live_retune = await self._build_live_retune() - else: - self._legacy_retune_cleanup = await self._build_legacy_retune_cleanup() await self._bootstrap_regression() await self._bootstrap_engine_plugins_if_needed() if self._live_retune is not None: await self._live_retune.start(self) - await self._start_legacy_retune_cleanup() + self._start_legacy_retune_check() if self.config.enable_runtime_reconfiguration: if self.runtime is None: @@ -289,64 +314,74 @@ async def _async_init(self) -> None: await self._shutdown_runtime() raise + @property + def may_hold_retune_lease(self) -> bool: + """Whether live retune, or the clean-up of a removed one, may hold + the retune leadership Lease (known once ``_async_init`` returned).""" + + return self._live_retune is not None or self._legacy_retune_task is not None + async def _build_live_retune(self) -> LiveRetuneHandle: - # Imported on use: the live retune stack talks to Kubernetes, and - # planners without retune never load it. - bootstrap = importlib.import_module("dynamo.planner.retune.live.bootstrap") - return await bootstrap.build_live_retune(self) + return await _live_retune_bootstrap().build_live_retune(self) - def _uses_kubernetes_connector(self) -> bool: + def _kubernetes_connector(self) -> Optional[KubernetesConnector]: + # Only some environments have a deployment controller. controller = getattr(self.environment, "controller", None) - if controller is None: - return False - try: - from dynamo.planner.connectors.kubernetes import KubernetesConnector - except ImportError: - return False - return isinstance(controller, KubernetesConnector) + return controller if isinstance(controller, KubernetesConnector) else None - async def _build_legacy_retune_cleanup(self) -> Optional[RetuneCleanupHandle]: - """§4.11 clean-up of a live retune whose config was removed. + def _start_legacy_retune_check(self) -> None: + """Look for a removed live retune in the background. - Only for a Kubernetes planner. ``None`` when nothing was left over; - it never fails startup: missing RBAC (403), a missing DGD or Lease - (404) and any other error are logged and skipped. + Only a Kubernetes planner without retune config. Startup never waits + for it, and it never fails the planner. """ - if not self._uses_kubernetes_connector(): - return None + connector = self._kubernetes_connector() + if self.config.retune is not None or connector is None: + return + task = asyncio.create_task( + self._legacy_retune_check(connector), name="planner-retune-legacy-check" + ) + task.add_done_callback(_log_legacy_retune_failure) + self._legacy_retune_task = task + + async def _legacy_retune_check(self, connector: KubernetesConnector) -> None: + """Clean up after a live retune whose config was removed. + + Reads the DGD's two retune Leases; only when one exists does it load + the live retune stack, which rolls back the routers that retune + deployed and then deletes its Requests and Leases. Without Lease RBAC + (the default chart) the read answers 403, logged at debug. + """ + + kube = KubernetesRetuneKube.from_kube_api(connector.kube_api) try: - bootstrap = importlib.import_module("dynamo.planner.retune.live.bootstrap") - return await asyncio.wait_for( - bootstrap.build_legacy_cleanup(self), - timeout=_LEGACY_RETUNE_CLEANUP_TIMEOUT_SECONDS, + leases = await asyncio.to_thread( + find_retune_leases, + kube, + connector.parent_dgd_name, + timeout_s=_LEGACY_RETUNE_CALL_TIMEOUT_SECONDS, ) - except Exception as exc: - logger.info("Live retune legacy clean-up skipped: %r", exc) - logger.debug("Live retune legacy clean-up setup failed", exc_info=True) - return None - - async def _start_legacy_retune_cleanup(self) -> None: - cleanup = self._legacy_retune_cleanup - if cleanup is None: + if not leases: + return + cleanup = await _live_retune_bootstrap().build_legacy_cleanup( + self, kube, timeout_s=_LEGACY_RETUNE_CALL_TIMEOUT_SECONDS + ) + except KubeError as error: + # ``status`` is set by the Kubernetes client wrapper only. + if getattr(error, "status", None) == 403: + logger.debug("Live retune legacy clean-up check skipped: %s", error) + else: + logger.info("Live retune legacy clean-up skipped: %s", error) return - try: - await cleanup.start(self) - except Exception as exc: - logger.info("Live retune legacy clean-up did not start: %r", exc) - logger.debug("Live retune legacy clean-up start failed", exc_info=True) - self._legacy_retune_cleanup = None - try: - await cleanup.stop() - except Exception: - logger.debug("Stopping the legacy clean-up failed", exc_info=True) + self._legacy_retune_cleanup = cleanup + await cleanup.start(self) async def replace_live_retune( self, inputs: Optional[RetuneInputs], *, generation: int, - last_decided_k: Optional[int], ) -> None: """Swap the engine's live retune controller (supervisor callback). @@ -363,12 +398,7 @@ async def replace_live_retune( engine = self._engine if not isinstance(engine, OrchestratorEngineAdapter): raise RuntimeError("live retune requires the orchestrator engine") - engine.replace_retune( - inputs, - generation=generation, - now_s=time.time(), - last_decided_k=last_decided_k, - ) + engine.replace_retune(inputs, generation=generation, now_s=time.time()) async def _shutdown_runtime(self) -> None: """Release initialized planner resources after normal or failed startup.""" @@ -398,16 +428,8 @@ async def _shutdown_runtime(self) -> None: except Exception: logger.exception("Failed to stop planner live dashboard") - engine = self._engine - self._engine = None - if engine is not None: - try: - await engine.shutdown() - except Exception: - logger.exception("Failed to stop planner engine") - - # After the engine: its shutdown closed the retune controller (and - # stopped its sweep), so the supervisor can release leadership. + # Before the engine: the supervisor aborts the retune sweep, waits + # for the engine's sweep thread and then releases leadership. live_retune = self._live_retune self._live_retune = None if live_retune is not None: @@ -416,6 +438,11 @@ async def _shutdown_runtime(self) -> None: except Exception: logger.exception("Failed to stop planner live retune") + legacy_retune_task = self._legacy_retune_task + self._legacy_retune_task = None + if legacy_retune_task is not None: + legacy_retune_task.cancel() + await asyncio.wait([legacy_retune_task]) legacy_retune_cleanup = self._legacy_retune_cleanup self._legacy_retune_cleanup = None if legacy_retune_cleanup is not None: @@ -424,6 +451,14 @@ async def _shutdown_runtime(self) -> None: except Exception: logger.exception("Failed to stop the live retune legacy clean-up") + engine = self._engine + self._engine = None + if engine is not None: + try: + await engine.shutdown() + except Exception: + logger.exception("Failed to stop planner engine") + effect_submission_task = self._effect_submission_task self._effect_submission_task = None if effect_submission_task is not None: diff --git a/components/src/dynamo/planner/core/types.py b/components/src/dynamo/planner/core/types.py index 195c625c7b15..49afd34ee41f 100644 --- a/components/src/dynamo/planner/core/types.py +++ b/components/src/dynamo/planner/core/types.py @@ -11,6 +11,7 @@ from __future__ import annotations +import copy import json import math from dataclasses import dataclass, field @@ -400,8 +401,9 @@ def __post_init__(self) -> None: object.__setattr__(self, "predictor_warmup", tuple(self.predictor_warmup)) if self.failure_reason is not None: _require_non_empty_string("failure_reason", self.failure_reason) - # A private copy: the record must not change with the sweep result. - object.__setattr__(self, "details", dict(self.details)) + # A private deep copy: the record must not change with the sweep + # result, whose details may nest mutable values. + object.__setattr__(self, "details", copy.deepcopy(self.details)) def to_dict(self) -> dict[str, Any]: """JSON-shaped record; ``predictor_warmup`` becomes its length.""" diff --git a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py index be3ab8179639..d5e792ad91bb 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py +++ b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py @@ -61,6 +61,7 @@ import logging from collections.abc import Callable, Iterable +from concurrent.futures import Future from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional, Sequence @@ -69,11 +70,9 @@ from dynamo.planner.monitoring.signal_health import PlannerSignalHealth if TYPE_CHECKING: - from concurrent.futures import Future - import grpc.aio - from dynamo.planner.retune.controller import RetuneController + from dynamo.planner.retune.controller import RetuneController, SweepExecutor from dynamo.planner.retune.live.contracts import HarvestAllowed, LaunchGate from dynamo.planner.core.budget import ( @@ -150,6 +149,10 @@ class RetuneHooks: phase_s: float = 0.0 +def _no_sweep() -> None: + """Queued behind the live sweeps by ``retune_sweeps_done``.""" + + class OrchestratorEngineAdapter: """``EngineProtocol``-compatible wrapper around the builtin plugin chain. @@ -355,9 +358,10 @@ def _factory(plugin_id, endpoint, *, in_process_instance=None): self._register_builtin_plugins() self._retune_hooks = retune_hooks self._retune: Optional[RetuneController] = None - # A replaced controller's sweep that may still be running while - # retune is disabled; the next generation's first launch waits for it. - self._retune_predecessor: Optional[Future[Any]] = None + # Live: the one sweep thread every controller generation submits to, + # so a rebuilt controller's sweep queues behind a dropped one and two + # generations never sweep at once. + self._retune_executor: Optional[SweepExecutor] = None if retune_hooks is None or self._config.retune is None: self._retune = self._build_retune_controller(retune_inputs) elif retune_inputs is None: @@ -385,15 +389,13 @@ def _build_retune_controller( retune_inputs: Optional[RetuneInputs], *, generation: int = 0, - predecessor: Optional[Future[Any]] = None, ) -> Optional[RetuneController]: """Build the retune controller iff ``config.retune`` is set. The controller validates the retune setup in its constructor, so an invalid one fails here, at engine construction. With live hooks the - controller gets them, ``generation`` and ``predecessor`` (the sweep - its first launch waits for); without (replay) it is built exactly as - before. + controller gets them, ``generation`` and the adapter's shared sweep + executor; without (replay) it is built exactly as before. """ if self._config.retune is None: return None @@ -406,7 +408,7 @@ def _build_retune_controller( # of the import chain of planners that never enable retune, and avoids # an import cycle through dynamo.replay, whose planner host imports # this module. - from dynamo.planner.retune.controller import RetuneController + from dynamo.planner.retune.controller import RetuneController, SweepExecutor hooks = self._retune_hooks if hooks is None: @@ -417,6 +419,8 @@ def _build_retune_controller( self._clock, self._scale_interval, ) + if self._retune_executor is None: + self._retune_executor = SweepExecutor() return RetuneController( self._config, retune_inputs, @@ -427,7 +431,7 @@ def _build_retune_controller( launch_gate=hooks.launch_gate, harvest_allowed=hooks.harvest_allowed, generation=generation, - predecessor=predecessor, + executor=self._retune_executor, ) @property @@ -443,21 +447,18 @@ def replace_retune( *, generation: int, now_s: float, - last_decided_k: Optional[int], ) -> None: """Swap the live retune controller without rebuilding the engine. The old controller's sweep is aborted (``drop_pending``) and the controller closed. With ``retune_inputs`` a new controller of - ``generation`` is built with the same hooks, anchored at ``now_s`` - and the hooks' phase, and rewound to ``last_decided_k`` when given; - with ``None`` the adapter stays disabled. The new controller's first - launch queues behind the old controller's sweep while that is still - running (``running_future``), also across disabled generations, so - two generations never sweep at once. A construction error is - raised after the old controller is gone, leaving the adapter - disabled. Callers serialize this with ticks (the planner's config - lock). + ``generation`` is built with the same hooks and anchored at ``now_s`` + and the hooks' phase; with ``None`` the adapter stays disabled. Every + generation submits to the adapter's one sweep thread, so the new + controller's first sweep queues behind the old one while that still + runs. A construction error is raised after the old controller is + gone, leaving the adapter disabled. Callers serialize this with + ticks (the planner's config lock). """ hooks = self._retune_hooks if self._config.retune is None or hooks is None: @@ -473,33 +474,34 @@ def replace_retune( old.close() except Exception: log.exception("retune controller close failed") - # After drop_pending and close: the sweep that still holds the - # old controller's thread (and, transitively, older ones). - running = old.running_future() - if running is not None: - self._retune_predecessor = running - if self._retune_predecessor is not None and self._retune_predecessor.done(): - self._retune_predecessor = None if retune_inputs is None: log.info( "retune controller generation %d: disabled (premises do not hold)", generation, ) return - controller = self._build_retune_controller( - retune_inputs, - generation=generation, - predecessor=self._retune_predecessor, - ) - assert controller is not None - # The new controller now carries it (and hands it on when replaced). - self._retune_predecessor = None + controller = self._build_retune_controller(retune_inputs, generation=generation) + if controller is None: + raise RuntimeError("replace_retune requires a live retune configuration") controller.start(now_s, hooks.phase_s) - if last_decided_k is not None: - controller.rewind(last_decided_k) self._retune = controller log.info("retune controller generation %d installed", generation) + def retune_sweeps_done(self) -> Optional[Future[None]]: + """A future done once every live sweep submitted so far returned. + + The sweep thread runs one sweep at a time in submission order, so a + no-op queued behind them completes after the last one. ``None`` + without live retune or once the engine is shut down. + """ + executor = self._retune_executor + if executor is None: + return None + try: + return executor.submit(_no_sweep) + except RuntimeError: # shut down + return None + @property def plugins_bootstrapped(self) -> bool: return self._plugins_bootstrapped @@ -966,6 +968,8 @@ async def shutdown(self) -> None: type(exc).__name__, exc, ) + if self._retune_executor is not None: + self._retune_executor.shutdown(wait=False, cancel_futures=True) await self._orchestrator.shutdown() # ------------------------------------------------------------------ diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index 81799e97da76..a11bdf849cfd 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -39,10 +39,10 @@ - ``drop_pending``/``rewind`` let the host abandon an in-flight sweep without consuming its window; ``restore_desired`` re-installs persisted router values. -- ``predecessor``/``running_future`` carry the single sweep thread across - controller generations: a rebuilt controller's first launch queues behind - the previous generation's still-running sweep, as a relaunch queues - behind a dropped sweep within one controller. +- ``executor`` is the live host's single sweep thread, shared by every + controller generation it builds, so two generations never sweep at once: + a launch queues behind a dropped sweep that still runs, and its launch + time moves to when the sweep really started. """ from __future__ import annotations @@ -55,7 +55,6 @@ from collections import deque from collections.abc import Callable, Mapping, Sequence from concurrent.futures import Future, ThreadPoolExecutor -from concurrent.futures import wait as wait_futures from dataclasses import dataclass, field from typing import Any @@ -110,6 +109,35 @@ SweepFn = Callable[..., SweepResult] +class SweepExecutor(ThreadPoolExecutor): + """The single sweep thread; ``busy`` while a submitted call is unfinished. + + A live host shares one across the controller generations it builds, so + their sweeps run one at a time in submission order. + """ + + def __init__(self) -> None: + super().__init__(max_workers=1, thread_name_prefix="planner-retune") + self._lock = threading.Lock() + self._unfinished = 0 + + def busy(self) -> bool: + with self._lock: + return self._unfinished > 0 + + def submit(self, fn: Callable[..., Any], /, *args: Any, **kwargs: Any) -> Future: + future = super().submit(fn, *args, **kwargs) + with self._lock: + self._unfinished += 1 + # Runs at once if the call already finished (or was cancelled). + future.add_done_callback(self._finished) + return future + + def _finished(self, _future: Future) -> None: + with self._lock: + self._unfinished -= 1 + + @dataclass(frozen=True) class RetuneTickResult: """What one ``on_tick`` call hands back to the engine. @@ -139,9 +167,10 @@ class _PendingSweep: ready_at_s: float | None # Set by the supervisor to stop this sweep (lost leadership, rebuild). abort: threading.Event | None = None - # When submitted behind a dropped (or predecessor generation's) sweep - # that was still running: ``(submitted_monotonic, [started_monotonic])``. - # ``launched_at_s`` then moves to the moment the sweep really started. + # Submitted while the sweep thread was still busy with a dropped sweep + # (of this or an earlier controller generation): + # ``(submitted_monotonic, [started_monotonic])``. ``launched_at_s`` then + # moves to the moment the sweep really started. queued: tuple[float, list[float]] | None = None @@ -162,10 +191,9 @@ class RetuneController: ``abort=`` when ``launch_gate`` is set), ``launch_gate`` and ``harvest_allowed`` are the supervisor's predicates (module docstring), and ``generation`` identifies this controller among the ones a live host - builds over its lifetime; every sweep request carries it. - ``predecessor`` is the previous generation's ``running_future()``: this - controller's first launch waits for it, so two generations never sweep - at once. + builds over its lifetime; every sweep request carries it. ``executor`` + is the host's single-thread sweep executor, shared across generations; + without one (replay) the controller owns its sweep thread. """ def __init__( @@ -180,7 +208,7 @@ def __init__( launch_gate: LaunchGate | None = None, harvest_allowed: HarvestAllowed | None = None, generation: int = 0, - predecessor: Future[SweepResult] | None = None, + executor: SweepExecutor | None = None, ) -> None: retune = config.retune if retune is None: @@ -192,8 +220,6 @@ def __init__( raise ValueError("RetuneController launch_gate requires a sweep_fn") if isinstance(generation, bool) or not isinstance(generation, int): raise ValueError("generation must be an integer") - if predecessor is not None and not isinstance(predecessor, Future): - raise ValueError("predecessor must be a concurrent.futures.Future") self._sweep_fn = sweep_fn self._launch_gate = launch_gate self._harvest_allowed = harvest_allowed @@ -265,15 +291,9 @@ def __init__( self._last_k = -1 self._history: deque[tuple[float, dict[str, int]]] = deque() self._pending: _PendingSweep | None = None - # A dropped sweep that may still hold the single sweep thread, or - # the previous generation's sweep that a first launch waits for. - self._dropped: Future[SweepResult] | None = ( - predecessor if predecessor is not None and not predecessor.done() else None - ) self._cancel = threading.Event() - self._executor = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="planner-retune" - ) + self._owns_executor = executor is None + self._executor = executor or SweepExecutor() logger.info( "retune enabled: interval=%ss grace=%ss %d grid points, router keys %s, " "planner keys %s, predictor candidates %s, objective %s, " @@ -338,8 +358,7 @@ async def on_tick( pending = self._pending if pending is not None and pending.future.done(): - if pending.queued is not None: - self._settle_queue_delay(pending) + self._settle_queue_delay(pending) if pending.ready_at_s is None: pending.decision_latency_s = self._decision_latency_s(pending.future) pending.ready_at_s = pending.launched_at_s + pending.decision_latency_s @@ -385,10 +404,17 @@ async def on_tick( ) def close(self) -> None: - """Kill the sweep workers and release the sweep thread.""" + """Kill the sweep workers and release the sweep thread. + + A shared (host-owned) executor stays up: only this controller's + sweep is cancelled, or told to stop if it already runs. + """ self._cancel.set() - self._executor.shutdown(wait=False, cancel_futures=True) + if self._owns_executor: + self._executor.shutdown(wait=False, cancel_futures=True) + elif self._pending is not None: + self._pending.future.cancel() def drop_pending(self) -> None: """Abandon the in-flight sweep without deciding its window. @@ -406,12 +432,7 @@ def drop_pending(self) -> None: self._pending = None if pending.abort is not None: pending.abort.set() - if not pending.future.cancel() and not pending.future.done(): - # Running, so any earlier dropped sweep has finished, or a - # predecessor generation's sweep is still running and this one - # waits for it: either way this one is now what a new launch - # would queue behind. - self._dropped = pending.future + pending.future.cancel() logger.info( "retune dropped the in-flight sweep of window %d", pending.request.window_index, @@ -449,24 +470,6 @@ def restore_desired(self, flat: Mapping[str, float]) -> None: self._router_map(router) self._desired_router = router - def running_future(self) -> Future[SweepResult] | None: - """The sweep that holds or will hold the sweep thread, if any. - - The pending sweep while it runs, else a dropped (or predecessor - generation's) sweep that is still running, else a pending sweep - that has not started; ``None`` once all of them are done. A host - replacing this controller calls it after ``drop_pending`` and - ``close`` and passes it as the successor's ``predecessor``. - """ - - pending = self._pending.future if self._pending is not None else None - if pending is not None and pending.running(): - return pending - for future in (self._dropped, pending): - if future is not None and not future.done(): - return future - return None - @property def generation(self) -> int: """The generation the host built this controller with.""" @@ -758,7 +761,8 @@ def _harvest_or_rewind( def _settle_queue_delay(self, pending: _PendingSweep) -> None: """Move a queued sweep's launch time to when it actually started.""" - assert pending.queued is not None + if pending.queued is None: + return submitted, started = pending.queued pending.queued = None if not started: @@ -928,32 +932,29 @@ def _submit( """Submit the sweep to the single sweep thread. ``run_sweep`` is looked up at call time so tests can substitute it. - A gated launch also passes the gate's abort event. Behind a dropped - sweep that still holds the thread, or a predecessor generation's - sweep that still runs on its own controller's thread, the call waits - for it and records when it really starts (``_settle_queue_delay``). + A gated (live) launch also passes the gate's abort event. While the + sweep thread still runs a dropped sweep, of this or an earlier + controller generation, the launch queues behind it and records when + it really starts (``_settle_queue_delay``). """ sweep_fn: SweepFn = self._sweep_fn if self._sweep_fn is not None else run_sweep - kwargs: dict[str, Any] = {"cancel": self._cancel} - if gated: - kwargs["abort"] = abort - if self._dropped is not None and self._dropped.done(): - self._dropped = None - if self._dropped is None: - return self._executor.submit(sweep_fn, request, **kwargs), None + if not gated: + return self._executor.submit(sweep_fn, request, cancel=self._cancel), None + cancel = self._cancel + if not self._executor.busy(): + future = self._executor.submit( + sweep_fn, request, cancel=cancel, abort=abort + ) + return future, None clock = self._clock - blocker = self._dropped started: list[float] = [] - def queued_sweep() -> SweepResult: - # A dropped sweep of this controller has already finished (one - # sweep thread); a predecessor generation's may still be running. - wait_futures((blocker,)) + def timed_sweep() -> SweepResult: started.append(clock.monotonic()) - return sweep_fn(request, **kwargs) + return sweep_fn(request, cancel=cancel, abort=abort) - future = self._executor.submit(queued_sweep) + future = self._executor.submit(timed_sweep) return future, (clock.monotonic(), started) def _decision_latency_s(self, future: Future[SweepResult]) -> float: diff --git a/components/src/dynamo/planner/retune/live/actuator.py b/components/src/dynamo/planner/retune/live/actuator.py index 3f2282beecfe..67280412bd08 100644 --- a/components/src/dynamo/planner/retune/live/actuator.py +++ b/components/src/dynamo/planner/retune/live/actuator.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Apply the retune router parameters to every frontend router (design §4.8). +"""Apply the retune router parameters to every frontend router. ``RouterActuator.reconcile`` runs one round: list the DGD's frontend pods, ``GET /router_policy`` on each, keep the routers of the DGD's dynamo @@ -32,6 +32,10 @@ unknown (revision > 0, never latched) is reset once first to learn it; a router whose initial differs from the startup map (P14) is never written, so planner-parsed values cannot leak into keys the sweep did not search. + A router is PATCHed only once the ``initials`` passed in (the persisted + ones) hold its initial: a router latched in this round is held, reported + in ``new_initials`` and not converged, so a process stopped before the + state write never leaves a router off an initial nobody recorded. - ``rollback``: reset only routers whose configuration differs from their latched initial. Never by revision: a reset itself bumps the revision. - ``observe``: no writes; latch initials and check parity only. @@ -678,6 +682,9 @@ def _plan( if initial is None: # Revision > 0 and never latched: learn the initial first. return ("reset", {}) + if not _same_map(state.initials.get(router.router_id), initial): + # Latched but not persisted yet: hold until the state records it. + return None if not _same_map(initial, self._startup_maps[router.worker_type]): # P14 failed for this router; writing explicit values would put # planner-parsed values into keys the sweep never searched. diff --git a/components/src/dynamo/planner/retune/live/bootstrap.py b/components/src/dynamo/planner/retune/live/bootstrap.py index 6231f759146b..8cf5347ec468 100644 --- a/components/src/dynamo/planner/retune/live/bootstrap.py +++ b/components/src/dynamo/planner/retune/live/bootstrap.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Build the live retune supervisor of a Kubernetes planner (design §4.2, §4.3). +"""Build the live retune supervisor of a Kubernetes planner. ``build_live_retune(host)`` runs in ``NativePlannerBase._async_init`` after ``environment.initialize()`` and before the engine is built. It reads the @@ -11,17 +11,17 @@ controller is built from (``None`` while P4 fails: the adapter then starts with retune disabled) and whose ``hooks()`` every controller gets. -Every collaborator (Kubernetes client, router actuator, reconciler, DGDR -executor, window cutter) is imported on use, so planners without live retune -never load them. ``build_supervisor`` takes them injectable for tests. - -``build_legacy_cleanup(host)`` is the §4.11 clean-up for a planner whose -retune was removed: if a previous live retune left its Leases behind, it -returns a supervisor in decommission mode that rolls back what that retune -deployed, then deletes its Requests and Leases. ``NativePlannerBase`` -builds it in ``_async_init`` for a Kubernetes planner without retune config, -starts it after the engine and stops it in ``_shutdown_runtime``; a failure -to build it never fails startup. +This module imports the whole live retune stack (Kubernetes client, router +actuator, reconciler, DGDR executor, window cutter). +``NativePlannerBase`` imports it only when the planner runs live retune or +must clean up after a removed one; ``build_supervisor`` takes the +collaborators injectable for tests. + +``build_legacy_cleanup(host, kube)`` is the clean-up for a planner whose +retune was removed. ``NativePlannerBase`` calls it in the background, only +after finding that a previous live retune left its Leases behind; it returns +a supervisor in decommission mode that rolls back what that retune deployed, +then deletes its Requests and Leases. """ from __future__ import annotations @@ -34,24 +34,40 @@ import socket import uuid from collections.abc import Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Optional - +from typing import Any, Optional + +from dynamo.llm import KvRouterConfig +from dynamo.planner.config.defaults import SubComponentType +from dynamo.planner.config.planner_config import ( + DgdrExecutorConfig, + PlannerConfig, + RetuneConfig, + RetuneEngineSpec, +) +from dynamo.planner.core.types import RetuneInputs +from dynamo.planner.errors import ComponentError +from dynamo.planner.monitoring.dgd_services import get_component_from_type_or_name +from dynamo.planner.retune.domains import expand_router_domain +from dynamo.planner.retune.live import frontend_config, window_cut +from dynamo.planner.retune.live.actuator import ( + AiohttpRouterPolicyClient, + RouterActuator, +) from dynamo.planner.retune.live.contracts import ( DgdRef, FrontendStartup, KubeAlreadyExists, - KubeError, KubeInvalid, PremiseReport, RetuneKube, RetuneMetricsSink, WindowCutter, ) -from dynamo.planner.retune.live.state import ( - fingerprint, - leadership_lease_name, - state_lease_name, -) +from dynamo.planner.retune.live.dgdr import DgdrContext, Shard, build_request +from dynamo.planner.retune.live.kube import KubernetesRetuneKube +from dynamo.planner.retune.live.metrics import RetuneMetrics +from dynamo.planner.retune.live.reconciler import Reconciler +from dynamo.planner.retune.live.state import fingerprint from dynamo.planner.retune.live.supervisor import ( ActuatorFactory, LiveRetuneSupervisor, @@ -60,10 +76,6 @@ worker_types_for, ) -if TYPE_CHECKING: - from dynamo.planner.config.planner_config import PlannerConfig - from dynamo.planner.core.types import RetuneInputs - logger = logging.getLogger(__name__) # Shard id and window of the P11 canary Request (dry run only, never stored). @@ -90,8 +102,6 @@ def __init__( self._worker_types = tuple(worker_types) def startup_flat(self, keys: Sequence[str]) -> dict[str, float]: - from dynamo.llm import KvRouterConfig - startup = KvRouterConfig.from_json(json.dumps(self._router_config)) serialized = json.loads(startup.to_json()) resolved = startup.policy_parameters("aggregated") @@ -102,8 +112,6 @@ def startup_flat(self, keys: Sequence[str]) -> dict[str, float]: return values def by_wt(self, flat: Mapping[str, float]) -> dict[str, dict[str, float]]: - from dynamo.llm import KvRouterConfig - config = KvRouterConfig.from_json(json.dumps({**self._router_config, **flat})) return { worker_type: { @@ -126,9 +134,6 @@ def worker_spec_replicas( An omitted ``replicas`` is the workload default of 1. """ - from dynamo.planner.config.defaults import SubComponentType - from dynamo.planner.monitoring.dgd_services import get_component_from_type_or_name - roles = ( (("prefill", SubComponentType.PREFILL), ("decode", SubComponentType.DECODE)) if planner_mode == "disagg" @@ -138,7 +143,7 @@ def worker_spec_replicas( for worker_type, role in roles: try: service = get_component_from_type_or_name(dict(dgd), role) - except Exception as error: + except ComponentError as error: logger.warning( "retune: cannot resolve the %s workers: %s", worker_type, error ) @@ -169,13 +174,7 @@ def build_retune_inputs( epoch milliseconds (origin 0, speedup 1). """ - from dynamo.planner.config.planner_config import RetuneEngineSpec - from dynamo.planner.core.types import RetuneInputs - - retune = config.retune - if retune is None or retune.dgdr is None: - raise ValueError("live retune requires retune.executor 'dgdr'") - dgdr = retune.dgdr + _, dgdr = _dgdr_retune(config) disagg = config.mode == "disagg" engine = RetuneEngineSpec( router_mode=startup.router_mode, # type: ignore[arg-type] @@ -208,8 +207,7 @@ def evaluate_spec( """ if frontend is None: - from dynamo.planner.retune.live import frontend_config as frontend - + frontend = frontend_config metadata = dgd.get("metadata") or {} name = str(metadata.get("name") or "") uid = metadata.get("uid") @@ -223,14 +221,7 @@ def evaluate_spec( ) parsed = frontend.parse_frontend(dgd, planner_mode=config.mode) startup = parsed if isinstance(parsed, FrontendStartup) else None - parse_error = None if startup is not None else str(parsed) - report: PremiseReport = frontend.evaluate_premises( - dgd, - planner_config=config, - startup=startup, - parse_error=parse_error, - planner_replicas=None, - ) + report: PremiseReport = frontend.evaluate_premises(dgd, planner_config=config) inputs = None if report.p4_ok and startup is not None: try: @@ -238,7 +229,7 @@ def evaluate_spec( config, startup, dgd_name=name, - model_name=_served_model(frontend, dgd), + model_name=frontend.served_model_name(dgd), ) except ValueError as error: logger.error("retune: building the live retune inputs failed: %s", error) @@ -260,16 +251,6 @@ def evaluate_spec( ) -def _served_model(frontend: Any, dgd: Mapping[str, Any]) -> Optional[str]: - served = getattr(frontend, "served_model_name", None) - if served is None: - return None - try: - return served(dgd) - except Exception: - return None - - # --------------------------------------------------------- DGDR pieces @@ -278,26 +259,16 @@ def dgdr_context_factory( ) -> Callable[[LiveRetuneSupervisor], Any]: """Bind a ``dgdr.DgdrContext`` to the supervisor's current DGD.""" - def factory(supervisor: LiveRetuneSupervisor) -> Any: - from dynamo.planner.retune.live.dgdr import DgdrContext - - retune = config.retune - assert retune is not None and retune.dgdr is not None - cutter: WindowCutter - if cut_window is not None: - cutter = cut_window - else: - from dynamo.planner.retune.live import window_cut - - cutter = window_cut.cut_window + def factory(supervisor: LiveRetuneSupervisor) -> DgdrContext: + retune, dgdr = _dgdr_retune(config) return DgdrContext( dgd=supervisor.dgd, - cfg=retune.dgdr, + cfg=dgdr, retune=retune, backend=str(config.backend), kube=supervisor.kube, registry=supervisor.registry, - cut_window=cutter, + cut_window=cut_window or window_cut.cut_window, is_leader=supervisor.is_leader, ) @@ -317,10 +288,7 @@ def dgdr_canary( """ def canary(supervisor: LiveRetuneSupervisor) -> bool: - from dynamo.planner.retune.live.dgdr import Shard, build_request - - retune = config.retune - assert retune is not None and retune.dgdr is not None + retune, dgdr = _dgdr_retune(config) replicas = supervisor.evaluation.spec_replicas point = supervisor.startup_flat() if not replicas or not point: @@ -334,7 +302,7 @@ def canary(supervisor: LiveRetuneSupervisor) -> bool: ) body = build_request( dgd=supervisor.dgd, - cfg=retune.dgdr, + cfg=dgdr, retune=retune, sla_ttft_ms=sla_ttft_ms, sla_itl_ms=sla_itl_ms, @@ -361,12 +329,9 @@ def canary(supervisor: LiveRetuneSupervisor) -> bool: def default_actuator_factory( kube: RetuneKube, metrics: RetuneMetricsSink ) -> ActuatorFactory: - def factory(dgd: DgdRef, startup_maps: dict[str, dict[str, float]]) -> Any: - from dynamo.planner.retune.live.actuator import ( - AiohttpRouterPolicyClient, - RouterActuator, - ) - + def factory( + dgd: DgdRef, startup_maps: dict[str, dict[str, float]] + ) -> RouterActuator: return RouterActuator( kube, AiohttpRouterPolicyClient(), dgd, startup_maps, metrics ) @@ -375,7 +340,11 @@ def factory(dgd: DgdRef, startup_maps: dict[str, dict[str, float]]) -> Any: def planner_identity() -> str: - """``/`` (§4.7.1).""" + """``/``: the Lease holder identity of this process. + + The nonce makes a restarted container a new holder, so it never renews + the Lease of its previous incarnation. + """ pod = ( os.environ.get("POD_NAME") @@ -392,26 +361,23 @@ def searched_router_keys(config: PlannerConfig) -> tuple[str, ...]: retune = config.retune if retune is None: return () - try: - from dynamo.planner.retune.domains import expand_router_domain + return tuple(sorted(expand_router_domain(retune))) - return tuple(sorted(expand_router_domain(retune))) - except Exception: - logger.exception("retune: expanding the router domain failed") - return () + +def _dgdr_retune(config: PlannerConfig) -> tuple[RetuneConfig, DgdrExecutorConfig]: + """``config.retune`` and its DGDR executor section, which must be set.""" + + retune = config.retune + if retune is None or retune.dgdr is None: + raise ValueError("live retune requires retune.executor 'dgdr'") + return retune, retune.dgdr def _runtime_namespace(host: Any) -> str: - environment = getattr(host, "environment", None) - getter = getattr(environment, "runtime_namespace", None) - if callable(getter): - try: - value = getter() - if value: - return str(value) - except Exception: - logger.debug("retune: runtime namespace unavailable", exc_info=True) - return str(host.config.namespace) + """The routers' dynamo namespace, else the planner's configured one.""" + + value = host.environment.runtime_namespace() + return str(value) if value else str(host.config.namespace) # --------------------------------------------------------------- build @@ -443,8 +409,6 @@ async def build_supervisor( ) dgdr = retune.dgdr if metrics is None: - from dynamo.planner.retune.live.metrics import RetuneMetrics - metrics = RetuneMetrics() dgd_obj = await asyncio.to_thread(kube.get_dgd, dgd_name) @@ -459,8 +423,6 @@ def evaluate(obj: dict) -> SpecEvaluation: evaluation = evaluate(dgd_obj) if reconciler is None: - from dynamo.planner.retune.live.reconciler import Reconciler - reconciler = Reconciler( kube, evaluation.dgd, dgdr, retune, dgdr.mount_path, metrics ) @@ -520,8 +482,6 @@ async def build_live_retune(host: Any) -> LiveRetuneSupervisor: f"{None if retune is None else retune.executor!r} is replay-only" ) connector = host.environment.controller - from dynamo.planner.retune.live.kube import KubernetesRetuneKube - kube = KubernetesRetuneKube.from_kube_api(connector.kube_api) return await build_supervisor( config, @@ -531,46 +491,28 @@ async def build_live_retune(host: Any) -> LiveRetuneSupervisor: ) -async def build_legacy_cleanup(host: Any) -> Optional[LiveRetuneSupervisor]: - """§4.11: a decommission supervisor if a removed retune left Leases. +async def build_legacy_cleanup( + host: Any, kube: RetuneKube, *, timeout_s: float = 10.0 +) -> LiveRetuneSupervisor: + """A decommission supervisor for the Leases a removed retune left. - ``None`` when there is nothing to clean up or the planner cannot read - Leases or the DGD (403/404: logged at info, never fatal). Run it like the - live supervisor (``start``/``stop``); it rolls back what the old retune - deployed, then deletes the DGD's retune Requests and both Leases and - stops itself (``decommissioned``). + ``NativePlannerBase`` calls it once it found a Lease of the DGD. Run it + like the live supervisor (``start``/``stop``): it rolls back what the + old retune deployed, then deletes the DGD's retune Requests and both + Leases and stops itself (``decommissioned``). Reading the DGD may raise + ``KubeError``. """ - config = host.config - if config.retune is not None: - return None - try: - connector = host.environment.controller - kube_api = connector.kube_api - dgd_name = connector.parent_dgd_name - except AttributeError: - return None - from dynamo.planner.retune.live.kube import KubernetesRetuneKube - - kube = KubernetesRetuneKube.from_kube_api(kube_api) - try: - leftovers = [ - await asyncio.to_thread(kube.get_lease, name) - for name in (leadership_lease_name(dgd_name), state_lease_name(dgd_name)) - ] - if not any(lease is not None for lease in leftovers): - return None - dgd_obj = await asyncio.to_thread(kube.get_dgd, dgd_name) - except KubeError as error: - logger.info("retune legacy clean-up skipped: %s", error) - return None + connector = host.environment.controller + dgd_name = connector.parent_dgd_name + dgd_obj = await asyncio.to_thread(kube.get_dgd, dgd_name, timeout_s=timeout_s) logger.info( "retune legacy clean-up: DGD %s has Leases of a removed live retune; " "rolling back what it deployed, then deleting its Requests and Leases", dgd_name, ) return build_decommission_supervisor( - config, + host.config, kube=kube, dgd=dgd_obj, dynamo_namespace=_runtime_namespace(host), @@ -592,10 +534,8 @@ def build_decommission_supervisor( """A supervisor that only rolls back and deletes (no retune config).""" if frontend is None: - from dynamo.planner.retune.live import frontend_config as frontend + frontend = frontend_config if metrics is None: - from dynamo.planner.retune.live.metrics import RetuneMetrics - metrics = RetuneMetrics() metadata = dgd.get("metadata") or {} ref = DgdRef( diff --git a/components/src/dynamo/planner/retune/live/metrics.py b/components/src/dynamo/planner/retune/live/metrics.py index 15a559acc4a7..b586425f4efa 100644 --- a/components/src/dynamo/planner/retune/live/metrics.py +++ b/components/src/dynamo/planner/retune/live/metrics.py @@ -5,7 +5,8 @@ ``RetuneMetrics`` implements ``contracts.RetuneMetricsSink``: decision, failure and skip counters, the supervisor's state, the scores of the latest -decision, router convergence, PATCH errors and trace health (design §4.10). +decision, router convergence, PATCH errors and trace health (the table in +the live retune guide's Metrics section). Every method is cheap and never raises, so callers on the event loop, in the sweep thread or in the actuator need no guard. diff --git a/components/src/dynamo/planner/retune/live/state.py b/components/src/dynamo/planner/retune/live/state.py index 640b4fd031a1..fbc338ab8c90 100644 --- a/components/src/dynamo/planner/retune/live/state.py +++ b/components/src/dynamo/planner/retune/live/state.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Persisted live retune state and the leadership Lease (design §4.7.1-§4.7.2). +"""Persisted live retune state and the leadership Lease. Two ``coordination.k8s.io/v1`` Leases per DGD, both owned by the DGD so the garbage collector removes them with it: @@ -74,6 +74,22 @@ def state_lease_name(dgd_name: str) -> str: return f"{dgd_name}-planner-retune-state" +def find_retune_leases( + kube: RetuneKube, dgd_name: str, *, timeout_s: float +) -> tuple[str, ...]: + """Names of the DGD's live retune Leases that exist. + + A planner without retune config calls it to find what a removed live + retune left behind. Raises ``KubeError`` (403 without Lease RBAC). + """ + + return tuple( + name + for name in (leadership_lease_name(dgd_name), state_lease_name(dgd_name)) + if kube.get_lease(name, timeout_s=timeout_s) is not None + ) + + def owner_reference(dgd: DgdRef) -> dict[str, Any]: """ownerReference to the DGD; the GC deletes the Leases with it.""" @@ -172,7 +188,25 @@ def fingerprint(config: PlannerConfig, startup: Optional[FrontendStartup]) -> st class StateFormatError(ValueError): - """The state annotations do not hold a readable snapshot.""" + """The state annotations do not hold a readable snapshot. + + The snapshot is treated as corrupt: the next claim write replaces it. + """ + + +class StateVersionError(Exception): + """The snapshot has a format version this planner does not know. + + Not a ``StateFormatError``: a newer planner's state is never treated as + corrupt and overwritten. Initialization refuses to proceed until a + planner that reads the version leads, or the state Lease is deleted. + """ + + +def _finite_number(value: Any) -> bool: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + return math.isfinite(value) def _float_map(value: Any, what: str) -> dict[str, float]: @@ -180,8 +214,8 @@ def _float_map(value: Any, what: str) -> dict[str, float]: raise StateFormatError(f"{what} must be an object") out: dict[str, float] = {} for key, item in value.items(): - if isinstance(item, bool) or not isinstance(item, (int, float)): - raise StateFormatError(f"{what}.{key} must be a number") + if not _finite_number(item): + raise StateFormatError(f"{what}.{key} must be a finite number") out[str(key)] = float(item) return out @@ -194,7 +228,7 @@ def _nested_map(value: Any, what: str) -> dict[str, dict[str, float]]: @dataclass(frozen=True) class RetuneState: - """The state snapshot of design §4.7.2. + """The persisted live retune state of one DGD. ``mode is None`` is "no state": only the claim (``dgd_uid`` and ``leader_term``) has been written. ``by_wt`` is the deployed per worker @@ -248,7 +282,9 @@ def from_annotations( ) -> Optional[RetuneState]: """Parse the snapshot; ``None`` when the lease holds none. - Raises ``StateFormatError`` for a snapshot that cannot be read. + Raises ``StateVersionError`` for a snapshot of another format version + (checked first, so nothing else of it is judged) and + ``StateFormatError`` for a snapshot that cannot be read. """ raw = (annotations or {}).get(STATE_ANNOTATION) @@ -260,6 +296,13 @@ def from_annotations( raise StateFormatError(f"state annotation is not JSON: {error}") from None if not isinstance(body, Mapping): raise StateFormatError("state annotation must be a JSON object") + version = body.get("version") + if isinstance(version, bool) or not isinstance(version, int): + raise StateFormatError("state.version must be an integer") + if version != STATE_FORMAT_VERSION: + raise StateVersionError( + f"state format version {version} is not {STATE_FORMAT_VERSION}" + ) dgd_uid = body.get("dgd_uid") if not isinstance(dgd_uid, str) or not dgd_uid: raise StateFormatError("state.dgd_uid must be a non-empty string") @@ -270,10 +313,11 @@ def from_annotations( if isinstance(deploy_seq, bool) or not isinstance(deploy_seq, int): raise StateFormatError("state.deploy_seq must be an integer") last = body.get("last_window_end_ms") - if last is not None and ( - isinstance(last, bool) or not isinstance(last, (int, float)) - ): - raise StateFormatError("state.last_window_end_ms must be a number") + if last is not None and not _finite_number(last): + raise StateFormatError("state.last_window_end_ms must be a finite number") + updated = body.get("updated_ms") + if updated is not None and not _finite_number(updated): + raise StateFormatError("state.updated_ms must be a finite number") history_raw = (annotations or {}).get(HISTORY_ANNOTATION) history: list[dict[str, Any]] = [] if history_raw: @@ -296,7 +340,7 @@ def from_annotations( by_wt=_nested_map(body.get("by_wt") or {}, "state.by_wt"), initials=_nested_map(body.get("initials") or {}, "state.initials"), last_window_end_ms=None if last is None else int(last), - updated_ms=int(body.get("updated_ms") or 0), + updated_ms=0 if updated is None else int(updated), history=tuple(history[-HISTORY_LIMIT:]), ) @@ -305,7 +349,7 @@ def deployed_off( state: Optional[RetuneState], startup_by_wt: Optional[Mapping[str, Mapping[str, float]]], ) -> bool: - """Whether the routers may be off their startup values (§4.7.3 step 4). + """Whether the routers may be off their startup values. ``rollback_pending ∨ by_wt ≠ startup``, false without a state, with one refinement: only an ``active`` record can have deployed anything. An @@ -328,7 +372,7 @@ def deployed_off( return not maps_equal(state.by_wt, startup_by_wt) -# -------------------------------------------------------- write rules (§4.7.2) +# ------------------------------------------------------------ write rules def claim(state: Optional[RetuneState], *, dgd_uid: str, term: str) -> RetuneState: @@ -529,6 +573,11 @@ def dgd(self) -> DgdRef: return self._dgd def read(self) -> StateSnapshot: + """Read the snapshot; an unreadable one comes back ``corrupt``. + + A snapshot of another format version raises ``StateVersionError``. + """ + lease = self._kube.get_lease(self.name, timeout_s=self._timeout_s) if lease is None: return StateSnapshot(state=None, resource_version=None, lease=None) @@ -606,7 +655,7 @@ def _new_body(self, annotations: Mapping[str, str]) -> dict[str, Any]: class LeaseLeadership: - """Leadership over ``-planner-retune`` (design §4.7.1). + """Leadership over ``-planner-retune``. ``step`` runs one renewal/acquisition attempt and returns what happened: diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py index 518b422cd51b..c08a1ada40d4 100644 --- a/components/src/dynamo/planner/retune/live/supervisor.py +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""``LiveRetuneSupervisor``: leadership, persisted state and actuation (design §4.7). +"""``LiveRetuneSupervisor``: leadership, persisted state and actuation. The retune controller decides exactly as in replay; the supervisor decides when it may launch and harvest, persists every decision before anything is @@ -26,19 +26,21 @@ - lease: renew or acquire ``-planner-retune`` every third of the lease duration. Becoming leader, re-acquiring after losing track, or a re-init - request enters initialization (§4.7.3) with ``term_seq + 1``; losing - leadership runs the local loss (§4.7.5). -- control: re-validate the spec premises (§4.7.7, rebuilding the controller - through ``host.replace_live_retune`` when the startup, ``spec_ok`` or P4 - change), then, as an initialized leader, the P11 canary and one actuator - round (§4.8), and it starts one reconciler round (§4.4.5). + request enters initialization (claim the state with ``term_seq + 1``, then + apply the first matching row of ``plan_initialization``); losing + leadership runs the local loss. +- control: re-read the DGD and re-validate the spec premises (rebuilding the + controller through ``host.replace_live_retune`` when the startup, + ``spec_ok`` or P4 change), then, as an initialized leader, the P11 canary + and one actuator round, and it starts one reconciler round. - reconciler: one round at a time in its own task, never awaited by the control loop, within ``reconcile_budget_s``. A round still running (a hung filesystem or slow API calls) skips the next ones; the reconciler stops before its next delete once this process is no longer the initialized leader of the round's epoch or the budget has passed. -- outbox: write each decision with the term and resourceVersion - preconditions (§4.7.4); N consecutive failures re-initialize. +- outbox: write each decision, conditional on the state's ``leader_term`` + being this process's term and on the resourceVersion read; N consecutive + failures re-initialize. Local leadership loss, a failed outbox, and a controller rebuild share one path: clear ``initialized``, stop the outbox writer and the in-flight @@ -48,8 +50,11 @@ The actuation target follows the state: ``rollback`` while ``rollback_pending`` (an advisory planner resets only routers whose initial -it latched, §4.8); ``observe`` (never write) in advisory; ``rollback`` when a -spec premise fails; otherwise ``apply`` the state's ``by_wt``. +it latched); ``observe`` (never write) in advisory; ``rollback`` when a spec +premise fails; otherwise ``apply`` the state's ``by_wt``. Like a decision, a +router's latched initial is persisted before the router is moved off it, so +a process stopped at any point leaves only routers whose initial the state +records. """ from __future__ import annotations @@ -65,9 +70,11 @@ import time from collections import deque from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence +from concurrent.futures import Future from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, Optional, Protocol, Union +from dynamo.planner.plugins.orchestrator.engine_adapter import RetuneHooks from dynamo.planner.retune.live.contracts import ( LABEL_DGD_UID, ActuationReport, @@ -85,12 +92,14 @@ RetuneMetricsSink, SweepRegistry, ) +from dynamo.planner.retune.live.dgdr import run_dgdr_sweep from dynamo.planner.retune.live.state import ( LeaseLeadership, RetuneMode, RetuneState, StateSnapshot, StateStore, + StateVersionError, append_history, claim, deploy, @@ -103,11 +112,11 @@ rollback_complete, rollback_start, ) +from dynamo.planner.retune.sweep import SweepRequest, SweepResult if TYPE_CHECKING: from dynamo.planner.config.planner_config import PlannerConfig from dynamo.planner.core.types import PlannerEffects, RetuneDecision, RetuneInputs - from dynamo.planner.retune.sweep import SweepRequest, SweepResult logger = logging.getLogger(__name__) @@ -184,7 +193,7 @@ def set_startup_maps(self, startup_maps: Mapping[str, Mapping[str, float]]) -> N # (DgdRef, startup maps per worker type) -> actuator ActuatorFactory = Callable[[DgdRef, dict[str, dict[str, float]]], ActuatorLike] -SweepRunner = Callable[..., "SweepResult"] +SweepRunner = Callable[..., SweepResult] class LiveRetuneHost(Protocol): @@ -193,15 +202,12 @@ class LiveRetuneHost(Protocol): ``replace_live_retune`` takes the planner's config lock and calls the engine adapter's ``replace_retune``. The supervisor reaches the current controller through ``host._engine.retune_controller`` and serializes its - calls on it with ``host._config_lock``. + calls on it with ``host._config_lock``; ``host._engine.retune_sweeps_done`` + tells it when the engine's sweep thread is idle. """ async def replace_live_retune( - self, - inputs: Optional[RetuneInputs], - *, - generation: int, - last_decided_k: Optional[int], + self, inputs: Optional[RetuneInputs], *, generation: int ) -> None: ... @@ -229,8 +235,8 @@ class SupervisorSettings: # stop (a blocked filesystem call cannot be interrupted; later rounds # wait for it instead of piling up threads). reconcile_budget_s: float = 120.0 - # Legacy clean-up (§4.11): roll back what an earlier retune deployed, - # then delete the Requests and both Leases. + # Legacy clean-up of a removed retune: roll back what it deployed, then + # delete the Requests and both Leases. decommission: bool = False def __post_init__(self) -> None: @@ -311,7 +317,7 @@ def window_index(window_end_ms: int, *, phase_s: float, interval_s: float) -> in return int(round((window_end_ms / 1000.0 - phase_s) / interval_s)) - 1 -# ----------------------------------------------------- initialization (§4.7.3) +# ---------------------------------------------------------- initialization InitRow = Literal[ "spec_not_ok", @@ -352,7 +358,18 @@ def plan_initialization( interval_ms: int, first_window_end_ms: int, ) -> InitPlan: - """The initialization table of design §4.7.3, first matching row wins. + """What a new term writes and restores; the first matching row wins. + + 1. ``spec_not_ok``: a spec premise fails. Start a rollback if the routers + may be off their startup (``deployed_off``), else record the startup. + 2. ``advisory_rollback``/``advisory_observe``: advisory. Roll back what + an active planner deployed, else only observe. + 3. ``active_restore``: an active, current (same mode and fingerprint), + complete deployment without a pending rollback is restored as is. + 4. ``promote``: with ``promote_last_advisory``, the last advisory + recommendation, if recent and of the same fingerprint, is deployed. + 5. ``active_reset``: otherwise deploy the startup point, with a rollback + pending when the routers may be off it. ``state`` is the snapshot after the claim write. A state without a deployment record is "no state". The first initialization of a DGD also @@ -469,7 +486,11 @@ def _promotable( now_ms: int, interval_ms: int, ) -> Optional[Mapping[str, Any]]: - """The last advisory recommendation, if §4.7.6 lets it be promoted.""" + """The last advisory recommendation, if it may be promoted. + + It must be the newest history entry, an ``advisory`` outcome of the same + fingerprint, at most one interval old, and cover every searched key. + """ if not state.history: return None @@ -633,10 +654,7 @@ def __init__( # (deploy_seq, monotonic since) of a converged rollback. self._rollback_since: Optional[tuple[int, float]] = None - # Sweeps. - self._aborts: list[threading.Event] = [] - self._aborts_lock = threading.Lock() - self._sweeps_in_flight = 0 + # Skipped launches already counted. self._skips_seen: deque[tuple[int, str]] = deque(maxlen=_SKIP_MEMORY) # Tasks and locks. @@ -750,11 +768,9 @@ def retune_inputs(self) -> Optional[RetuneInputs]: return self._evaluation.inputs - def hooks(self) -> Any: + def hooks(self) -> RetuneHooks: """``RetuneHooks`` for ``OrchestratorEngineAdapter``.""" - from dynamo.planner.plugins.orchestrator.engine_adapter import RetuneHooks - return RetuneHooks( sweep_fn=self.sweep_fn, launch_gate=self.launch_gate, @@ -769,7 +785,7 @@ def target(self) -> Optional[ActuationTarget]: if not self._initialized or state is None: return None if state.rollback_pending: - # §4.8: only an active planner resets a router it never latched + # Only an active planner resets a router it never latched # (revision > 0) to learn its initial; an advisory planner leaves # it alone. return ActuationTarget( @@ -815,14 +831,11 @@ def launch_gate(self, k: int, window: tuple[float, float, float]) -> GateResult: self._metrics.skipped(reason) logger.info("retune gate: window %d %s", k, reason) return GateResult(reason=reason) - abort = threading.Event() - with self._aborts_lock: - self._aborts = [event for event in self._aborts if not event.is_set()][-8:] - self._aborts.append(abort) + # Set by the controller's drop_pending (local loss, rebuild, stop). return GateResult( reason=None, spec_replicas=dict(self._evaluation.spec_replicas or {}), - abort=abort, + abort=threading.Event(), ) def harvest_allowed(self, generation: int) -> bool: @@ -843,35 +856,24 @@ def sweep_fn( ) -> SweepResult: """Run one DGDR sweep on the controller's sweep thread; never raises.""" - with self._aborts_lock: - self._sweeps_in_flight += 1 try: - runner = self._sweep_runner - if runner is None: - from dynamo.planner.retune.live.dgdr import run_dgdr_sweep - - runner = run_dgdr_sweep + runner = self._sweep_runner or run_dgdr_sweep if self._context_factory is None: raise RuntimeError("no DGDR sweep context factory is configured") context = self._context_factory(self) return runner(request, cancel=cancel, abort=abort, context=context) except Exception as error: # the sweep contract: never raise logger.exception("retune sweep of window %s failed", request.window_index) - from dynamo.planner.retune.sweep import SweepResult - return SweepResult( status="failed", error=f"{type(error).__name__}: {error}", failure_reason="sweep_error", ) - finally: - with self._aborts_lock: - self._sweeps_in_flight -= 1 def on_effects(self, effects: PlannerEffects) -> None: """Enqueue the tick's decisions; never raises into the tick.""" - decisions = list(getattr(effects, "retune_decisions", None) or ()) + decisions = list(effects.retune_decisions or ()) if not decisions: return if not (self.is_leader() and self._initialized): @@ -882,7 +884,7 @@ def on_effects(self, effects: PlannerEffects) -> None: ) self._rewind_now() return - router_parameters = getattr(effects, "router_parameters", None) + router_parameters = effects.router_parameters for decision in decisions: self._outbox.append( OutboxEntry( @@ -914,7 +916,7 @@ async def start(self, host: Any, *, background: bool = True) -> None: self._host = host controller = self._controller() if controller is not None: - self._generation = int(getattr(controller, "generation", self._generation)) + self._generation = int(controller.generation) self._controller_ok = controller is not None self._background = background self._publish() @@ -926,7 +928,14 @@ async def start(self, host: Any, *, background: bool = True) -> None: ] async def stop(self) -> None: - """Abort sweeps, wait for them briefly and release the Lease (§4.7.8).""" + """Abort the sweep, wait for it briefly and release the Lease. + + Runs before the engine shuts down. The local loss drops the + controller's sweep (``drop_pending`` sets its abort event); then the + engine's sweep thread is waited for, at most ``stop_timeout_s``, so + no sweep of this process creates a Request after the next leader + took over. + """ if self._stopped: return @@ -942,15 +951,16 @@ async def stop(self) -> None: await self._local_loss_locked("shutdown") except Exception: logger.exception("retune supervisor: shutdown clean-up failed") - self._abort_all() - deadline = self._monotonic() + self._settings.stop_timeout_s - while self._sweeps_in_flight > 0 and self._monotonic() < deadline: - await self._sleep(0.1) - if self._sweeps_in_flight > 0: - logger.warning( - "retune supervisor: %d sweep(s) still running at shutdown", - self._sweeps_in_flight, - ) + done = self._sweeps_done() + if done is not None: + try: + await asyncio.wait_for( + asyncio.wrap_future(done), timeout=self._settings.stop_timeout_s + ) + except asyncio.TimeoutError: + logger.warning( + "retune supervisor: a sweep is still running at shutdown" + ) try: await self._io(self._leadership.release) except Exception: @@ -1045,7 +1055,7 @@ async def outbox_round(self) -> None: return async def revalidate(self) -> None: - """§4.7.7: re-read the DGD and rebuild on a spec-class change.""" + """Re-read the DGD and rebuild on a spec-class change.""" try: dgd_obj = await self._io(self._kube.get_dgd, self._dgd.name) @@ -1105,9 +1115,7 @@ async def rebuild(self, evaluation: SpecEvaluation, reason: str) -> None: if host is not None: try: await host.replace_live_retune( - evaluation.inputs, - generation=self._generation, - last_decided_k=self._last_decided_k(), + evaluation.inputs, generation=self._generation ) self._controller_ok = evaluation.inputs is not None except Exception: @@ -1125,7 +1133,12 @@ async def rebuild(self, evaluation: SpecEvaluation, reason: str) -> None: # ------------------------------------------------- init and local loss async def _initialize_locked(self) -> bool: - """§4.7.3 under ``_init_lock``; ``True`` once initialized.""" + """Claim a new term and initialize under ``_init_lock``. + + Confirms leadership, claims the state (only ``leader_term``), writes + what ``plan_initialization`` decides, restores the controller and + rewinds it to the last persisted window. ``True`` once initialized. + """ await self._local_loss_locked("initialize") # Confirm on the API server right before the claim: a holder change @@ -1180,7 +1193,7 @@ async def _initialize_locked(self) -> bool: promote=settings.promote_last_advisory, now_ms=int(round(now_s * 1000.0)), interval_ms=int(round(settings.interval_s * 1000.0)), - # §4.7.2 first enable: skip the due window and the window in + # First enable: skip the due window and the window in # progress, whose trace may have begun mid-way. first_window_end_ms=current_window_end_ms( now_s, phase_s=settings.phase_s, interval_s=settings.interval_s @@ -1229,7 +1242,12 @@ async def _initialize_locked(self) -> bool: return True async def _read_own_snapshot(self) -> Optional[StateSnapshot]: - """Read the state; drop a snapshot of another DGD uid (§4.7.3 step 2).""" + """Read the state; drop a snapshot of another DGD uid. + + A recreated DGD of the same name finds the old uid's state: it is + deleted with that uid's Requests. A snapshot of an unknown format + version is never overwritten: the read fails (``None``). + """ try: snapshot = await self._io(self._store.read) @@ -1246,6 +1264,16 @@ async def _read_own_snapshot(self) -> Optional[StateSnapshot]: # Replaced by the claim (conditional on its resourceVersion). snapshot = dataclasses.replace(snapshot, state=None) return snapshot + except StateVersionError as error: + logger.error( + "retune init: the state Lease %s was written by another planner " + "version (%s); not initializing until a planner that reads it " + "leads or the Lease is deleted", + self._store.name, + error, + ) + self._metrics.failure("state_version") + return None except (KubeError, asyncio.TimeoutError) as error: logger.warning("retune init: reading the state failed: %s", error) return None @@ -1264,7 +1292,12 @@ async def _delete_foreign_requests(self, dgd_uid: str) -> None: self.registry.enqueue_delete([name for name in names if name]) async def _local_loss_locked(self, reason: str) -> None: - """§4.7.5 under ``_init_lock``.""" + """Local leadership loss under ``_init_lock``. + + Clears ``initialized``, stops the outbox writer and the in-flight + actuator/reconciler round, drops the queued decisions and the sweep, + and rewinds the controller to the last persisted window. + """ was = self._initialized or bool(self._outbox) or self._term is not None self._initialized = False @@ -1283,7 +1316,6 @@ async def _local_loss_locked(self, reason: str) -> None: dropped = len(self._outbox) self._outbox.clear() self._write_failures = 0 - self._abort_all() controller = self._controller() if controller is not None: async with self._host_lock(): @@ -1369,7 +1401,7 @@ async def _conditional_write(self, mutate: Mutation, what: str) -> WriteOutcome: async def _reread(self) -> Optional[StateSnapshot]: try: snapshot = await self._io(self._store.read) - except (KubeError, asyncio.TimeoutError) as error: + except (KubeError, asyncio.TimeoutError, StateVersionError) as error: logger.warning("retune: re-reading the state failed: %s", error) return None if snapshot.state is not None and snapshot.state.dgd_uid != self._dgd.uid: @@ -1419,10 +1451,9 @@ def mutate(state: RetuneState) -> Union[RetuneState, None, object]: def _count_decision(self, decision: RetuneDecision) -> None: self._metrics.decision(decision.outcome) - failure_reason = getattr(decision, "failure_reason", None) if decision.outcome in ("failed", "timeout"): - self._metrics.failure(failure_reason or decision.outcome) - details = getattr(decision, "details", None) or {} + self._metrics.failure(decision.failure_reason or decision.outcome) + details = decision.details # The DGDR executor reports the window cut under details["window"]. window = details.get("window") dropped = ( @@ -1448,7 +1479,7 @@ def _history_entry(self, entry: OutboxEntry) -> dict[str, Any]: "decision_id": decision.decision_id, "window_end_ms": int(round(decision.window_end_s * 1000.0)), "outcome": decision.outcome, - "failure_reason": getattr(decision, "failure_reason", None), + "failure_reason": decision.failure_reason, "reason": decision.reason, "fingerprint": entry.fingerprint, "baseline": _finite(decision.baseline_score), @@ -1460,7 +1491,7 @@ def _history_entry(self, entry: OutboxEntry) -> dict[str, Any]: item["by_wt"] = entry.router_parameters elif decision.outcome == "advisory": item["by_wt"] = self._resolve_by_wt(flat) - details = getattr(decision, "details", None) or {} + details = decision.details image = { key: details[key] for key in ("runtime_image", "runtime_version", "runtime_version_override") @@ -1486,7 +1517,7 @@ async def _leader_round(self) -> None: epoch = self._epoch if self._settings.decommission: await self._actuate() - await self._finish_decommission() + await self._finish_decommission(epoch) return await self._check_canary() await self._actuate() @@ -1496,10 +1527,42 @@ async def _leader_round(self) -> None: async def _actuate(self) -> None: target = self.target() - state = self.state - if target is None or state is None: + if target is None: return epoch = self._epoch + report = await self._reconcile_routers(target) + if report is None: + return + outcome = await self._persist_initials(report) + if ( + outcome == "ok" + and target.mode == "apply" + and report.new_initials + and not report.converged + and epoch == self._epoch + ): + # The actuator moves a router off its initial only once the state + # records that initial: a process stopped between the write and + # the state write would leave a router whose initial nobody + # knows, which an advisory or clean-up rollback never resets. + # The initials learned this round are persisted now, so the + # writes held for them go out in this round, not the next. + report = await self._reconcile_routers(target) + if report is None: + return + await self._persist_initials(report) + await self._track_rollback(target, report) + + async def _reconcile_routers( + self, target: ActuationTarget + ) -> Optional[ActuationReport]: + """One actuator pass against the persisted initials; ``None`` when + there is no state or a local loss overtook it.""" + + state = self.state + if state is None: + return None + epoch = self._epoch startup = self._evaluation.startup report = await self._actuator.reconcile( target, @@ -1507,7 +1570,7 @@ async def _actuate(self) -> None: expected_frontends=max(1, startup.replicas if startup is not None else 1), ) if epoch != self._epoch: - return + return None self._last_report = report if startup is not None and report.parity_ok is not None: if report.parity_ok != self._p14_ok: @@ -1515,16 +1578,20 @@ async def _actuate(self) -> None: "retune: P14 parity %s -> %s", self._p14_ok, report.parity_ok ) self._p14_ok = report.parity_ok - await self._persist_initials(report) - await self._track_rollback(target, report) + return report + + async def _persist_initials( + self, report: ActuationReport + ) -> Optional[WriteOutcome]: + """Merge the newly latched initials into the state and drop those of + routers that are gone; ``None`` when nothing changes.""" - async def _persist_initials(self, report: ActuationReport) -> None: state = self.state if state is None: - return + return None merged = merge_initials(state, report.new_initials, keep=report.seen_router_ids) if maps_equal(merged.initials, state.initials): - return + return None latched = dict(report.new_initials) keep = frozenset(report.seen_router_ids) @@ -1535,6 +1602,7 @@ def mutate(current: RetuneState) -> Union[RetuneState, None, object]: outcome = await self._write_state(mutate, "initials") if outcome == "lost_term": self._request_reinit("the state's leader_term is not ours") + return outcome async def _track_rollback( self, target: ActuationTarget, report: ActuationReport @@ -1599,7 +1667,7 @@ async def _check_canary(self) -> None: self._canary_at = now def _reconcile_allowed(self, epoch: int) -> bool: - """§4.4.5: only the initialized leader of ``epoch`` cleans up. + """Only the initialized leader of ``epoch`` deletes anything. Called from the reconciler's thread too (``is_leader`` is thread-safe, the rest are single reads). @@ -1697,12 +1765,19 @@ def _deadline_for(self, window_end_ms: int) -> float: settings = self._settings return window_end_ms / 1000.0 + settings.grace_s + settings.timeout_s - async def _finish_decommission(self) -> None: - """§4.11: once nothing is left to roll back, delete Requests and Leases.""" + async def _finish_decommission(self, epoch: int) -> None: + """Once nothing is left to roll back, delete Requests and Leases. + + Only as the initialized leader of ``epoch``, checked again before + every delete: a lost term (a failed state write of this round) or + lost leadership stops it, so it never deletes under another leader. + """ state = self.state if self.decommissioned or state is None or state.rollback_pending: return + if not self._reconcile_allowed(epoch): + return dgd = self._dgd try: requests = await self._io( @@ -1710,12 +1785,19 @@ async def _finish_decommission(self) -> None: ) for request in requests: name = (request.get("metadata") or {}).get("name") - if name: - await self._io(self._kube.delete_request, name) + if not name: + continue + if not self._reconcile_allowed(epoch): + return + await self._io(self._kube.delete_request, name) + if not self._reconcile_allowed(epoch): + return await self._io(self._store.delete) except (KubeError, asyncio.TimeoutError) as error: logger.warning("retune clean-up: deleting Requests/state failed: %s", error) return + if not self._reconcile_allowed(epoch): + return try: await self._io(self._kube.delete_lease, self._leadership.name) except (KubeError, asyncio.TimeoutError) as error: @@ -1800,9 +1882,7 @@ def _make_resolver( def _router_keys(self) -> tuple[str, ...]: controller = self._controller() - keys = ( - getattr(controller, "router_keys", None) if controller is not None else None - ) + keys = controller.router_keys if controller is not None else None return tuple(keys) if keys else self._settings.router_keys def _startup_by_wt(self) -> Optional[dict[str, dict[str, float]]]: @@ -1870,18 +1950,18 @@ def _rewind_now(self) -> None: if controller is not None: self._rewind(controller) - def _abort_all(self) -> None: - with self._aborts_lock: - aborts, self._aborts = self._aborts, [] - for event in aborts: - event.set() + def _engine(self) -> Any: + """The host's engine adapter; ``None`` before ``start`` or without one.""" + + return getattr(self._host, "_engine", None) def _controller(self) -> Any: - host = self._host - if host is None: - return None - engine = getattr(host, "_engine", None) - return getattr(engine, "retune_controller", None) + engine = self._engine() + return None if engine is None else engine.retune_controller + + def _sweeps_done(self) -> Optional[Future[None]]: + engine = self._engine() + return None if engine is None else engine.retune_sweeps_done() @contextlib.asynccontextmanager async def _host_lock(self) -> AsyncIterator[None]: diff --git a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py index 8d91f87a4945..269a14f4703f 100644 --- a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py +++ b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py @@ -591,13 +591,10 @@ def __init__(self, engine: OrchestratorEngineAdapter, clock: SimClock) -> None: self._config_lock = asyncio.Lock() self._clock = clock - async def replace_live_retune(self, inputs, *, generation, last_decided_k): + async def replace_live_retune(self, inputs, *, generation): async with self._config_lock: self._engine.replace_retune( - inputs, - generation=generation, - now_s=self._clock.time(), - last_decided_k=last_decided_k, + inputs, generation=generation, now_s=self._clock.time() ) @@ -704,16 +701,17 @@ async def tick(self) -> None: # Live, the sweep runs on the controller's thread while ticks go on; # finishing it here keeps the interleaving deterministic. controller = self.engine.retune_controller - pending = getattr(controller, "_pending", None) + pending = None if controller is None else controller._pending if pending is not None and not pending.future.done(): await asyncio.wait_for(asyncio.wrap_future(pending.future), timeout=60) async def stop(self) -> None: - """SIGTERM: ``_shutdown_runtime`` stops the engine, then the supervisor.""" + """SIGTERM: ``_shutdown_runtime`` stops the supervisor (it aborts the + sweep and waits for the sweep thread), then the engine.""" self.world.planners.remove(self) - await self.engine.shutdown() await self.supervisor.stop() + await self.engine.shutdown() def crash(self) -> None: """SIGKILL: nothing of this process runs again; its Lease stays.""" diff --git a/components/src/dynamo/planner/tests/retune_stubs.py b/components/src/dynamo/planner/tests/retune_stubs.py index c81dc5afdd7e..a225c96e3b5e 100644 --- a/components/src/dynamo/planner/tests/retune_stubs.py +++ b/components/src/dynamo/planner/tests/retune_stubs.py @@ -13,6 +13,7 @@ import sys import types +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from typing import Any @@ -59,8 +60,8 @@ def __init__(self, config, inputs, capabilities, clock, scale_interval_s, **hook StubController.construct_error = None raise error self.args = (config, inputs, capabilities, clock, scale_interval_s) - # Live hooks (sweep_fn, launch_gate, harvest_allowed, generation); - # empty when the host built the controller the replay way. + # Live hooks (sweep_fn, launch_gate, harvest_allowed, generation, + # executor); empty when the host built the controller the replay way. self.hooks = hooks self.config = config self.started: list[float] = [] @@ -72,9 +73,7 @@ def __init__(self, config, inputs, capabilities, clock, scale_interval_s, **hook self.results: list[StubTickResult] = [] self.closed = 0 self.close_error: Exception | None = None - self.dropped = 0 - self.rewinds: list[int] = [] - # Ordered lifecycle calls: "start", "rewind", "drop_pending", "close". + # Ordered lifecycle calls: "start", "drop_pending", "close". self.calls: list[str] = [] StubController.instances.append(self) @@ -88,17 +87,8 @@ def start(self, start_s: float, phase_s: float | None = None) -> None: self.calls.append("start") def drop_pending(self) -> None: - self.dropped += 1 self.calls.append("drop_pending") - def rewind(self, last_decided_k: int) -> None: - self.rewinds.append(last_decided_k) - self.calls.append("rewind") - - def running_future(self): - """No sweep thread: nothing for a successor to queue behind.""" - return None - async def on_tick(self, now_s, worker_counts): self.ticks.append((now_s, worker_counts)) self.seen_planner_values.append( @@ -116,11 +106,22 @@ def close(self) -> None: raise self.close_error +class StubSweepExecutor(ThreadPoolExecutor): + """The live sweep thread a host hands the stub (which never sweeps).""" + + def __init__(self) -> None: + super().__init__(max_workers=1) + + def busy(self) -> bool: + return False + + def install_stub_controller(monkeypatch) -> type[StubController]: """Install ``StubController`` as ``dynamo.planner.retune.controller``.""" module = types.ModuleType(CONTROLLER_MODULE) module.RetuneController = StubController # type: ignore[attr-defined] + module.SweepExecutor = StubSweepExecutor # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, CONTROLLER_MODULE, module) StubController.instances = [] StubController.construct_error = None diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py b/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py index 5f9311ae9c15..c4904fa75f12 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py @@ -3,11 +3,11 @@ """Unit tests for the live retune router actuator and its HTTP client. -``FakePolicyClient`` implements the ``/router_policy`` semantics of -``lib/kv-router/src/plugins/worker_selection/runtime.rs`` and -``lib/llm/src/http/service/router_policy.rs``: CAS on ``expected_revision``, -per-key PATCH merge, ``reset`` restores the startup values, and every -successful write bumps the revision. +``FakePolicyClient`` only records the calls and answers them: a successful +write sets the router to what the actuator asked for (the PATCHed keys, or +its initial after a reset) and bumps the revision. Error statuses are +injected by the tests; the server's own rules (``router_policy.rs``) are not +modelled here. """ from __future__ import annotations @@ -33,7 +33,6 @@ from dynamo.planner.retune.live.contracts import ( POD_LABEL_COMPONENT_TYPE, POD_LABEL_DGD_NAME, - POLICY_PARAMETER_KEYS, ActuationTarget, DgdRef, KubeError, @@ -62,13 +61,6 @@ "router_temperature": 0.0, "shared_cache_multiplier": 0.0, } -# overlap_score_credit = 0 also zeroes the inherited cache weights. -ZERO_CREDIT = { - **STARTUP, - "overlap_score_credit": 0.0, - "disk_cache_hit_weight": 0.0, - "host_cache_hit_weight": 0.0, -} CHOSEN = { **STARTUP, "overlap_score_credit": 0.5, @@ -82,12 +74,6 @@ "disk_cache_hit_weight": 0.0, "host_cache_hit_weight": 0.0, } -_CACHE_KEYS = ( - "overlap_score_credit", - "host_cache_hit_weight", - "disk_cache_hit_weight", - "shared_cache_multiplier", -) # ------------------------------------------------------------------ fakes @@ -102,7 +88,6 @@ class FakeRouter: revision: int = 0 current: Optional[dict[str, float]] = None runtime_config: bool = True # False: a custom policy, configuration null - plain_decode: bool = False # rejects nonzero cache inputs with 422 def __post_init__(self) -> None: if self.current is None: @@ -131,14 +116,14 @@ def add(self, router: FakeRouter) -> FakeRouter: class FakePolicyClient: - """In-memory frontends keyed by pod name.""" + """In-memory frontends keyed by pod name; records every call.""" def __init__(self) -> None: self.frontends: dict[str, FakeFrontend] = {} self.calls: list[tuple[Any, ...]] = [] - # Statuses returned by the next writes (any router) before applying. + # Statuses returned by the next writes (any router) instead of applying. self.write_failures: list[int] = [] - # Called before a write is applied: (op, pod name, router_id). + # Called before a write is answered: (op, pod name, router_id). self.before_write: Optional[Callable[[str, str, str], None]] = None self.raise_on_get: Optional[BaseException] = None self.active = 0 @@ -182,20 +167,19 @@ async def patch( self.calls.append( ("patch", pod.name, router_id, expected_revision, dict(parameters)) ) - return self._write("patch", pod, router_id, expected_revision, parameters) + return self._answer("patch", pod, router_id, parameters) async def reset( self, pod: PodInfo, router_id: str, expected_revision: int ) -> PolicyCallResult: self.calls.append(("reset", pod.name, router_id, expected_revision)) - return self._write("reset", pod, router_id, expected_revision, None) + return self._answer("reset", pod, router_id, None) - def _write( + def _answer( self, op: str, pod: PodInfo, router_id: str, - expected_revision: int, parameters: Optional[Mapping[str, float]], ) -> PolicyCallResult: if self.before_write is not None: @@ -205,23 +189,10 @@ def _write( router = self.frontend(pod.name).routers.get(router_id) if router is None: return PolicyCallResult(status=404, error="router instance not found") - if not router.runtime_config: - return PolicyCallResult(status=422, error="unsupported") - if expected_revision != router.revision: - return PolicyCallResult(status=409, error="policy revision conflict") if parameters is None: - updated = dict(router.initial) + router.current = dict(router.initial) else: - if not parameters: - return PolicyCallResult(status=400, error="empty") - updated = dict(router.current or {}) - for key, value in parameters.items(): - if key not in router.initial: - return PolicyCallResult(status=422, error=f"unknown {key}") - updated[key] = router.initial[key] if value is None else float(value) - if router.plain_decode and any(updated[key] != 0.0 for key in _CACHE_KEYS): - return PolicyCallResult(status=422, error="decode has no cache inputs") - router.current = updated + router.current = {**(router.current or {}), **parameters} router.revision += 1 return PolicyCallResult(status=200, router=router.info()) @@ -306,10 +277,6 @@ def apply(desired: Mapping[str, float], wt: str = "aggregated") -> ActuationTarg ROLLBACK = ActuationTarget(mode="rollback", include_not_ready=True) -def diff_keys(left: Mapping[str, float], right: Mapping[str, float]) -> set[str]: - return {key for key in POLICY_PARAMETER_KEYS if left[key] != right[key]} - - # ---------------------------------------------------------------- latching @@ -338,41 +305,40 @@ async def test_observe_latches_initials_and_never_writes() -> None: assert report.converged -async def test_new_initials_reported_until_persisted() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - - await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - # The state write has not landed: the router is at revision 1 now but - # its initial stays known locally (no learning reset) and is reported. - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - assert [call[0] for call in client.writes()] == ["patch"] - assert report.new_initials == {"r1": STARTUP} - assert report.converged +# ------------------------------------------------------------------- apply -async def test_observe_with_unknown_initial_writes_nothing() -> None: +async def test_apply_holds_a_router_until_its_initial_is_persisted() -> None: + # A process stopped between a PATCH and the state write of the router's + # initial would leave a router no rollback can restore: an advisory or + # clean-up planner never resets a router whose initial it does not know. actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add( - FakeRouter("r1", STARTUP, revision=4, current=dict(CHOSEN)) - ) + router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - for _ in range(3): - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + for _ in range(2): # the state write has not landed yet + report = await actuator.reconcile( + apply(CHOSEN), initials={}, expected_frontends=1 + ) + assert client.writes() == [] + assert report.new_initials == {"r1": STARTUP} assert not report.converged - assert report.parity_ok is None - assert report.new_initials == {} - assert client.writes() == [] - -# ------------------------------------------------------------------- apply + report = await actuator.reconcile( + apply(CHOSEN), initials={"r1": STARTUP}, expected_frontends=1 + ) + assert [call[0] for call in client.writes()] == ["patch"] + assert router.current == CHOSEN + assert report.converged and report.new_initials == {} async def test_apply_patches_only_differing_keys_with_explicit_values() -> None: actuator, _, client, _ = make_actuator([pod("fe-0")]) router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + initials = {"r1": dict(STARTUP)} - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) assert client.writes() == [ ( @@ -394,75 +360,46 @@ async def test_apply_patches_only_differing_keys_with_explicit_values() -> None: # Converged: the next rounds write nothing. for _ in range(2): report = await actuator.reconcile( - apply(CHOSEN), initials={"r1": STARTUP}, expected_frontends=1 + apply(CHOSEN), initials=initials, expected_frontends=1 ) assert report.converged assert len(client.writes()) == 1 -async def test_zero_credit_round_trip_restores_cache_weights() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - initials: dict[str, dict[str, float]] = {} - - report = await actuator.reconcile( - apply(ZERO_CREDIT), initials=initials, expected_frontends=1 - ) - initials.update(report.new_initials) - assert set(client.writes()[-1][4]) == diff_keys(STARTUP, ZERO_CREDIT) - assert router.current == ZERO_CREDIT - - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - # host/disk come back with the credit, as explicit values. - assert client.writes()[-1][4] == { - key: CHOSEN[key] for key in diff_keys(ZERO_CREDIT, CHOSEN) - } - assert router.current == CHOSEN - assert report.converged - - await actuator.reconcile( - apply(ZERO_CREDIT), initials=initials, expected_frontends=1 - ) - report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) - assert client.writes()[-1] == ("reset", "fe-0", "r1", 3) - assert router.current == STARTUP - assert router.current["host_cache_hit_weight"] == 0.75 - assert router.current["disk_cache_hit_weight"] == 0.25 - assert report.converged - - -async def test_apply_learns_unknown_initial_then_patches_in_one_round() -> None: +async def test_apply_learns_an_unknown_initial_with_a_reset_first() -> None: actuator, _, client, _ = make_actuator([pod("fe-0")]) tampered = {**STARTUP, "router_temperature": 0.9} router = client.frontend("fe-0").add( FakeRouter("r1", STARTUP, revision=2, current=tampered) ) + # A reset restores the router's initial and reports it; the PATCH waits + # for the state to record it. report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + assert client.writes() == [("reset", "fe-0", "r1", 2)] + assert router.current == STARTUP + assert report.new_initials == {"r1": STARTUP} + assert report.parity_ok is True + assert not report.converged + report = await actuator.reconcile( + apply(CHOSEN), initials=report.new_initials, expected_frontends=1 + ) assert [call[0] for call in client.writes()] == ["reset", "patch"] - assert client.writes()[0] == ("reset", "fe-0", "r1", 2) - assert client.writes()[1][3] == 3 assert router.current == CHOSEN - assert report.new_initials == {"r1": STARTUP} - assert report.parity_ok is True assert report.converged - await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - assert len(client.writes()) == 2 - async def test_parity_mismatch_blocks_apply_and_never_loops() -> None: actuator, _, client, _ = make_actuator([pod("fe-0"), pod("fe-1", ip="10.0.0.2")]) odd = {**STARTUP, "overlap_score_credit_decay": 0.5} client.frontend("fe-0").add(FakeRouter("good", STARTUP)) client.frontend("fe-1").add(FakeRouter("odd", odd)) + initials = {"good": dict(STARTUP), "odd": dict(odd)} for _ in range(3): report = await actuator.reconcile( - apply(CHOSEN), initials={}, expected_frontends=2 + apply(CHOSEN), initials=initials, expected_frontends=2 ) assert report.parity_ok is False assert len(report.parity_mismatches) == 1 @@ -476,20 +413,77 @@ async def test_parity_mismatch_blocks_apply_and_never_loops() -> None: assert client.frontend("fe-1").routers["odd"].current == odd -async def test_rollback_with_parity_mismatch_resets_to_each_routers_initial() -> None: +async def test_frontend_restart_fills_in_without_reset() -> None: + actuator, kube, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + initials = {"r1": dict(STARTUP)} + await actuator.reconcile(apply(CHOSEN), initials=initials, expected_frontends=1) + + # The frontend restarts: Ready is 0 for a while. + kube.pods = [pod("fe-0", ready=False)] + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) + assert not report.converged + + # A new pod with a fresh router at its startup values: latched, then + # PATCHed once the state records it; never reset. + kube.pods = [pod("fe-0b", ip="10.0.0.9")] + client.frontend("fe-0b").add(FakeRouter("r2", STARTUP)) + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) + assert report.new_initials == {"r2": STARTUP} + assert report.seen_router_ids == frozenset({"r2"}) + report = await actuator.reconcile( + apply(CHOSEN), initials=report.new_initials, expected_frontends=1 + ) + assert [call[0] for call in client.writes()] == ["patch", "patch"] + assert client.frontend("fe-0b").routers["r2"].current == CHOSEN + assert report.converged + + +async def test_disagg_needs_both_worker_types() -> None: + maps = {"prefill": dict(STARTUP), "decode": dict(DECODE_STARTUP)} + actuator, _, client, _ = make_actuator([pod("fe-0")], startup_maps=maps) + frontend = client.frontend("fe-0") + prefill = frontend.add(FakeRouter("p", STARTUP, worker_type="prefill")) + target = ActuationTarget( + mode="apply", + desired_by_wt={ + "prefill": dict(CHOSEN), + "decode": {**DECODE_STARTUP, "router_temperature": 0.2}, + }, + ) + initials = {"p": dict(STARTUP), "d": dict(DECODE_STARTUP)} + + report = await actuator.reconcile(target, initials=initials, expected_frontends=1) + assert not report.converged # no decode router yet + assert prefill.current == CHOSEN + + decode = frontend.add(FakeRouter("d", DECODE_STARTUP, worker_type="decode")) + report = await actuator.reconcile(target, initials=initials, expected_frontends=1) + assert decode.current == {**DECODE_STARTUP, "router_temperature": 0.2} + assert report.converged + assert report.parity_ok is True + + +async def test_only_the_dgd_namespace_and_planner_worker_types_are_managed() -> None: actuator, _, client, _ = make_actuator([pod("fe-0")]) - odd = {**STARTUP, "overlap_score_credit_decay": 0.5} - router = client.frontend("fe-0").add(FakeRouter("odd", odd)) - await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - router.current = {**odd, "router_temperature": 1.0} - router.revision = 1 + frontend = client.frontend("fe-0") + ours = frontend.add(FakeRouter("ours", STARTUP)) + foreign = frontend.add(FakeRouter("foreign", STARTUP, namespace="other-ns")) + encode = frontend.add(FakeRouter("encode", STARTUP, worker_type="encode")) - report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) + report = await actuator.reconcile( + apply(CHOSEN), initials={"ours": dict(STARTUP)}, expected_frontends=1 + ) - assert client.writes() == [("reset", "fe-0", "odd", 1)] - assert router.current == odd - assert report.converged # rollback converges to the latched initial - assert report.parity_ok is False + assert {call[2] for call in client.writes()} == {"ours"} + assert ours.current == CHOSEN + assert foreign.current == STARTUP and encode.current == STARTUP + assert report.seen_router_ids == frozenset({"ours"}) + assert report.converged # ---------------------------------------------------------------- rollback @@ -507,7 +501,6 @@ async def test_rollback_resets_only_deviating_routers_and_never_by_revision() -> assert client.writes() == [("reset", "fe-0", "b", 1)] assert deviating.current == STARTUP - assert deviating.revision == 2 assert at_initial.revision == 3 assert report.converged @@ -539,6 +532,54 @@ async def test_rollback_learns_unknown_initial_with_a_single_reset() -> None: assert len(client.writes()) == 1 +async def test_rollback_with_parity_mismatch_resets_to_each_routers_initial() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + odd = {**STARTUP, "overlap_score_credit_decay": 0.5} + router = client.frontend("fe-0").add(FakeRouter("odd", odd)) + await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + router.current = {**odd, "router_temperature": 1.0} + router.revision = 1 + + report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) + + assert client.writes() == [("reset", "fe-0", "odd", 1)] + assert router.current == odd + assert report.converged # rollback converges to the latched initial + assert report.parity_ok is False + + +async def test_rollback_without_learning_leaves_unknown_routers_alone() -> None: + actuator, _, client, _ = make_actuator([pod("fe-0")]) + frontend = client.frontend("fe-0") + # Changed by someone else before any planner latched it. + foreign = frontend.add( + FakeRouter("foreign", STARTUP, revision=4, current=dict(CHOSEN)) + ) + ours = frontend.add(FakeRouter("ours", STARTUP, revision=1, current=dict(CHOSEN))) + target = ActuationTarget( + mode="rollback", include_not_ready=True, learn_unknown=False + ) + + for _ in range(2): + report = await actuator.reconcile( + target, initials={"ours": STARTUP}, expected_frontends=1 + ) + assert report.converged + assert client.writes() == [("reset", "fe-0", "ours", 1)] + assert ours.current == STARTUP + assert foreign.current == CHOSEN and foreign.revision == 4 + assert report.new_initials == {} + assert report.parity_ok is None # the foreign router's initial is unknown + + # The default (active) target learns it with one reset. + report = await actuator.reconcile( + ROLLBACK, initials={"ours": STARTUP}, expected_frontends=1 + ) + assert client.writes()[-1] == ("reset", "fe-0", "foreign", 4) + assert foreign.current == STARTUP + assert report.converged + + async def test_rollback_targets_not_ready_pods_but_never_terminating_ones() -> None: pods = [ pod("ready", ip="10.0.0.1"), @@ -567,63 +608,33 @@ async def test_rollback_targets_not_ready_pods_but_never_terminating_ones() -> N assert {call[1] for call in client.calls} == {"ready"} -async def test_rollback_converges_without_an_unreachable_not_ready_pod() -> None: - actuator, _, client, _ = make_actuator( +async def test_unreachable_pods_and_convergence() -> None: + actuator, kube, client, _ = make_actuator( [pod("ready"), pod("crashing", ready=False, ip="10.0.0.2")] ) router = client.frontend("ready").add( FakeRouter("r1", STARTUP, revision=1, current=dict(CHOSEN)) ) - # CrashLoopBackOff: the GET is refused on every round. - client.frontend("crashing").get_failures = [0, 0, 0] - - for _ in range(3): + # CrashLoopBackOff: an unreachable NotReady pod never blocks convergence. + client.frontend("crashing").get_failures = [0, 0] + for _ in range(2): report = await actuator.reconcile( ROLLBACK, initials={"r1": STARTUP}, expected_frontends=1 ) assert report.converged assert report.errors == {"get:network": 1} assert router.current == STARTUP - assert client.writes() == [("reset", "ready", "r1", 1)] - -async def test_an_unreachable_ready_pod_blocks_convergence() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0"), pod("fe-1", ip="10.0.0.2")]) - client.frontend("fe-0").add(FakeRouter("r0", STARTUP)) - client.frontend("fe-1").add(FakeRouter("r1", STARTUP)) - client.frontend("fe-1").get_failures = [0] - - report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) - assert not report.converged - - report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) - assert report.converged - - -async def test_an_answering_not_ready_pod_is_still_checked() -> None: - actuator, _, client, _ = make_actuator( - [pod("ready"), pod("starting", ready=False, ip="10.0.0.2")] - ) - client.frontend("ready").add(FakeRouter("r0", STARTUP)) - stuck = client.frontend("starting").add( - FakeRouter("r1", STARTUP, revision=2, current=dict(CHOSEN)) + # An unreachable Ready pod does. + kube.pods = [pod("ready"), pod("crashing", ip="10.0.0.2")] + client.frontend("crashing").get_failures = [0] + report = await actuator.reconcile( + ROLLBACK, initials={"r1": STARTUP}, expected_frontends=1 ) - initials = {"r0": dict(STARTUP), "r1": dict(STARTUP)} - client.write_failures = [500] - - report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) - assert not report.converged # it answered, and it is not at its initial - assert stuck.current == CHOSEN - - report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) - assert report.converged - assert stuck.current == STARTUP + assert not report.converged -@pytest.mark.parametrize("phase", ["Failed", "Succeeded"]) -async def test_finished_pods_are_never_targeted_and_their_routers_are_gone( - phase: str, -) -> None: +async def test_finished_pods_are_never_targeted_and_their_routers_are_gone() -> None: actuator, kube, client, _ = make_actuator( [pod("fe-0"), pod("evicted", ip="10.0.0.2")] ) @@ -634,84 +645,37 @@ async def test_finished_pods_are_never_targeted_and_their_routers_are_gone( assert set(initials) == {"r0", "r-ev"} # Evicted: the pod object stays, with its IP, in a finished phase. - kube.pods = [pod("fe-0"), pod("evicted", ready=False, ip="10.0.0.2", phase=phase)] - client.frontend("evicted").get_failures = [0] + kube.pods = [ + pod("fe-0"), + pod("evicted", ready=False, ip="10.0.0.2", phase="Failed"), + ] client.calls.clear() report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) assert ("get", "evicted") not in client.calls assert report.converged - assert report.errors == {} assert report.seen_router_ids == frozenset({"r0"}) -async def test_rollback_without_learning_leaves_unknown_routers_alone() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - frontend = client.frontend("fe-0") - # Changed by someone else before any planner latched it. - foreign = frontend.add( - FakeRouter("foreign", STARTUP, revision=4, current=dict(CHOSEN)) - ) - ours = frontend.add(FakeRouter("ours", STARTUP, revision=1, current=dict(CHOSEN))) - target = ActuationTarget( - mode="rollback", include_not_ready=True, learn_unknown=False - ) - - for _ in range(2): - report = await actuator.reconcile( - target, initials={"ours": STARTUP}, expected_frontends=1 - ) - assert report.converged - assert client.writes() == [("reset", "fe-0", "ours", 1)] - assert ours.current == STARTUP - assert foreign.current == CHOSEN and foreign.revision == 4 - assert report.new_initials == {} - assert report.parity_ok is None # the foreign router's initial is unknown - - # The default (active) target learns it with one reset. - report = await actuator.reconcile( - ROLLBACK, initials={"ours": STARTUP}, expected_frontends=1 - ) - assert client.writes()[-1] == ("reset", "fe-0", "foreign", 4) - assert foreign.current == STARTUP - assert report.converged - - -async def test_apply_without_learning_never_resets_an_unknown_router() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add( - FakeRouter("r1", STARTUP, revision=2, current=dict(CHOSEN)) - ) - target = ActuationTarget( - mode="apply", - desired_by_wt={"aggregated": dict(ZERO_CREDIT)}, - learn_unknown=False, - ) - - report = await actuator.reconcile(target, initials={}, expected_frontends=1) - - assert client.writes() == [] - assert not report.converged - - # ----------------------------------------------------------- status codes async def test_conflict_regets_and_retries_once() -> None: actuator, _, client, metrics = make_actuator([pod("fe-0")]) router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - interfered = [] + client.write_failures = [409] def someone_else_writes(op: str, pod_name: str, router_id: str) -> None: - if not interfered: - interfered.append(True) + if client.write_failures: # the write that answers 409 router.current = {**STARTUP, "router_temperature": 0.7} router.revision += 1 client.before_write = someone_else_writes - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + report = await actuator.reconcile( + apply(CHOSEN), initials={"r1": STARTUP}, expected_frontends=1 + ) patches = [call for call in client.writes() if call[0] == "patch"] assert [call[3] for call in patches] == [0, 1] @@ -727,55 +691,18 @@ async def test_conflict_twice_gives_up_until_next_round() -> None: actuator, _, client, _ = make_actuator([pod("fe-0")]) router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) client.write_failures = [409, 409] + initials = {"r1": dict(STARTUP)} - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) assert len(client.writes()) == 2 assert not report.converged - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - assert router.current == CHOSEN - assert report.converged - - -async def test_patch_404_regets_and_patches_the_rebuilt_router() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - frontend = client.frontend("fe-0") - frontend.add(FakeRouter("old", STARTUP)) - - def rebuild(op: str, pod_name: str, router_id: str) -> None: - if router_id == "old": - del frontend.routers["old"] - frontend.add(FakeRouter("new", STARTUP)) - - client.before_write = rebuild - - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - - assert [(call[0], call[2]) for call in client.writes()] == [ - ("patch", "old"), - ("patch", "new"), - ] - assert frontend.routers["new"].current == CHOSEN - assert report.errors == {"patch:404": 1} - assert report.new_initials == {"new": STARTUP} - assert report.seen_router_ids == frozenset({"new"}) - assert report.converged - - -async def test_503_regets_then_retries() -> None: - actuator, _, client, metrics = make_actuator([pod("fe-0")]) - router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - client.write_failures = [503] - - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - - assert client.calls[:3] == [ - ("get", "fe-0"), - ("patch", "fe-0", "r1", 0, client.writes()[0][4]), - ("get", "fe-0"), - ] + report = await actuator.reconcile( + apply(CHOSEN), initials=initials, expected_frontends=1 + ) assert router.current == CHOSEN - assert metrics.patch_errors == ["503"] assert report.converged @@ -783,27 +710,37 @@ async def test_422_is_not_retried_until_the_target_changes() -> None: actuator, _, client, metrics = make_actuator([pod("fe-0")]) client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) client.write_failures = [422] + initials = {"r1": dict(STARTUP)} for _ in range(3): report = await actuator.reconcile( - apply(CHOSEN), initials={}, expected_frontends=1 + apply(CHOSEN), initials=initials, expected_frontends=1 ) assert not report.converged assert len(client.writes()) == 1 assert metrics.patch_errors == ["422"] other = {**CHOSEN, "router_temperature": 0.4} - report = await actuator.reconcile(apply(other), initials={}, expected_frontends=1) + report = await actuator.reconcile( + apply(other), initials=initials, expected_frontends=1 + ) assert len(client.writes()) == 2 assert report.converged -async def test_get_404_for_three_rounds_reports_admin_api_disabled() -> None: +async def test_get_404_for_three_consecutive_rounds_reports_admin_api_disabled() -> ( + None +): actuator, _, client, _ = make_actuator([pod("fe-0")]) frontend = client.frontend("fe-0") frontend.add(FakeRouter("r1", STARTUP)) - frontend.get_failures = [404, 404, 404] + # A 200 in between restarts the count. + frontend.get_failures = [404, 404] + for _ in range(3): + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + assert not report.admin_api_disabled + frontend.get_failures = [404, 404, 404] flags = [] for _ in range(3): report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) @@ -816,147 +753,40 @@ async def test_get_404_for_three_rounds_reports_admin_api_disabled() -> None: assert report.converged -async def test_get_404_counter_needs_consecutive_rounds() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - frontend = client.frontend("fe-0") - frontend.add(FakeRouter("r1", STARTUP)) - frontend.get_failures = [404, 404] - await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) # 200 - frontend.get_failures = [404] - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - assert not report.admin_api_disabled - - # ------------------------------------------------------------- convergence -async def test_not_converged_below_the_expected_ready_frontends() -> None: +async def test_convergence_needs_the_expected_ready_frontends_and_a_router() -> None: actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + frontend = client.frontend("fe-0") + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + assert not report.converged and report.parity_ok is None # no router + + frontend.add(FakeRouter("r1", STARTUP)) report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=2) assert not report.converged report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=0) assert report.converged # max(1, 0) -async def test_not_converged_without_any_router() -> None: +async def test_routers_present_needs_a_configured_router_not_the_target() -> None: actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0") - - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - assert not report.converged - assert report.parity_ok is None - - -async def test_routers_present_does_not_need_the_target_values() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) + frontend = client.frontend("fe-0") + frontend.add(FakeRouter("custom", STARTUP, runtime_config=False)) - # Present (P12) even though the routers do not match the apply target. - report = await actuator.reconcile( - ActuationTarget(mode="observe"), initials={}, expected_frontends=1 - ) - assert report.routers_present is True - report = await actuator.reconcile( - ActuationTarget(mode="observe"), initials={}, expected_frontends=2 - ) + report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) assert report.routers_present is False + assert not report.converged and client.writes() == [] - -async def test_routers_present_needs_a_configured_router() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("custom", STARTUP, runtime_config=False)) - + # Present (P12) even though the router does not match the apply target. + frontend.add(FakeRouter("r1", {**STARTUP, "router_temperature": 0.9})) report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) + assert report.routers_present is True + report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=2) assert report.routers_present is False -async def test_a_router_without_runtime_configuration_does_not_count() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("custom", STARTUP, runtime_config=False)) - - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - assert not report.converged - assert client.writes() == [] - - -async def test_disagg_needs_both_worker_types_and_respects_plain_decode() -> None: - maps = {"prefill": dict(STARTUP), "decode": dict(DECODE_STARTUP)} - actuator, _, client, _ = make_actuator([pod("fe-0")], startup_maps=maps) - frontend = client.frontend("fe-0") - prefill = frontend.add(FakeRouter("p", STARTUP, worker_type="prefill")) - - target = ActuationTarget( - mode="apply", - desired_by_wt={ - "prefill": dict(CHOSEN), - "decode": {**DECODE_STARTUP, "router_temperature": 0.2}, - }, - ) - report = await actuator.reconcile(target, initials={}, expected_frontends=1) - assert not report.converged # no decode router yet - assert prefill.current == CHOSEN - - decode = frontend.add( - FakeRouter("d", DECODE_STARTUP, worker_type="decode", plain_decode=True) - ) - report = await actuator.reconcile(target, initials={}, expected_frontends=1) - assert decode.current["router_temperature"] == 0.2 - assert decode.current["overlap_score_credit"] == 0.0 - assert report.converged - assert report.parity_ok is True - - -async def test_only_the_dgd_namespace_and_planner_worker_types_are_managed() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - frontend = client.frontend("fe-0") - ours = frontend.add(FakeRouter("ours", STARTUP)) - foreign = frontend.add(FakeRouter("foreign", STARTUP, namespace="other-ns")) - encode = frontend.add(FakeRouter("encode", STARTUP, worker_type="encode")) - - report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - - assert {call[2] for call in client.writes()} == {"ours"} - assert ours.current == CHOSEN - assert foreign.current == STARTUP and encode.current == STARTUP - assert report.seen_router_ids == frozenset({"ours"}) - assert report.converged - - -async def test_frontend_restart_fills_in_without_reset() -> None: - actuator, kube, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - initials: dict[str, dict[str, float]] = {} - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - initials.update(report.new_initials) - - # The frontend restarts: Ready is 0 for a while. - kube.pods = [pod("fe-0", ready=False)] - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - assert not report.converged - - # A new pod with a fresh router at its startup values: PATCH, no reset. - kube.pods = [pod("fe-0b", ip="10.0.0.9")] - client.frontend("fe-0b").add(FakeRouter("r2", STARTUP)) - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - assert [call[0] for call in client.writes()] == ["patch", "patch"] - assert client.frontend("fe-0b").routers["r2"].current == CHOSEN - assert report.converged - assert report.seen_router_ids == frozenset({"r2"}) - - -# -------------------------------------------------------- seen router ids - - async def test_seen_router_ids_drop_only_routers_that_are_provably_gone() -> None: pods = [pod("fe-0"), pod("fe-1", ip="10.0.0.2")] actuator, kube, client, _ = make_actuator(pods) @@ -980,24 +810,6 @@ async def test_seen_router_ids_drop_only_routers_that_are_provably_gone() -> Non assert report.seen_router_ids == frozenset({"a2"}) -async def test_persisted_initials_of_unknown_pods_survive_an_incomplete_view() -> None: - pods = [pod("fe-0"), pod("fe-1", ready=False, ip="10.0.0.2")] - actuator, _, client, _ = make_actuator(pods) - client.frontend("fe-0").add(FakeRouter("a", STARTUP)) - persisted = {"a": dict(STARTUP), "unknown": dict(STARTUP)} - - # apply does not query the NotReady fe-1, so "unknown" may live there. - report = await actuator.reconcile(OBSERVE, initials=persisted, expected_frontends=1) - assert report.seen_router_ids == frozenset({"a", "unknown"}) - - # rollback queries every live pod; "unknown" is nowhere: dropped. - client.frontend("fe-1") - report = await actuator.reconcile( - ROLLBACK, initials=persisted, expected_frontends=1 - ) - assert report.seen_router_ids == frozenset({"a"}) - - # --------------------------------------------------------- failure paths @@ -1061,12 +873,13 @@ async def test_invalid_apply_target_writes_nothing(target: ActuationTarget) -> N actuator, _, client, _ = make_actuator([pod("fe-0")]) client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - report = await actuator.reconcile(target, initials={}, expected_frontends=1) + report = await actuator.reconcile( + target, initials={"r1": STARTUP}, expected_frontends=1 + ) assert client.writes() == [] assert not report.converged assert report.errors == {"invalid_target": 1} - assert report.new_initials == {"r1": STARTUP} # still latched async def test_max_concurrency_bounds_parallel_gets() -> None: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py index 9f992c9842f2..4d2dbd239588 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py @@ -17,7 +17,7 @@ import os import signal import threading -from concurrent.futures import Future +from concurrent.futures import ThreadPoolExecutor from unittest.mock import patch import pytest @@ -75,18 +75,9 @@ def _hooks(**overrides) -> RetuneHooks: return RetuneHooks(**values) -def _stub_running_future(self): - """``running_future`` for the stub: ``self.running``, recorded in calls.""" - - self.calls.append("running_future") - return getattr(self, "running", None) - - @pytest.fixture def stub_controller(monkeypatch): - stub = install_stub_controller(monkeypatch) - monkeypatch.setattr(stub, "running_future", _stub_running_future, raising=False) - return stub + return install_stub_controller(monkeypatch) def _adapter(*, inputs=None, hooks=None, clock=None, **overrides): @@ -130,12 +121,13 @@ def test_live_controller_gets_the_hooks_and_the_phase(stub_controller): (controller,) = stub_controller.instances assert engine.retune_controller is controller + executor = controller.hooks.pop("executor") + assert isinstance(executor, ThreadPoolExecutor) assert controller.hooks == { "sweep_fn": hooks.sweep_fn, "launch_gate": hooks.launch_gate, "harvest_allowed": hooks.harvest_allowed, "generation": 0, - "predecessor": None, } engine.initial_tick(1_000.0) assert (controller.started, controller.phases) == ([1_000.0], [PHASE]) @@ -232,19 +224,19 @@ async def test_replace_retune_swaps_the_controller(stub_controller): (old,) = stub_controller.instances inputs = RetuneInputs(engine=retune_inputs().engine, trace_paths=("/trace/b",)) - engine.replace_retune(inputs, generation=4, now_s=500.0, last_decided_k=41) + engine.replace_retune(inputs, generation=4, now_s=500.0) old_, new = stub_controller.instances assert old_ is old - # Abort the old sweep first, close the old controller, then read the - # sweep still holding its thread (none here) for the new one to wait on. - assert old.calls == ["start", "drop_pending", "close", "running_future"] + # Abort the old sweep first, then close the old controller. + assert old.calls == ["start", "drop_pending", "close"] assert new.args[1] is inputs assert new.generation == 4 - assert new.hooks["predecessor"] is None + # Every generation submits to the adapter's one sweep thread. + assert new.hooks["executor"] is old.hooks["executor"] assert new.hooks["launch_gate"] is old.hooks["launch_gate"] - assert new.calls == ["start", "rewind"] - assert (new.started, new.phases, new.rewinds) == ([500.0], [PHASE], [41]) + assert new.calls == ["start"] + assert (new.started, new.phases) == ([500.0], [PHASE]) assert engine.retune_controller is new # The next tick runs the new controller only. @@ -264,83 +256,17 @@ async def test_replace_retune_swaps_the_controller(stub_controller): assert (old.closed, new.closed) == (1, 1) -def test_replace_retune_without_last_decided_k_does_not_rewind(stub_controller): - engine = _adapter(inputs=None) - - engine.replace_retune( - retune_inputs(), generation=1, now_s=10.0, last_decided_k=None - ) - - (controller,) = stub_controller.instances - assert controller.calls == ["start"] - assert engine.retune_controller is controller - - def test_replace_retune_with_no_inputs_disables_retune(stub_controller): engine = _adapter(inputs=retune_inputs()) (old,) = stub_controller.instances - engine.replace_retune(None, generation=2, now_s=10.0, last_decided_k=7) + engine.replace_retune(None, generation=2, now_s=10.0) assert engine.retune_controller is None - assert old.calls == ["drop_pending", "close", "running_future"] + assert old.calls == ["drop_pending", "close"] assert len(stub_controller.instances) == 1 -def test_replace_retune_queues_the_new_controller_behind_the_old_sweep( - stub_controller, -): - engine = _adapter(inputs=retune_inputs()) - (old,) = stub_controller.instances - old.running = Future() - - engine.replace_retune( - retune_inputs(), generation=1, now_s=10.0, last_decided_k=None - ) - - new = stub_controller.instances[1] - assert new.hooks["predecessor"] is old.running - # The new controller carries it on: replacing it asks it, not the adapter. - engine.replace_retune( - retune_inputs(), generation=2, now_s=20.0, last_decided_k=None - ) - assert stub_controller.instances[2].hooks["predecessor"] is None - - -def test_replace_retune_keeps_the_old_sweep_across_a_disabled_generation( - stub_controller, -): - engine = _adapter(inputs=retune_inputs()) - (old,) = stub_controller.instances - old.running = Future() - - engine.replace_retune(None, generation=1, now_s=10.0, last_decided_k=None) - stub_controller.construct_error = ValueError("zero startup credit") - with pytest.raises(ValueError, match="zero startup credit"): - engine.replace_retune( - retune_inputs(), generation=2, now_s=20.0, last_decided_k=None - ) - engine.replace_retune( - retune_inputs(), generation=3, now_s=30.0, last_decided_k=None - ) - - assert stub_controller.instances[1].hooks["predecessor"] is old.running - - -def test_replace_retune_forgets_an_old_sweep_that_has_finished(stub_controller): - engine = _adapter(inputs=retune_inputs()) - (old,) = stub_controller.instances - old.running = Future() - - engine.replace_retune(None, generation=1, now_s=10.0, last_decided_k=None) - old.running.set_result(None) - engine.replace_retune( - retune_inputs(), generation=2, now_s=20.0, last_decided_k=None - ) - - assert stub_controller.instances[1].hooks["predecessor"] is None - - def test_replace_retune_construction_failure_leaves_retune_disabled( stub_controller, ): @@ -349,9 +275,7 @@ def test_replace_retune_construction_failure_leaves_retune_disabled( stub_controller.construct_error = ValueError("zero startup credit") with pytest.raises(ValueError, match="zero startup credit"): - engine.replace_retune( - retune_inputs(), generation=3, now_s=10.0, last_decided_k=None - ) + engine.replace_retune(retune_inputs(), generation=3, now_s=10.0) assert old.closed == 1 assert engine.retune_controller is None @@ -364,9 +288,7 @@ def test_replace_retune_survives_an_old_controller_that_fails_to_close( (old,) = stub_controller.instances old.close_error = RuntimeError("worker stuck") - engine.replace_retune( - retune_inputs(), generation=1, now_s=10.0, last_decided_k=None - ) + engine.replace_retune(retune_inputs(), generation=1, now_s=10.0) assert engine.retune_controller is stub_controller.instances[1] @@ -380,7 +302,7 @@ def test_replace_retune_requires_live_hooks(stub_controller): ) with pytest.raises(RuntimeError, match="live retune"): - engine.replace_retune(None, generation=1, now_s=0.0, last_decided_k=None) + engine.replace_retune(None, generation=1, now_s=0.0) async def test_real_controller_with_live_hooks_anchors_at_the_phase(): @@ -431,6 +353,7 @@ def gate(k, window): assert [d.decision_id for d in decisions][:2] == ["retune-w60000", "retune-w120000"] +@pytest.mark.timeout(10) async def test_rebuilt_real_controllers_never_sweep_at_once(): """Two generations of one window never run their sweeps concurrently. @@ -495,9 +418,7 @@ def sweep(request, *, cancel, abort): await engine.tick(scheduled, TickInput(now_s=now, worker_counts=counts)) assert gen0_started.wait(30) - engine.replace_retune( - retune_inputs(), generation=1, now_s=now, last_decided_k=None - ) + engine.replace_retune(retune_inputs(), generation=1, now_s=now) await engine.tick(scheduled, TickInput(now_s=now + 1.0, worker_counts=counts)) controller = engine.retune_controller assert controller is not None and controller.generation == 1 @@ -558,6 +479,8 @@ def __init__(self, calls, *, inputs=None, hooks=None): self.host = None self.on_effects_error: Exception | None = None self.stop_error: Exception | None = None + # The host's engine when stop ran (it must still be up). + self.engine_at_stop = None def retune_inputs(self): self.calls.append("live.retune_inputs") @@ -572,6 +495,8 @@ async def start(self, host): async def stop(self): self.calls.append("live.stop") + if self.host is not None: + self.engine_at_stop = self.host._engine if self.stop_error is not None: raise self.stop_error @@ -671,7 +596,9 @@ async def bootstrap_regression(): finally: await planner._shutdown_runtime() - # Engine (and controller) first, then the supervisor, then the environment. + # The supervisor stops while the engine is up (it waits for the engine's + # sweep thread), then the engine closes the controller. + assert live.engine_at_stop is engine assert controller.closed == 1 assert environment.calls[-2:] == ["live.stop", "environment.shutdown"] assert planner._live_retune is None @@ -754,15 +681,17 @@ async def apply_effects(applied_effects): assert len(applied) == 2 +@pytest.mark.timeout(10) async def test_replace_live_retune_holds_the_config_lock(stub_controller): planner, _, _ = _planner(inputs=None) await planner._async_init() try: await planner._config_lock.acquire() task = asyncio.create_task( - planner.replace_live_retune(retune_inputs(), generation=2, last_decided_k=9) + planner.replace_live_retune(retune_inputs(), generation=2) ) - await asyncio.sleep(0.05) + for _ in range(5): # let the task run until it waits for the lock + await asyncio.sleep(0) assert not task.done() and stub_controller.instances == [] planner._config_lock.release() await asyncio.wait_for(task, timeout=5) @@ -770,7 +699,6 @@ async def test_replace_live_retune_holds_the_config_lock(stub_controller): (controller,) = stub_controller.instances assert planner._engine.retune_controller is controller # type: ignore[union-attr] assert controller.generation == 2 - assert controller.rewinds == [9] assert controller.phases == [PHASE] finally: await planner._shutdown_runtime() @@ -780,7 +708,7 @@ async def test_replace_live_retune_needs_the_orchestrator_engine(): planner, _, _ = _planner() with pytest.raises(RuntimeError, match="orchestrator engine"): - await planner.replace_live_retune(None, generation=1, last_decided_k=None) + await planner.replace_live_retune(None, generation=1) # --------------------------------------------------------------------------- @@ -885,10 +813,18 @@ async def run(): assert "Cannot install a planner SIGTERM handler" in caplog.text -async def test_start_planner_installs_sigterm_only_with_retune(monkeypatch): - seen: list[tuple[bool, str]] = [] +async def test_start_planner_installs_sigterm_only_when_a_retune_lease_may_be_held( + monkeypatch, +): + # With live retune, or the clean-up check of a removed one (every + # Kubernetes planner without retune config), the planner may hold the + # retune leadership Lease. + seen: list[bool] = [] class _Planner: + def __init__(self, may_hold: bool) -> None: + self.may_hold_retune_lease = may_hold + async def _async_init(self): pass @@ -904,16 +840,27 @@ def endpoint(self, _name): return _Endpoint() async def fake_run_until_sigterm(runtime, serve, run): - seen.append((True, type(runtime).__name__)) + seen.append(isinstance(runtime, _StartRuntime)) await asyncio.gather(serve, run) - monkeypatch.setattr(planner_main, "construct_planner", lambda **_: _Planner()) + planners = iter([_Planner(False), _Planner(True)]) + monkeypatch.setattr(planner_main, "construct_planner", lambda **_: next(planners)) monkeypatch.setattr(planner_main, "_run_until_sigterm", fake_run_until_sigterm) await planner_main.start_planner(_StartRuntime(), PlannerConfig(mode="agg")) assert seen == [] - await planner_main.start_planner(_StartRuntime(), _live_config()) - assert seen == [(True, "_StartRuntime")] + await planner_main.start_planner(_StartRuntime(), PlannerConfig(mode="agg")) + assert seen == [True] + + +async def test_may_hold_retune_lease(stub_controller): + planner, _, _ = _planner(inputs=None) + assert not planner.may_hold_retune_lease + await planner._async_init() + try: + assert planner.may_hold_retune_lease + finally: + await planner._shutdown_runtime() def test_hooks_are_frozen(): diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py index b02e5297c3d3..d909fb0ba052 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py @@ -7,9 +7,9 @@ Covers what a live host adds on top of the replay loop: epoch-phase anchoring, the launch gate, harvest gating and abandoned sweeps, -``drop_pending``/``rewind``, the predecessor generation's sweep, the DGD -spec replicas check, ``restore_desired`` and the structured failure reason -of a decision. +``drop_pending``/``rewind``, the sweep thread shared across generations, +the DGD spec replicas check, ``restore_desired`` and the structured failure +reason of a decision. """ from __future__ import annotations @@ -17,7 +17,6 @@ import json import threading from collections import deque -from concurrent.futures import Future import pytest @@ -37,7 +36,7 @@ WorkerCounts, ) from dynamo.planner.plugins.clock import VirtualClock, WallClock -from dynamo.planner.retune.controller import RetuneController +from dynamo.planner.retune.controller import RetuneController, SweepExecutor from dynamo.planner.retune.live.contracts import NON_DECISION_FAILURES, GateResult from dynamo.planner.retune.sweep import SweepResult @@ -192,7 +191,7 @@ def _live( start=T0, phase=0.0, config=None, - predecessor=None, + executor=None, ): controller = RetuneController( config or _config(), @@ -206,7 +205,7 @@ def _live( ), harvest_allowed=harvest, generation=generation, - predecessor=predecessor, + executor=executor, ) controller.start(start, phase) return controller @@ -670,8 +669,6 @@ def blocking(request): queued = controller._pending.future controller.drop_pending() assert queued.cancelled() - # Nothing holds the thread on its behalf: no queue delay to track. - assert controller._dropped is None release.set() blocker.result(timeout=30) finally: @@ -764,178 +761,14 @@ def blocking(request): # --------------------------------------------------------------------------- -# Predecessor generation and running_future +# The sweep thread shared across generations # --------------------------------------------------------------------------- -def _blocking_sweeps(release, started=None): - """``FakeSweeps`` whose first sweep runs until ``release`` is set.""" - - sweeps = FakeSweeps() - - def blocking(request): - if started is not None: - started.set() - release.wait(30) - return _flat()(request) - - sweeps.results.append(blocking) - sweeps.default = _flat() - return sweeps - - -async def test_a_first_launch_queues_behind_the_predecessor_generation(): - # The previous generation's sweep, still running on its own thread. - predecessor: Future = Future() - assert predecessor.set_running_or_notify_cancel() - ran = threading.Event() - sweeps = FakeSweeps() - - def record(request): - ran.set() - return _flat()(request) - - sweeps.default = record - clock = ManualClock() - controller = _live(sweeps, clock=clock, predecessor=predecessor) - try: - assert controller.running_future() is predecessor - clock.mono = 100.0 - launch = await _tick(controller, L_K) - assert launch.audit_events == [f"retune:launch w={K}"] - queued = controller._pending.future - # Waits for the predecessor; it holds no other thread, so it runs. - assert controller.running_future() is queued - assert not ran.wait(0.2) - assert sweeps.requests == [] - - clock.mono = 106.0 # the predecessor returns 6 s later - predecessor.set_result(SweepResult(status="failed")) - queued.result(timeout=30) - assert controller.running_future() is None - (decision,) = (await _tick(controller, L_K + 4 * STEP)).decisions - finally: - if not predecessor.done(): - predecessor.cancel() - controller.close() - - assert [r.window_index for r in sweeps.requests] == [K] - assert decision.launched_at_s == L_K + 6.0 - - -async def test_a_done_predecessor_does_not_delay_the_first_launch(): - predecessor: Future = Future() - predecessor.set_result(SweepResult(status="failed")) - sweeps = FakeSweeps() - sweeps.default = _flat() - controller = _live(sweeps, clock=WallClock(), predecessor=predecessor) - try: - assert controller.running_future() is None - await _tick(controller, L_K) - assert controller._pending.queued is None - controller._pending.future.result(timeout=30) - (decision,) = (await _tick(controller, L_K + 2 * STEP)).decisions - finally: - controller.close() - - assert decision.launched_at_s == L_K - - -async def test_a_dropped_launch_hands_the_predecessor_wait_on(): - # A launch dropped while it waits for the predecessor still holds this - # controller's thread: the relaunch, and a successor, queue behind it. - predecessor: Future = Future() - assert predecessor.set_running_or_notify_cancel() - sweeps = FakeSweeps() - sweeps.default = _flat() - controller = _live(sweeps, clock=WallClock(), predecessor=predecessor) - try: - await _tick(controller, L_K) - waiting = controller._pending.future - controller.drop_pending() - assert controller._dropped is waiting - assert controller.running_future() is waiting - controller.rewind(K - 1) - await _tick(controller, L_K + STEP) - relaunch = controller._pending.future - assert relaunch.running() is False - # The relaunch has not started: the waiting launch holds the thread. - assert controller.running_future() is waiting - - predecessor.set_result(SweepResult(status="failed")) - waiting.result(timeout=30) - relaunch.result(timeout=30) - assert controller.running_future() is None - finally: - if not predecessor.done(): - predecessor.cancel() - controller.close() - - assert [r.window_index for r in sweeps.requests] == [K, K] - - -async def test_running_future_tracks_the_sweep_holding_the_thread(): - release = threading.Event() - started = threading.Event() - sweeps = _blocking_sweeps(release, started) - controller = _live(sweeps, clock=WallClock()) - try: - assert controller.running_future() is None - await _tick(controller, L_K) - running = controller._pending.future - assert started.wait(30) - assert controller.running_future() is running - - controller.drop_pending() - assert controller.running_future() is running - controller.rewind(K - 1) - await _tick(controller, L_K + STEP) # queued behind the dropped one - queued = controller._pending.future - assert controller.running_future() is running - - release.set() - running.result(timeout=30) - queued.result(timeout=30) - assert controller.running_future() is None - finally: - release.set() - controller.close() - - -async def test_running_future_after_close_skips_a_cancelled_queued_sweep(): - # The host reads running_future after close: a queued sweep that close - # cancelled never runs, so only the running one is waited on. - release = threading.Event() - started = threading.Event() - sweeps = _blocking_sweeps(release, started) - controller = _live(sweeps, clock=WallClock()) - try: - await _tick(controller, L_K) - running = controller._pending.future - assert started.wait(30) - controller.drop_pending() - controller.rewind(K - 1) - await _tick(controller, L_K + STEP) - queued = controller._pending.future - controller.close() - assert queued.cancelled() - assert controller.running_future() is running - finally: - release.set() - controller.close() - running.result(timeout=30) - assert controller.running_future() is None - - -def test_predecessor_must_be_a_future(): - with pytest.raises(ValueError, match="predecessor must be"): - _live(FakeSweeps(), predecessor=object()) - - -async def test_generations_chained_by_predecessor_never_sweep_at_once(): - # Reviewer scenario: generation 0 is slow to see its abort while - # generation 1 relaunches the same window. Replacing generation 1 before - # it launched hands generation 0's sweep on to generation 2 as well. +async def test_generations_sharing_the_sweep_thread_never_sweep_at_once(): + # Generation 0 is slow to see its abort while generation 2 relaunches the + # same window; generation 1 was replaced before it launched. All three + # submit to the host's one sweep thread, which close() leaves running. lock = threading.Lock() running: list[int] = [] overlaps: list[list[int]] = [] @@ -959,35 +792,39 @@ def sweep(request, *, cancel, abort): with lock: running.remove(request.generation) - old = _live(sweep, clock=WallClock(), generation=0) - middle = new = None + executor = SweepExecutor() + clock = ManualClock() + controllers = [] try: + old = _live(sweep, clock=clock, generation=0, executor=executor) + controllers.append(old) + clock.mono = 100.0 await _tick(old, L_K) assert started.wait(30) old.drop_pending() old.close() - middle = _live( - sweep, clock=WallClock(), generation=1, predecessor=old.running_future() - ) + middle = _live(sweep, clock=clock, generation=1, executor=executor) + controllers.append(middle) middle.close() - new = _live( - sweep, clock=WallClock(), generation=2, predecessor=middle.running_future() - ) + new = _live(sweep, clock=clock, generation=2, executor=executor) + controllers.append(new) launched = await _tick(new, L_K + STEP) assert launched.audit_events == [f"retune:launch w={K}"] - assert new._pending.queued is not None relaunch = new._pending.future assert not relaunch.done() + clock.mono = 105.0 # generation 0 returns 5 s later release.set() relaunch.result(timeout=30) + (decision,) = (await _tick(new, L_K + 4 * STEP)).decisions finally: release.set() - for controller in (old, middle, new): - if controller is not None: - controller.close() + for controller in controllers: + controller.close() + executor.shutdown(wait=True) assert overlaps == [] assert order == [(0, K), (2, K)] + assert decision.launched_at_s == L_K + STEP + 5.0 # --------------------------------------------------------------------------- @@ -1204,6 +1041,8 @@ def test_decision_record_carries_failure_reason_and_a_private_details_copy(): failure_reason="crn_mismatch", details=details, ) + # A deep copy: neither the details nor a value nested in them is shared. + details["shards"][0]["state"] = "Failed" details["shards"] = [] record = decision.to_dict() diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py b/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py index 08e4b95d9009..4c4d8ccba13f 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py @@ -1,8 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""``dynamo_planner_retune_*`` metrics: names, labels and values per design -§4.10, collectors shared per registry, and methods that never raise.""" +"""``dynamo_planner_retune_*`` metrics: names, labels and values as the live +retune guide lists them, collectors shared per registry, and methods that +never raise.""" from __future__ import annotations diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py index 78ba876a2d6c..2ca376365249 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py @@ -7,6 +7,7 @@ import dataclasses import json +import math import pytest @@ -23,15 +24,18 @@ HISTORY_ANNOTATION, HISTORY_LIMIT, STATE_ANNOTATION, + STATE_FORMAT_VERSION, LeaseLeadership, RetuneState, StateFormatError, StateSnapshot, StateStore, + StateVersionError, append_history, claim, deploy, deployed_off, + find_retune_leases, fingerprint, has_decision, leader_term, @@ -212,32 +216,71 @@ def test_state_round_trips_through_annotations() -> None: assert RetuneState.from_annotations(annotations) == state +def _body(**fields) -> str: + return json.dumps({"version": STATE_FORMAT_VERSION, **fields}) + + def test_state_parsing_edge_cases() -> None: assert RetuneState.from_annotations(None) is None assert RetuneState.from_annotations({"other": "x"}) is None claimed = RetuneState.from_annotations( - {STATE_ANNOTATION: json.dumps({"dgd_uid": "u", "leader_term": "t"})} + {STATE_ANNOTATION: _body(dgd_uid="u", leader_term="t")} ) assert claimed == RetuneState(dgd_uid="u", leader_term="t") assert not claimed.has_deployment for bad in ( "{", "[]", - json.dumps({"leader_term": "t"}), - json.dumps({"dgd_uid": "u", "mode": "bogus"}), - json.dumps({"dgd_uid": "u", "flat": {"a": "x"}}), - json.dumps({"dgd_uid": "u", "deploy_seq": "3"}), + json.dumps({"dgd_uid": "u"}), # no version + _body(leader_term="t"), + _body(dgd_uid="u", mode="bogus"), + _body(dgd_uid="u", flat={"a": "x"}), + _body(dgd_uid="u", deploy_seq="3"), + # json.loads accepts NaN and Infinity; they are not numbers here. + _body(dgd_uid="u", flat={"a": math.nan}), + _body(dgd_uid="u", last_window_end_ms=math.nan), + _body(dgd_uid="u", last_window_end_ms=math.inf), + _body(dgd_uid="u", updated_ms="x"), + _body(dgd_uid="u", updated_ms=[1]), ): with pytest.raises(StateFormatError): RetuneState.from_annotations({STATE_ANNOTATION: bad}) # A damaged history never blocks the state. parsed = RetuneState.from_annotations( + {STATE_ANNOTATION: _body(dgd_uid="u"), HISTORY_ANNOTATION: "not json"} + ) + assert parsed is not None and parsed.history == () + + +def test_another_format_version_is_rejected_not_corrupt() -> None: + # Checked first: a newer planner's state is never judged by this + # planner's field rules (which would make it corrupt, then overwritten). + newer = json.dumps( + {"version": STATE_FORMAT_VERSION + 1, "dgd_uid": "u", "flat": {"a": "x"}} + ) + with pytest.raises(StateVersionError): + RetuneState.from_annotations({STATE_ANNOTATION: newer}) + assert not issubclass(StateVersionError, StateFormatError) + + kube = FakeRetuneKube() + store = StateStore(kube, DGD) + kube.create_lease( { - STATE_ANNOTATION: json.dumps({"dgd_uid": "u"}), - HISTORY_ANNOTATION: "not json", + "metadata": {"name": store.name, "annotations": {STATE_ANNOTATION: newer}}, + "spec": {}, } ) - assert parsed is not None and parsed.history == () + with pytest.raises(StateVersionError): + store.read() + + +def test_find_retune_leases() -> None: + kube = FakeRetuneKube() + assert find_retune_leases(kube, "graph", timeout_s=1.0) == () + kube.create_lease({"metadata": {"name": state_lease_name("graph")}, "spec": {}}) + assert find_retune_leases(kube, "graph", timeout_s=1.0) == ( + state_lease_name("graph"), + ) def test_deployed_off() -> None: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py index c071a10c1186..d298e021faee 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -1,14 +1,19 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -"""Unit tests for ``LiveRetuneSupervisor`` and its bootstrap, plus the §6 -interleaving harness that guards the §4.7 invariants. - -Everything runs on fakes: ``FakeRetuneKube`` (shared Leases), a fake frontend -whose routers follow the ``/router_policy`` PATCH/reset/revision semantics, -a fake actuator over it, a fake controller that drives the supervisor's -hooks as ``RetuneController.on_tick`` does, a fake planner host and clocks -the tests advance by hand. Blocking calls run inline. +# ruff: noqa: E402 +# Optional-dependency preflight must run before the supervisor imports the sweep. + +"""Unit tests for ``LiveRetuneSupervisor`` and its bootstrap, plus a seeded +interleaving harness that checks its invariants: at most one leader writes, +a decision is persisted before any router changes, no window is wasted, and +the routers settle on the persisted state. + +The supervisor drives the real engine adapter and ``RetuneController``; their +sweeps run on the adapter's sweep thread, in ``StubSweeps``, until the test +finishes them. Kubernetes is ``FakeRetuneKube`` (shared Leases), the routers +are ``FakeFrontend`` (PATCH merges and bumps the revision, reset restores +the initial) behind ``FakeActuator``, and clocks are advanced by hand. +Kubernetes calls run inline. """ from __future__ import annotations @@ -22,15 +27,35 @@ import math import random import threading -import time from collections import deque from types import SimpleNamespace from typing import Any, Optional import pytest -from dynamo.planner.core.types import RetuneDecision, RetuneParams +pytest.importorskip( + "aisimulate.sweeper", + reason="AI Simulate is an optional Dynamo simulation dependency", +) + +from dynamo.planner.config.planner_config import PlannerConfig, RetuneEngineSpec +from dynamo.planner.core.types import ( + EngineCapabilities, + RetuneDecision, + RetuneInputs, + RetuneParams, + WorkerCapabilities, + WorkerCounts, +) +from dynamo.planner.plugins.clock import WallClock +from dynamo.planner.plugins.orchestrator.engine_adapter import ( + OrchestratorEngineAdapter, + RetuneHooks, +) +from dynamo.planner.retune import controller as controller_module +from dynamo.planner.retune.controller import RetuneController from dynamo.planner.retune.live import bootstrap +from dynamo.planner.retune.live.actuator import RouterActuator from dynamo.planner.retune.live.contracts import ( LABEL_DGD_UID, ActuationReport, @@ -40,7 +65,9 @@ KubeConflict, KubeError, NullMetrics, + PolicyCallResult, PremiseReport, + RouterInfo, ) from dynamo.planner.retune.live.state import ( STATE_ANNOTATION, @@ -59,6 +86,7 @@ plan_initialization, window_index, ) +from dynamo.planner.retune.sweep import SweepResult from dynamo.planner.tests.live_retune_fakes import FakeRetuneKube from dynamo.planner.tests.unit.test_live_retune_state import live_config @@ -81,11 +109,17 @@ SCALE = "prefill_load_scale" TEMP = "router_temperature" KEYS = (CREDIT, SCALE, TEMP) -DOMAIN = {CREDIT: [0.5, 1.0], SCALE: [1.0], TEMP: [0.0, 0.2]} -VARIANTS = { - "a": {"aggregated": {CREDIT: 1.0, SCALE: 1.0, TEMP: 0.0, "host": 0.75}}, - "b": {"aggregated": {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.0, "host": 0.75}}, +# The searched router domain of live_config(), by AISimulate knob name. +ROUTER = { + "overlap_score_credit": {"choices": [0.5, 1.0]}, + "prefill_load_scale": 1.0, + "temperature": {"choices": [0.0, 0.2]}, } +# The same domain by live knob name. +DOMAIN = {CREDIT: [0.5, 1.0], SCALE: [1.0], TEMP: [0.0, 0.2]} +# Two frontend startups: the KvRouterConfig JSON the frontend parses to. +ROUTER_CONFIG = {"a": {CREDIT: 1.0}, "b": {CREDIT: 0.5}} +SPEC_COUNTS = WorkerCounts(ready_num_decode=2) # The due window at START_WALL. DUE0 = math.floor((START_WALL - GRACE) / INTERVAL) - 1 # The window in progress at START_WALL (the one after DUE0). First enable @@ -95,6 +129,35 @@ assert K0 == DUE0 + 1 +def resolver(variant: str) -> bootstrap.KvRouterResolver: + return bootstrap.KvRouterResolver(ROUTER_CONFIG[variant], ("aggregated",)) + + +def by_wt(variant: str, flat: dict[str, float]) -> dict[str, dict[str, float]]: + return resolver(variant).by_wt(flat) + + +def startup_flat(variant: str) -> dict[str, float]: + return resolver(variant).startup_flat(KEYS) + + +# policy_parameters("aggregated") of each startup: its routers' 8 parameters. +_DEFAULT_POLICY = { + "decode_active_request_weight": 0.0, + "disk_cache_hit_weight": 0.25, + "host_cache_hit_weight": 0.75, + "overlap_score_credit": 1.0, + "overlap_score_credit_decay": 0.0, + "prefill_load_scale": 1.0, + "router_temperature": 0.0, + "shared_cache_multiplier": 0.0, +} +VARIANTS = { + variant: {"aggregated": {**_DEFAULT_POLICY, **config}} + for variant, config in ROUTER_CONFIG.items() +} + + def window(k: int) -> tuple[float, float, float]: start = k * INTERVAL return start, start + INTERVAL, start + INTERVAL + GRACE @@ -104,14 +167,6 @@ def end_ms(k: int) -> int: return int(round(window(k)[1] * 1000)) -def by_wt(variant: str, flat: dict[str, float]) -> dict[str, dict[str, float]]: - return {wt: {**params, **flat} for wt, params in VARIANTS[variant].items()} - - -def startup_flat(variant: str) -> dict[str, float]: - return {key: VARIANTS[variant]["aggregated"][key] for key in KEYS} - - # ------------------------------------------------------------------ fakes @@ -131,6 +186,19 @@ def advance(self, seconds: float) -> None: self.mono += seconds +class AdapterClock(WallClock): + """The engine adapter's clock: monotonic time is the test's.""" + + def __init__(self, clock: Clock) -> None: + self._clock = clock + + def now(self) -> float: + return self._clock.time() + + def monotonic(self) -> float: + return self._clock.monotonic() + + async def inline(call): return call() @@ -218,7 +286,9 @@ def manual_patch(self, key: str, value: float) -> None: class FakeActuator: - """§4.8 over a ``FakeFrontend``: latch, apply/rollback/observe, parity.""" + """The router actuator's rules over a ``FakeFrontend``: latch, apply, + rollback, observe and parity. Like ``RouterActuator`` it PATCHes a router + only once the persisted ``initials`` hold its initial.""" def __init__( self, frontend: FakeFrontend, owner: str, advisory: bool, maps @@ -264,6 +334,8 @@ def initial(router: Router) -> Optional[dict[str, float]]: frontend.reset(router, self.owner, self.advisory, target.mode) self.latched[router.router_id] = dict(router.config) known = router.config + if initials.get(router.router_id) != known: + continue # held until the state records the initial if known != self.startup_maps.get(router.worker_type): continue desired = target.desired_by_wt[router.worker_type] @@ -312,122 +384,176 @@ def expected(router: Router): ) -class FakeResolver: - def __init__(self, maps) -> None: - self.maps = maps +class StubSweeps: + """The ``sweep_runner`` of every process: a sweep runs on its adapter's + sweep thread until the test finishes it, or until its abort or cancel + event is set. Results are made on the test's thread, so a seeded run + is deterministic.""" - def startup_flat(self, keys): - return {key: float(self.maps["aggregated"][key]) for key in keys} + @dataclasses.dataclass + class Running: + cancel: threading.Event + abort: Optional[threading.Event] + done: threading.Event = dataclasses.field(default_factory=threading.Event) + result: Optional[SweepResult] = None - def by_wt(self, flat): - return {wt: {**params, **flat} for wt, params in self.maps.items()} + def stopping(self) -> bool: + return self.cancel.is_set() or ( + self.abort is not None and self.abort.is_set() + ) + def __init__(self) -> None: + self._cond = threading.Condition() + self._running: dict[int, StubSweeps.Running] = {} + self._closed = False + self.requests: list[Any] = [] + + def __call__(self, request, *, cancel, abort, context): + running = StubSweeps.Running(cancel, abort) + with self._cond: + self._running[id(request)] = running + self.requests.append(request) + self._cond.notify_all() + try: + while not running.done.wait(0.002): + if self._closed or cancel.is_set(): + return SweepResult(status="failed", failure_reason="shutdown") + if abort is not None and abort.is_set(): + return SweepResult( + status="failed", failure_reason="lost_leadership" + ) + assert running.result is not None + return running.result + finally: + with self._cond: + del self._running[id(request)] + self._cond.notify_all() + + def finish(self, controller: RetuneController, result_for) -> bool: + """Finish the controller's pending sweep with ``result_for(request)`` + and wait until it returned; ``False`` when none is pending.""" + + pending = controller._pending + if pending is None or pending.future.done(): + return False + with self._cond: + assert self._cond.wait_for( + lambda: id(pending.request) in self._running or pending.future.done(), + timeout=10, + ) + running = self._running.get(id(pending.request)) + if running is None: + return False + running.result = result_for(pending.request) + running.done.set() + pending.future.result(timeout=10) + return True + + def quiesce(self, controllers) -> None: + """Wait until every stopped sweep returned and every pending sweep + runs, so no sweep starts after the test's clock moved on.""" + + def quiet() -> bool: + if any(running.stopping() for running in self._running.values()): + return False + for controller in controllers: + pending = controller._pending + if ( + pending is not None + and not pending.future.done() + and id(pending.request) not in self._running + ): + return False + return True + + with self._cond: + assert self._cond.wait_for(quiet, timeout=10) -class FakeController: - """The hook-facing surface of ``RetuneController`` plus a tick model.""" + def close(self) -> None: + with self._cond: + self._closed = True - def __init__(self, *, generation: int, variant: str) -> None: - self.generation = generation - self.variant = variant - self.router_keys = KEYS - self.router_domain = {key: list(values) for key, values in DOMAIN.items()} - self.desired = startup_flat(variant) - self.last_k = -1 - self.pending: Optional[dict[str, Any]] = None - self.closed = False - self.dropped: list[int] = [] - self.rewinds: list[int] = [] - self.restored: list[dict[str, float]] = [] - self.started: list[tuple[float, Optional[float]]] = [] - def start(self, start_s: float, phase_s: Optional[float] = None) -> None: - self.started.append((start_s, phase_s)) - self.last_k = -1 +def failed(request) -> SweepResult: + return SweepResult(status="failed", error="injected", failure_reason="crn_mismatch") - def drop_pending(self) -> None: - if self.pending is not None: - if self.pending["abort"] is not None: - self.pending["abort"].set() - self.dropped.append(self.pending["k"]) - self.pending = None - - def rewind(self, last_decided_k: int) -> None: - self.rewinds.append(last_decided_k) - if self.pending is None: - self.last_k = min(self.last_k, last_decided_k) - - def restore_desired(self, flat) -> None: - missing = set(self.router_keys) - set(flat) - if missing: - raise ValueError(f"missing {missing}") - self.desired = {key: float(flat[key]) for key in self.router_keys} - self.restored.append(dict(self.desired)) - def close(self) -> None: - self.closed = True - - def current(self) -> RetuneParams: - return RetuneParams(router=dict(self.desired)) - - # -- tick model (what on_tick does with the hooks) -- - - def tick(self, sup: LiveRetuneSupervisor, now_s: float, rng, advisory: bool): - decisions: list[RetuneDecision] = [] - pending = self.pending - if pending is not None and pending["done"]: - self.pending = None - k = pending["k"] - if pending["abort"] is not None and pending["abort"].is_set(): - # failed(lost_leadership): no decision, rewind. - self.dropped.append(k) - self.last_k = min(self.last_k, k - 1) - elif not sup.harvest_allowed(pending["generation"]): - self.dropped.append(k) - self.last_k = min(self.last_k, k - 1) - else: - decisions.append(self._decide(k, now_s, rng, advisory)) - due = math.floor((now_s - GRACE) / INTERVAL + 1e-9) - 1 - if due > self.last_k: - gate = sup.launch_gate(due, window(due)) - if gate.reason == "already_decided": - self.last_k = due - elif gate.reason is None: - assert gate.spec_replicas and gate.abort is not None - self.last_k = due - if self.pending is not None: - abort = self.pending["abort"] - # A busy behind a sweep the supervisor abandoned (abort - # set, not dropped) consumes the window for nothing. - assert ( - abort is None or not abort.is_set() - ), "busy behind an aborted sweep" - decisions.append(make_decision(due, "busy", self.current())) - else: - self.pending = { - "k": due, - "generation": self.generation, - "abort": gate.abort, - "done": False, - } - return SimpleNamespace( - retune_decisions=decisions, - router_parameters=by_wt(self.variant, self.desired), +def scored(best=None): + """A sweep result whose ``best`` candidate (else none) beats the rest.""" + + def result_for(request) -> SweepResult: + return SweepResult( + status="ok", + num_requests=10, + scores={ + candidate.key(): (2.0 if candidate.router == best else 1.0) + for candidate in request.candidates + }, + wall_s=1.0, ) - def _decide(self, k, now_s, rng, advisory) -> RetuneDecision: - previous = self.current() - chosen = {key: rng.choice(values) for key, values in DOMAIN.items()} - roll = rng.random() - if roll < 0.15: - return make_decision( - k, "failed", previous, failure_reason="replicas_changed" - ) - if roll < 0.35 or chosen == self.desired: - return make_decision(k, "kept", previous) - if advisory: - return make_decision(k, "advisory", previous, chosen=chosen) - self.desired = dict(chosen) - return make_decision(k, "applied", previous, chosen=chosen) + return result_for + + +class TracedController(RetuneController): + """The real controller, remembering the windows the supervisor dropped.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.dropped: list[int] = [] + + def drop_pending(self) -> None: + if self._pending is not None: + self.dropped.append(self._pending.request.window_index) + super().drop_pending() + + +@pytest.fixture(autouse=True) +def traced_controllers(monkeypatch): + # The engine adapter looks the controller class up when it builds one. + monkeypatch.setattr(controller_module, "RetuneController", TracedController) + yield + while WORLDS: + WORLDS.pop().close() + + +def controller_config(*, advisory: bool = False) -> PlannerConfig: + """The real controller's config, with this file's windows. + + Built with the replay executor: the live config's timeout must leave + 600 s before the next window, which 600 s windows cannot, and the + controller never reads the executor. + """ + + return PlannerConfig( + namespace="test-ns", + mode="agg", + advisory=advisory, + enable_load_scaling=False, + enable_throughput_scaling=False, + retune={ + "interval_seconds": INTERVAL, + "grace_seconds": GRACE, + "timeout_seconds": TIMEOUT, + "router": ROUTER, + }, + ) + + +def retune_inputs(variant: str) -> RetuneInputs: + return RetuneInputs( + engine=RetuneEngineSpec( + router_mode="kv_router", router_config=dict(ROUTER_CONFIG[variant]) + ), + trace_paths=("/trace/segments/graph",), + ) + + +def variant_of(inputs: RetuneInputs) -> str: + (variant,) = [ + v for v, c in ROUTER_CONFIG.items() if c == inputs.engine.router_config + ] + return variant def make_decision( @@ -463,41 +589,59 @@ def make_decision( class FakeHost: - """``NativePlannerBase`` as the supervisor sees it.""" + """``NativePlannerBase`` as the supervisor sees it: the config lock and + the real engine adapter, whose controllers share one sweep thread.""" - def __init__(self, spec: Spec, clock: Clock, controller: Optional[FakeController]): - self.spec = spec - self.clock = clock - self._engine = SimpleNamespace(retune_controller=controller) + def __init__( + self, + world: World, + hooks: RetuneHooks, + inputs: Optional[RetuneInputs], + *, + advisory: bool, + ) -> None: + self.clock = world.clock self._config_lock = asyncio.Lock() - self.replacements: list[tuple[Any, int, Optional[int]]] = [] - self.fail_replace = False - self.controllers: list[FakeController] = ( - [] if controller is None else [controller] + self._engine = OrchestratorEngineAdapter( + controller_config(advisory=advisory), + WorkerCapabilities( + decode=EngineCapabilities( + num_gpu=1, max_num_batched_tokens=2048, max_kv_tokens=16384 + ) + ), + clock=AdapterClock(world.clock), + retune_inputs=inputs, + retune_hooks=hooks, ) + if self.controller is not None: + # What the adapter's initial_tick does. + self.controller.start(world.clock.time(), hooks.phase_s) + # The frontend startup the current controller was built for. + self.variant = "a" if inputs is None else variant_of(inputs) + self.replacements: list[tuple[Any, int]] = [] + self.fail_replace = False @property - def controller(self) -> Optional[FakeController]: + def controller(self) -> Optional[TracedController]: return self._engine.retune_controller - async def replace_live_retune(self, inputs, *, generation, last_decided_k): + async def replace_live_retune(self, inputs, *, generation): async with self._config_lock: - self.replacements.append((inputs, generation, last_decided_k)) - old = self._engine.retune_controller - self._engine.retune_controller = None - if old is not None: - old.drop_pending() - old.close() - if inputs is None: - return + self.replacements.append((inputs, generation)) + now = self.clock.time() if self.fail_replace: + self._engine.replace_retune(None, generation=generation, now_s=now) raise ValueError("controller construction failed") - new = FakeController(generation=generation, variant=inputs.variant) - new.start(self.clock.time(), 0.0) - if last_decided_k is not None: - new.rewind(last_decided_k) - self._engine.retune_controller = new - self.controllers.append(new) + self._engine.replace_retune(inputs, generation=generation, now_s=now) + if inputs is not None: + self.variant = variant_of(inputs) + + def close(self) -> None: + if self.controller is not None: + self.controller.close() + executor = self._engine._retune_executor + if executor is not None: + executor.shutdown(wait=False, cancel_futures=True) class Spec: @@ -515,7 +659,7 @@ def __init__(self) -> None: def startup(self) -> FrontendStartup: return FrontendStartup( router_mode="kv_router", - router_config={}, + router_config=dict(ROUTER_CONFIG[self.variant]), startup_maps=copy.deepcopy(VARIANTS[self.variant]), frontend_component="Frontend", replicas=self.frontend_replicas, @@ -534,23 +678,30 @@ def evaluation(self, _dgd: Optional[dict] = None) -> SpecEvaluation: reasons=tuple(reasons), ), startup=self.startup(), - inputs=SimpleNamespace(variant=self.variant) if self.p4_ok else None, + inputs=retune_inputs(self.variant) if self.p4_ok else None, spec_replicas=None if self.replicas is None else dict(self.replicas), dgd=DgdRef(DGD_NAME, "test-ns", self.uid, self.dynamo_namespace), ) +# Worlds of the running test, closed after it (sweep threads end). +WORLDS: list[World] = [] + + class World: - """Shared kube, frontend, spec and clock of one test.""" + """Shared kube, frontend, spec, clock and sweeps of one test.""" def __init__(self, kube: Optional[FakeRetuneKube] = None) -> None: self.clock = Clock() self.kube = kube or FakeRetuneKube(now=self.clock.time) self.spec = Spec() self.frontend = FakeFrontend(VARIANTS["a"]) + self.sweeps = StubSweeps() + self.hosts: list[FakeHost] = [] self.kube.put_dgd( {"metadata": {"name": DGD_NAME, "uid": "uid-1"}, "spec": {"components": []}} ) + WORLDS.append(self) def shared_state(self) -> Optional[RetuneState]: lease = self.kube.leases.get(STATE_NAME) @@ -562,6 +713,11 @@ def holder(self) -> Optional[str]: lease = self.kube.leases.get(LEADER_NAME) return None if lease is None else lease["spec"].get("holderIdentity") + def close(self) -> None: + self.sweeps.close() + for host in self.hosts: + host.close() + @dataclasses.dataclass class Process: @@ -574,7 +730,7 @@ class Process: alive: bool = True @property - def controller(self) -> Optional[FakeController]: + def controller(self) -> Optional[TracedController]: return self.host.controller @@ -607,19 +763,14 @@ def make_process( canary=None, reconciler=None, sweep_runner=None, - context_factory=None, + context_factory=lambda supervisor: None, settings: Optional[SupervisorSettings] = None, blocking=None, + actuator_factory=None, ) -> Process: kube = kube or world.kube metrics = RecordingMetrics() evaluation = world.spec.evaluation() - controller = ( - FakeController(generation=0, variant=world.spec.variant) - if evaluation.inputs is not None - else None - ) - host = FakeHost(world.spec, world.clock, controller) settings = settings or settings_for(advisory=advisory, promote=promote) sup = LiveRetuneSupervisor( settings=settings, @@ -627,21 +778,24 @@ def make_process( evaluation=evaluation, identity=f"{name}/{len(world.kube.calls)}", premise_evaluator=world.spec.evaluation, - actuator_factory=lambda dgd, maps: FakeActuator( - world.frontend, name, advisory, maps - ), + actuator_factory=actuator_factory + or (lambda dgd, maps: FakeActuator(world.frontend, name, advisory, maps)), fingerprint_fn=lambda startup: f"fp-{None if startup is None else startup.fingerprint_source}", - resolver_factory=lambda startup: FakeResolver(startup.startup_maps), + resolver_factory=lambda startup: bootstrap.KvRouterResolver( + startup.router_config, ("aggregated",) + ), reconciler=reconciler, metrics=metrics, canary=canary, - sweep_runner=sweep_runner, + sweep_runner=sweep_runner or world.sweeps, context_factory=context_factory, clock=world.clock.time, monotonic=world.clock.monotonic, sleep=no_sleep, blocking=blocking or inline, ) + host = FakeHost(world, sup.hooks(), evaluation.inputs, advisory=advisory) + world.hosts.append(host) return Process(name, sup, host, metrics, advisory, kube) @@ -672,13 +826,18 @@ async def settle(processes: list[Process], world: World, rounds: int = 6) -> Non def enqueue(process: Process, *decisions: RetuneDecision) -> None: + """Hand the supervisor a tick's decisions with the controller's routers.""" + controller = process.controller - variant = controller.variant if controller is not None else "a" - desired = controller.desired if controller is not None else startup_flat("a") + desired = ( + controller.current_point().router + if controller is not None + else startup_flat("a") + ) process.sup.on_effects( SimpleNamespace( retune_decisions=list(decisions), - router_parameters=by_wt(variant, desired), + router_parameters=by_wt(process.host.variant, desired), ) ) @@ -916,7 +1075,7 @@ async def test_applied_decision_is_persisted_then_deployed() -> None: process = await leading(world) sup, controller = process.sup, process.controller chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - controller.desired = dict(chosen) + controller.restore_desired(chosen) enqueue(process, make_decision(K0 + 1, "applied", chosen=chosen)) assert sup.outbox_size == 1 assert process.metrics.of("decision") == [] # counted only once persisted @@ -1017,14 +1176,16 @@ async def test_consecutive_write_failures_reinitialize_and_rewind() -> None: process = await leading(world) sup, controller = process.sup, process.controller term_seq = sup.term_seq + # Window K0 + 2 launches (K0 + 1 is skipped: K0 + 2 is due already). + world.clock.wall = window(K0 + 2)[2] + await controller.on_tick(world.clock.time(), SPEC_COUNTS) + assert controller._pending.request.window_index == K0 + 2 world.kube.fail( "replace_lease", KubeError("timeout"), times=3, match=lambda target: target[0] == STATE_NAME, ) - controller.pending = {"k": K0 + 2, "generation": 0, "abort": None, "done": False} - controller.last_k = K0 + 2 enqueue(process, make_decision(K0 + 1, "kept")) for _ in range(3): await sup.outbox_round() @@ -1035,8 +1196,8 @@ async def test_consecutive_write_failures_reinitialize_and_rewind() -> None: assert sup.outbox_size == 0 # The sweep was dropped and the controller rewound to the last decided # window, so both windows can still run. - assert controller.pending is None and controller.dropped == [K0 + 2] - assert controller.last_k == K0 + assert controller._pending is None and controller.dropped == [K0 + 2] + assert controller._last_k == K0 assert world.shared_state().leader_term == sup.term @@ -1084,10 +1245,10 @@ async def test_decisions_are_dropped_and_rewound_without_leadership() -> None: world = World() process = await started(world) controller = process.controller - controller.last_k = K0 + 1 + controller._last_k = K0 + 1 enqueue(process, make_decision(K0 + 1, "kept")) assert process.sup.outbox_size == 0 - assert controller.rewinds and controller.last_k == -1 + assert controller._last_k == -1 # no state: every window is due again async def test_a_foreign_term_forces_reinitialization() -> None: @@ -1120,7 +1281,8 @@ async def test_a_late_write_of_an_earlier_term_fails() -> None: def steal_lease(world: World, identity: str) -> None: - """The leadership Lease changes hands without expiring (§4.7.9).""" + """The leadership Lease changes hands without expiring (an admin edit, + or a contender whose clock runs fast).""" lease = world.kube.leases[LEADER_NAME] lease["spec"]["holderIdentity"] = identity @@ -1169,14 +1331,16 @@ async def test_restart_restores_the_persisted_point() -> None: world = World() first = await leading(world) chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - first.controller.desired = dict(chosen) + first.controller.restore_desired(chosen) enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) await first.sup.outbox_round() await first.sup.stop() assert world.holder() is None # released second = await leading(world, "p2") - assert second.controller.restored == [chosen] - assert second.controller.rewinds[-1] == K0 + 1 + assert second.controller.current_point().router == chosen + # The persisted window is decided: the gate skips it, the next launches. + assert second.sup.launch_gate(K0 + 1, window(K0 + 1)).reason == "already_decided" + assert second.sup.launch_gate(K0 + 2, window(K0 + 2)).reason is None state = world.shared_state() assert state.leader_term == second.sup.term and state.deploy_seq == 2 assert second.sup.target().desired_by_wt == by_wt("a", chosen) @@ -1186,7 +1350,7 @@ async def test_fingerprint_change_resets_with_a_two_phase_rollback() -> None: world = World() first = await leading(world) chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - first.controller.desired = dict(chosen) + first.controller.restore_desired(chosen) enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) await first.sup.outbox_round() await first.sup.control_round() @@ -1200,7 +1364,7 @@ async def test_fingerprint_change_resets_with_a_two_phase_rollback() -> None: state = world.shared_state() assert state.rollback_pending and state.by_wt == VARIANTS["b"] assert state.fingerprint == "fp-b" and state.deploy_seq == 3 - assert second.controller.restored == [startup_flat("b")] + assert second.controller.current_point().router == startup_flat("b") assert second.sup.target().mode == "rollback" # Reset once, then converged: completes after a lease duration. resets = len(world.frontend.writes) @@ -1222,7 +1386,7 @@ async def test_applied_during_a_pending_rollback_wins() -> None: world = World() first = await leading(world) chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - first.controller.desired = dict(chosen) + first.controller.restore_desired(chosen) enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) await first.sup.outbox_round() await first.sup.stop() @@ -1232,8 +1396,9 @@ async def test_applied_during_a_pending_rollback_wins() -> None: assert world.shared_state().rollback_pending seq = world.shared_state().deploy_seq # An applied decision lands while the rollback is pending. - second.controller.desired = {CREDIT: 1.0, SCALE: 1.0, TEMP: 0.2} - enqueue(second, make_decision(K0 + 2, "applied", chosen=second.controller.desired)) + applied = {CREDIT: 1.0, SCALE: 1.0, TEMP: 0.2} + second.controller.restore_desired(applied) + enqueue(second, make_decision(K0 + 2, "applied", chosen=applied)) await second.sup.outbox_round() state = world.shared_state() assert not state.rollback_pending and state.deploy_seq == seq + 1 @@ -1256,7 +1421,7 @@ async def test_a_stale_rollback_complete_is_discarded(racing: str) -> None: world = World() first = await leading(world) chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - first.controller.desired = dict(chosen) + first.controller.restore_desired(chosen) enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) await first.sup.outbox_round() await first.sup.stop() @@ -1309,7 +1474,7 @@ async def test_advisory_switch_rolls_back_only_what_was_deployed() -> None: world = World() first = await leading(world) chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - first.controller.desired = dict(chosen) + first.controller.restore_desired(chosen) enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) await first.sup.outbox_round() await first.sup.control_round() @@ -1317,7 +1482,8 @@ async def test_advisory_switch_rolls_back_only_what_was_deployed() -> None: second = await leading(world, "p2", advisory=True) state = world.shared_state() assert state.mode == "advisory" and state.rollback_pending - assert second.controller.restored == [] # advisory never restores + # Advisory never restores a deployed point into the controller. + assert second.controller.current_point().router == startup_flat("a") await settle([second], world, rounds=5) state = world.shared_state() assert not state.rollback_pending @@ -1332,13 +1498,13 @@ async def test_advisory_switch_rolls_back_only_what_was_deployed() -> None: async def test_advisory_rollback_leaves_unlatched_routers_alone() -> None: - """§4.8: a router changed at revision > 0 before anyone latched it is - reset to learn its initial only in active mode.""" + """A router changed at revision > 0 before anyone latched it is reset to + learn its initial only in active mode.""" world = World() first = await leading(world) chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - first.controller.desired = dict(chosen) + first.controller.restore_desired(chosen) enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) await first.sup.outbox_round() await first.sup.control_round() @@ -1399,7 +1565,7 @@ async def test_promote_applies_the_last_advisory_recommendation() -> None: state = world.shared_state() assert state.mode == "active" and state.by_wt == by_wt("a", chosen) assert state.history[-1]["outcome"] == "promoted" - assert second.controller.restored == [chosen] + assert second.controller.current_point().router == chosen assert frontend_configs(world) == [by_wt("a", chosen)["aggregated"]] @@ -1444,7 +1610,7 @@ async def test_spec_flip_rebuilds_rolls_back_and_recovers() -> None: process = await leading(world) sup, host = process.sup, process.host chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - process.controller.desired = dict(chosen) + process.controller.restore_desired(chosen) enqueue(process, make_decision(K0 + 1, "applied", chosen=chosen)) await sup.outbox_round() await sup.control_round() @@ -1452,8 +1618,9 @@ async def test_spec_flip_rebuilds_rolls_back_and_recovers() -> None: # P8 breaks (a second frontend without replica sync). world.spec.spec_ok = False await sup.control_round() - assert host.replacements[-1][1:] == (1, K0 + 1) - assert old_controller.closed and sup.generation == 1 + assert host.replacements[-1][1] == 1 and sup.generation == 1 + assert process.controller is not old_controller + assert old_controller._cancel.is_set() # closed state = world.shared_state() assert state.rollback_pending and state.by_wt == VARIANTS["a"] assert sup.target().mode == "rollback" and not sup.ready() @@ -1476,7 +1643,7 @@ async def test_p4_failure_disables_until_it_recovers() -> None: assert not sup.ready() world.spec.p4_ok = True await sup.control_round() - assert host.replacements[-1][0].variant == "a" + assert variant_of(host.replacements[-1][0]) == "a" assert process.controller is not None and sup.generation == 1 await sup.control_round() assert sup.ready() @@ -1645,7 +1812,7 @@ async def test_a_hung_reconciler_never_blocks_revalidation_or_actuation() -> Non # The round runs in its own task; the control loop goes on and never # starts a second round while the first one's call still runs. chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - process.controller.desired = dict(chosen) + process.controller.restore_desired(chosen) enqueue(process, make_decision(K0 + 1, "applied", chosen=chosen)) await sup.outbox_round() await asyncio.wait_for(sup.control_round(), 5) @@ -1753,41 +1920,53 @@ def broken(*args, **kwargs): result = process.sup.sweep_fn(request, cancel=cancel, abort=abort) assert result.status == "failed" and result.failure_reason == "sweep_error" assert "boom" in result.error - no_context = await started(world, "p2", sweep_runner=runner) + no_context = await started(world, "p2", sweep_runner=runner, context_factory=None) result = no_context.sup.sweep_fn(request, cancel=cancel, abort=abort) assert result.status == "failed" -async def test_stop_aborts_waits_for_the_sweep_and_releases() -> None: +async def test_stop_aborts_the_sweep_and_waits_for_the_sweep_thread() -> None: world = World() - entered = threading.Event() - - def runner(request, *, cancel, abort, context): - entered.set() - assert abort.wait(5.0) - return "aborted" + process = await leading(world) + sup, controller = process.sup, process.controller + world.clock.wall = window(K0 + 1)[2] + await controller.on_tick(world.clock.time(), SPEC_COUNTS) + pending = controller._pending + world.sweeps.quiesce([controller]) # running on the sweep thread + abort = pending.abort - process = await leading( - world, sweep_runner=runner, context_factory=lambda sup: None - ) - sup = process.sup - gate = sup.launch_gate(K0 + 1, window(K0 + 1)) - thread = threading.Thread( - target=sup.sweep_fn, - args=(SimpleNamespace(window_index=K0 + 1),), - kwargs={"cancel": threading.Event(), "abort": gate.abort}, - ) - thread.start() - assert entered.wait(5.0) - process.sup._sleep = lambda s: asyncio.sleep(0.01) - process.sup._monotonic = time.monotonic await sup.stop() - thread.join(5.0) - assert gate.abort.is_set() and sup._sweeps_in_flight == 0 + + # drop_pending set the sweep's abort; stop waited for the sweep thread. + assert abort.is_set() and pending.future.done() + assert pending.future.result().failure_reason == "lost_leadership" assert world.holder() is None and not sup.is_leader() await sup.stop() # idempotent +async def test_stop_gives_up_on_a_sweep_that_never_returns(caplog) -> None: + world = World() + release = threading.Event() + + def stuck(request, *, cancel, abort, context): + release.wait(10) # ignores its abort + return SweepResult(status="failed") + + process = await leading( + world, sweep_runner=stuck, settings=settings_for(stop_timeout_s=0.05) + ) + sup, controller = process.sup, process.controller + world.clock.wall = window(K0 + 1)[2] + await controller.on_tick(world.clock.time(), SPEC_COUNTS) + try: + with caplog.at_level(logging.WARNING): + await sup.stop() + assert "a sweep is still running at shutdown" in caplog.text + assert world.holder() is None + finally: + release.set() + + async def test_background_loops_run_and_stop() -> None: world = World() process = make_process(world) @@ -1814,30 +1993,19 @@ async def test_hooks_are_the_adapter_hooks() -> None: world = World() process = await started(world) hooks = process.sup.hooks() - from dynamo.planner.plugins.orchestrator.engine_adapter import RetuneHooks - assert isinstance(hooks, RetuneHooks) assert hooks.launch_gate == process.sup.launch_gate assert hooks.harvest_allowed == process.sup.harvest_allowed assert hooks.sweep_fn == process.sup.sweep_fn and hooks.phase_s == 0.0 -async def test_decommission_rolls_back_then_deletes_everything() -> None: - world = World() - first = await leading(world) - chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - first.controller.desired = dict(chosen) - enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) - await first.sup.outbox_round() - await first.sup.control_round() - await first.sup.stop() - world.kube.create_request( - {"metadata": {"name": "rt-1", "labels": {LABEL_DGD_UID: "uid-1"}}} - ) +def decommission_supervisor(world: World) -> LiveRetuneSupervisor: + """The clean-up supervisor of a planner whose retune was removed.""" + frontend = SimpleNamespace( parse_frontend=lambda dgd, planner_mode: world.spec.startup() ) - sup = bootstrap.build_decommission_supervisor( + return bootstrap.build_decommission_supervisor( live_config(), kube=world.kube, dgd=world.kube.get_dgd(DGD_NAME), @@ -1852,6 +2020,21 @@ async def test_decommission_rolls_back_then_deletes_everything() -> None: sleep=no_sleep, blocking=inline, ) + + +async def test_decommission_rolls_back_then_deletes_everything() -> None: + world = World() + first = await leading(world) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first.controller.restore_desired(chosen) + enqueue(first, make_decision(K0 + 1, "applied", chosen=chosen)) + await first.sup.outbox_round() + await first.sup.control_round() + await first.sup.stop() + world.kube.create_request( + {"metadata": {"name": "rt-1", "labels": {LABEL_DGD_UID: "uid-1"}}} + ) + sup = decommission_supervisor(world) await sup.start(SimpleNamespace(), background=False) for _ in range(6): if sup.decommissioned: @@ -1866,6 +2049,30 @@ async def test_decommission_rolls_back_then_deletes_everything() -> None: await sup.stop() +async def test_decommission_never_deletes_after_losing_its_term() -> None: + world = World() + first = await leading(world, advisory=True) # nothing deployed + await first.sup.stop() + world.kube.create_request( + {"metadata": {"name": "rt-1", "labels": {LABEL_DGD_UID: "uid-1"}}} + ) + sup = decommission_supervisor(world) + await sup.start(SimpleNamespace(), background=False) + await sup.lease_round() + assert sup.initialized and not world.shared_state().rollback_pending + # Another replica claims the state; this round's initials write (a + # restarted frontend) finds out before the clean-up would delete. + store = StateStore(world.kube, sup.dgd) + snapshot = store.read() + store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) + world.frontend.restart(VARIANTS["a"]) + await sup.control_round() + assert not sup.decommissioned and not sup.initialized + assert set(world.kube.requests) == {"rt-1"} + assert set(world.kube.leases) == {STATE_NAME, LEADER_NAME} + await sup.stop() + + # -------------------------------------------------------------- bootstrap @@ -1923,6 +2130,7 @@ def test_build_retune_inputs_agg_and_disagg() -> None: def test_kv_router_resolver_matches_the_router_config() -> None: llm = pytest.importorskip("dynamo.llm") + assert VARIANTS == {variant: by_wt(variant, {}) for variant in ROUTER_CONFIG} resolver = bootstrap.KvRouterResolver({}, ("prefill", "decode")) flat = resolver.startup_flat(KEYS) default = llm.KvRouterConfig.from_json("{}") @@ -1945,19 +2153,17 @@ def __init__(self, startup: Optional[FrontendStartup], p4_ok=True, spec_ok=True) self.startup = startup self.p4_ok = p4_ok self.spec_ok = spec_ok - self.calls: list[dict] = [] def parse_frontend(self, dgd, *, planner_mode): return self.startup if self.startup is not None else "P4: shell-wrapped" - def evaluate_premises( - self, dgd, *, planner_config, startup, parse_error, planner_replicas - ): - self.calls.append({"startup": startup, "parse_error": parse_error}) + def evaluate_premises(self, dgd, *, planner_config): + parsed = self.startup is not None return PremiseReport( - p4_ok=self.p4_ok and startup is not None, - spec_ok=self.spec_ok and self.p4_ok and startup is not None, - reasons=() if startup is not None else (parse_error,), + p4_ok=self.p4_ok and parsed, + spec_ok=self.spec_ok and self.p4_ok and parsed, + reasons=() if parsed else ("P4: shell-wrapped",), + startup=self.startup, ) def served_model_name(self, dgd): @@ -1988,7 +2194,7 @@ def test_evaluate_spec() -> None: frontend=unparsed, ) assert evaluation.inputs is None and evaluation.startup is None - assert unparsed.calls[0]["parse_error"] == "P4: shell-wrapped" + assert evaluation.report.reasons == ("P4: shell-wrapped",) with pytest.raises(ValueError): bootstrap.evaluate_spec( {"metadata": {"name": "x"}}, @@ -2072,38 +2278,27 @@ async def fake_build(config, **kwargs): async def test_build_legacy_cleanup(monkeypatch) -> None: world = World() - from dynamo.planner.retune.live import kube as kube_module - + world.kube.put_dgd(dgd_object(replicas=2)) monkeypatch.setattr( - kube_module.KubernetesRetuneKube, - "from_kube_api", - classmethod(lambda cls, api: world.kube), + bootstrap.frontend_config, + "parse_frontend", + lambda dgd, planner_mode: "P4: no frontend", ) - config = live_config().model_copy(update={"retune": None}) + monkeypatch.setattr(bootstrap, "RetuneMetrics", NullMetrics) host = SimpleNamespace( - config=config, + config=live_config().model_copy(update={"retune": None}), environment=SimpleNamespace( - controller=SimpleNamespace(kube_api=object(), parent_dgd_name=DGD_NAME), + controller=SimpleNamespace(parent_dgd_name=DGD_NAME), + runtime_namespace=lambda: "", ), ) - assert await bootstrap.build_legacy_cleanup(host) is None # nothing left over - StateStore(world.kube, DgdRef(DGD_NAME, "test-ns", "uid-1", "dyn")).write( - RetuneState(dgd_uid="uid-1"), - StateStore(world.kube, DgdRef(DGD_NAME, "test-ns", "uid-1", "dyn")).read(), - ) - monkeypatch.setattr( - "dynamo.planner.retune.live.frontend_config.parse_frontend", - lambda dgd, planner_mode: "P4: no frontend", - ) - monkeypatch.setattr( - "dynamo.planner.retune.live.metrics.RetuneMetrics", lambda: NullMetrics() - ) - sup = await bootstrap.build_legacy_cleanup(host) - assert sup is not None and sup._settings.decommission - world.kube.fail("get_lease", KubeError("forbidden")) - assert await bootstrap.build_legacy_cleanup(host) is None - host.config = live_config() - assert await bootstrap.build_legacy_cleanup(host) is None + sup = await bootstrap.build_legacy_cleanup(host, world.kube) + assert sup._settings.decommission and sup.retune_inputs() is None + # The planner's configured namespace when the runtime has none. + assert sup.dgd == DgdRef(DGD_NAME, "test-ns", "uid-1", host.config.namespace) + world.kube.fail("get_dgd", KubeError("forbidden")) + with pytest.raises(KubeError): + await bootstrap.build_legacy_cleanup(host, world.kube) class PlannerEnvironment: @@ -2131,41 +2326,60 @@ async def shutdown(self) -> None: class FakeCleanup: - def __init__(self, calls: list[str], start_error: Optional[Exception] = None): + def __init__(self, calls: list[str]): self.calls = calls - self.start_error = start_error self.host = None async def start(self, host) -> None: self.calls.append("cleanup.start") self.host = host - if self.start_error is not None: - raise self.start_error async def stop(self) -> None: self.calls.append("cleanup.stop") -def planner_without_retune(monkeypatch, controller: Any, build): - """A ``NativePlannerBase`` without retune config whose legacy clean-up - is built by ``build(host)``; the engine bootstrap is a no-op.""" +def forbidden() -> KubeError: + error = KubeError("read leases/graph-planner-retune: HTTP 403: forbidden") + error.status = 403 # type: ignore[attr-defined] + return error + + +def planner_without_retune(monkeypatch, controller: Any, *, leases=(), build=None): + """A ``NativePlannerBase`` without retune config; the engine bootstrap + is a no-op. ``leases`` is what the Lease check finds (or raises) and + ``build(host, kube)`` builds the clean-up.""" from unittest.mock import patch - from dynamo.planner.core.base import NativePlannerBase + from dynamo.planner.core import base as base_module config = live_config( metric_reporting_prometheus_port=0, live_dashboard_port=0, control_api_port=0 ).model_copy(update={"retune": None}) environment = PlannerEnvironment(controller) + kube = object() - async def build_legacy_cleanup(host): + def find_retune_leases(found_kube, dgd_name, *, timeout_s): + assert found_kube is kube and dgd_name == DGD_NAME + environment.calls.append("leases.check") + if isinstance(leases, Exception): + raise leases + return tuple(leases) + + async def build_legacy_cleanup(host, found_kube, *, timeout_s): + assert found_kube is kube environment.calls.append("cleanup.build") return await build(host) + monkeypatch.setattr(base_module, "find_retune_leases", find_retune_leases) + monkeypatch.setattr( + base_module.KubernetesRetuneKube, + "from_kube_api", + classmethod(lambda cls, api: kube), + ) monkeypatch.setattr(bootstrap, "build_legacy_cleanup", build_legacy_cleanup) with patch("dynamo.planner.core.base.PlannerPrometheusMetrics"): - planner = NativePlannerBase(None, config, environment) + planner = base_module.NativePlannerBase(None, config, environment) async def bootstrap_engine(): environment.calls.append("engine") @@ -2183,10 +2397,15 @@ def kubernetes_connector() -> Any: from dynamo.planner.connectors.kubernetes import KubernetesConnector - return MagicMock(spec=KubernetesConnector) + connector = MagicMock(spec=KubernetesConnector) + connector.kube_api = object() + connector.parent_dgd_name = DGD_NAME + return connector -async def test_a_planner_without_retune_runs_the_legacy_cleanup(monkeypatch) -> None: +async def test_a_planner_without_retune_cleans_up_in_the_background( + monkeypatch, +) -> None: cleanups: list[FakeCleanup] = [] async def build(host): @@ -2194,15 +2413,17 @@ async def build(host): return cleanups[-1] planner, environment = planner_without_retune( - monkeypatch, kubernetes_connector(), build + monkeypatch, kubernetes_connector(), leases=(STATE_NAME,), build=build ) await planner._async_init() try: - # Built before the engine, started after it. - assert environment.calls == [ - "environment.initialize", + # Startup never waits for the check. + assert environment.calls == ["environment.initialize", "engine"] + assert planner.may_hold_retune_lease + await planner._legacy_retune_task + assert environment.calls[2:] == [ + "leases.check", "cleanup.build", - "engine", "cleanup.start", ] assert cleanups[0].host is planner and planner._live_retune is None @@ -2212,57 +2433,60 @@ async def build(host): assert planner._legacy_retune_cleanup is None -@pytest.mark.parametrize( - "outcome", - ["nothing_left", "forbidden", "not_found", "setup_error", "start_error"], -) -async def test_the_legacy_cleanup_never_fails_startup( +@pytest.mark.parametrize("outcome", ["nothing_left", "forbidden", "not_found", "bug"]) +async def test_the_legacy_check_never_fails_the_planner( monkeypatch, caplog, outcome ) -> None: + from dynamo.planner.core import base as base_module from dynamo.planner.retune.live.contracts import KubeNotFound + leases: Any = (LEADER_NAME,) + if outcome == "nothing_left": + leases = () + elif outcome == "forbidden": + leases = forbidden() + async def build(host): - if outcome == "nothing_left": - return None - if outcome == "forbidden": - raise KubeError("403 Forbidden: leases is forbidden") if outcome == "not_found": - raise KubeNotFound("404 Not Found") - if outcome == "setup_error": - raise RuntimeError("no kube config") - return FakeCleanup(environment.calls, start_error=RuntimeError("boom")) + raise KubeNotFound("read dynamographdeployments/graph: HTTP 404") + raise RuntimeError("a bug") + if outcome in ("nothing_left", "forbidden"): + # The live retune stack is never loaded. + monkeypatch.setattr(base_module, "_live_retune_bootstrap", None) planner, environment = planner_without_retune( - monkeypatch, kubernetes_connector(), build + monkeypatch, kubernetes_connector(), leases=leases, build=build ) - with caplog.at_level("INFO", logger="dynamo.planner.core.base"): + with caplog.at_level(logging.DEBUG, logger="dynamo.planner.core.base"): await planner._async_init() - try: - assert "engine" in environment.calls - assert planner._legacy_retune_cleanup is None - assert not [r for r in caplog.records if r.levelno >= logging.WARNING] - if outcome == "start_error": - assert environment.calls[-2:] == ["cleanup.start", "cleanup.stop"] - assert "legacy clean-up did not start" in caplog.text - elif outcome != "nothing_left": - assert "legacy clean-up skipped" in caplog.text - finally: - await planner._shutdown_runtime() - assert environment.calls.count("cleanup.stop") == (outcome == "start_error") - + try: + await asyncio.wait([planner._legacy_retune_task]) + assert planner._legacy_retune_cleanup is None + records = [r for r in caplog.records if "legacy clean-up" in r.message] + levels = {r.levelname for r in records} + if outcome == "nothing_left": + assert records == [] + elif outcome == "forbidden": + assert levels == {"DEBUG"} # the default chart: no Lease RBAC + elif outcome == "not_found": + assert levels == {"INFO"} and "skipped" in caplog.text + else: + assert levels == {"ERROR"} and "a bug" in caplog.text + finally: + await planner._shutdown_runtime() + assert "cleanup.stop" not in environment.calls -async def test_the_legacy_cleanup_is_kubernetes_only(monkeypatch) -> None: - async def build(host): - raise AssertionError("not a Kubernetes planner") +async def test_the_legacy_check_is_kubernetes_only(monkeypatch) -> None: for controller in (None, SimpleNamespace(kube_api=object())): - planner, environment = planner_without_retune(monkeypatch, controller, build) + planner, environment = planner_without_retune(monkeypatch, controller) await planner._async_init() try: - assert "cleanup.build" not in environment.calls - assert planner._legacy_retune_cleanup is None + assert planner._legacy_retune_task is None + assert not planner.may_hold_retune_lease finally: await planner._shutdown_runtime() + assert "leases.check" not in environment.calls async def test_dgdr_canary_and_context() -> None: @@ -2288,172 +2512,189 @@ async def test_dgdr_canary_and_context() -> None: assert context.is_leader() is True -# ------------------------------------------- with the real retune controller +# ------------------------------------------- initials before router writes -async def test_real_controller_launches_harvests_and_persists() -> None: - """The supervisor's hooks drive ``RetuneController`` end to end: gate, - launch with the spec replicas and the abort event, sweep_fn, harvest - gating, on_effects, outbox and the persisted deployment.""" +class FrontendPolicyClient: + """``RouterPolicyClient`` of the real ``RouterActuator`` over the + world's ``FakeFrontend`` (every pod answers with all of its routers).""" - pytest.importorskip("aisimulate.sweeper") - from dynamo.planner.core.types import EngineCapabilities, WorkerCapabilities - from dynamo.planner.plugins.clock import WallClock - from dynamo.planner.retune.controller import RetuneController - from dynamo.planner.retune.sweep import SweepResult + def __init__(self, world: World, writer: str) -> None: + self.frontend = world.frontend + self.dynamo_namespace = world.spec.dynamo_namespace + self.writer = writer + + def _info(self, router: Router) -> RouterInfo: + return RouterInfo( + router_id=router.router_id, + namespace=self.dynamo_namespace, + worker_type=router.worker_type, + revision=router.revision, + configuration=dict(router.config), + model="model", + ) + + async def get(self, pod) -> PolicyCallResult: + routers = tuple(self._info(r) for r in self.frontend.routers.values()) + return PolicyCallResult(status=200, routers=routers) + + async def patch(self, pod, router_id, expected_revision, parameters): + router = self.frontend.routers[router_id] + self.frontend.patch(router, dict(parameters), self.writer, False, "apply") + return PolicyCallResult(status=200, router=self._info(router)) + + async def reset(self, pod, router_id, expected_revision): + router = self.frontend.routers[router_id] + self.frontend.reset(router, self.writer, False, "rollback") + return PolicyCallResult(status=200, router=self._info(router)) + + +def real_actuators(world: World, writer: str): + world.kube.add_pod("frontend-0", dgd=DGD_NAME) + return lambda dgd, maps: RouterActuator( + world.kube, FrontendPolicyClient(world, writer), dgd, maps, NullMetrics() + ) + + +async def test_a_stop_before_the_initials_write_leaves_no_router_retuned() -> None: + """A router seen at revision 0 is latched and would be PATCHed in the same + round. Stopped (SIGTERM) before the state records its initial, the next + planner rolls back with an advisory target, which never resets a router + whose initial it does not know: so no router may be PATCHed before its + initial is persisted.""" - config = live_config() - retune = config.retune - interval = float(retune.interval_seconds) - grace = float(retune.grace_seconds) world = World() - startup = FrontendStartup( - router_mode="kv_router", - router_config={}, - startup_maps={}, - frontend_component="Frontend", - replicas=1, - replica_sync=False, - fingerprint_source="real", + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + first = await started(world, actuator_factory=real_actuators(world, "p1")) + store = StateStore(world.kube, first.sup.dgd) + await first.sup.lease_round() + snapshot = store.read() + # An active deployment is persisted, and the routers restart before the + # actuator has latched them (revision 0, initials unknown). + store.write( + dataclasses.replace( + snapshot.state, flat=chosen, by_wt=by_wt("a", chosen), deploy_seq=2 + ), + snapshot, ) - inputs = bootstrap.build_retune_inputs( - config, startup, dgd_name=DGD_NAME, model_name=None + first.sup._snapshot = store.read() + world.kube.fail( + "replace_lease", + KubeError("timeout"), + match=lambda target: target[0] == STATE_NAME, ) - resolver = bootstrap.KvRouterResolver({}, ("aggregated",)) - keys = bootstrap.searched_router_keys(config) - startup = dataclasses.replace( - startup, startup_maps=resolver.by_wt(resolver.startup_flat(keys)) + await first.sup.control_round() + assert world.frontend.writes == [] # held: the initial is not persisted + assert world.shared_state().initials == {} + await first.sup.stop() + + second = await leading( + world, "p2", advisory=True, actuator_factory=real_actuators(world, "p2") ) - world.frontend.restart(startup.startup_maps) - evaluation = SpecEvaluation( - report=PremiseReport(p4_ok=True, spec_ok=True), - startup=startup, - inputs=inputs, - spec_replicas={"aggregated": 2}, - dgd=DgdRef(DGD_NAME, "test-ns", "uid-1", "dyn"), + await settle([second], world, rounds=4) + assert frontend_configs(world) == [VARIANTS["a"]["aggregated"]] + assert not world.shared_state().rollback_pending + assert world.frontend.writes == [] + + +async def test_initials_are_persisted_before_the_routers_are_patched() -> None: + world = World() + process = await leading(world, actuator_factory=real_actuators(world, "p1")) + chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} + process.controller.restore_desired(chosen) + enqueue(process, make_decision(K0 + 1, "applied", chosen=chosen)) + await process.sup.outbox_round() + world.frontend.restart(VARIANTS["a"]) # new routers at revision 0 + (router_id,) = world.frontend.routers + patched_with: list[dict] = [] + original = world.frontend.patch + + def patch(router, params, *args): + patched_with.append(dict(world.shared_state().initials)) + original(router, params, *args) + + world.frontend.patch = patch # type: ignore[method-assign] + # One round: latch, persist the initial, then PATCH. + await process.sup.control_round() + assert patched_with and all(router_id in seen for seen in patched_with) + assert frontend_configs(world) == [by_wt("a", chosen)["aggregated"]] + assert process.sup.last_report.converged + + +async def test_a_state_of_an_unknown_format_version_is_never_overwritten() -> None: + world = World() + newer = json.dumps({"version": 99, "dgd_uid": "uid-1", "anything": "new"}) + world.kube.create_lease( + { + "metadata": {"name": STATE_NAME, "annotations": {STATE_ANNOTATION: newer}}, + "spec": {}, + } ) - requests = [] + process = await started(world) + for _ in range(2): + await process.sup.lease_round() + assert process.sup.is_leader() and not process.sup.initialized + lease = world.kube.leases[STATE_NAME] + assert lease["metadata"]["annotations"][STATE_ANNOTATION] == newer + assert process.metrics.of("failure") == ["state_version", "state_version"] + + +# ------------------------------------------- with the real retune controller + + +async def test_real_controller_launches_harvests_and_persists() -> None: + """The supervisor's hooks drive ``RetuneController`` end to end: gate, + launch with the spec replicas and the abort event, sweep_fn, harvest + gating, on_effects, outbox and the persisted deployment.""" + + world = World() + process = await leading(world) + sup, controller = process.sup, process.controller + assert sup.ready() and controller.router_keys == KEYS best = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - def runner(request, *, cancel, abort, context): - requests.append((request, abort)) - return SweepResult( - status="ok", - num_requests=100, - scores={ - candidate.key(): (2.0 if candidate.router == best else 1.0) - for candidate in request.candidates - }, - wall_s=1.0, - details={"runtime_image": "img:1"}, + async def tick(now_s): + world.clock.wall = now_s + result = await controller.on_tick(now_s, SPEC_COUNTS) + sup.on_effects( + SimpleNamespace( + retune_decisions=result.decisions, + router_parameters=result.router_parameters, + ) ) + await sup.outbox_round() + return result - settings = SupervisorSettings.from_config(config, router_keys=keys) - sup = LiveRetuneSupervisor( - settings=settings, - kube=world.kube, - evaluation=evaluation, - identity="p1/1", - premise_evaluator=lambda dgd: evaluation, - actuator_factory=lambda dgd, maps: FakeActuator( - world.frontend, "p1", False, maps - ), - fingerprint_fn=lambda s: "fp-real", - resolver_factory=lambda s: resolver, - metrics=RecordingMetrics(), - sweep_runner=runner, - context_factory=lambda supervisor: None, - clock=world.clock.time, - monotonic=world.clock.monotonic, - sleep=no_sleep, - blocking=inline, - ) - hooks = sup.hooks() - controller = RetuneController( - config, - inputs, - WorkerCapabilities( - decode=EngineCapabilities( - num_gpu=1, max_num_batched_tokens=2048, max_kv_tokens=16384 - ) - ), - WallClock(), - float(config.scheduling.scale_interval_seconds), - sweep_fn=hooks.sweep_fn, - launch_gate=hooks.launch_gate, - harvest_allowed=hooks.harvest_allowed, - generation=0, - ) - host = SimpleNamespace( - _engine=SimpleNamespace(retune_controller=controller), - _config_lock=asyncio.Lock(), - ) - try: - await sup.start(host, background=False) - controller.start(world.clock.time(), hooks.phase_s) - await sup.lease_round() - await sup.control_round() - assert sup.ready() and controller.router_keys == keys - - async def tick(now_s): - result = await controller.on_tick(now_s, None) - sup.on_effects( - SimpleNamespace( - retune_decisions=result.decisions, - router_parameters=result.router_parameters, - ) - ) - await sup.outbox_round() - return result - - # First enable recorded the window in progress at start as decided: - # neither it (its trace began mid-way) nor the due window launches. - now = world.clock.time() - first_due = math.floor((now - grace) / interval + 1e-9) - 1 - in_progress = math.floor(now / interval) - assert in_progress == first_due + 1 - await tick(now) - assert requests == [] - world.clock.wall = (in_progress + 1) * interval + grace - await tick(world.clock.wall) - assert requests == [] - # The next window launches at its L_k with the spec replicas. - launch_at = (in_progress + 2) * interval + grace - world.clock.wall = launch_at - await tick(launch_at) - for _ in range(200): - if requests: - break - await asyncio.sleep(0.01) - request, abort = requests[0] - assert request.window_index == in_progress + 1 - assert request.spec_replicas == {"aggregated": 2} and abort is not None - assert request.generation == 0 - # Harvested once done: applied, persisted, then deployed. - decided = None - for step in range(1, 200): - result = await tick(launch_at + step * 0.05 + 1000.0) - if result.decisions: - decided = result.decisions[0] - break - await asyncio.sleep(0.01) - assert decided is not None and decided.outcome == "applied" - assert decided.decision_id == f"retune-w{int(request.window_end_ms)}" - state = world.shared_state() - assert state.flat == best and state.deploy_seq == 2 - assert state.last_window_end_ms == int(request.window_end_ms) - assert state.by_wt == resolver.by_wt(best) - assert state.history[-1]["image"] == {"runtime_image": "img:1"} - await sup.control_round() - assert frontend_configs(world) == [resolver.by_wt(best)["aggregated"]] - # A restart restores the persisted point into a fresh controller. - controller.restore_desired(resolver.startup_flat(keys)) - sup._request_reinit("test") - await sup.lease_round() - assert controller.current_point().router == best - finally: - controller.close() - await sup.stop() + # First enable recorded the window in progress at start as decided: + # neither it (its trace began mid-way) nor the due window launches. + await tick(START_WALL) + await tick(window(K0)[2]) + assert controller._pending is None and world.sweeps.requests == [] + # The next window launches at its L_k with the spec replicas. + await tick(window(K0 + 1)[2]) + pending = controller._pending + assert pending is not None and pending.abort is not None + assert world.sweeps.finish(controller, scored(best)) + (request,) = world.sweeps.requests + assert request.window_index == K0 + 1 and request.generation == 0 + assert request.spec_replicas == {"aggregated": 2} and request.replicas_verified + # Harvested on the first tick at least a scale interval after the launch: + # applied, persisted, then deployed. + result = await tick(window(K0 + 1)[2] + 5.0) + (decided,) = result.decisions + assert decided.outcome == "applied" + assert decided.decision_id == f"retune-w{end_ms(K0 + 1)}" + state = world.shared_state() + assert state.flat == best and state.deploy_seq == 2 + assert state.last_window_end_ms == end_ms(K0 + 1) + assert state.by_wt == by_wt("a", best) + await sup.control_round() + assert frontend_configs(world) == [by_wt("a", best)["aggregated"]] + # A re-initialization restores the persisted point into the controller. + controller.restore_desired(startup_flat("a")) + sup._request_reinit("test") + await sup.lease_round() + assert controller.current_point().router == best # =================================================== interleaving harness @@ -2461,10 +2702,11 @@ async def tick(now_s): # Two planner processes share a FakeRetuneKube, a frontend and a clock. A # seeded random schedule interleaves lease renewals and failures, takeovers, # state writes that succeed, conflict, time out, time out but land, or land -# late, ticks that launch and harvest sweeps, frontend restarts and manual -# patches, spec changes (startup, spec_ok, P4), advisory switches, SIGTERM -# and SIGKILL. After every step the §6 invariants are asserted; at the end -# the system settles with no infinite resets or PATCHes. +# late, ticks of the real controllers that launch and harvest sweeps, +# frontend restarts and manual patches, spec changes (startup, spec_ok, P4), +# advisory switches, SIGTERM and SIGKILL. After every step the invariants +# are asserted; at the end the system settles with no infinite resets or +# PATCHes. class CheckedKube(FakeRetuneKube): @@ -2598,8 +2840,10 @@ def __init__(self, seed: int, *, advisory_only: bool = False) -> None: self.steps = 0 self.transients: dict[str, int] = {} # Monotonic time until which two processes may both believe they - # lead (the lease was taken over without expiring, §4.7.9). + # lead (the lease was taken over without expiring). self.stolen_until = -math.inf + self.launched = 0 + self.decided: list[str] = [] # -- processes -- @@ -2622,6 +2866,47 @@ async def spawn(self, slot: str) -> None: def live(self) -> list[Process]: return [p for p in self.processes.values() if p.alive] + # -- the real controllers -- + + async def tick(self, process: Process) -> None: + controller = process.controller + if controller is None: + return + had = controller._pending + result = await controller.on_tick(self.world.clock.time(), SPEC_COUNTS) + if controller._pending is not None and controller._pending is not had: + self.launched += 1 + for event in result.audit_events: + if event.startswith("retune:dropped w="): + controller.dropped.append(int(event.split("=", 1)[1])) + self.decided.extend(decision.outcome for decision in result.decisions) + process.sup.on_effects( + SimpleNamespace( + retune_decisions=result.decisions, + router_parameters=result.router_parameters, + ) + ) + + def finish_sweep(self, process: Process) -> None: + """The process's sweep returns: failed, flat (kept) or with a best + candidate (applied, or advisory).""" + + controller = process.controller + if controller is None: + return + roll = self.rng.random() + if roll < 0.15: + self.world.sweeps.finish(controller, failed) + return + best = None + if roll >= 0.35: + best = {key: self.rng.choice(values) for key, values in DOMAIN.items()} + self.world.sweeps.finish(controller, scored(best)) + + def quiesce(self) -> None: + controllers = [p.controller for p in self.live() if p.controller is not None] + self.world.sweeps.quiesce(controllers) + # -- events -- def _restore_transients(self) -> None: @@ -2670,17 +2955,10 @@ async def step(self) -> str: await sup.outbox_round() return f"outbox {slot}" if roll < 0.56: - controller = process.controller - if controller is not None and not controller.closed: - effects = controller.tick( - sup, self.world.clock.time(), rng, process.advisory - ) - sup.on_effects(effects) + await self.tick(process) return f"tick {slot}" if roll < 0.64: - controller = process.controller - if controller is not None and controller.pending is not None: - controller.pending["done"] = True + self.finish_sweep(process) return f"sweep_done {slot}" if roll < 0.78: await self.advance(rng.choice([5, 20, 20, 40, 60, 120, 240])) @@ -2716,10 +2994,10 @@ async def step(self) -> str: self._transient("p4_broken", self.world.spec, "p4_ok", False) return "p4_break" if roll < 0.945: - controller = process.controller - if controller is not None: - controller.close() + # SIGTERM: the supervisor stops while the engine is up, then the + # engine closes the controller. await sup.stop() + process.host.close() process.alive = False return f"sigterm {slot}" if roll < 0.951: @@ -2758,6 +3036,7 @@ async def advance(self, seconds: float) -> None: while remaining > 0: chunk = min(20.0, remaining) remaining -= chunk + self.quiesce() self.world.clock.advance(chunk) self._restore_transients() for process in self.live(): @@ -2768,16 +3047,11 @@ async def advance(self, seconds: float) -> None: await sup.outbox_round() if rng.random() < 0.6: await sup.control_round() - controller = process.controller - if controller is not None and not controller.closed: - if controller.pending is not None and rng.random() < 0.3: - controller.pending["done"] = True - if rng.random() < 0.6: - sup.on_effects( - controller.tick( - sup, self.world.clock.time(), rng, process.advisory - ) - ) + if rng.random() < 0.3: + self.finish_sweep(process) + if rng.random() < 0.6: + await self.tick(process) + self.quiesce() self.check("advance") # -- invariants -- @@ -2809,10 +3083,10 @@ def check(self, event: str) -> None: ) for k in controller.dropped: assert ( - controller.last_k < k - or controller.pending is not None + controller._last_k < k + or controller._pending is not None or k <= decided - ), (event, process.name, k, controller.last_k) + ), (event, process.name, k, controller._last_k) controller.dropped.clear() # An advisory planner never PATCHes, and resets only to finish a # rollback of what was deployed. @@ -2828,16 +3102,16 @@ def check(self, event: str) -> None: if snapshot.resource_version != lease["metadata"]["resourceVersion"]: continue # a late write landed; the next write re-reads assert shared.leader_term == sup.term, event - # State, controller baseline and target agree. + # State, controller point and target agree. if ( shared.mode == "active" and not shared.rollback_pending and sup.spec_ok and controller is not None ): - assert controller.desired == shared.flat, ( + assert controller.current_point().router == shared.flat, ( event, - controller.desired, + controller.current_point().router, shared.flat, ) target = sup.target() @@ -2864,6 +3138,7 @@ async def settle(self) -> None: await self.spawn("p1") live = self.live() for _ in range(12): + self.quiesce() world.clock.advance(20) for process in live: await process.sup.lease_round() @@ -2897,10 +3172,12 @@ async def run(self, steps: int) -> None: await self.spawn("p2") for _ in range(steps): event = await self.step() + self.quiesce() self.check(event) await self.settle() +@pytest.mark.timeout(120) @pytest.mark.parametrize("seed", range(24)) async def test_interleavings_keep_the_invariants(seed: int) -> None: harness = Harness(seed) @@ -2908,8 +3185,17 @@ async def test_interleavings_keep_the_invariants(seed: int) -> None: assert harness.world.kube.state_writes > 0 +@pytest.mark.timeout(120) @pytest.mark.parametrize("seed", range(6)) async def test_advisory_only_never_writes_to_routers(seed: int) -> None: harness = Harness(1000 + seed, advisory_only=True) await harness.run(steps=250) assert harness.world.frontend.writes == [] + + +async def test_the_interleavings_launch_and_decide_windows() -> None: + # The schedule reaches the real controller's launch and harvest paths. + harness = Harness(0) + await harness.run(steps=300) + assert harness.launched > 0 + assert {"applied", "kept", "failed"} <= set(harness.decided) diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index d7b5c5f2e2a2..39d76c2f182b 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -549,6 +549,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.enabled | bool | `true` | Whether to enable the Dynamo Kubernetes operator deployment | | dynamo-operator.upgradeCRD | bool | `true` | Whether the cluster-wide operator applies CRDs from its image through the crd-apply init container. When false, the chart installs no CRDs; apply them separately before starting a cluster-wide operator. Namespace-restricted installations must set this to false and use the CRDs managed by the cluster-wide operator. | | dynamo-operator.dgdrV1Beta2.enabled | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: Serve the unfinished DGDR v1beta2 Request API with conversion and install the Run and Candidate APIs. Run and Candidate objects are persisted in etcd; incompatible schema changes may make them unreadable or block future CRD upgrades. When true, the operator reconciles v1beta2 Requests into Runs and executes each Run as an AISimulate planner Job with a publisher sidecar that persists deployment Candidates. | +| dynamo-operator.dgdrV1Beta2.plannerLiveRetune | bool | `false` | EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: With dgdrV1Beta2.enabled, grant every Planner ServiceAccount what Planner live retune (retune.executor: dgdr) needs: create, read and delete DGDR v1beta2 Requests, read their Runs and Candidates, and read, create, update and delete any Lease in the namespace. | | dynamo-operator.natsAddr | string | `""` | NATS server address for operator communication. When empty, the operator uses bundled NATS only if global.nats.install=true; otherwise NATS is not configured. Format: `nats://hostname:4222` | | dynamo-operator.etcdAddr | string | `""` | etcd server address for an external etcd instance. Only needed when using external etcd without the bundled subchart. Format: `http://hostname:2379` or `https://hostname:2379` | | dynamo-operator.modelExpressURL | string | `""` | URL for the Model Express server if not deployed by this helm chart. This is ignored if Model Express server is installed by this helm chart (global.model-express.enabled is true). | diff --git a/deploy/helm/charts/platform/components/operator/templates/planner.yaml b/deploy/helm/charts/platform/components/operator/templates/planner.yaml index b6d69b7620fc..60a67ba8874c 100644 --- a/deploy/helm/charts/platform/components/operator/templates/planner.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/planner.yaml @@ -57,17 +57,20 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["list"] -{{- if .Values.dgdrV1Beta2.enabled }} +{{- if and .Values.dgdrV1Beta2.enabled .Values.dgdrV1Beta2.plannerLiveRetune }} # EXPERIMENTAL live retune (planner retune.executor: dgdr): sweep through DGDR -# v1beta2 Requests, read their Runs and Candidates, and hold the leadership -# and state Leases (read, create, replace and delete; never patch). Lease names -# are per DGD, so the Lease rule covers every Lease in the namespace. +# v1beta2 Requests, read their Runs and list their Candidates, and hold the +# leadership and state Leases (read, create, replace and delete; never patch). +# Lease names are per DGD, so the Lease rule covers every Lease in the namespace. - apiGroups: ["nvidia.com"] resources: ["dynamographdeploymentrequests"] - verbs: ["create", "get", "list", "watch", "delete"] + verbs: ["create", "get", "list", "delete"] - apiGroups: ["nvidia.com"] - resources: ["dynamographdeploymentruns", "dynamographdeploymentcandidates"] - verbs: ["get", "list", "watch"] + resources: ["dynamographdeploymentruns"] + verbs: ["get"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentcandidates"] + verbs: ["list"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "create", "update", "delete"] @@ -94,8 +97,9 @@ roleRef: # RoleBinding (internal/rbac/manager.go EnsureServiceAccountWithRBAC), so all # permissions — including pods/list — are effective only in the DGD's namespace. # RBAC escalation prevention lets the operator bind only permissions it holds, -# so every rule here must also be in the operator's role (with dgdrV1Beta2, the -# DGDR v1beta2 Run and Candidate rules of the operator's v1beta2 integration). +# so every rule here must also be in the operator's role (with +# dgdrV1Beta2.plannerLiveRetune, the DGDR v1beta2 Run and Candidate rules of +# the operator's v1beta2 integration). --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -125,17 +129,20 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["list"] -{{- if .Values.dgdrV1Beta2.enabled }} +{{- if and .Values.dgdrV1Beta2.enabled .Values.dgdrV1Beta2.plannerLiveRetune }} # EXPERIMENTAL live retune (planner retune.executor: dgdr): sweep through DGDR -# v1beta2 Requests, read their Runs and Candidates, and hold the leadership -# and state Leases (read, create, replace and delete; never patch). Lease names -# are per DGD, so the Lease rule covers every Lease in the namespace. +# v1beta2 Requests, read their Runs and list their Candidates, and hold the +# leadership and state Leases (read, create, replace and delete; never patch). +# Lease names are per DGD, so the Lease rule covers every Lease in the namespace. - apiGroups: ["nvidia.com"] resources: ["dynamographdeploymentrequests"] - verbs: ["create", "get", "list", "watch", "delete"] + verbs: ["create", "get", "list", "delete"] - apiGroups: ["nvidia.com"] - resources: ["dynamographdeploymentruns", "dynamographdeploymentcandidates"] - verbs: ["get", "list", "watch"] + resources: ["dynamographdeploymentruns"] + verbs: ["get"] +- apiGroups: ["nvidia.com"] + resources: ["dynamographdeploymentcandidates"] + verbs: ["list"] - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "create", "update", "delete"] diff --git a/deploy/helm/charts/platform/components/operator/values.yaml b/deploy/helm/charts/platform/components/operator/values.yaml index 269173f55baa..c8f6056082f7 100644 --- a/deploy/helm/charts/platform/components/operator/values.yaml +++ b/deploy/helm/charts/platform/components/operator/values.yaml @@ -40,6 +40,11 @@ dgdrV1Beta2: # Requests into Runs and executes each Run as an AISimulate planner Job with # a publisher sidecar that persists deployment Candidates. enabled: false + # -- EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: With dgdrV1Beta2.enabled, grant every + # Planner ServiceAccount what Planner live retune (retune.executor: dgdr) needs: + # create, read and delete DGDR v1beta2 Requests, read their Runs and Candidates, + # and read, create, update and delete any Lease in the namespace. + plannerLiveRetune: false # Environment variables to pass to operator Deployment. env: [] diff --git a/deploy/helm/charts/platform/tests/planner_rbac_test.yaml b/deploy/helm/charts/platform/tests/planner_rbac_test.yaml index 833743ebc58e..9e3cf99a18cf 100644 --- a/deploy/helm/charts/platform/tests/planner_rbac_test.yaml +++ b/deploy/helm/charts/platform/tests/planner_rbac_test.yaml @@ -115,10 +115,23 @@ tests: - patch documentIndex: 1 - # ── EXPERIMENTAL live retune (dgdrV1Beta2.enabled) ──────────────────────────── + # ── EXPERIMENTAL live retune (dgdrV1Beta2.plannerLiveRetune) ──────────────── - - it: omits live retune rules from the ClusterRole when dgdrV1Beta2 is disabled (default) + - it: omits live retune rules from the ClusterRole by default asserts: + - notContains: + path: rules + content: + apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - delete + documentIndex: 0 - notContains: path: rules content: @@ -130,9 +143,13 @@ tests: - create - get - list - - watch - delete documentIndex: 0 + + - it: omits live retune rules from the ClusterRole with dgdrV1Beta2 alone + set: + dynamo-operator.dgdrV1Beta2.enabled: true + asserts: - notContains: path: rules content: @@ -146,10 +163,55 @@ tests: - update - delete documentIndex: 0 + - notContains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentrequests + verbs: + - create + - get + - list + - delete + documentIndex: 0 - - it: grants DGDR v1beta2 and Lease access in the ClusterRole when dgdrV1Beta2 is enabled + - it: omits live retune rules from the ClusterRole with plannerLiveRetune alone + set: + dynamo-operator.dgdrV1Beta2.plannerLiveRetune: true + asserts: + - notContains: + path: rules + content: + apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - delete + documentIndex: 0 + - notContains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentrequests + verbs: + - create + - get + - list + - delete + documentIndex: 0 + + - it: grants DGDR v1beta2 and Lease access in the ClusterRole with plannerLiveRetune set: dynamo-operator.dgdrV1Beta2.enabled: true + dynamo-operator.dgdrV1Beta2.plannerLiveRetune: true asserts: - contains: path: rules @@ -162,7 +224,6 @@ tests: - create - get - list - - watch - delete documentIndex: 0 - contains: @@ -172,11 +233,18 @@ tests: - nvidia.com resources: - dynamographdeploymentruns - - dynamographdeploymentcandidates verbs: - get + documentIndex: 0 + - contains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentcandidates + verbs: - list - - watch documentIndex: 0 - contains: path: rules @@ -201,7 +269,13 @@ tests: verbs: - list documentIndex: 0 - # The planner reads, creates, replaces and deletes Leases; it never patches them. + + - it: omits live retune rules from the Role with dgdrV1Beta2 alone in namespace-restricted mode + set: + dynamo-operator.namespaceRestriction.enabled: true + dynamo-operator.upgradeCRD: false + dynamo-operator.dgdrV1Beta2.enabled: true + asserts: - notContains: path: rules content: @@ -213,34 +287,28 @@ tests: - get - create - update - - patch - delete - documentIndex: 0 - - - it: omits live retune rules from the Role when dgdrV1Beta2 is disabled in namespace-restricted mode - set: - dynamo-operator.namespaceRestriction.enabled: true - dynamo-operator.upgradeCRD: false - asserts: + documentIndex: 1 - notContains: path: rules content: apiGroups: - - coordination.k8s.io + - nvidia.com resources: - - leases + - dynamographdeploymentrequests verbs: - - get - create - - update + - get + - list - delete documentIndex: 1 - - it: grants DGDR v1beta2 and Lease access in the Role when dgdrV1Beta2 is enabled in namespace-restricted mode + - it: grants DGDR v1beta2 and Lease access in the Role with plannerLiveRetune in namespace-restricted mode set: dynamo-operator.namespaceRestriction.enabled: true dynamo-operator.upgradeCRD: false dynamo-operator.dgdrV1Beta2.enabled: true + dynamo-operator.dgdrV1Beta2.plannerLiveRetune: true asserts: - contains: path: rules @@ -253,7 +321,6 @@ tests: - create - get - list - - watch - delete documentIndex: 1 - contains: @@ -263,11 +330,18 @@ tests: - nvidia.com resources: - dynamographdeploymentruns - - dynamographdeploymentcandidates verbs: - get + documentIndex: 1 + - contains: + path: rules + content: + apiGroups: + - nvidia.com + resources: + - dynamographdeploymentcandidates + verbs: - list - - watch documentIndex: 1 - contains: path: rules @@ -282,18 +356,13 @@ tests: - update - delete documentIndex: 1 - # The planner reads, creates, replaces and deletes Leases; it never patches them. - - notContains: + - contains: path: rules content: apiGroups: - - coordination.k8s.io + - "" resources: - - leases + - pods verbs: - - get - - create - - update - - patch - - delete + - list documentIndex: 1 diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index a68de01cca42..6f1aa52a64ae 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -101,6 +101,11 @@ dynamo-operator: # reconciles v1beta2 Requests into Runs and executes each Run as an AISimulate # planner Job with a publisher sidecar that persists deployment Candidates. enabled: false + # -- EXPERIMENTAL, DEVELOPMENT CLUSTERS ONLY: With dgdrV1Beta2.enabled, grant every + # Planner ServiceAccount what Planner live retune (retune.executor: dgdr) needs: + # create, read and delete DGDR v1beta2 Requests, read their Runs and Candidates, + # and read, create, update and delete any Lease in the namespace. + plannerLiveRetune: false # Environment variables to pass to operator Deployment. env: [] diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 07845191d4c4..57ec5a378e38 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -54,7 +54,8 @@ This applies to all DGDR trace Runs, not only those the Planner creates. - **Operator with DGDR v1beta2 traces.** Install the platform chart with `dynamo-operator.dgdrV1Beta2.enabled=true` and an operator that accepts `spec.workload.trace`. - The same Helm value grants the Planner ServiceAccount the access it needs (see [RBAC](#rbac)). + Also set `dynamo-operator.dgdrV1Beta2.plannerLiveRetune=true` to grant the Planner + ServiceAccount the access it needs (see [RBAC](#rbac)). In a cluster-wide install, the operator must also hold the DGDR v1beta2 Run and Candidate permissions of [ai-dynamo/dynamo#17308](https://github.com/ai-dynamo/dynamo/pull/17308); see [RBAC](#rbac). @@ -192,7 +193,6 @@ retune: | `promote_last_advisory` | `false` | On switching to active, apply the last advisory recommendation if it is at most one interval old and the configuration is unchanged | | `lease_duration_seconds` | `60` | Leadership Lease duration, renewed every third of it | | `actuator_interval_seconds` | `15` | Period of the router and clean-up rounds | -| `trace_incomplete_threshold` | `0.05` | With `metric_pulling_prometheus_endpoint`, a window with this fraction fewer requests than the frontend served fails with `trace_incomplete` | For the other `retune` fields and the limits that `executor: dgdr` adds to them, see the [Retune field reference](../../reference/components/planner-configuration.mdx#retune-experimental). @@ -293,14 +293,20 @@ The Planner only warns when AISimulate does not list the model among its default parameters. The mode stored in the state alone does not show that. 2. Remove `retune` from the Planner configuration. 3. Remove the trace environment from the frontend. -4. Delete the Planner's Requests and both Leases: +4. Delete the Planner's Requests and both Leases. Select the Requests by the DGD's + `metadata.uid`: the `nvidia.com/planner-retune-dgd` label holds a hash instead of the name + when the name is longer than 63 characters. ```bash kubectl delete dynamographdeploymentrequests.nvidia.com -n \ - -l nvidia.com/planner-retune-dgd= + -l nvidia.com/planner-retune-dgd-uid=$(kubectl get dgd -n -o jsonpath='{.metadata.uid}') kubectl delete lease -n -planner-retune -planner-retune-state ``` + While the chart still grants the live retune RBAC, a Planner without `retune` does this on + its own: it finds the Leases in the background after it starts, resets any router the + removed retune changed, then deletes the Requests and both Leases. + 5. Delete `/` and `/` from the claim. Before upgrading the operator or its CRDs, switch to advisory or disable retune, then delete the @@ -319,16 +325,20 @@ retune Requests. ### RBAC -With `dynamo-operator.dgdrV1Beta2.enabled=true`, the platform chart adds these rules to the Planner Role -(namespace-restricted installs) and ClusterRole (cluster-wide installs): +With `dynamo-operator.dgdrV1Beta2.enabled=true` and +`dynamo-operator.dgdrV1Beta2.plannerLiveRetune=true`, the platform chart adds these rules to the +Planner Role (namespace-restricted installs) and ClusterRole (cluster-wide installs). Both values +default to `false`; `dgdrV1Beta2.enabled` alone grants the Planner nothing new. | API group | Resources | Verbs | |---|---|---| -| `nvidia.com` | `dynamographdeploymentrequests` | create, get, list, watch, delete | -| `nvidia.com` | `dynamographdeploymentruns`, `dynamographdeploymentcandidates` | get, list, watch | +| `nvidia.com` | `dynamographdeploymentrequests` | create, get, list, delete | +| `nvidia.com` | `dynamographdeploymentruns` | get | +| `nvidia.com` | `dynamographdeploymentcandidates` | list | | `coordination.k8s.io` | `leases` | get, create, update, delete | -The Planner also lists pods with its existing permission. These rules are namespace-wide: a +The chart has one Planner role for the whole install, so Planners without `retune` get these +rules too. The Planner also lists pods with its existing permission. These rules are namespace-wide: a Role or a RoleBinding of the ClusterRole has no `resourceNames`, because the Planner creates its Leases and Requests under per-DGD names. As a result, any workload that runs as the Planner ServiceAccount can: @@ -341,8 +351,8 @@ ServiceAccount can: In a cluster-wide install, the operator binds the Planner ClusterRole into each DGD namespace with a RoleBinding, and Kubernetes lets it bind only permissions it holds itself. Set -`dynamo-operator.dgdrV1Beta2.enabled=true` there only with an operator whose own role grants -`get`, `list` and `watch` on `dynamographdeploymentruns` and `dynamographdeploymentcandidates`, +`dynamo-operator.dgdrV1Beta2.plannerLiveRetune=true` there only with an operator whose own role +grants `get` on `dynamographdeploymentruns` and `list` on `dynamographdeploymentcandidates`, which the DGDR v1beta2 operator change in [ai-dynamo/dynamo#17308](https://github.com/ai-dynamo/dynamo/pull/17308) adds. Without it, the API server rejects the Planner RoleBinding in every namespace that does not have diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 676d889a58e7..0e7eb795578a 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -768,9 +768,6 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p Period of the router and clean-up rounds. Must be greater than 0. - - With `metric_pulling_prometheus_endpoint`, a window whose request count is below the frontend's by more than this fraction fails with `trace_incomplete`. Must be in [0, 1]. - From 03106cf4d3a7de87ca23cec08a3affa8549912c7 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 11:48:34 -0800 Subject: [PATCH 085/100] test(operator): trim redundant trace tests and drop the golden -update flag - The Request controller's CRD-rejection fixture uses the renamed spec.workload.trace.source.persistentVolumeClaim.subPath field path and its "subPath must be relative" message. - The trace-claim cancellation table keeps one running-Job row plus the success-criteria, failure-target, removed-claim and fail-during-deletion rows; the cancellation path reads no pods, so the pod-state rows and their pod scaffolding are gone. - The waiting-container table keeps one CreateContainerConfigError sweeper row and one ContainerCreating row; the diagnostic copies the message verbatim and PodInitializing shares ContainerCreating's case. - The carrier admission table drops the rows the aggregate row or the one structural subPath check already covers. The aggregate now uses a zero arrivalSpeedupRatio, so the boundary stays covered. - The lowering golden envtest no longer registers a package-level -update flag, which broke `go test ./... -update` in every other package. Like the operator's golden helpers, a mismatch writes .new next to the golden file and prints a diff; regenerate by running the test and moving the .new file over the golden. - TracePath's test drops the row that only repeated the non-.gz branch. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../internal/aisimulate/trace_test.go | 1 - .../dgdr_v1beta2_trace_golden_envtest_test.go | 40 +++---- ...ploymentrequest_v1beta2_controller_test.go | 4 +- ...phdeploymentrun_v1beta2_controller_test.go | 83 +------------ ...ploymentrequest_validation_envtest_test.go | 113 +----------------- 5 files changed, 26 insertions(+), 215 deletions(-) diff --git a/deploy/operator/internal/aisimulate/trace_test.go b/deploy/operator/internal/aisimulate/trace_test.go index 7214525bc46b..2183dcdd193e 100644 --- a/deploy/operator/internal/aisimulate/trace_test.go +++ b/deploy/operator/internal/aisimulate/trace_test.go @@ -86,7 +86,6 @@ func TestTracePathIsOneFixedFileUnderTheMount(t *testing.T) { for subPath, want := range map[string]string{ "retune/w-1.jsonl.gz": "/var/run/aisimulate/trace/trace.jsonl.gz", "retune/w-1.jsonl": "/var/run/aisimulate/trace/trace.jsonl", - "../../etc/passwd": "/var/run/aisimulate/trace/trace.jsonl", } { if got := TracePath(subPath); got != want { t.Errorf("TracePath(%q) = %q, want %q", subPath, got, want) diff --git a/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go b/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go index c8814cfe8e0e..4fae5536de87 100644 --- a/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go +++ b/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go @@ -10,21 +10,14 @@ package controller import ( "bytes" "encoding/json" - "flag" "os" - "path/filepath" "testing" nvidiacomv1beta2 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta2" + "github.com/pmezard/go-difflib/difflib" "sigs.k8s.io/controller-runtime/pkg/client" ) -// updateTraceRequestLoweringGolden rewrites the golden file instead of comparing with it: -// -// go test ./internal/controller -run '^TestDGDRV1beta2TraceRequestLoweringGolden$' -update -var updateTraceRequestLoweringGolden = flag.Bool("update", false, - "rewrite internal/aisimulate/testdata/trace_request_lowering.golden.json") - // traceRequestLoweringGoldenPath is read by the live-retune planner's tests, which check that the // planner builds request_spec and accepts run_input as the effective input of its Run. const traceRequestLoweringGoldenPath = "../aisimulate/testdata/trace_request_lowering.golden.json" @@ -32,7 +25,7 @@ const traceRequestLoweringGoldenPath = "../aisimulate/testdata/trace_request_low // liveRetuneShardRequestSpec is the spec of one grid shard the live-retune planner submits: its // build_request for an aggregated vLLM deployment with 3 replicas of tensor parallelism 2, the window // ending at 1793000000000 ms, and a 2x2x2 router grid. It omits workload.trace.format, which the -// CRD schema defaults. Change it together with the planner and rerun with -update. +// CRD schema defaults. Change it together with the planner and regenerate the golden file. const liveRetuneShardRequestSpec = `{ "model": {"name": "Qwen/Qwen3-32B"}, "backends": ["vllm"], @@ -92,7 +85,12 @@ const liveRetuneShardRequestSpec = `{ // The live-retune planner submits a trace Request and checks the input its Run executes. Both // sides share one golden file, so a change to the trace API, admission or lowering that the planner -// does not follow fails here or in the planner's tests. +// does not follow fails here or in the planner's tests. A mismatch writes the result next to the +// golden file as .new; to regenerate the golden file, run the test and move the .new file over it: +// +// go test ./internal/controller -run '^TestDGDRV1beta2TraceRequestLoweringGolden$' +// mv internal/aisimulate/testdata/trace_request_lowering.golden.json.new \ +// internal/aisimulate/testdata/trace_request_lowering.golden.json func TestDGDRV1beta2TraceRequestLoweringGolden(t *testing.T) { env := sharedEnv.ForTest(t) c := env.Client() @@ -142,21 +140,21 @@ func TestDGDRV1beta2TraceRequestLoweringGolden(t *testing.T) { } got = append(got, '\n') - t.Log("Compare with the golden file, or rewrite it with -update") - if *updateTraceRequestLoweringGolden { - if err := os.MkdirAll(filepath.Dir(traceRequestLoweringGoldenPath), 0o755); err != nil { - t.Fatalf("create the golden directory: %v", err) - } - if err := os.WriteFile(traceRequestLoweringGoldenPath, got, 0o644); err != nil { - t.Fatalf("write the golden file: %v", err) - } - } + t.Log("Compare with the golden file; a mismatch writes the result next to it as .new") want, err := os.ReadFile(traceRequestLoweringGoldenPath) if err != nil { t.Fatalf("read the golden file: %v", err) } if !bytes.Equal(got, want) { - t.Fatalf("%s is stale; rerun with -update and update the planner if needed\n got: %s\nwant: %s", - traceRequestLoweringGoldenPath, got, want) + newPath := traceRequestLoweringGoldenPath + ".new" + if err := os.WriteFile(newPath, got, 0o644); err != nil { + t.Fatalf("write %s: %v", newPath, err) + } + diff, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(string(want)), B: difflib.SplitLines(string(got)), + FromFile: traceRequestLoweringGoldenPath, ToFile: newPath, Context: 3, + }) + t.Fatalf("%s is stale; review %s, move it over the golden file and update the planner if needed:\n%s", + traceRequestLoweringGoldenPath, newPath, diff) } } diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go index 9e60fd469fdd..8c4153d0d754 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_v1beta2_controller_test.go @@ -747,8 +747,8 @@ func TestV1beta2RequestControllerPreservesInputOverridesForRunLowering(t *testin func TestV1beta2RequestControllerReportsARunTheCRDRejectsInsteadOfRetryingIt(t *testing.T) { invalid := apierrors.NewInvalid( schema.GroupKind{Group: nvidiacomv1beta2.GroupVersion.Group, Kind: "DynamoGraphDeploymentRun"}, "search-1", - field.ErrorList{field.Invalid(field.NewPath("spec", "workload", "trace", "source", "pvc", "path"), "../escape.jsonl", - "path must be relative to the claim root without empty, '.' or '..' segments")}, + field.ErrorList{field.Invalid(field.NewPath("spec", "workload", "trace", "source", "persistentVolumeClaim", "subPath"), "../escape.jsonl", + "subPath must be relative to the claim root without empty, '.' or '..' segments")}, ) tests := []struct { name string diff --git a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go index 8f5c79523b41..a761ed6a5090 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrun_v1beta2_controller_test.go @@ -2256,22 +2256,9 @@ func TestV1beta2RunControllerProtectsTheTraceClaimBeforeCreatingTheJob(t *testin } func TestV1beta2RunControllerCancelsARunWhoseTraceClaimIsDeleted(t *testing.T) { - waitingSweeper := []corev1.ContainerStatus{{Name: dgdrRunSweeperContainer, State: corev1.ContainerState{ - Waiting: &corev1.ContainerStateWaiting{Reason: "CreateContainerConfigError", Message: "failed to prepare subPath"}, - }}} - creatingSweeper := []corev1.ContainerStatus{{Name: dgdrRunSweeperContainer, State: corev1.ContainerState{ - Waiting: &corev1.ContainerStateWaiting{Reason: "ContainerCreating"}, - }}} - runningSweeper := []corev1.ContainerStatus{{Name: dgdrRunSweeperContainer, State: corev1.ContainerState{ - Running: &corev1.ContainerStateRunning{}, - }}} tests := []struct { name string status batchv1.JobStatus - // pod, when set, is the Job's pod as the cache shows it. - pod *corev1.PodStatus - // podNode binds the pod to a node. - podNode string // removeClaim removes the claim completely, as when its finalizers were removed by hand. removeClaim bool // failDuringDeletion marks the Job failed while it is being deleted, as the Job controller @@ -2279,40 +2266,8 @@ func TestV1beta2RunControllerCancelsARunWhoseTraceClaimIsDeleted(t *testing.T) { failDuringDeletion bool }{ { - name: "the Job counts a pod the pod cache does not show", - status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, - }, - { - name: "the pod is not scheduled", - status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, - pod: &corev1.PodStatus{Phase: corev1.PodPending}, - }, - { - name: "the pod is scheduled while its images are pulled", - status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, - pod: &corev1.PodStatus{Phase: corev1.PodPending, ContainerStatuses: creatingSweeper}, - podNode: "node-a", - }, - { - name: "the sweeper waits on a mount error", - status: batchv1.JobStatus{Active: 1, Ready: new(int32(0))}, - pod: &corev1.PodStatus{Phase: corev1.PodPending, ContainerStatuses: waitingSweeper}, - podNode: "node-a", - }, - { - name: "the sweeper runs", - status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, - pod: &corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: runningSweeper}, - podNode: "node-a", - }, - { - name: "the pod is ready while the pod cache still shows it unscheduled", + name: "the Job runs", status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, - pod: &corev1.PodStatus{Phase: corev1.PodPending}, - }, - { - name: "the Job tracks a succeeded pod it has not counted", - status: batchv1.JobStatus{UncountedTerminatedPods: &batchv1.UncountedTerminatedPods{Succeeded: []types.UID{"pod"}}}, }, { name: "the Job met its success criteria", @@ -2325,15 +2280,11 @@ func TestV1beta2RunControllerCancelsARunWhoseTraceClaimIsDeleted(t *testing.T) { { name: "the claim is gone after its finalizers were removed by hand", status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, - pod: &corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: runningSweeper}, - podNode: "node-a", removeClaim: true, }, { name: "the Job is marked failed while it is being deleted", status: batchv1.JobStatus{Active: 1, Ready: new(int32(1))}, - pod: &corev1.PodStatus{Phase: corev1.PodRunning, ContainerStatuses: runningSweeper}, - podNode: "node-a", failDuringDeletion: true, }, } @@ -2367,9 +2318,6 @@ func TestV1beta2RunControllerCancelsARunWhoseTraceClaimIsDeleted(t *testing.T) { if err := f.client.Status().Update(t.Context(), job); err != nil { t.Fatalf("update Job status: %v", err) } - if tt.pod != nil { - createDGDRRunJobPodForTest(t, f.client, job, tt.podNode, *tt.pod) - } f.reconcile() t.Log("Delete the claim") @@ -2855,33 +2803,12 @@ func TestV1beta2RunControllerReportsAWaitingContainerWithoutEndingTheRun(t *test wantReason string }{ { - name: "a subPath that resolves outside the claim", + name: "a sweeper whose trace mount cannot be prepared", container: dgdrRunSweeperContainer, reason: "CreateContainerConfigError", message: `failed to prepare subPath for volumeMount "trace" of container "sweeper"`, wantReason: dgdrRunReasonContainerWaiting, }, - { - name: "a missing file on a read-only claim", - container: dgdrRunSweeperContainer, - reason: "CreateContainerConfigError", - message: `failed to create subPath directory for volumeMount "trace" of container "sweeper"`, - wantReason: dgdrRunReasonContainerWaiting, - }, - { - name: "a kubelet message in other words", - container: dgdrRunSweeperContainer, - reason: "CreateContainerConfigError", - message: "the trace mount could not be prepared", - wantReason: dgdrRunReasonContainerWaiting, - }, - { - name: "another volume of the sweeper", - container: dgdrRunSweeperContainer, - reason: "CreateContainerConfigError", - message: `failed to prepare subPath for volumeMount "input" of container "sweeper"`, - wantReason: dgdrRunReasonContainerWaiting, - }, { name: "the publisher", container: dgdrRunPublisherContainer, @@ -2902,12 +2829,6 @@ func TestV1beta2RunControllerReportsAWaitingContainerWithoutEndingTheRun(t *test reason: "ContainerCreating", wantReason: dgdrRunReasonRunning, }, - { - name: "a container that waits for its pod to initialize", - container: dgdrRunPublisherContainer, - reason: "PodInitializing", - wantReason: dgdrRunReasonRunning, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go index c895f129a4a2..1019d45083de 100644 --- a/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go +++ b/deploy/operator/internal/webhook/validation/dynamographdeploymentrequest_validation_envtest_test.go @@ -896,57 +896,6 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { }, // A v1beta2 trace in a v1beta1 carrier stored without admission: CEL ratchets the unchanged // trace on the next v1beta2 write, so the structural webhook recursion rejects it. - { - name: "v1beta2 carrier trace subPath that leaves the claim", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("../escape.jsonl")}), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("../escape.jsonl")} - }), - seedWithoutWebhook: true, - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{ - `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "../escape.jsonl": subPath must be relative to the claim root without empty, '.' or '..' segments`, - }, - // CEL ratchets the unchanged trace and reports what it tolerated as a warning. - wantWarnings: []string{ - `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, - }, - }, - { - name: "v1beta2 carrier absolute trace subPath", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("/w.jsonl")}), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("/w.jsonl")} - }), - seedWithoutWebhook: true, - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{ - `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "/w.jsonl": subPath must be relative to the claim root without empty, '.' or '..' segments`, - }, - // CEL ratchets the unchanged trace and reports what it tolerated as a warning. - wantWarnings: []string{ - `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, - }, - }, - { - name: "v1beta2 carrier trace subPath with an empty segment", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("retune//w.jsonl")}), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": v1beta2TraceForAdmission("retune//w.jsonl")} - }), - seedWithoutWebhook: true, - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{ - `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "retune//w.jsonl": subPath must be relative to the claim root without empty, '.' or '..' segments`, - }, - // CEL ratchets the unchanged trace and reports what it tolerated as a warning. - wantWarnings: []string{ - `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "string": subPath must be relative to the claim root without empty, '.' or '..' segments`, - }, - }, { name: "v1beta2 carrier trace subPath with a NUL character", oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": v1beta2TraceForAdmission("retune/w\x00.jsonl")}), @@ -977,32 +926,6 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { "spec.workload.trace.source.persistentVolumeClaim.subPath: Too long: may not be more than 1024 characters", }, }, - { - name: "v1beta2 carrier trace claim that is not a DNS subdomain", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "w.jsonl"}}}}), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "w.jsonl"}}}} - }), - seedWithoutWebhook: true, - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{ - `spec.workload.trace.source.persistentVolumeClaim.claimName: Invalid value: "Bad_Claim": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, - }, - }, - { - name: "v1beta2 carrier trace format that is not admitted", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}}), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}} - }), - seedWithoutWebhook: true, - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{ - `spec: Invalid value: "": cannot be lowered to the public AISimulate recommendation input: spec.workload.trace.format: unsupported trace format "mooncake"`, - }, - }, { name: "v1beta2 carrier trace without a format receives the default on the next write", oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}}), @@ -1035,41 +958,11 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { `spec.workload.trace.source: Invalid value: "object": persistentVolumeClaim is required`, }, }, - { - name: "v1beta2 carrier trace beside a static workload", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}}), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}}} - }), - seedWithoutWebhook: true, - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{ - `spec.workload: Forbidden: static and trace are mutually exclusive`, - }, - // CEL ratchets the unchanged trace and reports what it tolerated as a warning. - wantWarnings: []string{ - `spec.workload: Invalid value: "object": static and trace are mutually exclusive`, - }, - }, - { - name: "v1beta2 carrier trace arrival speedup that is not positive", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}}), - request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"trace": map[string]any{"format": "Dynamo", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "request-traces", "subPath": "w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}} - }), - seedWithoutWebhook: true, - gpuDiscovery: true, - discovered: discoveredH200, - wantWebhook: []string{ - `spec.workload.trace.replay.arrivalSpeedupRatio: Invalid value: 0: must be greater than 0`, - }, - }, { name: "v1beta2 carrier trace errors are reported together in field order", - oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": -1.0}}}), + oldRequest: v1beta1CarrierForAdmission(map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}}), request: v1beta2DGDRForAdmission(func(spec map[string]any) { - spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": -1.0}}} + spec["workload"] = map[string]any{"static": map[string]any{}, "trace": map[string]any{"format": "mooncake", "source": map[string]any{"persistentVolumeClaim": map[string]any{"claimName": "Bad_Claim", "subPath": "../w.jsonl"}}, "replay": map[string]any{"arrivalSpeedupRatio": 0.0}}} }), seedWithoutWebhook: true, gpuDiscovery: true, @@ -1078,7 +971,7 @@ func TestDynamoGraphDeploymentRequestValidator_Validate(t *testing.T) { `spec.workload: Forbidden: static and trace are mutually exclusive`, `spec.workload.trace.source.persistentVolumeClaim.claimName: Invalid value: "Bad_Claim": a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')`, `spec.workload.trace.source.persistentVolumeClaim.subPath: Invalid value: "../w.jsonl": subPath must be relative to the claim root without empty, '.' or '..' segments`, - `spec.workload.trace.replay.arrivalSpeedupRatio: Invalid value: -1: must be greater than 0`, + `spec.workload.trace.replay.arrivalSpeedupRatio: Invalid value: 0: must be greater than 0`, `spec: Invalid value: "": cannot be lowered to the public AISimulate recommendation input: spec.workload.trace.format: unsupported trace format "mooncake"`, }, // CEL ratchets the unchanged trace and reports what it tolerated as a warning. From 0405a8f576a471e34e097b3550d743f247066f40 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 11:54:47 -0800 Subject: [PATCH 086/100] fix(planner): address second review on the live retune configuration - P4 runs the controller constructor's own startup router checks: RetuneController._router_domain and live retune's _startup_errors both call the new module-level controller.startup_router_domain, replacing the copy in frontend_config. - Guard build_frontend_parser's env isolation: a test sets every env var the parser registers to sentinels in os.environ and expects the empty-env defaults. - Premise tests build validated PlannerConfigs; drop the tests and the branches only impossible configs reached (no router domain, a slash-wrapped segments_dir, an unexpandable or non-'none' load model). - Drop retune.dgdr.startup_router_config and its P4 check. - balanced_shard_parts requires parallelism; drop the points-only ranking and its tests. - One P10 table row per branch. - contracts.py keeps FrontendStartup and PremiseReport; the other types and the live SweepRequest/SweepResult fields move to the PRs that use them. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../tests/test_frontend_parser_builder.py | 33 ++ .../dynamo/planner/config/planner_config.py | 29 +- .../src/dynamo/planner/retune/controller.py | 130 +++--- .../dynamo/planner/retune/live/contracts.py | 431 +----------------- .../planner/retune/live/frontend_config.py | 121 +---- components/src/dynamo/planner/retune/sweep.py | 26 +- .../tests/unit/test_live_retune_config.py | 28 +- .../unit/test_live_retune_frontend_config.py | 155 ++----- 8 files changed, 186 insertions(+), 767 deletions(-) diff --git a/components/src/dynamo/frontend/tests/test_frontend_parser_builder.py b/components/src/dynamo/frontend/tests/test_frontend_parser_builder.py index b095423b9765..5fc49e153b01 100644 --- a/components/src/dynamo/frontend/tests/test_frontend_parser_builder.py +++ b/components/src/dynamo/frontend/tests/test_frontend_parser_builder.py @@ -6,9 +6,11 @@ import argparse import logging import os +import sys import pytest +from dynamo.common.configuration.utils import env_or_default from dynamo.frontend.frontend_args import build_frontend_parser pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0] @@ -29,6 +31,37 @@ def test_reads_only_the_given_env(monkeypatch: pytest.MonkeyPatch) -> None: assert dict(os.environ) == before +def test_no_registered_env_var_is_read_from_os_environ( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # An argument registered without ``env=`` would take its default from + # the parsing process's os.environ. Record every env var the + # registrations look up, wherever ``env_or_default`` was imported. + registered: set[str] = set() + + def recording_env_or_default(env_var, default, value_type=None, *, env=None): + registered.add(env_var) + return env_or_default(env_var, default, value_type, env=env) + + for name, module in list(sys.modules.items()): + if name.startswith("dynamo.") and ( + vars(module).get("env_or_default") is env_or_default + ): + monkeypatch.setattr(module, "env_or_default", recording_env_or_default) + build_frontend_parser({}) + assert "DYN_ROUTER_MODE" in registered + for name in registered: + monkeypatch.delenv(name, raising=False) + defaults = vars(build_frontend_parser({})[0].parse_args([])) + + # Two sentinels, so that a leaked env var changes at least one default. + for sentinel in ("0", "1"): + for name in registered: + monkeypatch.setenv(name, sentinel) + parser, _ = build_frontend_parser({}) + assert vars(parser.parse_args([])) == defaults, sentinel + + def test_parses_without_notices_or_local_files( caplog: pytest.LogCaptureFixture, recwarn: pytest.WarningsRecorder ) -> None: diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index 2b2065b25226..2d31f1066996 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -916,13 +916,6 @@ class DgdrExecutorConfig(BaseModel): min_length=1, description="Claim-relative directory of window files: //.", ) - startup_router_config: dict[str, Any] | None = Field( - default=None, - description=( - "Optional startup KvRouterConfig JSON. It must equal what the planner " - "parses from the frontend's args and env; parsing is the source." - ), - ) model: str = Field( min_length=1, description="HF repo id the workers serve (worker --model); AISimulate resolves it.", @@ -987,9 +980,8 @@ class DgdrExecutorConfig(BaseModel): # DGDR v1beta2 ``spec.recommendation.maxCandidates`` maximum: one Request (a # grid shard) carries at most this many router points. DGDR_MAX_SHARD_POINTS = 100 -# Router knobs a DGDR v1beta2 Request can search, by public name -# (``contracts.DGDR_ROUTER_KNOBS``); the Request's router section has no -# place for the retune extension knobs. +# Router knobs a DGDR v1beta2 Request can search, by public name; the +# Request's router section has no place for the retune extension knobs. _DGDR_ROUTER_SECTION_KEYS = frozenset( { "overlap_score_credit", @@ -1040,7 +1032,7 @@ def dgdr_need_seconds( def balanced_shard_parts( - sizes: Sequence[int], max_shards: int, *, parallelism: int | None = None + sizes: Sequence[int], max_shards: int, *, parallelism: int ) -> tuple[int, ...]: """How many chunks to cut each router grid dimension into for DGDR sharding. @@ -1052,22 +1044,21 @@ def balanced_shard_parts( 1. whether the largest shard exceeds ``DGDR_MAX_SHARD_POINTS``, and when every split does, its points (so the caller reports the smallest); - 2. the largest shard's wall time: its ``dgdr_rounds`` with - ``parallelism``, its points without; + 2. the largest shard's wall time, its ``dgdr_rounds`` with + ``parallelism``; 3. the shard count, the largest shard's points, then the parts. P13 calls it with ``parallelism=run.parallelism``; the live executor must plan each window's shards with the same call, so that both see the - same largest shard. The default 3 x 9 x 4 grid with ``max_shards=3`` - gives ``(1, 3, 1)``, three shards of 36 points, with or without - parallelism. + same largest shard. The default 3 x 9 x 4 grid with ``max_shards=3`` and + ``parallelism=8`` gives ``(1, 3, 1)``, three shards of 36 points. """ if max_shards < 1: raise ValueError(f"max_shards must be at least 1, got {max_shards}") if any(size < 1 for size in sizes): raise ValueError(f"every grid dimension needs a value, got sizes {sizes}") - if parallelism is not None and parallelism < 1: + if parallelism < 1: raise ValueError(f"parallelism must be at least 1, got {parallelism}") def splits(index: int, budget: int): @@ -1082,8 +1073,8 @@ def rank(parts: tuple[int, ...]) -> tuple[Any, ...]: largest = largest_shard_points(sizes, parts) if largest > DGDR_MAX_SHARD_POINTS: return (True, largest, math.prod(parts), parts) - cost = largest if parallelism is None else dgdr_rounds(largest, parallelism) - return (False, cost, math.prod(parts), largest, parts) + rounds = dgdr_rounds(largest, parallelism) + return (False, rounds, math.prod(parts), largest, parts) return min(splits(0, max_shards), key=rank) diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index 4385d6ca3708..c82231f65bae 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -308,63 +308,11 @@ def current_point(self) -> RetuneParams: def _router_domain(self, retune: RetuneConfig) -> dict[str, list[float]]: if retune.router is None: return {} - engine = self._inputs.engine - startup = self._startup_router_json - # Only the builtin default policy is modelled identically. - if engine.router_mode != "kv_router": + if self._inputs.engine.router_mode != "kv_router": raise ValueError("retune.router requires engine router_mode='kv_router'") - if startup.get("router_policy_config") is not None: - raise ValueError( - "retune.router requires the builtin default router policy " - "(router_config.router_policy_config must be unset)" - ) - if startup.get("overlap_score_weight") is not None: - raise ValueError( - "retune.router cannot be used with the deprecated " - "overlap_score_weight, which overrides prefill_load_scale" - ) - if startup.get("conditional_disagg_enabled"): - raise ValueError( - "retune.router does not support conditional disaggregation: " - "replay and live resolve decode credit differently" - ) - - startup_maps = self._router_map({}) - # A worker type that starts with no overlap credit can never get it. - zero_credit = { - worker_type - for worker_type, parameters in startup_maps.items() - if worker_type != "decode" and parameters[OVERLAP_SCORE_CREDIT] == 0 - } - # The schema keeps prefill_load_model.type at its startup value; the - # serde form of the AIS load model is "aic". - startup_load_model = json.loads( - KvRouterConfig.from_json(json.dumps(startup)).to_json() - )["router_prefill_load_model"] - domain = expand_router_domain( - retune, - startup_load_model=startup_load_model, - zero_startup_credit=bool(zero_credit), + return startup_router_domain( + retune, self._startup_router_json, self._worker_types ) - # The schema admits only PolicyParameters names, with the bounds - # KvRouterConfig checks; resolving every candidate also guards - # against the two drifting apart. - for point in grid_points(domain, {}): - try: - maps = self._router_map(point.router) - except ValueError as error: - raise ValueError( - f"retune.router candidate {point.router} is not a valid " - f"router configuration: {error}" - ) from None - for worker_type in zero_credit: - if maps[worker_type][OVERLAP_SCORE_CREDIT] != 0: - raise ValueError( - f"retune.router candidate {point.router} gives the " - f"{worker_type} router overlap_score_credit > 0, but its " - "startup credit is 0 (a live router rejects that update)" - ) - return domain def _feasible_grid( self, @@ -846,6 +794,78 @@ def _check_inner_planner_isolation(config: PlannerConfig) -> None: ) +def startup_router_domain( + retune: RetuneConfig, + startup_router_json: Mapping[str, Any], + worker_types: Sequence[str], +) -> dict[str, list[float]]: + """The ``retune.router`` domain on a startup ``KvRouterConfig`` JSON. + + The startup-dependent router checks of the controller constructor, which + live retune's premise P4 runs on the frontend startup it parses, so that + a startup P4 accepts builds a controller. ``worker_types`` are the router + worker types of the planner mode. Raises ``ValueError`` when the startup + cannot be retuned or a candidate does not resolve on it. + """ + + # Only the builtin default policy is modelled identically. + if startup_router_json.get("router_policy_config") is not None: + raise ValueError( + "retune.router requires the builtin default router policy " + "(router_config.router_policy_config must be unset)" + ) + if startup_router_json.get("overlap_score_weight") is not None: + raise ValueError( + "retune.router cannot be used with the deprecated " + "overlap_score_weight, which overrides prefill_load_scale" + ) + if startup_router_json.get("conditional_disagg_enabled"): + raise ValueError( + "retune.router does not support conditional disaggregation: " + "replay and live resolve decode credit differently" + ) + + def router_maps(router: Mapping[str, float]) -> dict[str, dict[str, float]]: + config = KvRouterConfig.from_json(json.dumps({**startup_router_json, **router})) + return {wt: dict(config.policy_parameters(wt)) for wt in worker_types} + + # A worker type that starts with no overlap credit can never get it. + zero_credit = { + worker_type + for worker_type, parameters in router_maps({}).items() + if worker_type != "decode" and parameters[OVERLAP_SCORE_CREDIT] == 0 + } + # The schema keeps prefill_load_model.type at its startup value; the + # serde form of the AIS load model is "aic". + startup_load_model = json.loads( + KvRouterConfig.from_json(json.dumps(dict(startup_router_json))).to_json() + )["router_prefill_load_model"] + domain = expand_router_domain( + retune, + startup_load_model=startup_load_model, + zero_startup_credit=bool(zero_credit), + ) + # The schema admits only PolicyParameters names, with the bounds + # KvRouterConfig checks; resolving every candidate also guards against + # the two drifting apart. + for point in grid_points(domain, {}): + try: + maps = router_maps(point.router) + except ValueError as error: + raise ValueError( + f"retune.router candidate {point.router} is not a valid " + f"router configuration: {error}" + ) from None + for worker_type in zero_credit: + if maps[worker_type][OVERLAP_SCORE_CREDIT] != 0: + raise ValueError( + f"retune.router candidate {point.router} gives the " + f"{worker_type} router overlap_score_credit > 0, but its " + "startup credit is 0 (a live router rejects that update)" + ) + return domain + + def _candidate_error( config: PlannerConfig, updates: dict[str, Any], diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py index 173a6bf159ba..fa0b0d5144dc 100644 --- a/components/src/dynamo/planner/retune/live/contracts.py +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -1,212 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Types shared by the live retune modules. - -Every module in ``dynamo.planner.retune.live`` depends on this one and on -nothing else in the package except through the protocols defined here, so -each can be built and tested with fakes. -""" +"""Types shared by the live retune modules.""" from __future__ import annotations -import threading -from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass, field -from typing import Any, Literal, Optional, Protocol - -# The eight builtin default-policy parameters a frontend router exposes on -# GET/PATCH /router_policy, by their live (PATCH) names. -POLICY_PARAMETER_KEYS: tuple[str, ...] = ( - "decode_active_request_weight", - "disk_cache_hit_weight", - "host_cache_hit_weight", - "overlap_score_credit", - "overlap_score_credit_decay", - "prefill_load_scale", - "router_temperature", - "shared_cache_multiplier", -) - -# Router knobs a DGDR v1beta2 Request can search, AISimulate public name -> -# live name. ``dynamo.router.simulation.config.ROUTER_PUBLIC_TO_LIVE`` is the -# source of truth; this mirrors it for modules that must not import the -# router simulation stack. -DGDR_ROUTER_KNOBS: dict[str, str] = { - "overlap_score_credit": "overlap_score_credit", - "prefill_load_scale": "prefill_load_scale", - "temperature": "router_temperature", -} - -LABEL_PREFIX = "nvidia.com/planner-retune-" -LABEL_DGD = LABEL_PREFIX + "dgd" -LABEL_DGD_UID = LABEL_PREFIX + "dgd-uid" -LABEL_WINDOW = LABEL_PREFIX + "window" -LABEL_SHARD = LABEL_PREFIX + "shard" -LABEL_SPEC = LABEL_PREFIX + "spec" -LABEL_ATTEMPT = LABEL_PREFIX + "attempt" - -# Pod labels the operator sets (deploy/operator/internal/consts/consts.go). -POD_LABEL_DGD_NAME = "nvidia.com/dynamo-graph-deployment-name" -POD_LABEL_COMPONENT_TYPE = "nvidia.com/dynamo-component-type" - -DGDR_GROUP = "nvidia.com" -DGDR_VERSION = "v1beta2" -DGDR_REQUESTS = "dynamographdeploymentrequests" -DGDR_RUNS = "dynamographdeploymentruns" -DGDR_CANDIDATES = "dynamographdeploymentcandidates" -RUN_UID_LABEL = "nvidia.com/dgdr-run-uid" - -# SweepResult.failure_reason values that mean "this process stopped working -# on the window", not "the window failed": no decision is recorded and the -# controller rewinds so the window can be relaunched or adopted by the next -# leader. -NON_DECISION_FAILURES = frozenset({"shutdown", "lost_leadership"}) - -GateReason = Literal["not_leader", "not_ready", "already_decided"] - - -# --------------------------------------------------------------- controller - - -@dataclass(frozen=True) -class GateResult: - """What ``launch_gate`` tells the controller about a due window. - - ``reason`` is ``None`` to let the window launch (or record ``busy``). - ``not_leader``/``not_ready`` keep ``_last_k`` so the window is retried on - a later tick; ``already_decided`` advances it without a decision. - ``spec_replicas`` (worker type -> DGD spec replicas) and ``abort`` (set - by the supervisor when this process stops leading) are only meaningful - when ``reason`` is ``None``. - """ - - reason: Optional[GateReason] = None - spec_replicas: Optional[dict[str, int]] = None - abort: Optional[threading.Event] = None - - -# (k, (window_start_s, window_end_s, as_of_s)) -> GateResult -LaunchGate = Callable[[int, tuple[float, float, float]], GateResult] -# controller generation of the sweep's controller -> may it be harvested? -HarvestAllowed = Callable[[int], bool] - - -# ------------------------------------------------------------ kubernetes - - -class KubeError(Exception): - """A Kubernetes call failed (network, timeout, 5xx, unexpected status).""" - - -class KubeNotFound(KubeError): - pass - - -class KubeConflict(KubeError): - """409 on update (stale resourceVersion).""" - - -class KubeAlreadyExists(KubeError): - """409 on create (the name is taken).""" - - -class KubeInvalid(KubeError): - """422 / 400: the API server rejected the object (e.g. Strict pruning).""" - - -@dataclass(frozen=True) -class DgdRef: - """The DynamoGraphDeployment this planner serves.""" - - name: str - namespace: str # Kubernetes namespace - uid: str - dynamo_namespace: str # the routers' ``namespace`` on GET /router_policy - - -@dataclass(frozen=True) -class PodInfo: - name: str - uid: str - ip: Optional[str] - ready: bool - terminating: bool - labels: Mapping[str, str] = field(default_factory=dict) - # Container port named ``http`` of the frontend container, if any. - http_port: Optional[int] = None - # ``status.phase`` (Pending, Running, Succeeded, Failed, Unknown). - phase: Optional[str] = None - - -class RetuneKube(Protocol): - """Synchronous Kubernetes access for live retune. - - Every method takes an optional ``timeout_s`` (connect+read, applied as - the kubernetes client's ``_request_timeout`` tuple of ints) and raises - the ``Kube*`` errors above instead of client exceptions. Callers on the - event loop wrap calls in ``asyncio.to_thread``. - """ - - namespace: str - - def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: - ... - - def list_pods( - self, label_selector: str, *, timeout_s: float = 10.0 - ) -> list[PodInfo]: - ... - - def create_request( - self, - body: dict, - *, - dry_run: bool = False, - timeout_s: float = 10.0, - ) -> dict: - """Create with ``field_validation='Strict'`` (and ``dry_run='All'``).""" - ... - - def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - ... - - def list_requests( - self, label_selector: str, *, timeout_s: float = 10.0 - ) -> list[dict]: - ... - - def delete_request( - self, - name: str, - *, - propagation: Literal["Background", "Foreground"] = "Background", - timeout_s: float = 10.0, - ) -> None: - """Missing objects are not an error.""" - ... - - def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - ... - - def list_candidates( - self, label_selector: str, *, timeout_s: float = 10.0 - ) -> list[dict]: - ... - - def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - ... - - def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: - ... - - def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: - """Update with ``body['metadata']['resourceVersion']``; 409 -> KubeConflict.""" - ... - - def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: - ... - +from dataclasses import dataclass +from typing import Any, Optional # ------------------------------------------------------- frontend config @@ -244,228 +44,3 @@ class PremiseReport: reasons: tuple[str, ...] = () warnings: tuple[str, ...] = () startup: Optional[FrontendStartup] = None - - -# ---------------------------------------------------------- router policy - - -@dataclass(frozen=True) -class RouterInfo: - """One router on GET /router_policy.""" - - router_id: str - namespace: str - worker_type: str - revision: int - # The 8 PolicyParameters; None for a custom policy without runtime config. - configuration: Optional[dict[str, float]] - model: Optional[str] = None - - -@dataclass(frozen=True) -class PolicyCallResult: - """Outcome of one GET or PATCH. ``status`` is the HTTP status (0 on a - network error or timeout).""" - - status: int - routers: tuple[RouterInfo, ...] = () # GET - router: Optional[RouterInfo] = None # PATCH/reset response, when 200 - error: Optional[str] = None - - -class RouterPolicyClient(Protocol): - async def get(self, pod: PodInfo) -> PolicyCallResult: - ... - - async def patch( - self, - pod: PodInfo, - router_id: str, - expected_revision: int, - parameters: Mapping[str, float], - ) -> PolicyCallResult: - ... - - async def reset( - self, pod: PodInfo, router_id: str, expected_revision: int - ) -> PolicyCallResult: - """PATCH ``{router_id, expected_revision, reset: true}``.""" - ... - - -ActuationMode = Literal["apply", "rollback", "observe"] - - -@dataclass(frozen=True) -class ActuationTarget: - """What one actuator round converges to. - - ``apply``: PATCH explicit values so every router's 8 parameters equal - ``desired_by_wt[worker_type]``. ``rollback``: reset every router whose - configuration differs from its latched initial (never by revision). - ``observe``: no writes; latch initials and check parity only. - ``include_not_ready`` also targets NotReady, non-terminating pods. - ``learn_unknown`` lets the actuator reset a router whose initial was - never latched (revision > 0) once to learn it; that is active mode only. - With False (advisory) such a router is never written: rollback - leaves it alone and does not wait for it to converge. - """ - - mode: ActuationMode - desired_by_wt: Optional[dict[str, dict[str, float]]] = None - include_not_ready: bool = False - learn_unknown: bool = True - - -@dataclass(frozen=True) -class ActuationReport: - converged: bool - # router_id -> configuration latched this round (revision 0 or reset). - new_initials: dict[str, dict[str, float]] = field(default_factory=dict) - # router_ids seen this round (to prune stale initials). - seen_router_ids: frozenset[str] = frozenset() - # P14: every known initial equals startup_maps[worker_type]. - parity_ok: Optional[bool] = None - parity_mismatches: tuple[str, ...] = () - admin_api_disabled: bool = False - errors: dict[str, int] = field(default_factory=dict) # by status code / kind - # P12: enough Ready frontends, each with a configured router for every - # worker type, whatever their values. Gates launches; convergence does not. - routers_present: Optional[bool] = None - - -# ----------------------------------------------------------- reconciler - - -class SweepRegistry(Protocol): - """Thread-safe link between a running sweep and the reconciler. - - Protection is reference-counted per window: every ``protect`` call - takes one reference and needs one matching ``release``. A window, with - the union of every Request name given to it, stays protected while any - reference is held, so two sweeps of one window (a rebuilt controller - relaunching it while the old sweep winds down) cannot drop each - other's protection. - """ - - def protect(self, window_end_ms: int, request_names: Iterable[str]) -> None: - """Mark a window file and Requests as in use by this process (one - reference).""" - ... - - def release(self, window_end_ms: int) -> None: - """Drop one reference taken by ``protect``.""" - ... - - def enqueue_delete(self, request_names: Iterable[str]) -> None: - ... - - -# ---------------------------------------------------------------- window - - -@dataclass(frozen=True) -class WindowCutResult: - path: Optional[str] # None for an empty window (no file written) - num_requests: int - trace_block_size: Optional[int] - row_counts: dict[str, int] - dropped_members: int = 0 - dropped_bytes: int = 0 - reused: bool = False # the final file already existed (EEXIST) - - -class WindowCutter(Protocol): - def __call__( - self, - segments_root: str, - start_ms: int, - end_ms: int, - as_of_ms: int, - out_path: str, - *, - max_rows: int, - deadline_s: float, - cancel: Optional[threading.Event] = None, - ) -> WindowCutResult: - ... - - -# --------------------------------------------------------------- metrics - - -class RetuneMetricsSink(Protocol): - """``dynamo_planner_retune_*`` metrics; every method is cheap and safe.""" - - def decision(self, outcome: str) -> None: - ... - - def failure(self, reason: str) -> None: - ... - - def skipped(self, reason: str) -> None: - ... - - def supervisor_state( - self, - *, - leader: bool, - ready: bool, - initialized: bool, - rollback_pending: bool, - generation: int, - ) -> None: - ... - - def scores( - self, - *, - baseline: Optional[float], - best: Optional[float], - sweep_seconds: Optional[float], - missing_candidates: int, - ) -> None: - ... - - def router_converged(self, converged: bool) -> None: - ... - - def patch_error(self, code: str) -> None: - ... - - def trace_sink_missing(self, pod: str, missing: bool) -> None: - ... - - def dropped_members(self, count: int) -> None: - ... - - -class NullMetrics: - """A ``RetuneMetricsSink`` that records nothing (tests, disabled metrics).""" - - def decision(self, outcome: str) -> None: - pass - - def failure(self, reason: str) -> None: - pass - - def skipped(self, reason: str) -> None: - pass - - def supervisor_state(self, **_: Any) -> None: - pass - - def scores(self, **_: Any) -> None: - pass - - def router_converged(self, converged: bool) -> None: - pass - - def patch_error(self, code: str) -> None: - pass - - def trace_sink_missing(self, pod: str, missing: bool) -> None: - pass - - def dropped_members(self, count: int) -> None: - pass diff --git a/components/src/dynamo/planner/retune/live/frontend_config.py b/components/src/dynamo/planner/retune/live/frontend_config.py index fb7d74863f9c..c49a6ef7c71b 100644 --- a/components/src/dynamo/planner/retune/live/frontend_config.py +++ b/components/src/dynamo/planner/retune/live/frontend_config.py @@ -57,11 +57,8 @@ from dynamo.common.configuration.utils import ArgEnv from dynamo.frontend.frontend_args import build_frontend_parser from dynamo.llm import KvRouterConfig -from dynamo.planner.retune.domains import ( - OVERLAP_SCORE_CREDIT, - expand_router_domain, - grid_points, -) +from dynamo.planner.retune.controller import startup_router_domain +from dynamo.planner.retune.domains import expand_router_domain from dynamo.planner.retune.live.contracts import FrontendStartup, PremiseReport if TYPE_CHECKING: @@ -321,10 +318,6 @@ def evaluate_premises( else: p4.extend(analysis.p4_violations) p4.extend(_startup_errors(startup, retune, mode)) - if dgdr.startup_router_config is not None: - mismatch = _startup_config_mismatch(dgdr.startup_router_config, startup) - if mismatch is not None: - p4.append(mismatch) # A worker's advertised router configuration replaces the frontend's for # its worker set (watcher.rs effective_router_config), whatever the # frontend's flags say. @@ -1039,10 +1032,7 @@ def _startup_errors( f"P4: the frontend routes with {startup.router_mode!r}; live retune " "needs --router-mode kv" ] - try: - searched = set(_router_domain_keys(retune)) - except ValueError as error: - return [f"P4: cannot expand retune.router: {error}"] + searched = set(expand_router_domain(retune)) config = startup.router_config errors: list[str] = [] load_model = config.get(_LOAD_MODEL, "none") @@ -1065,12 +1055,18 @@ def _startup_errors( continue errors.append( f"P4: the frontend sets {key}={config[key]!r}; only the searched " - f"router knobs ({', '.join(sorted(searched)) or 'none'}) may differ " - "from their defaults" + f"router knobs ({', '.join(sorted(searched))}) may differ from " + "their defaults" ) if errors: return errors - return _controller_errors(retune, config, worker_types(planner_mode)) + # The controller constructor's own checks, so that P4 passes only for a + # startup a controller can be built on. + try: + startup_router_domain(retune, config, worker_types(planner_mode)) + except ValueError as error: + return [f"P4: retune.router does not fit the frontend startup: {error}"] + return [] def _worker_router_errors( @@ -1206,96 +1202,6 @@ def _worker_argv(component: Mapping[str, Any], env: _Env) -> tuple[list[str], bo return words, True -def _router_domain_keys(retune: RetuneConfig) -> list[str]: - if retune.router is None: - return [] - return list(expand_router_domain(retune)) - - -def _controller_errors( - retune: RetuneConfig, router_config: Mapping[str, Any], wts: tuple[str, ...] -) -> list[str]: - """The startup-dependent checks of ``RetuneController._router_domain``. - - A startup failing them would make the controller constructor raise, so - P4 rejects it before a controller is built. - """ - - if retune.router is None: - return [] - startup_json = canonical_json(dict(router_config)) - try: - startup_maps = _policy_maps(startup_json, wts) - load_model = json.loads(KvRouterConfig.from_json(startup_json).to_json())[ - _LOAD_MODEL - ] - except ValueError as error: - return [ - f"P4: the frontend startup is not a valid router configuration: {error}" - ] - # A worker type that starts with no overlap credit can never get it. - zero_credit = sorted( - wt - for wt, params in startup_maps.items() - if wt != "decode" and params[OVERLAP_SCORE_CREDIT] == 0 - ) - try: - domain = expand_router_domain( - retune, - startup_load_model=load_model, - zero_startup_credit=bool(zero_credit), - ) - except ValueError as error: - return [f"P4: retune.router does not fit the frontend startup: {error}"] - for point in grid_points(domain, {}): - try: - maps = _policy_maps(canonical_json({**router_config, **point.router}), wts) - except ValueError as error: - return [ - f"P4: retune.router candidate {point.router} is not a valid " - f"router configuration for this frontend: {error}" - ] - for wt in zero_credit: - if maps[wt][OVERLAP_SCORE_CREDIT] != 0: - return [ - f"P4: retune.router candidate {point.router} gives the {wt} " - "router overlap_score_credit > 0, but the frontend starts it " - "at 0 (a live router rejects that update)" - ] - return [] - - -def _startup_config_mismatch( - declared: Mapping[str, Any], startup: FrontendStartup -) -> Optional[str]: - """``retune.dgdr.startup_router_config`` must mean the parsed startup.""" - - try: - declared_full = json.loads( - KvRouterConfig.from_json(json.dumps(dict(declared))).to_json() - ) - except ValueError as error: - return ( - "P4: retune.dgdr.startup_router_config is not a KvRouterConfig " - f"JSON: {error}" - ) - parsed_full = json.loads( - KvRouterConfig.from_json(json.dumps(dict(startup.router_config))).to_json() - ) - keys = sorted( - key - for key in set(declared_full) | set(parsed_full) - if declared_full.get(key) != parsed_full.get(key) - ) - if not keys: - return None - diffs = "; ".join( - f"{key}: declared {declared_full.get(key)!r}, frontend {parsed_full.get(key)!r}" - for key in keys - ) - return f"P4: retune.dgdr.startup_router_config differs from the frontend ({diffs})" - - def _planner_replica_errors( components: Sequence[Mapping[str, Any]], warnings: list[str] ) -> list[str]: @@ -1372,9 +1278,8 @@ def literal(name: str) -> Optional[str]: if path is None: errors.append(f"P10: the frontend must set {ENV_TRACE_PATH}") elif mounts: - segments = segments_dir.strip("/") prefixes = [ - _join_dir(mount, segments, dgd_dir, f"$({ENV_POD_NAME})") + _join_dir(mount, segments_dir, dgd_dir, f"$({ENV_POD_NAME})") for mount in mounts for dgd_dir in (f"$({ENV_PARENT_DGD_NAME})", dgd_name) ] diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index d8cf1521f201..1dc508dac62f 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -171,17 +171,6 @@ class SweepRequest: ``retune.domains`` expands ``retune.planner.load_predictor``; its winner's predictor fields are fixed into every non-baseline candidate. ``objective`` is the metric ``score_summary`` scores each replay on. - - The remaining fields are set only by a live controller (``launch_gate`` - set) and keep their defaults in replay, where ``run_sweep`` ignores them: - ``router_domain`` is the controller's router domain (live knob name -> - values), ``deadline_s`` the absolute epoch deadline ``D_k = L_k + - timeout``, ``spec_replicas`` the DGD spec replicas by worker type the - gate reported, ``replicas_verified`` whether the recorded ready counts - cover the whole window (``False`` when the window began before the - controller started), and ``generation`` the launching controller's - generation. Abort events are passed to the sweep function, never stored - here, so the request stays picklable. """ window_index: int @@ -203,11 +192,6 @@ class SweepRequest: max_parallel: int timeout_s: float history_paths: tuple[str, ...] = () - router_domain: dict[str, tuple[float, ...]] | None = None - deadline_s: float | None = None - spec_replicas: dict[str, int] | None = None - replicas_verified: bool = True - generation: int = 0 def __post_init__(self) -> None: keys = [candidate.key() for candidate in self.candidates] @@ -223,8 +207,6 @@ def __post_init__(self) -> None: raise ValueError("max_parallel must be at least 1") if not self.timeout_s > 0: raise ValueError("timeout_s must be positive") - if self.deadline_s is not None and not math.isfinite(self.deadline_s): - raise ValueError("deadline_s must be finite") if self.predictor_candidates: labels = [label for label, _ in self.predictor_candidates] if len(set(labels)) != len(labels): @@ -256,11 +238,7 @@ class SweepResult: prepares it on ``warmup_observations`` as a host does before a swap; ``predictor_warmup_error`` is that preparation's error, if any. ``row_counts`` holds the window reader's counts and, when the history - was read, ``history_requests``. ``failure_reason`` is a stable, - machine-readable reason a live executor sets next to the free-text - ``error`` (``retune.live.contracts.NON_DECISION_FAILURES`` mean the - window was abandoned, not failed); ``details`` carries JSON-shaped - executor facts for the decision record. ``run_sweep`` sets neither. + was read, ``history_requests``. """ status: SweepStatus @@ -273,8 +251,6 @@ class SweepResult: predictor_warmup_error: str | None = None wall_s: float = 0.0 error: str | None = None - failure_reason: str | None = None - details: dict[str, Any] = field(default_factory=dict) def with_predictor_fields( diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py index 35e37715a33f..22008e851dec 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py @@ -547,26 +547,7 @@ def test_p13_uses_the_configured_run_parallelism_and_timeouts(): # --------------------------------------------------------------- helpers -@pytest.mark.parametrize( - ("sizes", "max_shards", "parts", "largest"), - [ - ((3, 9, 4), 3, (1, 3, 1), 36), - ((3, 9, 4), 2, (1, 1, 2), 54), - ((3, 9, 4), 1, (1, 1, 1), 108), - # 15 shards of 1 x 2 x 4 beat 12 of 1 x 9 x 1. - ((3, 9, 4), 16, (3, 5, 1), 8), - ((10,), 3, (3,), 4), - ((2, 2), 16, (2, 2), 1), - ((), 3, (), 1), - ], -) -def test_balanced_shard_parts(sizes, max_shards, parts, largest): - assert balanced_shard_parts(sizes, max_shards) == parts - assert largest_shard_points(sizes, parts) == largest - assert math.prod(parts) <= max_shards - - -def test_balanced_shard_parts_minimizes_the_largest_shard_exhaustively(): +def test_balanced_shard_parts_minimizes_the_rounds_exhaustively(): for sizes in ((3, 9, 4), (5, 7), (2, 3, 5), (12,), (1, 1, 30), (250,)): for max_shards in range(1, 17): best = min( @@ -577,8 +558,6 @@ def test_balanced_shard_parts_minimizes_the_largest_shard_exhaustively(): if math.prod((a, b, c)[: len(sizes)]) <= max_shards and all(p <= n for p, n in zip((a, b, c)[: len(sizes)], sizes)) ) - parts = balanced_shard_parts(sizes, max_shards) - assert largest_shard_points(sizes, parts) == best # With the Run parallelism, the split takes the fewest rounds, so # its largest shard fits and finishes exactly when the smallest # largest shard does: P13 and the executor agree. @@ -611,7 +590,7 @@ def test_balanced_shard_parts_with_parallelism(sizes, max_shards, parallelism, p @pytest.mark.parametrize( ("sizes", "max_shards", "parallelism"), - [((3, 0), 3, None), ((3,), 0, None), ((3,), 3, 0)], + [((3, 0), 3, 8), ((3,), 0, 8), ((3,), 3, 0)], ) def test_balanced_shard_parts_rejects_bad_input(sizes, max_shards, parallelism): with pytest.raises(ValueError): @@ -623,7 +602,8 @@ def test_default_router_grid_splits_into_three_shards_of_36(): assert retune is not None sizes = [len(values) for values in expand_router_domain(retune).values()] assert sizes == [3, 9, 4] - parts = balanced_shard_parts(sizes, 3) + assert retune.dgdr is not None + parts = balanced_shard_parts(sizes, 3, parallelism=retune.dgdr.run.parallelism) assert largest_shard_points(sizes, parts) == 36 <= DGDR_MAX_SHARD_POINTS diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py index 3faf3db93f2c..5c6928764f18 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py @@ -35,11 +35,7 @@ from dynamo.common.configuration.groups.kv_router_args import _KV_ROUTER_FIELDS from dynamo.frontend.frontend_args import FrontendArgGroup, FrontendConfig from dynamo.llm import KvRouterConfig -from dynamo.planner.config.planner_config import ( - DgdrExecutorConfig, - PlannerConfig, - RetuneConfig, -) +from dynamo.planner.config.planner_config import PlannerConfig from dynamo.planner.retune.live import frontend_config as fc from dynamo.planner.retune.live.contracts import FrontendStartup, PremiseReport @@ -74,19 +70,25 @@ "moe_tensor": 1, "moe_expert": 1, } -ENGINE_AGG = { - "mode": "aggregated", - "backend": "vllm", - "backend_version": "0.11.0", - "context_length": 32768, - "workers": { - "aggregated": { - "parallelism": {"preset": [dict(PRESET)]}, - "scheduler": {"max_batched_tokens": 8192, "max_sequences": 256}, - "kv_cache": {"block_size": 64, "capacity": {"memory_fraction": 0.9}}, - }, - "prefill": None, - "decode": None, +ENGINE_WORKER = { + "parallelism": {"preset": [PRESET]}, + "scheduler": {"max_batched_tokens": 8192, "max_sequences": 256}, + "kv_cache": {"block_size": 64, "capacity": {"memory_fraction": 0.9}}, +} +ENGINES = { + "agg": { + "mode": "aggregated", + "backend": "vllm", + "backend_version": "0.11.0", + "context_length": 32768, + "workers": {"aggregated": ENGINE_WORKER, "prefill": None, "decode": None}, + }, + "disagg": { + "mode": "disaggregated", + "backend": "vllm", + "backend_version": "0.11.0", + "context_length": 32768, + "workers": {"prefill": ENGINE_WORKER, "decode": ENGINE_WORKER}, }, } @@ -220,32 +222,30 @@ def dgd(*components: dict[str, Any], **spec: Any) -> dict[str, Any]: } -def retune_config(router: Any = "default", **dgdr: Any) -> RetuneConfig: - return RetuneConfig( - interval_seconds=21600.0, - timeout_seconds=20400.0, - router=router, - executor="dgdr", - dgdr=DgdrExecutorConfig( - **{ +def planner_config( + mode: str = "agg", router: Any = "default", **dgdr: Any +) -> PlannerConfig: + """A validated live retune config (P1-P3, P5 and P13 hold).""" + + return PlannerConfig( + namespace="test-ns", + environment="kubernetes", + mode=mode, + enable_load_scaling=False, + enable_throughput_scaling=False, + retune={ + "interval_seconds": 21600.0, + "timeout_seconds": 20400.0, + "router": router, + "executor": "dgdr", + "dgdr": { "claim_name": CLAIM, "model": MODEL, "sku": "h200_sxm", - "engine": copy.deepcopy(ENGINE_AGG), + "engine": copy.deepcopy(ENGINES[mode]), **dgdr, - } - ), - ) - - -def planner_config( - mode: str = "agg", router: Any = "default", **dgdr: Any -) -> PlannerConfig: - # model_construct keeps these tests independent of the live-config - # validators (see test_live_retune_config.py); evaluate_premises reads - # only mode and retune. - return PlannerConfig.model_construct( - mode=mode, retune=retune_config(router, **dgdr) + }, + }, ) @@ -807,19 +807,6 @@ def test_recipe_dgd_satisfies_every_premise(): assert not [w for w in report.warnings if w.startswith("P6")] -def test_validated_planner_config_satisfies_every_premise(): - config = PlannerConfig( - namespace="test-ns", - environment="kubernetes", - mode="agg", - enable_load_scaling=False, - enable_throughput_scaling=False, - retune=retune_config().model_dump(), - ) - report = evaluate(dgd(), config=config) - assert report.spec_ok, report.reasons - - def test_parse_error_fails_p4_only(): deployment = with_frontend( command=("/bin/sh", "-c"), args=("python3 -m dynamo.frontend",) @@ -976,17 +963,6 @@ def test_p4_unsearched_knob_when_router_domain_is_pinned(): assert report.p4_ok, report.reasons -def test_p4_without_router_search_requires_all_defaults(): - config = planner_config(router=None) - ok = evaluate(dgd(), config=config) - assert ok.p4_ok, ok.reasons - report = evaluate( - with_frontend(args=(*KV_ARGS, "--router-temperature", "0.5")), config=config - ) - assert any("router_temperature=0.5" in r for r in report.reasons) - assert "(none)" in report.reasons[0] - - def test_p4_zero_startup_credit_rejects_positive_candidates(): zero = (*KV_ARGS, "--router-kv-overlap-score-credit", "0") # The default domain collapses the omitted credit to [0.0]. @@ -1016,37 +992,6 @@ def test_p4_disagg_decode_credit_zero_is_not_a_zero_credit_startup(): assert set(report.startup.startup_maps) == {"prefill", "decode"} -def test_p4_written_load_model_must_match_startup(): - config = planner_config(router={"prefill_load_model": {"type": "ais"}}) - report = evaluate(dgd(), config=config) - assert not report.p4_ok - assert "prefill_load_model" in report.reasons[0] - - -@pytest.mark.parametrize( - ("declared", "match"), - [ - ({}, None), - ({"router_temperature": 0.5}, None), - ({"router_temperature": 0.5, "use_kv_events": True}, None), - ({"router_temperature": 0.4}, "router_temperature: declared 0.4, frontend 0.5"), - ({}, "router_temperature: declared 0.0"), - ({"not_a_field": 1}, "not a KvRouterConfig JSON"), - ], -) -def test_startup_router_config_must_match(declared, match): - config = planner_config(startup_router_config=declared) - args = (*KV_ARGS, "--router-temperature", "0.5") - if declared == {} and match is None: - args = KV_ARGS - report = evaluate(with_frontend(args=args), config=config) - if match is None: - assert report.p4_ok, report.reasons - else: - assert not report.p4_ok - assert any(match in r for r in report.reasons), report.reasons - - # ------------------------------------------- P4: worker router advertisement @@ -1550,12 +1495,14 @@ def test_p9_scaling_adapter_and_planner_replicas(): P10_CASES = [ - ({"DYN_REQUEST_TRACE": None}, None, "DYN_REQUEST_TRACE=1"), ({"DYN_REQUEST_TRACE": "0"}, None, "DYN_REQUEST_TRACE=1"), ({"DYN_REQUEST_TRACE": "true"}, None, None), - ({"DYN_REQUEST_TRACE_SINKS": None}, None, "DYN_REQUEST_TRACE_SINKS=file"), ({"DYN_REQUEST_TRACE_SINKS": "stderr"}, None, "DYN_REQUEST_TRACE_SINKS=file"), - ({"DYN_REQUEST_TRACE_SINKS": "jsonl_gz"}, None, "DYN_REQUEST_TRACE_SINKS=file"), + ( + {"DYN_REQUEST_TRACE_SINKS": "file,jsonl_gz"}, + None, + "DYN_REQUEST_TRACE_SINKS=file", + ), ({"DYN_REQUEST_TRACE_SINKS": "file,stderr"}, None, None), ({"DYN_REQUEST_TRACE_FILE_FORMAT": None}, None, None), ({"DYN_REQUEST_TRACE_FILE_FORMAT": "jsonl"}, None, "DYN_REQUEST_TRACE_FILE_FORMAT"), @@ -1632,15 +1579,12 @@ def test_p9_scaling_adapter_and_planner_replicas(): None, "must be a literal value", ), - ({"DYN_POD_NAME": None}, None, "DYN_POD_NAME from fieldRef"), - ({"DYN_POD_NAME": "fixed"}, None, "DYN_POD_NAME from fieldRef"), ( {"DYN_POD_NAME": {"valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}}}, None, "DYN_POD_NAME from fieldRef", ), ({"DYN_PARENT_DGD_K8S_NAME": "other"}, None, "overrides DYN_PARENT_DGD_K8S_NAME"), - ({}, {"volumes": []}, "no volume for claim"), ( {}, {"volumes": [{"name": "traces", "persistentVolumeClaim": {"claimName": "x"}}]}, @@ -1652,11 +1596,6 @@ def test_p9_scaling_adapter_and_planner_replicas(): {"mounts": [{"name": "traces", "mountPath": "/trace", "readOnly": True}]}, "read-only or through a subPath", ), - ( - {}, - {"mounts": [{"name": "traces", "mountPath": "/trace", "subPath": "segments"}]}, - "read-only or through a subPath", - ), ({}, {"mounts": [{"name": "traces", "mountPath": "/trace/"}]}, None), ( {}, @@ -1695,7 +1634,7 @@ def test_p10_custom_segments_dir_and_graph_env(): planner(), env=[{"name": "DYN_REQUEST_TRACE_FILE_PATH", "value": path}], ) - report = evaluate(deployment, config=planner_config(segments_dir="/rt/segs/")) + report = evaluate(deployment, config=planner_config(segments_dir="rt/segs")) assert report.spec_ok, report.reasons @@ -1719,7 +1658,7 @@ def test_p10_is_checked_when_parsing_fails(): def test_evaluate_requires_dgdr(): - config = PlannerConfig.model_construct(mode="agg", retune=None) + config = PlannerConfig(namespace="test-ns", environment="kubernetes", mode="agg") with pytest.raises(ValueError, match="retune.dgdr"): fc.evaluate_premises(dgd(), planner_config=config) From 49d2b8f0b525f55d7dfe3e02b4592aeb57563edf Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 12:08:04 -0800 Subject: [PATCH 087/100] fix(planner): address second review on the live retune window cut - Publish the window file with mode 0644: the Run's sweeper reads it as uid 1000 with supplemental group 65532 and no fsGroup, so 0600 fails every Run of a Planner that runs as another uid. - Excuse a truncated tail at the end of any segment, not only the newest: after an in-place container restart the sink continues in a new segment, so the crashed writer's last flush ends an older one. Drop the newest-segment plumbing and segment_index. - Remove the test-only temp_file_window_end, window_file_name and window_relpath. The reconciler imports TEMP_PREFIX, SEGMENT_SUFFIX and the temp-name regex from window_cut instead of redefining them, and its margins and Kubernetes timeout are module constants. - Re-add the contracts.py types this layer uses first (labels, Kubernetes access, SweepRegistry, the window cutter and the metrics sink). - Trim the module docstrings, the segment-walk errno table and the timing-dependent registry stress test. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/retune/live/contracts.py | 225 +++++++++++++++++- .../dynamo/planner/retune/live/reconciler.py | 83 ++----- .../dynamo/planner/retune/live/window_cut.py | 146 ++++-------- .../tests/unit/test_live_retune_reconciler.py | 70 +----- .../tests/unit/test_live_retune_window_cut.py | 118 ++++----- 5 files changed, 344 insertions(+), 298 deletions(-) diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py index fa0b0d5144dc..f280adc1222f 100644 --- a/components/src/dynamo/planner/retune/live/contracts.py +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -1,12 +1,125 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Types shared by the live retune modules.""" +"""Types shared by the live retune modules. + +Every module in ``dynamo.planner.retune.live`` depends on this one and on +nothing else in the package except through the protocols defined here, so +each can be built and tested with fakes. +""" from __future__ import annotations -from dataclasses import dataclass -from typing import Any, Optional +import threading +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal, Optional, Protocol + +LABEL_PREFIX = "nvidia.com/planner-retune-" +LABEL_DGD = LABEL_PREFIX + "dgd" +LABEL_DGD_UID = LABEL_PREFIX + "dgd-uid" +LABEL_WINDOW = LABEL_PREFIX + "window" + + +# ------------------------------------------------------------ kubernetes + + +class KubeError(Exception): + """A Kubernetes call failed (network, timeout, 5xx, unexpected status).""" + + +@dataclass(frozen=True) +class DgdRef: + """The DynamoGraphDeployment this planner serves.""" + + name: str + namespace: str # Kubernetes namespace + uid: str + dynamo_namespace: str # the routers' ``namespace`` on GET /router_policy + + +@dataclass(frozen=True) +class PodInfo: + name: str + uid: str + ip: Optional[str] + ready: bool + terminating: bool + labels: Mapping[str, str] = field(default_factory=dict) + # Container port named ``http`` of the frontend container, if any. + http_port: Optional[int] = None + # ``status.phase`` (Pending, Running, Succeeded, Failed, Unknown). + phase: Optional[str] = None + + +class RetuneKube(Protocol): + """Synchronous Kubernetes access for live retune. + + Every method takes an optional ``timeout_s`` (connect+read, applied as + the kubernetes client's ``_request_timeout`` tuple of ints) and raises + the ``Kube*`` errors above instead of client exceptions. Callers on the + event loop wrap calls in ``asyncio.to_thread``. + """ + + namespace: str + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + ... + + def list_pods( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[PodInfo]: + ... + + def create_request( + self, + body: dict, + *, + dry_run: bool = False, + timeout_s: float = 10.0, + ) -> dict: + """Create with ``field_validation='Strict'`` (and ``dry_run='All'``).""" + ... + + def get_request(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def list_requests( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + ... + + def delete_request( + self, + name: str, + *, + propagation: Literal["Background", "Foreground"] = "Background", + timeout_s: float = 10.0, + ) -> None: + """Missing objects are not an error.""" + ... + + def get_run(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def list_candidates( + self, label_selector: str, *, timeout_s: float = 10.0 + ) -> list[dict]: + ... + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + ... + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + """Update with ``body['metadata']['resourceVersion']``; 409 -> KubeConflict.""" + ... + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + ... + # ------------------------------------------------------- frontend config @@ -44,3 +157,109 @@ class PremiseReport: reasons: tuple[str, ...] = () warnings: tuple[str, ...] = () startup: Optional[FrontendStartup] = None + + +# ----------------------------------------------------------- reconciler + + +class SweepRegistry(Protocol): + """Thread-safe link between a running sweep and the reconciler. + + Protection is reference-counted per window: every ``protect`` call + takes one reference and needs one matching ``release``. A window, with + the union of every Request name given to it, stays protected while any + reference is held, so two sweeps of one window (a rebuilt controller + relaunching it while the old sweep winds down) cannot drop each + other's protection. + """ + + def protect(self, window_end_ms: int, request_names: Iterable[str]) -> None: + """Mark a window file and Requests as in use by this process (one + reference).""" + ... + + def release(self, window_end_ms: int) -> None: + """Drop one reference taken by ``protect``.""" + ... + + def enqueue_delete(self, request_names: Iterable[str]) -> None: + ... + + +# ---------------------------------------------------------------- window + + +@dataclass(frozen=True) +class WindowCutResult: + path: Optional[str] # None for an empty window (no file written) + num_requests: int + trace_block_size: Optional[int] + row_counts: dict[str, int] + dropped_members: int = 0 + dropped_bytes: int = 0 + reused: bool = False # the final file already existed (EEXIST) + + +class WindowCutter(Protocol): + def __call__( + self, + segments_root: str, + start_ms: int, + end_ms: int, + as_of_ms: int, + out_path: str, + *, + max_rows: int, + deadline_s: float, + cancel: Optional[threading.Event] = None, + ) -> WindowCutResult: + ... + + +# --------------------------------------------------------------- metrics + + +class RetuneMetricsSink(Protocol): + """``dynamo_planner_retune_*`` metrics; every method is cheap and safe.""" + + def decision(self, outcome: str) -> None: + ... + + def failure(self, reason: str) -> None: + ... + + def skipped(self, reason: str) -> None: + ... + + def supervisor_state( + self, + *, + leader: bool, + ready: bool, + initialized: bool, + rollback_pending: bool, + generation: int, + ) -> None: + ... + + def scores( + self, + *, + baseline: Optional[float], + best: Optional[float], + sweep_seconds: Optional[float], + missing_candidates: int, + ) -> None: + ... + + def router_converged(self, converged: bool) -> None: + ... + + def patch_error(self, code: str) -> None: + ... + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + ... + + def dropped_members(self, count: int) -> None: + ... diff --git a/components/src/dynamo/planner/retune/live/reconciler.py b/components/src/dynamo/planner/retune/live/reconciler.py index 767cea640033..614a3b47e2a9 100644 --- a/components/src/dynamo/planner/retune/live/reconciler.py +++ b/components/src/dynamo/planner/retune/live/reconciler.py @@ -15,42 +15,23 @@ - Segments, only when the frontend pod set is known and nonempty: never the highest-numbered segment of a live pod; other segments of a live pod, and the whole directory of a pod that no longer exists, once every file is - older than ``T + G + timeout + visibility margin``. A live pod's segments - are the files directly in its directory: the frontend premise check - requires the sink's file prefix to sit directly under - ``$(DYN_POD_NAME)/``. A pod directory that - cannot be read is skipped for the round; the other pods are still - collected. + older than ``T + G + timeout + VISIBILITY_MARGIN_S``. - Window files, never while protected: ``window_end_ms`` in ``[last_window_end_ms, due_window_end_ms]`` with ``now < D_k``, or in use by this process's sweep (``protect``). Otherwise deleted when at least one Request references the file and every referencing Request's current Run has ``Completed=True`` and no Job pod left, or when no Request references it and it is older than ``T + timeout``. -- Temp files older than ``cut_budget_seconds`` plus a margin, unless their - window is in use by this process's sweep. +- Temp files older than ``cut_budget_seconds + TEMP_MARGIN_S``, unless + their window is in use by this process's sweep. - Requests of this DGD name, never one this process's sweep protects: a terminal Request outside the newest ``keep_requests`` windows, a non-terminal Request whose window deadline has passed, a Request in the delete queue, and a Request of another DGD uid (the DGD was recreated). -``run_once(should_continue=...)`` asks the caller before every delete, -unlink or Kubernetes read and ends the round as soon as it says no -(leadership or the term was lost, or the round's budget passed, while it -ran on its worker thread). ``set_dgd`` follows the DGD identity when the DGD -is recreated under the same name. Conditions that persist across rounds -(a frontend without a trace directory, an unlabelled Request) are logged -when they start, not every round. - -``Reconciler`` is also the ``SweepRegistry`` the DGDR sweep uses to protect -its window and Requests and to queue deletions; those methods are thread-safe -(the sweep runs on the controller's executor thread). Protection is -reference-counted per window: each ``protect`` call takes one reference, -each ``release`` drops one, and the window with the union of every name -given to it stays protected while any reference is held. Two sweeps of the -same window (a rebuilt controller relaunching it while the old one winds -down) therefore cannot drop each other's protection. ``run_once`` blocks on -the filesystem and the Kubernetes API: call it with ``asyncio.to_thread``. +``Reconciler`` is also the ``SweepRegistry`` the DGDR sweep uses, from the +controller's executor thread, to protect its window and Requests and to +queue deletions. """ from __future__ import annotations @@ -76,18 +57,22 @@ RetuneKube, RetuneMetricsSink, ) +from dynamo.planner.retune.live.window_cut import ( + SEGMENT_SUFFIX, + TEMP_NAME_RE, + TEMP_PREFIX, +) logger = logging.getLogger(__name__) # Allowance for RWX attribute caching and the frontend's flush interval on # top of T + G + timeout before a segment counts as unused. -DEFAULT_VISIBILITY_MARGIN_S = 600.0 +VISIBILITY_MARGIN_S = 600.0 # Allowance on top of cut_budget_seconds before a temp file counts as # abandoned (the cut subprocess is killed at its deadline). -DEFAULT_TEMP_MARGIN_S = 300.0 +TEMP_MARGIN_S = 300.0 +KUBE_TIMEOUT_S = 10.0 -WINDOW_SUFFIX = ".jsonl.gz" -TEMP_PREFIX = ".tmp-" # Label the Job controller puts on a Job's pods (batchv1.JobNameLabel). JOB_NAME_LABEL = "batch.kubernetes.io/job-name" _FINISHED_POD_PHASES = frozenset({"Succeeded", "Failed"}) @@ -98,22 +83,10 @@ REQUEST_ACCEPTED_CONDITION = "Accepted" REQUEST_INVALID_SPEC_REASON = "InvalidSpec" -_WINDOW_RE = re.compile(r"^(\d+)" + re.escape(WINDOW_SUFFIX) + r"$") -_TEMP_RE = re.compile(r"^" + re.escape(TEMP_PREFIX) + r"(\d+)-") +# A window file: .jsonl.gz +_WINDOW_RE = re.compile(r"^(\d+)" + re.escape(SEGMENT_SUFFIX) + r"$") # A Rust gzip JSONL sink segment: ..jsonl.gz -_SEGMENT_RE = re.compile(r"^(.*)\.(\d{6,})\.jsonl\.gz$") - - -def window_file_name(window_end_ms: int) -> str: - return f"{int(window_end_ms)}{WINDOW_SUFFIX}" - - -def window_relpath(cfg: DgdrExecutorConfig, dgd_name: str, window_end_ms: int) -> str: - """Claim-relative path of a window file (the Request's trace path).""" - - return "/".join( - (cfg.windows_dir.strip("/"), dgd_name, window_file_name(window_end_ms)) - ) +_SEGMENT_RE = re.compile(r"^(.*)\.(\d{6,})" + re.escape(SEGMENT_SUFFIX) + r"$") def trace_pvc_source(spec: Any) -> tuple[Optional[str], Optional[str]]: @@ -204,27 +177,22 @@ def __init__( fs_root: str, metrics: RetuneMetricsSink, now: Callable[[], float] = time.time, - *, - visibility_margin_s: float = DEFAULT_VISIBILITY_MARGIN_S, - temp_margin_s: float = DEFAULT_TEMP_MARGIN_S, - kube_timeout_s: float = 10.0, ) -> None: self._kube = kube self._cfg = cfg self._fs_root = fs_root self._metrics = metrics self._now = now - self._timeout_s = float(kube_timeout_s) interval_s = float(retune.interval_seconds) sweep_timeout_s = float(retune.effective_timeout_seconds) self._segment_max_age_s = ( interval_s + float(retune.grace_seconds) + sweep_timeout_s - + float(visibility_margin_s) + + VISIBILITY_MARGIN_S ) self._window_max_age_s = interval_s + sweep_timeout_s - self._temp_max_age_s = float(cfg.cut_budget_seconds) + float(temp_margin_s) + self._temp_max_age_s = float(cfg.cut_budget_seconds) + TEMP_MARGIN_S self._lock = threading.Lock() # Guarded by _lock (set_dgd may run while a round is on its thread). @@ -351,7 +319,8 @@ def run_once( actuator's listing (None when it failed). ``should_continue`` is called before every delete, unlink or Kubernetes read; once it returns False (or raises) the round deletes and reads nothing more - and returns.""" + and returns. It blocks on the filesystem and the Kubernetes API: + call it with ``asyncio.to_thread``.""" now = float(self._now()) with self._lock: @@ -610,7 +579,7 @@ def _collect_window_entry( if not entry.is_file(follow_symlinks=False): return if entry.name.startswith(TEMP_PREFIX): - temp = _TEMP_RE.match(entry.name) + temp = TEMP_NAME_RE.match(entry.name) window = int(temp.group(1)) if temp is not None else None # A temp file of a window this process's sweep is cutting # belongs to the running cut subprocess. @@ -643,7 +612,7 @@ def _collect_window_entry( def _list_requests(self, dgd: DgdRef) -> list[_RequestView]: items = self._kube.list_requests( - f"{LABEL_DGD}={dgd.name}", timeout_s=self._timeout_s + f"{LABEL_DGD}={dgd.name}", timeout_s=KUBE_TIMEOUT_S ) views = [] prefix = self._cfg.windows_dir.strip("/") + "/" + dgd.name + "/" @@ -731,7 +700,7 @@ def _collect_requests( if not self._proceed(state): return try: - self._kube.delete_request(name, timeout_s=self._timeout_s) + self._kube.delete_request(name, timeout_s=KUBE_TIMEOUT_S) except KubeError as exc: logger.warning( "retune reconciler: deleting Request %s (%s) failed: %s", @@ -829,7 +798,7 @@ def _finished_with(self, state: _Round, view: _RequestView) -> bool: finished = job is None or not any( pod.phase not in _FINISHED_POD_PHASES for pod in self._kube.list_pods( - f"{JOB_NAME_LABEL}={job}", timeout_s=self._timeout_s + f"{JOB_NAME_LABEL}={job}", timeout_s=KUBE_TIMEOUT_S ) ) except KubeError as exc: @@ -863,7 +832,7 @@ def _current_run( if ref.get("name") ) for name in names: - run = self._kube.get_run(name, timeout_s=self._timeout_s) + run = self._kube.get_run(name, timeout_s=KUBE_TIMEOUT_S) if run is None: continue annotations = _dig(run, "metadata", "annotations") or {} diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py index 96d485a6775c..c428939e9bdd 100644 --- a/components/src/dynamo/planner/retune/live/window_cut.py +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -12,48 +12,17 @@ block-size rule and ``row_counts`` are those of ``retune.window.read_trace_window`` over the segments the cut reads, and the rows are the same bytes; only the output is gzip-compressed and the memory -is bounded. Only ``.jsonl.gz`` segments are read: the frontend's live trace -sink must write gzip, and the reconciler collects only those segments. - -* **Segments.** A row is appended to its segment when the request - completes, after it arrived, so a segment last modified before - ``start_ms`` holds no row of the window and is not read; the - ``SEGMENT_MTIME_MARGIN_S`` margin covers clock skew between the frontends, - which stamp arrivals, and the file server, which stamps modification - times. The reconciler keeps segments well past the window, so this skips - most of what it retains. -* **Members.** Each gzip member is decoded on its own and checked against its - CRC32 and ISIZE before any of its lines is read. A member that fails (cut - short by a segment still being written, a zero-filled hole a lagging RWX - client shows, a flipped bit) is dropped whole and the reader resynchronises - on the next gzip header, so one run of bad bytes may hold many members. - Runs of bad bytes above ``MAX_DROPPED_MEMBER_RATIO`` of the members, or - bad bytes above that ratio of the gzip bytes, fail the cut - (``trace_corrupt``); a run that reaches the end of a writer's newest - segment is its in-flight flush (or a crashed writer's last one) and is - counted but does not count toward either ratio. -* **Pass 1** parses every row of the segments it reads as the replay reader - does (a row it would reject, in a member whose CRC is good, fails the cut - as ``trace_corrupt``) and keeps only columns for each selected row (~45 B: - arrival, completion, a 128-bit digest of ``request_id``, where its line - is, block size, flags). Duplicates are removed by digest, keeping the - first in file order, whenever the columns outgrow ``max_rows`` by a slack, - so ``max_rows`` bounds memory and is checked on de-duplicated rows. -* **Pass 2** re-reads only the members that hold kept rows, in output order - (arrival, completion, then ``request_id`` within equal pairs), re-parses - each line with its hashes and streams ``_window_record`` rows to gzip. - Decoded lines wait in a bounded cache that evicts the member needed - furthest in the future. A member or row that no longer matches pass 1 - fails the cut (the segment changed). -* **Atomic output.** The file is one gzip member whose header comment holds - the cut's counts as JSON. It is written to ``.tmp--`` - next to ``out_path``, fsynced, made 0600 and hard-linked into place. An - existing ``out_path`` (another process cut the same window) is checked - before any segment is read and reused with the counts in its header, so a - later leader adopting a published window never cuts it again. -* **Child process.** ``cut_window`` runs the cut in a ``spawn`` child, like - the replay sweep's workers, and kills it at the deadline or on cancel, so - a hung RWX read or a runaway cut never holds the sweep thread. +is bounded. The cut runs in a child process, reads the segments in two +passes (``_Selection``, ``_write_rows``) and publishes the file atomically +(``_publish``). + +A gzip member that fails validation is dropped whole. Runs of bad bytes +above ``MAX_DROPPED_MEMBER_RATIO`` of the members, or bad bytes above that +ratio of the gzip bytes, fail the cut (``trace_corrupt``). A run that +reaches the end of a segment is a writer's in-flight flush, or the last +flush of a writer that crashed, and is counted but does not count toward +either ratio: after an in-place container restart the sink continues in a +new segment, so that tail can end any segment. """ from __future__ import annotations @@ -112,12 +81,15 @@ def write(self, data: bytes, /) -> Any: ] WINDOW_CUT_FAILURES: tuple[str, ...] = get_args(WindowCutFailure) -# Runs of bad bytes (other than in-flight tails) over all members, or their +# Runs of bad bytes (other than segment tails) over all members, or their # bytes over all gzip bytes, above which the cut fails as trace_corrupt. A run # counts once however many members it wiped out (a zero-filled hole); its # bytes measure how much of the trace it took. MAX_DROPPED_MEMBER_RATIO = 0.01 -# Segments last modified more than this before start_ms are not read. +# A row is appended when its request completes, after it arrived, so a +# segment last modified before start_ms holds no row of the window and is not +# read. The margin covers clock skew between the frontends, which stamp +# arrivals, and the file server, which stamps modification times. SEGMENT_MTIME_MARGIN_S = 300.0 # Window files are written fast rather than small: on hash-chain rows level # 1 is ~5x faster than level 6 for ~8% more bytes, and the cut budget pays @@ -125,10 +97,9 @@ def write(self, data: bytes, /) -> Any: OUTPUT_COMPRESSLEVEL = 1 # Temporary window files: ``.tmp--`` next to out_path. TEMP_PREFIX = ".tmp-" +TEMP_NAME_RE = re.compile(r"^" + re.escape(TEMP_PREFIX) + r"(\d+)-[0-9A-Za-z]+$") SEGMENT_SUFFIX = ".jsonl.gz" -_TEMP_NAME = re.compile(r"^\.tmp-(\d+)-([0-9A-Za-z]+)$") -_SEGMENT_INDEX = re.compile(r"\.(\d+)\.jsonl\.gz$") _GZIP_MAGIC = b"\x1f\x8b\x08" # ID1, ID2, CM=deflate _GZIP_WBITS = 16 + zlib.MAX_WBITS # one gzip member, header and trailer # The header of a window file: FLG=FCOMMENT, MTIME=0 (equal cuts are equal @@ -183,20 +154,6 @@ def temp_file_name(window_end_ms: float, nonce: Optional[str] = None) -> str: return f"{TEMP_PREFIX}{int(round(window_end_ms))}-{nonce}" -def temp_file_window_end(name: str) -> Optional[int]: - """The window end of a ``temp_file_name`` name; ``None`` for any other name.""" - - match = _TEMP_NAME.match(name) - return int(match.group(1)) if match else None - - -def segment_index(name: str) -> Optional[int]: - """The ``NNNNNN`` of a ``.NNNNNN.jsonl.gz`` segment name, else ``None``.""" - - match = _SEGMENT_INDEX.search(name) - return int(match.group(1)) if match else None - - # ------------------------------------------------------------- entry points @@ -306,12 +263,11 @@ def cut_window_inprocess( ) if result.path is None: return result - paths = [path for path, _ in files] created = _publish( out_path, temp_path, _counts_comment(result), - lambda sink: _write_rows(selection, paths, sink), + lambda sink: _write_rows(selection, files, sink), ) # Another process published it first: report the counts it holds. return result if created else _published(out_path) @@ -626,8 +582,8 @@ def _decode_exact_member(data: bytes) -> Optional[bytes]: # ---------------------------------------------------------------- pass 1 -def _segment_files(root: str) -> list[tuple[str, bool]]: - """Trace segments under ``root`` in replay-reader order, and which are newest. +def _segment_files(root: str) -> list[str]: + """Trace segments under ``root``, in replay-reader order. The files are the sorted ``*.jsonl.gz`` regular files under ``root``, in the order ``resolve_trace_files([root])`` reads them; plain ``*.jsonl`` @@ -636,9 +592,7 @@ def _segment_files(root: str) -> list[tuple[str, bool]]: on NFS: the reconciler deletes a dead pod's segments) is skipped; any other error listing or stat-ing one (EACCES, EIO, ETIMEDOUT on a soft mount, ...) fails the cut as ``window_cut_failed`` rather than silently - losing a pod's rows, as does any error on ``root`` itself. Each file - comes with whether it is the newest segment (highest ``NNNNNN``) of its - directory, the one its writer may still be appending to. + losing a pod's rows, as does any error on ``root`` itself. """ try: @@ -680,20 +634,12 @@ def walk_error(error: OSError) -> None: SEGMENT_SUFFIX, min(plain), ) - paths = sorted(found) - elif root.endswith(SEGMENT_SUFFIX): - paths = [Path(root)] - else: - raise WindowCutError( - "window_cut_failed", f"trace segment {root} is not a {SEGMENT_SUFFIX} file" - ) - newest: dict[Path, tuple[int, str]] = {} - for path in paths: - index = segment_index(path.name) - key = (-1 if index is None else index, path.name) - if path.parent not in newest or key > newest[path.parent]: - newest[path.parent] = key - return [(str(path), newest[path.parent][1] == path.name) for path in paths] + return [str(path) for path in sorted(found)] + if root.endswith(SEGMENT_SUFFIX): + return [root] + raise WindowCutError( + "window_cut_failed", f"trace segment {root} is not a {SEGMENT_SUFFIX} file" + ) def _is_regular_file(path: str) -> bool: @@ -717,12 +663,12 @@ def _unreadable(path: Any, error: OSError) -> WindowCutError: ) -# (file index, path, newest, item); an OSError item means the file vanished. -_TraceItem = tuple[int, str, bool, Union[_Member, _Garbage, OSError]] +# (file index, path, item); an OSError item means the file vanished. +_TraceItem = tuple[int, str, Union[_Member, _Garbage, OSError]] def _trace_items( - files: Sequence[tuple[str, bool]], not_before_s: float + files: Sequence[str], not_before_s: float ) -> Generator[_TraceItem, None, None]: """Every member and bad byte run of ``files``, in file order. @@ -733,7 +679,7 @@ def _trace_items( yields its error and the walk goes on; other I/O errors raise. """ - for file_index, (path, newest) in enumerate(files): + for file_index, path in enumerate(files): try: with open(path, "rb") as source: if os.fstat(source.fileno()).st_mtime < not_before_s: @@ -744,11 +690,11 @@ def _trace_items( ) continue for item in _GzipMemberReader(source): - yield file_index, path, newest, item + yield file_index, path, item except OSError as error: if error.errno not in _VANISHED: raise - yield file_index, path, newest, error + yield file_index, path, error class _Selection: @@ -758,7 +704,8 @@ class _Selection: (BLAKE2b of ``request_id``), line ``line[i]`` of member ``members[member[i]]``, ``block[i]`` and ``flags[i]``. Rows are in file order, the order the replay reader sees them, so de-duplication keeps the - same (first) row it does. + same (first) row it does. A row the replay reader would reject, in a + member whose CRC is good, fails the cut as ``trace_corrupt``. """ def __init__( @@ -805,7 +752,7 @@ def digest(self, row: int) -> bytes: def scan(self, items: Iterable[_TraceItem]) -> None: """Consume ``_trace_items`` (in file order).""" - for file_index, path, newest, item in items: + for file_index, path, item in items: if isinstance(item, _Member): self.valid_members += 1 self.gzip_bytes += item.end - item.offset @@ -813,7 +760,7 @@ def scan(self, items: Iterable[_TraceItem]) -> None: item.payload, file_index, path, item.offset, item.end ) elif isinstance(item, _Garbage): - self._dropped(path, item, newest=newest) + self._dropped(path, item) else: # The reconciler deletes only segments far older than any # window being cut; a row taken from this one fails pass 2. @@ -862,17 +809,17 @@ def finish(self) -> None: np.count_nonzero(flags & _FLAG_NO_TOTAL_TIME) ) - def _dropped(self, path: str, item: _Garbage, *, newest: bool) -> None: + def _dropped(self, path: str, item: _Garbage) -> None: size = item.end - item.offset self.dropped_members += 1 self.dropped_bytes += size self.gzip_bytes += size - if item.at_eof and newest: - # The flush its writer is appending, or a crashed writer's last - # one: at most one batch, written after the cut began. + if item.at_eof: + # The flush its writer is appending, or the last one of a writer + # that crashed (an in-place restart continues in a new segment). logger.debug( "retune window cut: %d trailing bytes of %s hold no complete " - "gzip member (in-flight write)", + "gzip member (in-flight or interrupted write)", size, path, ) @@ -1060,9 +1007,10 @@ def _publish( sink.close() raw.flush() os.fsync(raw.fileno()) - # Only the owner reads the window. The Run Job reads it as the planner - # image user (uid 1000), which the Planner also runs as by default. - os.chmod(temp_path, 0o600) + # The Run's sweeper reads the window as uid 1000 with supplemental + # group 65532 and no fsGroup, so it must be readable by others: 0600 + # would fail every Run of a Planner that runs as another uid. + os.chmod(temp_path, 0o644) try: os.link(temp_path, out_path) created = True @@ -1188,6 +1136,8 @@ def _write_rows(selection: _Selection, paths: Sequence[str], sink: _ByteSink) -> The order is ``read_trace_window``'s: (arrival, completion, request_id). The columns order rows by the first two; rows with an equal pair are collected and ordered by their ``request_id`` before they are written. + Only the members that hold kept rows are read again; one that no longer + matches pass 1 fails the cut (the segment changed). """ count = selection.num_rows diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py index 0706330640a2..c77c00d635ef 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py @@ -10,7 +10,6 @@ import os import shutil import stat -import threading from collections.abc import Callable from pathlib import Path from typing import Any, Optional @@ -29,9 +28,9 @@ from dynamo.planner.retune.live.reconciler import ( JOB_NAME_LABEL, RUN_GENERATION_ANNOTATION, + TEMP_MARGIN_S, + VISIBILITY_MARGIN_S, Reconciler, - window_file_name, - window_relpath, ) pytestmark = [ @@ -46,10 +45,9 @@ INTERVAL = 3600.0 GRACE = 120.0 TIMEOUT = 1800.0 -MARGIN = 600.0 -SEGMENT_AGE = INTERVAL + GRACE + TIMEOUT + MARGIN # 6120 -WINDOW_AGE = INTERVAL + TIMEOUT # 5400 -TEMP_AGE = 900.0 + 300.0 # cut_budget + temp margin +SEGMENT_AGE = INTERVAL + GRACE + TIMEOUT + VISIBILITY_MARGIN_S +WINDOW_AGE = INTERVAL + TIMEOUT +TEMP_AGE = 900.0 + TEMP_MARGIN_S # cut_budget_seconds + margin OLD = 100_000.0 # older than every threshold @@ -239,14 +237,7 @@ def make( kube = FakeKube() metrics = RecordingMetrics() reconciler = Reconciler( - kube, - DGD, - cfg, - retune, - str(tmp_path), - metrics, - now=lambda: NOW, - visibility_margin_s=MARGIN, + kube, DGD, cfg, retune, str(tmp_path), metrics, now=lambda: NOW ) return reconciler, kube, metrics @@ -443,7 +434,7 @@ def test_only_the_top_level_segments_of_a_live_pod_are_collected( def window_path(reconciler: Reconciler, window: int) -> Path: - return Path(reconciler.windows_root) / window_file_name(window) + return Path(reconciler.windows_root) / f"{window}.jsonl.gz" def test_protected_windows_are_never_deleted(tmp_path: Path) -> None: @@ -1049,50 +1040,3 @@ def test_an_old_sweeps_release_keeps_a_relaunched_sweeps_protection( report = run_once(reconciler, due=window_ms(0)) assert report.deleted_requests == ("rt-new", "rt-old") assert report.deleted_windows == (window,) - - -def test_registry_is_thread_safe_during_rounds(tmp_path: Path) -> None: - reconciler, kube, _ = make(tmp_path, keep_requests=0) - for k in range(10): - kube.add_request(f"rt-{k}", window_ms(20 + k), state="completed") - errors: list[BaseException] = [] - stop = threading.Event() - - def churn(offset: int) -> None: - try: - i = 0 - while not stop.is_set(): - window = 1_000 * offset + i % 50 - reconciler.protect(window, [f"x-{offset}-{i}"]) - reconciler.enqueue_delete([]) - reconciler.release(window) - i += 1 - except BaseException as exc: # surfaced below - errors.append(exc) - - threads = [threading.Thread(target=churn, args=(n,)) for n in range(4)] - for thread in threads: - thread.start() - try: - for _ in range(20): - run_once(reconciler, due=window_ms(0)) - finally: - stop.set() - for thread in threads: - thread.join() - assert errors == [] - assert reconciler.protected_windows() == frozenset() - assert kube.requests == {} - - -# ------------------------------------------------------------------ helpers - - -def test_window_relpath_matches_the_reconciler_layout(tmp_path: Path) -> None: - reconciler, _, _ = make(tmp_path) - cfg = DgdrExecutorConfig( - claim_name="traces", model="m", sku="s", engine={}, windows_dir="/w/x/" - ) - assert window_relpath(cfg, "dgd", 1234) == "w/x/dgd/1234.jsonl.gz" - assert reconciler.windows_root == str(tmp_path / "windows" / "dgd") - assert reconciler.segments_root == str(tmp_path / "segments" / "dgd") diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py index a0fcebb78c79..ccd2dfeb20e6 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -28,6 +28,7 @@ from dynamo.planner.retune.live.window_cut import ( MAX_DROPPED_MEMBER_RATIO, SEGMENT_MTIME_MARGIN_S, + TEMP_NAME_RE, WINDOW_CUT_FAILURES, WindowCutError, _decode_exact_member, @@ -38,9 +39,7 @@ _segment_files, cut_window, cut_window_inprocess, - segment_index, temp_file_name, - temp_file_window_end, ) from dynamo.planner.retune.window import ROW_COUNT_KEYS, read_trace_window @@ -391,7 +390,7 @@ def test_cut_matches_read_trace_window(tmp_path, seed): assert result.dropped_members == 0 assert result.dropped_bytes == 0 assert not result.reused - assert stat.S_IMODE(out.stat().st_mode) == 0o600 + assert stat.S_IMODE(out.stat().st_mode) == 0o644 assert _temp_files(out.parent) == [] @@ -603,38 +602,29 @@ def _cut_and_twin(tmp_path, build): return result -def test_in_flight_tail_of_the_newest_segment_is_not_corruption(tmp_path): - lost = _member([_row("in-flight", START + 900)]) +@pytest.mark.parametrize("truncated", ["req.000001", "req.000000"]) +def test_truncated_tail_of_a_segment_is_not_corruption(tmp_path, truncated): + # req.000001's tail is the flush its writer is appending. req.000000's is + # the last flush of a writer that crashed: after an in-place restart the + # sink continues in req.000001. + lost = _member([_row("lost", START + 900)]) def build(root, bad): - _write(root / "a" / "req.000000.jsonl.gz", *_tiny_members("old", 3)) - members = _tiny_members("new", 2, START + 100) - _write( - root / "a" / "req.000001.jsonl.gz", - *members, - *([lost[: len(lost) // 2]] if bad else []), - ) + for name, prefix in (("req.000000", "old"), ("req.000001", "new")): + tail = [lost[: len(lost) // 2]] if bad and name == truncated else [] + _write( + root / "a" / f"{name}.jsonl.gz", + *_tiny_members(prefix, 3), + *tail, + ) result = _cut_and_twin(tmp_path, build) - # 1 of 6 members is far above the ratio, but it is the writer's tail. + # 1 of 7 members is far above the ratio, but it is a writer's tail. assert result.dropped_members == 1 assert result.dropped_bytes == len(lost) // 2 -def test_truncated_tail_of_an_older_segment_is_corruption(tmp_path): - root = tmp_path / "segments" - lost = _member([_row("lost", START + 900)]) - _write(root / "a" / "req.000000.jsonl.gz", *_tiny_members("old", 3), lost[:20]) - _write(root / "a" / "req.000001.jsonl.gz", *_tiny_members("new", 3)) - - with pytest.raises(WindowCutError) as caught: - _cut(root, tmp_path) - assert caught.value.reason == "trace_corrupt" - assert "1 of 7 gzip members" in str(caught.value) - assert "req.000000.jsonl.gz@" in str(caught.value) - - @pytest.mark.parametrize("damage", ["partial", "crc", "hole"]) def test_damaged_members_within_the_ratio_are_dropped(tmp_path, damage): # 1 bad member among 100 is exactly the limit: the cut goes on. Its bytes @@ -676,8 +666,7 @@ def test_damaged_members_above_the_ratio_are_trace_corrupt(tmp_path): assert "2 of 100 gzip members" in str(caught.value) -@pytest.mark.parametrize("newest", [True, False]) -def test_hole_spanning_many_members_is_trace_corrupt(tmp_path, newest): +def test_hole_spanning_many_members_is_trace_corrupt(tmp_path): # A zero-filled hole over 20 of 200 members is one run of bad bytes: 1 of # 181 members is within the ratio, but 10% of the bytes is not. root = tmp_path / "segments" @@ -685,16 +674,13 @@ def test_hole_spanning_many_members_is_trace_corrupt(tmp_path, newest): hole = sum(map(len, members[50:70])) data = b"".join([*members[:50], bytes(hole), *members[70:]]) _write(root / "a" / "req.000000.jsonl.gz", data) - if not newest: - _write(root / "a" / "req.000001.jsonl.gz", *_tiny_members("next", 3)) - total_members = 181 + (0 if newest else 3) with pytest.raises(WindowCutError) as caught: _cut(root, tmp_path) assert caught.value.reason == "trace_corrupt" message = str(caught.value) - assert f"1 of {total_members} gzip members" in message + assert "1 of 181 gzip members" in message assert f"({hole} of " in message assert f"req.000000.jsonl.gz@{sum(map(len, members[:50]))}+{hole}" in message @@ -715,7 +701,7 @@ def build(root, bad): assert result.dropped_bytes == len(members[50]) -def test_segment_files_order_and_newest(tmp_path): +def test_segment_files_order(tmp_path): root = tmp_path / "segments" for name in ( "b/req.000999.jsonl.gz", @@ -730,10 +716,10 @@ def test_segment_files_order_and_newest(tmp_path): files = _segment_files(str(root)) assert files == [ - (str(root / "a/req.000000.jsonl.gz"), True), - (str(root / "b/req.000999.jsonl.gz"), False), - (str(root / "b/req.1000000.jsonl.gz"), True), - (str(root / "c/single.jsonl.gz"), True), + str(root / "a/req.000000.jsonl.gz"), + str(root / "b/req.000999.jsonl.gz"), + str(root / "b/req.1000000.jsonl.gz"), + str(root / "c/single.jsonl.gz"), ] @@ -785,7 +771,7 @@ def test_vanished_file_is_skipped(tmp_path, monkeypatch): _write(root / "a" / "req.000000.jsonl.gz", _member([_row("a", START)])) listed = _segment_files(str(root)) gone = str(root / "a" / "req.000001.jsonl.gz") - monkeypatch.setattr(window_cut, "_segment_files", lambda _: [*listed, (gone, True)]) + monkeypatch.setattr(window_cut, "_segment_files", lambda _: [*listed, gone]) result, _ = _cut(root, tmp_path) @@ -840,43 +826,28 @@ def test_unreadable_segments_root_fails(tmp_path): assert str(root) in str(caught.value) -@pytest.mark.parametrize("where", ["pod-list", "file-stat", "root-list", "root-stat"]) -@pytest.mark.parametrize( - "code", [errno.ENOENT, errno.ESTALE, errno.EIO, errno.ETIMEDOUT, errno.EACCES] -) -def test_segment_walk_skips_only_vanished_entries(tmp_path, monkeypatch, where, code): +@pytest.mark.parametrize("code", [errno.ENOENT, errno.EACCES]) +def test_segment_walk_skips_only_vanished_entries(tmp_path, monkeypatch, code): root = _two_pods(tmp_path / "segments") - target = { - "pod-list": root / "b", - "file-stat": root / "b" / "req.000000.jsonl.gz", - "root-list": root, - "root-stat": root, - }[where] - name = "scandir" if where.endswith("list") else "stat" - real = getattr(os, name) + pod = root / "b" + real = os.scandir def failing(path, *args, **kwargs): - if os.fspath(path) == str(target): + if os.fspath(path) == str(pod): raise OSError(code, os.strerror(code), os.fspath(path)) return real(path, *args, **kwargs) - monkeypatch.setattr(os, name, failing) - vanished = code in (errno.ENOENT, errno.ESTALE) + monkeypatch.setattr(os, "scandir", failing) - if vanished and not where.startswith("root"): + if code == errno.ENOENT: # The reconciler deleted pod b's segments under the walk. - assert _segment_files(str(root)) == [ - (str(root / "a" / "req.000000.jsonl.gz"), True) - ] + assert _segment_files(str(root)) == [str(root / "a" / "req.000000.jsonl.gz")] return with pytest.raises(WindowCutError) as caught: _segment_files(str(root)) assert caught.value.reason == "window_cut_failed" - if vanished and where == "root-stat": - assert "does not exist" in str(caught.value) - else: - assert str(target) in str(caught.value) - assert os.strerror(code) in str(caught.value) + assert str(pod) in str(caught.value) + assert os.strerror(code) in str(caught.value) # -------------------------------------------------------- atomic output @@ -941,7 +912,7 @@ def racing_link(source, target): assert result == _reused(winner, out) assert out.read_bytes() == winner_out.read_bytes() - assert temp_file_window_end(linked[0]) == END + assert linked[0].startswith(f".tmp-{END}-") assert _temp_files(out.parent) == [] @@ -961,7 +932,7 @@ def vanished_link(source, target): assert _temp_files(out.parent) == [] -def test_output_is_deterministic_and_0600_under_any_umask(tmp_path): +def test_output_is_deterministic_and_0644_under_any_umask(tmp_path): root = _random_trace(tmp_path / "segments", 8) first, second = _out(tmp_path, "a.jsonl.gz"), _out(tmp_path, "b.jsonl.gz") previous = os.umask(0o077) @@ -972,7 +943,7 @@ def test_output_is_deterministic_and_0600_under_any_umask(tmp_path): os.umask(previous) assert first.read_bytes() == second.read_bytes() - assert stat.S_IMODE(first.stat().st_mode) == 0o600 + assert stat.S_IMODE(first.stat().st_mode) == 0o644 with gzip.open(first, "rb") as source: assert source.read(1) # A comment (the counts) but no file name or mtime in the gzip header. @@ -1077,20 +1048,13 @@ def test_temp_path_must_be_next_to_out_path(tmp_path): ) -def test_temp_and_segment_names(): - name = temp_file_name(1_700_000_000_000.0, "ab12") - assert name == ".tmp-1700000000000-ab12" - assert temp_file_window_end(name) == 1_700_000_000_000 - assert temp_file_window_end(temp_file_name(5)) == 5 +def test_temp_file_name(): + assert temp_file_name(1_700_000_000_000.0, "ab12") == ".tmp-1700000000000-ab12" assert temp_file_name(5) != temp_file_name(5) - assert temp_file_window_end("1700000000000.jsonl.gz") is None - assert temp_file_window_end(".tmp-12-") is None + # The reconciler finds a temp file's window with TEMP_NAME_RE. + assert TEMP_NAME_RE.match(temp_file_name(5)).group(1) == "5" with pytest.raises(ValueError): temp_file_name(5, "../x") - assert segment_index("req.000042.jsonl.gz") == 42 - assert segment_index("req.1000000.jsonl.gz") == 1_000_000 - assert segment_index("req.000042.jsonl") is None - assert segment_index("req.jsonl.gz") is None def test_window_cut_error_reason_and_pickle(): From e396992f53dfac51a056787a82c33d358dc90c8e Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 12:22:43 -0800 Subject: [PATCH 088/100] fix(planner): address second review on the live retune DGDR sweep - Record why a Request without a Run waits (RunNameConflict, RunCreating, CancelingSupersededRun) in the shard status, as for a waiting Run, so a deadline that passes there reports the reason instead of a bare timeout. - Wait between polls on cancel alone and check abort when the wait ends; every mutation already checks is_leader(). The window cutter keeps one is_set() flag for both events. No helper threads. - Explain why every Request pins the direct generator: the planner reads only Candidate parameters and scores, and the AIC generator can reject a Candidate (#17581) and fail the Run and the window. - Drop the engine checks that P5 already enforces at config load (backend, workers mapping, single preset entry, positive integers); keep the spec_replicas role and replica checks, which depend on the live DGD. - plan_shards requires the Run parallelism, as balanced_shard_parts does. - Move the DGD and Lease access (RetuneKube methods, KubernetesRetuneKube, the fake and their tests) to the layer that uses it, and delete the unused FakeRetuneKube.update_pod and FakeDgdrOperator.run_to_completion. - Re-add the contracts.py types and SweepRequest/SweepResult fields this layer uses first. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/config/planner_config.py | 5 +- .../dynamo/planner/retune/live/contracts.py | 63 +++++--- .../src/dynamo/planner/retune/live/dgdr.py | 138 +++++++----------- .../src/dynamo/planner/retune/live/kube.py | 110 +------------- components/src/dynamo/planner/retune/sweep.py | 26 +++- .../dynamo/planner/tests/live_retune_fakes.py | 118 ++------------- .../tests/unit/test_live_retune_dgdr.py | 102 +++++++------ .../tests/unit/test_live_retune_kube.py | 79 ---------- 8 files changed, 201 insertions(+), 440 deletions(-) diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index 2d31f1066996..bd80efe68cd1 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -980,8 +980,9 @@ class DgdrExecutorConfig(BaseModel): # DGDR v1beta2 ``spec.recommendation.maxCandidates`` maximum: one Request (a # grid shard) carries at most this many router points. DGDR_MAX_SHARD_POINTS = 100 -# Router knobs a DGDR v1beta2 Request can search, by public name; the -# Request's router section has no place for the retune extension knobs. +# Router knobs a DGDR v1beta2 Request can search, by public name +# (``contracts.DGDR_ROUTER_KNOBS``); the Request's router section has no +# place for the retune extension knobs. _DGDR_ROUTER_SECTION_KEYS = frozenset( { "overlap_score_credit", diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py index 47912931d06b..5c6cb41f4bb5 100644 --- a/components/src/dynamo/planner/retune/live/contracts.py +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -10,15 +10,44 @@ from __future__ import annotations -import threading from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any, Literal, Optional, Protocol +# Router knobs a DGDR v1beta2 Request can search, AISimulate public name -> +# live name. ``dynamo.router.simulation.config.ROUTER_PUBLIC_TO_LIVE`` is the +# source of truth; this mirrors it for modules that must not import the +# router simulation stack. +DGDR_ROUTER_KNOBS: dict[str, str] = { + "overlap_score_credit": "overlap_score_credit", + "prefill_load_scale": "prefill_load_scale", + "temperature": "router_temperature", +} + LABEL_PREFIX = "nvidia.com/planner-retune-" LABEL_DGD = LABEL_PREFIX + "dgd" LABEL_DGD_UID = LABEL_PREFIX + "dgd-uid" LABEL_WINDOW = LABEL_PREFIX + "window" +LABEL_SHARD = LABEL_PREFIX + "shard" +LABEL_SPEC = LABEL_PREFIX + "spec" +LABEL_ATTEMPT = LABEL_PREFIX + "attempt" + +# Pod labels the operator sets (deploy/operator/internal/consts/consts.go). +POD_LABEL_DGD_NAME = "nvidia.com/dynamo-graph-deployment-name" +POD_LABEL_COMPONENT_TYPE = "nvidia.com/dynamo-component-type" + +DGDR_GROUP = "nvidia.com" +DGDR_VERSION = "v1beta2" +DGDR_REQUESTS = "dynamographdeploymentrequests" +DGDR_RUNS = "dynamographdeploymentruns" +DGDR_CANDIDATES = "dynamographdeploymentcandidates" +RUN_UID_LABEL = "nvidia.com/dgdr-run-uid" + +# SweepResult.failure_reason values that mean "this process stopped working +# on the window", not "the window failed": no decision is recorded and the +# controller rewinds so the window can be relaunched or adopted by the next +# leader. +NON_DECISION_FAILURES = frozenset({"shutdown", "lost_leadership"}) # ------------------------------------------------------------ kubernetes @@ -28,6 +57,22 @@ class KubeError(Exception): """A Kubernetes call failed (network, timeout, 5xx, unexpected status).""" +class KubeNotFound(KubeError): + pass + + +class KubeConflict(KubeError): + """409 on update (stale resourceVersion).""" + + +class KubeAlreadyExists(KubeError): + """409 on create (the name is taken).""" + + +class KubeInvalid(KubeError): + """422 / 400: the API server rejected the object (e.g. Strict pruning).""" + + @dataclass(frozen=True) class DgdRef: """The DynamoGraphDeployment this planner serves.""" @@ -63,9 +108,6 @@ class RetuneKube(Protocol): namespace: str - def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: - ... - def list_pods( self, label_selector: str, *, timeout_s: float = 10.0 ) -> list[PodInfo]: @@ -107,19 +149,6 @@ def list_candidates( ) -> list[dict]: ... - def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - ... - - def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: - ... - - def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: - """Update with ``body['metadata']['resourceVersion']``; 409 -> KubeConflict.""" - ... - - def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: - ... - # ------------------------------------------------------- frontend config diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py index ca4de57d66f4..c1618da83bc3 100644 --- a/components/src/dynamo/planner/retune/live/dgdr.py +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -28,10 +28,10 @@ The executor never deletes anything itself: Requests to remove go to the ``SweepRegistry`` deletion queue, which the reconciler drains while this process leads. Shutdown (``cancel``) and loss of leadership (``abort``) -return at once with ``failure_reason`` ``shutdown``/``lost_leadership`` and -leave every Request in place for the next leader to adopt: the wait between -polls ends as soon as either is set. Every mutating call first checks -``abort`` and ``context.is_leader()``. +return with ``failure_reason`` ``shutdown``/``lost_leadership`` and leave +every Request in place for the next leader to adopt. Shutdown ends the wait +between polls at once; ``abort`` is checked when it ends. Every mutating +call first checks ``abort`` and ``context.is_leader()``. A shard that holds the baseline, or the probe, is critical: when it fails the window fails. Any other failed shard only leaves its points unscored. @@ -162,8 +162,8 @@ def plan_shards( baseline: Mapping[str, float], startup: Mapping[str, float], max_shards: int, + parallelism: int, probe: bool = True, - parallelism: int | None = None, ) -> list[Shard]: """Split ``router_domain`` (live knob -> values) into Request shards. @@ -171,9 +171,9 @@ def plan_shards( ``DGDR_MAX_SHARD_POINTS`` points: each knob's sorted values are cut into contiguous, near-equal chunks by ``balanced_shard_parts``, the split the configuration check (P13) uses. It minimizes the largest shard's - wall time, then the shard count: with the Run ``parallelism`` the time - is its evaluation rounds (``dgdr_rounds``), so a shard that would not - shorten the window is not added; without it, the largest shard's size. + evaluation rounds at the Run ``parallelism`` (``dgdr_rounds``), then + the shard count, so a shard that would not shorten the window is not + added. ``baseline`` and ``startup`` (live knob -> value) each get a single-point shard when off the grid. With ``probe``, a ``probe`` shard repeats one fixed grid point (the @@ -343,8 +343,10 @@ def build_request( (unused roles are deleted with ``null``); the GPU budget is the sum of replicas times GPUs per worker (tensor x pipeline x attention_data). Router knobs use their AISimulate public names: ``{choices: [...]}`` - for a grid shard, a scalar for a point or probe shard. Raises - ``ValueError`` for an engine or shard the Request cannot express. + for a grid shard, a scalar for a point or probe shard. ``cfg.engine`` + is the one the configuration check (P5) accepted. Raises + ``ValueError`` for ``spec_replicas`` or a shard the Request cannot + express. """ engine = _pinned_engine(cfg.engine, spec_replicas, backend) @@ -388,6 +390,11 @@ def build_request( "candidate_timeout_seconds": cfg.run.candidate_timeout_seconds, }, "optimization": {"strict_sla": False}, + # The planner reads only each Candidate's parameters and + # score, never its DGD. The direct generator keeps retune + # Runs independent of the AIC generator, whose rendering can + # reject a Candidate (ai-dynamo/dynamo#17581: no explicit + # block size) and so fail the Run and the window. "dgdr_run": {"generator": "direct"}, } }, @@ -424,49 +431,33 @@ def build_request( def _pinned_engine( engine: Mapping[str, Any], spec_replicas: Mapping[str, int], backend: str ) -> dict[str, Any]: - pinned = copy.deepcopy(dict(engine)) + """``engine`` with the deployed backend and the DGD spec replicas. + + The configuration check (P5) has already pinned ``engine``: its backend, + if written, is ``backend``, and each worker role of the planner mode has + one parallelism preset entry of positive integers. ``spec_replicas`` + comes from the live DGD, so its roles and counts are checked here. + """ + roles = set(spec_replicas) if roles not in ({"aggregated"}, {"prefill", "decode"}): raise ValueError( "spec_replicas must name 'aggregated', or 'prefill' and 'decode'; " f"got {sorted(roles)}" ) - written = pinned.get("backend") - if written is None: - pinned["backend"] = backend - elif written != backend: - raise ValueError( - f"retune.dgdr.engine.backend {written!r} differs from the deployed " - f"backend {backend!r}" - ) - workers = pinned.get("workers") - if not isinstance(workers, dict): - raise ValueError("retune.dgdr.engine.workers must be a mapping") + pinned = copy.deepcopy(dict(engine)) + pinned["backend"] = backend + workers = pinned["workers"] for role in _ROLES: if role not in roles: workers[role] = None continue - worker = workers.get(role) - preset = ( - worker.get("parallelism", {}).get("preset") - if isinstance(worker, dict) and isinstance(worker.get("parallelism"), dict) - else None - ) - if ( - not isinstance(preset, list) - or len(preset) != 1 - or not isinstance(preset[0], dict) - ): - raise ValueError( - f"retune.dgdr.engine.workers.{role}.parallelism.preset must be a " - "list of exactly one entry" - ) replicas = int(spec_replicas[role]) if replicas < 1: raise ValueError( f"{role} has {replicas} spec replicas; at least 1 is needed" ) - preset[0]["replicas"] = replicas + workers[role]["parallelism"]["preset"][0]["replicas"] = replicas return pinned @@ -474,15 +465,7 @@ def _gpu_budget(engine: Mapping[str, Any], spec_replicas: Mapping[str, int]) -> budget = 0 for role, replicas in spec_replicas.items(): entry = engine["workers"][role]["parallelism"]["preset"][0] - per_worker = 1 - for knob in ("tensor", "pipeline", "attention_data"): - value = entry.get(knob, 1) - if isinstance(value, bool) or not isinstance(value, int) or value < 1: - raise ValueError( - f"retune.dgdr.engine.workers.{role}.parallelism.preset[0]." - f"{knob} must be a positive integer, got {value!r}" - ) - per_worker *= value + per_worker = entry["tensor"] * entry["pipeline"] * entry["attention_data"] budget += int(replicas) * per_worker return budget @@ -660,8 +643,8 @@ class DgdrContext: ``backend`` is the deployed backend (``vllm``, ``sglang``, ``trtllm``). ``is_leader`` is checked before every mutating call. ``now`` is the wall clock (``deadline_s`` is epoch seconds). Between polls the sweep - waits on ``cancel`` and ``abort``, so either ends the wait at once; - ``sleep``, when set, replaces that wait (a test's simulated clock). + waits on ``cancel``, so shutdown ends the wait at once; ``sleep``, when + set, replaces that wait (a test's simulated clock). """ dgd: DgdRef @@ -726,11 +709,8 @@ def __init__( class _Stop: - """Shutdown (``cancel``) or loss of leadership (``abort``), as one flag. - - The window cutter only asks ``is_set()``. ``wait`` is the sweep's pause - between polls and returns as soon as either event is set. - """ + """Shutdown (``cancel``) or loss of leadership (``abort``) as the one + ``is_set()`` flag the window cutter polls.""" def __init__( self, cancel: threading.Event, abort: Optional[threading.Event] @@ -740,31 +720,6 @@ def __init__( def is_set(self) -> bool: return any(event.is_set() for event in self._events) - def wait(self, timeout: float) -> bool: - """Block up to ``timeout`` seconds; True once either event is set. - - A ``threading.Event`` is waited on alone, so with two events each - gets a helper thread that waits on it for the same ``timeout`` and - wakes this call; no thread outlives the timeout. - """ - - if len(self._events) == 1: - return self._events[0].wait(timeout) - if self.is_set(): - return True - woken = threading.Event() - - def watch(event: threading.Event) -> None: - if event.wait(timeout): - woken.set() - - for event in self._events: - threading.Thread( - target=watch, args=(event,), name="retune-dgdr-wait", daemon=True - ).start() - woken.wait(timeout) - return self.is_set() - ShardState = Literal["create", "unknown", "wait", "done", "missing"] @@ -786,8 +741,10 @@ class _ShardRun: created: bool = False reason: Optional[str] = None message: Optional[str] = None - # The open Run's Completed reason and message (JobRunning, or why it - # waits: ContainerWaiting, TraceSourceMissing); None once it ends. + # Why the shard waits: the Request's Accepted reason and message while + # it has no Run (RunNameConflict, RunCreating, CancelingSupersededRun), + # then the open Run's Completed reason and message (JobRunning, + # ContainerWaiting, TraceSourceMissing); None once the Run ends. run_status: Optional[str] = None run_name: Optional[str] = None run_uid: Optional[str] = None @@ -812,8 +769,8 @@ def require_name(self) -> str: return self.name def status(self) -> Optional[str]: - """The open Run's status, else why the shard ended or its last - attempt was replaced.""" + """Why the open Request or Run waits, else why the shard ended or + its last attempt was replaced.""" return self.run_status or self.message @@ -964,12 +921,15 @@ def _timeout(self) -> float: return max(_MIN_K8S_TIMEOUT_S, min(_K8S_TIMEOUT_S, self.deadline - self._now())) def _pause(self, seconds: float) -> None: - """Wait between polls; shutdown or abort ends the wait at once.""" + """Wait between polls. Shutdown ends the wait at once; loss of + leadership is seen when it ends, and every mutating call checks + ``is_leader()`` first anyway.""" if self.ctx.sleep is not None: self.ctx.sleep(seconds) else: - self.stop.wait(seconds) + self.cancel.wait(seconds) + self._check_interrupt() def _check_interrupt(self) -> None: if self.cancel.is_set(): @@ -1495,7 +1455,12 @@ def _observe(self, run: _ShardRun) -> None: "invalid_spec", f"Request {run.name}: {accepted.get('message', '')}", ) - # RunCreating, RunNameConflict, CancelingSupersededRun: wait. + # RunCreating, RunNameConflict, CancelingSupersededRun: wait, and + # keep why, so a deadline that passes here reports it. + run.run_status = ( + f"Request {run.name} {accepted.get('reason')}: " + f"{accepted.get('message', '')}" + ) return try: found = self._find_run(obj) @@ -1503,6 +1468,7 @@ def _observe(self, run: _ShardRun) -> None: self._read_failed(f"get the Run of Request {run.name}", error) return if found is None: + run.run_status = None return # created, not yet visible run.run_name = found["metadata"]["name"] run_uid = found["metadata"].get("uid") diff --git a/components/src/dynamo/planner/retune/live/kube.py b/components/src/dynamo/planner/retune/live/kube.py index 74f82c93d28e..feca00c907d9 100644 --- a/components/src/dynamo/planner/retune/live/kube.py +++ b/components/src/dynamo/planner/retune/live/kube.py @@ -9,12 +9,8 @@ error carries the HTTP status as ``error.status`` (``None`` for a network error or timeout), so a caller can tell 403 from the rest. -Custom resources (the DGD at v1beta1, DGDR v1beta2 Requests, Runs and -Candidates) come back as the API server's JSON. Leases are read without the -client's model deserialization (``_preload_content=False``) so a Lease dict -round-trips unchanged: deserializing turns ``renewTime`` into a ``datetime`` -whose ``isoformat`` drops a zero fraction, which a later replace would send -back in a form the ``MicroTime`` parser rejects. +Custom resources (DGDR v1beta2 Requests, Runs and Candidates) come back as +the API server's JSON. """ from __future__ import annotations @@ -22,7 +18,7 @@ import json import logging import math -from collections.abc import Callable, Mapping +from collections.abc import Callable from typing import Any, Literal, Optional, TypeVar from kubernetes import client @@ -44,8 +40,6 @@ logger = logging.getLogger(__name__) -DGD_VERSION = "v1beta1" -DGD_PLURAL = "dynamographdeployments" # The frontend's container and HTTP port names (deploy/operator/internal/consts). MAIN_CONTAINER_NAME = "main" HTTP_PORT_NAME = "http" @@ -139,15 +133,6 @@ def _http_port(spec: Any) -> Optional[int]: return None -def _decode(response: Any) -> dict: - """The JSON body of a ``_preload_content=False`` response.""" - - data = response.data - if isinstance(data, bytes): - data = data.decode("utf-8") - return json.loads(data) if data else {} - - class KubernetesRetuneKube: """``contracts.RetuneKube`` for one namespace over ``api_client``. @@ -163,7 +148,6 @@ def __init__(self, namespace: str, api_client: client.ApiClient): self.api_client = api_client self._custom = client.CustomObjectsApi(self.api_client) self._core = client.CoreV1Api(self.api_client) - self._coordination = client.CoordinationV1Api(self.api_client) @classmethod def from_kube_api(cls, kube_api: Any) -> KubernetesRetuneKube: @@ -220,18 +204,6 @@ def _list_custom( ) return list(result.get("items") or []) - # ----------------------------------------------------------------- DGD - - def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: - found = self._get_custom(DGD_VERSION, DGD_PLURAL, name, timeout_s) - if found is None: - error = KubeNotFound( - f"DynamoGraphDeployment {self.namespace}/{name} not found" - ) - error.status = 404 # type: ignore[attr-defined] - raise error - return found - # ---------------------------------------------------------------- pods def list_pods( @@ -319,79 +291,3 @@ def list_candidates( return self._list_custom( DGDR_VERSION, DGDR_CANDIDATES, label_selector, timeout_s ) - - # -------------------------------------------------------------- Leases - - def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - try: - return self._call( - "read", - f"leases/{name}", - lambda: _decode( - self._coordination.read_namespaced_lease( - name=name, - namespace=self.namespace, - _preload_content=False, - _request_timeout=request_timeout(timeout_s), - ) - ), - ) - except KubeNotFound: - return None - - def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: - name = (body.get("metadata") or {}).get("name", "") - return self._call( - "create", - f"leases/{name}", - lambda: _decode( - self._coordination.create_namespaced_lease( - namespace=self.namespace, - body=_lease_body(body), - field_manager=FIELD_MANAGER, - _preload_content=False, - _request_timeout=request_timeout(timeout_s), - ) - ), - ) - - def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: - return self._call( - "update", - f"leases/{name}", - lambda: _decode( - self._coordination.replace_namespaced_lease( - name=name, - namespace=self.namespace, - body=_lease_body(body), - field_manager=FIELD_MANAGER, - _preload_content=False, - _request_timeout=request_timeout(timeout_s), - ) - ), - ) - - def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: - """Missing Leases are not an error.""" - - try: - self._call( - "delete", - f"leases/{name}", - lambda: self._coordination.delete_namespaced_lease( - name=name, - namespace=self.namespace, - _request_timeout=request_timeout(timeout_s), - ), - ) - except KubeNotFound: - pass - - -def _lease_body(body: Mapping[str, Any]) -> dict: - """A Lease body with its type fields set (the API server requires them).""" - - lease = dict(body) - lease.setdefault("apiVersion", "coordination.k8s.io/v1") - lease.setdefault("kind", "Lease") - return lease diff --git a/components/src/dynamo/planner/retune/sweep.py b/components/src/dynamo/planner/retune/sweep.py index 1dc508dac62f..d8cf1521f201 100644 --- a/components/src/dynamo/planner/retune/sweep.py +++ b/components/src/dynamo/planner/retune/sweep.py @@ -171,6 +171,17 @@ class SweepRequest: ``retune.domains`` expands ``retune.planner.load_predictor``; its winner's predictor fields are fixed into every non-baseline candidate. ``objective`` is the metric ``score_summary`` scores each replay on. + + The remaining fields are set only by a live controller (``launch_gate`` + set) and keep their defaults in replay, where ``run_sweep`` ignores them: + ``router_domain`` is the controller's router domain (live knob name -> + values), ``deadline_s`` the absolute epoch deadline ``D_k = L_k + + timeout``, ``spec_replicas`` the DGD spec replicas by worker type the + gate reported, ``replicas_verified`` whether the recorded ready counts + cover the whole window (``False`` when the window began before the + controller started), and ``generation`` the launching controller's + generation. Abort events are passed to the sweep function, never stored + here, so the request stays picklable. """ window_index: int @@ -192,6 +203,11 @@ class SweepRequest: max_parallel: int timeout_s: float history_paths: tuple[str, ...] = () + router_domain: dict[str, tuple[float, ...]] | None = None + deadline_s: float | None = None + spec_replicas: dict[str, int] | None = None + replicas_verified: bool = True + generation: int = 0 def __post_init__(self) -> None: keys = [candidate.key() for candidate in self.candidates] @@ -207,6 +223,8 @@ def __post_init__(self) -> None: raise ValueError("max_parallel must be at least 1") if not self.timeout_s > 0: raise ValueError("timeout_s must be positive") + if self.deadline_s is not None and not math.isfinite(self.deadline_s): + raise ValueError("deadline_s must be finite") if self.predictor_candidates: labels = [label for label, _ in self.predictor_candidates] if len(set(labels)) != len(labels): @@ -238,7 +256,11 @@ class SweepResult: prepares it on ``warmup_observations`` as a host does before a swap; ``predictor_warmup_error`` is that preparation's error, if any. ``row_counts`` holds the window reader's counts and, when the history - was read, ``history_requests``. + was read, ``history_requests``. ``failure_reason`` is a stable, + machine-readable reason a live executor sets next to the free-text + ``error`` (``retune.live.contracts.NON_DECISION_FAILURES`` mean the + window was abandoned, not failed); ``details`` carries JSON-shaped + executor facts for the decision record. ``run_sweep`` sets neither. """ status: SweepStatus @@ -251,6 +273,8 @@ class SweepResult: predictor_warmup_error: str | None = None wall_s: float = 0.0 error: str | None = None + failure_reason: str | None = None + details: dict[str, Any] = field(default_factory=dict) def with_predictor_fields( diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py index 09a1cbe4aa83..5a153b5a0639 100644 --- a/components/src/dynamo/planner/tests/live_retune_fakes.py +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -4,15 +4,14 @@ """In-memory fakes of the Kubernetes side of live retune. ``FakeRetuneKube`` implements ``contracts.RetuneKube`` over plain dicts with -the API-server behaviour the live modules depend on: ``resourceVersion`` -compare-and-swap on Lease replace, ``AlreadyExists`` on create, Strict field -validation and silent pruning on Request create, label selectors, cascading -Request deletion, and injectable faults (including writes that time out but -land). ``FakeDgdrOperator`` drives the Requests in a ``FakeRetuneKube`` -through the DGDR v1beta2 lifecycle (Accepted -> Run -> Completed with -Candidates) the way the operator at #17483 does, lowering each Request into -``Run.status.aisimulate.input`` with the same merge-patch rules. The -operator's golden +the API-server behaviour the live modules depend on: ``AlreadyExists`` on +create, Strict field validation and silent pruning on Request create, label +selectors, cascading Request deletion, and injectable faults (including +writes that time out but land). ``FakeDgdrOperator`` drives the Requests in +a ``FakeRetuneKube`` through the DGDR v1beta2 lifecycle (Accepted -> Run -> +Completed with Candidates) the way the operator at #17483 does, lowering +each Request into ``Run.status.aisimulate.input`` with the same merge-patch +rules. The operator's golden ``deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json`` is the oracle for that lowering; ``test_live_retune_dgdr`` checks ``lower_request_input`` against it. @@ -32,7 +31,7 @@ import uuid from collections import deque from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from typing import Any, Literal, Optional from dynamo.planner.retune.live.contracts import ( @@ -41,10 +40,8 @@ POD_LABEL_DGD_NAME, RUN_UID_LABEL, KubeAlreadyExists, - KubeConflict, KubeError, KubeInvalid, - KubeNotFound, PodInfo, ) @@ -71,7 +68,6 @@ } KubeMethod = Literal[ - "get_dgd", "list_pods", "create_request", "get_request", @@ -79,10 +75,6 @@ "delete_request", "get_run", "list_candidates", - "get_lease", - "create_lease", - "replace_lease", - "delete_lease", ] @@ -379,8 +371,8 @@ class KubeCall: class FakeRetuneKube: """In-memory ``contracts.RetuneKube`` (thread-safe). - Stores DGDs, pods (``PodInfo``), Requests, Runs, Candidates and Leases - as plain dicts. Objects come back as deep copies, so callers can never + Stores pods (``PodInfo``), Requests, Runs and Candidates as plain + dicts. Objects come back as deep copies, so callers can never alias the store. ``now`` stamps ``creationTimestamp``. Request create honours ``field_validation='Strict'``: a path listed in @@ -400,14 +392,12 @@ def __init__( ) -> None: self.namespace = namespace self.now = now - self.dgds: dict[str, dict] = {} self.pods: dict[str, PodInfo] = {} self.requests: dict[str, dict] = {} self.runs: dict[str, dict] = {} # Each Run's lowered input, recorded in its status by ``create_job``. self.run_inputs: dict[str, dict] = {} self.candidates: dict[str, dict] = {} - self.leases: dict[str, dict] = {} self.calls: list[KubeCall] = [] self.deleted_requests: list[tuple[str, str]] = [] self.unknown_fields: set[str] = set() @@ -495,25 +485,6 @@ def bump(self, obj: dict) -> None: obj["metadata"]["resourceVersion"] = str(next(self._rv)) - # --------------------------------------------------------------- DGD - - def put_dgd(self, dgd: dict) -> dict: - with self._lock: - stored = copy.deepcopy(dgd) - self._stamp( - stored, kind="DynamoGraphDeployment", api_version="nvidia.com/v1beta1" - ) - self.dgds[stored["metadata"]["name"]] = stored - return copy.deepcopy(stored) - - def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: - def operation() -> dict: - if name not in self.dgds: - raise KubeNotFound(f"DynamoGraphDeployment {name!r} not found") - return copy.deepcopy(self.dgds[name]) - - return self._call("get_dgd", name, {"timeout_s": timeout_s}, operation) - # -------------------------------------------------------------- pods def add_pod( @@ -546,12 +517,6 @@ def add_pod( self.pods[name] = pod return pod - def update_pod(self, name: str, **changes: Any) -> PodInfo: - with self._lock: - pod = replace(self.pods[name], **changes) - self.pods[name] = pod - return pod - def remove_pod(self, name: str) -> None: with self._lock: self.pods.pop(name, None) @@ -699,62 +664,6 @@ def operation() -> list[dict]: "list_candidates", label_selector, {"timeout_s": timeout_s}, operation ) - # ------------------------------------------------------------- Leases - - def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: - def operation() -> Optional[dict]: - found = self.leases.get(name) - return None if found is None else copy.deepcopy(found) - - return self._call("get_lease", name, {"timeout_s": timeout_s}, operation) - - def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: - def operation() -> dict: - stored = copy.deepcopy(body) - name = nested_get(stored, "metadata", "name") - if not name: - raise KubeInvalid("metadata.name is required") - if name in self.leases: - raise KubeAlreadyExists(f"leases {name!r} already exists") - stored["metadata"].pop("resourceVersion", None) - self._stamp(stored, kind="Lease", api_version="coordination.k8s.io/v1") - self.leases[name] = stored - return copy.deepcopy(stored) - - return self._call("create_lease", body, {"timeout_s": timeout_s}, operation) - - def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: - def operation() -> dict: - current = self.leases.get(name) - if current is None: - raise KubeNotFound(f"leases {name!r} not found") - sent = nested_get(body, "metadata", "resourceVersion") - if sent != current["metadata"]["resourceVersion"]: - raise KubeConflict( - f"leases {name!r}: resourceVersion {sent!r} is stale " - f"(current {current['metadata']['resourceVersion']!r})" - ) - stored = copy.deepcopy(body) - metadata = stored.setdefault("metadata", {}) - for key in ("uid", "creationTimestamp", "namespace", "generation"): - metadata[key] = current["metadata"].get(key) - metadata["name"] = name - stored.setdefault("apiVersion", current.get("apiVersion")) - stored.setdefault("kind", current.get("kind")) - self.bump(stored) - self.leases[name] = stored - return copy.deepcopy(stored) - - return self._call( - "replace_lease", (name, body), {"timeout_s": timeout_s}, operation - ) - - def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: - def operation() -> None: - self.leases.pop(name, None) - - return self._call("delete_lease", name, {"timeout_s": timeout_s}, operation) - # ------------------------------------------------- operator simulation def set_request_condition( @@ -1117,8 +1026,3 @@ def step(self) -> int: self.kube.complete_run(run_name, candidates=published) moved += 1 return moved - - def run_to_completion(self, max_steps: int = 10) -> None: - for _ in range(max_steps): - if not self.step(): - return diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py index c977c9535127..771b2acf7c07 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -478,7 +478,11 @@ def test_plan_shards_default_grid_is_three_balanced_shards_and_a_probe(): points = grid(DEFAULT_DOMAIN) baseline = points[0] shards = plan_shards( - DEFAULT_DOMAIN, baseline=baseline, startup=baseline, max_shards=3 + DEFAULT_DOMAIN, + baseline=baseline, + startup=baseline, + max_shards=3, + parallelism=8, ) grid_shards = [shard for shard in shards if shard.kind == "grid"] assert [shard.size for shard in grid_shards] == [36, 36, 36] @@ -533,18 +537,21 @@ def test_plan_shards_adds_off_grid_baseline_and_startup_points(): def test_plan_shards_splits_uneven_dimension_and_rejects_impossible_grids(): domain = {CREDIT: tuple(float(v) for v in range(250))} zero = {CREDIT: 0.0} - shards = plan_shards(domain, baseline=zero, startup=zero, max_shards=3, probe=False) + shards = plan_shards( + domain, baseline=zero, startup=zero, max_shards=3, parallelism=8, probe=False + ) assert [shard.size for shard in shards] == [84, 83, 83] assert shards[0].choices[CREDIT][0] == 0.0 assert shards[-1].choices[CREDIT][-1] == 249.0 with pytest.raises(ValueError, match="cannot be split"): - plan_shards(domain, baseline=zero, startup=zero, max_shards=2) + plan_shards(domain, baseline=zero, startup=zero, max_shards=2, parallelism=8) with pytest.raises(ValueError, match="cannot be split"): plan_shards( DEFAULT_DOMAIN, baseline=grid(DEFAULT_DOMAIN)[0], startup=grid(DEFAULT_DOMAIN)[0], max_shards=1, + parallelism=8, ) point = grid(SMALL_DOMAIN)[0] @@ -554,15 +561,12 @@ def sizes(**kwargs): ) return [shard.size for shard in shards] - # Without the Run parallelism the largest shard is minimized. - assert sizes(max_shards=16) == [1] * 8 - assert sizes(max_shards=3) == [4, 4] - # With it, a shard is added only when it saves an evaluation round - # (AISimulate runs the first candidate alone, so single points save one). + # A shard is added only when it saves an evaluation round (AISimulate + # runs the first candidate alone, so single points save one). assert sizes(max_shards=3, parallelism=8) == [8] assert sizes(max_shards=16, parallelism=8) == [1] * 8 assert sizes(max_shards=3, parallelism=1) == [4, 4] - # The default grid is three shards of 36 either way. + # The default grid is three shards of 36. default = plan_shards( DEFAULT_DOMAIN, baseline=grid(DEFAULT_DOMAIN)[0], @@ -576,12 +580,18 @@ def sizes(**kwargs): def test_plan_shards_validates_inputs_and_probe_falls_back_to_zero_temperature(): point = {CREDIT: 0.5, SCALE: 1.0} with pytest.raises(ValueError, match="do not match"): - plan_shards(SMALL_DOMAIN, baseline=point, startup=point, max_shards=3) + plan_shards( + SMALL_DOMAIN, baseline=point, startup=point, max_shards=3, parallelism=8 + ) with pytest.raises(ValueError, match="empty"): - plan_shards({}, baseline={}, startup={}, max_shards=3) + plan_shards({}, baseline={}, startup={}, max_shards=3, parallelism=8) domain = {CREDIT: (0.0, 1.0), SCALE: (1.0,), TEMP: (0.0,)} shards = plan_shards( - domain, baseline=grid(domain)[0], startup=grid(domain)[0], max_shards=3 + domain, + baseline=grid(domain)[0], + startup=grid(domain)[0], + max_shards=3, + parallelism=8, ) assert shards[-1].points == ({CREDIT: 1.0, SCALE: 1.0, TEMP: 0.0},) @@ -727,21 +737,11 @@ def test_build_request_point_shard_uses_scalars_and_disagg_budget(): @pytest.mark.parametrize( - "engine, replicas, backend, match", - [ - ({**AGG_ENGINE, "backend": "sglang"}, {"aggregated": 1}, "vllm", "differs"), - ( - {**AGG_ENGINE, "workers": {"aggregated": {}}}, - {"aggregated": 1}, - "vllm", - "preset", - ), - (AGG_ENGINE, {"prefill": 1}, "vllm", "spec_replicas"), - (AGG_ENGINE, {"aggregated": 0}, "vllm", "at least 1"), - ], + "replicas, match", + [({"prefill": 1}, "spec_replicas"), ({"aggregated": 0}, "at least 1")], ) -def test_build_request_rejects_engines_it_cannot_pin(engine, replicas, backend, match): - cfg = make_cfg(engine=engine) +def test_build_request_rejects_spec_replicas_it_cannot_pin(replicas, match): + cfg = make_cfg() with pytest.raises(ValueError, match=match): build_request( dgd=DGD, @@ -752,7 +752,7 @@ def test_build_request_rejects_engines_it_cannot_pin(engine, replicas, backend, window_end_ms=WINDOW_END_MS, shard=_shards()[0], spec_replicas=replicas, - backend=backend, + backend="vllm", attempt=1, ) @@ -1056,22 +1056,19 @@ def test_shutdown_returns_at_once_without_deleting(tmp_path): assert len(h.kube.requests) == 2 -@pytest.mark.parametrize( - "event, reason", [("cancel", "shutdown"), ("abort", "lost_leadership")] -) -def test_the_wait_between_polls_ends_when_the_sweep_is_stopped(tmp_path, event, reason): +def test_shutdown_ends_the_wait_between_polls(tmp_path): # Wall-clock time and the sweep's own wait: the poll interval is far - # longer than the test, so only the event can end the wait. + # longer than the test, so only shutdown can end the wait. h = Harness(tmp_path, operator=False) request = make_request(deadline_s=time.time() + TIMEOUT_S) - timer = threading.Timer(0.2, getattr(h, event).set) + timer = threading.Timer(0.2, h.cancel.set) timer.start() started = time.monotonic() try: result = h.sweep(request, now=time.time, sleep=None, poll_interval_s=600.0) finally: timer.cancel() - assert result.failure_reason == reason + assert result.failure_reason == "shutdown" assert time.monotonic() - started < 60.0 assert len(h.kube.requests) == 2 @@ -1355,6 +1352,33 @@ def operator(): assert_scores(result, request) +def test_a_run_name_conflict_until_the_deadline_reports_why(tmp_path): + # A recreated Request whose old Run still waits for garbage collection. + h = Harness(tmp_path, operator=False) + message = ( + 'Run "llm-rt-0123456789-1" exists and is not the Run of generation 1 ' + "of this Request; trying again once it is gone" + ) + + def operator(): + for name in list(h.kube.requests): + h.kube.set_request_condition( + name, status="False", reason="RunNameConflict", message=message + ) + + h.clock.hooks.append(operator) + result = h.sweep(make_request(deadline_s=LAUNCH_S + 900.0)) + assert result.status == "timeout" + assert result.failure_reason == "timeout" + for shard_id in ("grid-0", "probe"): + shard = result.details["shards"][shard_id] + assert shard["state"] == "wait" + assert shard["message"] == ( + f"Request {shard['request']} RunNameConflict: {message}" + ) + assert f"{shard_id}: {shard['message']}" in result.error + + def test_run_pending_until_the_deadline_times_out(tmp_path): h = Harness(tmp_path) # The operator never creates the Jobs. Like the real operator, it then @@ -1933,14 +1957,10 @@ def test_invalid_requests_fail_cleanly(tmp_path, kwargs, reason): def test_invalid_engine_fails_cleanly(tmp_path): - h = Harness( - tmp_path, - cfg=make_cfg( - engine={**AGG_ENGINE, "backend": "sglang"}, mount_path=str(tmp_path) - ), - ) - result = h.sweep(make_request()) + h = Harness(tmp_path) + result = h.sweep(make_request(spec_replicas={"aggregated": 0})) assert result.failure_reason == "invalid_engine" + assert h.created() == [] def test_deadline_defaults_to_the_request_timeout(tmp_path): diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py index 73f074f39084..f81c2e36b611 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py @@ -20,10 +20,8 @@ from dynamo.planner.retune.live.contracts import ( KubeAlreadyExists, - KubeConflict, KubeError, KubeInvalid, - KubeNotFound, PodInfo, RetuneKube, ) @@ -131,25 +129,6 @@ def test_satisfies_the_protocol(kube): KubernetesRetuneKube("", api_client=retune_kube.api_client) -def test_get_dgd(kube): - retune_kube, transport = kube - dgd = {"metadata": {"name": "llm", "uid": "u1"}, "spec": {"services": {}}} - transport.reply(dgd) - assert retune_kube.get_dgd("llm", timeout_s=4) == dgd - (call,) = transport.calls - assert call["method"] == "GET" - assert ( - path(call) - == f"apis/nvidia.com/v1beta1/namespaces/{NS}/dynamographdeployments/llm" - ) - assert call["timeout"] == (4, 4) - - transport.reply(api_error(404)) - with pytest.raises(KubeNotFound) as raised: - retune_kube.get_dgd("gone") - assert raised.value.status == 404 - - def test_create_request_is_strict_and_dry_run_on_request(kube): retune_kube, transport = kube body = {"metadata": {"name": "r1"}, "spec": {"model": {"name": "m"}}} @@ -331,64 +310,6 @@ def test_list_pods_builds_pod_info(kube): assert pods[3].http_port is None -def test_leases_round_trip_raw_json(kube): - retune_kube, transport = kube - lease = { - "apiVersion": "coordination.k8s.io/v1", - "kind": "Lease", - "metadata": {"name": "llm-planner-retune", "resourceVersion": "5"}, - "spec": { - "holderIdentity": "pod/abc", - "leaseDurationSeconds": 60, - "renewTime": "2026-10-28T12:00:00.000000Z", - }, - } - transport.reply(FakeResponse(json.dumps(lease).encode())) - # MicroTime strings come back verbatim, not re-serialized datetimes. - assert retune_kube.get_lease("llm-planner-retune") == lease - call = transport.calls[-1] - assert call["preload"] is False - assert ( - path(call) - == f"apis/coordination.k8s.io/v1/namespaces/{NS}/leases/llm-planner-retune" - ) - transport.reply(api_error(404)) - assert retune_kube.get_lease("gone") is None - - body = {"metadata": {"name": "l2"}, "spec": {"holderIdentity": "me"}} - transport.reply({**body, "metadata": {"name": "l2", "resourceVersion": "1"}}) - created = retune_kube.create_lease(body) - assert created["metadata"]["resourceVersion"] == "1" - call = transport.calls[-1] - assert call["method"] == "POST" - assert call["body"]["apiVersion"] == "coordination.k8s.io/v1" - assert call["body"]["kind"] == "Lease" - assert "apiVersion" not in body # the caller's dict is not mutated - transport.reply(api_error(409)) - with pytest.raises(KubeAlreadyExists): - retune_kube.create_lease(body) - - transport.reply(lease) - retune_kube.replace_lease("llm-planner-retune", lease, timeout_s=2) - call = transport.calls[-1] - assert call["method"] == "PUT" - assert call["body"]["metadata"]["resourceVersion"] == "5" - assert call["timeout"] == (2, 2) - transport.reply(api_error(409, "the object has been modified")) - with pytest.raises(KubeConflict) as raised: - retune_kube.replace_lease("llm-planner-retune", lease) - assert raised.value.status == 409 - transport.reply(api_error(404)) - with pytest.raises(KubeNotFound): - retune_kube.replace_lease("gone", lease) - - transport.reply({"kind": "Status", "status": "Success"}) - retune_kube.delete_lease("l2") - assert transport.calls[-1]["method"] == "DELETE" - transport.reply(api_error(404)) - retune_kube.delete_lease("l2") - - def test_from_kube_api_shares_the_connector_client(): api_client = client.ApiClient(client.Configuration()) kube_api = SimpleNamespace( From 9ed9ebe853a4913ae68cca70ebd93a54cea5a1bb Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 12:55:03 -0800 Subject: [PATCH 089/100] fix(planner): re-add the live retune DGD and Lease access moved up from #17736 The #17736 review fixes moved the Lease and DGD support to the layer that uses it, and the merge applied those removals without a conflict. Restore this layer's contracts.py, kube.py and their test, and re-add the fake's DGD and Lease stores. FakeRetuneKube.update_pod and FakeDgdrOperator.run_to_completion stay deleted: nothing calls them. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/retune/live/contracts.py | 179 +++++++++++++++++- .../src/dynamo/planner/retune/live/kube.py | 110 ++++++++++- .../dynamo/planner/tests/live_retune_fakes.py | 105 +++++++++- .../tests/unit/test_live_retune_kube.py | 79 ++++++++ 4 files changed, 459 insertions(+), 14 deletions(-) diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py index 5c6cb41f4bb5..fd3d0347a45a 100644 --- a/components/src/dynamo/planner/retune/live/contracts.py +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -10,10 +10,24 @@ from __future__ import annotations -from collections.abc import Iterable, Mapping +import threading +from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from typing import Any, Literal, Optional, Protocol +# The eight builtin default-policy parameters a frontend router exposes on +# GET/PATCH /router_policy, by their live (PATCH) names. +POLICY_PARAMETER_KEYS: tuple[str, ...] = ( + "decode_active_request_weight", + "disk_cache_hit_weight", + "host_cache_hit_weight", + "overlap_score_credit", + "overlap_score_credit_decay", + "prefill_load_scale", + "router_temperature", + "shared_cache_multiplier", +) + # Router knobs a DGDR v1beta2 Request can search, AISimulate public name -> # live name. ``dynamo.router.simulation.config.ROUTER_PUBLIC_TO_LIVE`` is the # source of truth; this mirrors it for modules that must not import the @@ -49,6 +63,34 @@ # leader. NON_DECISION_FAILURES = frozenset({"shutdown", "lost_leadership"}) +GateReason = Literal["not_leader", "not_ready", "already_decided"] + + +# --------------------------------------------------------------- controller + + +@dataclass(frozen=True) +class GateResult: + """What ``launch_gate`` tells the controller about a due window. + + ``reason`` is ``None`` to let the window launch (or record ``busy``). + ``not_leader``/``not_ready`` keep ``_last_k`` so the window is retried on + a later tick; ``already_decided`` advances it without a decision. + ``spec_replicas`` (worker type -> DGD spec replicas) and ``abort`` (set + by the supervisor when this process stops leading) are only meaningful + when ``reason`` is ``None``. + """ + + reason: Optional[GateReason] = None + spec_replicas: Optional[dict[str, int]] = None + abort: Optional[threading.Event] = None + + +# (k, (window_start_s, window_end_s, as_of_s)) -> GateResult +LaunchGate = Callable[[int, tuple[float, float, float]], GateResult] +# controller generation of the sweep's controller -> may it be harvested? +HarvestAllowed = Callable[[int], bool] + # ------------------------------------------------------------ kubernetes @@ -108,6 +150,9 @@ class RetuneKube(Protocol): namespace: str + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + ... + def list_pods( self, label_selector: str, *, timeout_s: float = 10.0 ) -> list[PodInfo]: @@ -149,6 +194,19 @@ def list_candidates( ) -> list[dict]: ... + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + ... + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + ... + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + """Update with ``body['metadata']['resourceVersion']``; 409 -> KubeConflict.""" + ... + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + ... + # ------------------------------------------------------- frontend config @@ -188,6 +246,94 @@ class PremiseReport: startup: Optional[FrontendStartup] = None +# ---------------------------------------------------------- router policy + + +@dataclass(frozen=True) +class RouterInfo: + """One router on GET /router_policy.""" + + router_id: str + namespace: str + worker_type: str + revision: int + # The 8 PolicyParameters; None for a custom policy without runtime config. + configuration: Optional[dict[str, float]] + model: Optional[str] = None + + +@dataclass(frozen=True) +class PolicyCallResult: + """Outcome of one GET or PATCH. ``status`` is the HTTP status (0 on a + network error or timeout).""" + + status: int + routers: tuple[RouterInfo, ...] = () # GET + router: Optional[RouterInfo] = None # PATCH/reset response, when 200 + error: Optional[str] = None + + +class RouterPolicyClient(Protocol): + async def get(self, pod: PodInfo) -> PolicyCallResult: + ... + + async def patch( + self, + pod: PodInfo, + router_id: str, + expected_revision: int, + parameters: Mapping[str, float], + ) -> PolicyCallResult: + ... + + async def reset( + self, pod: PodInfo, router_id: str, expected_revision: int + ) -> PolicyCallResult: + """PATCH ``{router_id, expected_revision, reset: true}``.""" + ... + + +ActuationMode = Literal["apply", "rollback", "observe"] + + +@dataclass(frozen=True) +class ActuationTarget: + """What one actuator round converges to. + + ``apply``: PATCH explicit values so every router's 8 parameters equal + ``desired_by_wt[worker_type]``. ``rollback``: reset every router whose + configuration differs from its latched initial (never by revision). + ``observe``: no writes; latch initials and check parity only. + ``include_not_ready`` also targets NotReady, non-terminating pods. + ``learn_unknown`` lets the actuator reset a router whose initial was + never latched (revision > 0) once to learn it; that is active mode only. + With False (advisory) such a router is never written: rollback + leaves it alone and does not wait for it to converge. + """ + + mode: ActuationMode + desired_by_wt: Optional[dict[str, dict[str, float]]] = None + include_not_ready: bool = False + learn_unknown: bool = True + + +@dataclass(frozen=True) +class ActuationReport: + converged: bool + # router_id -> configuration latched this round (revision 0 or reset). + new_initials: dict[str, dict[str, float]] = field(default_factory=dict) + # router_ids seen this round (to prune stale initials). + seen_router_ids: frozenset[str] = frozenset() + # P14: every known initial equals startup_maps[worker_type]. + parity_ok: Optional[bool] = None + parity_mismatches: tuple[str, ...] = () + admin_api_disabled: bool = False + errors: dict[str, int] = field(default_factory=dict) # by status code / kind + # P12: enough Ready frontends, each with a configured router for every + # worker type, whatever their values. Gates launches; convergence does not. + routers_present: Optional[bool] = None + + # ----------------------------------------------------------- reconciler @@ -299,3 +445,34 @@ def trace_sink_missing(self, pod: str, missing: bool) -> None: def dropped_members(self, count: int) -> None: ... + + +class NullMetrics: + """A ``RetuneMetricsSink`` that records nothing (tests, disabled metrics).""" + + def decision(self, outcome: str) -> None: + pass + + def failure(self, reason: str) -> None: + pass + + def skipped(self, reason: str) -> None: + pass + + def supervisor_state(self, **_: Any) -> None: + pass + + def scores(self, **_: Any) -> None: + pass + + def router_converged(self, converged: bool) -> None: + pass + + def patch_error(self, code: str) -> None: + pass + + def trace_sink_missing(self, pod: str, missing: bool) -> None: + pass + + def dropped_members(self, count: int) -> None: + pass diff --git a/components/src/dynamo/planner/retune/live/kube.py b/components/src/dynamo/planner/retune/live/kube.py index feca00c907d9..74f82c93d28e 100644 --- a/components/src/dynamo/planner/retune/live/kube.py +++ b/components/src/dynamo/planner/retune/live/kube.py @@ -9,8 +9,12 @@ error carries the HTTP status as ``error.status`` (``None`` for a network error or timeout), so a caller can tell 403 from the rest. -Custom resources (DGDR v1beta2 Requests, Runs and Candidates) come back as -the API server's JSON. +Custom resources (the DGD at v1beta1, DGDR v1beta2 Requests, Runs and +Candidates) come back as the API server's JSON. Leases are read without the +client's model deserialization (``_preload_content=False``) so a Lease dict +round-trips unchanged: deserializing turns ``renewTime`` into a ``datetime`` +whose ``isoformat`` drops a zero fraction, which a later replace would send +back in a form the ``MicroTime`` parser rejects. """ from __future__ import annotations @@ -18,7 +22,7 @@ import json import logging import math -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Any, Literal, Optional, TypeVar from kubernetes import client @@ -40,6 +44,8 @@ logger = logging.getLogger(__name__) +DGD_VERSION = "v1beta1" +DGD_PLURAL = "dynamographdeployments" # The frontend's container and HTTP port names (deploy/operator/internal/consts). MAIN_CONTAINER_NAME = "main" HTTP_PORT_NAME = "http" @@ -133,6 +139,15 @@ def _http_port(spec: Any) -> Optional[int]: return None +def _decode(response: Any) -> dict: + """The JSON body of a ``_preload_content=False`` response.""" + + data = response.data + if isinstance(data, bytes): + data = data.decode("utf-8") + return json.loads(data) if data else {} + + class KubernetesRetuneKube: """``contracts.RetuneKube`` for one namespace over ``api_client``. @@ -148,6 +163,7 @@ def __init__(self, namespace: str, api_client: client.ApiClient): self.api_client = api_client self._custom = client.CustomObjectsApi(self.api_client) self._core = client.CoreV1Api(self.api_client) + self._coordination = client.CoordinationV1Api(self.api_client) @classmethod def from_kube_api(cls, kube_api: Any) -> KubernetesRetuneKube: @@ -204,6 +220,18 @@ def _list_custom( ) return list(result.get("items") or []) + # ----------------------------------------------------------------- DGD + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + found = self._get_custom(DGD_VERSION, DGD_PLURAL, name, timeout_s) + if found is None: + error = KubeNotFound( + f"DynamoGraphDeployment {self.namespace}/{name} not found" + ) + error.status = 404 # type: ignore[attr-defined] + raise error + return found + # ---------------------------------------------------------------- pods def list_pods( @@ -291,3 +319,79 @@ def list_candidates( return self._list_custom( DGDR_VERSION, DGDR_CANDIDATES, label_selector, timeout_s ) + + # -------------------------------------------------------------- Leases + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + try: + return self._call( + "read", + f"leases/{name}", + lambda: _decode( + self._coordination.read_namespaced_lease( + name=name, + namespace=self.namespace, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + except KubeNotFound: + return None + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + name = (body.get("metadata") or {}).get("name", "") + return self._call( + "create", + f"leases/{name}", + lambda: _decode( + self._coordination.create_namespaced_lease( + namespace=self.namespace, + body=_lease_body(body), + field_manager=FIELD_MANAGER, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + return self._call( + "update", + f"leases/{name}", + lambda: _decode( + self._coordination.replace_namespaced_lease( + name=name, + namespace=self.namespace, + body=_lease_body(body), + field_manager=FIELD_MANAGER, + _preload_content=False, + _request_timeout=request_timeout(timeout_s), + ) + ), + ) + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + """Missing Leases are not an error.""" + + try: + self._call( + "delete", + f"leases/{name}", + lambda: self._coordination.delete_namespaced_lease( + name=name, + namespace=self.namespace, + _request_timeout=request_timeout(timeout_s), + ), + ) + except KubeNotFound: + pass + + +def _lease_body(body: Mapping[str, Any]) -> dict: + """A Lease body with its type fields set (the API server requires them).""" + + lease = dict(body) + lease.setdefault("apiVersion", "coordination.k8s.io/v1") + lease.setdefault("kind", "Lease") + return lease diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py index 5a153b5a0639..d78ea5d21fdd 100644 --- a/components/src/dynamo/planner/tests/live_retune_fakes.py +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -4,14 +4,15 @@ """In-memory fakes of the Kubernetes side of live retune. ``FakeRetuneKube`` implements ``contracts.RetuneKube`` over plain dicts with -the API-server behaviour the live modules depend on: ``AlreadyExists`` on -create, Strict field validation and silent pruning on Request create, label -selectors, cascading Request deletion, and injectable faults (including -writes that time out but land). ``FakeDgdrOperator`` drives the Requests in -a ``FakeRetuneKube`` through the DGDR v1beta2 lifecycle (Accepted -> Run -> -Completed with Candidates) the way the operator at #17483 does, lowering -each Request into ``Run.status.aisimulate.input`` with the same merge-patch -rules. The operator's golden +the API-server behaviour the live modules depend on: ``resourceVersion`` +compare-and-swap on Lease replace, ``AlreadyExists`` on create, Strict field +validation and silent pruning on Request create, label selectors, cascading +Request deletion, and injectable faults (including writes that time out but +land). ``FakeDgdrOperator`` drives the Requests in a ``FakeRetuneKube`` +through the DGDR v1beta2 lifecycle (Accepted -> Run -> Completed with +Candidates) the way the operator at #17483 does, lowering each Request into +``Run.status.aisimulate.input`` with the same merge-patch rules. The +operator's golden ``deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json`` is the oracle for that lowering; ``test_live_retune_dgdr`` checks ``lower_request_input`` against it. @@ -40,8 +41,10 @@ POD_LABEL_DGD_NAME, RUN_UID_LABEL, KubeAlreadyExists, + KubeConflict, KubeError, KubeInvalid, + KubeNotFound, PodInfo, ) @@ -68,6 +71,7 @@ } KubeMethod = Literal[ + "get_dgd", "list_pods", "create_request", "get_request", @@ -75,6 +79,10 @@ "delete_request", "get_run", "list_candidates", + "get_lease", + "create_lease", + "replace_lease", + "delete_lease", ] @@ -371,8 +379,8 @@ class KubeCall: class FakeRetuneKube: """In-memory ``contracts.RetuneKube`` (thread-safe). - Stores pods (``PodInfo``), Requests, Runs and Candidates as plain - dicts. Objects come back as deep copies, so callers can never + Stores DGDs, pods (``PodInfo``), Requests, Runs, Candidates and Leases + as plain dicts. Objects come back as deep copies, so callers can never alias the store. ``now`` stamps ``creationTimestamp``. Request create honours ``field_validation='Strict'``: a path listed in @@ -392,12 +400,14 @@ def __init__( ) -> None: self.namespace = namespace self.now = now + self.dgds: dict[str, dict] = {} self.pods: dict[str, PodInfo] = {} self.requests: dict[str, dict] = {} self.runs: dict[str, dict] = {} # Each Run's lowered input, recorded in its status by ``create_job``. self.run_inputs: dict[str, dict] = {} self.candidates: dict[str, dict] = {} + self.leases: dict[str, dict] = {} self.calls: list[KubeCall] = [] self.deleted_requests: list[tuple[str, str]] = [] self.unknown_fields: set[str] = set() @@ -485,6 +495,25 @@ def bump(self, obj: dict) -> None: obj["metadata"]["resourceVersion"] = str(next(self._rv)) + # --------------------------------------------------------------- DGD + + def put_dgd(self, dgd: dict) -> dict: + with self._lock: + stored = copy.deepcopy(dgd) + self._stamp( + stored, kind="DynamoGraphDeployment", api_version="nvidia.com/v1beta1" + ) + self.dgds[stored["metadata"]["name"]] = stored + return copy.deepcopy(stored) + + def get_dgd(self, name: str, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + if name not in self.dgds: + raise KubeNotFound(f"DynamoGraphDeployment {name!r} not found") + return copy.deepcopy(self.dgds[name]) + + return self._call("get_dgd", name, {"timeout_s": timeout_s}, operation) + # -------------------------------------------------------------- pods def add_pod( @@ -664,6 +693,62 @@ def operation() -> list[dict]: "list_candidates", label_selector, {"timeout_s": timeout_s}, operation ) + # ------------------------------------------------------------- Leases + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + def operation() -> Optional[dict]: + found = self.leases.get(name) + return None if found is None else copy.deepcopy(found) + + return self._call("get_lease", name, {"timeout_s": timeout_s}, operation) + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + stored = copy.deepcopy(body) + name = nested_get(stored, "metadata", "name") + if not name: + raise KubeInvalid("metadata.name is required") + if name in self.leases: + raise KubeAlreadyExists(f"leases {name!r} already exists") + stored["metadata"].pop("resourceVersion", None) + self._stamp(stored, kind="Lease", api_version="coordination.k8s.io/v1") + self.leases[name] = stored + return copy.deepcopy(stored) + + return self._call("create_lease", body, {"timeout_s": timeout_s}, operation) + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + def operation() -> dict: + current = self.leases.get(name) + if current is None: + raise KubeNotFound(f"leases {name!r} not found") + sent = nested_get(body, "metadata", "resourceVersion") + if sent != current["metadata"]["resourceVersion"]: + raise KubeConflict( + f"leases {name!r}: resourceVersion {sent!r} is stale " + f"(current {current['metadata']['resourceVersion']!r})" + ) + stored = copy.deepcopy(body) + metadata = stored.setdefault("metadata", {}) + for key in ("uid", "creationTimestamp", "namespace", "generation"): + metadata[key] = current["metadata"].get(key) + metadata["name"] = name + stored.setdefault("apiVersion", current.get("apiVersion")) + stored.setdefault("kind", current.get("kind")) + self.bump(stored) + self.leases[name] = stored + return copy.deepcopy(stored) + + return self._call( + "replace_lease", (name, body), {"timeout_s": timeout_s}, operation + ) + + def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + def operation() -> None: + self.leases.pop(name, None) + + return self._call("delete_lease", name, {"timeout_s": timeout_s}, operation) + # ------------------------------------------------- operator simulation def set_request_condition( diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py index f81c2e36b611..73f074f39084 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py @@ -20,8 +20,10 @@ from dynamo.planner.retune.live.contracts import ( KubeAlreadyExists, + KubeConflict, KubeError, KubeInvalid, + KubeNotFound, PodInfo, RetuneKube, ) @@ -129,6 +131,25 @@ def test_satisfies_the_protocol(kube): KubernetesRetuneKube("", api_client=retune_kube.api_client) +def test_get_dgd(kube): + retune_kube, transport = kube + dgd = {"metadata": {"name": "llm", "uid": "u1"}, "spec": {"services": {}}} + transport.reply(dgd) + assert retune_kube.get_dgd("llm", timeout_s=4) == dgd + (call,) = transport.calls + assert call["method"] == "GET" + assert ( + path(call) + == f"apis/nvidia.com/v1beta1/namespaces/{NS}/dynamographdeployments/llm" + ) + assert call["timeout"] == (4, 4) + + transport.reply(api_error(404)) + with pytest.raises(KubeNotFound) as raised: + retune_kube.get_dgd("gone") + assert raised.value.status == 404 + + def test_create_request_is_strict_and_dry_run_on_request(kube): retune_kube, transport = kube body = {"metadata": {"name": "r1"}, "spec": {"model": {"name": "m"}}} @@ -310,6 +331,64 @@ def test_list_pods_builds_pod_info(kube): assert pods[3].http_port is None +def test_leases_round_trip_raw_json(kube): + retune_kube, transport = kube + lease = { + "apiVersion": "coordination.k8s.io/v1", + "kind": "Lease", + "metadata": {"name": "llm-planner-retune", "resourceVersion": "5"}, + "spec": { + "holderIdentity": "pod/abc", + "leaseDurationSeconds": 60, + "renewTime": "2026-10-28T12:00:00.000000Z", + }, + } + transport.reply(FakeResponse(json.dumps(lease).encode())) + # MicroTime strings come back verbatim, not re-serialized datetimes. + assert retune_kube.get_lease("llm-planner-retune") == lease + call = transport.calls[-1] + assert call["preload"] is False + assert ( + path(call) + == f"apis/coordination.k8s.io/v1/namespaces/{NS}/leases/llm-planner-retune" + ) + transport.reply(api_error(404)) + assert retune_kube.get_lease("gone") is None + + body = {"metadata": {"name": "l2"}, "spec": {"holderIdentity": "me"}} + transport.reply({**body, "metadata": {"name": "l2", "resourceVersion": "1"}}) + created = retune_kube.create_lease(body) + assert created["metadata"]["resourceVersion"] == "1" + call = transport.calls[-1] + assert call["method"] == "POST" + assert call["body"]["apiVersion"] == "coordination.k8s.io/v1" + assert call["body"]["kind"] == "Lease" + assert "apiVersion" not in body # the caller's dict is not mutated + transport.reply(api_error(409)) + with pytest.raises(KubeAlreadyExists): + retune_kube.create_lease(body) + + transport.reply(lease) + retune_kube.replace_lease("llm-planner-retune", lease, timeout_s=2) + call = transport.calls[-1] + assert call["method"] == "PUT" + assert call["body"]["metadata"]["resourceVersion"] == "5" + assert call["timeout"] == (2, 2) + transport.reply(api_error(409, "the object has been modified")) + with pytest.raises(KubeConflict) as raised: + retune_kube.replace_lease("llm-planner-retune", lease) + assert raised.value.status == 409 + transport.reply(api_error(404)) + with pytest.raises(KubeNotFound): + retune_kube.replace_lease("gone", lease) + + transport.reply({"kind": "Status", "status": "Success"}) + retune_kube.delete_lease("l2") + assert transport.calls[-1]["method"] == "DELETE" + transport.reply(api_error(404)) + retune_kube.delete_lease("l2") + + def test_from_kube_api_shares_the_connector_client(): api_client = client.ApiClient(client.Configuration()) kube_api = SimpleNamespace( From dd46cb54d6f64ab825567313f31ea83fdcf1795c Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 12:55:24 -0800 Subject: [PATCH 090/100] fix(planner): address second review on enabling live retune - Router actuator: a router missing from its pod's GET is not gone (router_policy.rs leaves out a router whose control call timed out or failed). A router counts as gone only once every pod instance it may run on is gone: no longer listed, finished, or restarted (PodInfo.restarts, the main container's restartCount; a restarted frontend has new router ids). An initial persisted by an earlier process may run on any frontend listed when this process first misses it. An advisory or clean-up rollback therefore keeps the initial of a router omitted from GET and still resets it. - Decommission deletes the state Lease conditional on the cached snapshot's resourceVersion, and the leadership Lease on the resourceVersion of this process's last write (delete_lease takes a resource_version precondition). A state claimed by another term since then stays and the supervisor re-initializes. - State reads drop history entries that canonical_json cannot write back (non-objects, NaN, infinities) and reject a non-string fingerprint, so a state write never raises on them. - The SIGTERM handler is installed once the planner may hold a retune Lease: at start with live retune, or when the clean-up check of a removed one finds a Lease. That check retries a transient API error once after 15 s; 403 and 404 are not retried. - The live sweep thread is a plain single-thread executor. Harvest waits for the sweep's future, so the queued-launch bookkeeping (queued, _settle_queue_delay, timed_sweep, SweepExecutor.busy) only moved launched_at_s; with P1 (latency 0, or measured at scale 1) that never changed when a decision lands. - Tests: the supervisor tests drive the real RouterActuator through FrontendPolicyClient (FakeActuator is gone) and the seeded interleaving harness is dropped; the deterministic single-writer, no-wasted-window and advisory tests stay. The e2e policy fake keeps only the revision compare-and-swap and the merge. The actuator, adapter and controller hook tests are cut to convergence, rollback, P14, the 409 retry, advisory and the host contract, and the metrics tests to the exposition names, the zero-initialized series, trace_sink_missing removal and collector sharing. The RBAC chart tests count the rules of each case. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/planner/__main__.py | 49 +- components/src/dynamo/planner/core/base.py | 77 +- .../plugins/orchestrator/engine_adapter.py | 12 +- .../src/dynamo/planner/retune/controller.py | 88 +- .../dynamo/planner/retune/live/actuator.py | 45 +- .../dynamo/planner/retune/live/contracts.py | 16 +- .../src/dynamo/planner/retune/live/kube.py | 27 +- .../src/dynamo/planner/retune/live/state.py | 57 +- .../dynamo/planner/retune/live/supervisor.py | 32 +- .../tests/integration/test_live_retune_e2e.py | 60 +- .../dynamo/planner/tests/live_retune_fakes.py | 39 +- .../src/dynamo/planner/tests/retune_stubs.py | 12 - .../tests/unit/test_live_retune_actuator.py | 548 +++-------- .../tests/unit/test_live_retune_adapter.py | 295 ++---- .../unit/test_live_retune_controller_hooks.py | 488 +--------- .../tests/unit/test_live_retune_kube.py | 15 + .../tests/unit/test_live_retune_metrics.py | 226 +---- .../tests/unit/test_live_retune_state.py | 29 +- .../tests/unit/test_live_retune_supervisor.py | 876 ++++-------------- .../platform/tests/planner_rbac_test.yaml | 134 +-- 20 files changed, 755 insertions(+), 2370 deletions(-) diff --git a/components/src/dynamo/planner/__main__.py b/components/src/dynamo/planner/__main__.py index bbb7c268e35b..7c8e24807527 100644 --- a/components/src/dynamo/planner/__main__.py +++ b/components/src/dynamo/planner/__main__.py @@ -18,7 +18,7 @@ import logging import signal from collections.abc import Awaitable -from typing import Any +from typing import Any, Optional from pydantic import BaseModel @@ -56,26 +56,33 @@ async def generate(request: RequestType): if not planner.may_hold_retune_lease: await asyncio.gather(serve, planner.run()) return - await _run_until_sigterm(runtime, serve, planner.run()) + await _run_until_sigterm(runtime, serve, planner.run(), planner.retune_lease_found) async def _run_until_sigterm( - runtime: Any, serve: Awaitable[Any], run: Awaitable[None] + runtime: Any, + serve: Awaitable[Any], + run: Awaitable[None], + lease_found: asyncio.Event, ) -> None: - """Serve and run the planner; SIGTERM stops the planner cleanly. + """Serve and run the planner; once ``lease_found`` is set, SIGTERM stops + the planner cleanly. Live retune, and the clean-up of a removed one, hold a leadership Lease, and live retune may have a sweep in flight. Without a handler SIGTERM kills the process before ``run()``'s ``finally`` stops the sweep and - releases the Lease, so the next leader would wait out the Lease. The handler cancels only the run - task; once its shutdown finished, the runtime is shut down to stop - serving the endpoint and the call returns normally. + releases the Lease, so the next leader would wait out the Lease. The + handler is installed when the planner may hold the Lease: at once with + live retune, or when the clean-up check finds a Lease. It cancels only + the run task; once its shutdown finished, the runtime is shut down to + stop serving the endpoint and the call returns normally. """ loop = asyncio.get_running_loop() run_task = asyncio.ensure_future(run) serve_task = asyncio.ensure_future(serve) terminating = False + installed = False def on_sigterm() -> None: nonlocal terminating @@ -85,13 +92,24 @@ def on_sigterm() -> None: logger.info("SIGTERM received; stopping the planner") run_task.cancel() - try: - loop.add_signal_handler(signal.SIGTERM, on_sigterm) - installed = True - except (NotImplementedError, RuntimeError, ValueError) as exc: - # No signal handlers on this platform or off the main thread. - logger.warning("Cannot install a planner SIGTERM handler: %s", exc) - installed = False + def install() -> None: + nonlocal installed + try: + loop.add_signal_handler(signal.SIGTERM, on_sigterm) + installed = True + except (NotImplementedError, RuntimeError, ValueError) as exc: + # No signal handlers on this platform or off the main thread. + logger.warning("Cannot install a planner SIGTERM handler: %s", exc) + + async def install_once_found() -> None: + await lease_found.wait() + install() + + installer: Optional[asyncio.Future[None]] = None + if lease_found.is_set(): + install() + else: + installer = asyncio.ensure_future(install_once_found()) try: await asyncio.gather(serve_task, run_task) except asyncio.CancelledError: @@ -107,6 +125,9 @@ def on_sigterm() -> None: "Planner endpoint stopped with an error: %s", serve_task.exception() ) finally: + if installer is not None and not installer.done(): + installer.cancel() + await asyncio.wait([installer]) if installed: loop.remove_signal_handler(signal.SIGTERM) diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index ea11f47bbb6d..60ef11f2e2a1 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -58,7 +58,7 @@ runtime_candidate_errors, runtime_configuration_errors, ) -from dynamo.planner.retune.live.contracts import KubeError +from dynamo.planner.retune.live.contracts import KubeError, KubeNotFound from dynamo.planner.retune.live.kube import KubernetesRetuneKube from dynamo.planner.retune.live.state import find_retune_leases from dynamo.runtime import DistributedRuntime @@ -138,6 +138,9 @@ async def stop(self) -> None: # The background check for a removed live retune reads two Leases, then, # only if one exists, the DGD; each Kubernetes call is bounded by this. _LEGACY_RETUNE_CALL_TIMEOUT_SECONDS = 10.0 +# The check runs once per start; a transient API error is retried once after +# this long (403 and 404 are not). +_LEGACY_RETUNE_RETRY_SECONDS = 15.0 # A tick waits briefly for the common fast submission path, then leaves the # task running. It is deliberately not cancelled: a remote request may have # been accepted even when its response has not arrived, so retrying could @@ -258,6 +261,7 @@ def __init__( # whether a removed live retune left its Leases, and cleans up after it. self._legacy_retune_task: Optional[asyncio.Task[None]] = None self._legacy_retune_cleanup: Optional[RetuneCleanupHandle] = None + self._retune_lease_found = asyncio.Event() async def _async_init(self) -> None: # Shutdown is safe for a partially initialized environment and is @@ -271,6 +275,7 @@ async def _async_init(self) -> None: # Before the first _ensure_engine (regression bootstrap builds # the engine): the engine takes the live retune inputs/hooks. self._live_retune = await self._build_live_retune() + self._retune_lease_found.set() await self._bootstrap_regression() await self._bootstrap_engine_plugins_if_needed() @@ -321,6 +326,14 @@ def may_hold_retune_lease(self) -> bool: return self._live_retune is not None or self._legacy_retune_task is not None + @property + def retune_lease_found(self) -> asyncio.Event: + """Set once this planner may hold a retune Lease: at start with live + retune configured, or when the clean-up check of a removed one finds + a Lease. ``__main__`` installs its SIGTERM handler then.""" + + return self._retune_lease_found + async def _build_live_retune(self) -> LiveRetuneHandle: return await _live_retune_bootstrap().build_live_retune(self) @@ -351,32 +364,56 @@ async def _legacy_retune_check(self, connector: KubernetesConnector) -> None: Reads the DGD's two retune Leases; only when one exists does it load the live retune stack, which rolls back the routers that retune deployed and then deletes its Requests and Leases. Without Lease RBAC - (the default chart) the read answers 403, logged at debug. + (the default chart) the read answers 403, logged at debug. Any other + API error but a 404 is retried once. """ kube = KubernetesRetuneKube.from_kube_api(connector.kube_api) - try: - leases = await asyncio.to_thread( - find_retune_leases, - kube, - connector.parent_dgd_name, - timeout_s=_LEGACY_RETUNE_CALL_TIMEOUT_SECONDS, - ) - if not leases: - return - cleanup = await _live_retune_bootstrap().build_legacy_cleanup( - self, kube, timeout_s=_LEGACY_RETUNE_CALL_TIMEOUT_SECONDS - ) - except KubeError as error: - # ``status`` is set by the Kubernetes client wrapper only. - if getattr(error, "status", None) == 403: - logger.debug("Live retune legacy clean-up check skipped: %s", error) - else: - logger.info("Live retune legacy clean-up skipped: %s", error) + cleanup: Optional[RetuneCleanupHandle] = None + for retry in (True, False): + try: + cleanup = await self._find_legacy_retune( + kube, connector.parent_dgd_name + ) + break + except KubeError as error: + # ``status`` is set by the Kubernetes client wrapper only. + if getattr(error, "status", None) == 403: + logger.debug("Live retune legacy clean-up check skipped: %s", error) + return + if not retry or isinstance(error, KubeNotFound): + logger.info("Live retune legacy clean-up skipped: %s", error) + return + logger.info( + "Live retune legacy clean-up check failed; retrying in %.0f s: %s", + _LEGACY_RETUNE_RETRY_SECONDS, + error, + ) + await asyncio.sleep(_LEGACY_RETUNE_RETRY_SECONDS) + if cleanup is None: return self._legacy_retune_cleanup = cleanup await cleanup.start(self) + async def _find_legacy_retune( + self, kube: KubernetesRetuneKube, dgd_name: str + ) -> Optional[RetuneCleanupHandle]: + """The clean-up of a removed live retune; ``None`` without its Leases.""" + + leases = await asyncio.to_thread( + find_retune_leases, + kube, + dgd_name, + timeout_s=_LEGACY_RETUNE_CALL_TIMEOUT_SECONDS, + ) + if not leases: + return None + # The clean-up takes the leadership Lease: stop cleanly on SIGTERM. + self._retune_lease_found.set() + return await _live_retune_bootstrap().build_legacy_cleanup( + self, kube, timeout_s=_LEGACY_RETUNE_CALL_TIMEOUT_SECONDS + ) + async def replace_live_retune( self, inputs: Optional[RetuneInputs], diff --git a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py index d5e792ad91bb..2a93b58c8457 100644 --- a/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py +++ b/components/src/dynamo/planner/plugins/orchestrator/engine_adapter.py @@ -61,7 +61,7 @@ import logging from collections.abc import Callable, Iterable -from concurrent.futures import Future +from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Optional, Sequence @@ -72,7 +72,7 @@ if TYPE_CHECKING: import grpc.aio - from dynamo.planner.retune.controller import RetuneController, SweepExecutor + from dynamo.planner.retune.controller import RetuneController from dynamo.planner.retune.live.contracts import HarvestAllowed, LaunchGate from dynamo.planner.core.budget import ( @@ -361,7 +361,7 @@ def _factory(plugin_id, endpoint, *, in_process_instance=None): # Live: the one sweep thread every controller generation submits to, # so a rebuilt controller's sweep queues behind a dropped one and two # generations never sweep at once. - self._retune_executor: Optional[SweepExecutor] = None + self._retune_executor: Optional[ThreadPoolExecutor] = None if retune_hooks is None or self._config.retune is None: self._retune = self._build_retune_controller(retune_inputs) elif retune_inputs is None: @@ -408,7 +408,7 @@ def _build_retune_controller( # of the import chain of planners that never enable retune, and avoids # an import cycle through dynamo.replay, whose planner host imports # this module. - from dynamo.planner.retune.controller import RetuneController, SweepExecutor + from dynamo.planner.retune.controller import RetuneController hooks = self._retune_hooks if hooks is None: @@ -420,7 +420,9 @@ def _build_retune_controller( self._scale_interval, ) if self._retune_executor is None: - self._retune_executor = SweepExecutor() + self._retune_executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="planner-retune" + ) return RetuneController( self._config, retune_inputs, diff --git a/components/src/dynamo/planner/retune/controller.py b/components/src/dynamo/planner/retune/controller.py index a152cffd07b4..1f0b263e85df 100644 --- a/components/src/dynamo/planner/retune/controller.py +++ b/components/src/dynamo/planner/retune/controller.py @@ -41,8 +41,8 @@ router values. - ``executor`` is the live host's single sweep thread, shared by every controller generation it builds, so two generations never sweep at once: - a launch queues behind a dropped sweep that still runs, and its launch - time moves to when the sweep really started. + a launch queues behind a dropped sweep that still runs. Harvest waits for + the queued sweep's future, so the queue delay needs no bookkeeping. """ from __future__ import annotations @@ -109,35 +109,6 @@ SweepFn = Callable[..., SweepResult] -class SweepExecutor(ThreadPoolExecutor): - """The single sweep thread; ``busy`` while a submitted call is unfinished. - - A live host shares one across the controller generations it builds, so - their sweeps run one at a time in submission order. - """ - - def __init__(self) -> None: - super().__init__(max_workers=1, thread_name_prefix="planner-retune") - self._lock = threading.Lock() - self._unfinished = 0 - - def busy(self) -> bool: - with self._lock: - return self._unfinished > 0 - - def submit(self, fn: Callable[..., Any], /, *args: Any, **kwargs: Any) -> Future: - future = super().submit(fn, *args, **kwargs) - with self._lock: - self._unfinished += 1 - # Runs at once if the call already finished (or was cancelled). - future.add_done_callback(self._finished) - return future - - def _finished(self, _future: Future) -> None: - with self._lock: - self._unfinished -= 1 - - @dataclass(frozen=True) class RetuneTickResult: """What one ``on_tick`` call hands back to the engine. @@ -167,11 +138,6 @@ class _PendingSweep: ready_at_s: float | None # Set by the supervisor to stop this sweep (lost leadership, rebuild). abort: threading.Event | None = None - # Submitted while the sweep thread was still busy with a dropped sweep - # (of this or an earlier controller generation): - # ``(submitted_monotonic, [started_monotonic])``. ``launched_at_s`` then - # moves to the moment the sweep really started. - queued: tuple[float, list[float]] | None = None class RetuneController: @@ -208,7 +174,7 @@ def __init__( launch_gate: LaunchGate | None = None, harvest_allowed: HarvestAllowed | None = None, generation: int = 0, - executor: SweepExecutor | None = None, + executor: ThreadPoolExecutor | None = None, ) -> None: retune = config.retune if retune is None: @@ -293,7 +259,9 @@ def __init__( self._pending: _PendingSweep | None = None self._cancel = threading.Event() self._owns_executor = executor is None - self._executor = executor or SweepExecutor() + self._executor = executor or ThreadPoolExecutor( + max_workers=1, thread_name_prefix="planner-retune" + ) logger.info( "retune enabled: interval=%ss grace=%ss %d grid points, router keys %s, " "planner keys %s, predictor candidates %s, objective %s, " @@ -358,7 +326,6 @@ async def on_tick( pending = self._pending if pending is not None and pending.future.done(): - self._settle_queue_delay(pending) if pending.ready_at_s is None: pending.decision_latency_s = self._decision_latency_s(pending.future) pending.ready_at_s = pending.launched_at_s + pending.decision_latency_s @@ -706,20 +673,6 @@ def _harvest_or_rewind( return None return self._decide(pending, result, now_s) - def _settle_queue_delay(self, pending: _PendingSweep) -> None: - """Move a queued sweep's launch time to when it actually started.""" - - if pending.queued is None: - return - submitted, started = pending.queued - pending.queued = None - if not started: - return # never ran (cancelled at close); keep the submit time - delay = max(0.0, started[0] - submitted) - pending.launched_at_s += delay - if pending.ready_at_s is not None: - pending.ready_at_s += delay - def _replicas_for_launch( self, start_s: float, end_s: float, spec_replicas: Mapping[str, int] | None ) -> tuple[dict[str, int] | None, bool, str | None]: @@ -816,7 +769,7 @@ async def _launch( **self._live_request_fields(as_of_s, spec_replicas, verified), ) abort = gate.abort if gate is not None else None - future, queued = self._submit(request, abort, gated=gate is not None) + future = self._submit(request, abort, gated=gate is not None) if isinstance(self._clock, VirtualClock): # Virtual time is frozen while the sweep runs; wait for it here. await asyncio.wait([asyncio.wrap_future(future)]) @@ -835,7 +788,6 @@ async def _launch( decision_latency_s=latency, ready_at_s=None if latency is None else now_s + latency, abort=abort, - queued=queued, ) logger.info( "retune launched window %d [%s, %s) with %d candidates, replicas %s%s", @@ -876,34 +828,22 @@ def _submit( abort: threading.Event | None, *, gated: bool, - ) -> tuple[Future[SweepResult], tuple[float, list[float]] | None]: + ) -> Future[SweepResult]: """Submit the sweep to the single sweep thread. ``run_sweep`` is looked up at call time so tests can substitute it. A gated (live) launch also passes the gate's abort event. While the sweep thread still runs a dropped sweep, of this or an earlier - controller generation, the launch queues behind it and records when - it really starts (``_settle_queue_delay``). + controller generation, the launch queues behind it; the decision + waits for its future either way. """ sweep_fn: SweepFn = self._sweep_fn if self._sweep_fn is not None else run_sweep if not gated: - return self._executor.submit(sweep_fn, request, cancel=self._cancel), None - cancel = self._cancel - if not self._executor.busy(): - future = self._executor.submit( - sweep_fn, request, cancel=cancel, abort=abort - ) - return future, None - clock = self._clock - started: list[float] = [] - - def timed_sweep() -> SweepResult: - started.append(clock.monotonic()) - return sweep_fn(request, cancel=cancel, abort=abort) - - future = self._executor.submit(timed_sweep) - return future, (clock.monotonic(), started) + return self._executor.submit(sweep_fn, request, cancel=self._cancel) + return self._executor.submit( + sweep_fn, request, cancel=self._cancel, abort=abort + ) def _decision_latency_s(self, future: Future[SweepResult]) -> float: """Delay from launch to decision; ``measured`` needs a done sweep.""" diff --git a/components/src/dynamo/planner/retune/live/actuator.py b/components/src/dynamo/planner/retune/live/actuator.py index 67280412bd08..154e1c0dbce4 100644 --- a/components/src/dynamo/planner/retune/live/actuator.py +++ b/components/src/dynamo/planner/retune/live/actuator.py @@ -112,6 +112,8 @@ DEFAULT_CLIENT_TIMEOUT_S = ROUTER_CONTROL_TIMEOUT_S + 3.0 # Pod phases after which no container runs again. _FINISHED_POD_PHASES = frozenset({"Succeeded", "Failed"}) +# (pod uid, restarts): one run of a frontend; its routers live only as long. +_PodInstance = tuple[str, Optional[int]] # ----------------------------------------------------------------- client @@ -341,8 +343,10 @@ def __init__( self._admin_disabled_rounds = int(admin_disabled_rounds) # router_id -> initial configuration latched by this process. self._latched: dict[str, dict[str, float]] = {} - # router_id -> name of the pod it was last seen on. - self._router_pod: dict[str, str] = {} + # router_id -> the pod instances it may run on: the one it was last + # seen on, or for an initial not seen by this process, every + # frontend listed when it was first missing. + self._router_hosts: dict[str, frozenset[_PodInstance]] = {} # router_id -> signature of a write the router rejected with 422; # the same write is not retried until the target changes. self._rejected: dict[str, str] = {} @@ -591,7 +595,7 @@ def _in_scope( continue if router.worker_type not in self._startup_maps: continue - self._router_pod[router.router_id] = pod.name + self._router_hosts[router.router_id] = frozenset({_instance(pod)}) if router.configuration is None: scoped.append(router) continue @@ -796,33 +800,34 @@ def _seen_router_ids( ) -> set[str]: """Routers observed this round plus known ones not provably gone. - A known router is gone when its pod no longer exists (or has - finished: Succeeded/Failed) or answered GET without it; with an - unknown pod, when every live frontend with an IP answered GET this - round. + A GET answers without the routers whose control call timed out or + failed (``router_policy.rs``), so a router missing from its pod's + answer may still be there. A known router is gone only once every + pod instance it may run on is gone: no longer listed, finished + (Succeeded/Failed), or restarted (a restarted frontend registers new + routers). """ observed = { router.router_id for outcome in outcomes for router in outcome.routers } - existing = {pod.name for pod in pods if pod.phase not in _FINISHED_POD_PHASES} - answered = {outcome.pod.name for outcome in outcomes if outcome.ok} - complete = all(pod.name in answered for pod in pods if _serving(pod) and pod.ip) + present = frozenset( + _instance(pod) for pod in pods if pod.phase not in _FINISHED_POD_PHASES + ) seen = set(observed) for router_id in set(state.initials) | set(self._latched): if router_id in observed: continue - pod_name = self._router_pod.get(router_id) - if pod_name is None: - gone = complete - else: - gone = pod_name not in existing or pod_name in answered - if not gone: + # A persisted initial this process has not seen on any pod: its + # router runs on a frontend listed now (a pod never comes back), + # or on none. + hosts = self._router_hosts.setdefault(router_id, present) + if hosts & present: seen.add(router_id) return seen def _prune(self, seen: set[str]) -> None: - for cache in (self._latched, self._router_pod, self._rejected): + for cache in (self._latched, self._router_hosts, self._rejected): for router_id in [key for key in cache if key not in seen]: del cache[router_id] @@ -843,6 +848,12 @@ def _track_admin_api( # ---------------------------------------------------------------- helpers +def _instance(pod: PodInfo) -> _PodInstance: + """One run of a pod's frontend: its uid and container restart count.""" + + return (pod.uid, pod.restarts) + + def _serving(pod: PodInfo) -> bool: """Not terminating and not finished: its frontend may still answer.""" diff --git a/components/src/dynamo/planner/retune/live/contracts.py b/components/src/dynamo/planner/retune/live/contracts.py index fd3d0347a45a..50c743cd65ea 100644 --- a/components/src/dynamo/planner/retune/live/contracts.py +++ b/components/src/dynamo/planner/retune/live/contracts.py @@ -137,6 +137,9 @@ class PodInfo: http_port: Optional[int] = None # ``status.phase`` (Pending, Running, Succeeded, Failed, Unknown). phase: Optional[str] = None + # ``restartCount`` of the ``main`` container, if reported. A restarted + # frontend registers new routers (new router ids). + restarts: Optional[int] = None class RetuneKube(Protocol): @@ -204,7 +207,15 @@ def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> di """Update with ``body['metadata']['resourceVersion']``; 409 -> KubeConflict.""" ... - def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + def delete_lease( + self, + name: str, + *, + resource_version: Optional[str] = None, + timeout_s: float = 10.0, + ) -> None: + """Missing Leases are not an error. With ``resource_version`` the + delete is conditional on it; a changed Lease -> KubeConflict.""" ... @@ -322,7 +333,8 @@ class ActuationReport: converged: bool # router_id -> configuration latched this round (revision 0 or reset). new_initials: dict[str, dict[str, float]] = field(default_factory=dict) - # router_ids seen this round (to prune stale initials). + # router_ids seen this round, or known and not provably gone; the + # initials of every other router are pruned. seen_router_ids: frozenset[str] = frozenset() # P14: every known initial equals startup_maps[worker_type]. parity_ok: Optional[bool] = None diff --git a/components/src/dynamo/planner/retune/live/kube.py b/components/src/dynamo/planner/retune/live/kube.py index 74f82c93d28e..3746f699b1b0 100644 --- a/components/src/dynamo/planner/retune/live/kube.py +++ b/components/src/dynamo/planner/retune/live/kube.py @@ -124,9 +124,19 @@ def _pod_info(pod: Any) -> PodInfo: labels=dict(metadata.labels or {}), http_port=_http_port(pod.spec), phase=status.phase if status is not None else None, + restarts=_main_restarts(status), ) +def _main_restarts(status: Any) -> Optional[int]: + """``restartCount`` of the main container, if its status is reported.""" + + for container in (status.container_statuses if status is not None else None) or (): + if container.name == MAIN_CONTAINER_NAME: + return int(container.restart_count) + return None + + def _http_port(spec: Any) -> Optional[int]: """The ``http`` container port of the main container, else of any container.""" @@ -371,9 +381,21 @@ def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> di ), ) - def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: - """Missing Leases are not an error.""" + def delete_lease( + self, + name: str, + *, + resource_version: Optional[str] = None, + timeout_s: float = 10.0, + ) -> None: + """Missing Leases are not an error. ``resource_version`` makes the + delete conditional (a 409 -> ``KubeConflict`` when it changed).""" + options = None + if resource_version is not None: + options = client.V1DeleteOptions( + preconditions=client.V1Preconditions(resource_version=resource_version) + ) try: self._call( "delete", @@ -381,6 +403,7 @@ def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: lambda: self._coordination.delete_namespaced_lease( name=name, namespace=self.namespace, + body=options, _request_timeout=request_timeout(timeout_s), ), ) diff --git a/components/src/dynamo/planner/retune/live/state.py b/components/src/dynamo/planner/retune/live/state.py index fbc338ab8c90..c44a2d4a23bb 100644 --- a/components/src/dynamo/planner/retune/live/state.py +++ b/components/src/dynamo/planner/retune/live/state.py @@ -220,6 +220,19 @@ def _float_map(value: Any, what: str) -> dict[str, float]: return out +def _writable_entry(item: Any) -> bool: + """A history entry that ``canonical_json`` can write back: a JSON object + without NaN or infinities (``json.loads`` accepts both).""" + + if not isinstance(item, Mapping): + return False + try: + canonical_json(item) + except ValueError: + return False + return True + + def _nested_map(value: Any, what: str) -> dict[str, dict[str, float]]: if not isinstance(value, Mapping): raise StateFormatError(f"{what} must be an object") @@ -309,6 +322,9 @@ def from_annotations( mode = body.get("mode") if mode is not None and mode not in _MODES: raise StateFormatError(f"state.mode must be one of {_MODES} or null") + fingerprint = body.get("fingerprint") + if fingerprint is not None and not isinstance(fingerprint, str): + raise StateFormatError("state.fingerprint must be a string or null") deploy_seq = body.get("deploy_seq", 0) if isinstance(deploy_seq, bool) or not isinstance(deploy_seq, int): raise StateFormatError("state.deploy_seq must be an integer") @@ -328,11 +344,17 @@ def from_annotations( logger.warning("retune state: unreadable history annotation dropped") parsed = [] if isinstance(parsed, list): - history = [dict(item) for item in parsed if isinstance(item, Mapping)] + # A NaN in one entry would make every later write raise. + history = [dict(item) for item in parsed if _writable_entry(item)] + if len(history) < len(parsed): + logger.warning( + "retune state: %d malformed history entries dropped", + len(parsed) - len(history), + ) return cls( dgd_uid=dgd_uid, leader_term=str(body.get("leader_term") or ""), - fingerprint=body.get("fingerprint"), + fingerprint=fingerprint, mode=mode, rollback_pending=bool(body.get("rollback_pending", False)), deploy_seq=int(deploy_seq), @@ -630,8 +652,18 @@ def write(self, state: RetuneState, snapshot: StateSnapshot) -> StateSnapshot: lease=replaced, ) - def delete(self) -> None: - self._kube.delete_lease(self.name, timeout_s=self._timeout_s) + def delete(self, snapshot: StateSnapshot) -> None: + """Delete the state Lease read or written as ``snapshot``. + + Conditional on its resourceVersion: a Lease changed since (another + term claimed it) raises ``KubeConflict`` and stays. + """ + + self._kube.delete_lease( + self.name, + resource_version=snapshot.resource_version, + timeout_s=self._timeout_s, + ) def _new_body(self, annotations: Mapping[str, str]) -> dict[str, Any]: return { @@ -706,6 +738,8 @@ def __init__( self._held = False self._leader_until = -math.inf self.transitions = 0 + # resourceVersion of this process's last successful write. + self.resource_version: Optional[str] = None # (holderIdentity, resourceVersion) of a foreign lease, and when this # process first saw it unchanged. self._observed: Optional[tuple[str, Optional[str]]] = None @@ -779,7 +813,7 @@ def step(self) -> LeaseStep: new_holder = (updated.get("spec") or {}).get("holderIdentity") or "" if new_holder != self.identity: return self._lose() - self._lead(t_send, transitions) + self._lead(t_send, transitions, updated) return "renewed" if was_held else "acquired" lost = self.held @@ -838,11 +872,12 @@ def release(self) -> bool: # ------------------------------------------------------------ helpers - def _lead(self, t_send: float, transitions: int) -> None: + def _lead(self, t_send: float, transitions: int, written: dict[str, Any]) -> None: with self._lock: self._held = True self._leader_until = t_send + self.lease_duration_s - self.margin_s self.transitions = transitions + self.resource_version = (written.get("metadata") or {}).get("resourceVersion") self._observed = None def _lose(self) -> LeaseStep: @@ -870,7 +905,7 @@ def _create(self, t_send: float) -> LeaseStep: }, } try: - self._kube.create_lease(body, timeout_s=self._timeout_s) + created = self._kube.create_lease(body, timeout_s=self._timeout_s) except (KubeAlreadyExists, KubeConflict): return self._lose() except KubeError as error: @@ -878,7 +913,7 @@ def _create(self, t_send: float) -> LeaseStep: "retune leadership: creating %s failed: %s", self.name, error ) return "error" - self._lead(t_send, 0) + self._lead(t_send, 0, created) logger.info( "retune leadership: acquired new lease %s as %s", self.name, self.identity ) @@ -889,7 +924,9 @@ def _take_over( ) -> LeaseStep: body = self._body(lease, transitions=transitions + 1, acquire=True) try: - self._kube.replace_lease(self.name, body, timeout_s=self._timeout_s) + written = self._kube.replace_lease( + self.name, body, timeout_s=self._timeout_s + ) except KubeConflict: return "lost" if lost else "standby" except KubeError as error: @@ -897,7 +934,7 @@ def _take_over( "retune leadership: taking over %s failed: %s", self.name, error ) return "error" - self._lead(t_send, transitions + 1) + self._lead(t_send, transitions + 1, written) logger.info( "retune leadership: acquired lease %s as %s (transitions=%d)", self.name, diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py index c08a1ada40d4..6f5fe8c8c9ca 100644 --- a/components/src/dynamo/planner/retune/live/supervisor.py +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -1257,7 +1257,7 @@ async def _read_own_snapshot(self) -> Optional[StateSnapshot]: snapshot.state.dgd_uid, self._dgd.uid, ) - await self._io(self._store.delete) + await self._io(self._store.delete, snapshot) await self._delete_foreign_requests(snapshot.state.dgd_uid) snapshot = StateSnapshot(state=None, resource_version=None, lease=None) elif snapshot.corrupt: @@ -1771,10 +1771,19 @@ async def _finish_decommission(self, epoch: int) -> None: Only as the initialized leader of ``epoch``, checked again before every delete: a lost term (a failed state write of this round) or lost leadership stops it, so it never deletes under another leader. + The local clock is not the only fence: each Lease delete is + conditional on the resourceVersion this process last read or wrote, + so it fails once another term has claimed the state or taken over + the leadership Lease. """ - state = self.state - if self.decommissioned or state is None or state.rollback_pending: + snapshot = self._snapshot + if ( + self.decommissioned + or snapshot is None + or snapshot.state is None + or snapshot.state.rollback_pending + ): return if not self._reconcile_allowed(epoch): return @@ -1792,15 +1801,28 @@ async def _finish_decommission(self, epoch: int) -> None: await self._io(self._kube.delete_request, name) if not self._reconcile_allowed(epoch): return - await self._io(self._store.delete) + await self._io(self._store.delete, snapshot) + except KubeConflict: + logger.warning( + "retune clean-up: the state changed since this term read it; " + "not deleting it" + ) + self._request_reinit("the state changed under the clean-up") + return except (KubeError, asyncio.TimeoutError) as error: logger.warning("retune clean-up: deleting Requests/state failed: %s", error) return if not self._reconcile_allowed(epoch): return try: - await self._io(self._kube.delete_lease, self._leadership.name) + await self._io( + self._kube.delete_lease, + self._leadership.name, + resource_version=self._leadership.resource_version, + ) except (KubeError, asyncio.TimeoutError) as error: + # A KubeConflict: the Lease changed since this process wrote it. + # It stays for the DGD's garbage collection or a later clean-up. logger.warning("retune clean-up: deleting the lease failed: %s", error) self._leadership.drop() self._initialized = False diff --git a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py index 269a14f4703f..2d312f0059a8 100644 --- a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py +++ b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py @@ -13,9 +13,8 @@ Fake: the Kubernetes API (``FakeRetuneKube``, with ``FakeDgdrOperator`` driving each DGDR v1beta2 Request to a Run and its Candidates), the -frontends' ``/router_policy`` (revision-checked PATCH and reset with the -semantics of ``router_policy.rs`` and ``worker_selection/runtime.rs``), the -trace claim (a temporary directory of real ``jsonl.gz`` segments) and the +frontends' ``/router_policy`` (a revision-checked PATCH that merges, and a +reset to the startup values), the trace claim (a temporary directory of real ``jsonl.gz`` segments) and the clocks. The test advances the clocks in 15 s steps; each step runs every live planner the way ``NativePlannerBase`` and the supervisor's background tasks do: lease round, the due planner tick (its decisions go to the @@ -31,7 +30,6 @@ import dataclasses import gzip import json -import math import os import time import uuid @@ -383,11 +381,17 @@ def __init__(self, kube: FakeRetuneKube) -> None: def start(self, pod: str) -> None: self.kube.add_pod(pod, dgd=DGD_NAME, component_type="frontend", ready=True) - self.restart(pod) + self._register(pod) def restart(self, pod: str) -> None: - """A new frontend process: a new router id at revision 0 with the - startup values (``RouterPolicyRegistry::register``).""" + """A new frontend process in the pod (a container restart).""" + + self.kube.restart_pod(pod) + self._register(pod) + + def _register(self, pod: str) -> None: + """A new router id at revision 0 with the startup values + (``RouterPolicyRegistry::register``).""" startup = startup_map() self.routers[pod] = [ @@ -413,13 +417,9 @@ def persisted_state(self) -> Optional[RetuneState]: class FakeRouterPolicyClient: - """``RouterPolicyClient`` with the ``/router_policy`` semantics of - ``lib/llm/src/http/service/router_policy.rs`` and - ``lib/kv-router/src/plugins/worker_selection/runtime.rs``: an empty patch - is 400, an unknown router 404, a stale ``expected_revision`` 409, an - unknown parameter or an invalid value 422; a PATCH replaces only the - keys it names (``null`` restores that key's startup value), a reset - restores every startup value, and both bump the revision.""" + """``RouterPolicyClient`` over ``FakeFrontends``: a stale + ``expected_revision`` is 409; a PATCH replaces only the keys it names, a + reset restores every startup value, and both bump the revision.""" def __init__(self, frontends: FakeFrontends, writer: str) -> None: self.frontends = frontends @@ -485,39 +485,13 @@ def _apply( expected_revision: int, parameters: Optional[dict[str, Any]], ) -> tuple[int, Optional[FakeRouter], Optional[str]]: - if parameters is not None and not parameters: - return 400, None, "provide either nonempty parameters or reset: true" - router = next( - ( - r - for r in self.frontends.routers.get(pod, ()) - if r.router_id == router_id - ), - None, - ) - if router is None: - return 404, None, "router instance not found" + (router,) = [r for r in self.frontends.routers[pod] if r.router_id == router_id] if expected_revision != router.revision: return 409, None, "policy revision conflict" if parameters is None: - updated = dict(router.initial) + router.parameters = dict(router.initial) else: - updated = dict(router.parameters) - for key, value in parameters.items(): - if key not in router.initial: - return 422, None, f"unknown parameter {key!r}" - if value is None: - updated[key] = router.initial[key] - continue - if ( - isinstance(value, bool) - or not isinstance(value, (int, float)) - or not math.isfinite(value) - or value < 0 - ): - return 422, None, f"invalid value {value!r} for {key!r}" - updated[key] = float(value) - router.parameters = updated + router.parameters = {**router.parameters, **parameters} router.revision += 1 return 200, router, None diff --git a/components/src/dynamo/planner/tests/live_retune_fakes.py b/components/src/dynamo/planner/tests/live_retune_fakes.py index d78ea5d21fdd..a026b23e3019 100644 --- a/components/src/dynamo/planner/tests/live_retune_fakes.py +++ b/components/src/dynamo/planner/tests/live_retune_fakes.py @@ -32,7 +32,7 @@ import uuid from collections import deque from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Literal, Optional from dynamo.planner.retune.live.contracts import ( @@ -527,6 +527,7 @@ def add_pod( ip: str | None = None, http_port: int | None = 8000, labels: Mapping[str, str] | None = None, + restarts: int | None = 0, ) -> PodInfo: all_labels = dict(labels or {}) if dgd is not None: @@ -541,11 +542,21 @@ def add_pod( terminating=terminating, labels=all_labels, http_port=http_port, + restarts=restarts, ) with self._lock: self.pods[name] = pod return pod + def restart_pod(self, name: str) -> PodInfo: + """The pod's ``main`` container restarted.""" + + with self._lock: + pod = self.pods[name] + pod = replace(pod, restarts=(pod.restarts or 0) + 1) + self.pods[name] = pod + return pod + def remove_pod(self, name: str) -> None: with self._lock: self.pods.pop(name, None) @@ -743,11 +754,31 @@ def operation() -> dict: "replace_lease", (name, body), {"timeout_s": timeout_s}, operation ) - def delete_lease(self, name: str, *, timeout_s: float = 10.0) -> None: + def delete_lease( + self, + name: str, + *, + resource_version: Optional[str] = None, + timeout_s: float = 10.0, + ) -> None: def operation() -> None: - self.leases.pop(name, None) + current = self.leases.get(name) + if current is None: + return + found = current["metadata"]["resourceVersion"] + if resource_version is not None and resource_version != found: + raise KubeConflict( + f"leases {name!r}: precondition resourceVersion " + f"{resource_version!r} is stale (current {found!r})" + ) + del self.leases[name] - return self._call("delete_lease", name, {"timeout_s": timeout_s}, operation) + return self._call( + "delete_lease", + name, + {"resource_version": resource_version, "timeout_s": timeout_s}, + operation, + ) # ------------------------------------------------- operator simulation diff --git a/components/src/dynamo/planner/tests/retune_stubs.py b/components/src/dynamo/planner/tests/retune_stubs.py index a225c96e3b5e..d9d137c369d8 100644 --- a/components/src/dynamo/planner/tests/retune_stubs.py +++ b/components/src/dynamo/planner/tests/retune_stubs.py @@ -13,7 +13,6 @@ import sys import types -from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from typing import Any @@ -106,22 +105,11 @@ def close(self) -> None: raise self.close_error -class StubSweepExecutor(ThreadPoolExecutor): - """The live sweep thread a host hands the stub (which never sweeps).""" - - def __init__(self) -> None: - super().__init__(max_workers=1) - - def busy(self) -> bool: - return False - - def install_stub_controller(monkeypatch) -> type[StubController]: """Install ``StubController`` as ``dynamo.planner.retune.controller``.""" module = types.ModuleType(CONTROLLER_MODULE) module.RetuneController = StubController # type: ignore[attr-defined] - module.SweepExecutor = StubSweepExecutor # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, CONTROLLER_MODULE, module) StubController.instances = [] StubController.construct_error = None diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py b/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py index c4904fa75f12..4a1237b96b3b 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_actuator.py @@ -12,12 +12,11 @@ from __future__ import annotations -import asyncio import json import re import socket from collections.abc import Callable, Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, Optional @@ -109,6 +108,8 @@ class FakeFrontend: routers: dict[str, FakeRouter] = field(default_factory=dict) # Statuses returned by the next GETs before the real answer. get_failures: list[int] = field(default_factory=list) + # Routers GET leaves out (their control call timed out). + timed_out: set[str] = field(default_factory=set) def add(self, router: FakeRouter) -> FakeRouter: self.routers[router.router_id] = router @@ -125,10 +126,6 @@ def __init__(self) -> None: self.write_failures: list[int] = [] # Called before a write is answered: (op, pod name, router_id). self.before_write: Optional[Callable[[str, str, str], None]] = None - self.raise_on_get: Optional[BaseException] = None - self.active = 0 - self.max_active = 0 - self.get_delay_s = 0.0 def frontend(self, pod: str) -> FakeFrontend: return self.frontends.setdefault(pod, FakeFrontend()) @@ -138,24 +135,19 @@ def writes(self) -> list[tuple[Any, ...]]: async def get(self, pod: PodInfo) -> PolicyCallResult: self.calls.append(("get", pod.name)) - if self.raise_on_get is not None: - raise self.raise_on_get - self.active += 1 - self.max_active = max(self.max_active, self.active) - try: - if self.get_delay_s: - await asyncio.sleep(self.get_delay_s) - frontend = self.frontends.get(pod.name) - if frontend is None: - return PolicyCallResult(status=0, error="connection refused") - if frontend.get_failures: - return PolicyCallResult(status=frontend.get_failures.pop(0)) - return PolicyCallResult( - status=200, - routers=tuple(router.info() for router in frontend.routers.values()), - ) - finally: - self.active -= 1 + frontend = self.frontends.get(pod.name) + if frontend is None: + return PolicyCallResult(status=0, error="connection refused") + if frontend.get_failures: + return PolicyCallResult(status=frontend.get_failures.pop(0)) + return PolicyCallResult( + status=200, + routers=tuple( + router.info() + for router in frontend.routers.values() + if router.router_id not in frontend.timed_out + ), + ) async def patch( self, @@ -236,6 +228,7 @@ def pod( terminating: bool = False, ip: Optional[str] = "10.0.0.1", phase: Optional[str] = None, + restarts: int = 0, ) -> PodInfo: return PodInfo( name=name, @@ -246,6 +239,7 @@ def pod( labels={POD_LABEL_DGD_NAME: DGD.name, POD_LABEL_COMPONENT_TYPE: "frontend"}, http_port=8000, phase=phase, + restarts=restarts, ) @@ -253,18 +247,12 @@ def make_actuator( pods: list[PodInfo], *, startup_maps: Optional[dict[str, dict[str, float]]] = None, - max_concurrency: int = 8, ) -> tuple[RouterActuator, FakeKube, FakePolicyClient, RecordingMetrics]: kube = FakeKube(pods) client = FakePolicyClient() metrics = RecordingMetrics() actuator = RouterActuator( - kube, - client, - DGD, - startup_maps or {"aggregated": dict(STARTUP)}, - metrics, - max_concurrency=max_concurrency, + kube, client, DGD, startup_maps or {"aggregated": dict(STARTUP)}, metrics ) return actuator, kube, client, metrics @@ -413,36 +401,6 @@ async def test_parity_mismatch_blocks_apply_and_never_loops() -> None: assert client.frontend("fe-1").routers["odd"].current == odd -async def test_frontend_restart_fills_in_without_reset() -> None: - actuator, kube, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - initials = {"r1": dict(STARTUP)} - await actuator.reconcile(apply(CHOSEN), initials=initials, expected_frontends=1) - - # The frontend restarts: Ready is 0 for a while. - kube.pods = [pod("fe-0", ready=False)] - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - assert not report.converged - - # A new pod with a fresh router at its startup values: latched, then - # PATCHed once the state records it; never reset. - kube.pods = [pod("fe-0b", ip="10.0.0.9")] - client.frontend("fe-0b").add(FakeRouter("r2", STARTUP)) - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - assert report.new_initials == {"r2": STARTUP} - assert report.seen_router_ids == frozenset({"r2"}) - report = await actuator.reconcile( - apply(CHOSEN), initials=report.new_initials, expected_frontends=1 - ) - assert [call[0] for call in client.writes()] == ["patch", "patch"] - assert client.frontend("fe-0b").routers["r2"].current == CHOSEN - assert report.converged - - async def test_disagg_needs_both_worker_types() -> None: maps = {"prefill": dict(STARTUP), "decode": dict(DECODE_STARTUP)} actuator, _, client, _ = make_actuator([pod("fe-0")], startup_maps=maps) @@ -514,40 +472,6 @@ async def test_rollback_resets_only_deviating_routers_and_never_by_revision() -> assert len(client.writes()) == 1 -async def test_rollback_learns_unknown_initial_with_a_single_reset() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - router = client.frontend("fe-0").add( - FakeRouter("r1", STARTUP, revision=5, current=dict(CHOSEN)) - ) - - report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) - assert client.writes() == [("reset", "fe-0", "r1", 5)] - assert report.new_initials == {"r1": STARTUP} - assert router.current == STARTUP - assert report.converged - - for _ in range(2): - report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) - assert report.converged - assert len(client.writes()) == 1 - - -async def test_rollback_with_parity_mismatch_resets_to_each_routers_initial() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - odd = {**STARTUP, "overlap_score_credit_decay": 0.5} - router = client.frontend("fe-0").add(FakeRouter("odd", odd)) - await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - router.current = {**odd, "router_temperature": 1.0} - router.revision = 1 - - report = await actuator.reconcile(ROLLBACK, initials={}, expected_frontends=1) - - assert client.writes() == [("reset", "fe-0", "odd", 1)] - assert router.current == odd - assert report.converged # rollback converges to the latched initial - assert report.parity_ok is False - - async def test_rollback_without_learning_leaves_unknown_routers_alone() -> None: actuator, _, client, _ = make_actuator([pod("fe-0")]) frontend = client.frontend("fe-0") @@ -580,82 +504,42 @@ async def test_rollback_without_learning_leaves_unknown_routers_alone() -> None: assert report.converged -async def test_rollback_targets_not_ready_pods_but_never_terminating_ones() -> None: +async def test_rollback_targets_not_ready_pods_and_waits_only_for_ready_ones() -> None: pods = [ pod("ready", ip="10.0.0.1"), pod("not-ready", ready=False, ip="10.0.0.2"), + pod("crashing", ready=False, ip="10.0.0.3"), pod("pending", ready=False, ip=None), pod("leaving", terminating=True, ip="10.0.0.4"), ] - actuator, _, client, _ = make_actuator(pods) + actuator, kube, client, _ = make_actuator(pods) initials = {} for name in ("ready", "not-ready", "leaving"): client.frontend(name).add( FakeRouter(f"r-{name}", STARTUP, revision=1, current=dict(CHOSEN)) ) initials[f"r-{name}"] = dict(STARTUP) + # CrashLoopBackOff: an unreachable NotReady pod never blocks convergence. + client.frontend("crashing").get_failures = [0] report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) - assert sorted(call[1] for call in client.writes()) == ["not-ready", "ready"] assert ("get", "leaving") not in client.calls assert ("get", "pending") not in client.calls - assert report.converged - - # apply never targets NotReady pods. - client.calls.clear() - await actuator.reconcile(apply(CHOSEN), initials=initials, expected_frontends=1) - assert {call[1] for call in client.calls} == {"ready"} - - -async def test_unreachable_pods_and_convergence() -> None: - actuator, kube, client, _ = make_actuator( - [pod("ready"), pod("crashing", ready=False, ip="10.0.0.2")] - ) - router = client.frontend("ready").add( - FakeRouter("r1", STARTUP, revision=1, current=dict(CHOSEN)) - ) - # CrashLoopBackOff: an unreachable NotReady pod never blocks convergence. - client.frontend("crashing").get_failures = [0, 0] - for _ in range(2): - report = await actuator.reconcile( - ROLLBACK, initials={"r1": STARTUP}, expected_frontends=1 - ) - assert report.converged - assert report.errors == {"get:network": 1} - assert router.current == STARTUP + assert report.converged and report.errors == {"get:network": 1} - # An unreachable Ready pod does. - kube.pods = [pod("ready"), pod("crashing", ip="10.0.0.2")] + # An unreachable Ready pod does block it; apply never targets NotReady pods. + kube.pods = [pod("ready"), pod("crashing", ip="10.0.0.3")] client.frontend("crashing").get_failures = [0] + client.calls.clear() report = await actuator.reconcile( - ROLLBACK, initials={"r1": STARTUP}, expected_frontends=1 + apply(CHOSEN), initials=initials, expected_frontends=1 ) assert not report.converged - - -async def test_finished_pods_are_never_targeted_and_their_routers_are_gone() -> None: - actuator, kube, client, _ = make_actuator( - [pod("fe-0"), pod("evicted", ip="10.0.0.2")] - ) - client.frontend("fe-0").add(FakeRouter("r0", STARTUP)) - client.frontend("evicted").add(FakeRouter("r-ev", STARTUP)) - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - initials = dict(report.new_initials) - assert set(initials) == {"r0", "r-ev"} - - # Evicted: the pod object stays, with its IP, in a finished phase. - kube.pods = [ - pod("fe-0"), - pod("evicted", ready=False, ip="10.0.0.2", phase="Failed"), - ] + kube.pods = pods client.calls.clear() - - report = await actuator.reconcile(ROLLBACK, initials=initials, expected_frontends=1) - - assert ("get", "evicted") not in client.calls - assert report.converged - assert report.seen_router_ids == frozenset({"r0"}) + await actuator.reconcile(apply(CHOSEN), initials=initials, expected_frontends=1) + assert {call[1] for call in client.calls} == {"ready"} # ----------------------------------------------------------- status codes @@ -687,104 +571,28 @@ def someone_else_writes(op: str, pod_name: str, router_id: str) -> None: assert report.converged -async def test_conflict_twice_gives_up_until_next_round() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - router = client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - client.write_failures = [409, 409] - initials = {"r1": dict(STARTUP)} - - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - assert len(client.writes()) == 2 - assert not report.converged - - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - assert router.current == CHOSEN - assert report.converged - - -async def test_422_is_not_retried_until_the_target_changes() -> None: - actuator, _, client, metrics = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - client.write_failures = [422] - initials = {"r1": dict(STARTUP)} - - for _ in range(3): - report = await actuator.reconcile( - apply(CHOSEN), initials=initials, expected_frontends=1 - ) - assert not report.converged - assert len(client.writes()) == 1 - assert metrics.patch_errors == ["422"] - - other = {**CHOSEN, "router_temperature": 0.4} - report = await actuator.reconcile( - apply(other), initials=initials, expected_frontends=1 - ) - assert len(client.writes()) == 2 - assert report.converged - - -async def test_get_404_for_three_consecutive_rounds_reports_admin_api_disabled() -> ( - None -): - actuator, _, client, _ = make_actuator([pod("fe-0")]) - frontend = client.frontend("fe-0") - frontend.add(FakeRouter("r1", STARTUP)) - # A 200 in between restarts the count. - frontend.get_failures = [404, 404] - for _ in range(3): - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - assert not report.admin_api_disabled - - frontend.get_failures = [404, 404, 404] - flags = [] - for _ in range(3): - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - flags.append(report.admin_api_disabled) - assert report.errors == {"get:404": 1} - assert flags == [False, False, True] - - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - assert not report.admin_api_disabled - assert report.converged - - # ------------------------------------------------------------- convergence -async def test_convergence_needs_the_expected_ready_frontends_and_a_router() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - frontend = client.frontend("fe-0") - - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - assert not report.converged and report.parity_ok is None # no router - - frontend.add(FakeRouter("r1", STARTUP)) - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=2) - assert not report.converged - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=0) - assert report.converged # max(1, 0) - - -async def test_routers_present_needs_a_configured_router_not_the_target() -> None: +async def test_convergence_and_presence_need_the_expected_frontends() -> None: actuator, _, client, _ = make_actuator([pod("fe-0")]) frontend = client.frontend("fe-0") frontend.add(FakeRouter("custom", STARTUP, runtime_config=False)) + # No configured router: neither converged nor present (P12). report = await actuator.reconcile(apply(CHOSEN), initials={}, expected_frontends=1) - assert report.routers_present is False - assert not report.converged and client.writes() == [] + assert not report.converged and report.routers_present is False + assert report.parity_ok is None and client.writes() == [] - # Present (P12) even though the router does not match the apply target. + # Present even though the router does not match the apply target. frontend.add(FakeRouter("r1", {**STARTUP, "router_temperature": 0.9})) report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - assert report.routers_present is True + assert report.converged and report.routers_present is True report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=2) - assert report.routers_present is False + assert not report.converged and report.routers_present is False + + +# ------------------------------------------------------------ gone routers async def test_seen_router_ids_drop_only_routers_that_are_provably_gone() -> None: @@ -797,131 +605,65 @@ async def test_seen_router_ids_drop_only_routers_that_are_provably_gone() -> Non initials = dict(report.new_initials) assert report.seen_router_ids == frozenset({"a", "b"}) - # fe-1 does not answer: b is kept; a left fe-0, which answered: dropped. - del frontend0.routers["a"] - frontend0.add(FakeRouter("a2", STARTUP)) + # GET leaves out a router whose control call timed out, and a pod that + # does not answer shows none: both are kept, as on a list failure. + frontend0.timed_out.add("a") client.frontend("fe-1").get_failures = [0] report = await actuator.reconcile(OBSERVE, initials=initials, expected_frontends=2) - assert report.seen_router_ids == frozenset({"a2", "b"}) + assert report.seen_router_ids == frozenset({"a", "b"}) + kube.fail = KubeError("apiserver down") + report = await actuator.reconcile(OBSERVE, initials=initials, expected_frontends=2) + assert report.errors == {"list_pods": 1} and actuator.last_pods is None + assert report.seen_router_ids == frozenset({"a", "b"}) + kube.fail = None - # fe-1 is gone from the listing: b goes too. - kube.pods = [pod("fe-0")] + # fe-0's frontend restarted with a new router; fe-1 was evicted (the pod + # object stays, with its IP, in a finished phase and is never asked). + del frontend0.routers["a"] + frontend0.timed_out.clear() + frontend0.add(FakeRouter("a2", STARTUP)) + kube.pods = [ + pod("fe-0", restarts=1), + pod("fe-1", ready=False, ip="10.0.0.2", phase="Failed"), + ] + client.calls.clear() report = await actuator.reconcile(OBSERVE, initials=initials, expected_frontends=1) + assert ("get", "fe-1") not in client.calls assert report.seen_router_ids == frozenset({"a2"}) -# --------------------------------------------------------- failure paths - - -async def test_list_pods_failure_reports_without_raising() -> None: - actuator, kube, _, metrics = make_actuator([pod("fe-0")]) - kube.fail = KubeError("apiserver down") - - report = await actuator.reconcile( - OBSERVE, initials={"r1": STARTUP}, expected_frontends=1 - ) - - assert not report.converged - assert report.errors == {"list_pods": 1} - assert report.seen_router_ids == frozenset({"r1"}) - assert actuator.last_pods is None - assert metrics.converged == [False] - - -async def test_unexpected_client_exception_never_escapes() -> None: - actuator, _, client, metrics = make_actuator([pod("fe-0")]) - client.raise_on_get = RuntimeError("bug") - - report = await actuator.reconcile( - apply(CHOSEN), initials={"r1": STARTUP}, expected_frontends=1 +async def test_a_router_missing_from_get_is_still_rolled_back() -> None: + # A clean-up (advisory, learn_unknown off) planner restarts with the + # initial of a retuned router persisted. Its first GET leaves the router + # out (control call timed out): the initial must survive, or the router + # comes back at revision > 0, is left alone, and stays retuned. + target = ActuationTarget( + mode="rollback", include_not_ready=True, learn_unknown=False ) + actuator, kube, client, _ = make_actuator([pod("fe-0")]) + frontend = client.frontend("fe-0") + router = frontend.add(FakeRouter("r1", STARTUP, revision=1, current=dict(CHOSEN))) + frontend.timed_out.add("r1") + initials = {"r1": dict(STARTUP)} - assert not report.converged - assert report.errors == {"internal": 1} + report = await actuator.reconcile(target, initials=initials, expected_frontends=1) assert report.seen_router_ids == frozenset({"r1"}) - assert metrics.converged == [False] - - -async def test_cancellation_propagates() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.raise_on_get = asyncio.CancelledError() - - with pytest.raises(asyncio.CancelledError): - await actuator.reconcile(OBSERVE, initials={}, expected_frontends=1) - - -@pytest.mark.parametrize( - "target", - [ - ActuationTarget(mode="apply"), - ActuationTarget(mode="apply", desired_by_wt={"prefill": dict(CHOSEN)}), - ActuationTarget( - mode="apply", - desired_by_wt={"aggregated": {**CHOSEN, "router_temperature": -1.0}}, - ), - ActuationTarget( - mode="apply", - desired_by_wt={ - "aggregated": { - k: v for k, v in CHOSEN.items() if k != "prefill_load_scale" - } - }, - ), - ], -) -async def test_invalid_apply_target_writes_nothing(target: ActuationTarget) -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - client.frontend("fe-0").add(FakeRouter("r1", STARTUP)) - - report = await actuator.reconcile( - target, initials={"r1": STARTUP}, expected_frontends=1 - ) - - assert client.writes() == [] - assert not report.converged - assert report.errors == {"invalid_target": 1} - - -async def test_max_concurrency_bounds_parallel_gets() -> None: - pods = [pod(f"fe-{i}", ip=f"10.0.0.{i + 1}") for i in range(6)] - actuator, _, client, _ = make_actuator(pods, max_concurrency=2) - for i in range(6): - client.frontend(f"fe-{i}").add(FakeRouter(f"r{i}", STARTUP)) - client.get_delay_s = 0.01 - - report = await actuator.reconcile(OBSERVE, initials={}, expected_frontends=6) - - assert client.max_active == 2 - assert report.converged - assert actuator.last_pods is not None and len(actuator.last_pods) == 6 - - -def test_constructor_validation() -> None: - kube, client, metrics = FakeKube(), FakePolicyClient(), RecordingMetrics() - with pytest.raises(ValueError): - RouterActuator(kube, client, DGD, {}, metrics) - with pytest.raises(ValueError): - RouterActuator(kube, client, DGD, {"aggregated": STARTUP}, metrics, 0) - actuator = RouterActuator( - kube, client, DGD, {"prefill": STARTUP, "decode": DECODE_STARTUP}, metrics - ) - assert actuator.worker_types == ("prefill", "decode") - actuator.set_startup_maps({"aggregated": STARTUP}) - assert actuator.worker_types == ("aggregated",) - + assert not report.converged and client.writes() == [] -async def test_close_closes_the_client_when_it_can() -> None: - actuator, _, client, _ = make_actuator([pod("fe-0")]) - await actuator.close() # FakePolicyClient has no close: no-op + frontend.timed_out.clear() + initials = {k: v for k, v in initials.items() if k in report.seen_router_ids} + report = await actuator.reconcile(target, initials=initials, expected_frontends=1) + assert client.writes() == [("reset", "fe-0", "r1", 1)] + assert router.current == STARTUP and report.converged - http = AiohttpRouterPolicyClient() - actuator = RouterActuator( - FakeKube(), http, DGD, {"aggregated": STARTUP}, RecordingMetrics() - ) - session = http._ensure_session() # what the first call does - assert not session.closed - await actuator.close() - assert session.closed and http._session is None + # Once every frontend listed at the first miss is gone, so is the router. + actuator, kube, client, _ = make_actuator([pod("fe-0")]) + client.frontend("fe-0").add(FakeRouter("r2", STARTUP)) + report = await actuator.reconcile(target, initials=initials, expected_frontends=1) + assert report.seen_router_ids == frozenset({"r1", "r2"}) + kube.pods = [pod("fe-new", ip="10.0.0.3")] + report = await actuator.reconcile(target, initials=initials, expected_frontends=1) + assert report.seen_router_ids == frozenset() # ------------------------------------------------------------- HTTP client @@ -1015,13 +757,11 @@ async def frontend_server(): await server.close() -async def test_client_get_parses_the_router_list(frontend_server) -> None: +async def test_client_speaks_the_router_policy_wire_format(frontend_server) -> None: frontend, client, target = frontend_server result = await client.get(target) - assert result.status == 200 and result.error is None - assert frontend.requests == [("GET", {})] # no ?model= first, custom = result.routers assert first == RouterInfo( router_id="6f1c", @@ -1033,13 +773,10 @@ async def test_client_get_parses_the_router_list(frontend_server) -> None: ) assert custom.configuration is None and custom.revision == 0 - -async def test_client_patch_sends_explicit_parameters(frontend_server) -> None: - frontend, client, target = frontend_server - - result = await client.patch(target, "6f1c", 3, {"overlap_score_credit": 0}) - + patched = await client.patch(target, "6f1c", 3, {"overlap_score_credit": 0}) + reset = await client.reset(target, "6f1c", 7) assert frontend.requests == [ + ("GET", {}), # no ?model= ( "PATCH", { @@ -1047,101 +784,32 @@ async def test_client_patch_sends_explicit_parameters(frontend_server) -> None: "expected_revision": 3, "parameters": {"overlap_score_credit": 0.0}, }, - ) - ] - assert result.status == 200 - assert result.router is not None - assert result.router.revision == 4 - assert result.router.configuration["overlap_score_credit"] == 0.0 - - -async def test_client_reset_sends_reset_true_without_parameters( - frontend_server, -) -> None: - frontend, client, target = frontend_server - - result = await client.reset(target, "6f1c", 7) - - assert frontend.requests == [ - ("PATCH", {"router_id": "6f1c", "expected_revision": 7, "reset": True}) + ), + ("PATCH", {"router_id": "6f1c", "expected_revision": 7, "reset": True}), ] - assert result.status == 200 and result.router.revision == 8 + assert patched.router is not None and patched.router.revision == 4 + assert patched.router.configuration["overlap_score_credit"] == 0.0 + assert reset.status == 200 and reset.router.revision == 8 -async def test_client_reports_the_error_status_and_message(frontend_server) -> None: +async def test_client_errors_are_statuses(frontend_server) -> None: frontend, client, target = frontend_server frontend.patch_status = 409 - result = await client.patch(target, "6f1c", 1, {"router_temperature": 0.1}) - - assert result.status == 409 - assert result.router is None + assert (result.status, result.router) == (409, None) assert result.error == "policy revision conflict: expected 1, current 2" + for body in ("not json", json.dumps({"routers": "x"}), json.dumps({"other": []})): + frontend.list_body = body + result = await client.get(target) + assert result.status == 0 + assert result.error and result.error.startswith("malformed GET response") -@pytest.mark.parametrize( - "body", ["not json", json.dumps({"routers": "x"}), json.dumps({"other": []})] -) -async def test_client_malformed_body_is_status_zero(frontend_server, body) -> None: - frontend, client, target = frontend_server - frontend.list_body = body - - result = await client.get(target) - - assert result.status == 0 - assert result.error and result.error.startswith("malformed GET response") - - -async def test_client_network_error_is_status_zero() -> None: with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) - port = sock.getsockname()[1] - client = AiohttpRouterPolicyClient(timeout_s=1.0) - target = PodInfo( - name="fe", - uid="u", - ip="127.0.0.1", - ready=True, - terminating=False, - http_port=port, - ) - try: - result = await client.get(target) - finally: - await client.close() - assert result.status == 0 - assert result.error - - -async def test_client_without_pod_ip_or_parameters_sends_nothing() -> None: - client = AiohttpRouterPolicyClient() - no_ip = PodInfo(name="fe", uid="u", ip=None, ready=False, terminating=False) - result = await client.get(no_ip) - assert result.status == 0 and "no IP" in (result.error or "") - with_ip = PodInfo(name="fe", uid="u", ip="10.0.0.1", ready=True, terminating=False) - result = await client.patch(with_ip, "r", 0, {}) - assert result.status == 0 and result.error == "empty parameters" - await client.close() - - -def test_client_url() -> None: - client = AiohttpRouterPolicyClient(scheme="http") - base = dict(name="fe", uid="u", ready=True, terminating=False) - assert ( - client.url(PodInfo(ip="10.1.2.3", http_port=8123, **base)) - == "http://10.1.2.3:8123/router_policy" - ) - assert ( - client.url(PodInfo(ip="10.1.2.3", **base)) - == "http://10.1.2.3:8000/router_policy" - ) - assert client.url(PodInfo(ip="fd00::5", http_port=80, **base)) == ( - "http://[fd00::5]:80/router_policy" - ) - with pytest.raises(ValueError): - AiohttpRouterPolicyClient(scheme="ftp") - with pytest.raises(ValueError): - AiohttpRouterPolicyClient(timeout_s=0) + closed_port = sock.getsockname()[1] + result = await client.get(replace(target, http_port=closed_port)) + assert result.status == 0 and result.error def test_client_timeout_outlasts_the_server_control_timeout() -> None: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py index 4d2dbd239588..5e3db54465b3 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py @@ -12,7 +12,6 @@ from __future__ import annotations import asyncio -import dataclasses import logging import os import signal @@ -35,7 +34,7 @@ WorkerCounts, ) from dynamo.planner.environment.state import DeploymentState -from dynamo.planner.plugins.clock import VirtualClock, WallClock +from dynamo.planner.plugins.clock import WallClock from dynamo.planner.plugins.orchestrator.engine_adapter import ( OrchestratorEngineAdapter, RetuneHooks, @@ -133,70 +132,30 @@ def test_live_controller_gets_the_hooks_and_the_phase(stub_controller): assert (controller.started, controller.phases) == ([1_000.0], [PHASE]) -def test_replay_controller_is_built_and_started_as_before(stub_controller): - engine = OrchestratorEngineAdapter( - retune_only_config(), - caps(), - clock=VirtualClock(), - retune_inputs=retune_inputs(), - ) - - (controller,) = stub_controller.instances - assert controller.hooks == {} - engine.initial_tick(5.0) - assert (controller.started, controller.phases) == ([5.0], [None]) - - -def test_replay_without_inputs_still_fails_at_construction(stub_controller): - with pytest.raises(ValueError, match="requires retune inputs"): - OrchestratorEngineAdapter(retune_only_config(), caps(), clock=VirtualClock()) - - -def test_hooks_without_retune_config_build_nothing(stub_controller): - engine = OrchestratorEngineAdapter( - PlannerConfig(mode="agg"), - caps(), - clock=WallClock(), - retune_inputs=retune_inputs(), - retune_hooks=_hooks(), - ) - - assert engine.retune_controller is None - assert stub_controller.instances == [] - - -def test_live_without_inputs_starts_disabled(stub_controller, caplog): +def test_live_without_inputs_or_a_valid_controller_starts_disabled( + stub_controller, caplog +): with caplog.at_level(logging.WARNING): engine = _adapter(inputs=None) - - assert engine.retune_controller is None - assert stub_controller.instances == [] + assert engine.retune_controller is None and stub_controller.instances == [] assert "retune stays disabled" in caplog.text engine.initial_tick(0.0) # no controller to start - -def test_live_controller_construction_failure_starts_disabled(stub_controller, caplog): stub_controller.construct_error = ValueError("retune.router requires kv_router") - with caplog.at_level(logging.ERROR): engine = _adapter(inputs=retune_inputs()) - assert engine.retune_controller is None assert "requires kv_router" in caplog.text -@pytest.mark.parametrize( - "ready, overrides", - [(16, {}), (1, {"min_endpoint": 2})], - ids=["above-default-max-gpu-budget", "below-min-endpoint"], -) async def test_disabled_retune_only_planner_never_reconciles_replicas( - stub_controller, ready, overrides + stub_controller, ): # retune_only follows the config, not the controller: a disabled live - # retune keeps the deployment's replicas and stays quiet. - engine = _adapter(inputs=None, optimization_target="sla", **overrides) - counts = WorkerCounts(ready_num_decode=ready, expected_num_decode=ready) + # retune keeps the deployment's replicas (here below min_endpoint) and + # stays quiet. + engine = _adapter(inputs=None, optimization_target="sla", min_endpoint=2) + counts = WorkerCounts(ready_num_decode=1, expected_num_decode=1) scheduled = engine.initial_tick(0.0) for _ in range(10): @@ -267,92 +226,6 @@ def test_replace_retune_with_no_inputs_disables_retune(stub_controller): assert len(stub_controller.instances) == 1 -def test_replace_retune_construction_failure_leaves_retune_disabled( - stub_controller, -): - engine = _adapter(inputs=retune_inputs()) - (old,) = stub_controller.instances - stub_controller.construct_error = ValueError("zero startup credit") - - with pytest.raises(ValueError, match="zero startup credit"): - engine.replace_retune(retune_inputs(), generation=3, now_s=10.0) - - assert old.closed == 1 - assert engine.retune_controller is None - - -def test_replace_retune_survives_an_old_controller_that_fails_to_close( - stub_controller, -): - engine = _adapter(inputs=retune_inputs()) - (old,) = stub_controller.instances - old.close_error = RuntimeError("worker stuck") - - engine.replace_retune(retune_inputs(), generation=1, now_s=10.0) - - assert engine.retune_controller is stub_controller.instances[1] - - -def test_replace_retune_requires_live_hooks(stub_controller): - engine = OrchestratorEngineAdapter( - retune_only_config(), - caps(), - clock=VirtualClock(), - retune_inputs=retune_inputs(), - ) - - with pytest.raises(RuntimeError, match="live retune"): - engine.replace_retune(None, generation=1, now_s=0.0) - - -async def test_real_controller_with_live_hooks_anchors_at_the_phase(): - pytest.importorskip( - "aisimulate.sweeper", - reason="AI Simulate is an optional Dynamo simulation dependency", - ) - from dynamo.planner.retune.sweep import SweepResult - - def sweep(request, *, cancel, abort): - return SweepResult( - status="ok", - num_requests=1, - scores={candidate.key(): 1.0 for candidate in request.candidates}, - ) - - gate_calls = [] - - def gate(k, window): - gate_calls.append((k, window)) - return GateResult(spec_replicas={"aggregated": 2}) - - engine = OrchestratorEngineAdapter( - retune_only_config(), - caps(), - clock=VirtualClock(), - retune_inputs=retune_inputs(), - retune_hooks=_hooks(sweep_fn=sweep, launch_gate=gate, phase_s=0.0), - ) - controller = engine.retune_controller - assert controller is not None and controller.generation == 0 - counts = WorkerCounts(ready_num_decode=2, expected_num_decode=2) - try: - # Started mid-window W_1 = [60, 120): W_1 launches at L_1 = 120 + grace. - scheduled = engine.initial_tick(90.0) - decisions = [] - while scheduled.at_s <= 240.0: - effects = await engine.tick( - scheduled, TickInput(now_s=scheduled.at_s, worker_counts=counts) - ) - decisions.extend(effects.retune_decisions) - assert effects.next_tick is not None - scheduled = effects.next_tick - finally: - await engine.shutdown() - - assert gate_calls[0][0] == 0 - assert [d.decision_id for d in decisions][:2] == ["retune-w60000", "retune-w120000"] - - @pytest.mark.timeout(10) async def test_rebuilt_real_controllers_never_sweep_at_once(): """Two generations of one window never run their sweeps concurrently. @@ -360,7 +233,7 @@ async def test_rebuilt_real_controllers_never_sweep_at_once(): Generation 0's sweep of window k is dropped but, like a DGDR sweep that polls, only returns a while later; generation 1 relaunches window k in the meantime. Its sweep must start only after generation 0's returned, - and its decision's launch time moves to that start. + and it is decided once it is done. """ pytest.importorskip( @@ -440,8 +313,6 @@ def sweep(request, *, cancel, abort): assert [g for g, _ in order] == [0, 1] and k0 == k1 (decision,) = effects.retune_decisions assert decision.window_index == k0 - # Launched when generation 0's sweep returned, not at the tick. - assert decision.launched_at_s > now + 1.0 # --------------------------------------------------------------------------- @@ -478,7 +349,6 @@ def __init__(self, calls, *, inputs=None, hooks=None): self.effects: list[PlannerEffects] = [] self.host = None self.on_effects_error: Exception | None = None - self.stop_error: Exception | None = None # The host's engine when stop ran (it must still be up). self.engine_at_stop = None @@ -497,8 +367,6 @@ async def stop(self): self.calls.append("live.stop") if self.host is not None: self.engine_at_stop = self.host._engine - if self.stop_error is not None: - raise self.stop_error def on_effects(self, effects): self.effects.append(effects) @@ -562,12 +430,6 @@ def test_live_planner_rejects_the_local_executor(): _base(_live_config("local"), _Environment()) -def test_live_planner_accepts_the_dgdr_executor(): - planner, _, _ = _planner() - - assert planner._live_retune is None - - async def test_async_init_builds_the_engine_with_live_retune_and_starts_it( stub_controller, ): @@ -604,17 +466,6 @@ async def bootstrap_regression(): assert planner._live_retune is None -async def test_async_init_with_failing_premises_starts_disabled(stub_controller): - planner, _, live = _planner(inputs=None) - - await planner._async_init() - try: - assert planner._engine.retune_controller is None # type: ignore[union-attr] - assert live.host is planner - finally: - await planner._shutdown_runtime() - - async def test_failed_init_still_stops_live_retune(stub_controller): planner, environment, live = _planner(inputs=retune_inputs()) @@ -629,18 +480,6 @@ async def failing_start(_host): assert environment.calls[-2:] == ["live.stop", "environment.shutdown"] -async def test_shutdown_logs_a_failing_live_retune_stop(stub_controller, caplog): - planner, environment, live = _planner(inputs=None) - await planner._async_init() - live.stop_error = RuntimeError("lease release timed out") - - with caplog.at_level(logging.ERROR): - await planner._shutdown_runtime() - - assert "Failed to stop planner live retune" in caplog.text - assert environment.calls[-1] == "environment.shutdown" - - class _FakeEngine: def __init__(self, effects): self.effects = effects @@ -704,13 +543,6 @@ async def test_replace_live_retune_holds_the_config_lock(stub_controller): await planner._shutdown_runtime() -async def test_replace_live_retune_needs_the_orchestrator_engine(): - planner, _, _ = _planner() - - with pytest.raises(RuntimeError, match="orchestrator engine"): - await planner.replace_live_retune(None, generation=1) - - # --------------------------------------------------------------------------- # SIGTERM # --------------------------------------------------------------------------- @@ -726,6 +558,17 @@ def shutdown(self) -> None: self.stopped.set() +def _found() -> asyncio.Event: + found = asyncio.Event() + found.set() + return found + + +def _sigterm_installed() -> bool: + loop = asyncio.get_running_loop() + return signal.SIGTERM in loop._signal_handlers # type: ignore[attr-defined] + + async def test_sigterm_cancels_the_planner_run_and_stops_serving(): runtime = _Runtime() events: list[str] = [] @@ -744,86 +587,91 @@ async def run(): await asyncio.sleep(0) events.append("run finally") - main = asyncio.create_task(planner_main._run_until_sigterm(runtime, serve(), run())) + main = asyncio.create_task( + planner_main._run_until_sigterm(runtime, serve(), run(), _found()) + ) await asyncio.wait_for(running.wait(), timeout=5) - loop = asyncio.get_running_loop() - assert signal.SIGTERM in loop._signal_handlers # type: ignore[attr-defined] + assert _sigterm_installed() os.kill(os.getpid(), signal.SIGTERM) os.kill(os.getpid(), signal.SIGTERM) # a second one changes nothing await asyncio.wait_for(main, timeout=5) assert events == ["run finally", "serve stopped"] assert runtime.shutdowns == 1 - assert signal.SIGTERM not in loop._signal_handlers # type: ignore[attr-defined] + assert not _sigterm_installed() -async def test_without_sigterm_a_run_failure_propagates(): +async def test_sigterm_is_handled_only_once_a_retune_lease_is_found(): + # The clean-up check of a removed live retune runs in the background; + # a planner whose check finds no Lease keeps the default SIGTERM. runtime = _Runtime() + found = asyncio.Event() + running = asyncio.Event() async def serve(): await runtime.stopped.wait() async def run(): - raise RuntimeError("tick failed") + running.set() + await asyncio.sleep(3600) - with pytest.raises(RuntimeError, match="tick failed"): - await planner_main._run_until_sigterm(runtime, serve(), run()) - assert runtime.shutdowns == 0 - loop = asyncio.get_running_loop() - assert signal.SIGTERM not in loop._signal_handlers # type: ignore[attr-defined] - runtime.shutdown() # let the serve task finish + main = asyncio.create_task( + planner_main._run_until_sigterm(runtime, serve(), run(), found) + ) + await asyncio.wait_for(running.wait(), timeout=5) + assert not _sigterm_installed() + found.set() + for _ in range(3): + await asyncio.sleep(0) + assert _sigterm_installed() + os.kill(os.getpid(), signal.SIGTERM) + await asyncio.wait_for(main, timeout=5) + assert runtime.shutdowns == 1 and not _sigterm_installed() -async def test_outer_cancellation_is_not_mistaken_for_sigterm(): +async def test_run_failure_and_outer_cancellation_propagate(): runtime = _Runtime() - started = asyncio.Event() async def serve(): await runtime.stopped.wait() + async def failing(): + raise RuntimeError("tick failed") + + with pytest.raises(RuntimeError, match="tick failed"): + await planner_main._run_until_sigterm(runtime, serve(), failing(), _found()) + assert runtime.shutdowns == 0 and not _sigterm_installed() + + started = asyncio.Event() + async def run(): started.set() await asyncio.sleep(3600) - main = asyncio.create_task(planner_main._run_until_sigterm(runtime, serve(), run())) + # Never found: the waiting installer is cancelled with the call. + main = asyncio.create_task( + planner_main._run_until_sigterm(runtime, serve(), run(), asyncio.Event()) + ) await asyncio.wait_for(started.wait(), timeout=5) main.cancel() with pytest.raises(asyncio.CancelledError): await main assert runtime.shutdowns == 0 + runtime.shutdown() # let the serve tasks finish -async def test_platforms_without_signal_handlers_still_run(monkeypatch, caplog): - runtime = _Runtime() - loop = asyncio.get_running_loop() - - def unsupported(*_args): - raise NotImplementedError - - monkeypatch.setattr(loop, "add_signal_handler", unsupported) - - async def serve(): - return None - - async def run(): - return None - - with caplog.at_level(logging.WARNING): - await planner_main._run_until_sigterm(runtime, serve(), run()) - assert "Cannot install a planner SIGTERM handler" in caplog.text - - -async def test_start_planner_installs_sigterm_only_when_a_retune_lease_may_be_held( +async def test_start_planner_runs_the_sigterm_path_only_when_a_lease_may_be_held( monkeypatch, ): # With live retune, or the clean-up check of a removed one (every # Kubernetes planner without retune config), the planner may hold the # retune leadership Lease. - seen: list[bool] = [] + seen: list[asyncio.Event] = [] class _Planner: def __init__(self, may_hold: bool) -> None: self.may_hold_retune_lease = may_hold + self.retune_lease_found = asyncio.Event() async def _async_init(self): pass @@ -839,8 +687,8 @@ class _StartRuntime: def endpoint(self, _name): return _Endpoint() - async def fake_run_until_sigterm(runtime, serve, run): - seen.append(isinstance(runtime, _StartRuntime)) + async def fake_run_until_sigterm(runtime, serve, run, lease_found): + seen.append(lease_found) await asyncio.gather(serve, run) planners = iter([_Planner(False), _Planner(True)]) @@ -850,20 +698,15 @@ async def fake_run_until_sigterm(runtime, serve, run): await planner_main.start_planner(_StartRuntime(), PlannerConfig(mode="agg")) assert seen == [] await planner_main.start_planner(_StartRuntime(), PlannerConfig(mode="agg")) - assert seen == [True] + assert len(seen) == 1 and not seen[0].is_set() -async def test_may_hold_retune_lease(stub_controller): +async def test_live_retune_finds_its_lease_at_start(stub_controller): planner, _, _ = _planner(inputs=None) assert not planner.may_hold_retune_lease await planner._async_init() try: assert planner.may_hold_retune_lease + assert planner.retune_lease_found.is_set() finally: await planner._shutdown_runtime() - - -def test_hooks_are_frozen(): - hooks = _hooks() - with pytest.raises(dataclasses.FrozenInstanceError): - hooks.phase_s = 1.0 # type: ignore[misc] diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py index d909fb0ba052..a0f52a33e496 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_controller_hooks.py @@ -7,9 +7,9 @@ Covers what a live host adds on top of the replay loop: epoch-phase anchoring, the launch gate, harvest gating and abandoned sweeps, -``drop_pending``/``rewind``, the sweep thread shared across generations, -the DGD spec replicas check, ``restore_desired`` and the structured failure -reason of a decision. +``drop_pending``/``rewind``, the DGD spec replicas check, ``restore_desired`` +and the structured failure reason of a decision. The sweep thread shared +across generations is tested with the engine adapter. """ from __future__ import annotations @@ -25,18 +25,16 @@ reason="AI Simulate is an optional Dynamo simulation dependency", ) -import dynamo.planner.retune.controller as controller_module from dynamo.llm import KvRouterConfig from dynamo.planner.config.planner_config import PlannerConfig, RetuneEngineSpec from dynamo.planner.core.types import ( - RetuneDecision, RetuneInputs, RetuneParams, WorkerCapabilities, WorkerCounts, ) from dynamo.planner.plugins.clock import VirtualClock, WallClock -from dynamo.planner.retune.controller import RetuneController, SweepExecutor +from dynamo.planner.retune.controller import RetuneController from dynamo.planner.retune.live.contracts import NON_DECISION_FAILURES, GateResult from dynamo.planner.retune.sweep import SweepResult @@ -128,16 +126,6 @@ def __call__(self, generation): return self.answers.popleft() if self.answers else self.default -class ManualClock(WallClock): - """A wall clock (no inline await) whose monotonic time tests set.""" - - def __init__(self): - self.mono = 0.0 - - def monotonic(self) -> float: - return self.mono - - def _scored(score_of, *, wall_s=0.5, **extra): def build(request): return SweepResult( @@ -223,42 +211,6 @@ def _window_ms(k): return k * INTERVAL * 1000.0, (k + 1) * INTERVAL * 1000.0 -# --------------------------------------------------------------------------- -# Construction and properties -# --------------------------------------------------------------------------- - - -def test_launch_gate_requires_a_sweep_fn(): - with pytest.raises(ValueError, match="launch_gate requires a sweep_fn"): - RetuneController( - _config(), - _inputs(), - WorkerCapabilities(), - VirtualClock(), - STEP, - launch_gate=Gate(), - ) - - -def test_generation_router_keys_and_domain_are_exposed(): - controller = _live(FakeSweeps(), generation=7) - try: - assert controller.generation == 7 - assert controller.router_keys == ( - "overlap_score_credit", - "prefill_load_scale", - "router_temperature", - ) - assert controller.router_domain == { - key: list(values) for key, values in ROUTER_DOMAIN.items() - } - # A copy: callers cannot change the controller's domain. - controller.router_domain["router_temperature"].append(9.0) - assert controller.router_domain["router_temperature"] == [0.0, 0.5] - finally: - controller.close() - - # --------------------------------------------------------------------------- # Phase anchoring # --------------------------------------------------------------------------- @@ -297,40 +249,6 @@ async def test_phase_anchoring_gives_restart_stable_windows_and_ids(): second.close() -async def test_replay_start_keeps_its_decision_ids(monkeypatch): - sweeps = FakeSweeps() - sweeps.default = _flat() - monkeypatch.setattr(controller_module, "run_sweep", sweeps) - controller = RetuneController( - _config(), _inputs(), WorkerCapabilities(), VirtualClock(), STEP - ) - controller.start(10.0) - try: - await _tick(controller, 75.0) - (decision,) = (await _tick(controller, 80.0)).decisions - finally: - controller.close() - - assert decision.decision_id == "retune-10-w0" - (request,) = sweeps.requests - # Replay requests keep the live-only fields at their defaults. - assert request.router_domain is None - assert request.deadline_s is None - assert request.spec_replicas is None - assert request.replicas_verified is True - assert request.generation == 0 - assert sweeps.aborts == [None] - - -def test_phase_must_be_finite(): - controller = _live(FakeSweeps()) - try: - with pytest.raises(ValueError, match="phase_s must be finite"): - controller.start(T0, float("nan")) - finally: - controller.close() - - # --------------------------------------------------------------------------- # Launch gate # --------------------------------------------------------------------------- @@ -441,26 +359,6 @@ def blocking(request): assert len(sweeps.requests) == 1 -@pytest.mark.parametrize( - "bad", - [RuntimeError("supervisor bug"), GateResult(reason="bogus")], # type: ignore[arg-type] - ids=["raises", "unknown-reason"], -) -async def test_a_failing_gate_holds_the_window(bad): - sweeps = FakeSweeps() - sweeps.default = _flat() - gate = Gate(bad, default=GateResult(spec_replicas=SPEC)) - controller = _live(sweeps, gate=gate) - try: - held = await _tick(controller, L_K) - launched = await _tick(controller, L_K + STEP) - finally: - controller.close() - - assert held.decisions == [] and held.audit_events == [] - assert launched.audit_events == [f"retune:launch w={K}"] - - # --------------------------------------------------------------------------- # Harvest gating and abandoned sweeps # --------------------------------------------------------------------------- @@ -486,24 +384,6 @@ async def test_refused_harvest_drops_the_result_and_relaunches_the_window(): assert [(d.outcome, d.window_index) for d in decided.decisions] == [("kept", K)] -async def test_harvest_allowed_that_raises_counts_as_refused(): - sweeps = FakeSweeps() - sweeps.default = _flat() - - def harvest(_generation): - raise RuntimeError("lease read failed") - - controller = _live(sweeps, harvest=harvest) - try: - await _tick(controller, L_K) - refused = await _tick(controller, L_K + STEP) - finally: - controller.close() - - assert refused.decisions == [] - assert f"retune:dropped w={K}" in refused.audit_events - - @pytest.mark.parametrize("reason", sorted(NON_DECISION_FAILURES)) async def test_non_decision_failures_record_nothing_and_rewind(reason): sweeps = FakeSweeps() @@ -540,69 +420,6 @@ async def test_abandoned_window_is_not_relaunched_once_its_successor_is_due(): assert result.audit_events == [f"retune:dropped w={K}", f"retune:launch w={K + 1}"] -@pytest.mark.parametrize( - "result, outcome", - [ - ( - SweepResult( - status="failed", - error="probe scores differ", - failure_reason="crn_mismatch", - details={"probe": {"expected": 1.0, "got": 1.5}}, - ), - "failed", - ), - ( - SweepResult( - status="timeout", - error="window deadline", - failure_reason="timeout", - details={"requests": ["a", "b"]}, - ), - "timeout", - ), - ( - _flat(failure_reason=None, details={"runtime_image": "ais:0.13"}), - "kept", - ), - ], - ids=["failed", "timeout", "kept"], -) -async def test_failure_reason_and_details_reach_the_decision(result, outcome): - sweeps = FakeSweeps() - sweeps.default = result - controller = _live(sweeps) - try: - await _tick(controller, L_K) - (decision,) = (await _tick(controller, L_K + STEP)).decisions - finally: - controller.close() - - expected = result if isinstance(result, SweepResult) else result(sweeps.requests[0]) - assert decision.outcome == outcome - assert decision.failure_reason == expected.failure_reason - assert decision.details == expected.details - record = decision.to_dict() - assert record["failure_reason"] == expected.failure_reason - assert record["details"] == expected.details - - -async def test_a_raising_sweep_fn_is_a_failed_decision_without_reason(): - def raises(_request, *, cancel, abort): - raise RuntimeError("kube unreachable") - - controller = _live(raises) - try: - await _tick(controller, L_K) - (decision,) = (await _tick(controller, L_K + STEP)).decisions - finally: - controller.close() - - assert decision.outcome == "failed" - assert decision.reason == "RuntimeError: kube unreachable" - assert decision.failure_reason is None - - # --------------------------------------------------------------------------- # drop_pending and rewind # --------------------------------------------------------------------------- @@ -638,9 +455,9 @@ def blocking(request): relaunch = await _tick(controller, L_K + STEP) release.set() dropped.result(timeout=30) + # The relaunch queued behind the dropped sweep; it is decided once + # its own sweep is done. controller._pending.future.result(timeout=30) - # The relaunch queued behind the dropped sweep, so it launched (and - # becomes ready) a little after L_K + STEP. decided = await _tick(controller, L_K + 3 * STEP) finally: release.set() @@ -652,181 +469,6 @@ def blocking(request): assert [(d.outcome, d.window_index) for d in decided.decisions] == [("kept", K)] -async def test_drop_pending_cancels_a_sweep_that_has_not_started(): - release = threading.Event() - sweeps = FakeSweeps() - - def blocking(request): - release.wait(30) - return _flat()(request) - - sweeps.default = blocking - controller = _live(sweeps, clock=WallClock()) - try: - # Occupy the sweep thread so the next submission stays queued. - blocker = controller._executor.submit(release.wait, 30) - await _tick(controller, L_K) - queued = controller._pending.future - controller.drop_pending() - assert queued.cancelled() - release.set() - blocker.result(timeout=30) - finally: - release.set() - controller.close() - - assert sweeps.requests == [] - - -def test_drop_pending_and_rewind_without_a_sweep(): - controller = _live(FakeSweeps()) - try: - controller.drop_pending() - controller._last_k = K - controller.rewind(K + 3) - assert controller._last_k == K - controller.rewind(K - 2) - assert controller._last_k == K - 2 - finally: - controller.close() - - -async def test_a_sweep_queued_behind_a_dropped_one_launches_when_it_starts(): - release = threading.Event() - sweeps = FakeSweeps() - - def blocking(request): - release.wait(30) - return _flat()(request) - - sweeps.results.append(blocking) - sweeps.default = _flat() - clock = ManualClock() - controller = _live(sweeps, clock=clock) - try: - clock.mono = 100.0 - await _tick(controller, L_K) - dropped = controller._pending.future - controller.drop_pending() - controller.rewind(K - 1) - await _tick(controller, L_K + STEP) # submitted at monotonic 100 - clock.mono = 107.0 # the dropped sweep returns 7 s later - release.set() - dropped.result(timeout=30) - controller._pending.future.result(timeout=30) - (decision,) = (await _tick(controller, L_K + 4 * STEP)).decisions - finally: - release.set() - controller.close() - - assert decision.launched_at_s == L_K + STEP + 7.0 - - -async def test_repeated_drops_keep_tracking_the_sweep_holding_the_thread(): - release = threading.Event() - sweeps = FakeSweeps() - - def blocking(request): - release.wait(30) - return _flat()(request) - - sweeps.results.append(blocking) - sweeps.default = _flat() - clock = ManualClock() - controller = _live(sweeps, clock=clock) - try: - clock.mono = 100.0 - await _tick(controller, L_K) - running = controller._pending.future - controller.drop_pending() - controller.rewind(K - 1) - await _tick(controller, L_K + STEP) # queued behind the running one - queued = controller._pending.future - controller.drop_pending() - assert queued.cancelled() - controller.rewind(K - 1) - await _tick(controller, L_K + 2 * STEP) # still behind the first one - clock.mono = 104.0 - release.set() - running.result(timeout=30) - controller._pending.future.result(timeout=30) - (decision,) = (await _tick(controller, L_K + 5 * STEP)).decisions - finally: - release.set() - controller.close() - - # The cancelled sweep never ran; the last one started when the first ended. - assert len(sweeps.requests) == 2 - assert decision.launched_at_s == L_K + 2 * STEP + 4.0 - - -# --------------------------------------------------------------------------- -# The sweep thread shared across generations -# --------------------------------------------------------------------------- - - -async def test_generations_sharing_the_sweep_thread_never_sweep_at_once(): - # Generation 0 is slow to see its abort while generation 2 relaunches the - # same window; generation 1 was replaced before it launched. All three - # submit to the host's one sweep thread, which close() leaves running. - lock = threading.Lock() - running: list[int] = [] - overlaps: list[list[int]] = [] - order: list[tuple[int, int]] = [] - release = threading.Event() - started = threading.Event() - - def sweep(request, *, cancel, abort): - with lock: - running.append(request.generation) - order.append((request.generation, request.window_index)) - if len(running) > 1: - overlaps.append(list(running)) - try: - if request.generation == 0: - started.set() - release.wait(30) - return SweepResult(status="failed", failure_reason="lost_leadership") - return _flat()(request) - finally: - with lock: - running.remove(request.generation) - - executor = SweepExecutor() - clock = ManualClock() - controllers = [] - try: - old = _live(sweep, clock=clock, generation=0, executor=executor) - controllers.append(old) - clock.mono = 100.0 - await _tick(old, L_K) - assert started.wait(30) - old.drop_pending() - old.close() - middle = _live(sweep, clock=clock, generation=1, executor=executor) - controllers.append(middle) - middle.close() - new = _live(sweep, clock=clock, generation=2, executor=executor) - controllers.append(new) - launched = await _tick(new, L_K + STEP) - assert launched.audit_events == [f"retune:launch w={K}"] - relaunch = new._pending.future - assert not relaunch.done() - clock.mono = 105.0 # generation 0 returns 5 s later - release.set() - relaunch.result(timeout=30) - (decision,) = (await _tick(new, L_K + 4 * STEP)).decisions - finally: - release.set() - for controller in controllers: - controller.close() - executor.shutdown(wait=True) - - assert overlaps == [] - assert order == [(0, K), (2, K)] - assert decision.launched_at_s == L_K + STEP + 5.0 - - # --------------------------------------------------------------------------- # DGD spec replicas # --------------------------------------------------------------------------- @@ -851,24 +493,6 @@ def _requests_for_k(sweeps): return [request for request in sweeps.requests if request.window_index == K] -async def test_replicas_verified_when_the_counts_cover_the_window(): - sweeps = FakeSweeps() - sweeps.default = _flat() - start = T0 - INTERVAL - 2 * STEP - controller = _live(sweeps, start=start) - try: - decisions = await _run_window( - controller, lambda _now: 2, start=start, until=L_K - ) - finally: - controller.close() - - assert decisions == [] - (request,) = _requests_for_k(sweeps) - assert request.replicas_verified is True - assert request.replicas == request.spec_replicas == SPEC - - async def test_a_ready_count_change_in_the_window_fails_it(): sweeps = FakeSweeps() sweeps.default = _flat() @@ -947,24 +571,6 @@ async def test_window_started_before_the_controller_launches_unverified(): assert request.replicas == SPEC -async def test_unverified_window_still_fails_on_an_observed_change(): - sweeps = FakeSweeps() - sweeps.default = _flat() - start = T0 - INTERVAL / 2 - controller = _live(sweeps, start=start) - try: - decisions = await _run_window( - controller, lambda _now: 3, start=start, until=L_K - ) - finally: - controller.close() - - assert [(d.outcome, d.failure_reason) for _, d in decisions] == [ - ("failed", "replicas_changed") - ] - assert _requests_for_k(sweeps) == [] - - # --------------------------------------------------------------------------- # restore_desired # --------------------------------------------------------------------------- @@ -991,85 +597,3 @@ async def test_restore_desired_sets_the_deployed_point_but_not_startup(): # The restored point is the baseline; startup stays a candidate. assert request.baseline == BEST assert STARTUP in request.candidates - - -@pytest.mark.parametrize( - "flat, match", - [ - ( - {"overlap_score_credit": 0.0, "prefill_load_scale": 1.0}, - "router_temperature", - ), - ({**BEST.router, "router_temperature": -1.0}, "non-negative"), - ({**BEST.router, "router_temperature": float("inf")}, "finite"), - ], - ids=["missing-key", "negative", "infinite"], -) -def test_restore_desired_rejects_bad_values_and_keeps_the_desired_point(flat, match): - controller = _live(FakeSweeps()) - try: - with pytest.raises(ValueError, match=match): - controller.restore_desired(flat) - assert controller.current_point() == STARTUP - finally: - controller.close() - - -# --------------------------------------------------------------------------- -# Decision record -# --------------------------------------------------------------------------- - - -def test_decision_record_carries_failure_reason_and_a_private_details_copy(): - details = {"shards": [{"id": "g0", "state": "Completed"}]} - decision = RetuneDecision( - decision_id="retune-w1800000000000", - outcome="failed", - window_index=K, - window_start_s=T0 - INTERVAL, - window_end_s=T0, - as_of_s=L_K, - launched_at_s=L_K, - decided_at_s=L_K + STEP, - num_requests=0, - num_candidates=2, - previous=STARTUP, - chosen=STARTUP, - baseline_score=None, - best_score=None, - reason="probe scores differ", - failure_reason="crn_mismatch", - details=details, - ) - # A deep copy: neither the details nor a value nested in them is shared. - details["shards"][0]["state"] = "Failed" - details["shards"] = [] - - record = decision.to_dict() - assert record["failure_reason"] == "crn_mismatch" - assert record["details"] == {"shards": [{"id": "g0", "state": "Completed"}]} - json.dumps(record) - with pytest.raises(ValueError, match="failure_reason"): - RetuneDecision(**{**decision.__dict__, "failure_reason": ""}) - - -def test_replay_decision_record_has_empty_failure_fields(): - decision = RetuneDecision( - decision_id="retune-0-w0", - outcome="kept", - window_index=0, - window_start_s=0.0, - window_end_s=60.0, - as_of_s=65.0, - launched_at_s=65.0, - decided_at_s=70.0, - num_requests=1, - num_candidates=1, - previous=STARTUP, - chosen=STARTUP, - baseline_score=1.0, - best_score=1.0, - ) - - record = decision.to_dict() - assert (record["failure_reason"], record["details"]) == (None, {}) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py index 73f074f39084..c85fb01dba1b 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_kube.py @@ -285,10 +285,18 @@ def _pod( {"type": "PodScheduled", "status": "True"}, {"type": "Ready", "status": "True" if ready else "False"}, ], + "containerStatuses": [ + {"name": "sidecar", "restartCount": 7, **_CONTAINER_STATUS}, + {"name": "main", "restartCount": 2, **_CONTAINER_STATUS}, + ], }, } +# The fields V1ContainerStatus requires besides name and restartCount. +_CONTAINER_STATUS = {"image": "img", "imageID": "", "ready": True} + + def test_list_pods_builds_pod_info(kube): retune_kube, transport = kube transport.reply( @@ -325,6 +333,7 @@ def test_list_pods_builds_pod_info(kube): terminating=False, labels={"nvidia.com/dynamo-component-type": "frontend"}, http_port=8000, # the main container's port wins + restarts=2, # the main container's ) assert (pods[1].ready, pods[1].terminating, pods[1].ip) == (False, True, None) assert pods[2].http_port == 8080 @@ -385,8 +394,14 @@ def test_leases_round_trip_raw_json(kube): transport.reply({"kind": "Status", "status": "Success"}) retune_kube.delete_lease("l2") assert transport.calls[-1]["method"] == "DELETE" + assert transport.calls[-1]["body"] is None transport.reply(api_error(404)) retune_kube.delete_lease("l2") + # Conditional on a resourceVersion: a changed Lease is a conflict. + transport.reply(api_error(409, "Precondition failed: ResourceVersion")) + with pytest.raises(KubeConflict): + retune_kube.delete_lease("l2", resource_version="5") + assert transport.calls[-1]["body"]["preconditions"] == {"resourceVersion": "5"} def test_from_kube_api_shares_the_connector_client(): diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py b/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py index 4c4d8ccba13f..9d08e1c81a80 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_metrics.py @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""``dynamo_planner_retune_*`` metrics: names, labels and values as the live -retune guide lists them, collectors shared per registry, and methods that -never raise.""" +"""``dynamo_planner_retune_*`` metrics: the names the live retune guide +lists, the series that start at zero, the ``trace_sink_missing`` series a +recovered pod drops, and collectors shared per registry.""" from __future__ import annotations @@ -13,7 +13,6 @@ from prometheus_client import REGISTRY, CollectorRegistry, generate_latest from dynamo.planner.core.types import RETUNE_OUTCOMES -from dynamo.planner.retune.live.contracts import NullMetrics, RetuneMetricsSink from dynamo.planner.retune.live.metrics import PREFIX, RetuneMetrics pytestmark = [ @@ -33,42 +32,30 @@ def _value(registry: CollectorRegistry, name: str, **labels: str) -> float | Non return registry.get_sample_value(f"{PREFIX}_{name}", labels) -def test_metrics_implement_the_sink_protocol(registry): - metrics: RetuneMetricsSink = RetuneMetrics(registry) - null: RetuneMetricsSink = NullMetrics() +def test_exposition_uses_the_documented_names(registry): + metrics = RetuneMetrics(registry) + metrics.trace_sink_missing("frontend-0", True) + text = generate_latest(registry).decode() + for name in ( - "decision", - "failure", - "skipped", - "supervisor_state", - "scores", + "decisions_total", + "failures_total", + "skipped_total", + "leader", + "ready", + "initialized", + "rollback_pending", + "controller_generation", + "baseline_score", + "best_score", + "sweep_seconds", + "missing_candidates", "router_converged", - "patch_error", + "router_patch_errors_total", "trace_sink_missing", - "dropped_members", + "trace_dropped_members_total", ): - assert callable(getattr(metrics, name)) - assert callable(getattr(null, name)) - - -def test_counters_count_by_label(registry): - metrics = RetuneMetrics(registry) - - metrics.decision("applied") - metrics.decision("applied") - metrics.decision("kept") - metrics.failure("crn_mismatch") - metrics.skipped("not_leader") - metrics.patch_error("409") - metrics.patch_error("timeout") - metrics.patch_error("409") - - assert _value(registry, "decisions_total", outcome="applied") == 2 - assert _value(registry, "decisions_total", outcome="kept") == 1 - assert _value(registry, "failures_total", reason="crn_mismatch") == 1 - assert _value(registry, "skipped_total", reason="not_leader") == 1 - assert _value(registry, "router_patch_errors_total", code="409") == 2 - assert _value(registry, "router_patch_errors_total", code="timeout") == 1 + assert f"\n# HELP {PREFIX}_{name} " in f"\n{text}", name def test_known_outcomes_and_skip_reasons_start_at_zero(registry): @@ -84,195 +71,34 @@ def test_known_outcomes_and_skip_reasons_start_at_zero(registry): assert value is not None and math.isnan(value) -def test_supervisor_state_gauges(registry): - metrics = RetuneMetrics(registry) - - metrics.supervisor_state( - leader=True, - ready=False, - initialized=True, - rollback_pending=True, - generation=3, - ) - assert _value(registry, "leader") == 1 - assert _value(registry, "ready") == 0 - assert _value(registry, "initialized") == 1 - assert _value(registry, "rollback_pending") == 1 - assert _value(registry, "controller_generation") == 3 - - metrics.supervisor_state( - leader=False, - ready=True, - initialized=False, - rollback_pending=False, - generation=4, - ) - assert _value(registry, "leader") == 0 - assert _value(registry, "ready") == 1 - assert _value(registry, "initialized") == 0 - assert _value(registry, "rollback_pending") == 0 - assert _value(registry, "controller_generation") == 4 - - -def test_scores_set_nan_for_unknown_values(registry): - metrics = RetuneMetrics(registry) - - metrics.scores(baseline=1.5, best=2.25, sweep_seconds=3600.0, missing_candidates=2) - assert _value(registry, "baseline_score") == 1.5 - assert _value(registry, "best_score") == 2.25 - assert _value(registry, "sweep_seconds") == 3600.0 - assert _value(registry, "missing_candidates") == 2 - - metrics.scores(baseline=None, best=None, sweep_seconds=None, missing_candidates=0) - for name in ("baseline_score", "best_score", "sweep_seconds"): - value = _value(registry, name) - assert value is not None and math.isnan(value) - assert _value(registry, "missing_candidates") == 0 - - -def test_router_converged_gauge(registry): - metrics = RetuneMetrics(registry) - - metrics.router_converged(True) - assert _value(registry, "router_converged") == 1 - metrics.router_converged(False) - assert _value(registry, "router_converged") == 0 - - def test_trace_sink_missing_drops_the_series_once_the_sink_is_back(registry): metrics = RetuneMetrics(registry) metrics.trace_sink_missing("frontend-0", True) metrics.trace_sink_missing("frontend-1", True) - assert _value(registry, "trace_sink_missing", pod="frontend-0") == 1 - assert _value(registry, "trace_sink_missing", pod="frontend-1") == 1 - metrics.trace_sink_missing("frontend-0", False) assert _value(registry, "trace_sink_missing", pod="frontend-0") is None assert _value(registry, "trace_sink_missing", pod="frontend-1") == 1 # A pod that never had a series is not an error. metrics.trace_sink_missing("frontend-9", False) - assert _value(registry, "trace_sink_missing", pod="frontend-9") is None - - -def test_dropped_members_counts_only_positive_counts(registry): - metrics = RetuneMetrics(registry) - - metrics.dropped_members(3) - metrics.dropped_members(0) - metrics.dropped_members(-1) - metrics.dropped_members(2) - assert _value(registry, "trace_dropped_members_total") == 5 -def test_exposition_uses_the_design_names(registry): - metrics = RetuneMetrics(registry) - metrics.trace_sink_missing("frontend-0", True) - text = generate_latest(registry).decode() - - for name in ( - "decisions_total", - "failures_total", - "skipped_total", - "leader", - "ready", - "initialized", - "rollback_pending", - "controller_generation", - "baseline_score", - "best_score", - "sweep_seconds", - "missing_candidates", - "router_converged", - "router_patch_errors_total", - "trace_sink_missing", - "trace_dropped_members_total", - ): - assert f"\n# HELP {PREFIX}_{name} " in f"\n{text}", name - - -def test_instances_on_one_registry_share_collectors(registry): +def test_instances_share_the_collectors_of_their_registry(registry): first = RetuneMetrics(registry) second = RetuneMetrics(registry) - first.decision("applied") second.decision("applied") assert _value(registry, "decisions_total", outcome="applied") == 2 - -def test_default_registry_tolerates_reinstantiation(): # The planner serves prometheus_client.REGISTRY; a controller rebuild or # a second test must not hit "Duplicated timeseries". - first = RetuneMetrics() - second = RetuneMetrics() before = ( REGISTRY.get_sample_value(f"{PREFIX}_failures_total", {"reason": "probe"}) or 0.0 ) - first.failure("probe") - second.failure("probe") + RetuneMetrics().failure("probe") + RetuneMetrics().failure("probe") assert ( REGISTRY.get_sample_value(f"{PREFIX}_failures_total", {"reason": "probe"}) == before + 2 ) - - -def test_empty_labels_read_unknown(registry): - metrics = RetuneMetrics(registry) - - metrics.failure("") - metrics.patch_error("") - assert _value(registry, "failures_total", reason="unknown") == 1 - assert _value(registry, "router_patch_errors_total", code="unknown") == 1 - - -class _Broken: - def labels(self, *args, **kwargs): - raise RuntimeError("broken") - - def set(self, *args, **kwargs): - raise RuntimeError("broken") - - def inc(self, *args, **kwargs): - raise RuntimeError("broken") - - def remove(self, *args, **kwargs): - raise RuntimeError("broken") - - -def test_methods_never_raise(registry, monkeypatch): - metrics = RetuneMetrics(registry) - for name in vars(metrics._m): - monkeypatch.setattr(metrics._m, name, _Broken()) - - metrics.decision("applied") - metrics.failure("k8s_error") - metrics.skipped("not_ready") - metrics.supervisor_state( - leader=True, ready=True, initialized=True, rollback_pending=False, generation=1 - ) - metrics.scores(baseline=1.0, best=2.0, sweep_seconds=3.0, missing_candidates=0) - metrics.router_converged(True) - metrics.patch_error("503") - metrics.trace_sink_missing("frontend-0", True) - metrics.trace_sink_missing("frontend-0", False) - metrics.dropped_members(1) - - -def test_bad_values_do_not_raise(registry): - metrics = RetuneMetrics(registry) - - metrics.scores( - baseline="not a number", # type: ignore[arg-type] - best=None, - sweep_seconds=None, - missing_candidates=0, - ) - metrics.dropped_members(None) # type: ignore[arg-type] - metrics.supervisor_state( - leader=True, - ready=True, - initialized=True, - rollback_pending=False, - generation="x", # type: ignore[arg-type] - ) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py index 2ca376365249..642f58d3343a 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py @@ -242,6 +242,7 @@ def test_state_parsing_edge_cases() -> None: _body(dgd_uid="u", last_window_end_ms=math.inf), _body(dgd_uid="u", updated_ms="x"), _body(dgd_uid="u", updated_ms=[1]), + _body(dgd_uid="u", fingerprint=math.nan), ): with pytest.raises(StateFormatError): RetuneState.from_annotations({STATE_ANNOTATION: bad}) @@ -252,6 +253,27 @@ def test_state_parsing_edge_cases() -> None: assert parsed is not None and parsed.history == () +def test_unwritable_history_entries_are_dropped_on_read() -> None: + # json.loads accepts NaN and Infinity, canonical_json does not: one such + # entry kept would make every later state write raise. + history = [ + {"decision_id": "ok", "best": 1.5}, + {"decision_id": "nan", "best": math.nan}, + {"decision_id": "inf", "by_wt": {"aggregated": {"t": math.inf}}}, + "not an object", + ] + parsed = RetuneState.from_annotations( + { + STATE_ANNOTATION: _body(dgd_uid="u"), + HISTORY_ANNOTATION: json.dumps(history), + } + ) + assert parsed is not None + assert [entry["decision_id"] for entry in parsed.history] == ["ok"] + written = append_history(parsed, {"decision_id": "next"}).to_annotations() + assert json.loads(written[HISTORY_ANNOTATION])[0] == history[0] + + def test_another_format_version_is_rejected_not_corrupt() -> None: # Checked first: a newer planner's state is never judged by this # planner's field rules (which would make it corrupt, then overwritten). @@ -421,7 +443,10 @@ def test_store_creates_reads_and_conditionally_replaces() -> None: assert store.read().state.leader_term == "t2" assert updated.resource_version != read.resource_version - store.delete() + # The delete is conditional too: a stale snapshot leaves the Lease. + with pytest.raises(KubeConflict): + store.delete(read) + store.delete(store.read()) assert store.read().lease is None @@ -479,6 +504,8 @@ def test_leadership_create_renew_and_expire_locally() -> None: clock.advance(20) assert a.step() == "renewed" assert a.leader_until == clock.mono + 45.0 + # What a conditional delete of the Lease needs. + assert a.resource_version == kube.leases[a.name]["metadata"]["resourceVersion"] # Renewal failures keep leadership only until leader_until. kube.fail("get_lease", times=10) clock.advance(20) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py index d298e021faee..80b7541d4418 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -3,17 +3,15 @@ # ruff: noqa: E402 # Optional-dependency preflight must run before the supervisor imports the sweep. -"""Unit tests for ``LiveRetuneSupervisor`` and its bootstrap, plus a seeded -interleaving harness that checks its invariants: at most one leader writes, -a decision is persisted before any router changes, no window is wasted, and -the routers settle on the persisted state. - -The supervisor drives the real engine adapter and ``RetuneController``; their -sweeps run on the adapter's sweep thread, in ``StubSweeps``, until the test -finishes them. Kubernetes is ``FakeRetuneKube`` (shared Leases), the routers -are ``FakeFrontend`` (PATCH merges and bumps the revision, reset restores -the initial) behind ``FakeActuator``, and clocks are advanced by hand. -Kubernetes calls run inline. +"""Unit tests for ``LiveRetuneSupervisor`` and its bootstrap. + +The supervisor drives the real engine adapter, ``RetuneController`` and +``RouterActuator``; sweeps run on the adapter's sweep thread, in +``StubSweeps``, until the test finishes them. Kubernetes is +``FakeRetuneKube`` (shared Leases), the routers are ``FakeFrontend`` (PATCH +merges and bumps the revision, reset restores the initial) behind +``FrontendPolicyClient``, and clocks are advanced by hand. Kubernetes calls +run inline. """ from __future__ import annotations @@ -25,9 +23,7 @@ import json import logging import math -import random import threading -from collections import deque from types import SimpleNamespace from typing import Any, Optional @@ -244,20 +240,26 @@ class Router: manual: bool = False +FRONTEND_POD = "frontend-0" + + class FakeFrontend: - """Routers with the runtime.rs semantics: PATCH merges and bumps the - revision, reset restores the initial and bumps the revision.""" + """The DGD's frontend pod and its routers, with the runtime.rs + semantics: PATCH merges and bumps the revision, reset restores the + initial and bumps the revision. A restart is a new frontend process in + the same pod (a container restart): new routers at revision 0.""" - def __init__(self, maps: dict[str, dict[str, float]]) -> None: + def __init__(self, kube: FakeRetuneKube, maps: dict[str, dict[str, float]]): self._ids = itertools.count() + self.kube = kube self.down = False - # (writer, advisory, kind, target mode, router_id) - self.writes: list[tuple[str, bool, str, str, str]] = [] - self.manual_patches = 0 + # (writer, advisory, kind, router_id) + self.writes: list[tuple[str, bool, str, str]] = [] self.routers: dict[str, Router] = {} - self.restart(maps) + kube.add_pod(FRONTEND_POD, dgd=DGD_NAME) + self._start(maps) - def restart(self, maps: dict[str, dict[str, float]]) -> None: + def _start(self, maps: dict[str, dict[str, float]]) -> None: self.routers = {} for worker_type, params in maps.items(): router_id = f"r{next(self._ids)}" @@ -265,123 +267,79 @@ def restart(self, maps: dict[str, dict[str, float]]) -> None: router_id, worker_type, 0, dict(params), dict(params) ) - def patch(self, router: Router, params, writer, advisory, mode) -> None: - router.config.update(params) - router.revision += 1 - router.manual = False - self.writes.append((writer, advisory, "patch", mode, router.router_id)) + def restart(self, maps: dict[str, dict[str, float]]) -> None: + self.kube.restart_pod(FRONTEND_POD) + self._start(maps) - def reset(self, router: Router, writer, advisory, mode) -> None: - router.config = dict(router.initial) + def write(self, router: Router, kind: str, params, writer, advisory) -> None: + if kind == "reset": + router.config = dict(router.initial) + else: + router.config.update(params) router.revision += 1 - router.manual = False - self.writes.append((writer, advisory, "reset", mode, router.router_id)) + self.writes.append((writer, advisory, kind, router.router_id)) def manual_patch(self, key: str, value: float) -> None: - self.manual_patches += 1 for router in self.routers.values(): router.config[key] = value router.revision += 1 - router.manual = True -class FakeActuator: - """The router actuator's rules over a ``FakeFrontend``: latch, apply, - rollback, observe and parity. Like ``RouterActuator`` it PATCHes a router - only once the persisted ``initials`` hold its initial.""" +class FrontendPolicyClient: + """``RouterPolicyClient`` of the real ``RouterActuator`` over the + world's ``FakeFrontend``: revision compare-and-swap, then the write. + ``writer`` and ``advisory`` label the writes it makes.""" - def __init__( - self, frontend: FakeFrontend, owner: str, advisory: bool, maps - ) -> None: - self.frontend = frontend - self.owner = owner + def __init__(self, world: World, writer: str, advisory: bool) -> None: + self.world = world + self.writer = writer self.advisory = advisory - self.startup_maps = {wt: dict(p) for wt, p in maps.items()} - self.latched: dict[str, dict[str, float]] = {} - self.targets: list[ActuationTarget] = [] - self.last_pods = () - def set_startup_maps(self, maps) -> None: - self.startup_maps = {wt: dict(p) for wt, p in maps.items()} + def _info(self, router: Router) -> RouterInfo: + return RouterInfo( + router_id=router.router_id, + namespace=self.world.spec.dynamo_namespace, + worker_type=router.worker_type, + revision=router.revision, + configuration=dict(router.config), + model="model", + ) - async def reconcile(self, target, *, initials, expected_frontends): - self.targets.append(target) - frontend = self.frontend + async def get(self, pod) -> PolicyCallResult: + frontend = self.world.frontend if frontend.down: - return ActuationReport( - converged=False, - seen_router_ids=frozenset(initials) | frozenset(self.latched), - errors={"list_pods": 1}, - ) - routers = list(frontend.routers.values()) - - def initial(router: Router) -> Optional[dict[str, float]]: - return self.latched.get(router.router_id) or initials.get(router.router_id) - - for router in routers: - if router.revision == 0 and initial(router) is None: - self.latched[router.router_id] = dict(router.config) - for router in routers: - known = initial(router) - if target.mode == "rollback": - if known is None and not target.learn_unknown: - continue # advisory: never written, never waited for - if known is None or router.config != known: - frontend.reset(router, self.owner, self.advisory, target.mode) - self.latched[router.router_id] = dict(router.config) - elif target.mode == "apply": - if known is None: - frontend.reset(router, self.owner, self.advisory, target.mode) - self.latched[router.router_id] = dict(router.config) - known = router.config - if initials.get(router.router_id) != known: - continue # held until the state records the initial - if known != self.startup_maps.get(router.worker_type): - continue - desired = target.desired_by_wt[router.worker_type] - diff = {k: v for k, v in desired.items() if router.config.get(k) != v} - if diff: - frontend.patch(router, diff, self.owner, self.advisory, target.mode) - - def expected(router: Router): - if target.mode == "apply": - return target.desired_by_wt[router.worker_type] - return initial(router) - - converged = bool(routers) and all( - expected(router) is not None and router.config == expected(router) - for router in routers - if not ( - target.mode == "rollback" - and not target.learn_unknown - and initial(router) is None - ) - ) - known = [router for router in routers if initial(router) is not None] - mismatched = [ - router - for router in known - if initial(router) != self.startup_maps.get(router.worker_type) - ] - if mismatched: - parity: Optional[bool] = False - elif known and len(known) == len(routers): - parity = True - else: - parity = None - seen = frozenset(router.router_id for router in routers) - self.latched = {rid: cfg for rid, cfg in self.latched.items() if rid in seen} - new_initials = { - rid: dict(cfg) - for rid, cfg in self.latched.items() - if initials.get(rid) != cfg - } - return ActuationReport( - converged=converged, - new_initials=new_initials, - seen_router_ids=seen, - parity_ok=parity, - ) + return PolicyCallResult(status=0, error="connection refused") + routers = tuple(self._info(r) for r in frontend.routers.values()) + return PolicyCallResult(status=200, routers=routers) + + async def patch(self, pod, router_id, expected_revision, parameters): + return self._write(router_id, expected_revision, "patch", dict(parameters)) + + async def reset(self, pod, router_id, expected_revision): + return self._write(router_id, expected_revision, "reset", None) + + def _write(self, router_id, expected_revision, kind, params) -> PolicyCallResult: + frontend = self.world.frontend + router = frontend.routers.get(router_id) + if router is None: + return PolicyCallResult(status=404, error="router instance not found") + if router.revision != expected_revision: + return PolicyCallResult(status=409, error="policy revision conflict") + frontend.write(router, kind, params, self.writer, self.advisory) + return PolicyCallResult(status=200, router=self._info(router)) + + +def real_actuators(world: World, writer: str, advisory: bool = False): + """The ``actuator_factory`` of a process: ``RouterActuator`` over the + world's frontend.""" + + return lambda dgd, maps: RouterActuator( + world.kube, + FrontendPolicyClient(world, writer, advisory), + dgd, + maps, + NullMetrics(), + ) class StubSweeps: @@ -695,7 +653,7 @@ def __init__(self, kube: Optional[FakeRetuneKube] = None) -> None: self.clock = Clock() self.kube = kube or FakeRetuneKube(now=self.clock.time) self.spec = Spec() - self.frontend = FakeFrontend(VARIANTS["a"]) + self.frontend = FakeFrontend(self.kube, VARIANTS["a"]) self.sweeps = StubSweeps() self.hosts: list[FakeHost] = [] self.kube.put_dgd( @@ -778,8 +736,7 @@ def make_process( evaluation=evaluation, identity=f"{name}/{len(world.kube.calls)}", premise_evaluator=world.spec.evaluation, - actuator_factory=actuator_factory - or (lambda dgd, maps: FakeActuator(world.frontend, name, advisory, maps)), + actuator_factory=actuator_factory or real_actuators(world, name, advisory), fingerprint_fn=lambda startup: f"fp-{None if startup is None else startup.fingerprint_source}", resolver_factory=lambda startup: bootstrap.KvRouterResolver( startup.router_config, ("aggregated",) @@ -1710,7 +1667,7 @@ async def test_an_actuator_round_overtaken_by_a_local_loss_is_discarded() -> Non report_before = sup.last_report rv = world.kube.leases[STATE_NAME]["metadata"]["resourceVersion"] - class OvertakenActuator(FakeActuator): + class OvertakenActuator(RouterActuator): async def reconcile(self, target, *, initials, expected_frontends): sup._epoch += 1 # a local loss ran while this round awaited I/O return ActuationReport( @@ -1720,7 +1677,13 @@ async def reconcile(self, target, *, initials, expected_frontends): parity_ok=False, ) - sup._actuator = OvertakenActuator(world.frontend, "p1", False, VARIANTS["a"]) + sup._actuator = OvertakenActuator( + world.kube, + FrontendPolicyClient(world, "p1", False), + sup.dgd, + VARIANTS["a"], + NullMetrics(), + ) await sup.control_round() assert sup.last_report is report_before and sup._p14_ok is not False assert world.kube.leases[STATE_NAME]["metadata"]["resourceVersion"] == rv @@ -2011,9 +1974,7 @@ def decommission_supervisor(world: World) -> LiveRetuneSupervisor: dgd=world.kube.get_dgd(DGD_NAME), dynamo_namespace="dyn", metrics=NullMetrics(), - actuator_factory=lambda dgd, maps: FakeActuator( - world.frontend, "gc", True, maps - ), + actuator_factory=real_actuators(world, "gc", advisory=True), frontend=frontend, clock=world.clock.time, monotonic=world.clock.monotonic, @@ -2049,8 +2010,9 @@ async def test_decommission_rolls_back_then_deletes_everything() -> None: await sup.stop() -async def test_decommission_never_deletes_after_losing_its_term() -> None: - world = World() +async def decommissioning(world: World) -> LiveRetuneSupervisor: + """An initialized clean-up supervisor with nothing to roll back.""" + first = await leading(world, advisory=True) # nothing deployed await first.sup.stop() world.kube.create_request( @@ -2060,6 +2022,12 @@ async def test_decommission_never_deletes_after_losing_its_term() -> None: await sup.start(SimpleNamespace(), background=False) await sup.lease_round() assert sup.initialized and not world.shared_state().rollback_pending + return sup + + +async def test_decommission_never_deletes_after_losing_its_term() -> None: + world = World() + sup = await decommissioning(world) # Another replica claims the state; this round's initials write (a # restarted frontend) finds out before the clean-up would delete. store = StateStore(world.kube, sup.dgd) @@ -2073,6 +2041,29 @@ async def test_decommission_never_deletes_after_losing_its_term() -> None: await sup.stop() +async def test_decommission_deletes_only_the_leases_it_last_saw() -> None: + # The local lease clock still says leader and no write of the round + # notices the other term: only the delete preconditions stop it. + world = World() + sup = await decommissioning(world) + store = StateStore(world.kube, sup.dgd) + snapshot = store.read() + store.write(dataclasses.replace(snapshot.state, leader_term="9/x/1"), snapshot) + await sup.control_round() + assert not sup.decommissioned and not sup.initialized + assert set(world.kube.leases) == {STATE_NAME, LEADER_NAME} + assert world.shared_state().leader_term == "9/x/1" + await sup.stop() + + world = World() + sup = await decommissioning(world) + steal_lease(world, "intruder/0") + await sup.control_round() + assert sup.decommissioned and set(world.kube.leases) == {LEADER_NAME} + assert world.holder() == "intruder/0" + await sup.stop() + + # -------------------------------------------------------------- bootstrap @@ -2223,9 +2214,7 @@ async def test_build_supervisor_wires_the_collaborators() -> None: metrics=NullMetrics(), identity="pod/1", frontend=frontend, - actuator_factory=lambda dgd, maps: FakeActuator( - world.frontend, "x", False, maps - ), + actuator_factory=real_actuators(world, "x"), reconciler=reconciler, canary=lambda s: True, ) @@ -2346,8 +2335,9 @@ def forbidden() -> KubeError: def planner_without_retune(monkeypatch, controller: Any, *, leases=(), build=None): """A ``NativePlannerBase`` without retune config; the engine bootstrap - is a no-op. ``leases`` is what the Lease check finds (or raises) and - ``build(host, kube)`` builds the clean-up.""" + is a no-op. ``leases`` is what the Lease check finds (or raises), a list + of those for consecutive checks, and ``build(host, kube)`` builds the + clean-up.""" from unittest.mock import patch @@ -2359,12 +2349,15 @@ def planner_without_retune(monkeypatch, controller: Any, *, leases=(), build=Non environment = PlannerEnvironment(controller) kube = object() + outcomes = list(leases) if isinstance(leases, list) else None + def find_retune_leases(found_kube, dgd_name, *, timeout_s): assert found_kube is kube and dgd_name == DGD_NAME environment.calls.append("leases.check") - if isinstance(leases, Exception): - raise leases - return tuple(leases) + found = leases if outcomes is None else outcomes.pop(0) + if isinstance(found, Exception): + raise found + return tuple(found) async def build_legacy_cleanup(host, found_kube, *, timeout_s): assert found_kube is kube @@ -2420,7 +2413,10 @@ async def build(host): # Startup never waits for the check. assert environment.calls == ["environment.initialize", "engine"] assert planner.may_hold_retune_lease + assert not planner.retune_lease_found.is_set() await planner._legacy_retune_task + # The clean-up holds the leadership Lease: SIGTERM must stop it. + assert planner.retune_lease_found.is_set() assert environment.calls[2:] == [ "leases.check", "cleanup.build", @@ -2462,6 +2458,11 @@ async def build(host): try: await asyncio.wait([planner._legacy_retune_task]) assert planner._legacy_retune_cleanup is None + assert environment.calls.count("leases.check") == 1 # no retry + # The default SIGTERM stays unless a Lease was found. + assert planner.retune_lease_found.is_set() is ( + outcome in ("not_found", "bug") + ) records = [r for r in caplog.records if "legacy clean-up" in r.message] levels = {r.levelname for r in records} if outcome == "nothing_left": @@ -2477,6 +2478,33 @@ async def build(host): assert "cleanup.stop" not in environment.calls +async def test_the_legacy_check_retries_a_transient_error_once(monkeypatch) -> None: + from dynamo.planner.core import base as base_module + + monkeypatch.setattr(base_module, "_LEGACY_RETUNE_RETRY_SECONDS", 0.0) + cleanups: list[FakeCleanup] = [] + + async def build(host): + cleanups.append(FakeCleanup(environment.calls)) + return cleanups[-1] + + for leases, found in ( + ([KubeError("timeout"), (STATE_NAME,)], True), + ([KubeError("timeout"), KubeError("timeout")], False), + ): + planner, environment = planner_without_retune( + monkeypatch, kubernetes_connector(), leases=leases, build=build + ) + await planner._async_init() + try: + await planner._legacy_retune_task + assert environment.calls.count("leases.check") == 2 + assert (planner._legacy_retune_cleanup is not None) is found + assert planner.retune_lease_found.is_set() is found + finally: + await planner._shutdown_runtime() + + async def test_the_legacy_check_is_kubernetes_only(monkeypatch) -> None: for controller in (None, SimpleNamespace(kube_api=object())): planner, environment = planner_without_retune(monkeypatch, controller) @@ -2515,47 +2543,6 @@ async def test_dgdr_canary_and_context() -> None: # ------------------------------------------- initials before router writes -class FrontendPolicyClient: - """``RouterPolicyClient`` of the real ``RouterActuator`` over the - world's ``FakeFrontend`` (every pod answers with all of its routers).""" - - def __init__(self, world: World, writer: str) -> None: - self.frontend = world.frontend - self.dynamo_namespace = world.spec.dynamo_namespace - self.writer = writer - - def _info(self, router: Router) -> RouterInfo: - return RouterInfo( - router_id=router.router_id, - namespace=self.dynamo_namespace, - worker_type=router.worker_type, - revision=router.revision, - configuration=dict(router.config), - model="model", - ) - - async def get(self, pod) -> PolicyCallResult: - routers = tuple(self._info(r) for r in self.frontend.routers.values()) - return PolicyCallResult(status=200, routers=routers) - - async def patch(self, pod, router_id, expected_revision, parameters): - router = self.frontend.routers[router_id] - self.frontend.patch(router, dict(parameters), self.writer, False, "apply") - return PolicyCallResult(status=200, router=self._info(router)) - - async def reset(self, pod, router_id, expected_revision): - router = self.frontend.routers[router_id] - self.frontend.reset(router, self.writer, False, "rollback") - return PolicyCallResult(status=200, router=self._info(router)) - - -def real_actuators(world: World, writer: str): - world.kube.add_pod("frontend-0", dgd=DGD_NAME) - return lambda dgd, maps: RouterActuator( - world.kube, FrontendPolicyClient(world, writer), dgd, maps, NullMetrics() - ) - - async def test_a_stop_before_the_initials_write_leaves_no_router_retuned() -> None: """A router seen at revision 0 is latched and would be PATCHed in the same round. Stopped (SIGTERM) before the state records its initial, the next @@ -2565,7 +2552,7 @@ async def test_a_stop_before_the_initials_write_leaves_no_router_retuned() -> No world = World() chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} - first = await started(world, actuator_factory=real_actuators(world, "p1")) + first = await started(world) store = StateStore(world.kube, first.sup.dgd) await first.sup.lease_round() snapshot = store.read() @@ -2588,9 +2575,7 @@ async def test_a_stop_before_the_initials_write_leaves_no_router_retuned() -> No assert world.shared_state().initials == {} await first.sup.stop() - second = await leading( - world, "p2", advisory=True, actuator_factory=real_actuators(world, "p2") - ) + second = await leading(world, "p2", advisory=True) await settle([second], world, rounds=4) assert frontend_configs(world) == [VARIANTS["a"]["aggregated"]] assert not world.shared_state().rollback_pending @@ -2599,7 +2584,7 @@ async def test_a_stop_before_the_initials_write_leaves_no_router_retuned() -> No async def test_initials_are_persisted_before_the_routers_are_patched() -> None: world = World() - process = await leading(world, actuator_factory=real_actuators(world, "p1")) + process = await leading(world) chosen = {CREDIT: 0.5, SCALE: 1.0, TEMP: 0.2} process.controller.restore_desired(chosen) enqueue(process, make_decision(K0 + 1, "applied", chosen=chosen)) @@ -2607,13 +2592,14 @@ async def test_initials_are_persisted_before_the_routers_are_patched() -> None: world.frontend.restart(VARIANTS["a"]) # new routers at revision 0 (router_id,) = world.frontend.routers patched_with: list[dict] = [] - original = world.frontend.patch + original = world.frontend.write - def patch(router, params, *args): - patched_with.append(dict(world.shared_state().initials)) - original(router, params, *args) + def write(router, kind, *args): + if kind == "patch": + patched_with.append(dict(world.shared_state().initials)) + original(router, kind, *args) - world.frontend.patch = patch # type: ignore[method-assign] + world.frontend.write = write # type: ignore[method-assign] # One round: latch, persist the initial, then PATCH. await process.sup.control_round() assert patched_with and all(router_id in seen for seen in patched_with) @@ -2695,507 +2681,3 @@ async def tick(now_s): sup._request_reinit("test") await sup.lease_round() assert controller.current_point().router == best - - -# =================================================== interleaving harness -# -# Two planner processes share a FakeRetuneKube, a frontend and a clock. A -# seeded random schedule interleaves lease renewals and failures, takeovers, -# state writes that succeed, conflict, time out, time out but land, or land -# late, ticks of the real controllers that launch and harvest sweeps, -# frontend restarts and manual patches, spec changes (startup, spec_ok, P4), -# advisory switches, SIGTERM and SIGKILL. After every step the invariants -# are asserted; at the end the system settles with no infinite resets or -# PATCHes. - - -class CheckedKube(FakeRetuneKube): - """Asserts at every successful state write that a term change (a claim) - is made by the leadership Lease holder, and that a write never changes - the term otherwise (an earlier term's write cannot land after a claim).""" - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.state_writes = 0 - self.late = False - # Identity of the process whose call is executing (set by ProcessKube). - self.writer: Optional[str] = None - - def _holder(self) -> Optional[str]: - lease = self.leases.get(LEADER_NAME) - return None if lease is None else lease["spec"].get("holderIdentity") - - def _check(self, before: Optional[dict], after: dict) -> None: - def term(obj: Optional[dict]) -> Optional[str]: - if obj is None: - return None - raw = (obj["metadata"].get("annotations") or {}).get(STATE_ANNOTATION) - return None if raw is None else json.loads(raw).get("leader_term") - - new = term(after) - identity = None if not new else new.split("/", 1)[1].rsplit("/", 1)[0] - if self.writer is not None and new: - # At most one leader writes: a write lands only under its - # writer's own term (an earlier term's writer must fail). - assert identity == self.writer, (identity, self.writer) - if new != term(before): - assert not self.late, "a late write changed the leader_term" - assert identity == self._holder(), (identity, self._holder()) - self.state_writes += 1 - - def replace_lease(self, name, body, *, timeout_s=10.0): - before = copy.deepcopy(self.leases.get(name)) - result = super().replace_lease(name, body, timeout_s=timeout_s) - if name == STATE_NAME: - self._check(before, result) - return result - - def create_lease(self, body, *, timeout_s=10.0): - result = super().create_lease(body, timeout_s=timeout_s) - if result["metadata"]["name"] == STATE_NAME: - self._check(None, result) - return result - - -class ProcessKube: - """One process's view of the shared kube, with its own faults.""" - - def __init__(self, shared: CheckedKube) -> None: - self._shared = shared - self.namespace = shared.namespace - self.identity: Optional[str] = None - self.lease_get_failures = 0 - self.state_faults: deque[str] = deque() - self.deferred: list[tuple[str, dict]] = [] - - def __getattr__(self, attribute: str) -> Any: - return getattr(self._shared, attribute) - - def _as_writer(self, call): - self._shared.writer = self.identity - try: - return call() - finally: - self._shared.writer = None - - def create_lease(self, body, *, timeout_s=10.0): - return self._as_writer( - lambda: self._shared.create_lease(body, timeout_s=timeout_s) - ) - - def get_lease(self, name, *, timeout_s=10.0): - if name == LEADER_NAME and self.lease_get_failures > 0: - self.lease_get_failures -= 1 - raise KubeError("injected renewal failure") - return self._shared.get_lease(name, timeout_s=timeout_s) - - def replace_lease(self, name, body, *, timeout_s=10.0): - if name == STATE_NAME and self.state_faults: - fault = self.state_faults.popleft() - current = self._shared.leases.get(name) - same_term = current is not None and ( - json.loads(current["metadata"]["annotations"][STATE_ANNOTATION]).get( - "leader_term" - ) - == json.loads(body["metadata"]["annotations"][STATE_ANNOTATION]).get( - "leader_term" - ) - ) - if fault == "timeout": - raise KubeError("injected timeout") - if fault == "conflict": - raise KubeConflict("injected conflict") - if fault == "landed": - self._as_writer( - lambda: self._shared.replace_lease(name, body, timeout_s=timeout_s) - ) - raise KubeError("injected timeout (landed)") - if fault == "defer" and same_term: - self.deferred.append((name, copy.deepcopy(body))) - raise KubeError("injected timeout (still in flight)") - return self._as_writer( - lambda: self._shared.replace_lease(name, body, timeout_s=timeout_s) - ) - - def land_deferred(self) -> None: - while self.deferred: - name, body = self.deferred.pop(0) - self._shared.late = True - try: - self._as_writer(lambda: self._shared.replace_lease(name, body)) - except KubeConflict: - pass - finally: - self._shared.late = False - - -class Harness: - def __init__(self, seed: int, *, advisory_only: bool = False) -> None: - self.rng = random.Random(seed) - self.advisory_only = advisory_only - self.world = World(CheckedKube()) - self.world.kube.now = self.world.clock.time - self.processes: dict[str, Process] = {} - self.spawned = 0 - self.steps = 0 - self.transients: dict[str, int] = {} - # Monotonic time until which two processes may both believe they - # lead (the lease was taken over without expiring). - self.stolen_until = -math.inf - self.launched = 0 - self.decided: list[str] = [] - - # -- processes -- - - async def spawn(self, slot: str) -> None: - self.spawned += 1 - kube = ProcessKube(self.world.kube) - advisory = True if self.advisory_only else self.rng.random() < 0.3 - promote = self.rng.random() < 0.5 - process = make_process( - self.world, - f"{slot}-{self.spawned}", - advisory=advisory, - promote=promote, - kube=kube, - ) - kube.identity = process.sup.identity - await process.sup.start(process.host, background=False) - self.processes[slot] = process - - def live(self) -> list[Process]: - return [p for p in self.processes.values() if p.alive] - - # -- the real controllers -- - - async def tick(self, process: Process) -> None: - controller = process.controller - if controller is None: - return - had = controller._pending - result = await controller.on_tick(self.world.clock.time(), SPEC_COUNTS) - if controller._pending is not None and controller._pending is not had: - self.launched += 1 - for event in result.audit_events: - if event.startswith("retune:dropped w="): - controller.dropped.append(int(event.split("=", 1)[1])) - self.decided.extend(decision.outcome for decision in result.decisions) - process.sup.on_effects( - SimpleNamespace( - retune_decisions=result.decisions, - router_parameters=result.router_parameters, - ) - ) - - def finish_sweep(self, process: Process) -> None: - """The process's sweep returns: failed, flat (kept) or with a best - candidate (applied, or advisory).""" - - controller = process.controller - if controller is None: - return - roll = self.rng.random() - if roll < 0.15: - self.world.sweeps.finish(controller, failed) - return - best = None - if roll >= 0.35: - best = {key: self.rng.choice(values) for key, values in DOMAIN.items()} - self.world.sweeps.finish(controller, scored(best)) - - def quiesce(self) -> None: - controllers = [p.controller for p in self.live() if p.controller is not None] - self.world.sweeps.quiesce(controllers) - - # -- events -- - - def _restore_transients(self) -> None: - """Faults of the frontend and the spec last a few steps, then heal.""" - - world, now = self.world, self.steps - for name, attribute, healthy in ( - ("frontend_down", "down", False), - ("spec_broken", "spec_ok", True), - ("p4_broken", "p4_ok", True), - ): - until = self.transients.get(name) - if until is not None and now >= until: - target = world.frontend if attribute == "down" else world.spec - setattr(target, attribute, healthy) - del self.transients[name] - - def _transient(self, name: str, target: Any, attribute: str, broken: Any) -> None: - if name not in self.transients: - setattr(target, attribute, broken) - self.transients[name] = self.steps + self.rng.randint(2, 25) - - async def step(self) -> str: - rng = self.rng - self.steps += 1 - self._restore_transients() - slot = rng.choice(["p1", "p2"]) - process = self.processes.get(slot) - if process is None or not process.alive: - if rng.random() < 0.5: - await self.spawn(slot) - return f"spawn {slot}" - slot = "p2" if slot == "p1" else "p1" - process = self.processes.get(slot) - if process is None or not process.alive: - return "idle" - sup = process.sup - roll = rng.random() - if roll < 0.14: - await sup.lease_round() - return f"lease {slot}" - if roll < 0.28: - await sup.control_round() - return f"control {slot}" - if roll < 0.38: - await sup.outbox_round() - return f"outbox {slot}" - if roll < 0.56: - await self.tick(process) - return f"tick {slot}" - if roll < 0.64: - self.finish_sweep(process) - return f"sweep_done {slot}" - if roll < 0.78: - await self.advance(rng.choice([5, 20, 20, 40, 60, 120, 240])) - return "advance" - if roll < 0.82: - kind = rng.choice(["timeout", "conflict", "landed", "defer"]) - process.kube.state_faults.extend([kind] * rng.choice([1, 1, 2, 3])) - return f"fault {kind} {slot}" - if roll < 0.84: - process.kube.land_deferred() - return f"land {slot}" - if roll < 0.855: - process.kube.lease_get_failures += rng.choice([1, 2, 4]) - return f"renew_fail {slot}" - if roll < 0.87: - self.world.frontend.restart(VARIANTS[self.world.spec.variant]) - return "frontend_restart" - if roll < 0.89: - self.world.frontend.manual_patch(TEMP, rng.choice([0.3, 0.7])) - return "manual_patch" - if roll < 0.905: - self._transient("frontend_down", self.world.frontend, "down", True) - return "frontend_down" - if roll < 0.915: - spec = self.world.spec - spec.variant = "b" if spec.variant == "a" else "a" - self.world.frontend.restart(VARIANTS[spec.variant]) - return "startup_change" - if roll < 0.93: - self._transient("spec_broken", self.world.spec, "spec_ok", False) - return "spec_break" - if roll < 0.937: - self._transient("p4_broken", self.world.spec, "p4_ok", False) - return "p4_break" - if roll < 0.945: - # SIGTERM: the supervisor stops while the engine is up, then the - # engine closes the controller. - await sup.stop() - process.host.close() - process.alive = False - return f"sigterm {slot}" - if roll < 0.951: - process.alive = False - return f"sigkill {slot}" - if roll < 0.957: - self.steal() - return "steal" - return "noop" - - def steal(self) -> None: - """The leadership Lease changes hands without expiring (an admin edit, - or a contender with a skewed clock): to the other live process or - to an intruder that never renews.""" - - lease = self.world.kube.leases.get(LEADER_NAME) - if lease is None: - return - holder = lease["spec"].get("holderIdentity") - others = [p.sup.identity for p in self.live() if p.sup.identity != holder] - lease["spec"]["holderIdentity"] = ( - self.rng.choice(others) - if others and self.rng.random() < 0.7 - else "intruder/0" - ) - lease["spec"]["leaseTransitions"] = lease["spec"].get("leaseTransitions", 0) + 1 - self.world.kube.bump(lease) - self.stolen_until = self.world.clock.monotonic() + LEASE - - async def advance(self, seconds: float) -> None: - """Time passes in chunks of at most 20s while the background loops - mostly keep running (each may miss a chunk).""" - - rng = self.rng - remaining = seconds - while remaining > 0: - chunk = min(20.0, remaining) - remaining -= chunk - self.quiesce() - self.world.clock.advance(chunk) - self._restore_transients() - for process in self.live(): - sup = process.sup - if rng.random() < 0.85: - await sup.lease_round() - if rng.random() < 0.6: - await sup.outbox_round() - if rng.random() < 0.6: - await sup.control_round() - if rng.random() < 0.3: - self.finish_sweep(process) - if rng.random() < 0.6: - await self.tick(process) - self.quiesce() - self.check("advance") - - # -- invariants -- - - def check(self, event: str) -> None: - world = self.world - live = self.live() - leaders = [p.name for p in live if p.sup.is_leader()] - if world.clock.monotonic() >= self.stolen_until: - assert len(leaders) <= 1, (event, leaders) - shared = world.shared_state() - if shared is not None and shared.rollback_pending: - # A pending rollback always records the startup (or nothing). - assert shared.by_wt in (VARIANTS["a"], VARIANTS["b"], {}), ( - event, - shared.by_wt, - ) - for process in live: - sup, controller = process.sup, process.controller - # No window is wasted: a dropped sweep or decision leaves its - # window launchable. - if controller is not None: - decided = ( - -1 - if shared is None or shared.last_window_end_ms is None - else window_index( - shared.last_window_end_ms, phase_s=0.0, interval_s=INTERVAL - ) - ) - for k in controller.dropped: - assert ( - controller._last_k < k - or controller._pending is not None - or k <= decided - ), (event, process.name, k, controller._last_k) - controller.dropped.clear() - # An advisory planner never PATCHes, and resets only to finish a - # rollback of what was deployed. - for writer, advisory, kind, mode, _ in world.frontend.writes: - if advisory: - assert kind == "reset" and mode == "rollback", (event, writer, kind) - if not (sup.is_leader() and sup.initialized and sup.outbox_size == 0): - continue - snapshot = sup._snapshot - if snapshot is None or shared is None: - continue - lease = world.kube.leases[STATE_NAME] - if snapshot.resource_version != lease["metadata"]["resourceVersion"]: - continue # a late write landed; the next write re-reads - assert shared.leader_term == sup.term, event - # State, controller point and target agree. - if ( - shared.mode == "active" - and not shared.rollback_pending - and sup.spec_ok - and controller is not None - ): - assert controller.current_point().router == shared.flat, ( - event, - controller.current_point().router, - shared.flat, - ) - target = sup.target() - if shared.rollback_pending: - assert target.mode == "rollback", event - elif process.advisory: - assert target.mode == "observe", event - elif not sup.spec_ok: - assert target.mode == "rollback", event - else: - assert target.desired_by_wt == shared.by_wt, event - - async def settle(self) -> None: - world = self.world - world.frontend.down = False - world.spec.spec_ok = True - world.spec.p4_ok = True - self.transients.clear() - for process in self.live(): - process.kube.state_faults.clear() - process.kube.lease_get_failures = 0 - process.kube.deferred.clear() - if not self.live(): - await self.spawn("p1") - live = self.live() - for _ in range(12): - self.quiesce() - world.clock.advance(20) - for process in live: - await process.sup.lease_round() - await process.sup.outbox_round() - await process.sup.control_round() - self.check("settle") - writes = len(world.frontend.writes) - for _ in range(4): - world.clock.advance(20) - for process in live: - await process.sup.lease_round() - await process.sup.outbox_round() - await process.sup.control_round() - self.check("settle") - # No infinite reset or PATCH. - assert len(world.frontend.writes) == writes, world.frontend.writes[writes:] - leader = next((p for p in live if p.sup.is_leader()), None) - assert leader is not None and leader.sup.initialized - state = world.shared_state() - # Two-phase rollbacks complete once converged. - assert not state.rollback_pending - target = leader.sup.target() - configs = frontend_configs(world) - if target.mode == "apply": - assert configs == [state.by_wt["aggregated"]] - elif target.mode == "rollback": - assert configs == [VARIANTS[world.spec.variant]["aggregated"]] - - async def run(self, steps: int) -> None: - await self.spawn("p1") - await self.spawn("p2") - for _ in range(steps): - event = await self.step() - self.quiesce() - self.check(event) - await self.settle() - - -@pytest.mark.timeout(120) -@pytest.mark.parametrize("seed", range(24)) -async def test_interleavings_keep_the_invariants(seed: int) -> None: - harness = Harness(seed) - await harness.run(steps=300) - assert harness.world.kube.state_writes > 0 - - -@pytest.mark.timeout(120) -@pytest.mark.parametrize("seed", range(6)) -async def test_advisory_only_never_writes_to_routers(seed: int) -> None: - harness = Harness(1000 + seed, advisory_only=True) - await harness.run(steps=250) - assert harness.world.frontend.writes == [] - - -async def test_the_interleavings_launch_and_decide_windows() -> None: - # The schedule reaches the real controller's launch and harvest paths. - harness = Harness(0) - await harness.run(steps=300) - assert harness.launched > 0 - assert {"applied", "kept", "failed"} <= set(harness.decided) diff --git a/deploy/helm/charts/platform/tests/planner_rbac_test.yaml b/deploy/helm/charts/platform/tests/planner_rbac_test.yaml index 9e3cf99a18cf..c1ba096a9eb4 100644 --- a/deploy/helm/charts/platform/tests/planner_rbac_test.yaml +++ b/deploy/helm/charts/platform/tests/planner_rbac_test.yaml @@ -116,96 +116,32 @@ tests: documentIndex: 1 # ── EXPERIMENTAL live retune (dgdrV1Beta2.plannerLiveRetune) ──────────────── + # The base role has 6 rules; live retune adds exactly 4. Counting the rules + # also catches an extra verb or a split rule. - it: omits live retune rules from the ClusterRole by default asserts: - - notContains: + - lengthEqual: path: rules - content: - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - create - - update - - delete - documentIndex: 0 - - notContains: - path: rules - content: - apiGroups: - - nvidia.com - resources: - - dynamographdeploymentrequests - verbs: - - create - - get - - list - - delete + count: 6 documentIndex: 0 - it: omits live retune rules from the ClusterRole with dgdrV1Beta2 alone set: dynamo-operator.dgdrV1Beta2.enabled: true asserts: - - notContains: - path: rules - content: - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - create - - update - - delete - documentIndex: 0 - - notContains: + - lengthEqual: path: rules - content: - apiGroups: - - nvidia.com - resources: - - dynamographdeploymentrequests - verbs: - - create - - get - - list - - delete + count: 6 documentIndex: 0 - it: omits live retune rules from the ClusterRole with plannerLiveRetune alone set: dynamo-operator.dgdrV1Beta2.plannerLiveRetune: true asserts: - - notContains: - path: rules - content: - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - create - - update - - delete - documentIndex: 0 - - notContains: + - lengthEqual: path: rules - content: - apiGroups: - - nvidia.com - resources: - - dynamographdeploymentrequests - verbs: - - create - - get - - list - - delete + count: 6 documentIndex: 0 - it: grants DGDR v1beta2 and Lease access in the ClusterRole with plannerLiveRetune @@ -213,6 +149,10 @@ tests: dynamo-operator.dgdrV1Beta2.enabled: true dynamo-operator.dgdrV1Beta2.plannerLiveRetune: true asserts: + - lengthEqual: + path: rules + count: 10 + documentIndex: 0 - contains: path: rules content: @@ -259,16 +199,6 @@ tests: - update - delete documentIndex: 0 - - contains: - path: rules - content: - apiGroups: - - "" - resources: - - pods - verbs: - - list - documentIndex: 0 - it: omits live retune rules from the Role with dgdrV1Beta2 alone in namespace-restricted mode set: @@ -276,31 +206,9 @@ tests: dynamo-operator.upgradeCRD: false dynamo-operator.dgdrV1Beta2.enabled: true asserts: - - notContains: - path: rules - content: - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - create - - update - - delete - documentIndex: 1 - - notContains: + - lengthEqual: path: rules - content: - apiGroups: - - nvidia.com - resources: - - dynamographdeploymentrequests - verbs: - - create - - get - - list - - delete + count: 6 documentIndex: 1 - it: grants DGDR v1beta2 and Lease access in the Role with plannerLiveRetune in namespace-restricted mode @@ -310,6 +218,10 @@ tests: dynamo-operator.dgdrV1Beta2.enabled: true dynamo-operator.dgdrV1Beta2.plannerLiveRetune: true asserts: + - lengthEqual: + path: rules + count: 10 + documentIndex: 1 - contains: path: rules content: @@ -356,13 +268,3 @@ tests: - update - delete documentIndex: 1 - - contains: - path: rules - content: - apiGroups: - - "" - resources: - - pods - verbs: - - list - documentIndex: 1 From 180e779c72155c4bc795fa54b055fd81828a5c8a Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Mon, 2 Nov 2026 12:55:25 -0800 Subject: [PATCH 091/100] docs(planner): update the live retune guide for the lower-layer review fixes - Drop retune.dgdr.startup_router_config from the guide and the configuration reference; the field is gone. - The window file is written with mode 0644, readable by the Run Job's UID 1000 and group 65532, so the Planner no longer has to run as UID 1000. - The clean-up check of a removed retune retries once after a transient API error. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../kubernetes/auto-deployment/planner-live-retune.md | 10 +++++----- .../reference/components/planner-configuration.mdx | 3 --- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 57ec5a378e38..a95ef3d05e52 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -66,8 +66,8 @@ This applies to all DGDR trace Runs, not only those the Planner creates. `retune.dgdr.model` must be a public Hugging Face repository id that AISimulate can resolve. - **A ReadWriteMany claim** in the DGD namespace, dedicated to traces. Create `segments/` and `windows/` at its root, writable by the user the frontend and Planner containers run as. The - Planner writes each window file with mode `0600`, and the Run Job reads it as the planner image - user (UID 1000), so run the Planner as that user, which is its default. + Planner writes each window file with mode `0644`, so the Run Job, which reads it as UID 1000 + with supplemental group 65532, can read it whatever user the Planner runs as. - **A KV-router frontend with default routing fields**, and workers that do not set their own router mode. See [Deployment rules](#deployment-rules). - **One Planner replica.** @@ -182,7 +182,6 @@ retune: | `model` | required | Hugging Face repository id the workers serve; the Requests' `spec.model.name` | | `sku` | required | The Requests' `spec.hardware.gpu.sku` | | `engine` | required | The pinned AISimulate engine section; see [Engine section](#engine-section) | -| `startup_router_config` | `null` | Optional startup `KvRouterConfig` JSON; when set it must equal what the Planner parses from the frontend | | `cut_budget_seconds` | `900` | Deadline of the window cut subprocess | | `job_startup_seconds` | `300` | Run Job start-up time reserved when deciding whether a shard can still finish | | `run.parallelism` | `8` | AISimulate `optimizer.parallelism` of each Run; Job resources may lower it | @@ -304,8 +303,9 @@ The Planner only warns when AISimulate does not list the model among its default ``` While the chart still grants the live retune RBAC, a Planner without `retune` does this on - its own: it finds the Leases in the background after it starts, resets any router the - removed retune changed, then deletes the Requests and both Leases. + its own: it looks for the Leases in the background after it starts (once more after a + transient API error), resets any router the removed retune changed, then deletes the + Requests and both Leases. 5. Delete `/` and `/` from the claim. diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 0e7eb795578a..add5c428987e 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -730,9 +730,6 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p The pinned AISimulate `engine` section, written as each Request's `spec.aisimulate.inputOverrides.engine`. Every value must be a scalar, each worker role of `mode` sets one `parallelism.preset` entry and both scheduler knobs, and `replicas` must not be set: the Planner fills them in from the DGD spec at each launch. See [Engine section](../../kubernetes/auto-deployment/planner-live-retune.md#engine-section). - - Optional startup `KvRouterConfig` JSON. When set, it must equal what the Planner parses from the frontend's arguments and environment; the parsed value is the source of truth. - Deadline of the subprocess that cuts a window file from the segments. Must be greater than 0. From 7d56f6aa2fd9779d5270f1ad0a64adbd6e3ef049 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 13:23:01 -0800 Subject: [PATCH 092/100] fix(planner): renew live retune leases before the local leadership deadline - lease_duration_seconds must be at least 37 s: the 15 s lease margin plus two 11 s renewal I/O budgets with the 5 s Kubernetes request timeout. One timing policy in planner_config (the live_retune_*_seconds helpers) backs SupervisorSettings and the DgdrExecutorConfig validator, and the schema states the minimum. - While leading, the lease loop starts the next renewal at least one I/O budget before leader_until (LeaseLeadership.renewal_wait_s), so a renewal that uses its whole budget still lands inside the term. A failed renewal is retried one request timeout after it started. - Initialization keeps renewing the lease: a keepalive shares the lease loop's schedule and step lock, and ends even when the initializing task is cancelled. A lapse during initialization (LeaseLeadership acquisitions / leads_since) aborts it, and the next round initializes the new term. - LeaseLeadership no longer caps the margin at two thirds of the lease; it rejects a margin that is not below the lease duration. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/config/planner_config.py | 63 +++- .../src/dynamo/planner/retune/live/state.py | 68 +++- .../dynamo/planner/retune/live/supervisor.py | 172 +++++++-- .../tests/unit/test_live_retune_config.py | 24 ++ .../tests/unit/test_live_retune_state.py | 128 ++++++- .../tests/unit/test_live_retune_supervisor.py | 351 +++++++++++++++++- .../auto-deployment/planner-live-retune.md | 2 +- .../components/planner-configuration.mdx | 2 +- 8 files changed, 760 insertions(+), 50 deletions(-) diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index d1b613f021fe..87190bc1f49f 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -878,6 +878,41 @@ class DgdrRunConfig(BaseModel): ) +# Kubernetes request timeout of the live retune supervisor +# (``SupervisorSettings``). +LIVE_RETUNE_K8S_TIMEOUT_SECONDS = 5.0 + + +def live_retune_lease_margin_seconds(k8s_timeout_s: float) -> float: + """How long before the leadership Lease could expire local leadership + ends.""" + + return 3.0 * k8s_timeout_s + + +def live_retune_io_timeout_seconds(k8s_timeout_s: float) -> float: + """I/O budget of one blocking Kubernetes call, such as a whole renewal + (a GET and a replace): a backstop, the client applies the request + timeout too.""" + + return 2.0 * k8s_timeout_s + 1.0 + + +def live_retune_min_lease_seconds(k8s_timeout_s: float) -> float: + """Shortest leadership Lease the live supervisor can keep renewing. + + Local leadership ends the lease margin before the Lease could expire, + counted from the send of the renewal that granted it. That renewal may + use a whole I/O budget, and the next one starts at least one I/O budget + before local leadership ends, so two budgets must fit. Lease rounds are + single renewals: an initialization renews concurrently instead of + delaying the next one. + """ + + margin_s = live_retune_lease_margin_seconds(k8s_timeout_s) + return margin_s + 2.0 * live_retune_io_timeout_seconds(k8s_timeout_s) + + class DgdrExecutorConfig(BaseModel): """Live retune through DGDR v1beta2 (``retune.executor: dgdr``). @@ -961,9 +996,18 @@ class DgdrExecutorConfig(BaseModel): ) lease_duration_seconds: float = Field( default=60.0, - ge=15, allow_inf_nan=False, - description="Leadership Lease duration; renewed every third of it.", + # Not ge=: it would reject before _lease_fits_two_renewals explains. + json_schema_extra={ + "minimum": live_retune_min_lease_seconds(LIVE_RETUNE_K8S_TIMEOUT_SECONDS) + }, + description=( + "Leadership Lease duration. Renewed every third of it, or sooner " + "so that a renewal that uses its whole I/O budget ends before " + "local leadership does (the duration minus the lease margin). " + "At least 37 s: the 15 s lease margin plus two 11 s renewal I/O " + "budgets, with the 5 s Kubernetes request timeout." + ), ) actuator_interval_seconds: float = Field( default=15.0, @@ -972,6 +1016,21 @@ class DgdrExecutorConfig(BaseModel): description="Period of the router actuator and reconciler rounds.", ) + @field_validator("lease_duration_seconds") + @classmethod + def _lease_fits_two_renewals(cls, value: float) -> float: + timeout_s = LIVE_RETUNE_K8S_TIMEOUT_SECONDS + minimum_s = live_retune_min_lease_seconds(timeout_s) + if value < minimum_s: + raise ValueError( + "retune.dgdr.lease_duration_seconds must be at least the lease " + "margin + 2 x the I/O budget of one renewal " + f"({live_retune_lease_margin_seconds(timeout_s):g}s + 2 x " + f"{live_retune_io_timeout_seconds(timeout_s):g}s = {minimum_s:g}s " + f"with the {timeout_s:g}s Kubernetes request timeout)" + ) + return value + # Live retune (``retune.executor: dgdr``) limits. A sweep must end this long # before the next window's launch: harvest, the decision write and clean-up diff --git a/components/src/dynamo/planner/retune/live/state.py b/components/src/dynamo/planner/retune/live/state.py index c44a2d4a23bb..fa1a8cac51e5 100644 --- a/components/src/dynamo/planner/retune/live/state.py +++ b/components/src/dynamo/planner/retune/live/state.py @@ -702,7 +702,12 @@ class LeaseLeadership: ``is_leader`` holds from a successful write until ``leader_until`` = send time + lease duration - ``margin_s``, all on the local monotonic clock: a contender takes over only after watching the holder unchanged - for a whole lease duration, which starts after the send. + for a whole lease duration, which starts after the send. ``margin_s`` + must be >= 0 and less than the lease duration. The caller must start + each renewal early enough for it to end before ``leader_until`` + (``renewal_wait_s``): a renewal whose read of the Lease returns at or + after ``leader_until`` reports ``acquired`` (a new term), never + ``renewed``. """ def __init__( @@ -729,14 +734,16 @@ def __init__( self._duration_int = max(1, int(math.ceil(lease_duration_s))) if margin_s is None: margin_s = 3.0 * timeout_s - # Never leave less than a third of the duration to lead in. - self.margin_s = min(float(margin_s), self.lease_duration_s * 2.0 / 3.0) + if not 0.0 <= margin_s < self.lease_duration_s: + raise ValueError("margin_s must be >= 0 and less than lease_duration_s") + self.margin_s = float(margin_s) self._timeout_s = float(timeout_s) self._monotonic = monotonic self._clock = clock self._lock = threading.Lock() self._held = False self._leader_until = -math.inf + self._acquisitions = 0 self.transitions = 0 # resourceVersion of this process's last successful write. self.resource_version: Optional[str] = None @@ -761,6 +768,44 @@ def leader_until(self) -> float: with self._lock: return self._leader_until + @property + def acquisitions(self) -> int: + """How many steps have reported ``acquired``.""" + + with self._lock: + return self._acquisitions + + def leads_since(self, acquisitions: int) -> bool: + """``is_leader`` with no ``acquired`` step since ``acquisitions``. + + Checked atomically with the ``leader_until`` a re-acquisition sets, + so a step still in flight on another thread cannot hide its lapse. + """ + + with self._lock: + return ( + self._held + and self._monotonic() < self._leader_until + and self._acquisitions == acquisitions + ) + + def renewal_wait_s( + self, *, interval_s: float, budget_s: float, floor_s: float = 0.0 + ) -> float: + """Time until the next ``step``. + + ``interval_s``, but while leading no later than ``budget_s`` before + ``leader_until`` (at once if that has passed), so that a renewal + taking up to ``budget_s`` ends while this process still leads. + Never less than ``floor_s``, which takes precedence. + """ + + now = self._monotonic() + with self._lock: + leading = self._held and now < self._leader_until + latest_s = self._leader_until - budget_s - now + return max(floor_s, min(interval_s, latest_s) if leading else interval_s) + def set_owner(self, dgd: DgdRef) -> None: """The DGD the Lease's ownerReference names from now on.""" @@ -813,7 +858,7 @@ def step(self) -> LeaseStep: new_holder = (updated.get("spec") or {}).get("holderIdentity") or "" if new_holder != self.identity: return self._lose() - self._lead(t_send, transitions, updated) + self._lead(t_send, transitions, updated, acquired=not was_held) return "renewed" if was_held else "acquired" lost = self.held @@ -872,10 +917,19 @@ def release(self) -> bool: # ------------------------------------------------------------ helpers - def _lead(self, t_send: float, transitions: int, written: dict[str, Any]) -> None: + def _lead( + self, + t_send: float, + transitions: int, + written: dict[str, Any], + *, + acquired: bool, + ) -> None: with self._lock: self._held = True self._leader_until = t_send + self.lease_duration_s - self.margin_s + if acquired: + self._acquisitions += 1 self.transitions = transitions self.resource_version = (written.get("metadata") or {}).get("resourceVersion") self._observed = None @@ -913,7 +967,7 @@ def _create(self, t_send: float) -> LeaseStep: "retune leadership: creating %s failed: %s", self.name, error ) return "error" - self._lead(t_send, 0, created) + self._lead(t_send, 0, created, acquired=True) logger.info( "retune leadership: acquired new lease %s as %s", self.name, self.identity ) @@ -934,7 +988,7 @@ def _take_over( "retune leadership: taking over %s failed: %s", self.name, error ) return "error" - self._lead(t_send, transitions + 1, written) + self._lead(t_send, transitions + 1, written, acquired=True) logger.info( "retune leadership: acquired lease %s as %s (transitions=%d)", self.name, diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py index 6f5fe8c8c9ca..8229d7c6dfee 100644 --- a/components/src/dynamo/planner/retune/live/supervisor.py +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -25,10 +25,13 @@ thread with a timeout): - lease: renew or acquire ``-planner-retune`` every third of the lease - duration. Becoming leader, re-acquiring after losing track, or a re-init - request enters initialization (claim the state with ``term_seq + 1``, then - apply the first matching row of ``plan_initialization``); losing - leadership runs the local loss. + duration; while leading, early enough that a renewal using its whole I/O + budget still ends before local leadership does. Becoming leader, + re-acquiring after losing track, or a re-init request enters + initialization (claim the state with ``term_seq + 1``, then apply the + first matching row of ``plan_initialization``); the Lease is still renewed + on the same schedule while it runs, and a renewal that reports a lapse + aborts it. Losing leadership runs the local loss. - control: re-read the DGD and re-validate the spec premises (rebuilding the controller through ``host.replace_live_retune`` when the startup, ``spec_ok`` or P4 change), then, as an initialized leader, the P11 canary @@ -74,6 +77,12 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, Optional, Protocol, Union +from dynamo.planner.config.planner_config import ( + LIVE_RETUNE_K8S_TIMEOUT_SECONDS, + live_retune_io_timeout_seconds, + live_retune_lease_margin_seconds, + live_retune_min_lease_seconds, +) from dynamo.planner.plugins.orchestrator.engine_adapter import RetuneHooks from dynamo.planner.retune.live.contracts import ( LABEL_DGD_UID, @@ -95,6 +104,7 @@ from dynamo.planner.retune.live.dgdr import run_dgdr_sweep from dynamo.planner.retune.live.state import ( LeaseLeadership, + LeaseStep, RetuneMode, RetuneState, StateSnapshot, @@ -227,7 +237,7 @@ class SupervisorSettings: router_keys: tuple[str, ...] = () lease_duration_s: float = 60.0 actuator_interval_s: float = 15.0 - k8s_timeout_s: float = 5.0 + k8s_timeout_s: float = LIVE_RETUNE_K8S_TIMEOUT_SECONDS write_failure_limit: int = 3 canary_interval_s: float = 600.0 stop_timeout_s: float = 10.0 @@ -245,22 +255,37 @@ def __post_init__(self) -> None: "lease_duration_s", "actuator_interval_s", "reconcile_budget_s", + "k8s_timeout_s", ): value = getattr(self, name) if not (math.isfinite(value) and value > 0): raise ValueError(f"{name} must be positive and finite") if self.write_failure_limit < 1: raise ValueError("write_failure_limit must be >= 1") + minimum_s = live_retune_min_lease_seconds(self.k8s_timeout_s) + if self.lease_duration_s < minimum_s: + raise ValueError( + f"lease_duration_s {self.lease_duration_s:g}s is too short for " + f"k8s_timeout_s {self.k8s_timeout_s:g}s: it must be at least the " + f"lease margin ({self.lease_margin_s:g}s) + 2 x the I/O budget " + f"of one renewal ({self.io_timeout_s:g}s) = {minimum_s:g}s" + ) @property def renew_interval_s(self) -> float: return self.lease_duration_s / 3.0 + @property + def lease_margin_s(self) -> float: + """How long before the Lease could expire local leadership ends.""" + + return live_retune_lease_margin_seconds(self.k8s_timeout_s) + @property def io_timeout_s(self) -> float: """Backstop on one blocking call; the client applies its own.""" - return 2.0 * self.k8s_timeout_s + 1.0 + return live_retune_io_timeout_seconds(self.k8s_timeout_s) @classmethod def from_config( @@ -608,7 +633,7 @@ def __init__( evaluation.dgd, identity=identity, lease_duration_s=settings.lease_duration_s, - margin_s=3.0 * settings.k8s_timeout_s, + margin_s=settings.lease_margin_s, timeout_s=settings.k8s_timeout_s, monotonic=monotonic, clock=clock, @@ -660,6 +685,9 @@ def __init__( # Tasks and locks. self._init_lock = asyncio.Lock() self._write_lock = asyncio.Lock() + # The lease loop and an initialization both step the leadership + # Lease; their GET/replace pairs must not race on its resourceVersion. + self._lease_step_lock = asyncio.Lock() self._tasks: list[asyncio.Task[None]] = [] self._outbox_task: Optional[asyncio.Task[None]] = None self._round_task: Optional[asyncio.Task[None]] = None @@ -971,16 +999,15 @@ async def stop(self) -> None: # ------------------------------------------------------------- rounds - async def lease_round(self) -> None: - """One renewal/acquisition; initialize or run the local loss.""" + async def lease_round(self) -> Optional[LeaseStep]: + """One renewal/acquisition; initialize or run the local loss. + + Returns the renewal/acquisition step, ``None`` once stopping. + """ if self._stopping: - return - try: - step = await self._io(self._leadership.step) - except Exception as error: - logger.warning("retune lease round failed: %s", error) - step = "error" + return None + step = await self._lease_step() try: if self.is_leader(): if ( @@ -999,6 +1026,7 @@ async def lease_round(self) -> None: except Exception: logger.exception("retune lease round: initialization failed") self._publish() + return step async def control_round(self) -> None: """Re-validate the spec; as an initialized leader, actuate and start @@ -1138,31 +1166,50 @@ async def _initialize_locked(self) -> bool: Confirms leadership, claims the state (only ``leader_term``), writes what ``plan_initialization`` decides, restores the controller and rewinds it to the last persisted window. ``True`` once initialized. + The leadership Lease is renewed meanwhile; a renewal that reports a + lapse (``acquired``) aborts the initialization. """ await self._local_loss_locked("initialize") # Confirm on the API server right before the claim: a holder change # this process has not seen yet (it believes it leads until # leader_until) aborts the initialization instead of claiming. - try: - await self._io(self._leadership.step) - except Exception as error: - logger.warning("retune init: confirming leadership failed: %s", error) + started = self._monotonic() + step = await self._lease_step() if not self.is_leader(): return False + acquisitions = self._leadership.acquisitions + keepalive = asyncio.create_task( + self._renew_while_initializing(step, started), + name="planner-retune-init-lease", + ) + try: + return await self._claim_locked(acquisitions) + finally: + await self._end_renewals(keepalive) + + async def _claim_locked(self, acquisitions: int) -> bool: + """``_initialize_locked`` after the confirming renewal; gives up once + a local loss or a re-acquisition since ``acquisitions`` made the new + term stale.""" + self._term_seq += 1 self._reinit_requested = False term = f"{self._leadership.transitions}/{self._identity}/{self._term_seq}" self._term = term epoch = self._epoch settings = self._settings + + def current() -> bool: + return epoch == self._epoch and self._leadership.leads_since(acquisitions) + async with self._write_lock: snapshot = await self._read_own_snapshot() if snapshot is None: return False # Claim: only leader_term, conditional on the resourceVersion. for attempt in range(_CLAIM_ATTEMPTS): - if not self.is_leader() or epoch != self._epoch: + if not current(): return False claimed = claim(snapshot.state, dgd_uid=self._dgd.uid, term=term) try: @@ -1200,7 +1247,7 @@ async def _initialize_locked(self) -> bool: ), ) if plan.write: - if not self.is_leader() or epoch != self._epoch: + if not current(): return False try: snapshot = await self._io(self._store.write, plan.state, snapshot) @@ -1230,7 +1277,7 @@ async def _initialize_locked(self) -> bool: return False if controller is not None: self._rewind(controller) - if epoch != self._epoch or not self.is_leader(): + if not current(): return False self._initialized = True self._rollback_since = None @@ -2045,13 +2092,90 @@ def _publish(self) -> None: async def _lease_loop(self) -> None: while not self._stopping: + started = self._monotonic() + step: Optional[LeaseStep] = "error" try: - await self.lease_round() + step = await self.lease_round() except asyncio.CancelledError: raise except Exception: logger.exception("retune lease loop iteration failed") - await self._wait(self._lease_wake, self._settings.renew_interval_s) + wait_s = self._lease_wait_s(failed=step == "error", started=started) + await self._wait(self._lease_wake, wait_s) + + def _lease_wait_s(self, *, failed: bool, started: float) -> float: + """Time until the next renewal after an attempt begun at ``started``. + + While leading, after a successful attempt the next starts no later + than one I/O budget before ``leader_until`` (at once if that has + passed), so that it ends while this process still leads. After a + failed attempt the next starts no sooner than one request timeout + after the failed one started, even within one I/O budget of + ``leader_until`` or past it: a fast-failing API server is not polled + in a busy loop, and a retry after a slow failure is not pushed back + further. + """ + + settings = self._settings + floor_s = 0.0 + if failed: + elapsed_s = self._monotonic() - started + floor_s = max(0.0, settings.k8s_timeout_s - elapsed_s) + return self._leadership.renewal_wait_s( + interval_s=settings.renew_interval_s, + budget_s=settings.io_timeout_s, + floor_s=floor_s, + ) + + async def _lease_step(self) -> LeaseStep: + """One ``LeaseLeadership.step``; ``error`` if it raised.""" + + async with self._lease_step_lock: + try: + return await self._io(self._leadership.step) + except Exception as error: + logger.warning("retune lease renewal failed: %s", error) + return "error" + + async def _renew_while_initializing(self, step: LeaseStep, started: float) -> None: + """Renew on the lease loop's schedule while an initialization runs. + + The lease loop waits for the initialization (or for ``_init_lock`` + during a rebuild), which may outlast the local lifetime left after + its confirming renewal. ``step`` and ``started`` are that renewal's. + A lapse (``acquired``) requests a re-initialization; the leadership + re-acquired is still renewed until the stale initialization ends. + """ + + while True: + await self._sleep( + self._lease_wait_s(failed=step == "error", started=started) + ) + if self._stopping: + return + started = self._monotonic() + step = await self._lease_step() + if step == "acquired": + self._request_reinit("leadership lapsed during initialization") + elif step not in ("renewed", "error"): + return + + async def _end_renewals(self, keepalive: asyncio.Task[None]) -> None: + """Cancel ``_renew_while_initializing`` and wait for it to end. + + A renewal in flight is waited for first: cancelled, its thread would + go on and race the next step on the resourceVersion. If this task is + cancelled meanwhile (``stop``), the renewal is cancelled in flight, + as the lease loop's own step is, rather than left running. + """ + + try: + async with self._lease_step_lock: + await self._cancel(keepalive) + except asyncio.CancelledError: + keepalive.cancel() + await asyncio.wait([keepalive]) + raise async def _control_loop(self) -> None: while not self._stopping: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py index 22008e851dec..f8be5d5f5364 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py @@ -27,12 +27,14 @@ from dynamo.planner.config.planner_config import ( DGDR_MAX_SHARD_POINTS, + LIVE_RETUNE_K8S_TIMEOUT_SECONDS, DgdrExecutorConfig, PlannerConfig, balanced_shard_parts, dgdr_need_seconds, dgdr_rounds, largest_shard_points, + live_retune_min_lease_seconds, ) from dynamo.planner.retune.domains import expand_router_domain @@ -499,6 +501,28 @@ def test_live_retune_rejects_unsafe_claim_paths(overrides, match): _rejects(match, _retune(dgdr=_dgdr(**overrides))) +# --------------------------------------------------------- leadership Lease + + +def test_live_retune_lease_fits_two_renewals_before_its_margin(): + minimum = live_retune_min_lease_seconds(LIVE_RETUNE_K8S_TIMEOUT_SECONDS) + # 3 x 5 s margin + 2 x (2 x 5 s + 1 s) I/O budget. + assert minimum == 37.0 + for lease in (15.0, minimum - 0.001): + _rejects( + r"retune.dgdr.lease_duration_seconds must be at least the lease " + r"margin \+ 2 x the I/O budget of one renewal \(15s \+ 2 x 11s = 37s " + r"with the 5s Kubernetes request timeout\)", + _retune(dgdr=_dgdr(lease_duration_seconds=lease)), + ) + config = _config(_retune(dgdr=_dgdr(lease_duration_seconds=minimum))) + assert config.retune is not None and config.retune.dgdr is not None + assert config.retune.dgdr.lease_duration_seconds == minimum + schema = DgdrExecutorConfig.model_json_schema()["properties"] + assert schema["lease_duration_seconds"]["minimum"] == minimum + assert "At least 37 s" in schema["lease_duration_seconds"]["description"] + + # -------------------------------------------------------------------- P13 diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py index 642f58d3343a..dd9ee79b4a8d 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py @@ -8,10 +8,18 @@ import dataclasses import json import math +from collections.abc import Callable +from typing import Optional import pytest -from dynamo.planner.config.planner_config import PlannerConfig +from dynamo.planner.config.planner_config import ( + LIVE_RETUNE_K8S_TIMEOUT_SECONDS, + PlannerConfig, + live_retune_io_timeout_seconds, + live_retune_lease_margin_seconds, + live_retune_min_lease_seconds, +) from dynamo.planner.retune.live.contracts import ( LABEL_DGD, LABEL_DGD_UID, @@ -613,20 +621,120 @@ def test_leadership_conflicts_and_errors() -> None: assert not a.release() and not a.is_leader() -def test_leadership_validates_arguments_and_caps_the_margin() -> None: +def test_leadership_validates_arguments() -> None: kube = FakeRetuneKube() clock = FakeClock() with pytest.raises(ValueError): LeaseLeadership(kube, DGD, identity="") with pytest.raises(ValueError): LeaseLeadership(kube, DGD, identity="a", lease_duration_s=0) - capped = leadership(kube, clock) - assert capped.margin_s == 15.0 - tiny = LeaseLeadership( - kube, DGD, identity="a", lease_duration_s=15.0, margin_s=100.0 + # No time left to lead in; the default margin is three request timeouts. + for margin_s in (None, 15.0, 100.0, -1.0, math.nan): + with pytest.raises(ValueError, match="margin_s"): + LeaseLeadership( + kube, DGD, identity="a", lease_duration_s=15.0, margin_s=margin_s + ) + assert leadership(kube, clock).margin_s == 15.0 + short = LeaseLeadership( + kube, DGD, identity="a", lease_duration_s=15.0, margin_s=10.0 ) - assert tiny.margin_s == 10.0 - tiny.set_owner(dataclasses.replace(DGD, uid="uid-2")) - assert tiny.step() == "acquired" - owner = kube.leases[tiny.name]["metadata"]["ownerReferences"][0] + assert short.margin_s == 10.0 + short.set_owner(dataclasses.replace(DGD, uid="uid-2")) + assert short.step() == "acquired" + owner = kube.leases[short.name]["metadata"]["ownerReferences"][0] assert owner["uid"] == "uid-2" + + +# The shortest lease the live supervisor accepts, with its request timeout. +K8S_TIMEOUT = LIVE_RETUNE_K8S_TIMEOUT_SECONDS +MIN_LEASE = live_retune_min_lease_seconds(K8S_TIMEOUT) + + +def minimum_leadership(kube, clock) -> LeaseLeadership: + return LeaseLeadership( + kube, + DGD, + identity="a/1", + lease_duration_s=MIN_LEASE, + margin_s=live_retune_lease_margin_seconds(K8S_TIMEOUT), + timeout_s=K8S_TIMEOUT, + monotonic=clock.monotonic, + clock=clock.time, + ) + + +INTERVAL = MIN_LEASE / 3.0 +BUDGET = live_retune_io_timeout_seconds(K8S_TIMEOUT) + + +def renewal_wait(a: LeaseLeadership) -> float: + """The supervisor's lease loop wait after a successful renewal.""" + + return a.renewal_wait_s(interval_s=INTERVAL, budget_s=BUDGET) + + +def test_leadership_renews_the_minimum_lease_in_one_term_until_it_lapses() -> None: + clock = FakeClock() + kube = FakeRetuneKube() + a = minimum_leadership(kube, clock) + assert renewal_wait(a) == INTERVAL + assert a.step() == "acquired" and a.acquisitions == 1 + acquired = kube.leases[a.name]["spec"]["acquireTime"] + # One I/O budget before leader_until, sooner than the renew interval. + margin = live_retune_lease_margin_seconds(K8S_TIMEOUT) + assert renewal_wait(a) == MIN_LEASE - margin - BUDGET < INTERVAL + for _ in range(10): + clock.advance(renewal_wait(a)) + assert a.leads_since(1) + assert a.step() == "renewed" + spec = kube.leases[a.name]["spec"] + assert a.transitions == 0 and spec["leaseTransitions"] == 0 + assert spec["acquireTime"] == acquired + # A failed attempt's floor wins over the cap, even past leader_until. + clock.advance(a.leader_until - clock.mono - 1.0) + assert a.renewal_wait_s(interval_s=INTERVAL, budget_s=BUDGET, floor_s=5.0) == 5.0 + clock.advance(5.0) + assert not a.is_leader() and renewal_wait(a) == INTERVAL + assert a.step() == "acquired" and a.acquisitions == 2 + assert a.is_leader() and not a.leads_since(1) + + +class SlowLeaseKube(FakeRetuneKube): + """Every Lease call takes ``latency_s`` of ``clock``. A replace calls + ``probe`` when its response is due, before the caller gets it.""" + + def __init__(self, clock: FakeClock, latency_s: float) -> None: + super().__init__(now=clock.time) + self.clock = clock + self.latency_s = latency_s + self.probe: Optional[Callable[[], bool]] = None + self.probes: list[bool] = [] + + def get_lease(self, name: str, *, timeout_s: float = 10.0) -> Optional[dict]: + self.clock.advance(self.latency_s) + return super().get_lease(name, timeout_s=timeout_s) + + def create_lease(self, body: dict, *, timeout_s: float = 10.0) -> dict: + self.clock.advance(self.latency_s) + return super().create_lease(body, timeout_s=timeout_s) + + def replace_lease(self, name: str, body: dict, *, timeout_s: float = 10.0) -> dict: + self.clock.advance(self.latency_s) + if self.probe is not None: + self.probes.append(self.probe()) + return super().replace_lease(name, body, timeout_s=timeout_s) + + +def test_leadership_never_lapses_while_slow_renewals_are_in_flight() -> None: + # The GET and the replace together take just under the I/O budget that + # bounds one renewal (the supervisor's backstop around step()); the + # client's (connect, read) timeout alone does not cap a call at K8S_TIMEOUT. + clock = FakeClock() + kube = SlowLeaseKube(clock, latency_s=BUDGET / 2 - 0.01) + a = minimum_leadership(kube, clock) + kube.probe = a.is_leader + assert a.step() == "acquired" + for _ in range(10): + clock.advance(renewal_wait(a)) + assert a.step() == "renewed" + assert kube.probes == [True] * 10 and a.transitions == 0 diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py index 80b7541d4418..33fd7793d098 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -34,7 +34,12 @@ reason="AI Simulate is an optional Dynamo simulation dependency", ) -from dynamo.planner.config.planner_config import PlannerConfig, RetuneEngineSpec +from dynamo.planner.config.planner_config import ( + LIVE_RETUNE_K8S_TIMEOUT_SECONDS, + PlannerConfig, + RetuneEngineSpec, + live_retune_min_lease_seconds, +) from dynamo.planner.core.types import ( EngineCapabilities, RetuneDecision, @@ -199,8 +204,10 @@ async def inline(call): return call() -async def no_sleep(_seconds: float) -> None: - await asyncio.sleep(0) +async def frozen_sleep(_seconds: float) -> None: + """The test's clock moves only when the test moves it: no sleep ends.""" + + await asyncio.get_running_loop().create_future() class RecordingMetrics(NullMetrics): @@ -748,7 +755,7 @@ def make_process( context_factory=context_factory, clock=world.clock.time, monotonic=world.clock.monotonic, - sleep=no_sleep, + sleep=frozen_sleep, blocking=blocking or inline, ) host = FakeHost(world, sup.hooks(), evaluation.inputs, advisory=advisory) @@ -1952,6 +1959,340 @@ async def test_background_loops_run_and_stop() -> None: assert process.metrics.state["leader"] is False +# ------------------------------------------------------------ lease timing + +MIN_LEASE = live_retune_min_lease_seconds(LIVE_RETUNE_K8S_TIMEOUT_SECONDS) + + +def test_settings_require_a_lease_that_fits_two_renewals() -> None: + for lease in (15.0, MIN_LEASE - 0.001): + with pytest.raises( + ValueError, + match=r"k8s_timeout_s 5s: it must be at least the lease margin " + r"\(15s\) \+ 2 x the I/O budget of one renewal \(11s\) = 37s", + ): + settings_for(lease_duration_s=lease) + for timeout in (0.0, -1.0, math.inf, math.nan): + with pytest.raises(ValueError, match="k8s_timeout_s must be positive"): + settings_for(k8s_timeout_s=timeout) + settings = settings_for(lease_duration_s=MIN_LEASE) + assert settings.lease_margin_s == 15.0 and settings.io_timeout_s == 11.0 + # The legacy clean-up supervisor's settings (the defaults). + SupervisorSettings(interval_s=3600.0, grace_s=0.0, timeout_s=3000.0) + dgdr = live_config().retune.dgdr.model_dump() + dgdr["lease_duration_seconds"] = MIN_LEASE + config = live_config(retune={"dgdr": dgdr}) + assert SupervisorSettings.from_config(config).lease_duration_s == MIN_LEASE + + +class SlowLeases: + """``world.kube``, but each read or write of a Lease named in + ``latency_s`` first advances the test's clock by its latency.""" + + def __init__(self, world: World) -> None: + self._kube = world.kube + self._clock = world.clock + self.latency_s: dict[str, float] = {} + + def __getattr__(self, name: str) -> Any: + return getattr(self._kube, name) + + def _wait(self, lease: str) -> None: + self._clock.advance(self.latency_s.get(lease, 0.0)) + + def get_lease(self, name: str, **kwargs: Any) -> Optional[dict]: + self._wait(name) + return self._kube.get_lease(name, **kwargs) + + def create_lease(self, body: dict, **kwargs: Any) -> dict: + self._wait(body["metadata"]["name"]) + return self._kube.create_lease(body, **kwargs) + + def replace_lease(self, name: str, body: dict, **kwargs: Any) -> dict: + self._wait(name) + return self._kube.replace_lease(name, body, **kwargs) + + +class LeaseLoop: + """The supervisor's own lease loop, one round at a time: each of its + waits advances the test's clock by the time the loop asked for (a + re-init request does not cut it short). + + Every other sleep (an initialization's renewals) ends once Kubernetes + calls have advanced the clock past it: each call runs inline, then lets + the sleepers it made due run before its caller goes on. + """ + + def __init__(self, sup: LiveRetuneSupervisor, clock: Clock) -> None: + self._sup = sup + self._clock = clock + self._asleep = asyncio.Event() + self._wake = asyncio.Event() + self._task: Optional[asyncio.Task[None]] = None + self._sleepers: list[tuple[float, asyncio.Future[None]]] = [] + self.waits: list[float] = [] + + async def __aenter__(self) -> LeaseLoop: + self._sup._wait = self._wait # type: ignore[method-assign] + self._sup._sleep = self._sleep + self._sup._blocking = self._run + self._task = asyncio.create_task(self._sup._lease_loop()) + await self._round_done() + return self + + async def __aexit__(self, *exc_info: Any) -> None: + assert self._task is not None + self._task.cancel() + await asyncio.wait([self._task]) + + async def _wait(self, _event: Optional[asyncio.Event], seconds: float) -> None: + self.waits.append(seconds) + self._asleep.set() + await self._wake.wait() + self._wake.clear() + self._clock.advance(seconds) + + async def _sleep(self, seconds: float) -> None: + if seconds <= 0: + return + future = asyncio.get_running_loop().create_future() + self._sleepers.append((self._clock.monotonic() + seconds, future)) + await future + + async def _run(self, call): + try: + return call() + finally: + now = self._clock.monotonic() + for deadline, future in self._sleepers: + if deadline <= now and not future.done(): + future.set_result(None) + self._sleepers = [ + (deadline, future) + for deadline, future in self._sleepers + if not future.done() + ] + await asyncio.sleep(0) + + async def _round_done(self) -> None: + await asyncio.wait_for(self._asleep.wait(), timeout=10) + self._asleep.clear() + + async def next_round(self) -> None: + """Let the loop wait as asked, then run its next round.""" + + self._wake.set() + await self._round_done() + + +async def test_renewals_after_a_slow_initialization_keep_the_term() -> None: + world = World() + kube = SlowLeases(world) + process = await started(world, kube=kube) + sup, controller = process.sup, process.controller + # The state read, claim and plan write after the confirming renewal take + # 27 s, each within its 11 s I/O budget: more than the local lifetime + # (45 s) minus the renew interval (20 s). + kube.latency_s[STATE_NAME] = 9.0 + async with LeaseLoop(sup, world.clock) as loop: + assert sup.initialized + kube.latency_s.clear() + term, term_seq = sup.term, sup.term_seq + await sup.control_round() + assert sup.ready() + world.clock.wall = window(K0 + 1)[2] + await controller.on_tick(world.clock.time(), SPEC_COUNTS) + pending = controller._pending + assert pending is not None + for _ in range(6): + await loop.next_round() + assert sup.is_leader() and sup.initialized + assert (sup.term, sup.term_seq) == (term, term_seq) + assert controller._pending is pending and not pending.abort.is_set() + assert controller.dropped == [] + + +async def test_minimum_lease_renewals_keep_the_term_and_the_sweep() -> None: + world = World() + kube = SlowLeases(world) + settings = settings_for(lease_duration_s=MIN_LEASE) + process = await leading(world, kube=kube, settings=settings) + sup, controller = process.sup, process.controller + term, term_seq = sup.term, sup.term_seq + world.clock.wall = window(K0 + 1)[2] + await controller.on_tick(world.clock.time(), SPEC_COUNTS) + pending = controller._pending + assert pending is not None + # Every leadership Lease call takes its whole request timeout. + kube.latency_s[LEADER_NAME] = settings.k8s_timeout_s + async with LeaseLoop(sup, world.clock) as loop: + for _ in range(8): + await loop.next_round() + assert sup.is_leader() and sup.initialized + assert (sup.term, sup.term_seq) == (term, term_seq) + assert world.kube.leases[LEADER_NAME]["spec"]["leaseTransitions"] == 0 + assert controller._pending is pending and not pending.abort.is_set() + assert controller.dropped == [] + + +@pytest.mark.parametrize("latency_s", [4.0, 5.0]) +async def test_minimum_lease_initialization_with_slow_leases_keeps_the_term( + latency_s: float, +) -> None: + world = World() + kube = SlowLeases(world) + settings = settings_for(lease_duration_s=MIN_LEASE) + process = await started(world, kube=kube, settings=settings) + sup, controller = process.sup, process.controller + # Without renewals in between, local leadership would end soon after the + # state read, claim and plan write that follow the confirming renewal + # (4 s per Lease call: before the next renewal is sent), or before they + # end (5 s, the whole request timeout). + kube.latency_s[LEADER_NAME] = latency_s + kube.latency_s[STATE_NAME] = latency_s + async with LeaseLoop(sup, world.clock) as loop: + assert sup.is_leader() and sup.initialized + term, term_seq = sup.term, sup.term_seq + del kube.latency_s[STATE_NAME] + await sup.control_round() + assert sup.ready() + world.clock.wall = window(K0 + 1)[2] + await controller.on_tick(world.clock.time(), SPEC_COUNTS) + pending = controller._pending + assert pending is not None + for _ in range(8): + await loop.next_round() + assert sup.is_leader() and sup.initialized + assert (sup.term, sup.term_seq) == (term, term_seq) + assert controller._pending is pending and not pending.abort.is_set() + assert controller.dropped == [] + + +async def test_a_lapse_during_initialization_aborts_it_and_reinitializes_once() -> None: + world = World() + kube = SlowLeases(world) + settings = settings_for(lease_duration_s=MIN_LEASE) + process = await started(world, kube=kube, settings=settings) + sup = process.sup + kube.latency_s[LEADER_NAME] = 1.0 + kube.latency_s[STATE_NAME] = 9.0 + # From the new term on (set before its claim), reading the leadership + # Lease fails until local leadership has ended: the next renewal that + # succeeds re-acquires while the initialization of that term still runs. + world.kube.fail( + "get_lease", + KubeError("unavailable"), + times=100, + match=lambda target: target == LEADER_NAME + and sup.term is not None + and sup.is_leader(), + ) + async with LeaseLoop(sup, world.clock) as loop: + assert sup.is_leader() and not sup.initialized + assert sup.term_seq == 1 + world.kube.clear_faults() + kube.latency_s.clear() + for _ in range(3): + await loop.next_round() + assert sup.is_leader() and sup.initialized + assert sup.term_seq == 2 + assert world.shared_state().leader_term == sup.term + + +async def test_a_cancelled_initialization_leaves_no_renewal_behind() -> None: + world = World() + process = await started(world) + sup = process.sup + steps = 0 + in_flight, gate = asyncio.Event(), asyncio.Event() + + async def gated(call): + # The third leadership step, after the acquiring and the confirming + # ones, is the initialization's first renewal: it stays in flight, + # and the initialization goes on only once it is. + nonlocal steps + if getattr(call, "func", None) == sup._leadership.step: + steps += 1 + if steps == 3: + in_flight.set() + await gate.wait() + elif steps >= 2: + await in_flight.wait() + return call() + + sup._blocking = gated + sup._sleep = lambda _seconds: asyncio.sleep(0) + task = asyncio.create_task(sup.lease_round()) + for _ in range(1000): + if sup.initialized: + break + await asyncio.sleep(0) + # Initialized; the round now waits for the renewal in flight. + assert sup.initialized and in_flight.is_set() and not task.done() + task.cancel() + await asyncio.wait([task]) + pending = [ + other + for other in asyncio.all_tasks() + if other.get_name() == "planner-retune-init-lease" + ] + assert pending == [] + await sup.stop() + gate.set() + await asyncio.sleep(0) + assert world.holder() is None and not sup.is_leader() + + +async def test_failed_renewals_are_retried_a_request_timeout_apart() -> None: + world = World() + process = await leading(world) + sup = process.sup + world.kube.fail( + "get_lease", + KubeError("unavailable"), + times=100, + match=lambda target: target == LEADER_NAME, + ) + async with LeaseLoop(sup, world.clock) as loop: + for _ in range(5): + await loop.next_round() + # The renew interval, then one I/O budget before leader_until (45 s after + # the last renewal), then one request timeout apart until it passed. + assert loop.waits == [20.0, 14.0, 5.0, 5.0, 5.0, 20.0] + assert sup.term is None and not sup.initialized + + +async def test_a_retry_after_a_slow_failure_keeps_the_minimum_lease() -> None: + world = World() + kube = SlowLeases(world) + settings = settings_for(lease_duration_s=MIN_LEASE) + process = await leading(world, kube=kube, settings=settings) + sup, controller = process.sup, process.controller + term, term_seq = sup.term, sup.term_seq + world.clock.wall = window(K0 + 1)[2] + await controller.on_tick(world.clock.time(), SPEC_COUNTS) + pending = controller._pending + assert pending is not None + kube.latency_s[LEADER_NAME] = 2.0 + async with LeaseLoop(sup, world.clock) as loop: + # The renewal one I/O budget before leader_until fails after its + # whole request timeout; the retry goes at once and still renews. + kube.latency_s[LEADER_NAME] = settings.k8s_timeout_s + world.kube.fail( + "get_lease", + KubeError("unavailable"), + match=lambda target: target == LEADER_NAME, + ) + await loop.next_round() + kube.latency_s[LEADER_NAME] = 2.0 + await loop.next_round() + assert sup.is_leader() and sup.initialized + assert (sup.term, sup.term_seq) == (term, term_seq) + # leader_until is 22 s after each renewal's send; a renewal takes 4 s. + assert loop.waits == [7.0, 0.0, 7.0] + assert controller._pending is pending and not pending.abort.is_set() + + async def test_hooks_are_the_adapter_hooks() -> None: world = World() process = await started(world) @@ -1978,7 +2319,7 @@ def decommission_supervisor(world: World) -> LiveRetuneSupervisor: frontend=frontend, clock=world.clock.time, monotonic=world.clock.monotonic, - sleep=no_sleep, + sleep=frozen_sleep, blocking=inline, ) diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index a95ef3d05e52..6a4b2e333156 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -190,7 +190,7 @@ retune: | `keep_requests` | `5` | Windows whose finished Requests are kept for audit | | `max_window_rows` | `5000000` | A larger window fails with `window_too_large` | | `promote_last_advisory` | `false` | On switching to active, apply the last advisory recommendation if it is at most one interval old and the configuration is unchanged | -| `lease_duration_seconds` | `60` | Leadership Lease duration, renewed every third of it | +| `lease_duration_seconds` | `60` | Leadership Lease duration, renewed every third of it or sooner; at least `37`, so two renewals of up to 11 s each fit before the Planner stops leading, 15 s before the Lease expires | | `actuator_interval_seconds` | `15` | Period of the router and clean-up rounds | For the other `retune` fields and the limits that `executor: dgdr` adds to them, see the diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index add5c428987e..47e9087e0a27 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -760,7 +760,7 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p On switching from `advisory: true` to `false`, apply the last advisory recommendation if it is at most one interval old and the configuration is unchanged. - Duration of the leadership Lease, renewed every third of it. Must be at least 15. + Duration of the leadership Lease, renewed every third of it or sooner. Must be at least 37: the Planner stops acting as leader 15 seconds (three 5-second Kubernetes request timeouts) before the Lease expires, and two renewals of up to 11 seconds each must fit before then. Period of the router and clean-up rounds. Must be greater than 0. From 147797688541284d19473b0d9aa0e6ea8981a6a9 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 13:23:01 -0800 Subject: [PATCH 093/100] fix(planner): reject runtime updates of the live retune scoring SLA Live retune fixes its scoring SLA when its controller and state fingerprint are built, and a runtime configuration update does not rebuild them. With retune.sla_ttft_ms or retune.sla_itl_ms unset, a runtime update of ttft_ms or itl_ms was accepted and reported while live retune kept scoring and creating Requests with the old SLA. While live retune runs, runtime candidate validation rejects (HTTP 422) an update that changes the effective scoring SLA and leaves the config untouched. The message says to restart the planner with the new values, or with the unset retune SLA overrides set to keep those fields changeable at runtime. Planners without live retune are unaffected. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/planner/core/base.py | 3 + .../src/dynamo/planner/reconfiguration.py | 31 +++++ .../tests/unit/test_live_retune_adapter.py | 126 +++++++++++++++++- .../auto-deployment/planner-live-retune.md | 7 + .../components/planner-configuration.mdx | 5 + 5 files changed, 171 insertions(+), 1 deletion(-) diff --git a/components/src/dynamo/planner/core/base.py b/components/src/dynamo/planner/core/base.py index 60ef11f2e2a1..86b2d198a81c 100644 --- a/components/src/dynamo/planner/core/base.py +++ b/components/src/dynamo/planner/core/base.py @@ -55,6 +55,7 @@ from dynamo.planner.offline.trace_data import extract_traffic_observations_from_trace from dynamo.planner.reconfiguration import ( PlannerReconfiguration, + live_retune_sla_errors, runtime_candidate_errors, runtime_configuration_errors, ) @@ -583,6 +584,8 @@ async def get_min_endpoints(self) -> dict[str, object]: def validate_runtime_candidate(self, candidate: PlannerConfig) -> None: errors = runtime_candidate_errors(candidate, self._build_worker_capabilities()) + if self._live_retune is not None: + errors.extend(live_retune_sla_errors(self.config, candidate)) if errors: raise ValueError("; ".join(errors)) self._reconfigurable_engine() diff --git a/components/src/dynamo/planner/reconfiguration.py b/components/src/dynamo/planner/reconfiguration.py index fe7827b36c1f..4a50fcfc1d12 100644 --- a/components/src/dynamo/planner/reconfiguration.py +++ b/components/src/dynamo/planner/reconfiguration.py @@ -244,6 +244,37 @@ def runtime_candidate_errors( ) +def live_retune_sla_errors( + config: PlannerConfig, candidate: PlannerConfig +) -> list[str]: + """Return why ``candidate`` cannot replace ``config`` under live retune. + + The live retune controller and its state fingerprint take the scoring SLA + when they are built and are not rebuilt on a runtime update, so a change + of an inherited ``ttft_ms`` or ``itl_ms`` would never reach them. + """ + + if config.retune is None or candidate.retune is None: + return [] + current = config.retune.scoring_sla(config.ttft_ms, config.itl_ms) + proposed = candidate.retune.scoring_sla(candidate.ttft_ms, candidate.itl_ms) + changed = [ + field + for field, old, new in zip(("ttft_ms", "itl_ms"), current, proposed) + if old != new + ] + if not changed: + return [] + fields = " and ".join(changed) + overrides = " and ".join(f"retune.sla_{field}" for field in changed) + return [ + f"cannot change {fields} at runtime while retune.executor is 'dgdr': " + f"live retune inherits its scoring SLA from {fields} ({overrides} unset); " + f"restart the planner with the new {fields}, or restart it with " + f"{overrides} set to keep {fields} changeable at runtime" + ] + + def prepare_predictor( config: PlannerConfig, capabilities: WorkerCapabilities, diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py index 5e3db54465b3..34cbd95a6f71 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_adapter.py @@ -6,7 +6,8 @@ The retune controller is the recording stub of ``retune_stubs`` (one test builds the real one), and the live supervisor is a fake ``LiveRetuneHandle``, so these tests cover only the host contract: the disabled state, -``replace_retune``, the base's supervisor lifecycle and the SIGTERM path. +``replace_retune``, the base's supervisor lifecycle, its runtime +configuration API and the SIGTERM path. """ from __future__ import annotations @@ -39,6 +40,7 @@ OrchestratorEngineAdapter, RetuneHooks, ) +from dynamo.planner.reconfiguration import ReconfigureRequest, candidate_config from dynamo.planner.retune.live.contracts import GateResult from dynamo.planner.tests.retune_stubs import ( ROUTER_MAP, @@ -48,6 +50,7 @@ retune_inputs, retune_only_config, ) +from dynamo.planner.tests.unit.test_live_retune_state import live_config pytestmark = [ pytest.mark.gpu_0, @@ -543,6 +546,127 @@ async def test_replace_live_retune_holds_the_config_lock(stub_controller): await planner._shutdown_runtime() +# --------------------------------------------------------------------------- +# Runtime reconfiguration +# --------------------------------------------------------------------------- + + +def _sla_planner(**retune) -> NativePlannerBase: + """A planner whose validated dgdr config scores with SLA 500/50 ms by + default, with its live retune supervisor installed.""" + + config = live_config( + ttft_ms=500.0, + itl_ms=50.0, + retune=retune, + metric_reporting_prometheus_port=0, + live_dashboard_port=0, + control_api_port=0, + ) + planner = _base(config, _Environment()) + planner._live_retune = FakeLiveRetune([]) + return planner + + +async def _patch(planner: NativePlannerBase, changes: dict) -> dict: + """PATCH ``changes`` through the runtime API; an accepted update is + awaited and answered with its operation.""" + + reconfiguration = planner.reconfiguration + request_id = f"update-{planner._config_generation}" + body = ReconfigureRequest( + instance_id=reconfiguration.instance_id, + expected_revision=planner._config_generation, + request_id=request_id, + config=changes, + ).model_dump() + response = await reconfiguration.handle_request( + {"method": "PATCH", "path": "config", "body": body} + ) + if response["status"] != 202: + return response + await reconfiguration._task + return {"status": 202, "body": reconfiguration.get_operation(request_id)} + + +@pytest.mark.parametrize( + ("retune", "changes", "inherited"), + [ + ({}, {"ttft_ms": 250.0}, "ttft_ms"), + ({}, {"itl_ms": 25.0}, "itl_ms"), + ({}, {"ttft_ms": 250.0, "itl_ms": 25.0}, "ttft_ms and itl_ms"), + ({}, {"ttft_ms": 250.0, "max_gpu_budget": 16}, "ttft_ms"), + ({"sla_ttft_ms": 400.0}, {"itl_ms": 25.0}, "itl_ms"), + ], +) +async def test_live_retune_rejects_a_runtime_change_of_its_inherited_sla( + retune, changes, inherited +): + planner = _sla_planner(**retune) + # The update passes every other runtime check: only live retune stops it. + candidate = candidate_config(planner.config, changes) + assert {field: getattr(candidate, field) for field in changes} == changes + before = await planner.reconfiguration.get_config() + + response = await _patch(planner, changes) + + assert response["status"] == 422 + overrides = " and ".join(f"retune.sla_{f}" for f in inherited.split(" and ")) + assert response["body"]["error"] == ( + f"cannot change {inherited} at runtime while retune.executor is 'dgdr': " + f"live retune inherits its scoring SLA from {inherited} ({overrides} unset); " + f"restart the planner with the new {inherited}, or restart it with " + f"{overrides} set to keep {inherited} changeable at runtime" + ) + assert await planner.reconfiguration.get_config() == before + assert (planner.config.ttft_ms, planner.config.itl_ms) == (500.0, 50.0) + await planner._shutdown_runtime() + + +@pytest.mark.parametrize( + ("retune", "changes"), + [ + ( + {"sla_ttft_ms": 400.0, "sla_itl_ms": 40.0}, + {"ttft_ms": 250.0, "itl_ms": 25.0}, + ), + ({"sla_ttft_ms": 400.0}, {"ttft_ms": 250.0}), + ({}, {"ttft_ms": 500.0, "max_gpu_budget": 16}), + ], +) +async def test_live_retune_accepts_runtime_updates_that_keep_its_scoring_sla( + retune, changes +): + planner = _sla_planner(**retune) + config = planner.config + assert config.retune is not None + scoring_sla = config.retune.scoring_sla(config.ttft_ms, config.itl_ms) + + response = await _patch(planner, changes) + + assert response["body"]["state"] == "applied" + assert {field: getattr(config, field) for field in changes} == changes + assert config.retune.scoring_sla(config.ttft_ms, config.itl_ms) == scoring_sla + await planner._shutdown_runtime() + + +async def test_planner_without_live_retune_changes_its_sla_at_runtime(): + config = PlannerConfig( + namespace="test-ns", + mode="agg", + metric_reporting_prometheus_port=0, + live_dashboard_port=0, + control_api_port=0, + ) + planner = _base(config, _Environment()) + + response = await _patch(planner, {"ttft_ms": 250.0, "itl_ms": 25.0}) + + assert response["body"]["state"] == "applied" + assert (planner.config.ttft_ms, planner.config.itl_ms) == (250.0, 25.0) + await planner._shutdown_runtime() + + # --------------------------------------------------------------------------- # SIGTERM # --------------------------------------------------------------------------- diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 6a4b2e333156..c610c2600af5 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -196,6 +196,13 @@ retune: For the other `retune` fields and the limits that `executor: dgdr` adds to them, see the [Retune field reference](../../reference/components/planner-configuration.mdx#retune-experimental). +Live retune takes its scoring SLA when it starts: `retune.sla_ttft_ms` and `retune.sla_itl_ms`, +or the Planner's `ttft_ms` and `itl_ms` where they are unset. An update of `ttft_ms` or `itl_ms` +through the +[runtime reconfiguration API](../../reference/components/planner-configuration.mdx#system-port-reconfiguration-api) +that would change that SLA fails with HTTP `422`. To change the SLA, restart the Planner with the +new values; to keep `ttft_ms` and `itl_ms` changeable at runtime, set both `retune` SLA fields. + ### Engine section `retune.dgdr.engine` becomes `spec.aisimulate.inputOverrides.engine` of every Request, so every diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index 47e9087e0a27..ac3612f9298a 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -476,6 +476,11 @@ Fields retain their normal mode-specific meaning. For example, the explicit queu thresholds are used by `optimization_target: load`. Predictor replacement requires `optimization_target: sla` with throughput scaling enabled. Scheduling intervals, AIS performance models, deployment topology, and other fields are rejected by this API. +With [live retune](../../kubernetes/auto-deployment/planner-live-retune.md) (`retune.executor: dgdr`), +an update of `ttft_ms` or `itl_ms` that changes the retune scoring SLA, because `retune.sla_ttft_ms` +or `retune.sla_itl_ms` is unset, is rejected with HTTP `422`. To change the SLA, restart the +Planner with the new values; to keep `ttft_ms` and `itl_ms` changeable at runtime, start it with +both `retune` SLA fields set. Use the instance ID and revision returned by GET when submitting an update: From b24f1164fca7bd416a5425c0ad5f1b2c0a0179ab Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 13:23:01 -0800 Subject: [PATCH 094/100] fix(planner): bind live retune window files and Requests to the trace selection Window files and Requests were keyed by the window end alone, so a sweep with another window start or as_of could reuse a file or Request cut for a different trace selection. - Window files are named //-.jsonl.gz, where the selection is the first 12 hex digits of a SHA-256 over the window start, end and as_of. build_request takes window_start_ms and as_of_ms, so the Request's subPath, spec hash and name follow the selection, and Requests of another selection are deleted, not adopted. - The cut records the selection in the gzip header and reuses a published file only for the same selection. - The reconciler recognizes the new names as well as the old ones. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/config/planner_config.py | 6 +- .../dynamo/planner/retune/live/bootstrap.py | 2 + .../src/dynamo/planner/retune/live/dgdr.py | 78 +++++++++-- .../dynamo/planner/retune/live/reconciler.py | 10 +- .../dynamo/planner/retune/live/window_cut.py | 73 +++++++++-- .../tests/integration/test_live_retune_e2e.py | 2 +- .../tests/unit/test_live_retune_dgdr.py | 124 +++++++++++++++++- .../tests/unit/test_live_retune_reconciler.py | 52 ++++++++ .../tests/unit/test_live_retune_window_cut.py | 85 ++++++++++-- .../trace_request_lowering.golden.json | 2 +- .../dgdr_v1beta2_trace_golden_envtest_test.go | 7 +- .../auto-deployment/planner-live-retune.md | 8 +- .../components/planner-configuration.mdx | 2 +- 13 files changed, 396 insertions(+), 55 deletions(-) diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index 87190bc1f49f..132a08bb4935 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -949,7 +949,11 @@ class DgdrExecutorConfig(BaseModel): windows_dir: str = Field( default="windows", min_length=1, - description="Claim-relative directory of window files: //.", + description=( + "Claim-relative directory of window files: " + "//-.jsonl.gz, where " + " is a digest of the window's start, end and cutoff." + ), ) model: str = Field( min_length=1, diff --git a/components/src/dynamo/planner/retune/live/bootstrap.py b/components/src/dynamo/planner/retune/live/bootstrap.py index 8cf5347ec468..274a73123f18 100644 --- a/components/src/dynamo/planner/retune/live/bootstrap.py +++ b/components/src/dynamo/planner/retune/live/bootstrap.py @@ -306,7 +306,9 @@ def canary(supervisor: LiveRetuneSupervisor) -> bool: retune=retune, sla_ttft_ms=sla_ttft_ms, sla_itl_ms=sla_itl_ms, + window_start_ms=CANARY_WINDOW_END_MS - 1, window_end_ms=CANARY_WINDOW_END_MS, + as_of_ms=CANARY_WINDOW_END_MS, shard=shard, spec_replicas=replicas, backend=str(config.backend), diff --git a/components/src/dynamo/planner/retune/live/dgdr.py b/components/src/dynamo/planner/retune/live/dgdr.py index c1618da83bc3..232f58c0c37f 100644 --- a/components/src/dynamo/planner/retune/live/dgdr.py +++ b/components/src/dynamo/planner/retune/live/dgdr.py @@ -12,10 +12,12 @@ probe that evaluates one fixed grid point a second time. 2. Adopt the Requests a previous leader created for this window, matched by ``(dgd-uid, window, shard, spec hash)``; others of the window are queued - for deletion. A window whose every shard is adopted as Succeeded is - harvested without cutting it again. An adopted attempt whose Run failed - or ran another input is replaced by attempt+1 while the shard still - fits before the deadline. + for deletion. The spec names the window file, whose name holds the + trace selection (``window_claim_path``), so a Request of another window + start or cutoff with the same end is never adopted. A window whose + every shard is adopted as Succeeded is harvested without cutting it + again. An adopted attempt whose Run failed or ran another input is + replaced by attempt+1 while the shard still fits before the deadline. 3. Cut the window file (a subprocess, ``context.cut_window``), then create one Request per remaining shard with Strict field validation and check that the API server kept every submitted field. @@ -294,10 +296,36 @@ def spec_hash(spec: Mapping[str, Any]) -> str: return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:16] -def window_claim_path(cfg: DgdrExecutorConfig, dgd: DgdRef, window_end_ms: int) -> str: - """Claim-relative path of a window file: ``//.jsonl.gz``.""" +def window_selection(window_start_ms: int, window_end_ms: int, as_of_ms: int) -> str: + """First 12 hex digits of the SHA-256 of ``||`` (epoch ms).""" - return f"{cfg.windows_dir.strip('/')}/{dgd.name}/{int(window_end_ms)}.jsonl.gz" + key = f"{int(window_start_ms)}|{int(window_end_ms)}|{int(as_of_ms)}" + return hashlib.sha256(key.encode("utf-8")).hexdigest()[:12] + + +def window_claim_path( + cfg: DgdrExecutorConfig, + dgd: DgdRef, + *, + window_start_ms: int, + window_end_ms: int, + as_of_ms: int, +) -> str: + """Claim-relative path of a window file: + ``//-.jsonl.gz``. + + The Request spec names this path, so the selection (``[start, end)`` + as of ``as_of``) is part of the spec hash and the Request name: a + window cut or a Request of another selection with the same end is + never reused. The reconciler reads the window end from the leading + digits. + """ + + selection = window_selection(window_start_ms, window_end_ms, as_of_ms) + return ( + f"{cfg.windows_dir.strip('/')}/{dgd.name}/" + f"{int(window_end_ms)}-{selection}.jsonl.gz" + ) def request_name( @@ -330,7 +358,9 @@ def build_request( retune: RetuneConfig, sla_ttft_ms: float, sla_itl_ms: float, + window_start_ms: int, window_end_ms: int, + as_of_ms: int, shard: Shard, spec_replicas: Mapping[str, int], backend: str, @@ -338,8 +368,10 @@ def build_request( ) -> dict: """The DGDR v1beta2 Request that replays one shard of a window. - The engine is ``cfg.engine`` with every leaf a scalar and each used - role's ``parallelism.preset[0].replicas`` set from ``spec_replicas`` + The trace is the window file of ``[window_start_ms, window_end_ms)`` as + of ``as_of_ms`` (``window_claim_path``). The engine is ``cfg.engine`` + with every leaf a scalar and each used role's + ``parallelism.preset[0].replicas`` set from ``spec_replicas`` (unused roles are deleted with ``null``); the GPU budget is the sum of replicas times GPUs per worker (tensor x pipeline x attention_data). Router knobs use their AISimulate public names: ``{choices: [...]}`` @@ -368,7 +400,13 @@ def build_request( "source": { "persistentVolumeClaim": { "claimName": cfg.claim_name, - "subPath": window_claim_path(cfg, dgd, window_end_ms), + "subPath": window_claim_path( + cfg, + dgd, + window_start_ms=window_start_ms, + window_end_ms=window_end_ms, + as_of_ms=as_of_ms, + ), } }, } @@ -818,10 +856,20 @@ def __init__( if request.deadline_s is not None else started + request.timeout_s ) + # Rounded once: the window file's path, the Request specs and the + # cut must agree on the selection. + self.window_start_ms = int(round(request.window_start_ms)) self.window_end_ms = int(round(request.window_end_ms)) + self.as_of_ms = int(round(request.as_of_ms)) # Registry references this sweep took; each is released at the end. self._protections = 0 - self.claim_path = window_claim_path(self.cfg, context.dgd, self.window_end_ms) + self.claim_path = window_claim_path( + self.cfg, + context.dgd, + window_start_ms=self.window_start_ms, + window_end_ms=self.window_end_ms, + as_of_ms=self.as_of_ms, + ) self.local_path = os.path.join(self.cfg.mount_path, self.claim_path) self.shards: dict[str, _ShardRun] = {} self.cut_done = False @@ -1020,7 +1068,9 @@ def _prepare(self) -> None: retune=self.ctx.retune, sla_ttft_ms=request.sla_ttft_ms, sla_itl_ms=request.sla_itl_ms, + window_start_ms=self.window_start_ms, window_end_ms=self.window_end_ms, + as_of_ms=self.as_of_ms, shard=shard, spec_replicas=replicas, backend=self.ctx.backend, @@ -1076,7 +1126,9 @@ def _build(self, run: _ShardRun) -> dict: retune=self.ctx.retune, sla_ttft_ms=self.request.sla_ttft_ms, sla_itl_ms=self.request.sla_itl_ms, + window_start_ms=self.window_start_ms, window_end_ms=self.window_end_ms, + as_of_ms=self.as_of_ms, shard=run.shard, spec_replicas=self.details["replicas"], backend=self.ctx.backend, @@ -1306,9 +1358,9 @@ def _cut(self) -> None: os.makedirs(os.path.dirname(self.local_path), exist_ok=True) result: WindowCutResult = self.ctx.cut_window( segments_root, - int(round(self.request.window_start_ms)), + self.window_start_ms, self.window_end_ms, - int(round(self.request.as_of_ms)), + self.as_of_ms, self.local_path, max_rows=self.cfg.max_window_rows, deadline_s=cut_deadline_s, diff --git a/components/src/dynamo/planner/retune/live/reconciler.py b/components/src/dynamo/planner/retune/live/reconciler.py index 614a3b47e2a9..a3ce986b1b10 100644 --- a/components/src/dynamo/planner/retune/live/reconciler.py +++ b/components/src/dynamo/planner/retune/live/reconciler.py @@ -8,9 +8,9 @@ claim, ``T``/``G`` the retune interval/grace and ``timeout`` the sweep timeout:: - ////.NNNNNN.jsonl.gz frontend segments - ///.jsonl.gz window files - ///.tmp-- cut temp files + ////.NNNNNN.jsonl.gz frontend segments + ///-.jsonl.gz window files + ///.tmp-- cut temp files - Segments, only when the frontend pod set is known and nonempty: never the highest-numbered segment of a live pod; other segments of a live pod, and @@ -83,8 +83,8 @@ REQUEST_ACCEPTED_CONDITION = "Accepted" REQUEST_INVALID_SPEC_REASON = "InvalidSpec" -# A window file: .jsonl.gz -_WINDOW_RE = re.compile(r"^(\d+)" + re.escape(SEGMENT_SUFFIX) + r"$") +# A window file: [-].jsonl.gz +_WINDOW_RE = re.compile(r"^(\d+)(?:-[0-9a-f]+)?" + re.escape(SEGMENT_SUFFIX) + r"$") # A Rust gzip JSONL sink segment: ..jsonl.gz _SEGMENT_RE = re.compile(r"^(.*)\.(\d{6,})" + re.escape(SEGMENT_SUFFIX) + r"$") diff --git a/components/src/dynamo/planner/retune/live/window_cut.py b/components/src/dynamo/planner/retune/live/window_cut.py index a744413da301..a44006201487 100644 --- a/components/src/dynamo/planner/retune/live/window_cut.py +++ b/components/src/dynamo/planner/retune/live/window_cut.py @@ -14,7 +14,9 @@ rows are the same bytes; only the output is gzip-compressed and the memory is bounded. The cut runs in a child process, reads the segments in two passes (``_Selection``, ``_write_rows``) and publishes the file atomically -(``_publish``). +(``_publish``). The file's gzip header records its selection (``start_ms``, +``end_ms``, ``as_of_ms`` in whole ms) and counts; a file already at the +output path is reused only for the same selection. A gzip member that fails validation is dropped whole. Runs of bad bytes above ``MAX_DROPPED_MEMBER_RATIO`` of the members, or bad bytes above that @@ -103,7 +105,7 @@ def write(self, data: bytes, /) -> Any: _GZIP_WBITS = 16 + zlib.MAX_WBITS # one gzip member, header and trailer # The header of a window file: FLG=FCOMMENT, MTIME=0 (equal cuts are equal # bytes), XFL=0, OS=unknown. The zero-terminated comment that follows holds -# the cut's counts. +# the cut's selection and counts. _WINDOW_HEADER = _GZIP_MAGIC + b"\x10" + bytes(4) + b"\x00\xff" _WINDOW_HEADER_READ = 1 << 16 _READ_CHUNK = 1 << 20 @@ -224,9 +226,12 @@ def cut_window_inprocess( """Cut a window file in this process; the body of ``cut_window``'s child. An empty window writes no file (``path=None``). When ``out_path`` already - exists, no segment is read: the result carries the counts in its header - (``reused=True``, nothing dropped). ``temp_path`` names the temporary - file (it must be in ``out_path``'s directory); by default a fresh + exists, no segment is read: if its header records the same selection + (``start_ms``, ``end_ms`` and ``as_of_ms`` rounded to whole ms), the + result carries the counts in its header (``reused=True``, nothing + dropped); a file of another selection, or one that records none, fails + as ``window_cut_failed``. ``temp_path`` names the temporary file (it + must be in ``out_path``'s directory); by default a fresh ``temp_file_name`` there. Failures raise ``WindowCutError``. """ @@ -243,7 +248,9 @@ def cut_window_inprocess( if os.path.lexists(out_path): # Another process (an earlier leader, an adopted Request's cut) # already published this window; DGDR Requests name it by path. - return _published(out_path) + return _published( + out_path, start_ms=start_ms, end_ms=end_ms, as_of_ms=as_of_ms + ) files = _segment_files(os.fspath(segments_root)) selection = _Selection(start_ms, end_ms, as_of_ms, max_rows) items = _trace_items(files, start_ms / 1000.0 - SEGMENT_MTIME_MARGIN_S) @@ -265,11 +272,15 @@ def cut_window_inprocess( created = _publish( out_path, temp_path, - _counts_comment(result), + _counts_comment( + result, start_ms=start_ms, end_ms=end_ms, as_of_ms=as_of_ms + ), lambda sink: _write_rows(selection, files, sink), ) + if created: + return result # Another process published it first: report the counts it holds. - return result if created else _published(out_path) + return _published(out_path, start_ms=start_ms, end_ms=end_ms, as_of_ms=as_of_ms) except OSError as error: raise WindowCutError( "window_cut_failed", f"{type(error).__name__}: {error}" @@ -1072,11 +1083,29 @@ def close(self) -> None: self._raw.write(struct.pack(" bytes: - """The header comment of a window file: the counts of its cut, as ASCII - JSON (which never holds the NUL that ends a gzip comment).""" +def _selection_bounds( + start_ms: float, end_ms: float, as_of_ms: float +) -> dict[str, int]: + return { + "start_ms": int(round(start_ms)), + "end_ms": int(round(end_ms)), + "as_of_ms": int(round(as_of_ms)), + } + + +def _describe_selection(bounds: Mapping[str, Any]) -> str: + return f"[{bounds['start_ms']}, {bounds['end_ms']}) as of {bounds['as_of_ms']}" + + +def _counts_comment( + result: WindowCutResult, *, start_ms: float, end_ms: float, as_of_ms: float +) -> bytes: + """The header comment of a window file: the selection and the counts of + its cut, as ASCII JSON (which never holds the NUL that ends a gzip + comment).""" counts = { + "selection": _selection_bounds(start_ms, end_ms, as_of_ms), "num_requests": result.num_requests, "trace_block_size": result.trace_block_size, "row_counts": result.row_counts, @@ -1084,11 +1113,15 @@ def _counts_comment(result: WindowCutResult) -> bytes: return json.dumps(counts, sort_keys=True, separators=(",", ":")).encode("ascii") -def _published(path: str) -> WindowCutResult: +def _published( + path: str, *, start_ms: float, end_ms: float, as_of_ms: float +) -> WindowCutResult: """The result of the window file already at ``path``, from its header. Nothing was dropped by this call: it read no segment. A file without - the counts of a cut in its header fails as ``window_cut_failed``. + the counts of a cut in its header, or cut for another selection than + ``[start_ms, end_ms)`` as of ``as_of_ms`` (whole ms), or recording no + selection, fails as ``window_cut_failed``. """ with open(path, "rb") as source: @@ -1116,6 +1149,20 @@ def _published(path: str) -> WindowCutResult: f"window file {path} exists but its gzip header holds no cut " f"counts: {error}", ) from error + wanted = _selection_bounds(start_ms, end_ms, as_of_ms) + found = counts.get("selection") + if found != wanted: + if not (isinstance(found, dict) and set(found) == set(wanted)): + raise WindowCutError( + "window_cut_failed", + f"window file {path} exists but its gzip header holds no " + "selection, so it is not reused", + ) + raise WindowCutError( + "window_cut_failed", + f"window file {path} was cut for {_describe_selection(found)}, " + f"not {_describe_selection(wanted)}", + ) return WindowCutResult( path=path, num_requests=num_requests, diff --git a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py index 2d312f0059a8..e601b19bf81a 100644 --- a/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py +++ b/components/src/dynamo/planner/tests/integration/test_live_retune_e2e.py @@ -548,7 +548,7 @@ def window_files(self) -> set[int]: if not self.windows.is_dir(): return set() return { - int(name.split(".", 1)[0]) + int(name.split(".", 1)[0].split("-", 1)[0]) for name in os.listdir(self.windows) if name.endswith(".jsonl.gz") } diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py index 771b2acf7c07..f6fcf3c1f8f1 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_dgdr.py @@ -13,7 +13,9 @@ from __future__ import annotations +import dataclasses import gzip +import hashlib import itertools import json import math @@ -62,6 +64,8 @@ request_name, run_dgdr_sweep, spec_hash, + window_claim_path, + window_selection, ) from dynamo.planner.retune.live.reconciler import trace_pvc_source from dynamo.planner.retune.sweep import SweepRequest, score_summary @@ -94,6 +98,12 @@ GRACE_S = 120.0 TIMEOUT_S = 3000.0 LAUNCH_S = WINDOW_END_MS / 1000.0 + GRACE_S +WINDOW_START_MS = WINDOW_END_MS - int(INTERVAL_S * 1000) +AS_OF_MS = WINDOW_END_MS + int(GRACE_S * 1000) +WINDOW_FILE = ( + f"windows/llm/{WINDOW_END_MS}-" + f"{window_selection(WINDOW_START_MS, WINDOW_END_MS, AS_OF_MS)}.jsonl.gz" +) POLL_S = 5.0 DGD = DgdRef(name="llm", namespace="test-ns", uid="dgd-uid-1", dynamo_namespace="dyn") AGG_ENGINE = { @@ -623,7 +633,9 @@ def test_build_request_aggregated_grid_shard(): retune=retune, sla_ttft_ms=500, sla_itl_ms=30, + window_start_ms=WINDOW_START_MS, window_end_ms=WINDOW_END_MS, + as_of_ms=AS_OF_MS, shard=shard, spec_replicas={"aggregated": 3}, backend="vllm", @@ -642,7 +654,7 @@ def test_build_request_aggregated_grid_shard(): "source": { "persistentVolumeClaim": { "claimName": "request-traces", - "subPath": f"windows/llm/{WINDOW_END_MS}.jsonl.gz", + "subPath": WINDOW_FILE, } }, } @@ -713,7 +725,9 @@ def test_build_request_point_shard_uses_scalars_and_disagg_budget(): retune=retune, sla_ttft_ms=200, sla_itl_ms=20, + window_start_ms=WINDOW_START_MS, window_end_ms=WINDOW_END_MS, + as_of_ms=AS_OF_MS, shard=probe, spec_replicas={"prefill": 2, "decode": 4}, backend="vllm", @@ -749,7 +763,9 @@ def test_build_request_rejects_spec_replicas_it_cannot_pin(replicas, match): retune=make_retune(cfg), sla_ttft_ms=1, sla_itl_ms=1, + window_start_ms=WINDOW_START_MS, window_end_ms=WINDOW_END_MS, + as_of_ms=AS_OF_MS, shard=_shards()[0], spec_replicas=replicas, backend="vllm", @@ -769,7 +785,9 @@ def test_build_request_requires_the_three_dgdr_knobs(): retune=make_retune(cfg), sla_ttft_ms=1, sla_itl_ms=1, + window_start_ms=WINDOW_START_MS, window_end_ms=WINDOW_END_MS, + as_of_ms=AS_OF_MS, shard=shard, spec_replicas={"aggregated": 1}, backend="vllm", @@ -777,7 +795,9 @@ def test_build_request_requires_the_three_dgdr_knobs(): ) -def _body(shard_index=0, **kwargs): +def _body( + shard_index=0, *, window_start_ms=WINDOW_START_MS, as_of_ms=AS_OF_MS, **kwargs +): cfg = make_cfg(**kwargs) return build_request( dgd=DGD, @@ -785,7 +805,9 @@ def _body(shard_index=0, **kwargs): retune=make_retune(cfg), sla_ttft_ms=500, sla_itl_ms=30, + window_start_ms=window_start_ms, window_end_ms=WINDOW_END_MS, + as_of_ms=as_of_ms, shard=_shards()[shard_index], spec_replicas={"aggregated": 3}, backend="vllm", @@ -793,6 +815,47 @@ def _body(shard_index=0, **kwargs): ) +def test_window_file_is_named_by_its_end_and_selection_digest(): + key = f"{WINDOW_START_MS}|{WINDOW_END_MS}|{AS_OF_MS}".encode() + digest = hashlib.sha256(key).hexdigest()[:12] + assert window_selection(WINDOW_START_MS, WINDOW_END_MS, AS_OF_MS) == digest + path = window_claim_path( + make_cfg(windows_dir="retune/windows"), + DGD, + window_start_ms=WINDOW_START_MS, + window_end_ms=WINDOW_END_MS, + as_of_ms=AS_OF_MS, + ) + assert path == f"retune/windows/llm/{WINDOW_END_MS}-{digest}.jsonl.gz" + + +def _identity(**selection: int) -> tuple[Optional[str], str, str]: + """The window file, spec hash and name of a grid shard's Request.""" + + body = _body(**selection) + _, path = trace_pvc_source(body["spec"]) + return path, body["metadata"]["labels"][LABEL_SPEC], body["metadata"]["name"] + + +@pytest.mark.parametrize( + "selection", + [ + {"window_start_ms": WINDOW_END_MS - 1_800_000}, + {"as_of_ms": WINDOW_END_MS + 300_000}, + ], + ids=["start", "as_of"], +) +def test_request_identity_follows_the_trace_selection(selection): + same = _identity() + assert _identity() == same + assert same[0] == WINDOW_FILE + path, digest, name = _identity(**selection) + assert path is not None and path.startswith(f"windows/llm/{WINDOW_END_MS}-") + assert path != same[0] + assert digest != same[1] + assert name != same[2] + + def test_input_read_back_accepts_the_operator_lowering_and_finds_drift(): body = _body() body["metadata"]["namespace"] = "test-ns" @@ -947,9 +1010,7 @@ def test_fresh_window_creates_cuts_harvests_and_scores(tmp_path): ) # The window was cut once, into the claim path the Requests name. (cut,) = h.cutter.calls - assert cut["out_path"] == str( - tmp_path / "trace" / f"windows/llm/{WINDOW_END_MS}.jsonl.gz" - ) + assert cut["out_path"] == str(tmp_path / "trace" / WINDOW_FILE) assert cut["segments_root"] == "/trace/segments/llm" assert cut["start_ms"] == WINDOW_END_MS - 3_600_000 assert cut["end_ms"] == WINDOW_END_MS @@ -1019,6 +1080,55 @@ def test_window_adopted_as_succeeded_is_not_cut_again(tmp_path): assert all(shard["adopted"] for shard in second.details["shards"].values()) +@pytest.mark.parametrize( + "change", + [ + # Restarted with half the interval: the window ends at the same time + # but starts later. + { + "window_index": 15, + "window_start_ms": WINDOW_END_MS - INTERVAL_S * 500.0, + "history_start_ms": WINDOW_END_MS - INTERVAL_S * 500.0, + }, + # Restarted with a longer grace: the same window, a later cutoff. + {"as_of_ms": WINDOW_END_MS + 2 * GRACE_S * 1000.0}, + ], + ids=["interval", "grace"], +) +def test_changed_selection_at_a_shared_end_is_not_adopted(tmp_path, change): + h = Harness(tmp_path) + first = h.sweep(make_request()) + assert first.status == "ok", first.error + old = dict(h.kube.requests) + (first_cut,) = h.cutter.calls + + h.cutter = FakeCutter() + request = dataclasses.replace(make_request(), **change) + second = h.sweep(request) + + assert second.status == "ok", second.error + assert len(h.cutter.calls) == 1, "the new selection was not cut" + (cut,) = h.cutter.calls + assert (cut["start_ms"], cut["end_ms"], cut["as_of_ms"]) == ( + round(request.window_start_ms), + WINDOW_END_MS, + round(request.as_of_ms), + ) + assert cut["out_path"] != first_cut["out_path"] + assert not any(shard["adopted"] for shard in second.details["shards"].values()) + assert sorted(h.registry.deleted) == sorted(old) + new = [obj for name, obj in h.kube.requests.items() if name not in old] + assert len(new) == len(old) + new_files = {trace_pvc_source(obj["spec"])[1] for obj in new} + old_files = {trace_pvc_source(obj["spec"])[1] for obj in old.values()} + assert {str(tmp_path / "trace" / path) for path in new_files} == {cut["out_path"]} + assert new_files.isdisjoint(old_files) + new_specs = {obj["metadata"]["labels"][LABEL_SPEC] for obj in new} + old_specs = {obj["metadata"]["labels"][LABEL_SPEC] for obj in old.values()} + assert new_specs.isdisjoint(old_specs) + assert_scores(second, request) + + def test_lost_leadership_leaves_requests_for_the_next_leader(tmp_path): h = Harness(tmp_path) h.operator.max_phase = "started" @@ -1268,7 +1378,9 @@ def test_name_taken_by_a_foreign_request_fails_a_critical_shard(tmp_path): retune=h.retune, sla_ttft_ms=500, sla_itl_ms=30, + window_start_ms=WINDOW_START_MS, window_end_ms=WINDOW_END_MS, + as_of_ms=AS_OF_MS, shard=_shards()[0], spec_replicas={"aggregated": 3}, backend="vllm", @@ -1998,7 +2110,7 @@ def real_cut(*args, deadline_s, **kwargs): result = h.sweep(request, cut_window=real_cut) assert result.status == "ok", result.error assert result.num_requests == 5 - window_file = tmp_path / "trace" / "windows" / "llm" / f"{WINDOW_END_MS}.jsonl.gz" + window_file = tmp_path / "trace" / WINDOW_FILE with gzip.open(window_file, "rt", encoding="utf-8") as lines: assert len(lines.readlines()) == 5 _, claim_path = trace_pvc_source(h.created()[0]["spec"]) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py index c77c00d635ef..58988605bdfd 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_reconciler.py @@ -437,6 +437,10 @@ def window_path(reconciler: Reconciler, window: int) -> Path: return Path(reconciler.windows_root) / f"{window}.jsonl.gz" +def selection_path(reconciler: Reconciler, window: int, digest: str) -> Path: + return Path(reconciler.windows_root) / f"{window}-{digest}.jsonl.gz" + + def test_protected_windows_are_never_deleted(tmp_path: Path) -> None: reconciler, _, _ = make(tmp_path) last, due = window_ms(2), int((NOW - 60) * 1000) @@ -568,6 +572,54 @@ def test_unreferenced_window_needs_the_fallback_age(tmp_path: Path) -> None: assert young.exists() and not old.exists() and unknown.exists() +def test_windows_named_by_their_selection_are_protected_by_their_end( + tmp_path: Path, +) -> None: + reconciler, _, _ = make(tmp_path) + last, due = window_ms(2), int((NOW - 60) * 1000) + due_path = touch(selection_path(reconciler, due, "0123456789ab"), OLD) + stale_path = touch(selection_path(reconciler, window_ms(5), "0123456789ab"), OLD) + legacy_path = touch(window_path(reconciler, window_ms(6)), OLD) + + report = run_once(reconciler, last=last, due=due, pods=[frontend("fe-0")]) + + assert due_path.exists() # undecided and before its deadline + assert not stale_path.exists() and not legacy_path.exists() + assert report.deleted_windows == (window_ms(6), window_ms(5)) + + +def test_a_window_named_by_its_selection_goes_once_its_referrer_is_finished( + tmp_path: Path, +) -> None: + reconciler, kube, _ = make(tmp_path, keep_requests=10) + # Outside the protected range and before its deadline, so only its + # referrer decides. + window = int((NOW - 60) * 1000) + path = touch(selection_path(reconciler, window, "0123456789ab"), 10) + other_selection = touch(selection_path(reconciler, window, "ba9876543210"), 10) + kube.add_request( + "rt-a", + window, + state="running", + job_pods=1, + sub_path=f"windows/dgd/{path.name}", + ) + + def round_(): + return run_once(reconciler, last=window_ms(0), due=window_ms(0)) + + round_() + assert path.exists() and other_selection.exists() + + kube.add_request( + "rt-a", window, state="completed", sub_path=f"windows/dgd/{path.name}" + ) + report = round_() + assert report.deleted_windows == (window,) + assert not path.exists() + assert other_selection.exists() # unreferenced but young + + def test_windows_are_kept_when_requests_cannot_be_listed(tmp_path: Path) -> None: reconciler, kube, _ = make(tmp_path) path = touch(window_path(reconciler, window_ms(6)), OLD) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py index ccd2dfeb20e6..c686a1b42be9 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_window_cut.py @@ -859,10 +859,20 @@ def _reused(result, path): ) +def _header_comment(path): + data = path.read_bytes() + return json.loads(data[10 : data.index(b"\x00", 10)]) + + def test_published_window_is_reused_without_reading_the_trace(tmp_path): root = _random_trace(tmp_path / "segments", 5) first, out = _cut(root, tmp_path) published = out.read_bytes() + assert _header_comment(out)["selection"] == { + "start_ms": START, + "end_ms": END, + "as_of_ms": AS_OF, + } # A later leader adopting the window must not fail it on segments that # changed, broke or were collected since it was published. shutil.rmtree(root) @@ -874,6 +884,40 @@ def test_published_window_is_reused_without_reading_the_trace(tmp_path): assert _temp_files(out.parent) == [] +@pytest.mark.parametrize( + "selection", + [{"start": START + 1000}, {"as_of": AS_OF + 100}], + ids=["start", "as_of"], +) +def test_published_window_of_another_selection_is_not_reused(tmp_path, selection): + root = _random_trace(tmp_path / "segments", 5) + _, out = _cut(root, tmp_path) + published = out.read_bytes() + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path, **selection) + + assert caught.value.reason == "window_cut_failed" + assert f"was cut for [{START}, {END}) as of {AS_OF}, not [" in str(caught.value) + assert out.read_bytes() == published + assert _temp_files(out.parent) == [] + + +def test_published_window_without_a_selection_is_not_reused(tmp_path): + root = _random_trace(tmp_path / "segments", 5) + _, out = _cut(root, tmp_path) + data = out.read_bytes() + end = data.index(b"\x00", 10) + counts = _header_comment(out) + del counts["selection"] + out.write_bytes(data[:10] + json.dumps(counts).encode("ascii") + data[end:]) + + with pytest.raises(WindowCutError) as caught: + _cut(root, tmp_path) + assert caught.value.reason == "window_cut_failed" + assert "holds no selection" in str(caught.value) + + def test_a_window_file_without_cut_counts_fails(tmp_path): root = _random_trace(tmp_path / "segments", 5) out = _out(tmp_path) @@ -885,6 +929,21 @@ def test_a_window_file_without_cut_counts_fails(tmp_path): assert "holds no cut counts" in str(caught.value) +def _lose_link_race_to(monkeypatch, winner_out): + """Make the cut's link find ``winner_out`` already published at its target.""" + + real_link = os.link + linked = [] + + def racing_link(source, target): + linked.append(os.path.basename(source)) + real_link(winner_out, target) # another process published first + return real_link(source, target) + + monkeypatch.setattr(os, "link", racing_link) + return linked + + def test_link_race_reports_the_winners_counts(tmp_path, monkeypatch): root = _random_trace(tmp_path / "segments", 6) winner_out = _out(tmp_path, "winner.jsonl.gz") @@ -898,15 +957,7 @@ def test_link_race_reports_the_winners_counts(tmp_path, monkeypatch): ) assert winner.num_requests != _golden(root, tmp_path)[0].num_requests out = _out(tmp_path) - real_link = os.link - linked = [] - - def racing_link(source, target): - linked.append(os.path.basename(source)) - real_link(winner_out, target) # another process published first - return real_link(source, target) - - monkeypatch.setattr(os, "link", racing_link) + linked = _lose_link_race_to(monkeypatch, winner_out) result = cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) @@ -916,6 +967,22 @@ def racing_link(source, target): assert _temp_files(out.parent) == [] +def test_link_race_won_by_another_selection_fails(tmp_path, monkeypatch): + root = _random_trace(tmp_path / "segments", 6) + winner_out = _out(tmp_path, "winner.jsonl.gz") + cut_window_inprocess(root, START + 1000, END, AS_OF, winner_out, max_rows=10**6) + out = _out(tmp_path) + linked = _lose_link_race_to(monkeypatch, winner_out) + + with pytest.raises(WindowCutError) as caught: + cut_window_inprocess(root, START, END, AS_OF, out, max_rows=10**6) + + assert caught.value.reason == "window_cut_failed" + assert f"was cut for [{START + 1000}, {END}) as of {AS_OF}" in str(caught.value) + assert linked and out.read_bytes() == winner_out.read_bytes() + assert _temp_files(out.parent) == [] + + def test_link_enoent_is_window_cut_failed(tmp_path, monkeypatch): root = _random_trace(tmp_path / "segments", 7) out = _out(tmp_path) diff --git a/deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json b/deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json index d3be2a10bd42..59560e0daec5 100644 --- a/deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json +++ b/deploy/operator/internal/aisimulate/testdata/trace_request_lowering.golden.json @@ -17,7 +17,7 @@ "source": { "persistentVolumeClaim": { "claimName": "request-traces", - "subPath": "windows/llm/1793000000000.jsonl.gz" + "subPath": "windows/llm/1793000000000-ba081802aba3.jsonl.gz" } } } diff --git a/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go b/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go index 4fae5536de87..ceaa601ab181 100644 --- a/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go +++ b/deploy/operator/internal/controller/dgdr_v1beta2_trace_golden_envtest_test.go @@ -24,8 +24,9 @@ const traceRequestLoweringGoldenPath = "../aisimulate/testdata/trace_request_low // liveRetuneShardRequestSpec is the spec of one grid shard the live-retune planner submits: its // build_request for an aggregated vLLM deployment with 3 replicas of tensor parallelism 2, the window -// ending at 1793000000000 ms, and a 2x2x2 router grid. It omits workload.trace.format, which the -// CRD schema defaults. Change it together with the planner and regenerate the golden file. +// [1792996400000, 1793000000000) ms as of 1793000120000 ms, whose selection digest ends the subPath, +// and a 2x2x2 router grid. It omits workload.trace.format, which the CRD schema defaults. Change it +// together with the planner and regenerate the golden file. const liveRetuneShardRequestSpec = `{ "model": {"name": "Qwen/Qwen3-32B"}, "backends": ["vllm"], @@ -35,7 +36,7 @@ const liveRetuneShardRequestSpec = `{ "source": { "persistentVolumeClaim": { "claimName": "request-traces", - "subPath": "windows/llm/1793000000000.jsonl.gz" + "subPath": "windows/llm/1793000000000-ba081802aba3.jsonl.gz" } } } diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index c610c2600af5..1c6f24aa7b88 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -28,8 +28,12 @@ runs in retune-only mode and never scales. 1. **Record.** Each frontend writes a `request_end` record per request to its own directory on a shared ReadWriteMany claim (`///`). 2. **Cut a window.** At `L_k = window end + grace_seconds`, the Planner merges the records of the - window into one canonical trace file, `//.jsonl.gz`, in a - subprocess bounded by `cut_budget_seconds`. + window into one canonical trace file, + `//-.jsonl.gz`, in a subprocess bounded by + `cut_budget_seconds`. `` is a digest of the window's start, end and cutoff + (`window end + grace_seconds`), so a Planner restarted with another `interval_seconds` or + `grace_seconds` cuts its own file and creates its own Requests instead of reusing those of a + window with the same end. 3. **Search.** The Planner splits the router grid into at most `run.max_shards` Cartesian shards of at most 100 points and creates one DGDR v1beta2 Request per shard. Each Request replays the window file on the pinned `retune.dgdr.engine` with the replica counts of the DGD spec. One more diff --git a/docs/fern/pages/reference/components/planner-configuration.mdx b/docs/fern/pages/reference/components/planner-configuration.mdx index ac3612f9298a..33cd2bb24810 100644 --- a/docs/fern/pages/reference/components/planner-configuration.mdx +++ b/docs/fern/pages/reference/components/planner-configuration.mdx @@ -721,7 +721,7 @@ The Planner runs the builtin plugin pipeline by default. `scheduling` controls p Claim-relative directory of the frontend trace segments, written as `///`. `/`-separated names without empty, `.` or `..` segments. `segments_dir` and `windows_dir` must not contain one another. - Claim-relative directory of the window files, written as `//.jsonl.gz`. Same rules as `segments_dir`. + Claim-relative directory of the window files, written as `//-.jsonl.gz`, where `` is a digest of the window's start, end and cutoff. Same rules as `segments_dir`. Window phase. Window `k` is `[phase_seconds + k × interval_seconds, phase_seconds + (k+1) × interval_seconds)` in epoch seconds, so a restarted Planner continues the same windows. Must be 0 or greater. From 1c666b7cc8544040190d0c964eef3d6ba4a5e419 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 13:23:01 -0800 Subject: [PATCH 095/100] fix(planner): apply the worker shell default and reject opaque worker args in P4 P4 reads worker commands the way the operator runs them and fails closed when a value it cannot read could form a router option. - A worker without a command runs /bin/sh -c (the operator's getCommonContainer default) unless the pod runs the native "runtime" sidecar, so its one-string script is split into shell words before the router parser runs. - Env is readable, inert (POD_*, non-annotation fieldRef, resourceFieldRef) or dangerous (ConfigMap, Secret, annotation fieldRef, and names only an envFrom source may define). In exec form a dangerous reference passes only attached to its option (--opt=$(VAR)) or as the value of --speculative-config. - Shell scripts are scanned after Kubernetes substitution: readable env is substituted, an unquoted dangerous $VAR fails, and a quoted one fails as a standalone argument of the worker module, eval or a nested sh -c. A script's own assignment does not clear explicit dangerous env. Shells behind launchers (env, tini, dumb-init) and shell options with operands (-o pipefail) are recognized. - Command substitution and backticks are not evaluated; the live retune guide's Limitations table lists what P4 does not read. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../planner/retune/live/frontend_config.py | 922 ++++++++++++++++-- .../unit/test_live_retune_frontend_config.py | 484 ++++++++- .../auto-deployment/planner-live-retune.md | 7 +- 3 files changed, 1295 insertions(+), 118 deletions(-) diff --git a/components/src/dynamo/planner/retune/live/frontend_config.py b/components/src/dynamo/planner/retune/live/frontend_config.py index c49a6ef7c71b..b1a765405bc1 100644 --- a/components/src/dynamo/planner/retune/live/frontend_config.py +++ b/components/src/dynamo/planner/retune/live/frontend_config.py @@ -17,17 +17,26 @@ parses them with the frontend's own parser (``build_frontend_parser``), which reads env defaults from that env alone, so the planner process's environment is neither read nor modified. A shell-wrapped command, a - ``--router-policy-config`` file, router env set through ``valueFrom`` and + ``--router-policy-config`` file, router env the planner cannot read and an argument whose unreadable ``$(VAR)`` could set a router flag make the - frontend unparseable. ``envFrom`` sources are assumed to set no router - env; ``evaluate_premises`` warns about them, and P14 compares the - routers' policy parameters with the parsed startup. + frontend unparseable. +- Env set through ``valueFrom`` is unreadable. A ConfigMap, Secret or + annotation value could also be or create an option ("dangerous"), as + could a ``$(VAR)`` the env leaves undefined that an ``envFrom`` source + may define (its ``prefix`` plus a ConfigMap or Secret key). Any other + ``fieldRef`` (the operator's ``POD_*`` among them) and a + ``resourceFieldRef`` never start with ``-`` or hold whitespace, so they + stay unreadable values but never stand for an option ("inert"). + ``envFrom`` sources are otherwise assumed to set no router env; + ``evaluate_premises`` warns about them, and P14 compares the routers' + policy parameters with the parsed startup. - ``evaluate_premises`` checks P4 and P7-P10 and warns on P6 and P7; P14 is the router actuator's. P4 also parses every worker component's router - flags and env the same way: a worker that resolves a router mode - advertises its own router configuration, which replaces the frontend's - for its worker set. ``p4_ok`` alone decides whether a retune controller - can be built. + flags and env the same way (``_worker_router_errors``): a worker that + resolves a router mode advertises its own router configuration, which + replaces the frontend's for its worker set, and one where dangerous env + could stand for an option fails too. ``p4_ok`` alone decides whether a + retune controller can be built. Every function is pure (a dict in, a value out, no Kubernetes calls) and never reads the planner's own environment. @@ -77,12 +86,32 @@ DEFAULT_FRONTEND_COMMAND = ("python3",) DEFAULT_FRONTEND_ARGS = ("-m", "dynamo.frontend") FRONTEND_MODULE = "dynamo.frontend" +# The operator's worker invocation (component_common.go getCommonContainer), +# kept by graph.go's merge under an empty authored command. A native-sidecar +# main, one whose pod template has an init container named RUNTIME_CONTAINER, +# skips that base container (graph.go: native-sidecar engines retain their +# image entrypoint), so its args stay an exec-form argv. +DEFAULT_WORKER_COMMAND = ("/bin/sh", "-c") +# consts.go RuntimeContainerName. +RUNTIME_CONTAINER = "runtime" +# Worker options known to take exactly one value, so an unreadable value +# right after one is that value: were it itself an option, the listed one +# would lack its value and the worker would fail at startup. The worker's +# router parser knows only the router flags, not the arity of engine or +# runtime flags (--enforce-eager takes none, --connector a variable number), +# so only these exempt a value; the recipes need no other. Before adding an +# option, check its nargs and action in every backend that accepts it. +_WORKER_VALUE_OPTIONS = frozenset({"--speculative-config"}) # Env the operator injects into every component (component_common.go # getCommonContainer); user env of the same name overrides it. ENV_PARENT_DGD_NAME = "DYN_PARENT_DGD_K8S_NAME" ENV_PARENT_DGD_NAMESPACE = "DYN_PARENT_DGD_K8S_NAMESPACE" -_OPERATOR_FIELD_REF_ENV = ("POD_NAME", "POD_NAMESPACE", "POD_UID") +_OPERATOR_FIELD_REF_ENV = { + "POD_NAME": "metadata.name", + "POD_NAMESPACE": "metadata.namespace", + "POD_UID": "metadata.uid", +} # The frontend's request trace recipe (P10). ENV_POD_NAME = "DYN_POD_NAME" @@ -168,12 +197,40 @@ # runtime_status.go ResolveComponentRuntimeStatus, in its priority order. _MODEL_FLAGS = ("--served-model-name", "--model-name", "--model", "--model-path") _SHELLS = frozenset({"sh", "bash", "dash", "zsh", "ash", "ksh"}) +# Shell option letters that take the next argument (-o pipefail, -O extglob), +# and bash's long options that do. +_SHELL_OPERAND_LETTERS = frozenset("oO") +_SHELL_LONG_OPTIONS_WITH_VALUE = frozenset({"--init-file", "--rcfile"}) # Python options that may precede ``-m``; the second set takes a value. _PYTHON_FLAGS = frozenset({"-u", "-B", "-E", "-I", "-O", "-OO", "-P", "-q", "-s", "-S"}) _PYTHON_FLAGS_WITH_VALUE = frozenset({"-X", "-W"}) -# Marks an argument part that references env the planner cannot read. -_OPAQUE = "\x00opaque:" -_OPAQUE_REPR = repr(_OPAQUE)[1:-1] +# Python's -m, alone or closing a cluster of flags (-um), with its module +# attached or in the next word. +_PYTHON_MODULE_OPTION = re.compile(r"-[A-Za-z]*?m(.*)") +# Marks an argument part that references env the planner cannot read, as +# "\x00": _OPAQUE where the value could be or create an +# option, _INERT where it cannot. +_UNREADABLE = "\x00unreadable" +_OPAQUE = _UNREADABLE + ":" +_INERT = _UNREADABLE + "-inert:" +# A marker, also as argparse quotes it with repr. +_MARKER = re.compile(r"(?:\x00|\\x00)unreadable(?:-inert)?:(.*?)(?:\x00|\\x00|$)") +# The keys a ConfigMap or Secret may hold, so what an envFrom source defines. +_ENV_FROM_KEY = re.compile(r"[-._a-zA-Z0-9]+") + +# Shell syntax the worker script scan reads. +_SHELL_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +_SHELL_ASSIGNMENT = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)\+?=") +_SHELL_BLANKS = frozenset(" \t") +_SHELL_OPERATORS = frozenset("\n;&|()<>") +# Reserved words that may precede a command's name. +_SHELL_RESERVED = frozenset( + {"!", "{", "do", "elif", "else", "if", "then", "time", "until", "while"} +) +_SHELL_DECLARATIONS = frozenset({"declare", "export", "local", "readonly", "typeset"}) +# Commands that parse a quoted word again or turn it into env, so that one +# word can still reach the worker's options or env. +_SHELL_REPARSING = frozenset({".", "env", "eval", "export", "source"}) class FrontendParseError(ValueError): @@ -422,14 +479,30 @@ class _Env: ``entries`` is the merged raw env keyed by name (operator-injected, then graph ``env``, then the container's; later wins). ``values`` holds each name whose value the planner can compute, after ``$(VAR)`` expansion; - ``opaque`` the names it cannot (``valueFrom``, or a reference to one). + ``opaque`` the names it cannot (``valueFrom``, or a reference to + unreadable env), and ``inert`` those of them whose value can never be or + create an option. """ entries: Mapping[str, Mapping[str, Any]] values: Mapping[str, str] opaque: frozenset[str] + inert: frozenset[str] env_from: tuple[Mapping[str, Any], ...] + @property + def env_from_prefixes(self) -> tuple[str, ...]: + return _env_from_prefixes(self.env_from) + + def expand(self, value: str) -> str: + return _expand( + value, + self.values, + self.opaque, + inert=self.inert, + env_from_prefixes=self.env_from_prefixes, + ) + @dataclass(frozen=True) class _Analysis: @@ -472,6 +545,8 @@ def _analyze(dgd: Mapping[str, Any], planner_mode: str) -> _Analysis: "argv": list(argv), "values": dict(env.values), "opaque": sorted(env.opaque), + "inert": sorted(env.inert), + "env_from_prefixes": list(env.env_from_prefixes), } ), worker_types(planner_mode), @@ -519,6 +594,8 @@ def _parse_cached(key: str, wts: tuple[str, ...]) -> _Parsed: inputs["values"], frozenset(inputs["opaque"]), wts, + inert=frozenset(inputs["inert"]), + env_from_prefixes=tuple(inputs["env_from_prefixes"]), ) except FrontendParseError as error: return _Parsed(error=str(error)) @@ -537,6 +614,9 @@ def _parse( values: Mapping[str, str], opaque: frozenset[str], wts: tuple[str, ...], + *, + inert: frozenset[str] = frozenset(), + env_from_prefixes: tuple[str, ...] = (), ) -> _Parsed: args = _frontend_args(argv) parser, registered = _frontend_parser(values) @@ -553,14 +633,17 @@ def relevant(name: str) -> bool: if hidden: raise FrontendParseError( f"P4: frontend router env {', '.join(hidden)} is set through valueFrom " - "(or references such a value); the planner cannot read it, so set " - "it with a literal value" + "(or references such a value, or a $(VAR) an envFrom source may " + "define); the planner cannot read it, so set it with a literal value" ) # An unreadable $(VAR) becomes a marker the parser carries as a value: a # non-router flag keeps it unconverted, a router flag fails the parse, - # and one left over where a flag could stand fails below. - expanded = [_expand_arg(arg, values, opaque) for arg in args] + # and a dangerous one left over where a flag could stand fails below. + expanded = [ + _expand(arg, values, opaque, inert=inert, env_from_prefixes=env_from_prefixes) + for arg in args + ] namespace, unknown = parser.parse_known_args(expanded) hidden_flag = _opaque_flag_position(unknown) if hidden_flag is not None: @@ -573,7 +656,7 @@ def relevant(name: str) -> bool: unresolved = sorted( dest for dest, value in vars(namespace).items() - if dest in _RELEVANT_DESTS and _OPAQUE in str(value) + if dest in _RELEVANT_DESTS and _UNREADABLE in str(value) ) if unresolved: raise FrontendParseError( @@ -760,7 +843,8 @@ def _component_env(dgd: Mapping[str, Any], spec: Mapping[str, Any]) -> _Env: The operator injects ``DYN_PARENT_DGD_K8S_*`` and ``POD_*``, MergeEnvs overlays graph ``env`` and the main container's ``env`` and sorts the - result by name; Kubernetes expands ``$(VAR)`` against earlier names. + result by name; Kubernetes expands ``$(VAR)`` against earlier names and + every ``envFrom`` key. """ metadata = dgd.get("metadata") or {} @@ -774,52 +858,103 @@ def _component_env(dgd: Mapping[str, Any], spec: Mapping[str, Any]) -> _Env: "value": str(metadata.get("namespace", "")), }, } - for name in _OPERATOR_FIELD_REF_ENV: - entries[name] = {"name": name, "valueFrom": {"fieldRef": {}}} + for name, path in _OPERATOR_FIELD_REF_ENV.items(): + entries[name] = {"name": name, "valueFrom": {"fieldRef": {"fieldPath": path}}} main = _main_container(spec) for entry in [*_graph_env(dgd), *(main.get("env") or [])]: if isinstance(entry, Mapping) and entry.get("name"): entries[str(entry["name"])] = entry + env_from = tuple( + source for source in main.get("envFrom") or [] if isinstance(source, Mapping) + ) + prefixes = _env_from_prefixes(env_from) values: dict[str, str] = {} opaque: set[str] = set() + inert: set[str] = set() for name in sorted(entries): entry = entries[name] - expanded = ( - None - if entry.get("valueFrom") is not None - else _expand(str(entry.get("value") or ""), values, opaque) - ) - if expanded is None: + source = entry.get("valueFrom") + if source is not None: opaque.add(name) - else: + if _inert_source(source): + inert.add(name) + continue + expanded = _expand( + str(entry.get("value") or ""), + values, + opaque, + inert=inert, + env_from_prefixes=prefixes, + ) + if _UNREADABLE not in expanded: values[name] = expanded - env_from = tuple( - source for source in main.get("envFrom") or [] if isinstance(source, Mapping) - ) + continue + opaque.add(name) + # Literal text around inert values may still make an option. + if _OPAQUE not in expanded and not ( + expanded.startswith("-") or any(char.isspace() for char in expanded) + ): + inert.add(name) return _Env( entries=entries, values=values, opaque=frozenset(opaque), + inert=frozenset(inert), env_from=env_from, ) +def _inert_source(source: Any) -> bool: + """Whether a ``valueFrom`` value can never be or create an option. + + A fieldRef other than an annotation (a name, namespace, uid, label, IP, + node or service account name) and a resourceFieldRef (a quantity) never + start with ``-`` or hold whitespace; a ConfigMap or Secret key and an + annotation can hold anything. + """ + + if not isinstance(source, Mapping): + return False + if isinstance(source.get("resourceFieldRef"), Mapping): + return True + field_ref = source.get("fieldRef") + if not isinstance(field_ref, Mapping): + return False + return not str(field_ref.get("fieldPath") or "").startswith("metadata.annotations") + + +def _env_from_prefixes(env_from: Iterable[Mapping[str, Any]]) -> tuple[str, ...]: + return tuple(sorted({str(source.get("prefix") or "") for source in env_from})) + + +def _env_from_may_define(name: str, prefixes: Iterable[str]) -> bool: + """Whether an ``envFrom`` source with one of ``prefixes`` could define ``name``.""" + + return any( + name.startswith(prefix) and _ENV_FROM_KEY.fullmatch(name[len(prefix) :]) + for prefix in prefixes + ) + + def _expand( value: str, values: Mapping[str, str], opaque: Iterable[str], *, - mark_opaque: bool = False, -) -> Optional[str]: - """Kubernetes ``$(VAR)`` expansion. - - ``$$`` escapes a ``$`` and a reference to an undefined name stays as - written. A reference to an opaque name makes the result ``None``, or, - with ``mark_opaque``, is replaced by a marker. + inert: Iterable[str] = (), + env_from_prefixes: Iterable[str] = (), +) -> str: + """Kubernetes ``$(VAR)`` expansion, marking what the planner cannot read. + + ``$$`` escapes a ``$``. A reference to an ``opaque`` name, or to an + undefined one an ``envFrom`` source may define, is replaced by a marker: + ``_INERT`` for an ``inert`` name, ``_OPAQUE`` otherwise. Any other + undefined reference stays as written. """ - hidden = set(opaque) + hidden, harmless = set(opaque), set(inert) + prefixes = tuple(env_from_prefixes) out: list[str] = [] index = 0 while index < len(value): @@ -831,10 +966,12 @@ def _expand( close = value.find(")", index + 2) if close != -1: name = value[index + 2 : close] - if name in hidden: - if not mark_opaque: - return None - out.append(f"{_OPAQUE}{name}\x00") + if name in harmless: + out.append(f"{_INERT}$({name})\x00") + elif name in hidden or ( + name not in values and _env_from_may_define(name, prefixes) + ): + out.append(f"{_OPAQUE}$({name})\x00") else: out.append(values.get(name, value[index : close + 1])) index = close + 1 @@ -844,54 +981,66 @@ def _expand( return "".join(out) -def _expand_arg(arg: str, values: Mapping[str, str], opaque: Iterable[str]) -> str: - """Expand one argument, marking opaque references for the parser.""" - - expanded = _expand(arg, values, opaque, mark_opaque=True) - assert expanded is not None - return expanded - - -def _opaque_flag_position(unknown: Sequence[str]) -> Optional[str]: - """A left-over argument whose unreadable env could stand for a flag. - - The frontend parser consumes the value of every flag it knows, so a - marked argument it leaves over sits where a flag could, unless it - follows a left-over option (a flag the frontend hands to its chat - processor, and this its value). A left-over option whose name is - marked always could. +def _opaque_flag_position( + unknown: Sequence[str], *, value_options: Optional[frozenset[str]] = None +) -> Optional[str]: + """A left-over argument whose dangerous env could stand for a flag. + + ``unknown`` is what the frontend parser, or a worker's router parser, + leaves over in exec form. The parser consumes the value of every flag it + knows, so a dangerous argument it leaves over sits where a flag could, + unless it is the value of a left-over option, and a left-over option + whose name is unreadable always could. Inert env never stands for a + flag, and ``--option=$(VAR)`` is the option's value whatever it holds. + + A reference expands to exactly one argument, so it is a value after an + option that takes exactly one: one that starts with ``-`` makes that + option's own parser fail at startup. With ``value_options`` unset, every + left-over option counts as one (the frontend hands them to its chat + processor); a worker passes the options known to take one value, since + its router parser cannot tell an engine flag that takes none + (``--enforce-eager``) or optional values (``--connector``), which the + worker would let the reference follow as a flag. """ + def takes_value(option: Optional[str]) -> bool: + if option is None: + return False + if value_options is not None: + return option in value_options + return option.startswith("-") and "=" not in option + previous: Optional[str] = None for token in unknown: if token.startswith("-"): - if _OPAQUE in token.split("=", 1)[0]: + if _UNREADABLE in token.split("=", 1)[0]: return token - elif token.startswith(_OPAQUE) and not ( - previous is not None - and previous.startswith("-") - and "=" not in previous - and _OPAQUE not in previous - ): + elif token.startswith(_OPAQUE) and not takes_value(previous): return token previous = token return None +def _env_from_note( + reference: Optional[str], opaque: frozenset[str], prefixes: Sequence[str] +) -> str: + """Says why a reference is unreadable when only an envFrom source may define it.""" + + name = (reference or "").strip("$()") + if name in opaque or not _env_from_may_define(name, prefixes): + return "" + return " (the container's env leaves it to an envFrom source)" + + def _opaque_reference(text: str) -> Optional[str]: - """``$(NAME)`` for the first opaque marker in ``text``, or ``None``. + """The reference (``$(NAME)``, ``$NAME``) of the first marker in ``text``. argparse quotes a rejected value with ``repr``, so the marker may appear escaped. """ - for marker, end_marker in ((_OPAQUE, "\x00"), (_OPAQUE_REPR, "\\x00")): - index = text.find(marker) - if index != -1: - start = index + len(marker) - end = text.find(end_marker, start) - return f"$({text[start:end] if end != -1 else text[start:]})" - return None + match = _MARKER.search(text) + return None if match is None else match.group(1) def _env_from_may_set_router_env(source: Mapping[str, Any]) -> bool: @@ -935,7 +1084,7 @@ def _worker_parse_error_message(message: str) -> str: class _ArgsParser(argparse.ArgumentParser): """An ``ArgumentParser`` that raises instead of printing and exiting. - A value carrying the opaque marker skips type conversion and the + A value carrying an unreadable marker skips type conversion and the choices check unless its destination is in ``relevant``, so an unreadable value of a flag that does not matter here parses, while one of a flag that does fails with a message naming the reference. @@ -962,7 +1111,7 @@ def exit(self, status: int = 0, message: Optional[str] = None): # type: ignore[ def _opaque_irrelevant(self, action: argparse.Action, value: Any) -> bool: return ( isinstance(value, str) - and _OPAQUE in value + and _UNREADABLE in value and action.dest not in self._relevant ) @@ -1080,6 +1229,12 @@ def _worker_router_errors( ``DYN_ROUTER_MODE``, from its container or the graph env) puts a ``RouterConfig`` in its model card, and the frontend routes that worker set with it instead of its own flags; replay would model the frontend's. + A worker whose mode the planner cannot rule out fails too: one where + dangerous env could stand for an option. In exec form that is a + reference neither attached to its option (``--option=$(VAR)``) nor + after one of ``_WORKER_VALUE_OPTIONS``; in shell form any dangerous + ``$(VAR)``, which Kubernetes substitutes before the shell parses, and + the script references ``_shell_expand`` marks. """ errors: list[str] = [] @@ -1097,6 +1252,7 @@ def _worker_router_errors( "shell": shell, "values": dict(env.values), "opaque": sorted(env.opaque), + "env_from_prefixes": list(env.env_from_prefixes), } ) ) @@ -1130,6 +1286,7 @@ def _worker_router_cached(key: str) -> Optional[str]: bool(inputs["shell"]), inputs["values"], frozenset(inputs["opaque"]), + tuple(inputs["env_from_prefixes"]), ) except FrontendParseError as error: return str(error) @@ -1146,6 +1303,7 @@ def _worker_router_mode( shell: bool, values: Mapping[str, str], opaque: frozenset[str], + env_from_prefixes: Sequence[str], ) -> Optional[str]: parser, registered = _worker_router_parser(values) mode_env = sorted( @@ -1155,8 +1313,8 @@ def _worker_router_mode( if hidden: return ( f"sets {', '.join(hidden)} through valueFrom (or a reference to such " - "a value), so the planner cannot tell whether it advertises a router " - "configuration" + "a value, or a $(VAR) an envFrom source may define), so the planner " + "cannot tell whether it advertises a router configuration" ) # A shell script can also export the env inline. inline = sorted( @@ -1165,41 +1323,600 @@ def _worker_router_mode( ) if inline: return f"sets {', '.join(inline)} in its shell command" - namespace, _ = parser.parse_known_args(list(argv)) + namespace, remainder = parser.parse_known_args(list(argv)) mode = getattr(namespace, _ROUTER_MODE_DEST, None) - if mode is None: + if mode is not None: + return ( + f"advertises its own router configuration (router mode {mode!r} from " + f"--router-mode or {ENV_ROUTER_MODE})" + ) + if shell: + # Kubernetes substitutes $(VAR) into the script before the shell + # parses it, so a dangerous one can become several words anywhere; + # _shell_expand marks only the shell's own references that can. + opaque_word = next((word for word in argv if _OPAQUE in word), None) + if opaque_word is None: + return None + reference = _opaque_reference(opaque_word) + return ( + f"has a shell script that references {reference} where it could " + "become an option, and the planner cannot read it" + f"{_env_from_note(reference, opaque, env_from_prefixes)}, so it could " + "select a router mode; give the value literally, quote the reference " + '("$VAR") outside the worker\'s arguments, or attach it to its ' + 'option (--option="$VAR")' + ) + hidden_flag = _opaque_flag_position(remainder, value_options=_WORKER_VALUE_OPTIONS) + if hidden_flag is None: return None + reference = _opaque_reference(hidden_flag) return ( - f"advertises its own router configuration (router mode {mode!r} from " - f"--router-mode or {ENV_ROUTER_MODE})" + f"has an argument that references {reference}, which the planner cannot " + f"read{_env_from_note(reference, opaque, env_from_prefixes)} and which " + "is neither attached to an option nor the value of one known to take a value " + f"({', '.join(sorted(_WORKER_VALUE_OPTIONS))}), so it could select a " + "router mode; give the value literally or attach it to its option " + "(--option=$(VAR))" ) def _worker_argv(component: Mapping[str, Any], env: _Env) -> tuple[list[str], bool]: """A worker's argv after ``$(VAR)`` expansion, and whether a shell runs it. - Unreadable references stay marked. A shell-wrapped command is split - into the script's words, so the flags of the command it runs are seen. + An empty command is the operator's ``/bin/sh -c``, except for a + native-sidecar main, which keeps its image entrypoint and so runs its + args. A shell runs the command when it starts with one, or when a + shell's ``-c`` follows a launcher (``env bash -c``, ``tini -- sh -c``); + then its script's own references are resolved (``_shell_expand``) and + every part is split into words, so the flags of the commands it runs are + seen. Unreadable ``$(VAR)`` references stay marked. A standalone ``--`` + is dropped: a launcher consumes it, while the router parser would read + every later flag as a positional. """ main = _main_container(component) - parts: list[str] = [] - for part in [*(main.get("command") or []), *(main.get("args") or [])]: - expanded = _expand(str(part), env.values, env.opaque, mark_opaque=True) - assert expanded is not None - parts.append(expanded) - shell = bool(parts) and ( - posixpath.basename(parts[0]) in _SHELLS or parts[1:2] == ["-c"] + command = list(main.get("command") or []) + if not command and not _native_sidecar(component): + command = list(DEFAULT_WORKER_COMMAND) + parts = [env.expand(str(part)) for part in [*command, *(main.get("args") or [])]] + script = _shell_script_index(parts) + shell = script is not None or ( + bool(parts) + and (posixpath.basename(parts[0]) in _SHELLS or parts[1:2] == ["-c"]) ) - if not shell: - return parts, False - words: list[str] = [] - for part in parts: - try: - words.extend(shlex.split(part)) - except ValueError: - words.extend(part.split()) - return words, True + words = parts + if shell: + words = [] + for index, part in enumerate(parts): + if index == script: + part = _shell_expand(part, env) + try: + words.extend(shlex.split(part)) + except ValueError: + words.extend(part.split()) + return [word for word in words if word != "--"], shell + + +def _native_sidecar(component: Mapping[str, Any]) -> bool: + """Whether the operator runs the component's Dynamo runtime as a native sidecar. + + v1beta1_helpers.go GetComponentDynamoSidecar: an init container named + ``runtime`` in the pod template, or in the leader role's. + """ + + templates = [component] + for role in component.get("roles") or []: + if isinstance(role, Mapping) and role.get("name") == "leader": + templates.append(role) + return any( + isinstance(container, Mapping) and container.get("name") == RUNTIME_CONTAINER + for template in templates + for container in _pod_spec(template).get("initContainers") or [] + ) + + +def _shell_script_index(parts: Sequence[str]) -> Optional[int]: + """The index of the script a shell's ``-c`` runs in ``parts``, if any. + + It is ``len(parts)`` when ``parts`` end before the script. + """ + + for index, part in enumerate(parts): + if posixpath.basename(part) in _SHELLS: + script = _shell_options_end(parts, index + 1) + if script is not None: + return script + return None + + +def _shell_options_end(parts: Sequence[str], index: int) -> Optional[int]: + """Where the options a shell reads from ``index`` on end, if they include ``-c``. + + The shell reads ``-`` and ``+`` clusters up to ``--``, ``-`` or a word + that is none; each ``o`` or ``O`` in a cluster takes the next argument + (``-euo pipefail``). With ``-c`` the script is the first word after them. + """ + + command = False + while index < len(parts): + option = parts[index] + if option[:1] not in ("-", "+"): + break + index += 1 + if option in ("-", "--"): + break + if option.startswith("--"): + if option in _SHELL_LONG_OPTIONS_WITH_VALUE: + index += 1 + continue + command |= option.startswith("-") and "c" in option + index += sum(letter in _SHELL_OPERAND_LETTERS for letter in option[1:]) + return index if command else None + + +def _shell_expand(script: str, env: _Env) -> str: + """A worker script with the shell's own parameter references resolved. + + Runs on the text the shell parses, after Kubernetes ``$(VAR)`` + substitution, and keeps it apart from each ``$NAME``, ``${NAME}`` and + ``${NAMEword}`` outside single quotes, comments and heredocs: + + - literal env becomes its value as the shell passes it on (split into + words when unquoted), so the router parser sees an option it holds; + - dangerous env becomes ``_OPAQUE`` where its value could be or create + an option of the worker: unquoted, which the shell splits into words; + anywhere in an argument of ``eval`` or of a shell given ``-c`` (also + behind a launcher), which parse it again; or quoted as a whole + argument of the command that runs the worker (``-m dynamo.``) or + of one that turns it into code or env (source, env, export, a shell), + unless it follows one of ``_WORKER_VALUE_OPTIONS``. An assignment + neither splits nor passes it on, but leaves its variable dangerous; + - other unreadable env becomes ``_INERT``; + - script variables, image env and special parameters stay as written; + - a default or alternate word is scanned in their place wherever the + shell may use it. + + The scan does not follow control flow, so an assignment may not run: + it leaves literal env and explicit dangerous env of its name, and a + dangerous earlier assignment, in place. Command substitution, backticks + and arithmetic are neither evaluated nor rejected, and neither are + scripts the worker sources or runs, or literal text that ``eval`` or a + nested shell parses again; a script variable counts as dangerous only + when the script assigns it from dangerous env directly (not through an + array, positional parameters, a function argument or a command's + output). + """ + + return _ShellScan(script, env).run() + + +class _ShellScan: + """One pass of ``_shell_expand`` over a script.""" + + def __init__(self, script: str, env: _Env) -> None: + self.script = script + self.env = env + self.out: list[str] = [] + self.index = 0 + self.quoted = False + # The current word as far as the scan knows it: an expansion it does + # not evaluate adds nothing, since it may be empty. + self.word = "" + self.in_word = False + self.assignment = False + self.dangerous = False + # The known words of the current simple command, and its name. + self.words: list[str] = [] + self.command: Optional[str] = None + self.command_dangerous = False + self.redirect = False + self.heredocs: list[tuple[str, bool]] = [] + # Script variables, and whether each was assigned dangerous env. + self.variables: dict[str, bool] = {} + + def run(self) -> str: + while self.index < len(self.script): + # A default or alternate word is spliced into the script. + script = self.script + char = script[self.index] + if char == "\x00": + end = script.find("\x00", self.index + 1) + self.dangerous |= script.startswith(_OPAQUE, self.index) + self.emit(len(script) if end == -1 else end + 1, "") + elif not self.quoted and char in _SHELL_BLANKS: + self.end_word() + self.copy(self.index + 1) + elif not self.quoted and char in _SHELL_OPERATORS: + self.operator() + elif not self.quoted and char == "#" and not self.in_word: + end = script.find("\n", self.index) + self.copy(len(script) if end == -1 else end) + elif char == "\\": + self.escape() + elif char == "'" and not self.quoted: + end = script.find("'", self.index + 1) + end = len(script) if end == -1 else end + 1 + self.emit(end, script[self.index + 1 : end - 1]) + elif char == '"': + self.emit(self.index + 1, "") + self.quoted = not self.quoted + elif char == "`": + self.emit(_backtick_end(script, self.index + 1), "") + elif char == "$": + self.dollar() + else: + self.emit(self.index + 1, char) + self.end_word() + self.end_command() + return "".join(self.out) + + def copy(self, end: int) -> None: + self.out.append(self.script[self.index : end]) + self.index = end + + def emit(self, end: int, known: str, text: Optional[str] = None) -> None: + """Add the script up to ``end`` (or ``text`` for it) to the current word.""" + + if not self.in_word: + self.in_word = True + self.assignment = self.command is None and bool( + _SHELL_ASSIGNMENT.match(self.script, self.index) + ) + self.out.append(self.script[self.index : end] if text is None else text) + self.word += known + self.index = end + + def end_word(self) -> None: + if not self.in_word: + return + word = self.word + self.command_dangerous |= self.dangerous + if self.redirect: + self.redirect = False + else: + match = _SHELL_ASSIGNMENT.match(word) + if match and (self.assignment or self.command in _SHELL_DECLARATIONS): + self.assign(match.group(1), self.dangerous) + elif _SHELL_NAME.fullmatch(word) and ( + self.command == "read" + or (self.command == "for" and self.words == ["for"]) + ): + self.assign(word, False) + self.words.append(word) + if ( + self.command is None + and not self.assignment + and word not in _SHELL_RESERVED + ): + self.command = word + self.word, self.in_word, self.assignment, self.dangerous = ( + "", + False, + False, + False, + ) + + def end_command(self) -> None: + if self.command == "for" and len(self.words) > 1 and self.command_dangerous: + self.variables[self.words[1]] = True + self.words, self.command = [], None + self.command_dangerous = self.redirect = False + + def assign(self, name: str, dangerous: bool) -> None: + # The scan does not follow control flow, so the assignment may not + # run: it never clears explicit dangerous env or an earlier + # dangerous assignment. + env = self.env + self.variables[name] = ( + dangerous + or self.variables.get(name, False) + or (name in env.opaque and name not in env.inert) + ) + + def operator(self) -> None: + script, index = self.script, self.index + self.end_word() + if script.startswith("<<", index) and not script.startswith("<<<", index): + heredoc, end = _shell_heredoc(script, index) + if heredoc is not None: + self.heredocs.append(heredoc) + self.copy(end) + return + if script[index] in "<>" or script.startswith("&>", index): + # A redirection, whose target is no argument. + end = index + 1 + while end < len(script) and script[end] in "<>&|": + end += 1 + self.copy(end) + self.redirect = True + return + self.end_command() + self.copy(index + 1) + if script[index] == "\n": + for delimiter, strip in self.heredocs: + self.copy(_shell_heredoc_end(script, self.index, delimiter, strip)) + self.heredocs.clear() + + def escape(self) -> None: + index = self.index + following = self.script[index + 1 : index + 2] + if following == "\n": + # A line continuation joins the lines. + self.index += 2 + elif self.quoted and following not in ('"', "\\", "$", "`"): + self.emit(index + 1, "\\") + else: + self.emit(index + 2, following) + + def dollar(self) -> None: + script, index = self.script, self.index + following = script[index + 1 : index + 2] + name = _SHELL_NAME.match(script, index + 1) + if following == "(": + # Command substitution or arithmetic, not evaluated. + self.emit(_shell_substitution_end(script, index + 2), "") + elif following == "{": + self.braced() + elif name is not None: + self.reference(name.group(0), name.end()) + elif following and following in "@*#?$!-0123456789": + self.emit(index + 2, "") + else: + self.emit(index + 1, "$") + + def braced(self) -> None: + script, index = self.script, self.index + close = _shell_brace_end(script, index + 2) + if close == -1: + self.emit(len(script), "") + return + body = script[index + 2 : close] + name = _SHELL_NAME.match(body) + if name is None: + # A length, an indirection or a positional parameter. + self.emit(close + 1, "") + return + operator = body[name.end() :] + word_operator = operator.lstrip(":")[:1] + if word_operator in ("-", "=", "+"): + word = operator[operator.index(word_operator) + 1 :] + kind = self.kind(name.group(0)) + value = self.env.values.get(name.group(0), "") + known_set = kind == "readable" and ( + bool(value) or not operator.startswith(":") + ) + # The shell expands the word where it replaces the value: an + # alternate one where the value may be set, a default one where + # it may be unset and is not unreadable env. + if word_operator == "+": + if known_set or kind != "readable": + self.script = script[:index] + word + script[close + 1 :] + else: + self.index = close + 1 + return + if not known_set and kind not in ("dangerous", "inert"): + self.script = script[:index] + word + script[close + 1 :] + return + self.reference(name.group(0), close + 1, operator) + + def reference(self, name: str, end: int, operator: str = "") -> None: + kind = self.kind(name) + if kind == "readable": + value = self.env.values[name] + # The value, unless a default word replaces an empty one or an + # operator derives another value from it. + if operator[:1] in ("", "-", "=", "?") or ( + operator[:2] in (":-", ":=", ":?") and value + ): + self.literal(value, end) + else: + self.emit(end, "") + elif kind == "unknown": + self.emit(end, "") + else: + dangerous = kind == "dangerous" + self.dangerous |= dangerous + marker = _OPAQUE if dangerous and self.option_position() else _INERT + self.emit(end, "", f"{marker}${name}\x00") + + def kind(self, name: str) -> str: + env = self.env + assigned = self.variables.get(name) + if assigned: + return "dangerous" + if name in env.values: + # The script's own assignment may not run either. + return "readable" + if assigned is not None: + return "unknown" + if name in env.inert: + return "inert" + if name in env.opaque or _env_from_may_define(name, env.env_from_prefixes): + return "dangerous" + return "unknown" + + def literal(self, value: str, end: int) -> None: + if self.quoted: + self.emit(end, value, value.replace("\\", "\\\\").replace('"', '\\"')) + elif self.in_word and self.assignment: + self.emit(end, value, shlex.quote(value)) + else: + # The shell splits an unquoted value into words. + pieces = value.split() + for position, piece in enumerate(pieces): + if position or value[:1].isspace(): + self.end_word() + self.out.append(" ") + self.emit(self.index, piece, shlex.quote(piece)) + if value[-1:].isspace(): + self.end_word() + self.out.append(" ") + self.index = end + + def option_position(self) -> bool: + """Whether a dangerous value here could be or create an option of the worker.""" + + if self.in_word and self.assignment: + return False + if not self.quoted or self.reparses(): + return True + lead = self.word if self.in_word else "" + if lead: + # Attached to its option, or inside one's name. + return lead.startswith("-") and "=" not in lead + if self.words and self.words[-1] in _WORKER_VALUE_OPTIONS: + return False + if self.command is None: + # The command's name: one word, a program to run. + return False + if ( + self.command in _SHELL_REPARSING + or posixpath.basename(self.command) in _SHELLS + ): + return True + words = self.words + for word, following in zip(words, [*words[1:], ""]): + module = _PYTHON_MODULE_OPTION.fullmatch(word) + if module is not None and (module.group(1) or following).startswith( + "dynamo." + ): + return True + return False + + def reparses(self) -> bool: + """Whether the current command parses its arguments again as a script. + + ``eval`` does, and so does a shell given ``-c``, also behind a + launcher (``exec``, ``nohup``, ``env``), whose positional parameters + its script may expand unquoted. + """ + + return self.command == "eval" or _shell_script_index(self.words) is not None + + +def _backtick_end(script: str, index: int) -> int: + """The index after the backtick closing one opened before ``index``.""" + + while index < len(script): + if script[index] == "\\": + index += 2 + elif script[index] == "`": + return index + 1 + else: + index += 1 + return len(script) + + +def _shell_brace_end(script: str, index: int) -> int: + """The index of the ``}`` closing a ``${`` whose body starts at ``index``, or -1.""" + + depth = 0 + while index < len(script): + if script[index] == "\\": + index += 2 + continue + if script.startswith("${", index): + depth += 1 + index += 2 + continue + if script[index] == "}": + if depth == 0: + return index + depth -= 1 + index += 1 + return -1 + + +def _shell_substitution_end(script: str, index: int) -> int: + """The index after the ``)`` closing a ``$(`` whose body starts at ``index``.""" + + depth = 0 + quote = "" + heredocs: list[tuple[str, bool]] = [] + word_start = True + while index < len(script): + char = script[index] + if quote == "'": + quote = "" if char == "'" else quote + index += 1 + continue + if char == "\\": + index += 2 + word_start = False + continue + if char == "`": + index = _backtick_end(script, index + 1) + continue + if script.startswith("$(", index): + index = _shell_substitution_end(script, index + 2) + continue + if quote == '"': + quote = "" if char == '"' else quote + index += 1 + continue + if char in "'\"": + quote = char + elif char == "#" and word_start: + end = script.find("\n", index) + index = len(script) if end == -1 else end + continue + elif script.startswith("<<", index) and not script.startswith("<<<", index): + heredoc, index = _shell_heredoc(script, index) + if heredoc is not None: + heredocs.append(heredoc) + continue + elif char == "\n" and heredocs: + index += 1 + for delimiter, strip in heredocs: + index = _shell_heredoc_end(script, index, delimiter, strip) + heredocs.clear() + word_start = True + continue + elif char == "(": + depth += 1 + elif char == ")": + if depth == 0: + return index + 1 + depth -= 1 + word_start = char in _SHELL_BLANKS or char in _SHELL_OPERATORS + index += 1 + return len(script) + + +def _shell_heredoc(script: str, index: int) -> tuple[Optional[tuple[str, bool]], int]: + """The heredoc ``<<`` at ``index`` opens, and the index after its delimiter. + + The heredoc is its delimiter and whether ``<<-`` strips leading tabs. + """ + + index += 2 + strip = script.startswith("-", index) + if strip: + index += 1 + while index < len(script) and script[index] in _SHELL_BLANKS: + index += 1 + start = index + while ( + index < len(script) + and script[index] not in _SHELL_BLANKS + and script[index] not in _SHELL_OPERATORS + ): + index += 1 + delimiter = re.sub(r"[\"'\\]", "", script[start:index]) + return ((delimiter, strip) if delimiter else None), index + + +def _shell_heredoc_end(script: str, index: int, delimiter: str, strip: bool) -> int: + """The index after a heredoc body starting at ``index`` and its delimiter line.""" + + while index < len(script): + end = script.find("\n", index) + end = len(script) if end == -1 else end + line = script[index:end] + index = end + 1 + if (line.lstrip("\t") if strip else line) == delimiter: + break + return min(index, len(script)) def _planner_replica_errors( @@ -1440,8 +2157,7 @@ def _model_flags( main = _main_container(component) tokens: list[str] = [] for part in [*(main.get("command") or []), *(main.get("args") or [])]: - text = _expand(str(part), env.values, env.opaque, mark_opaque=True) - assert text is not None + text = env.expand(str(part)) try: tokens.extend(shlex.split(text)) except ValueError: @@ -1458,7 +2174,7 @@ def _model_flags( resolved = ( value and "$" not in value - and _OPAQUE not in value + and _UNREADABLE not in value and not value.startswith("-") ) found.append((flag, value if resolved else None)) diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py index 5c6928764f18..d994b218fdca 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py @@ -622,6 +622,9 @@ def test_opaque_reference_in_a_typed_non_router_arg_is_fine(args): (*KV_ARGS, "--tool-call-parser", "--$(HTTP_PORT)", "$(HTTP_PORT)"), "could set a router flag", ), + # Inert env is still unreadable as a router value or an option name. + ((*KV_ARGS, "--router-temp", "$(POD_UID)"), "router arg references $(POD_UID)"), + ((*KV_ARGS, "--$(POD_NAME)"), "could set a router flag"), ], ) def test_opaque_reference_that_could_set_a_router_flag_is_unparseable(args, match): @@ -630,6 +633,50 @@ def test_opaque_reference_that_could_set_a_router_flag_is_unparseable(args, matc assert match in error +@pytest.mark.parametrize( + "args", + [ + (*KV_ARGS, "$(POD_NAME)"), + (*KV_ARGS, "--router-kv-events", "$(POD_UID)"), + (*KV_ARGS, "$(NODE)"), + ], +) +def test_inert_reference_where_a_flag_could_stand_is_fine(args): + # A pod name, uid or node name never starts with "-". + env = trace_env(NODE={"valueFrom": {"fieldRef": {"fieldPath": "spec.nodeName"}}}) + assert parse(with_frontend(args=args, env=env)).router_config == {} + + +@pytest.mark.parametrize( + ("args", "prefix", "match"), + [ + ((*KV_ARGS, "$(ROUTER_ARG)"), None, "could set a router flag"), + ( + (*KV_ARGS, "--router-temperature", "$(T)"), + None, + "router arg references $(T)", + ), + ((*KV_ARGS, "$(APP_ROUTER_ARG)"), "APP_", "could set a router flag"), + # A prefix that cannot produce the name, and command substitution. + ((*KV_ARGS, "$(ROUTER_ARG)"), "APP_", None), + ((*KV_ARGS, "--metrics-prefix", "$(seq 1 3)"), None, None), + ], +) +def test_env_from_may_define_an_undefined_reference(args, prefix, match): + deployment = with_frontend(args=args) + source: dict[str, Any] = {"configMapRef": {"name": "router-options"}} + if prefix is not None: + source["prefix"] = prefix + deployment["spec"]["components"][0]["podTemplate"]["spec"]["containers"][0][ + "envFrom" + ] = [source] + result = fc.parse_frontend(deployment, planner_mode="agg") + if match is None: + assert isinstance(result, FrontendStartup), result + else: + assert isinstance(result, str) and match in result, result + + def test_router_env_from_value_from_is_unparseable(): env = trace_env( DYN_ROUTER_TEMPERATURE={ @@ -996,11 +1043,16 @@ def test_p4_disagg_decode_credit_zero_is_not_a_zero_credit_startup(): SECRET = {"valueFrom": {"secretKeyRef": {"name": "s", "key": "k"}}} +# A Secret that may hold any key. +ENV_FROM = [{"secretRef": {"name": "worker-launch-options"}}] +# A worker launch in a shell script. +WORKER = f"python3 -m dynamo.vllm --model {MODEL}" def router_worker( *args: str, env: list[dict[str, Any]] | None = None, + env_from: list[dict[str, Any]] | None = None, name: str = "agg", type_: str = "worker", ) -> dict[str, Any]: @@ -1009,17 +1061,54 @@ def router_worker( main["args"] = [*main["args"], *args] if env is not None: main["env"] = env + if env_from is not None: + main["envFrom"] = env_from return component -def shell_worker(script: str) -> dict[str, Any]: +def shell_worker( + script: str, + *, + command: tuple[str, ...] | None = ("/bin/sh", "-c"), + env: list[dict[str, Any]] | None = None, + env_from: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """A worker running ``script``; ``command=None`` omits the command.""" + component = worker() - component["podTemplate"]["spec"]["containers"][0].update( - command=["/bin/sh", "-c"], args=[script] - ) + main = component["podTemplate"]["spec"]["containers"][0] + main["args"] = [script] + if command is None: + del main["command"] + else: + main["command"] = list(command) + if env is not None: + main["env"] = env + if env_from is not None: + main["envFrom"] = env_from return component +def field_ref_env(name: str, path: str) -> dict[str, Any]: + return {"name": name, "valueFrom": {"fieldRef": {"fieldPath": path}}} + + +def router_arg_env() -> list[dict[str, Any]]: + """ROUTER_ARG from a ConfigMap the planner cannot read.""" + + return [ + { + "name": "ROUTER_ARG", + "valueFrom": { + "configMapKeyRef": { + "name": "worker-launch-options", + "key": "router-arg", + } + }, + } + ] + + def minimax_prefill_worker(*router_args: str) -> dict[str, Any]: """The PrefillWorker of recipes/minimax-m3/vllm/disagg-gb200-agentic.""" @@ -1117,6 +1206,50 @@ def minimax_prefill_worker(*router_args: str) -> dict[str, Any]: [{"name": "DYN_ROUTER_TEMPERATURE", "value": "hot"}], "invalid router env", ), + # An unreadable argument where an option could stand, e.g. a + # ConfigMap value "--router-mode=kv". + ( + ("--no-router-kv-events", "$(ROUTER_ARG)"), + router_arg_env(), + "has an argument that references $(ROUTER_ARG)", + ), + ( + ("$(ROUTER_ARG)",), + router_arg_env(), + "has an argument that references $(ROUTER_ARG)", + ), + ( + ("--$(A_MODE)=kv",), + [{"name": "A_MODE", **SECRET}], + "has an argument that references $(A_MODE)", + ), + ( + ("--router-$(A_MODE)", "kv"), + [{"name": "A_MODE", **SECRET}], + "has an argument that references $(A_MODE)", + ), + ( + ("--speculative-config", "$(A_SECRET)", "$(A_SECRET)"), + [{"name": "A_SECRET", **SECRET}], + "has an argument that references $(A_SECRET)", + ), + # After an engine flag that takes no value or optional values, the + # worker would read "--router-mode=kv" as a flag. + ( + ("--enforce-eager", "$(ROUTER_ARG)"), + router_arg_env(), + "has an argument that references $(ROUTER_ARG)", + ), + ( + ("--connector", "$(ROUTER_ARG)"), + router_arg_env(), + "has an argument that references $(ROUTER_ARG)", + ), + ( + ("$(NOTE)",), + [field_ref_env("NOTE", "metadata.annotations['router']")], + "has an argument that references $(NOTE)", + ), ], ) def test_p4_rejects_a_worker_that_advertises_a_router_config(args, env, match): @@ -1135,12 +1268,15 @@ def test_p4_rejects_a_worker_that_advertises_a_router_config(args, env, match): (("--no-router-kv-events", "--router-temperature", "0.5"), None), ((), [{"name": "DYN_ROUTER_TEMPERATURE", "value": "0.5"}]), ((), [{"name": "DYN_ROUTER_MODE_HINT", "value": "kv"}]), - # Unreadable env in a non-mode argument. + # Unreadable env as the value of a non-mode router flag, of an + # option known to take one, or attached to its option. (("--router-temperature", "$(A_SECRET)"), [{"name": "A_SECRET", **SECRET}]), - ( - ("--speculative-config", "$(A_SECRET)", "$(A_SECRET)"), - [{"name": "A_SECRET", **SECRET}], - ), + (("--speculative-config", "$(A_SECRET)"), [{"name": "A_SECRET", **SECRET}]), + (("--speculative-config", "$(ROUTER_ARG)"), router_arg_env()), + (("--some-opt=$(ROUTER_ARG)",), router_arg_env()), + # Inert env never stands for an option. + (("$(POD_NAME)",), None), + (("--enforce-eager", "$(MY_POD)"), [field_ref_env("MY_POD", "metadata.name")]), ], ) def test_p4_accepts_workers_that_advertise_nothing(args, env): @@ -1148,6 +1284,83 @@ def test_p4_accepts_workers_that_advertise_nothing(args, env): assert report.p4_ok and report.spec_ok, report.reasons +@pytest.mark.parametrize( + ("command", "args", "env_from", "match"), + [ + # A Secret or ConfigMap key may define what env leaves undefined. + ( + ("python3",), + ("$(ROUTER_ARG)",), + ENV_FROM, + "references $(ROUTER_ARG), which the planner cannot read (the " + "container's env leaves it to an envFrom source)", + ), + (("/bin/sh", "-c"), ("$(ROUTER_ARG)",), ENV_FROM, "references $(ROUTER_ARG)"), + (None, ("$(ROUTER_ARG)",), ENV_FROM, "references $(ROUTER_ARG)"), + # Image env too, which a shell splits unless quoted; a script's own + # assignment overrides what such a source may define. + ( + ("/bin/sh", "-c"), + (f"export LD_LIBRARY_PATH=/opt/lib:$LD_LIBRARY_PATH; exec {WORKER}",), + ENV_FROM, + "references $LD_LIBRARY_PATH where it could become an option, and " + "the planner cannot read it (the container's env leaves it to an " + "envFrom source)", + ), + ( + ("/bin/sh", "-c"), + (f'export LD_LIBRARY_PATH="/opt/lib:$LD_LIBRARY_PATH"; exec {WORKER}',), + ENV_FROM, + None, + ), + (("/bin/sh", "-c"), (f"ARGS=; exec {WORKER} $ARGS",), ENV_FROM, None), + ( + ("python3",), + ("$(APP_ROUTER_ARG)",), + [{**ENV_FROM[0], "prefix": "APP_"}], + "references $(APP_ROUTER_ARG)", + ), + # A prefix that cannot produce the name, and the worker's own env. + (("python3",), ("$(ROUTER_ARG)",), [{**ENV_FROM[0], "prefix": "APP_"}], None), + (("python3",), ("--model", "$(MODEL_ID)"), ENV_FROM, None), + ], +) +def test_p4_worker_reference_an_env_from_source_may_define( + command, args, env_from, match +): + component = worker() + main = component["podTemplate"]["spec"]["containers"][0] + main.update( + args=["-m", "dynamo.vllm", *args] if command == ("python3",) else list(args), + env=[{"name": "MODEL_ID", "value": MODEL}], + envFrom=env_from, + ) + if command is None: + del main["command"] + else: + main["command"] = list(command) + report = evaluate(dgd(frontend(), component, planner())) + rejected = reasons_matching(report, "P4: worker component agg ") + if match is None: + assert report.p4_ok, report.reasons + else: + assert any(match in reason for reason in rejected), report.reasons + + +def test_p4_checks_an_untyped_worker(): + report = evaluate( + dgd( + frontend(), + router_worker("--router-mode", "kv", name="w", type_=""), + planner(), + ) + ) + assert any( + "router mode 'kv'" in reason + for reason in reasons_matching(report, "P4: worker component w ") + ), report.reasons + + def test_p4_rejects_graph_env_router_mode_reaching_the_workers(): # recipes/solar-open2-250b: a graph-level DYN_ROUTER_MODE reaches the # frontend and every worker. @@ -1200,10 +1413,40 @@ def test_p4_checks_the_real_recipe_prefill_worker(): assert fc.served_model_name(inheriting) == "nvidia/MiniMax-M3-NVFP4" +@pytest.mark.parametrize( + "command", + # The operator's /bin/sh -c applies to an omitted or empty command; a + # launcher may start the shell. + [ + ("/bin/sh", "-c"), + None, + (), + ("/usr/bin/env", "bash", "-c"), + ("tini", "--", "/bin/sh", "-c"), + ("dumb-init", "--", "bash", "-c"), + ("/bin/bash", "-o", "pipefail", "-c"), + ("bash", "-euo", "pipefail", "-c"), + ], + ids=[ + "explicit", + "omitted", + "empty", + "env-bash", + "tini-sh", + "dumb-init-bash", + "bash-o-pipefail", + "bash-euo-pipefail", + ], +) @pytest.mark.parametrize( ("script", "match"), [ (f"python3 -m dynamo.vllm --model {MODEL} --router-mode kv", "router mode"), + ( + f"python3 -m dynamo.vllm --model {MODEL} --router-mode kv " + "--no-router-kv-events", + "router mode 'kv'", + ), ( f"export DYN_ROUTER_MODE=kv && exec python3 -m dynamo.vllm --model {MODEL}", "sets DYN_ROUTER_MODE in its shell command", @@ -1216,8 +1459,8 @@ def test_p4_checks_the_real_recipe_prefill_worker(): (f"python3 -m dynamo.vllm --model {MODEL} --no-router-kv-events", None), ], ) -def test_p4_shell_wrapped_worker(script, match): - report = evaluate(dgd(frontend(), shell_worker(script), planner())) +def test_p4_shell_wrapped_worker(command, script, match): + report = evaluate(dgd(frontend(), shell_worker(script, command=command), planner())) rejected = reasons_matching(report, "P4: worker component agg ") if match is None: assert report.p4_ok, report.reasons @@ -1226,6 +1469,204 @@ def test_p4_shell_wrapped_worker(script, match): assert any(match in reason for reason in rejected), report.reasons +@pytest.mark.parametrize( + "command", + # A shell option's operand (pipefail) is no script. + [ + ("/bin/sh", "-c"), + ("bash", "-lc"), + None, + ("/bin/bash", "-o", "pipefail", "-c"), + ("bash", "-euo", "pipefail", "-c"), + ], + ids=["sh", "bash-lc", "omitted", "bash-o-pipefail", "bash-euo-pipefail"], +) +@pytest.mark.parametrize( + ("script", "match"), + [ + ("$(WORKER_CMD)", "has a shell script that references $(WORKER_CMD)"), + (f"{WORKER} $(ROUTER_ARG)", "has a shell script that references $(ROUTER_ARG)"), + # Kubernetes substitutes $(VAR) before the shell parses quotes, so + # even an option's value could carry --router-mode. + ( + "python3 -m dynamo.vllm --speculative-config $(CFG)", + "has a shell script that references $(CFG)", + ), + ( + f"{WORKER} --router-temperature $(CFG)", + "has a shell script that references $(CFG)", + ), + ("python3 -m dynamo.vllm --model $(MODEL_NAME)", None), + # The shell splits an unquoted variable into words. + (f"{WORKER} $ROUTER_ARG", "has a shell script that references $ROUTER_ARG"), + (f"{WORKER} ${{ROUTER_ARG}}", "has a shell script that references $ROUTER_ARG"), + ( + "python3 -m dynamo.vllm --speculative-config $SPECULATIVE_CONFIG", + "has a shell script that references $SPECULATIVE_CONFIG", + ), + ( + f"X=$ROUTER_ARG; exec {WORKER} $X", + "has a shell script that references $X", + ), + # An assignment may not run, so it clears neither env nor an + # earlier dangerous value. + ( + 'if [ -z "$ROUTER_ARG" ]; then ROUTER_ARG=x; fi; ' + f"exec {WORKER} $ROUTER_ARG", + "has a shell script that references $ROUTER_ARG", + ), + ( + f'[ -n "$ROUTER_ARG" ] || ROUTER_ARG=""; exec {WORKER} $ROUTER_ARG', + "has a shell script that references $ROUTER_ARG", + ), + ( + f'X=$ROUTER_ARG; [ -n "$X" ] || X=; exec {WORKER} $X', + "has a shell script that references $X", + ), + ( + f'[ -n "$OVERRIDE" ] && EXTRA="$OVERRIDE"; exec {WORKER} $EXTRA', + "router mode 'kv'", + ), + # A quoted one stays one word: an option's value, unless it stands + # alone as an argument of the worker. + (f'{WORKER} "$ROUTER_ARG"', "has a shell script that references $ROUTER_ARG"), + ( + f'exec env A=1 {WORKER} "${{ROUTER_ARG:-}}"', + "has a shell script that references $ROUTER_ARG", + ), + ( + f'exec python3 -um dynamo.vllm --model {MODEL} "$ROUTER_ARG"', + "has a shell script that references $ROUTER_ARG", + ), + # eval and a shell's -c parse their arguments again, also behind a + # launcher and within a larger word. + ('eval "$ROUTER_ARG"', "has a shell script that references $ROUTER_ARG"), + ( + f'eval "{WORKER} $ROUTER_ARG"', + "has a shell script that references $ROUTER_ARG", + ), + ('exec sh -c "$ROUTER_ARG"', "has a shell script that references $ROUTER_ARG"), + ( + f'exec sh -c "{WORKER} $ROUTER_ARG"', + "has a shell script that references $ROUTER_ARG", + ), + ( + f"exec bash -c 'exec {WORKER} $@' bash \"$ROUTER_ARG\"", + "has a shell script that references $ROUTER_ARG", + ), + (f'{WORKER} --speculative-config="${{SPECULATIVE_CONFIG}}"', None), + (f'{WORKER} --speculative-config "$SPECULATIVE_CONFIG"', None), + (f'{WORKER} "--some-opt=$ROUTER_ARG"', None), + (f'[ -n "$ROUTER_ARG" ] && echo "$ROUTER_ARG"; exec {WORKER}', None), + (f'export LD_LIBRARY_PATH="/opt/lib:$ROUTER_ARG"; exec {WORKER}', None), + # Literal env reached through the shell is read, and so are the + # default and alternate words the shell may use. + (f"{WORKER} $EXTRA", "router mode 'kv'"), + (f'{WORKER} --router-mode "${{MODE:-random}}"', "router mode 'kv'"), + (f"{WORKER} ${{MODE:+--router-mode=kv}}", "router mode 'kv'"), + ( + f"{WORKER} ${{UNSET:-$ROUTER_ARG}}", + "has a shell script that references $ROUTER_ARG", + ), + # Inert env never stands for an option. + (f'{WORKER} $POD_NAME "$MY_POD" $(MY_POD) $(POD_UID)', None), + (f'[ "$POD_INDEX" = "0" ] && exec {WORKER}', None), + (f"{WORKER} $NOTE", "has a shell script that references $NOTE"), + # Neither evaluated nor rejected: command substitution and script + # variables built from it. + (f"{WORKER} $(cat /etc/router-args)", None), + (f"ARGS=`cat /etc/router-args`; {WORKER} $ARGS", None), + ], +) +def test_p4_shell_wrapped_worker_with_unreadable_env(command, script, match): + env = [ + {"name": name, "valueFrom": {"configMapKeyRef": {"name": "c", "key": "k"}}} + for name in ("CFG", "ROUTER_ARG", "SPECULATIVE_CONFIG", "WORKER_CMD") + ] + env += [ + {"name": "MODEL_NAME", "value": MODEL}, + {"name": "EXTRA", "value": "--router-mode kv"}, + {"name": "MODE", "value": "kv"}, + field_ref_env("MY_POD", "metadata.name"), + field_ref_env("POD_INDEX", "metadata.labels['grove.io/podclique-pod-index']"), + field_ref_env("NOTE", "metadata.annotations['router']"), + ] + report = evaluate( + dgd(frontend(), shell_worker(script, command=command, env=env), planner()) + ) + rejected = reasons_matching(report, "P4: worker component agg ") + if match is None: + assert report.p4_ok, report.reasons + else: + assert not report.p4_ok + assert any(match in reason for reason in rejected), report.reasons + assert all("on the frontend only" in reason for reason in rejected) + + +@pytest.mark.parametrize( + ("command", "args", "match"), + [ + # A launcher drops its "--", so the worker still sees later flags. + ( + ("tini", "--", "python3", "-m", "dynamo.vllm"), + ("--router-mode", "kv"), + "router mode 'kv'", + ), + ( + ("dumb-init", "--"), + ("python3", "-m", "dynamo.vllm", "--router-mode", "kv"), + "router mode 'kv'", + ), + ( + ("/bin/sh", "-c"), + ("exec tini -- python3 -m dynamo.vllm --router-mode kv",), + "router mode 'kv'", + ), + (("tini", "--", "python3", "-m", "dynamo.vllm"), ("--model", MODEL), None), + ], +) +def test_p4_worker_behind_a_launcher(command, args, match): + component = worker() + component["podTemplate"]["spec"]["containers"][0].update( + command=list(command), args=list(args) + ) + report = evaluate(dgd(frontend(), component, planner())) + rejected = reasons_matching(report, "P4: worker component agg ") + if match is None: + assert report.p4_ok, report.reasons + else: + assert any(match in reason for reason in rejected), report.reasons + + +@pytest.mark.parametrize("native_sidecar", [True, False]) +def test_p4_native_sidecar_main_keeps_its_args_as_exec_form(native_sidecar): + # The main keeps its image entrypoint instead of the operator's + # /bin/sh -c, so an option's attached value is not a shell script. + component = worker() + spec = component["podTemplate"]["spec"] + main = spec["containers"][0] + del main["command"] + main.update( + args=["--model-path=$(MODEL)", "--tp", "2"], + env=[ + { + "name": "MODEL", + "valueFrom": {"configMapKeyRef": {"name": "c", "key": "model"}}, + } + ], + ) + if native_sidecar: + spec["initContainers"] = [{"name": "runtime", "restartPolicy": "Always"}] + report = evaluate(dgd(frontend(), component, planner())) + if native_sidecar: + assert report.p4_ok, report.reasons + else: + assert any( + "has a shell script that references $(MODEL)" in reason + for reason in reasons_matching(report, "P4: worker component agg ") + ), report.reasons + + def test_worker_check_ignores_the_planner_env(monkeypatch): monkeypatch.setenv("DYN_ROUTER_MODE", "kv") fc._worker_router_cached.cache_clear() @@ -1246,6 +1687,8 @@ def test_worker_check_ignores_the_planner_env(monkeypatch): (("--router-mode", "least-loaded"), {"DYN_ROUTER_MODE": "kv"}), (("--no-router-kv-events", "--router-temperature", "0.5"), {}), ((), {"DYN_ROUTER_TEMPERATURE": "0.5"}), + # What an unreadable "$(ROUTER_ARG)" after --no-router-kv-events can be. + (("--no-router-kv-events", "--router-mode=kv"), {}), ] @@ -1715,11 +2158,26 @@ def test_worker_types_and_canonical_json(): ("$(A", "$(A"), ("$", "$"), ("a$$b", "a$b"), - ("$(SECRET)", None), + ("x-$(SECRET)", "x-{opaque}$(SECRET)\x00"), + ("$(POD)", "{inert}$(POD)\x00"), + # Undefined names an envFrom source with prefix APP_ may define. + ("$(APP_ARG)", "{opaque}$(APP_ARG)\x00"), + ("$(APP_a.b-c)", "{opaque}$(APP_a.b-c)\x00"), + # The container's env wins over envFrom. + ("$(APP_DEFINED)", "2"), + ("$(ARG)", "$(ARG)"), + ("$(APP_ seq 1 60)", "$(APP_ seq 1 60)"), ], ) def test_kubernetes_expansion(value, expected): - assert fc._expand(value, {"A": "1"}, {"SECRET"}) == expected + expanded = fc._expand( + value, + {"A": "1", "APP_DEFINED": "2"}, + {"SECRET", "POD"}, + inert={"POD"}, + env_from_prefixes=("APP_",), + ) + assert expanded == expected.format(opaque=fc._OPAQUE, inert=fc._INERT) def test_parse_is_cached_per_spec(): diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 1c6f24aa7b88..6707fe7dabfe 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -73,7 +73,8 @@ This applies to all DGDR trace Runs, not only those the Planner creates. Planner writes each window file with mode `0644`, so the Run Job, which reads it as UID 1000 with supplemental group 65532, can read it whatever user the Planner runs as. - **A KV-router frontend with default routing fields**, and workers that do not set their own - router mode. See [Deployment rules](#deployment-rules). + router mode or pass a value the Planner cannot read where an option could stand. See + [Deployment rules](#deployment-rules). - **One Planner replica.** ## Configure the frontend @@ -263,7 +264,7 @@ retries: | Rule | Requirement | |---|---| -| P4 | The frontend runs `--router-mode kv` with every routing field the Request cannot express at its default, `kv_event_lag_ms` unset or 0, no `DYN_ROUTER_WORKER_SELECTION_POLICY`, a `none` prefill load model, router env given as plain values (no `valueFrom`), and a command that is not wrapped in a shell. The Planner cannot read `envFrom` sources: it assumes they set no router env, and only warns about a source whose `prefix` could produce a `DYN_` name. Keep router env out of `envFrom`: P14 catches only a changed policy parameter. No worker advertises its own router configuration: worker components set neither `--router-mode` nor `DYN_ROUTER_MODE`, directly or through the graph-level `spec.env`, because a worker's advertised configuration replaces the frontend's for its workers | +| P4 | The frontend runs `--router-mode kv` with every routing field the Request cannot express at its default, `kv_event_lag_ms` unset or 0, no `DYN_ROUTER_WORKER_SELECTION_POLICY`, a `none` prefill load model, router env given as plain values (no `valueFrom`), and a command that is not wrapped in a shell. The Planner cannot read `envFrom` sources: it assumes they set no router env, only warns about a source whose `prefix` could produce a `DYN_` name, and treats a `$(VAR)` that only an `envFrom` source could define as unreadable. Keep router env out of `envFrom`: P14 catches only a changed policy parameter. No worker advertises its own router configuration: worker components set neither `--router-mode` nor `DYN_ROUTER_MODE`, directly or through the graph-level `spec.env`, because a worker's advertised configuration replaces the frontend's for its workers. The Planner reads a worker's command as the operator runs it: an omitted `command` is `/bin/sh -c` with the first of `args` as the script, unless the pod has a native `runtime` sidecar, and a shell started through `env`, `tini --` or `dumb-init --` is a script too. A value the Planner cannot read and that could hold an option (a ConfigMap, Secret or annotation `valueFrom`, or a `$(VAR)` that the container's `env` leaves to an `envFrom` source) must not stand where a worker option could: attach it to its option (`--option=$(VAR)`) or pass it right after `--speculative-config`. In a worker shell script any such `$(VAR)` fails, and so does a `$VAR` or `${VAR}` of such a value that is unquoted, stands alone as a worker argument, or sits in text that `eval` or a nested `sh -c` parses again; quote it outside the worker's arguments and attach it to its option among them (`--option="$VAR"`). The Planner does not follow the script's conditions, so the script's own assignment to a ConfigMap, Secret or annotation variable does not clear it. With an `envFrom` source that has no `prefix`, every name the container's `env` leaves undefined counts as such a value, image env such as `PATH` and `LD_LIBRARY_PATH` included, so quote those references too (`export LD_LIBRARY_PATH="/usr/local/nvidia/lib64:$LD_LIBRARY_PATH"`). Other `fieldRef` values, such as the pod name, are fine anywhere. The Planner does not evaluate command substitution, backticks or the scripts a worker runs, so keep router flags out of them | | P7 | The DGD serves one model; the frontend is not an EPP, not a `frontendSidecar`, and has no TLS | | P8 | One frontend replica, or `--router-replica-sync` | | P9 | No `scalingAdapter` on worker components; one Planner replica | @@ -394,6 +395,8 @@ The Planner exports these on `metric_reporting_prometheus_port`: | Retune with load or throughput scaling, or of Planner parameters | A DGDR Run has no base Planner configuration, warm-up or predictor pre-sweep | | Frontends with non-default routing fields beyond the three knobs, or `kv_event_lag_ms` other than 0 | The DGDR router section expresses only the three knobs, and the Run does not pass a lag | | Workers that set their own router mode (`--router-mode` or `DYN_ROUTER_MODE`) | A worker's advertised router configuration replaces the frontend's for its workers, and the Run replays only the frontend's | +| Worker arguments that pass a ConfigMap, Secret, annotation or `envFrom` value where an option could stand, including an unquoted `$VAR` of one in a worker shell script or one that `eval` or a nested shell parses again (P4 fails) | The Planner cannot read the value, so it cannot rule out a router mode the worker would advertise | +| Router flags that a worker produces through shell command substitution, backticks, script variables, positional parameters, literal text that `eval` or a nested shell parses again, or a script file (P4 does not see them) | The Planner reads the worker's spec and does not run its script | | Raising `overlap_score_credit` on a frontend that starts it at 0: an omitted credit then searches only 0, and a written positive value fails P4 | A live router rejects a positive credit after starting at 0 | | Private, fine-tuned or local-path models | The Run Job has no Hugging Face credentials or model mount | | Several frontends without router replica sync | AISimulate simulates one router | From f070334ff82e629e25f090255fb19ced647866fe Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 13:23:01 -0800 Subject: [PATCH 096/100] fix(examples): quote the GKE worker scripts' PATH exports The workers take envFrom from a Secret without a prefix, so live retune's P4 cannot rule out that the Secret defines LD_LIBRARY_PATH or PATH, and an unquoted reference in an export could split into more words in a shell that does not treat export as a declaration. Quoting changes nothing for bash and keeps the examples eligible for live retune. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- examples/deployments/GKE/sglang/disagg.yaml | 8 ++++---- examples/deployments/GKE/vllm/disagg.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/deployments/GKE/sglang/disagg.yaml b/examples/deployments/GKE/sglang/disagg.yaml index dda3390cddac..6cacd6c2acb2 100644 --- a/examples/deployments/GKE/sglang/disagg.yaml +++ b/examples/deployments/GKE/sglang/disagg.yaml @@ -20,8 +20,8 @@ spec: containers: - args: - | - export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:$LD_LIBRARY_PATH - export PATH=$PATH:/usr/local/nvidia/bin:/usr/local/nvidia/lib64 + export LD_LIBRARY_PATH="/usr/local/nvidia/lib64:$LD_LIBRARY_PATH" + export PATH="$PATH:/usr/local/nvidia/bin:/usr/local/nvidia/lib64" /sbin/ldconfig nvidia-smi exec python3 -m dynamo.sglang --model-path Qwen/Qwen3-0.6B --served-model-name Qwen/Qwen3-0.6B --page-size 16 --tp 1 --trust-remote-code --skip-tokenizer-init --disaggregation-mode decode --disaggregation-transfer-backend nixl --disaggregation-bootstrap-port "12345" --host "0.0.0.0" @@ -45,8 +45,8 @@ spec: containers: - args: - |- - export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:$LD_LIBRARY_PATH - export PATH=$PATH:/usr/local/nvidia/bin:/usr/local/nvidia/lib64 + export LD_LIBRARY_PATH="/usr/local/nvidia/lib64:$LD_LIBRARY_PATH" + export PATH="$PATH:/usr/local/nvidia/bin:/usr/local/nvidia/lib64" /sbin/ldconfig nvidia-smi exec python3 -m dynamo.sglang --model-path Qwen/Qwen3-0.6B --served-model-name Qwen/Qwen3-0.6B --page-size 16 --tp 1 --trust-remote-code --skip-tokenizer-init --disaggregation-mode prefill --disaggregation-transfer-backend nixl --disaggregation-bootstrap-port "12345" --host "0.0.0.0" diff --git a/examples/deployments/GKE/vllm/disagg.yaml b/examples/deployments/GKE/vllm/disagg.yaml index 206ff36cb908..8538fe401a54 100644 --- a/examples/deployments/GKE/vllm/disagg.yaml +++ b/examples/deployments/GKE/vllm/disagg.yaml @@ -21,8 +21,8 @@ spec: containers: - args: - | - export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:$LD_LIBRARY_PATH - export PATH=$PATH:/usr/local/nvidia/bin:/usr/local/nvidia/lib64 + export LD_LIBRARY_PATH="/usr/local/nvidia/lib64:$LD_LIBRARY_PATH" + export PATH="$PATH:/usr/local/nvidia/bin:/usr/local/nvidia/lib64" /sbin/ldconfig exec python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --disaggregation-mode decode command: @@ -45,8 +45,8 @@ spec: containers: - args: - | - export LD_LIBRARY_PATH=/usr/local/nvidia/lib64:$LD_LIBRARY_PATH - export PATH=$PATH:/usr/local/nvidia/bin:/usr/local/nvidia/lib64 + export LD_LIBRARY_PATH="/usr/local/nvidia/lib64:$LD_LIBRARY_PATH" + export PATH="$PATH:/usr/local/nvidia/bin:/usr/local/nvidia/lib64" /sbin/ldconfig exec python3 -m dynamo.vllm --model Qwen/Qwen3-0.6B --disaggregation-mode prefill command: From a1437c01c5da52034456c43a0440755c0ed75b55 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 13:23:02 -0800 Subject: [PATCH 097/100] fix(planner): reject context-parallel presets the pinned AISimulate cannot run The pinned AISimulate's ParallelismPresetConfig has no decode_context, and its parallel mapping rejects prefill_context above 1 as predict-only, so such a preset passed P5 and then failed the Run. - P5 accepts prefill_context only when omitted or 1, and rejects decode_context with its own message. - A comment lists what to widen together when the pin moves. - The pytest marker report stubs aisimulate.config.engine. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../dynamo/planner/config/planner_config.py | 20 ++++- .../tests/unit/test_live_retune_config.py | 73 ++++++++++++++++++- .../auto-deployment/planner-live-retune.md | 2 + tests/report_pytest_markers.py | 1 + 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/components/src/dynamo/planner/config/planner_config.py b/components/src/dynamo/planner/config/planner_config.py index 132a08bb4935..784b198460f7 100644 --- a/components/src/dynamo/planner/config/planner_config.py +++ b/components/src/dynamo/planner/config/planner_config.py @@ -1060,7 +1060,13 @@ def _lease_fits_two_renewals(cls, value: float) -> float: # ``parallelism.preset`` entry keys: AISimulate requires all of these plus # ``replicas``, which the executor sets from the DGD spec at each launch. _DGDR_PRESET_KEYS = ("tensor", "pipeline", "attention_data", "moe_tensor", "moe_expert") -_DGDR_PRESET_OPTIONAL_KEYS = ("prefill_context", "decode_context") +# The pinned AISimulate recommend runs prefill_context only at 1, and its +# preset entry schema has no decode_context. When the pin moves, update this +# tuple together with _validate_dgdr_parallelism (its decode_context branch, +# the "optionally prefill_context: 1" message and the prefill_context == 1 +# check), test_pinned_aisimulate_rejects_decode_context and the preset rules +# in docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md. +_DGDR_PRESET_OPTIONAL_KEYS = ("prefill_context",) # Scheduler knobs AISimulate searches over a default domain when omitted. _DGDR_SCHEDULER_KNOBS = ("max_batched_tokens", "max_sequences") @@ -1232,6 +1238,11 @@ def _validate_dgdr_parallelism(parallelism: Any, path: str) -> None: "single preset entry (P5)" ) entry = preset[0] + if "decode_context" in entry: + raise ValueError( + f"{path}.preset[0].decode_context is not accepted (P5); the pinned " + "AISimulate preset schema has no decode_context" + ) missing = [key for key in _DGDR_PRESET_KEYS if key not in entry] unknown = sorted( set(entry) - set(_DGDR_PRESET_KEYS) - set(_DGDR_PRESET_OPTIONAL_KEYS) @@ -1239,6 +1250,7 @@ def _validate_dgdr_parallelism(parallelism: Any, path: str) -> None: if missing or unknown: raise ValueError( f"{path}.preset[0] must set exactly {', '.join(_DGDR_PRESET_KEYS)} " + "and optionally prefill_context: 1 " f"(P5); missing={missing}, unknown={unknown}" ) for key, value in entry.items(): @@ -1246,6 +1258,12 @@ def _validate_dgdr_parallelism(parallelism: Any, path: str) -> None: raise ValueError( f"{path}.preset[0].{key} must be a positive integer (P5), got {value!r}" ) + if entry.get("prefill_context", 1) != 1: + raise ValueError( + f"{path}.preset[0].prefill_context must be 1 or omitted (P5); " + "AISimulate recommend rejects context parallelism above 1, " + f"got {entry['prefill_context']!r}" + ) def _check_dgdr_engine_scalars(value: Any, path: tuple[str, ...]) -> None: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py index f8be5d5f5364..ef2cbddf43d4 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_config.py @@ -23,6 +23,7 @@ reason="AI Simulate is an optional Dynamo simulation dependency", ) +from aisimulate.config.engine import ParallelismRecommendationConfig from pydantic import ValidationError from dynamo.planner.config.planner_config import ( @@ -37,6 +38,7 @@ live_retune_min_lease_seconds, ) from dynamo.planner.retune.domains import expand_router_domain +from dynamo.planner.retune.live import dgdr pytestmark = [ pytest.mark.gpu_0, @@ -354,7 +356,25 @@ def _rejects_engine(match: str, engine: dict, mode: str = "agg") -> None: ( ("workers", "aggregated", "parallelism", "preset", 0, "expert"), 1, - r"preset\[0\] must set exactly .*unknown=\['expert'\]", + r"preset\[0\] must set exactly .*moe_expert and optionally " + r"prefill_context: 1 \(P5\); missing=\[\], unknown=\['expert'\]", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "decode_context"), + 1, + r"preset\[0\].decode_context is not accepted \(P5\); the pinned " + r"AISimulate preset schema has no decode_context", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "prefill_context"), + 2, + r"preset\[0\].prefill_context must be 1 or omitted \(P5\); AISimulate " + r"recommend rejects context parallelism above 1, got 2", + ), + ( + ("workers", "aggregated", "parallelism", "preset", 0, "prefill_context"), + 0, + r"preset\[0\].prefill_context must be a positive integer \(P5\), got 0", ), ( ("workers", "aggregated", "scheduler", "max_batched_tokens"), @@ -474,12 +494,59 @@ def test_p5_accepts_unused_roles_absent_or_null(): assert _config(_retune(dgdr=_dgdr(engine=engine))).retune is not None -def test_p5_accepts_the_context_parallel_preset_knobs(): +def test_p5_accepts_prefill_context_one(): engine = _engine() - engine["workers"]["aggregated"]["parallelism"]["preset"][0]["decode_context"] = 1 + engine["workers"]["aggregated"]["parallelism"]["preset"][0]["prefill_context"] = 1 assert _config(_retune(dgdr=_dgdr(engine=engine))).retune is not None +def test_p5_rejects_decode_context_on_a_disaggregated_role(): + engine = _engine("disagg") + engine["workers"]["decode"]["parallelism"]["preset"][0]["decode_context"] = 1 + _rejects_engine( + r"engine.workers.decode.parallelism.preset\[0\].decode_context is not " + r"accepted \(P5\)", + engine, + mode="disagg", + ) + + +def _spec_replicas(mode: str) -> dict[str, int]: + return {"aggregated": 2} if mode == "agg" else {"prefill": 1, "decode": 3} + + +def _validate_with_pinned_aisimulate(engine: dict, mode: str) -> None: + spec_replicas = _spec_replicas(mode) + pinned = dgdr._pinned_engine(engine, spec_replicas, "vllm") + for role in spec_replicas: + ParallelismRecommendationConfig.model_validate( + pinned["workers"][role]["parallelism"] + ) + + +@pytest.mark.parametrize("mode", ["agg", "disagg"]) +@pytest.mark.parametrize("prefill_context", [None, 1]) +def test_p5_presets_validate_against_the_pinned_aisimulate(mode, prefill_context): + engine = _engine(mode) + if prefill_context is not None: + for role in _spec_replicas(mode): + preset = engine["workers"][role]["parallelism"]["preset"] + preset[0]["prefill_context"] = prefill_context + config = _config(_retune(dgdr=_dgdr(engine=engine)), mode=mode) + assert config.retune is not None and config.retune.dgdr is not None + + _validate_with_pinned_aisimulate(config.retune.dgdr.engine, mode) + + +def test_pinned_aisimulate_rejects_decode_context(): + # Why P5 rejects decode_context; once a pin bump makes this fail, P5 may + # admit it. + engine = _engine() + engine["workers"]["aggregated"]["parallelism"]["preset"][0]["decode_context"] = 1 + with pytest.raises(ValidationError, match=r"decode_context\s+Extra inputs"): + _validate_with_pinned_aisimulate(engine, "agg") + + # -------------------------------------------------------------- claim paths diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 6707fe7dabfe..6388bb7e75a5 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -219,6 +219,8 @@ AISimulate searches by default, would make the Run search engine settings as wel `parallelism.preset` to a list of exactly one entry with `tensor`, `pipeline`, `attention_data`, `moe_tensor` and `moe_expert`, and sets `scheduler.max_batched_tokens` and `scheduler.max_sequences`. +- The preset entry may also set `prefill_context`, but only to `1`, and must not set + `decode_context`: the pinned AISimulate recommendation does not run context parallelism. - Do not set `replicas` anywhere: the Planner fills them in from the DGD spec at each launch and sets `spec.hardware.gpu.budget` to their GPU total. - Write roles the mode does not use as `null`, or leave them out. diff --git a/tests/report_pytest_markers.py b/tests/report_pytest_markers.py index 12a59d76ad99..067320487ffa 100755 --- a/tests/report_pytest_markers.py +++ b/tests/report_pytest_markers.py @@ -117,6 +117,7 @@ "aisimulate.config", "aisimulate.config.cli", "aisimulate.config.common", + "aisimulate.config.engine", "aisimulate.config_adapter", "aisimulate.generator.api", "aisimulate.generator.enumerate", From b7f957f5e8413c089b9444826e1ffca6faa6c763 Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 15:12:13 -0800 Subject: [PATCH 098/100] fix(planner): read live retune worker launches as the operator renders them P4 misjudged several worker shapes: - Operator env (DYN_NAMESPACE, DYN_COMPONENT, a worker's DYN_SYSTEM_*, NIXL_TELEMETRY_* and DYN_FORWARDPASS_METRIC_PORT, a multinode component's DYNAMO_RANK and DYNAMO_LEADER_ADDRESS) counts as inert container env, never as an envFrom value. - A worker whose launches are only in role pod templates is checked role by role, and P7 reads its model flags there too. - A native-sidecar worker is skipped: its main runs the engine. - An unreadable value right after --model, --model-path, --served-model-name or --model-name is that option's value; on the frontend, only --tool-call-parser, --reasoning-parser and --chat-template exempt a separate value. - The script scan sees env, export, source, eval and shells by path and behind exec, nohup, command or builtin, and a redirection target other than a here-string is never an option. - Messages name a script variable's source, give shell-form $(VAR) its own advice and say when an option's name is unreadable; the docs list the stdin, xargs and su -c forms P4 does not read. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../planner/retune/live/frontend_config.py | 447 ++++++++++++------ .../unit/test_live_retune_frontend_config.py | 355 ++++++++++++-- .../auto-deployment/planner-live-retune.md | 6 +- 3 files changed, 625 insertions(+), 183 deletions(-) diff --git a/components/src/dynamo/planner/retune/live/frontend_config.py b/components/src/dynamo/planner/retune/live/frontend_config.py index b1a765405bc1..8a2e6b95e361 100644 --- a/components/src/dynamo/planner/retune/live/frontend_config.py +++ b/components/src/dynamo/planner/retune/live/frontend_config.py @@ -24,19 +24,22 @@ annotation value could also be or create an option ("dangerous"), as could a ``$(VAR)`` the env leaves undefined that an ``envFrom`` source may define (its ``prefix`` plus a ConfigMap or Secret key). Any other - ``fieldRef`` (the operator's ``POD_*`` among them) and a - ``resourceFieldRef`` never start with ``-`` or hold whitespace, so they - stay unreadable values but never stand for an option ("inert"). - ``envFrom`` sources are otherwise assumed to set no router env; - ``evaluate_premises`` warns about them, and P14 compares the routers' - policy parameters with the parsed startup. + ``fieldRef`` (the operator's ``POD_*`` among them), a + ``resourceFieldRef`` and the operator env of ``_operator_inert_env`` + (``DYN_NAMESPACE`` among them) never start with ``-`` or hold + whitespace, so they stay unreadable values but never stand for an option + ("inert"). ``envFrom`` sources are otherwise assumed to set no router + env; ``evaluate_premises`` warns about them, and P14 compares the + routers' policy parameters with the parsed startup. - ``evaluate_premises`` checks P4 and P7-P10 and warns on P6 and P7; P14 is the router actuator's. P4 also parses every worker component's router - flags and env the same way (``_worker_router_errors``): a worker that - resolves a router mode advertises its own router configuration, which - replaces the frontend's for its worker set, and one where dangerous env - could stand for an option fails too. ``p4_ok`` alone decides whether a - retune controller can be built. + flags and env the same way (``_worker_router_errors``), each role's pod + template for a component that has only those, and skips a component + with a native ``runtime`` sidecar, whose main runs the engine: a worker + that resolves a router mode advertises its own router configuration, + which replaces the frontend's for its worker set, and one where + dangerous env could stand for an option fails too. ``p4_ok`` alone + decides whether a retune controller can be built. Every function is pure (a dict in, a value out, no Kubernetes calls) and never reads the planner's own environment. @@ -87,21 +90,31 @@ DEFAULT_FRONTEND_ARGS = ("-m", "dynamo.frontend") FRONTEND_MODULE = "dynamo.frontend" # The operator's worker invocation (component_common.go getCommonContainer), -# kept by graph.go's merge under an empty authored command. A native-sidecar -# main, one whose pod template has an init container named RUNTIME_CONTAINER, -# skips that base container (graph.go: native-sidecar engines retain their -# image entrypoint), so its args stay an exec-form argv. +# kept by graph.go's merge under an empty authored command. DEFAULT_WORKER_COMMAND = ("/bin/sh", "-c") -# consts.go RuntimeContainerName. +# consts.go RuntimeContainerName: an init container of this name makes the +# component a native sidecar, whose main runs the engine. RUNTIME_CONTAINER = "runtime" -# Worker options known to take exactly one value, so an unreadable value -# right after one is that value: were it itself an option, the listed one -# would lack its value and the worker would fail at startup. The worker's -# router parser knows only the router flags, not the arity of engine or -# runtime flags (--enforce-eager takes none, --connector a variable number), -# so only these exempt a value; the recipes need no other. Before adding an -# option, check its nargs and action in every backend that accepts it. -_WORKER_VALUE_OPTIONS = frozenset({"--speculative-config"}) +# runtime_status.go ResolveComponentRuntimeStatus, in its priority order. +_MODEL_FLAGS = ("--served-model-name", "--model-name", "--model", "--model-path") +# Worker options known to take at least one value, so an unreadable value +# right after one is a value: were it itself an option, the listed one would +# lack its value and the worker would fail at startup. The worker's router +# parser knows only the router flags, not the arity of engine or runtime +# flags (--enforce-eager takes none, --connector a variable number), so only +# these exempt a value. Checked arities: vLLM --model (one) and +# --served-model-name (nargs "+", whose first value this is); +# SGLang --model-path and --served-model-name (one); TensorRT-LLM --model, +# --model-path and --served-model-name (one); the mocker's --model-path and +# --model-name (one); --speculative-config (one JSON value). Before adding +# an option, check its nargs and action in every backend that accepts it. +_WORKER_VALUE_OPTIONS = frozenset({"--speculative-config", *_MODEL_FLAGS}) +# Left-over frontend options known to take exactly one value, in the sglang +# chat processor and in vLLM's. Other left-over options go to the chat +# processor too, where some take none (vLLM's --enable-auto-tool-choice). +_FRONTEND_VALUE_OPTIONS = frozenset( + {"--chat-template", "--reasoning-parser", "--tool-call-parser"} +) # Env the operator injects into every component (component_common.go # getCommonContainer); user env of the same name overrides it. @@ -112,6 +125,27 @@ "POD_NAMESPACE": "metadata.namespace", "POD_UID": "metadata.uid", } +# Operator env whose value the planner does not compute but which never +# starts with "-" or holds whitespace: getCommonContainer's Dynamo namespace +# and component type; on a worker, component_worker.go GetBaseContainer's +# system server, health check, NIXL telemetry and forward-pass metric +# settings; on a multinode component, graph.go addMultinodeTopologyEnvVars' +# leader hostname and rank. Container env wins over envFrom, so no envFrom +# source defines them. Names the operator sets only in some configurations +# (NATS_SERVER, ETCD_ENDPOINTS, DYN_DISCOVERY_BACKEND, CONTAINER_NAME, the +# TLS paths) are not modeled. +_OPERATOR_INERT_ENV = ("DYN_COMPONENT", "DYN_NAMESPACE") +_OPERATOR_WORKER_INERT_ENV = ( + "DYN_FORWARDPASS_METRIC_PORT", + "DYN_HEALTH_CHECK_ENABLED", + "DYN_SYSTEM_ENABLED", + "DYN_SYSTEM_PORT", + "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS", + "NIXL_TELEMETRY_ENABLE", + "NIXL_TELEMETRY_EXPORTER", + "NIXL_TELEMETRY_PROMETHEUS_PORT", +) +_OPERATOR_MULTINODE_INERT_ENV = ("DYNAMO_LEADER_ADDRESS", "DYNAMO_RANK") # The frontend's request trace recipe (P10). ENV_POD_NAME = "DYN_POD_NAME" @@ -194,8 +228,6 @@ "conditional_disagg_config", } ) -# runtime_status.go ResolveComponentRuntimeStatus, in its priority order. -_MODEL_FLAGS = ("--served-model-name", "--model-name", "--model", "--model-path") _SHELLS = frozenset({"sh", "bash", "dash", "zsh", "ash", "ksh"}) # Shell option letters that take the next argument (-o pipefail, -O extglob), # and bash's long options that do. @@ -231,6 +263,8 @@ # Commands that parse a quoted word again or turn it into env, so that one # word can still reach the worker's options or env. _SHELL_REPARSING = frozenset({".", "env", "eval", "export", "source"}) +# Commands that run their first operand as the command. +_SHELL_LAUNCHERS = frozenset({"builtin", "command", "exec", "nohup"}) class FrontendParseError(ValueError): @@ -293,7 +327,8 @@ def served_model_names( Uses the operator's ``status.components[].servedModelName`` while the status is current, else the first of ``--served-model-name``, ``--model-name``, ``--model`` and ``--model-path`` in the main - container's command and args (the operator's order), after Kubernetes + container's command and args (the operator's order; of each role's pod + template, for a component with role templates only), after Kubernetes ``$(VAR)`` expansion against the component's env, skipping values that reference env the planner cannot read. """ @@ -306,7 +341,7 @@ def served_model_names( continue name = str(component.get("name", "")) model = (projected.get(name) or {}).get("servedModelName") or _flag_model( - _model_flags(component, _component_env(dgd, component)) + _component_model_flags(dgd, component) ) if model: models[name] = str(model) @@ -645,12 +680,14 @@ def relevant(name: str) -> bool: for arg in args ] namespace, unknown = parser.parse_known_args(expanded) - hidden_flag = _opaque_flag_position(unknown) + hidden_flag = _opaque_flag_position(unknown, value_options=_FRONTEND_VALUE_OPTIONS) if hidden_flag is not None: original = dict(zip(expanded, args)).get(hidden_flag, hidden_flag) raise FrontendParseError( f"P4: frontend arg {original!r} references env the planner cannot " - "read and is not the value of a frontend flag, so it could set a " + "read and is neither attached to an option nor the value of a " + "frontend flag or of one known to take a value " + f"({', '.join(sorted(_FRONTEND_VALUE_OPTIONS))}), so it could set a " "router flag" ) unresolved = sorted( @@ -841,29 +878,36 @@ def _frontend_args(argv: Sequence[str]) -> list[str]: def _component_env(dgd: Mapping[str, Any], spec: Mapping[str, Any]) -> _Env: """Merge a component's env the way the operator renders it. - The operator injects ``DYN_PARENT_DGD_K8S_*`` and ``POD_*``, MergeEnvs - overlays graph ``env`` and the main container's ``env`` and sorts the - result by name; Kubernetes expands ``$(VAR)`` against earlier names and - every ``envFrom`` key. + The operator injects ``DYN_PARENT_DGD_K8S_*``, ``POD_*`` and the inert + names of ``_operator_inert_env``, except into a native-sidecar main, + which gets no base container; MergeEnvs overlays graph ``env`` and the + main container's ``env`` and sorts the result by name; Kubernetes + expands ``$(VAR)`` against earlier names and every ``envFrom`` key. """ metadata = dgd.get("metadata") or {} - entries: dict[str, Mapping[str, Any]] = { - ENV_PARENT_DGD_NAME: { + entries: dict[str, Mapping[str, Any]] = {} + operator_inert: set[str] = set() + if not _native_sidecar(spec): + entries[ENV_PARENT_DGD_NAME] = { "name": ENV_PARENT_DGD_NAME, "value": str(metadata.get("name", "")), - }, - ENV_PARENT_DGD_NAMESPACE: { + } + entries[ENV_PARENT_DGD_NAMESPACE] = { "name": ENV_PARENT_DGD_NAMESPACE, "value": str(metadata.get("namespace", "")), - }, - } - for name, path in _OPERATOR_FIELD_REF_ENV.items(): - entries[name] = {"name": name, "valueFrom": {"fieldRef": {"fieldPath": path}}} + } + for name, path in _OPERATOR_FIELD_REF_ENV.items(): + entries[name] = { + "name": name, + "valueFrom": {"fieldRef": {"fieldPath": path}}, + } + operator_inert.update(_operator_inert_env(spec)) main = _main_container(spec) for entry in [*_graph_env(dgd), *(main.get("env") or [])]: if isinstance(entry, Mapping) and entry.get("name"): entries[str(entry["name"])] = entry + operator_inert.discard(str(entry["name"])) env_from = tuple( source for source in main.get("envFrom") or [] if isinstance(source, Mapping) ) @@ -872,7 +916,11 @@ def _component_env(dgd: Mapping[str, Any], spec: Mapping[str, Any]) -> _Env: values: dict[str, str] = {} opaque: set[str] = set() inert: set[str] = set() - for name in sorted(entries): + for name in sorted({*entries, *operator_inert}): + if name in operator_inert: + opaque.add(name) + inert.add(name) + continue entry = entries[name] source = entry.get("valueFrom") if source is not None: @@ -905,6 +953,23 @@ def _component_env(dgd: Mapping[str, Any], spec: Mapping[str, Any]) -> _Env: ) +def _operator_inert_env(spec: Mapping[str, Any]) -> list[str]: + """The ``_OPERATOR_*INERT_ENV`` names the operator sets on a component. + + The worker names need an explicit worker type (component_common.go + ComponentDefaultsFactory gives an untyped component the common base + container only), the multinode ones a ``multinode`` component, whose + ``nodeCount`` defaults to 2. + """ + + names = list(_OPERATOR_INERT_ENV) + if _type(spec) and _type(spec) in WORKER_TYPES: + names.extend(_OPERATOR_WORKER_INERT_ENV) + if isinstance(spec.get("multinode"), Mapping): + names.extend(_OPERATOR_MULTINODE_INERT_ENV) + return names + + def _inert_source(source: Any) -> bool: """Whether a ``valueFrom`` value can never be or create an option. @@ -982,7 +1047,7 @@ def _expand( def _opaque_flag_position( - unknown: Sequence[str], *, value_options: Optional[frozenset[str]] = None + unknown: Sequence[str], *, value_options: frozenset[str] ) -> Optional[str]: """A left-over argument whose dangerous env could stand for a flag. @@ -993,29 +1058,22 @@ def _opaque_flag_position( whose name is unreadable always could. Inert env never stands for a flag, and ``--option=$(VAR)`` is the option's value whatever it holds. - A reference expands to exactly one argument, so it is a value after an - option that takes exactly one: one that starts with ``-`` makes that - option's own parser fail at startup. With ``value_options`` unset, every - left-over option counts as one (the frontend hands them to its chat - processor); a worker passes the options known to take one value, since - its router parser cannot tell an engine flag that takes none - (``--enforce-eager``) or optional values (``--connector``), which the - worker would let the reference follow as a flag. + A reference expands to exactly one argument, so right after one of + ``value_options``, the left-over options known to take a value, it is + that value: one that starts with ``-`` makes that option's own parser + fail at startup. After any other left-over option it could be parsed as + a flag: the frontend hands its left-over options to its chat processor + and a worker to its engine, and both have flags that take no value + (vLLM's ``--enable-auto-tool-choice``, ``--enforce-eager``) or + optional values (``--connector``). """ - def takes_value(option: Optional[str]) -> bool: - if option is None: - return False - if value_options is not None: - return option in value_options - return option.startswith("-") and "=" not in option - previous: Optional[str] = None for token in unknown: if token.startswith("-"): if _UNREADABLE in token.split("=", 1)[0]: return token - elif token.startswith(_OPAQUE) and not takes_value(previous): + elif token.startswith(_OPAQUE) and previous not in value_options: return token previous = token return None @@ -1024,7 +1082,12 @@ def takes_value(option: Optional[str]) -> bool: def _env_from_note( reference: Optional[str], opaque: frozenset[str], prefixes: Sequence[str] ) -> str: - """Says why a reference is unreadable when only an envFrom source may define it.""" + """Says why a reference to env (``$(NAME)``, ``$NAME``) is unreadable. + + Empty unless the name is not opaque env and an envFrom source may + define it. A script variable's reference, which names its source + (``$NAME(=)``), never gets the note. + """ name = (reference or "").strip("$()") if name in opaque or not _env_from_may_define(name, prefixes): @@ -1235,35 +1298,51 @@ def _worker_router_errors( after one of ``_WORKER_VALUE_OPTIONS``; in shell form any dangerous ``$(VAR)``, which Kubernetes substitutes before the shell parses, and the script references ``_shell_expand`` marks. + + A component with role pod templates and no component ``podTemplate`` + is checked role by role, each role's template as its pods run it. A + component with a native ``runtime`` sidecar is skipped: its main runs + the engine with its image entrypoint, and the sidecar registers no + router configuration. """ errors: list[str] = [] env_from: list[str] = [] for component in components: - if _type(component) not in WORKER_TYPES: + if _type(component) not in WORKER_TYPES or _native_sidecar(component): continue name = str(component.get("name", "")) - env = _component_env(dgd, component) - argv, shell = _worker_argv(component, env) - detail = _worker_router_cached( - canonical_json( - { - "argv": argv, - "shell": shell, - "values": dict(env.values), - "opaque": sorted(env.opaque), - "env_from_prefixes": list(env.env_from_prefixes), - } - ) - ) - if detail is not None: - errors.append( - f"P4: worker component {name} {detail}; a worker's router " - "configuration replaces the frontend's for its workers, so set " - f"--router-mode and {ENV_ROUTER_MODE} on the frontend only (a " - "graph-level env entry reaches every worker)" + roles = [ + (role, spec, _component_env(dgd, spec)) + for role, spec in _role_specs(component) + ] + for role, spec, env in roles: + argv, shell = _worker_argv(spec, env) + detail = _worker_router_cached( + canonical_json( + { + "argv": argv, + "shell": shell, + "values": dict(env.values), + "opaque": sorted(env.opaque), + "env_from_prefixes": list(env.env_from_prefixes), + } + ) ) - if any(_env_from_may_set_router_env(source) for source in env.env_from): + if detail is not None: + label = name if role is None else f"{name} (role {role})" + errors.append( + f"P4: worker component {label} {detail}; a worker's router " + "configuration replaces the frontend's for its workers, so set " + f"--router-mode and {ENV_ROUTER_MODE} on the frontend only (a " + "graph-level env entry reaches every worker)" + ) + break + if any( + _env_from_may_set_router_env(source) + for _, _, env in roles + for source in env.env_from + ): env_from.append(name) if env_from: warnings.append( @@ -1337,47 +1416,73 @@ def _worker_router_mode( opaque_word = next((word for word in argv if _OPAQUE in word), None) if opaque_word is None: return None - reference = _opaque_reference(opaque_word) + reference = _opaque_reference(opaque_word) or "" + note = _env_from_note(reference, opaque, env_from_prefixes) + if reference.startswith("$("): + return ( + f"has a shell script that references {reference}, which " + "Kubernetes substitutes into the script before the shell parses " + "it, so its value could become an option wherever it stands, and " + f"the planner cannot read it{note}, so it could select a router " + "mode; give the value literally or use the shell's own reference " + f'attached to its option (--option="${reference[2:-1]}")' + ) return ( - f"has a shell script that references {reference} where it could " - "become an option, and the planner cannot read it" - f"{_env_from_note(reference, opaque, env_from_prefixes)}, so it could " - "select a router mode; give the value literally, quote the reference " - '("$VAR") outside the worker\'s arguments, or attach it to its ' - 'option (--option="$VAR")' + f"has a shell script that references {_describe_reference(reference)} " + f"where it could become an option, and the planner cannot read it{note}, " + "so it could select a router mode; give the value literally, quote the " + 'reference ("$VAR") outside the worker\'s arguments, or attach it to ' + 'its option (--option="$VAR")' ) hidden_flag = _opaque_flag_position(remainder, value_options=_WORKER_VALUE_OPTIONS) if hidden_flag is None: return None reference = _opaque_reference(hidden_flag) + note = _env_from_note(reference, opaque, env_from_prefixes) + if hidden_flag.startswith("-"): + return ( + f"has an option whose name references {reference}, which the planner " + f"cannot read{note}, so it could select a router mode; spell the " + "option's name literally" + ) return ( f"has an argument that references {reference}, which the planner cannot " - f"read{_env_from_note(reference, opaque, env_from_prefixes)} and which " - "is neither attached to an option nor the value of one known to take a value " + f"read{note} and which is neither attached to an option nor the value of " + "one known to take a value " f"({', '.join(sorted(_WORKER_VALUE_OPTIONS))}), so it could select a " "router mode; give the value literally or attach it to its option " "(--option=$(VAR))" ) +def _describe_reference(reference: str) -> str: + """A shell reference as a message shows it, with its source when it has one. + + ``_ShellScan`` writes a script variable assigned from dangerous env as + ``$NAME(=)``, since a marker holds no whitespace. + """ + + match = re.fullmatch(r"(\$[A-Za-z_][A-Za-z0-9_]*)\(=(.*)\)", reference) + if match is None: + return reference + return f"{match.group(1)} (assigned from {match.group(2)})" + + def _worker_argv(component: Mapping[str, Any], env: _Env) -> tuple[list[str], bool]: """A worker's argv after ``$(VAR)`` expansion, and whether a shell runs it. - An empty command is the operator's ``/bin/sh -c``, except for a - native-sidecar main, which keeps its image entrypoint and so runs its - args. A shell runs the command when it starts with one, or when a - shell's ``-c`` follows a launcher (``env bash -c``, ``tini -- sh -c``); - then its script's own references are resolved (``_shell_expand``) and - every part is split into words, so the flags of the commands it runs are - seen. Unreadable ``$(VAR)`` references stay marked. A standalone ``--`` - is dropped: a launcher consumes it, while the router parser would read - every later flag as a positional. + An empty command is the operator's ``/bin/sh -c``. A shell runs the + command when it starts with one, or when a shell's ``-c`` follows a + launcher (``env bash -c``, ``tini -- sh -c``); then its script's own + references are resolved (``_shell_expand``) and every part is split + into words, so the flags of the commands it runs are seen. Unreadable + ``$(VAR)`` references stay marked. A standalone ``--`` is dropped: a + launcher consumes it, while the router parser would read every later + flag as a positional. """ main = _main_container(component) - command = list(main.get("command") or []) - if not command and not _native_sidecar(component): - command = list(DEFAULT_WORKER_COMMAND) + command = list(main.get("command") or []) or list(DEFAULT_WORKER_COMMAND) parts = [env.expand(str(part)) for part in [*command, *(main.get("args") or [])]] script = _shell_script_index(parts) shell = script is not None or ( @@ -1415,6 +1520,31 @@ def _native_sidecar(component: Mapping[str, Any]) -> bool: ) +def _role_specs( + component: Mapping[str, Any], +) -> list[tuple[Optional[str], Mapping[str, Any]]]: + """The component as each of its pods runs it, with the role's name. + + v1beta1_helpers.go EffectiveComponentForRole: a component ``podTemplate``, + or no role ``podTemplate``, serves every pod (role ``None`` here); + otherwise each role with a ``podTemplate`` runs that template. The + roles stay in each copy, so ``_native_sidecar`` still reads the + leader's. + """ + + roles = [ + role + for role in component.get("roles") or [] + if isinstance(role, Mapping) and isinstance(role.get("podTemplate"), Mapping) + ] + if isinstance(component.get("podTemplate"), Mapping) or not roles: + return [(None, component)] + return [ + (str(role.get("name", "")), {**component, "podTemplate": role["podTemplate"]}) + for role in roles + ] + + def _shell_script_index(parts: Sequence[str]) -> Optional[int]: """The index of the script a shell's ``-c`` runs in ``parts``, if any. @@ -1463,16 +1593,23 @@ def _shell_expand(script: str, env: _Env) -> str: - literal env becomes its value as the shell passes it on (split into words when unquoted), so the router parser sees an option it holds; - - dangerous env becomes ``_OPAQUE`` where its value could be or create - an option of the worker: unquoted, which the shell splits into words; - anywhere in an argument of ``eval`` or of a shell given ``-c`` (also - behind a launcher), which parse it again; or quoted as a whole - argument of the command that runs the worker (``-m dynamo.``) or - of one that turns it into code or env (source, env, export, a shell), - unless it follows one of ``_WORKER_VALUE_OPTIONS``. An assignment - neither splits nor passes it on, but leaves its variable dangerous; + - dangerous env (a ConfigMap, Secret or annotation ``valueFrom``, or a + name the container's env leaves to an ``envFrom`` source, which with + an unprefixed source includes image env such as ``PATH``) becomes + ``_OPAQUE`` where its value could be or create an option of the + worker: unquoted, which the shell splits into words; anywhere in an + argument of ``eval`` or of a shell given ``-c`` (also behind a + launcher), which parse it again; or quoted as a whole argument of the + command that runs the worker (``-m dynamo.``) or of one that turns + it into code or env (source, env, export, a shell, named by path or + behind exec, nohup, command or builtin), unless it follows one of + ``_WORKER_VALUE_OPTIONS``. A redirection target other than a + here-string names a file and never counts. An assignment neither + splits nor passes it on, but leaves its variable dangerous, and a + reference to that variable names the env it was assigned from; - other unreadable env becomes ``_INERT``; - - script variables, image env and special parameters stay as written; + - script variables, special parameters and image env that no + ``envFrom`` source may define stay as written; - a default or alternate word is scanned in their place wherever the shell may use it. @@ -1480,11 +1617,12 @@ def _shell_expand(script: str, env: _Env) -> str: it leaves literal env and explicit dangerous env of its name, and a dangerous earlier assignment, in place. Command substitution, backticks and arithmetic are neither evaluated nor rejected, and neither are - scripts the worker sources or runs, or literal text that ``eval`` or a - nested shell parses again; a script variable counts as dangerous only - when the script assigns it from dangerous env directly (not through an - array, positional parameters, a function argument or a command's - output). + scripts the worker sources or runs, literal text that ``eval`` or a + nested shell parses again, text that a shell or ``xargs`` reads from a + heredoc, a here-string or a pipe, or a script ``su -c`` runs; a script + variable counts as dangerous only when the script assigns it from + dangerous env directly (not through an array, positional parameters, a + function argument or a command's output). """ return _ShellScan(script, env).run() @@ -1510,9 +1648,14 @@ def __init__(self, script: str, env: _Env) -> None: self.command: Optional[str] = None self.command_dangerous = False self.redirect = False + self.here_string = False self.heredocs: list[tuple[str, bool]] = [] # Script variables, and whether each was assigned dangerous env. self.variables: dict[str, bool] = {} + # The first dangerous reference in the current word, and the one + # each dangerous script variable was assigned from. + self.source: Optional[str] = None + self.sources: dict[str, str] = {} def run(self) -> str: while self.index < len(self.script): @@ -1521,8 +1664,13 @@ def run(self) -> str: char = script[self.index] if char == "\x00": end = script.find("\x00", self.index + 1) - self.dangerous |= script.startswith(_OPAQUE, self.index) - self.emit(len(script) if end == -1 else end + 1, "") + end = len(script) if end == -1 else end + 1 + if script.startswith(_OPAQUE, self.index): + self.dangerous = True + self.source = self.source or _opaque_reference( + script[self.index : end] + ) + self.emit(end, "") elif not self.quoted and char in _SHELL_BLANKS: self.end_word() self.copy(self.index + 1) @@ -1572,7 +1720,7 @@ def end_word(self) -> None: word = self.word self.command_dangerous |= self.dangerous if self.redirect: - self.redirect = False + self.redirect = self.here_string = False else: match = _SHELL_ASSIGNMENT.match(word) if match and (self.assignment or self.command in _SHELL_DECLARATIONS): @@ -1595,12 +1743,13 @@ def end_word(self) -> None: False, False, ) + self.source = None def end_command(self) -> None: if self.command == "for" and len(self.words) > 1 and self.command_dangerous: self.variables[self.words[1]] = True self.words, self.command = [], None - self.command_dangerous = self.redirect = False + self.command_dangerous = self.redirect = self.here_string = False def assign(self, name: str, dangerous: bool) -> None: # The scan does not follow control flow, so the assignment may not @@ -1612,6 +1761,8 @@ def assign(self, name: str, dangerous: bool) -> None: or self.variables.get(name, False) or (name in env.opaque and name not in env.inert) ) + if dangerous and self.source is not None: + self.sources.setdefault(name, self.source) def operator(self) -> None: script, index = self.script, self.index @@ -1623,10 +1774,12 @@ def operator(self) -> None: self.copy(end) return if script[index] in "<>" or script.startswith("&>", index): - # A redirection, whose target is no argument. + # A redirection, whose target is never an argument of the + # command, unless a here-string feeds it the target's text. end = index + 1 while end < len(script) and script[end] in "<>&|": end += 1 + self.here_string = script.startswith("<<<", index) self.copy(end) self.redirect = True return @@ -1716,8 +1869,14 @@ def reference(self, name: str, end: int, operator: str = "") -> None: else: dangerous = kind == "dangerous" self.dangerous |= dangerous + # A marker holds no whitespace: _worker_argv splits the script. + shown = f"${name}" + if self.variables.get(name) and name in self.sources: + shown += f"(={self.sources[name]})" + if dangerous: + self.source = self.source or shown marker = _OPAQUE if dangerous and self.option_position() else _INERT - self.emit(end, "", f"{marker}${name}\x00") + self.emit(end, "", f"{marker}{shown}\x00") def kind(self, name: str) -> str: env = self.env @@ -1758,6 +1917,9 @@ def option_position(self) -> bool: if self.in_word and self.assignment: return False + if self.redirect and not self.here_string: + # A redirection target names a file or descriptor. + return False if not self.quoted or self.reparses(): return True lead = self.word if self.in_word else "" @@ -1769,10 +1931,8 @@ def option_position(self) -> bool: if self.command is None: # The command's name: one word, a program to run. return False - if ( - self.command in _SHELL_REPARSING - or posixpath.basename(self.command) in _SHELLS - ): + launched = self.launched() + if launched in _SHELL_REPARSING or launched in _SHELLS: return True words = self.words for word, following in zip(words, [*words[1:], ""]): @@ -1783,15 +1943,26 @@ def option_position(self) -> bool: return True return False + def launched(self) -> str: + """The current command's name by basename, past ``_SHELL_LAUNCHERS``.""" + + if self.command is None: + return "" + for word in self.words[self.words.index(self.command) :]: + name = posixpath.basename(word) + if not word.startswith("-") and name not in _SHELL_LAUNCHERS: + return name + return "" + def reparses(self) -> bool: """Whether the current command parses its arguments again as a script. - ``eval`` does, and so does a shell given ``-c``, also behind a - launcher (``exec``, ``nohup``, ``env``), whose positional parameters - its script may expand unquoted. + ``eval`` does, also behind ``_SHELL_LAUNCHERS``, and so does a shell + given ``-c``, also behind a launcher (``exec``, ``nohup``, ``env``), + whose positional parameters its script may expand unquoted. """ - return self.command == "eval" or _shell_script_index(self.words) is not None + return self.launched() == "eval" or _shell_script_index(self.words) is not None def _backtick_end(script: str, index: int) -> int: @@ -2181,6 +2352,18 @@ def _model_flags( return found +def _component_model_flags( + dgd: Mapping[str, Any], component: Mapping[str, Any] +) -> list[tuple[str, Optional[str]]]: + """``_model_flags`` of every pod template the component runs (``_role_specs``).""" + + return [ + flag + for _, spec in _role_specs(component) + for flag in _model_flags(spec, _component_env(dgd, spec)) + ] + + def _flag_model(flags: Sequence[tuple[str, Optional[str]]]) -> Optional[str]: """The operator's served model: the first flag of ``_MODEL_FLAGS`` given.""" @@ -2206,7 +2389,7 @@ def _worker_model_names( ) if served: names.add(str(served)) - for _, value in _model_flags(component, _component_env(dgd, component)): + for _, value in _component_model_flags(dgd, component): if value: names.update(part for part in re.split(r"[\s,]+", value) if part) return names diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py index d994b218fdca..544339bc85fb 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_frontend_config.py @@ -587,6 +587,14 @@ def test_opaque_reference_in_a_non_router_arg_is_fine(): "$(HTTP_PORT)", ), (*KV_ARGS, "--dyn-chat-processor", "vllm", "--tool-call-parser=$(HTTP_PORT)"), + # Attached to an option that takes no value, it is the option's value. + ( + *KV_ARGS, + "--dyn-chat-processor", + "vllm", + "--enable-auto-tool-choice", + "--some-vllm-opt=$(HTTP_PORT)", + ), ], ) def test_opaque_reference_in_a_typed_non_router_arg_is_fine(args): @@ -622,6 +630,31 @@ def test_opaque_reference_in_a_typed_non_router_arg_is_fine(args): (*KV_ARGS, "--tool-call-parser", "--$(HTTP_PORT)", "$(HTTP_PORT)"), "could set a router flag", ), + # The chat processor's flag takes no value, so the frontend would + # parse what follows it as its own flag. + ( + ( + *KV_ARGS, + "--dyn-chat-processor", + "vllm", + "--enable-auto-tool-choice", + "$(HTTP_PORT)", + ), + "could set a router flag", + ), + ( + ( + *KV_ARGS, + "--dyn-chat-processor", + "vllm", + "--tool-call-parser", + "hermes", + "--enable-auto-tool-choice", + "--trust-remote-code", + "$(HTTP_PORT)", + ), + "could set a router flag", + ), # Inert env is still unreadable as a router value or an option name. ((*KV_ARGS, "--router-temp", "$(POD_UID)"), "router arg references $(POD_UID)"), ((*KV_ARGS, "--$(POD_NAME)"), "could set a router flag"), @@ -1221,18 +1254,31 @@ def minimax_prefill_worker(*router_args: str) -> dict[str, Any]: ( ("--$(A_MODE)=kv",), [{"name": "A_MODE", **SECRET}], - "has an argument that references $(A_MODE)", + "has an option whose name references $(A_MODE)", ), ( ("--router-$(A_MODE)", "kv"), [{"name": "A_MODE", **SECRET}], - "has an argument that references $(A_MODE)", + "has an option whose name references $(A_MODE)", + ), + # Inert env could still spell an option's name. + ( + ("--x-$(MY_POD)=1",), + [field_ref_env("MY_POD", "metadata.name")], + "has an option whose name references $(MY_POD)", ), ( ("--speculative-config", "$(A_SECRET)", "$(A_SECRET)"), [{"name": "A_SECRET", **SECRET}], "has an argument that references $(A_SECRET)", ), + # Only the first value of --served-model-name (nargs "+") is exempt: + # a later one starting with "-" is parsed as a flag. + ( + ("--served-model-name", "x", "$(ROUTER_ARG)"), + router_arg_env(), + "has an argument that references $(ROUTER_ARG)", + ), # After an engine flag that takes no value or optional values, the # worker would read "--router-mode=kv" as a flag. ( @@ -1273,6 +1319,9 @@ def test_p4_rejects_a_worker_that_advertises_a_router_config(args, env, match): (("--router-temperature", "$(A_SECRET)"), [{"name": "A_SECRET", **SECRET}]), (("--speculative-config", "$(A_SECRET)"), [{"name": "A_SECRET", **SECRET}]), (("--speculative-config", "$(ROUTER_ARG)"), router_arg_env()), + (("--model", "$(ROUTER_ARG)"), router_arg_env()), + (("--model-path", "$(ROUTER_ARG)"), router_arg_env()), + (("--served-model-name", "$(ROUTER_ARG)"), router_arg_env()), (("--some-opt=$(ROUTER_ARG)",), router_arg_env()), # Inert env never stands for an option. (("$(POD_NAME)",), None), @@ -1285,60 +1334,123 @@ def test_p4_accepts_workers_that_advertise_nothing(args, env): @pytest.mark.parametrize( - ("command", "args", "env_from", "match"), + ("args", "env_from", "match"), [ # A Secret or ConfigMap key may define what env leaves undefined. ( - ("python3",), ("$(ROUTER_ARG)",), ENV_FROM, "references $(ROUTER_ARG), which the planner cannot read (the " "container's env leaves it to an envFrom source)", ), - (("/bin/sh", "-c"), ("$(ROUTER_ARG)",), ENV_FROM, "references $(ROUTER_ARG)"), - (None, ("$(ROUTER_ARG)",), ENV_FROM, "references $(ROUTER_ARG)"), + ( + ("$(APP_ROUTER_ARG)",), + [{**ENV_FROM[0], "prefix": "APP_"}], + "references $(APP_ROUTER_ARG)", + ), + # A prefix that cannot produce the name, the worker's own env, and + # the value of an option known to take one. + (("$(ROUTER_ARG)",), [{**ENV_FROM[0], "prefix": "APP_"}], None), + (("--model", "$(MODEL_ID)"), ENV_FROM, None), + (("--model", "$(MODEL_NAME)"), ENV_FROM, None), + # The operator sets these on the container, whose env wins over + # envFrom, and none of them holds an option. + (("--draft-endpoint", "$(DYN_NAMESPACE)/backend/generate"), ENV_FROM, None), + (("--x", "$(DYN_COMPONENT)", "$(DYN_SYSTEM_PORT)"), ENV_FROM, None), + # The operator sets this one only for some discovery configurations. + (("$(NATS_SERVER)",), ENV_FROM, "references $(NATS_SERVER)"), + ], +) +def test_p4_worker_reference_an_env_from_source_may_define(args, env_from, match): + component = router_worker( + *args, env=[{"name": "MODEL_ID", "value": MODEL}], env_from=env_from + ) + report = evaluate(dgd(frontend(), component, planner())) + rejected = reasons_matching(report, "P4: worker component agg ") + if match is None: + assert report.p4_ok, report.reasons + else: + assert any(match in reason for reason in rejected), report.reasons + + +@pytest.mark.parametrize( + ("command", "script", "match"), + [ + (("/bin/sh", "-c"), "$(ROUTER_ARG)", "references $(ROUTER_ARG)"), + (None, "$(ROUTER_ARG)", "references $(ROUTER_ARG)"), # Image env too, which a shell splits unless quoted; a script's own # assignment overrides what such a source may define. ( ("/bin/sh", "-c"), - (f"export LD_LIBRARY_PATH=/opt/lib:$LD_LIBRARY_PATH; exec {WORKER}",), - ENV_FROM, + f"export LD_LIBRARY_PATH=/opt/lib:$LD_LIBRARY_PATH; exec {WORKER}", "references $LD_LIBRARY_PATH where it could become an option, and " "the planner cannot read it (the container's env leaves it to an " "envFrom source)", ), ( ("/bin/sh", "-c"), - (f'export LD_LIBRARY_PATH="/opt/lib:$LD_LIBRARY_PATH"; exec {WORKER}',), - ENV_FROM, + f'export LD_LIBRARY_PATH="/opt/lib:$LD_LIBRARY_PATH"; exec {WORKER}', None, ), - (("/bin/sh", "-c"), (f"ARGS=; exec {WORKER} $ARGS",), ENV_FROM, None), - ( - ("python3",), - ("$(APP_ROUTER_ARG)",), - [{**ENV_FROM[0], "prefix": "APP_"}], - "references $(APP_ROUTER_ARG)", - ), - # A prefix that cannot produce the name, and the worker's own env. - (("python3",), ("$(ROUTER_ARG)",), [{**ENV_FROM[0], "prefix": "APP_"}], None), - (("python3",), ("--model", "$(MODEL_ID)"), ENV_FROM, None), + (("/bin/sh", "-c"), f"ARGS=; exec {WORKER} $ARGS", None), + # A redirection target is a file, never an argument. + (("/bin/sh", "-c"), f'exec {WORKER} > "$HOME/worker.log" 2>&1', None), + # Operator env the container's env defines. + (None, f"echo $DYN_NAMESPACE $DYN_COMPONENT; exec {WORKER}", None), + (None, f"exec {WORKER} --endpoint dyn://$DYN_NAMESPACE.backend.generate", None), + (None, f"echo $DYN_SYSTEM_PORT; exec {WORKER}", None), ], ) -def test_p4_worker_reference_an_env_from_source_may_define( - command, args, env_from, match +def test_p4_worker_script_reference_an_env_from_source_may_define( + command, script, match ): - component = worker() - main = component["podTemplate"]["spec"]["containers"][0] - main.update( - args=["-m", "dynamo.vllm", *args] if command == ("python3",) else list(args), + component = shell_worker( + script, + command=command, env=[{"name": "MODEL_ID", "value": MODEL}], - envFrom=env_from, + env_from=ENV_FROM, ) - if command is None: - del main["command"] + report = evaluate(dgd(frontend(), component, planner())) + rejected = reasons_matching(report, "P4: worker component agg ") + if match is None: + assert report.p4_ok, report.reasons else: - main["command"] = list(command) + assert any(match in reason for reason in rejected), report.reasons + + +@pytest.mark.parametrize( + "env_from", [ENV_FROM, [{**ENV_FROM[0], "prefix": "APP_"}]], ids=["bare", "APP_"] +) +def test_p4_names_the_env_a_dangerous_script_variable_holds(env_from): + component = shell_worker( + f"X=$ROUTER_ARG; exec {WORKER} $X", env=router_arg_env(), env_from=env_from + ) + report = evaluate(dgd(frontend(), component, planner())) + (reason,) = reasons_matching(report, "P4: worker component agg ") + assert "references $X (assigned from $ROUTER_ARG) where" in reason + assert "envFrom source" not in reason + + +@pytest.mark.parametrize( + ("env", "match"), + [ + (None, None), + # User env of an operator name overrides the operator's. + ( + [ + { + "name": "DYN_NAMESPACE", + "valueFrom": {"configMapKeyRef": {"name": "c", "key": "ns"}}, + } + ], + "references $(DYN_NAMESPACE)", + ), + ], +) +def test_p4_user_env_overrides_operator_env(env, match): + component = router_worker( + "--draft-endpoint", "$(DYN_NAMESPACE)/backend/generate", env=env + ) report = evaluate(dgd(frontend(), component, planner())) rejected = reasons_matching(report, "P4: worker component agg ") if match is None: @@ -1347,6 +1459,17 @@ def test_p4_worker_reference_an_env_from_source_may_define( assert any(match in reason for reason in rejected), report.reasons +def test_p4_untyped_component_gets_no_worker_operator_env(): + # ComponentDefaultsFactory gives an untyped component the common base + # container only, so an envFrom source may define DYN_SYSTEM_PORT. + component = router_worker("$(DYN_SYSTEM_PORT)", env_from=ENV_FROM, type_="") + report = evaluate(dgd(frontend(), component, planner())) + assert any( + "references $(DYN_SYSTEM_PORT)" in reason + for reason in reasons_matching(report, "P4: worker component agg ") + ), report.reasons + + def test_p4_checks_an_untyped_worker(): report = evaluate( dgd( @@ -1506,7 +1629,7 @@ def test_p4_shell_wrapped_worker(command, script, match): ), ( f"X=$ROUTER_ARG; exec {WORKER} $X", - "has a shell script that references $X", + "has a shell script that references $X (assigned from $ROUTER_ARG)", ), # An assignment may not run, so it clears neither env nor an # earlier dangerous value. @@ -1554,6 +1677,48 @@ def test_p4_shell_wrapped_worker(command, script, match): f"exec bash -c 'exec {WORKER} $@' bash \"$ROUTER_ARG\"", "has a shell script that references $ROUTER_ARG", ), + # source, env, export and a shell turn one quoted word into code or + # env, also by path and behind exec, nohup, command or builtin. + ( + f'env "$ROUTER_ARG" {WORKER}', + "has a shell script that references $ROUTER_ARG", + ), + ( + f'exec env "$ROUTER_ARG" {WORKER}', + "has a shell script that references $ROUTER_ARG", + ), + ( + f'exec env A=1 "$ROUTER_ARG" {WORKER}', + "has a shell script that references $ROUTER_ARG", + ), + ( + f'/usr/bin/env "$ROUTER_ARG" {WORKER}', + "has a shell script that references $ROUTER_ARG", + ), + ( + f'nohup env "$ROUTER_ARG" {WORKER}', + "has a shell script that references $ROUTER_ARG", + ), + ( + f'command export "$ROUTER_ARG"; exec {WORKER}', + "has a shell script that references $ROUTER_ARG", + ), + ( + 'command eval "$ROUTER_ARG"', + "has a shell script that references $ROUTER_ARG", + ), + (f'cp -r /cfg/. "$ROUTER_ARG"; exec {WORKER}', None), + # A redirection target names a file, unless a here-string feeds + # its text to the command. + (f'exec {WORKER} > "$CFG" 2>&1', None), + (f"exec {WORKER} 2>$CFG", None), + ( + f'exec {WORKER} > "$CFG" "$ROUTER_ARG"', + "has a shell script that references $ROUTER_ARG", + ), + ("exec bash <<< $ROUTER_ARG", "has a shell script that references $ROUTER_ARG"), + # The value of an option known to take one. + ('exec python3 -m dynamo.vllm --model "$ROUTER_ARG"', None), (f'{WORKER} --speculative-config="${{SPECULATIVE_CONFIG}}"', None), (f'{WORKER} --speculative-config "$SPECULATIVE_CONFIG"', None), (f'{WORKER} "--some-opt=$ROUTER_ARG"', None), @@ -1638,32 +1803,126 @@ def test_p4_worker_behind_a_launcher(command, args, match): assert any(match in reason for reason in rejected), report.reasons -@pytest.mark.parametrize("native_sidecar", [True, False]) -def test_p4_native_sidecar_main_keeps_its_args_as_exec_form(native_sidecar): - # The main keeps its image entrypoint instead of the operator's - # /bin/sh -c, so an option's attached value is not a shell script. +MODEL_FROM_CONFIG_MAP = [ + {"name": "MODEL", "valueFrom": {"configMapKeyRef": {"name": "c", "key": "model"}}} +] +RUNTIME_SIDECAR = {"name": "runtime", "restartPolicy": "Always"} + + +@pytest.mark.parametrize( + ("command", "args"), + [ + (None, ["--model-path", "$(MODEL)", "--tp", "2"]), + (["python3", "-m", "tensorrt_llm.commands.serve"], ["$(MODEL)"]), + (None, ["--model-path=$(MODEL)", "--router-mode", "kv"]), + ], +) +@pytest.mark.parametrize("leader_only", [False, True], ids=["component", "leader"]) +def test_p4_skips_native_sidecar_mains(command, args, leader_only): + # The main runs the engine with its image entrypoint, and the runtime + # sidecar registers no router configuration. component = worker() spec = component["podTemplate"]["spec"] main = spec["containers"][0] - del main["command"] - main.update( - args=["--model-path=$(MODEL)", "--tp", "2"], - env=[ + main.update(args=args, env=MODEL_FROM_CONFIG_MAP) + if command is None: + del main["command"] + else: + main["command"] = command + if leader_only: + component["multinode"] = {"nodeCount": 2} + component["roles"] = [ { - "name": "MODEL", - "valueFrom": {"configMapKeyRef": {"name": "c", "key": "model"}}, + "name": "leader", + "podTemplate": {"spec": {"initContainers": [RUNTIME_SIDECAR]}}, } - ], - ) - if native_sidecar: - spec["initContainers"] = [{"name": "runtime", "restartPolicy": "Always"}] + ] + else: + spec["initContainers"] = [RUNTIME_SIDECAR] + report = evaluate(dgd(frontend(), component, planner())) + assert report.p4_ok, report.reasons + + +def test_p4_reads_an_omitted_command_without_a_native_sidecar_as_a_script(): + component = worker() + main = component["podTemplate"]["spec"]["containers"][0] + del main["command"] + main.update(args=["--model-path=$(MODEL)", "--tp", "2"], env=MODEL_FROM_CONFIG_MAP) report = evaluate(dgd(frontend(), component, planner())) - if native_sidecar: + (reason,) = reasons_matching(report, "P4: worker component agg ") + # Quoting or attaching never helps a $(VAR): Kubernetes substitutes it + # before the shell parses the script. + assert "has a shell script that references $(MODEL), which Kubernetes" in reason + assert '(--option="$MODEL")' in reason + assert "quote the reference" not in reason + + +def role_worker( + leader: dict[str, Any], worker_: dict[str, Any], **extra: Any +) -> dict[str, Any]: + """A multinode worker whose launches are only in its role pod templates.""" + + roles = [] + for name, main in (("leader", leader), ("worker", worker_)): + container = {"name": "main", "command": ["python3", "-m", "dynamo.vllm"]} + container.update(main) + roles.append( + {"name": name, "podTemplate": {"spec": {"containers": [container]}}} + ) + return { + "name": "agg", + "type": "worker", + "replicas": 1, + "multinode": {"nodeCount": 2}, + "roles": roles, + **extra, + } + + +ROLE_ARGS = ["--model", MODEL, "--tensor-parallel-size", "16"] + + +@pytest.mark.parametrize( + ("leader", "worker_", "match"), + [ + ( + {"args": [*ROLE_ARGS, "--router-mode", "kv"]}, + {"args": ROLE_ARGS}, + "agg (role leader) advertises its own router configuration", + ), + ( + {"args": ROLE_ARGS}, + {"args": ROLE_ARGS, "env": [{"name": "DYN_ROUTER_MODE", "value": "kv"}]}, + "agg (role worker) advertises its own router configuration", + ), + ( + {"args": [*ROLE_ARGS, "$(ROUTER_ARG)"], "env": router_arg_env()}, + {"args": ROLE_ARGS}, + "agg (role leader) has an argument that references $(ROUTER_ARG)", + ), + ({"args": ROLE_ARGS}, {"args": ROLE_ARGS}, None), + # The operator sets the rank, so envFrom cannot. + ( + {"args": ROLE_ARGS, "envFrom": ENV_FROM}, + { + "args": [*ROLE_ARGS, "--node-rank", "$(DYNAMO_RANK)"], + "envFrom": ENV_FROM, + }, + None, + ), + ], +) +def test_p4_checks_each_role_pod_template(leader, worker_, match): + report = evaluate(dgd(frontend(), role_worker(leader, worker_), planner())) + if match is None: assert report.p4_ok, report.reasons + assert fc.served_model_names(dgd(role_worker(leader, worker_)))[0] == { + "agg": MODEL + } else: assert any( - "has a shell script that references $(MODEL)" in reason - for reason in reasons_matching(report, "P4: worker component agg ") + match in reason + for reason in reasons_matching(report, "P4: worker component ") ), report.reasons diff --git a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md index 6388bb7e75a5..117db53ca8d0 100644 --- a/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md +++ b/docs/fern/pages/kubernetes/auto-deployment/planner-live-retune.md @@ -266,7 +266,7 @@ retries: | Rule | Requirement | |---|---| -| P4 | The frontend runs `--router-mode kv` with every routing field the Request cannot express at its default, `kv_event_lag_ms` unset or 0, no `DYN_ROUTER_WORKER_SELECTION_POLICY`, a `none` prefill load model, router env given as plain values (no `valueFrom`), and a command that is not wrapped in a shell. The Planner cannot read `envFrom` sources: it assumes they set no router env, only warns about a source whose `prefix` could produce a `DYN_` name, and treats a `$(VAR)` that only an `envFrom` source could define as unreadable. Keep router env out of `envFrom`: P14 catches only a changed policy parameter. No worker advertises its own router configuration: worker components set neither `--router-mode` nor `DYN_ROUTER_MODE`, directly or through the graph-level `spec.env`, because a worker's advertised configuration replaces the frontend's for its workers. The Planner reads a worker's command as the operator runs it: an omitted `command` is `/bin/sh -c` with the first of `args` as the script, unless the pod has a native `runtime` sidecar, and a shell started through `env`, `tini --` or `dumb-init --` is a script too. A value the Planner cannot read and that could hold an option (a ConfigMap, Secret or annotation `valueFrom`, or a `$(VAR)` that the container's `env` leaves to an `envFrom` source) must not stand where a worker option could: attach it to its option (`--option=$(VAR)`) or pass it right after `--speculative-config`. In a worker shell script any such `$(VAR)` fails, and so does a `$VAR` or `${VAR}` of such a value that is unquoted, stands alone as a worker argument, or sits in text that `eval` or a nested `sh -c` parses again; quote it outside the worker's arguments and attach it to its option among them (`--option="$VAR"`). The Planner does not follow the script's conditions, so the script's own assignment to a ConfigMap, Secret or annotation variable does not clear it. With an `envFrom` source that has no `prefix`, every name the container's `env` leaves undefined counts as such a value, image env such as `PATH` and `LD_LIBRARY_PATH` included, so quote those references too (`export LD_LIBRARY_PATH="/usr/local/nvidia/lib64:$LD_LIBRARY_PATH"`). Other `fieldRef` values, such as the pod name, are fine anywhere. The Planner does not evaluate command substitution, backticks or the scripts a worker runs, so keep router flags out of them | +| P4 | The frontend runs `--router-mode kv` with every routing field the Request cannot express at its default, `kv_event_lag_ms` unset or 0, no `DYN_ROUTER_WORKER_SELECTION_POLICY`, a `none` prefill load model, router env given as plain values (no `valueFrom`), and a command that is not wrapped in a shell. The Planner cannot read `envFrom` sources: it assumes they set no router env, only warns about a source whose `prefix` could produce a `DYN_` name, and treats a `$(VAR)` that only an `envFrom` source could define as unreadable. Keep router env out of `envFrom`: P14 catches only a changed policy parameter. No worker advertises its own router configuration: worker components set neither `--router-mode` nor `DYN_ROUTER_MODE`, directly or through the graph-level `spec.env`, because a worker's advertised configuration replaces the frontend's for its workers. The Planner reads a worker's command as the operator runs it: an omitted `command` is `/bin/sh -c` with the first of `args` as the script, and a shell started through `env`, `tini --` or `dumb-init --` is a script too. For a worker whose launches are in role pod templates (`roles[].podTemplate`, no component `podTemplate`), it reads each role's main container and env. A worker with a native `runtime` sidecar is not checked: its main runs the engine, and the sidecar advertises no router configuration. A value the Planner cannot read and that could hold an option (a ConfigMap, Secret or annotation `valueFrom`, or a `$(VAR)` that the container's `env` leaves to an `envFrom` source) must not stand where a worker option could: attach it to its option (`--option=$(VAR)`) or pass it as the separate value right after `--model`, `--model-path`, `--served-model-name`, `--model-name` or `--speculative-config`. A frontend argument follows the same rule, with `--tool-call-parser`, `--reasoning-parser` and `--chat-template` as the options whose separate value it may be. In a worker shell script any such `$(VAR)` fails, since Kubernetes substitutes it before the shell parses the script; use the shell's own `$VAR` instead. A `$VAR` or `${VAR}` of such a value fails when it is unquoted, stands alone as an argument of the worker or of `env`, `export`, `source` or a shell (also by path and behind `exec`, `nohup`, `command` or `builtin`), or sits in text that `eval` or a shell given `-c` parses again; quote it outside the worker's arguments and attach it to its option among them (`--option="$VAR"`). A redirection target, such as `> "$LOG"`, is fine. The Planner does not follow the script's conditions, so the script's own assignment to a ConfigMap, Secret or annotation variable does not clear it. With an `envFrom` source that has no `prefix`, every name the container's `env` leaves undefined counts as such a value, image env such as `PATH` and `LD_LIBRARY_PATH` included, so quote those references too (`export LD_LIBRARY_PATH="/usr/local/nvidia/lib64:$LD_LIBRARY_PATH"`). Names the operator sets on the container do not count: `DYN_NAMESPACE`, `DYN_COMPONENT`, `DYN_PARENT_DGD_K8S_*` and `POD_*`, a worker's `DYN_SYSTEM_*`, `DYN_HEALTH_CHECK_ENABLED`, `NIXL_TELEMETRY_*` and `DYN_FORWARDPASS_METRIC_PORT`, and a multinode component's `DYNAMO_RANK` and `DYNAMO_LEADER_ADDRESS`. Other `fieldRef` values, such as the pod name, may stand anywhere except inside an option's name (`--x-$(POD_NAME)`) or as the value of a frontend router flag. The Planner does not evaluate command substitution, backticks, the scripts a worker runs, text that a shell or `xargs` reads from a heredoc, a here-string or a pipe, or a script that `su -c` runs, so keep router flags and such values out of them | | P7 | The DGD serves one model; the frontend is not an EPP, not a `frontendSidecar`, and has no TLS | | P8 | One frontend replica, or `--router-replica-sync` | | P9 | No `scalingAdapter` on worker components; one Planner replica | @@ -397,8 +397,8 @@ The Planner exports these on `metric_reporting_prometheus_port`: | Retune with load or throughput scaling, or of Planner parameters | A DGDR Run has no base Planner configuration, warm-up or predictor pre-sweep | | Frontends with non-default routing fields beyond the three knobs, or `kv_event_lag_ms` other than 0 | The DGDR router section expresses only the three knobs, and the Run does not pass a lag | | Workers that set their own router mode (`--router-mode` or `DYN_ROUTER_MODE`) | A worker's advertised router configuration replaces the frontend's for its workers, and the Run replays only the frontend's | -| Worker arguments that pass a ConfigMap, Secret, annotation or `envFrom` value where an option could stand, including an unquoted `$VAR` of one in a worker shell script or one that `eval` or a nested shell parses again (P4 fails) | The Planner cannot read the value, so it cannot rule out a router mode the worker would advertise | -| Router flags that a worker produces through shell command substitution, backticks, script variables, positional parameters, literal text that `eval` or a nested shell parses again, or a script file (P4 does not see them) | The Planner reads the worker's spec and does not run its script | +| Worker arguments that pass a ConfigMap, Secret, annotation or `envFrom` value where an option could stand, including an unquoted `$VAR` of one in a worker shell script or one that `eval` or a shell given `-c` parses again (P4 fails) | The Planner cannot read the value, so it cannot rule out a router mode the worker would advertise | +| Router flags that a worker produces through shell command substitution, backticks, script variables, positional parameters, literal text that `eval` or a nested shell parses again, or a script file, and ConfigMap, Secret, annotation or `envFrom` values in text that a shell or `xargs` reads from a heredoc, a here-string or a pipe, or that `su -c` runs (P4 does not see them) | The Planner reads the worker's spec and does not run its script | | Raising `overlap_score_credit` on a frontend that starts it at 0: an omitted credit then searches only 0, and a written positive value fails P4 | A live router rejects a positive credit after starting at 0 | | Private, fine-tuned or local-path models | The Run Job has no Hugging Face credentials or model mount | | Several frontends without router replica sync | AISimulate simulates one router | From 76630c6009bda6cf85cab6d5626a00ee781696cb Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 15:12:14 -0800 Subject: [PATCH 099/100] fix(planner): keep a released live retune Lease released A renewal thread still running when stop() released the Lease could take it over again, so the successor waited a full lease duration. LeaseLeadership.release() is now final: a step that reads the Lease after it stands by and writes nothing. - The cancelled-initialization test runs the late step in a real thread. - New tests cover a lapse reported after initialization and the step lock between a rebuild and the lease loop. - The live_retune_sla_errors docstring says when an inherited SLA change would reach the controller. Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- .../src/dynamo/planner/reconfiguration.py | 9 +- .../src/dynamo/planner/retune/live/state.py | 18 ++- .../dynamo/planner/retune/live/supervisor.py | 7 +- .../tests/unit/test_live_retune_state.py | 15 +++ .../tests/unit/test_live_retune_supervisor.py | 109 ++++++++++++++++-- 5 files changed, 144 insertions(+), 14 deletions(-) diff --git a/components/src/dynamo/planner/reconfiguration.py b/components/src/dynamo/planner/reconfiguration.py index 4a50fcfc1d12..a8874a0d243f 100644 --- a/components/src/dynamo/planner/reconfiguration.py +++ b/components/src/dynamo/planner/reconfiguration.py @@ -249,9 +249,12 @@ def live_retune_sla_errors( ) -> list[str]: """Return why ``candidate`` cannot replace ``config`` under live retune. - The live retune controller and its state fingerprint take the scoring SLA - when they are built and are not rebuilt on a runtime update, so a change - of an inherited ``ttft_ms`` or ``itl_ms`` would never reach them. + The live retune controller and the supervisor's state fingerprint read + the scoring SLA only when they are built: at startup, or at a rebuild for + a spec, P4 or DGD change. A runtime change of an inherited ``ttft_ms`` or + ``itl_ms`` would leave the running controller scoring with the old SLA + until such a rebuild, which would then apply the new SLA and, through its + new fingerprint, reset the persisted retune state. """ if config.retune is None or candidate.retune is None: diff --git a/components/src/dynamo/planner/retune/live/state.py b/components/src/dynamo/planner/retune/live/state.py index fa1a8cac51e5..c3bbdef4f898 100644 --- a/components/src/dynamo/planner/retune/live/state.py +++ b/components/src/dynamo/planner/retune/live/state.py @@ -695,7 +695,7 @@ class LeaseLeadership: expired foreign holder, or its own lease after it had lost track). - ``renewed``: it led and still does. - ``lost``: it led and another holder now owns the lease. - - ``standby``: another holder leads. + - ``standby``: another holder leads, or ``release`` has run. - ``error``: the API call failed; leadership, if any, lasts until ``leader_until``. @@ -744,6 +744,8 @@ def __init__( self._held = False self._leader_until = -math.inf self._acquisitions = 0 + # Set by ``release``: a step that reads the Lease after it never leads. + self._released = False self.transitions = 0 # resourceVersion of this process's last successful write. self.resource_version: Optional[str] = None @@ -829,6 +831,10 @@ def step(self) -> LeaseStep: "retune leadership: reading lease %s failed: %s", self.name, error ) return "error" + with self._lock: + if self._released: + # A step still running on another thread when release ran. + return "standby" if lease is None: return self._create(t_send) @@ -889,8 +895,16 @@ def step(self) -> LeaseStep: return self._take_over(lease, t_send, transitions, lost) def release(self) -> bool: - """Clear the holder if it is still this process (conditional update).""" + """Clear the holder if it is still this process (conditional update). + + Final: a ``step`` whose read of the Lease returns after this started + reports ``standby`` and writes nothing. A step that read the Lease + earlier can still write; this clears that write only if it lands + before this method's own read. + """ + with self._lock: + self._released = True held = self.held self.drop() if not held: diff --git a/components/src/dynamo/planner/retune/live/supervisor.py b/components/src/dynamo/planner/retune/live/supervisor.py index 8229d7c6dfee..d69ece31fbaa 100644 --- a/components/src/dynamo/planner/retune/live/supervisor.py +++ b/components/src/dynamo/planner/retune/live/supervisor.py @@ -2165,8 +2165,11 @@ async def _end_renewals(self, keepalive: asyncio.Task[None]) -> None: A renewal in flight is waited for first: cancelled, its thread would go on and race the next step on the resourceVersion. If this task is - cancelled meanwhile (``stop``), the renewal is cancelled in flight, - as the lease loop's own step is, rather than left running. + cancelled meanwhile (``stop``), the renewal task is cancelled without + waiting, as the lease loop's own step is: the renewal's thread may + still finish its step, and once ``stop`` has released the + ``LeaseLeadership``, a step that reads the Lease after that is a + no-op. """ try: diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py index dd9ee79b4a8d..74f7e4c6a260 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_state.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_state.py @@ -600,6 +600,21 @@ def test_empty_holder_is_taken_over_at_once_and_release_is_conditional() -> None assert holder(kube) == "b/1" +def test_a_step_after_release_never_leads_again() -> None: + clock = FakeClock() + kube = FakeRetuneKube() + a = leadership(kube, clock, identity="a/1") + assert a.step() == "acquired" + assert a.release() and holder(kube) is None + # A step still running when the Lease was released stands by instead of + # taking the empty holder over, or creating the Lease again. + assert a.step() == "standby" + assert holder(kube) is None and not a.is_leader() + del kube.leases[a.name] + assert a.step() == "standby" + assert a.name not in kube.leases and not a.is_leader() + + def test_leadership_conflicts_and_errors() -> None: clock = FakeClock() kube = FakeRetuneKube() diff --git a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py index 33fd7793d098..e9864dddfd15 100644 --- a/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py +++ b/components/src/dynamo/planner/tests/unit/test_live_retune_supervisor.py @@ -82,6 +82,7 @@ LiveRetuneSupervisor, SpecEvaluation, SupervisorSettings, + _to_thread, current_window_end_ms, due_window_end_ms, plan_initialization, @@ -2204,18 +2205,28 @@ async def test_a_cancelled_initialization_leaves_no_renewal_behind() -> None: process = await started(world) sup = process.sup steps = 0 - in_flight, gate = asyncio.Event(), asyncio.Event() + in_flight = asyncio.Event() + gate, returned = threading.Event(), threading.Event() + late: list[str] = [] + + def gated_step(call): + gate.wait(10) + try: + late.append(call()) + finally: + returned.set() async def gated(call): # The third leadership step, after the acquiring and the confirming - # ones, is the initialization's first renewal: it stays in flight, - # and the initialization goes on only once it is. + # ones, is the initialization's first renewal: it runs on a thread, + # as the production runner's steps do, and blocks there until the + # gate opens. The initialization goes on only once it is in flight. nonlocal steps if getattr(call, "func", None) == sup._leadership.step: steps += 1 if steps == 3: in_flight.set() - await gate.wait() + return await _to_thread(lambda: gated_step(call)) elif steps >= 2: await in_flight.wait() return call() @@ -2227,7 +2238,7 @@ async def gated(call): if sup.initialized: break await asyncio.sleep(0) - # Initialized; the round now waits for the renewal in flight. + # Initialized; the round waits for the renewal in flight. assert sup.initialized and in_flight.is_set() and not task.done() task.cancel() await asyncio.wait([task]) @@ -2238,9 +2249,93 @@ async def gated(call): ] assert pending == [] await sup.stop() + assert world.holder() is None + # The cancelled renewal's thread goes on and reads the Lease after stop + # released it: it neither takes it over nor leads. gate.set() - await asyncio.sleep(0) - assert world.holder() is None and not sup.is_leader() + assert await asyncio.to_thread(returned.wait, 10) + assert late == ["standby"] + assert world.holder() is None and not sup._leadership.is_leader() + + +async def test_a_lapse_reported_after_initialization_ends_the_term() -> None: + world = World() + process = await started(world) + sup = process.sup + steps = 0 + in_flight, gate = asyncio.Event(), asyncio.Event() + + async def gated(call): + # The third leadership step, the initialization's first renewal, + # stays in flight until the initialization has finished and returns + # after local leadership ended: it reports a lapse (acquired). + nonlocal steps + if getattr(call, "func", None) == sup._leadership.step: + steps += 1 + if steps == 3: + in_flight.set() + await gate.wait() + world.clock.advance(LEASE) + elif steps >= 2: + await in_flight.wait() + return call() + + sup._blocking = gated + sup._sleep = lambda _seconds: asyncio.sleep(0) + task = asyncio.create_task(sup.lease_round()) + for _ in range(1000): + if sup.initialized: + break + await asyncio.sleep(0) + assert sup.initialized and in_flight.is_set() + term_seq = sup.term_seq + # The round waits for the renewal in flight. + for _ in range(100): + await asyncio.sleep(0) + assert not task.done() + gate.set() + assert await task == "acquired" + lease = world.kube.leases[LEADER_NAME] + assert sup._leadership.resource_version == lease["metadata"]["resourceVersion"] + # The lapse ends the term the initialization started. + assert sup._leadership.acquisitions == 2 and not sup.initialized + sup._blocking = inline + await sup.lease_round() + assert sup.initialized and sup.term_seq == term_seq + 1 + assert world.shared_state().leader_term == sup.term + + +async def test_leadership_steps_of_a_rebuild_and_the_lease_loop_never_overlap() -> None: + world = World() + process = await leading(world) + sup = process.sup + active = overlaps = cancelled = 0 + + async def counting(call): + # Every leadership step yields to the event loop while in flight. + nonlocal active, overlaps, cancelled + if getattr(call, "func", None) != sup._leadership.step: + return call() + active += 1 + if active > 1: + overlaps += 1 + try: + for _ in range(5): + await asyncio.sleep(0) + return call() + except asyncio.CancelledError: + cancelled += 1 + raise + finally: + active -= 1 + + sup._blocking = counting + # P8 breaks: the rebuild re-initializes next to a lease round. + world.spec.spec_ok = False + world.clock.advance(20) + await asyncio.gather(sup.control_round(), sup.lease_round()) + assert sup.generation == 1 and sup.is_leader() and sup.initialized + assert overlaps == 0 and cancelled == 0 async def test_failed_renewals_are_retried_a_request_timeout_apart() -> None: From 7ca7737ed36e724b9d9d064c6b2fc218c4a53e4d Mon Sep 17 00:00:00 2001 From: hongkuanz Date: Tue, 3 Nov 2026 16:33:14 -0800 Subject: [PATCH 100/100] fix(planner): narrow worker P4 reference type for mypy Co-Authored-By: Claude Opus 5.5 Signed-off-by: hongkuanz --- components/src/dynamo/planner/retune/live/frontend_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/src/dynamo/planner/retune/live/frontend_config.py b/components/src/dynamo/planner/retune/live/frontend_config.py index 8a2e6b95e361..fba1b3204ed9 100644 --- a/components/src/dynamo/planner/retune/live/frontend_config.py +++ b/components/src/dynamo/planner/retune/live/frontend_config.py @@ -1437,7 +1437,7 @@ def _worker_router_mode( hidden_flag = _opaque_flag_position(remainder, value_options=_WORKER_VALUE_OPTIONS) if hidden_flag is None: return None - reference = _opaque_reference(hidden_flag) + reference = _opaque_reference(hidden_flag) or "" note = _env_from_note(reference, opaque, env_from_prefixes) if hidden_flag.startswith("-"): return (