From 102d6dc893087a1d9646bd1752ff71e62a94eba8 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Sat, 3 Oct 2026 19:25:35 +0800 Subject: [PATCH 01/13] fix: distinguish FPM worker probe sidecars from rank artifacts Signed-off-by: Yiming Liu --- .../collector/fpm_forward/native_artifact.py | 5 ++++ .../test_fpm_worker_probe_artifact.py | 26 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 python/aisimulate/tests/unit/collector/test_fpm_worker_probe_artifact.py diff --git a/python/aisimulate/collector/fpm_forward/native_artifact.py b/python/aisimulate/collector/fpm_forward/native_artifact.py index 402ba4037..2e544d165 100644 --- a/python/aisimulate/collector/fpm_forward/native_artifact.py +++ b/python/aisimulate/collector/fpm_forward/native_artifact.py @@ -471,6 +471,11 @@ def _rank_artifacts(raw_root: Path) -> list[tuple[Path, dict[str, Any]]]: payload = json.loads(path.read_text()) if not isinstance(payload, dict): raise TypeError(f"native benchmark artifact is not a mapping: {path}") + if path.stem.endswith("_merged_worker_probe"): + # Dynamo retains post-run RPC evidence separately from timing rows. + if payload.get("schema") != "dynamo.fpm.benchmark_worker_probe" or payload.get("schema_version") != 1: + raise ValueError(f"unsupported native worker probe sidecar: {path}") + continue if payload.get("artifact_type") == "merged" or path.stem.endswith("_merged"): continue artifacts.append((path, payload)) diff --git a/python/aisimulate/tests/unit/collector/test_fpm_worker_probe_artifact.py b/python/aisimulate/tests/unit/collector/test_fpm_worker_probe_artifact.py new file mode 100644 index 000000000..cd4b87bdf --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_fpm_worker_probe_artifact.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest +from collector.fpm_forward.native_artifact import _rank_artifacts + +pytestmark = pytest.mark.unit + + +def test_worker_probe_sidecar_is_not_a_native_rank(tmp_path): + rank = {"artifact_type": "rank", "dp": {"rank": 0}} + (tmp_path / "benchmark_dp0.json").write_text(json.dumps(rank)) + (tmp_path / "benchmark_merged.json").write_text(json.dumps({"artifact_type": "merged"})) + sidecar = tmp_path / "benchmark_merged_worker_probe.json" + sidecar.write_text(json.dumps({"schema": "dynamo.fpm.benchmark_worker_probe", "schema_version": 1})) + assert _rank_artifacts(tmp_path) == [(tmp_path / "benchmark_dp0.json", rank)] + assert sidecar.exists() + + +@pytest.mark.parametrize("payload", [{}, {"schema": "dynamo.fpm.benchmark_worker_probe", "schema_version": 2}]) +def test_unrecognized_worker_probe_envelope_fails(tmp_path, payload): + (tmp_path / "benchmark_merged_worker_probe.json").write_text(json.dumps(payload)) + with pytest.raises(ValueError, match="worker probe sidecar"): + _rank_artifacts(tmp_path) From eadc40ff6e1d294d692a44254784ae2709eddcce Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Sat, 3 Oct 2026 23:12:34 +0800 Subject: [PATCH 02/13] fix: wait for native FPM artifact finalization Keep each collector engine alive until its local merged benchmark artifact and any advertised worker-probe sidecar are recorded. Bind completion artifacts to the run and local DP shard, retain legacy producers without sidecars, and preserve recorded partial or failed probe evidence for quality validation. Bound post-rank finalization independently of the collection budget, preserve raw timings on failure, and cover delayed merge/probe output, both two-node completion orders, stale artifacts, and deadline failures. Validation: 350 collector tests passed; independently rechecked against failed full-grid and successful P/D smoke artifacts. Signed-off-by: Yiming Liu --- .../collector/fpm_forward/runtime/fpm_exec.sh | 103 ++++- .../tests/unit/collector/test_fpm_exec.py | 360 +++++++++++++++++- .../collector/test_fpm_runtime_wrapper.py | 11 + 3 files changed, 458 insertions(+), 16 deletions(-) diff --git a/python/aisimulate/collector/fpm_forward/runtime/fpm_exec.sh b/python/aisimulate/collector/fpm_forward/runtime/fpm_exec.sh index bc3737c5e..2f1f46aa8 100644 --- a/python/aisimulate/collector/fpm_forward/runtime/fpm_exec.sh +++ b/python/aisimulate/collector/fpm_forward/runtime/fpm_exec.sh @@ -46,6 +46,14 @@ if [[ ! "${FPM_READINESS_TIMEOUT_SECONDS:-}" =~ ^[1-9][0-9]*$ ]] || echo "FPM_READINESS_TIMEOUT_SECONDS must be an integer from 1 through 3600" >&2 exit 2 fi +# Merge and the producer's bounded worker probe must not consume hours of an +# unused collection budget after every local rank has finished timing. +finalization_timeout_seconds="${FPM_FINALIZATION_TIMEOUT_SECONDS-120}" +if [[ ! "$finalization_timeout_seconds" =~ ^[1-9][0-9]*$ ]] || + (( ${#finalization_timeout_seconds} > 4 || finalization_timeout_seconds > 3600 )); then + echo "FPM_FINALIZATION_TIMEOUT_SECONDS must be an integer from 1 through 3600" >&2 + exit 2 +fi if [[ -f "${workdir}/fpm_memory_worker.py" ]]; then # The wrappers are staged alongside this script and delegate all execution @@ -189,7 +197,17 @@ if (( ! fpm_is_follower )); then for ((dp_rank=local_dp_start; dp_rank&2 exit 1 @@ -208,21 +226,29 @@ start_rank = int(sys.argv[1]) expected_mode = sys.argv[2] expected_schema_version = int(sys.argv[3]) allowed_point_types = {"prefill", "decode"} if expected_mode == "agg" else {expected_mode} +pending_status = 10 def invalid(path, message): print(f"Invalid FPM benchmark result {path}: {message}", file=sys.stderr) raise SystemExit(20) -for offset, raw_path in enumerate(sys.argv[4:]): - path = pathlib.Path(raw_path) +def read_result(path): + if path.is_symlink(): + invalid(path, "result must not be a symlink") if not path.is_file() or path.stat().st_size == 0: - raise SystemExit(10) + raise SystemExit(pending_status) try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): - raise SystemExit(10) + raise SystemExit(pending_status) if not isinstance(value, dict): invalid(path, f"top-level JSON must be an object, got {type(value).__name__}") + return value + +paths = [pathlib.Path(raw_path) for raw_path in sys.argv[4:]] +run_identity = None +for offset, path in enumerate(paths): + value = read_result(path) expected_rank = start_rank + offset if (value.get("schema_version") != expected_schema_version or value.get("status") != "complete" or value.get("valid") is not True): @@ -266,6 +292,54 @@ for offset, raw_path in enumerate(sys.argv[4:]): observed_ranks.add(rank) if observed_ranks != {expected_rank}: invalid(path, f"FPM dp_ranks {sorted(observed_ranks)!r} != [{expected_rank}]") + identity = (value.get("run_id"), value.get("grid_digest")) + if not all(isinstance(part, str) and part for part in identity): + invalid(path, "missing run_id or grid_digest") + if run_identity is not None and identity != run_identity: + invalid(path, "run_id or grid_digest differs across local ranks") + run_identity = identity + +# Rank files precede the launcher's merge and optional post-run worker probe. +# Signal a separate finalization deadline once all local ranks are complete. +# A sidecar proves probe recording, not worker restoration; observed failures +# and partial coverage remain consumer evidence. +pending_status = 11 +first_path = paths[0] +merged_path = first_path.with_name(f"{first_path.stem}_merged{first_path.suffix}") +merged = read_result(merged_path) +if (merged.get("artifact_type") != "merged" + or merged.get("schema_version") != expected_schema_version + or merged.get("status") != "complete" or merged.get("valid") is not True): + invalid(merged_path, "merged result is not complete and valid with the expected schema") +if (merged.get("run_id"), merged.get("grid_digest")) != run_identity: + invalid(merged_path, "run_id or grid_digest differs from local rank results") +if not isinstance(merged.get("config"), dict) or merged["config"].get("mode") != expected_mode: + invalid(merged_path, "benchmark mode differs from local rank results") +rank_files = [str(path) for path in paths] +source_ranks = list(range(start_rank, start_rank + len(paths))) +if (merged.get("rank_files") != rank_files or merged.get("merged_output_path") != str(merged_path) + or not isinstance(merged.get("dp"), dict) or merged["dp"].get("source_ranks") != source_ranks): + invalid(merged_path, "merged paths or source ranks do not bind the local DP shard") + +# Older producers have no sidecar contract. Never infer one from an engine +# block or require a file they cannot write. +if "worker_probe_path" in merged: + probe_path = merged_path.with_name(f"{merged_path.stem}_worker_probe{merged_path.suffix}") + if merged["worker_probe_path"] != str(probe_path): + invalid(merged_path, "worker_probe_path does not name the local merged artifact's sidecar") + probe = read_result(probe_path) + if (probe.get("schema") != "dynamo.fpm.benchmark_worker_probe" or probe.get("schema_version") != 1 + or probe.get("run_id") != run_identity[0] + or probe.get("merged_output_path") != str(merged_path) or probe.get("rank_files") != rank_files): + invalid(probe_path, "worker probe schema or run/path binding differs from the merged result") + if not isinstance(probe.get("merged"), dict) or not isinstance(probe.get("ranks"), dict): + invalid(probe_path, "worker probe merged and ranks views must be objects") + if set(probe["ranks"]) != {str(rank) for rank in source_ranks}: + invalid(probe_path, "worker probe rank views do not cover the local DP shard") + for rank, rank_file in zip(source_ranks, rank_files): + view = probe["ranks"][str(rank)] + if not isinstance(view, dict) or view.get("rank_file") != rank_file: + invalid(probe_path, f"worker probe rank {rank} does not bind its rank file") PY } @@ -335,6 +409,7 @@ if (( fpm_is_follower )); then fi deadline=$((SECONDS + FPM_WAIT_TIMEOUT_SECONDS)) +finalization_deadline=0 while true; do set +e @@ -347,13 +422,17 @@ while true; do if (( result_status == 20 )); then exit 1 fi - if (( result_status != 10 )); then - # 10 is the checker's only keep-waiting signal; any other status - # (python traceback, exec failure, OOM-kill) is checker breakage + if (( result_status != 10 && result_status != 11 )); then + # 10 means rank collection pending; 11 means artifact finalization pending. + # Any other status (python traceback, exec failure, OOM-kill) is checker breakage # and must fail now instead of burning the whole wait deadline. echo "FPM result checker failed with unexpected status ${result_status}" >&2 exit 1 fi + if (( result_status == 11 && finalization_deadline == 0 )); then + finalization_deadline=$((SECONDS + finalization_timeout_seconds)) + echo "Local FPM rank outputs complete; allowing ${finalization_timeout_seconds}s for artifact finalization" >&2 + fi if ! kill -0 "$engine_pid" 2>/dev/null; then set +e wait "$engine_pid" @@ -361,11 +440,15 @@ while true; do set -e terminate_engine "$engine_pid" engine_pid="" - echo "Engine exited before writing all FPM benchmark outputs" >&2 + echo "Engine exited before writing all FPM benchmark outputs (rank, merged, and advertised worker probe)" >&2 if (( engine_status == 0 )); then exit 1; else exit "$engine_status"; fi fi if (( SECONDS >= deadline )); then - echo "Timed out waiting for all FPM benchmark outputs" >&2 + echo "Timed out waiting for all FPM benchmark outputs (rank, merged, and advertised worker probe)" >&2 + exit 124 + fi + if (( finalization_deadline > 0 && SECONDS >= finalization_deadline )); then + echo "Timed out finalizing FPM benchmark artifacts after valid local rank outputs; raw timings retained" >&2 exit 124 fi sleep 2 diff --git a/python/aisimulate/tests/unit/collector/test_fpm_exec.py b/python/aisimulate/tests/unit/collector/test_fpm_exec.py index 6c2ee4e36..e77523626 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_exec.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_exec.py @@ -106,6 +106,10 @@ def _benchmark_result( for point_type in point_types ] return { + "artifact_type": "rank", + "run_id": "test-run", + "grid_digest": "test-grid", + "dp": {"rank": dp_rank}, "schema_version": schema_version, "status": status, "valid": valid, @@ -121,7 +125,48 @@ def _benchmark_result( } -def _writer_engine_body(files: dict[str, object], *, delay: float = 0.0) -> str: +def _finalized_artifacts(files: dict[str, object], *, probe_resolution: str | None = None) -> dict[str, object]: + """A launcher merges its local DP shard before its optional worker probe.""" + files = dict(files) + if not files or not all( + isinstance(value, dict) and value.get("artifact_type") == "rank" for value in files.values() + ): + return files + first_path = Path(next(iter(files))) + merged_path = first_path.with_name(f"{first_path.stem}_merged{first_path.suffix}") + ranks = [value["dp"]["rank"] for value in files.values()] + merged = { + **next(iter(files.values())), + "artifact_type": "merged", + "rank_files": list(files), + "merged_output_path": str(merged_path), + "dp": {"source_ranks": ranks}, + } + if probe_resolution is not None: + probe_path = merged_path.with_name(f"{merged_path.stem}_worker_probe{merged_path.suffix}") + merged["worker_probe_path"] = str(probe_path) + probe = { + "schema": "dynamo.fpm.benchmark_worker_probe", + "schema_version": 1, + "run_id": merged["run_id"], + "merged_output_path": str(merged_path), + "rank_files": list(files), + "merged": {"resolution": probe_resolution}, + "ranks": { + str(rank): {"rank_file": path, "resolution": probe_resolution} + for rank, path in zip(ranks, files, strict=True) + }, + } + files[str(merged_path)] = merged + files[str(probe_path)] = probe + else: + files[str(merged_path)] = merged + return files + + +def _writer_engine_body(files: dict[str, object], *, delay: float = 0.0, finalize: bool = True) -> str: + if finalize: + files = _finalized_artifacts(files) serialized = json.dumps(dict(files)) return f"""\ import json @@ -468,8 +513,9 @@ def test_engine_progress_bars_are_kept_out_of_the_exec_stream(tmp_path): ) stderr_bar = "Loading safetensors checkpoint shards: 4% Completed | 2/47 [00:21<06:43, 8.96s/it]" staged = _stage(tmp_path, run_script="") - payload = json.dumps(_benchmark_result()) + payloads = json.dumps(_finalized_artifacts({str(staged.benchmark_output): _benchmark_result()})) engine_body = f"""\ +import json import pathlib import signal import sys @@ -481,7 +527,8 @@ def test_engine_progress_bars_are_kept_out_of_the_exec_stream(tmp_path): print("engine stderr survives the stream filter", file=sys.stderr) sys.stdout.flush() sys.stderr.flush() -pathlib.Path({str(staged.benchmark_output)!r}).write_text({payload!r}) +for path, payload in json.loads({payloads!r}).items(): + pathlib.Path(path).write_text(json.dumps(payload)) signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) while True: time.sleep(0.1) @@ -568,6 +615,7 @@ def test_fpm_exec_bounds_stubborn_engine_shutdown(tmp_path, result_valid, expect pid_path = tmp_path / "engine.pid" child_pid_path = tmp_path / "engine-child.pid" staged = _stage(tmp_path, run_script="", engine_grace_seconds=1) + payloads = json.dumps(_finalized_artifacts({str(staged.benchmark_output): result})) engine_body = f"""\ import json import os @@ -590,9 +638,10 @@ def test_fpm_exec_bounds_stubborn_engine_shutdown(tmp_path, result_valid, expect stderr=subprocess.DEVNULL, ) pathlib.Path(os.environ["FAKE_ENGINE_CHILD_PID_PATH"]).write_text(str(child.pid)) -path = pathlib.Path({str(staged.benchmark_output)!r}) -path.parent.mkdir(parents=True, exist_ok=True) -path.write_text({json.dumps(result)!r}) +for raw_path, payload in json.loads({payloads!r}).items(): + path = pathlib.Path(raw_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload)) while True: time.sleep(0.1) """ @@ -875,6 +924,304 @@ def accept_report() -> None: assert received == [b"1"] +# --------------------------------------------------------------------------- +# post-collection merge/probe completion +# --------------------------------------------------------------------------- + + +def _check_artifacts(paths: list[str], *, start_rank: int = 0): + """Exercise the shipped checker directly, without GPU or engine startup.""" + checker = FPM_EXEC.read_text().split("check_result_files() {", 1)[1].split("<<'PY'\n", 1)[1].split("\nPY\n", 1)[0] + return subprocess.run( + ["python3", "-", str(start_rank), "prefill", str(FPM_NATIVE_BENCHMARK_RESULT_SCHEMA_VERSION), *paths], + input=checker, + text=True, + capture_output=True, + check=False, + ) + + +def _write_artifacts(files: dict[str, object]) -> None: + for path, value in files.items(): + Path(path).write_text(json.dumps(value)) + + +@pytest.mark.parametrize("start_rank", [0, 4]) +def test_completion_checker_waits_for_local_merge_and_advertised_probe(tmp_path, start_rank): + paths = fpm_expected_result_paths(str(tmp_path / "benchmark.json"), start_rank // 4, 4) + ranks = {path: _benchmark_result(dp_rank=start_rank + offset) for offset, path in enumerate(paths)} + artifacts = _finalized_artifacts(ranks, probe_resolution="worker_probe_partial") + merged_path, probe_path = list(artifacts)[-2:] + _write_artifacts(ranks) + + assert _check_artifacts(paths, start_rank=start_rank).returncode == 11 + _write_artifacts({merged_path: artifacts[merged_path]}) + assert _check_artifacts(paths, start_rank=start_rank).returncode == 11 + _write_artifacts({probe_path: artifacts[probe_path]}) + assert _check_artifacts(paths, start_rank=start_rank).returncode == 0 + + +@pytest.mark.parametrize("resolution", [None, "worker_probe_partial", "probe_failed: TimeoutError: "]) +def test_completion_checker_preserves_legacy_and_fail_soft_probe_results(tmp_path, resolution): + path = str(tmp_path / "benchmark.json") + artifacts = _finalized_artifacts({path: _benchmark_result()}, probe_resolution=resolution) + if resolution is None: + artifacts[list(artifacts)[-1]]["engine"] = {"resolution": "legacy_inline_probe"} + _write_artifacts(artifacts) + + result = _check_artifacts([path]) + + assert result.returncode == 0, result.stderr + # Completion does not upgrade a partial or failed observation to success. + if resolution is not None: + assert json.loads(Path(list(artifacts)[-1]).read_text())["merged"]["resolution"] == resolution + + +@pytest.mark.parametrize( + "defect", + [ + "rank-run-id", + "merged-run-id", + "merged-grid", + "merged-ranks", + "merged-rank-files", + "merged-path", + "merged-status", + "merged-type", + "probe-path", + "probe-schema", + "probe-run-id", + "probe-rank-files", + "probe-merged-path", + "probe-rank-binding", + "probe-top-level", + ], +) +def test_completion_checker_rejects_malformed_or_stale_bindings(tmp_path, defect): + paths = fpm_expected_result_paths(str(tmp_path / "benchmark.json"), 1, 2) + ranks = {path: _benchmark_result(dp_rank=2 + offset) for offset, path in enumerate(paths)} + artifacts = _finalized_artifacts(ranks, probe_resolution="worker_probe_partial") + merged_path, probe_path = list(artifacts)[-2:] + merged, probe = artifacts[merged_path], artifacts[probe_path] + if defect == "rank-run-id": + artifacts[paths[1]]["run_id"] = "another-run" + elif defect == "merged-run-id": + merged["run_id"] = "another-run" + elif defect == "merged-grid": + merged["grid_digest"] = "another-grid" + elif defect == "merged-ranks": + merged["dp"]["source_ranks"] = [0, 1] + elif defect == "merged-rank-files": + merged["rank_files"] = paths[:1] + elif defect == "merged-path": + merged["merged_output_path"] = str(tmp_path / "other_merged.json") + elif defect == "merged-status": + merged["status"] = "partial" + elif defect == "merged-type": + merged["artifact_type"] = "rank" + elif defect == "probe-path": + merged["worker_probe_path"] = str(tmp_path / "unrelated.json") + elif defect == "probe-schema": + probe["schema"] = "unrelated" + elif defect == "probe-run-id": + probe["run_id"] = "another-run" + elif defect == "probe-rank-files": + probe["rank_files"] = list(reversed(paths)) + elif defect == "probe-merged-path": + probe["merged_output_path"] = str(tmp_path / "other_merged.json") + elif defect == "probe-rank-binding": + probe["ranks"]["2"]["rank_file"] = paths[1] + elif defect == "probe-top-level": + artifacts[probe_path] = [] + _write_artifacts(artifacts) + + result = _check_artifacts(paths, start_rank=2) + + assert result.returncode == 20, result.stderr + assert "Invalid FPM benchmark result" in result.stderr + + +@pytest.mark.parametrize("resolution", [None, "worker_probe_partial", "probe_failed: TimeoutError: "]) +def test_engine_is_not_terminated_before_delayed_artifact_finalization(tmp_path, resolution): + staged = _stage(tmp_path, run_script="") + artifacts = _finalized_artifacts({str(staged.benchmark_output): _benchmark_result()}, probe_resolution=resolution) + body = _writer_engine_body(artifacts, delay=2.5, finalize=False) + (staged.workdir / "run.sh").write_text(_setsid_run_script(tmp_path, body)) + + completed = _run(staged) + + assert completed.returncode == 0, completed.stderr + assert {path: json.loads(Path(path).read_text()) for path in artifacts} == artifacts + + +@pytest.mark.parametrize("missing", ["merge", "probe"]) +def test_missing_finalization_is_bounded_and_preserves_rank_timings(tmp_path, missing): + staged = _stage(tmp_path, run_script="", env_overrides={"FPM_WAIT_TIMEOUT_SECONDS": 12000}) + rank_path = str(staged.benchmark_output) + ranks = {rank_path: _benchmark_result()} + artifacts = _finalized_artifacts(ranks, probe_resolution="worker_probe_partial") + if missing == "merge": + artifacts = ranks + else: + del artifacts[list(artifacts)[-1]] + body = _writer_engine_body(artifacts, finalize=False) + # A broken finalization deadline fails this test in eight seconds instead + # of retaining the fake engine for the entire collection budget. + body = body.replace("while True:", "for _ in range(80):") + (staged.workdir / "run.sh").write_text(_setsid_run_script(tmp_path, body)) + + started = time.monotonic() + completed = _run(staged, extra_env={"FPM_FINALIZATION_TIMEOUT_SECONDS": "1"}) + + assert completed.returncode == 124, completed.stderr + assert "Timed out finalizing FPM benchmark artifacts" in completed.stderr + assert "raw timings retained" in completed.stderr + assert time.monotonic() - started < 7 + assert json.loads(Path(rank_path).read_text()) == ranks[rank_path] + + +def test_finalization_budget_does_not_start_during_rank_collection(tmp_path): + staged = _stage(tmp_path, run_script="", env_overrides={"FPM_WAIT_TIMEOUT_SECONDS": 10}) + body = "import time\ntime.sleep(3)\n" + _writer_engine_body({str(staged.benchmark_output): _benchmark_result()}) + (staged.workdir / "run.sh").write_text(_setsid_run_script(tmp_path, body)) + + completed = _run(staged, extra_env={"FPM_FINALIZATION_TIMEOUT_SECONDS": "1"}) + + assert completed.returncode == 0, completed.stderr + + +def test_finalization_budget_cannot_extend_collection_deadline(tmp_path): + staged = _stage(tmp_path, run_script="", env_overrides={"FPM_WAIT_TIMEOUT_SECONDS": 1}) + body = _writer_engine_body({str(staged.benchmark_output): _benchmark_result()}, finalize=False) + (staged.workdir / "run.sh").write_text(_setsid_run_script(tmp_path, body)) + + completed = _run(staged, extra_env={"FPM_FINALIZATION_TIMEOUT_SECONDS": "1200"}) + + assert completed.returncode == 124, completed.stderr + assert "Timed out waiting for all FPM benchmark outputs" in completed.stderr + + +@pytest.mark.parametrize("value", ["", "0", "-1", "3601", "not-an-integer", "999999999999999999999999999999999999"]) +def test_invalid_finalization_budget_fails_before_startup(tmp_path, value): + staged = _stage(tmp_path, run_script="#!/bin/bash\nexit 0\n") + + completed = _run(staged, extra_env={"FPM_FINALIZATION_TIMEOUT_SECONDS": value}) + + assert completed.returncode == 2, completed.stderr + assert "FPM_FINALIZATION_TIMEOUT_SECONDS must be an integer from 1 through 3600" in completed.stderr + assert not staged.etcd_trace.exists() + + +@pytest.mark.parametrize("suffix", ["_merged", "_merged_worker_probe"]) +def test_fpm_exec_refuses_stale_finalization_artifacts_before_launch(tmp_path, suffix): + started = tmp_path / "engine-started" + staged = _stage(tmp_path, run_script=f"touch {shlex.quote(str(started))}\n") + path = staged.benchmark_output.with_name(f"benchmark{suffix}.json") + path.write_text("stale") + + completed = _run(staged) + + assert completed.returncode == 1, completed.stderr + assert "Refusing to overwrite existing benchmark output" in completed.stderr + assert path.read_text() == "stale" + assert not started.exists() + + +@pytest.mark.parametrize("engine_exit", [0, 37]) +def test_engine_exit_during_finalization_is_a_failure_with_timings_preserved(tmp_path, engine_exit): + staged = _stage(tmp_path, run_script="") + rank = _benchmark_result() + body = ( + "import pathlib, sys\n" + f"pathlib.Path({str(staged.benchmark_output)!r}).write_text({json.dumps(rank)!r})\n" + f"sys.exit({engine_exit})\n" + ) + (staged.workdir / "run.sh").write_text(_setsid_run_script(tmp_path, body)) + + completed = _run(staged) + + assert completed.returncode == (engine_exit or 1), completed.stderr + assert "Engine exited before writing all FPM benchmark outputs" in completed.stderr + assert json.loads(staged.benchmark_output.read_text()) == rank + + +@pytest.mark.parametrize("slow_node", [0, 1]) +def test_two_node_teardown_waits_for_both_local_artifact_finalizations(tmp_path, slow_node): + nodes = [] + for node in range(2): + directory = tmp_path / f"node{node}" + directory.mkdir() + nodes.append( + _stage( + directory, + run_script="", + env_overrides={ + "FPM_NODE_COUNT": 2, + "FPM_DATA_PARALLEL_SIZE": 2, + "FPM_NODE_RANK": node, + }, + ) + ) + # Both launched wrappers share the leader's readiness and barrier ports, + # but keep separate result volumes, as in the two-node Slurm collection. + follower_script = nodes[1].script.read_text() + follower_script = follower_script.replace(str(nodes[1].etcd_port), str(nodes[0].etcd_port)) + follower_script = follower_script.replace(str(nodes[1].barrier_port), str(nodes[0].barrier_port)) + nodes[1].script.write_text(follower_script) + finalized = [] + for node, staged in enumerate(nodes): + rank_path = fpm_expected_result_paths(str(staged.benchmark_output), node, 1)[0] + artifacts = _finalized_artifacts( + {rank_path: _benchmark_result(dp_rank=node)}, probe_resolution="worker_probe_partial" + ) + finalized.append(list(artifacts)[-1]) + body = f"""\ +import json +import pathlib +import signal +import sys +import time + +def stop(*_): + probes = pathlib.Path({str(tmp_path)!r}).glob("node*/results/*_worker_probe.json") + pathlib.Path({str(tmp_path / f"stopped{node}.json")!r}).write_text( + json.dumps([path.is_file() for path in probes]) + ) + sys.exit(0) + +signal.signal(signal.SIGTERM, stop) +for index, (path, payload) in enumerate(json.loads({json.dumps(artifacts)!r}).items()): + if index > 0: + time.sleep({2.5 if node == slow_node else 0.1}) + pathlib.Path(path).write_text(json.dumps(payload)) +while True: + time.sleep(0.1) +""" + (staged.workdir / "run.sh").write_text(_setsid_run_script(staged.workdir.parent, body)) + processes = [ + subprocess.Popen( + ["bash", str(staged.script)], + env={**staged.env, "FPM_COMPLETION_BARRIER_TIMEOUT_SECONDS": "20"}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + for staged in nodes + ] + try: + for process in processes: + _, stderr = process.communicate(timeout=40) + assert process.returncode == 0, stderr + finally: + for process in processes: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + assert all(Path(path).is_file() for path in finalized) + for node in range(2): + assert json.loads((tmp_path / f"stopped{node}.json").read_text()) == [True, True] + + # --------------------------------------------------------------------------- # contract pins # --------------------------------------------------------------------------- @@ -893,6 +1240,7 @@ def test_fpm_exec_consumes_only_contract_environment(): assert {"FPM_NODE_RANK", "FPM_MASTER_ADDR", "FPM_BENCHMARK_OUTPUT_PATH"} <= consumed allowed = set(FPM_ENV_EXPORTED_VARS) | { "FPM_COMPLETION_BARRIER_TIMEOUT_SECONDS", + "FPM_FINALIZATION_TIMEOUT_SECONDS", "FPM_READINESS_TIMEOUT_SECONDS", "FPM_SLURM_CPUS_PER_TASK", "FPM_SLURM_CPU_BIND", diff --git a/python/aisimulate/tests/unit/collector/test_fpm_runtime_wrapper.py b/python/aisimulate/tests/unit/collector/test_fpm_runtime_wrapper.py index f4cd3ff4b..050109c1d 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_runtime_wrapper.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_runtime_wrapper.py @@ -63,6 +63,8 @@ def _write_executable(path: Path, content: str) -> None: def _valid_result() -> dict: return { + "run_id": "test-run", + "grid_digest": "test-grid", "schema_version": FPM_NATIVE_BENCHMARK_RESULT_SCHEMA_VERSION, "status": "complete", "valid": True, @@ -204,6 +206,14 @@ def test_memory_observer_pythonpath_is_available_before_engine_launch(tmp_path, def test_fpm_exec_leader_starts_etcd_and_cleanup_stops_it(tmp_path): output_path = tmp_path / "results" / "benchmark.json" + merged_path = output_path.with_name("benchmark_merged.json") + merged = { + **_valid_result(), + "artifact_type": "merged", + "rank_files": [str(output_path)], + "merged_output_path": str(merged_path), + "dp": {"source_ranks": [0]}, + } staged = _stage( tmp_path, # Write the complete result, then idle: the engine must stay alive @@ -212,6 +222,7 @@ def test_fpm_exec_leader_starts_etcd_and_cleanup_stops_it(tmp_path): run_script=( "#!/usr/bin/env bash\n" f"printf '%s' {shlex.quote(json.dumps(_valid_result()))} > {shlex.quote(str(output_path))}\n" + f"printf '%s' {shlex.quote(json.dumps(merged))} > {shlex.quote(str(merged_path))}\n" "exec sleep 300\n" ), ) From 8bc44d6d4993efc0ecf1260acb507aa57821ca0d Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Sun, 4 Oct 2026 00:48:15 +0800 Subject: [PATCH 03/13] fix: propagate Slurm worker failures in completion order Observe failed node futures immediately so existing command-scope cancellation stops live peer steps before pool shutdown. Preserve the original error and every node's output logs. Add a real-process regression with a failing second node and live first node, plus multi-node success coverage. All 67 Slurm unit tests and Ruff checks pass. Signed-off-by: Yiming Liu --- .../aisimulate/collector/fpm_forward/slurm.py | 6 +- .../tests/unit/collector/test_fpm_slurm.py | 86 +++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/python/aisimulate/collector/fpm_forward/slurm.py b/python/aisimulate/collector/fpm_forward/slurm.py index de2b718ac..89834b7e3 100644 --- a/python/aisimulate/collector/fpm_forward/slurm.py +++ b/python/aisimulate/collector/fpm_forward/slurm.py @@ -19,7 +19,7 @@ import subprocess import time import uuid -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from aisimulate.fpm_contract import FPM_BENCHMARK_RESULT_GLOB @@ -326,7 +326,9 @@ def run(unit: str) -> None: pool = ThreadPoolExecutor(max_workers=len(pods)) try: futures = [pool.submit(scope.run, run, pod) for pod in pods] - for future in futures: + # A later node can fail while an earlier one is still waiting for + # rendezvous. Observe that failure before joining the live peer. + for future in as_completed(futures): future.result() except BaseException as error: # Worker threads do not receive the main thread's interrupt. diff --git a/python/aisimulate/tests/unit/collector/test_fpm_slurm.py b/python/aisimulate/tests/unit/collector/test_fpm_slurm.py index b600928b7..101df16c0 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_slurm.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_slurm.py @@ -304,6 +304,92 @@ def test_wait_ready_rejects_terminal_or_foreign_allocation(runner, monkeypatch, assert runner.hosts == [] +@pytest.mark.skipif(os.name != "posix", reason="POSIX transport process groups") +def test_execute_observes_later_node_failure_while_first_node_is_alive(runner, monkeypatch): + from collector.fpm_forward import runner as campaign + + readiness = runner.cell_dir / "first-node-ready" + children, failures, cancellations = {}, {}, [] + original_popen = subprocess.Popen + original_cancel = campaign.CommandScope.cancel + monkeypatch.setattr(campaign, "_COMMAND_TERMINATION_GRACE_SECONDS", 0.1) + + def popen(args, **kwargs): + child = original_popen(args, **kwargs) + children[args[-1]] = child + return child + + def cancel(scope): + cancellations.append((scope, children["node0000"].poll() is None)) + return original_cancel(scope) + + def command(unit, _command, *, timeout): + script = ( + "import pathlib,sys,time; " + "print(sys.argv[-1] + ' stdout', flush=True); " + "print(sys.argv[-1] + ' stderr', file=sys.stderr, flush=True); " + ) + if unit == "node0000": + script += "pathlib.Path(sys.argv[1]).write_text('ready'); time.sleep(30)" + else: + deadline = time.monotonic() + 5 + while not readiness.exists(): + assert time.monotonic() < deadline, "first node did not start" + time.sleep(0.005) + script += "sys.exit(17)" + try: + return campaign._run_command([sys.executable, "-c", script, str(readiness), unit], timeout=timeout) + except subprocess.CalledProcessError as error: + failures[unit] = error + raise + + monkeypatch.setattr(subprocess, "Popen", popen) + monkeypatch.setattr(campaign.CommandScope, "cancel", cancel) + monkeypatch.setattr(runner, "_exec", command) + try: + # A broken executor is bounded by the first node's command timeout. + with pytest.raises(subprocess.CalledProcessError) as caught: + runner.execute(["node0000", "node0001"], timeout_seconds=5) + finally: + for child in children.values(): + if child.poll() is None: + child.kill() + child.wait(timeout=2) + + assert caught.value is failures["node0001"] + assert caught.value.returncode == 17 + assert len(cancellations) == 1 and cancellations[0][1] + scope = cancellations[0][0] + assert scope.cancelled and scope.inflight == 0 and not scope.processes + assert children["node0000"].returncode == -signal.SIGTERM + assert children["node0001"].returncode == 17 + for unit in ("node0000", "node0001"): + for stream in ("stdout", "stderr"): + assert (runner.cell_dir / "logs" / f"{unit}.{stream}.log").read_text() == f"{unit} {stream}\n" + with campaign._ACTIVE_COMMANDS_LOCK: + assert not campaign._ACTIVE_COMMANDS + + +def test_execute_retains_all_successful_node_logs(runner, monkeypatch): + second_started = threading.Event() + finished = [] + + def command(unit, _command, *, timeout): + if unit == "node0000": + assert second_started.wait(5) + else: + second_started.set() + finished.append(unit) + return subprocess.CompletedProcess([], 0, stdout=f"{unit} stdout\n", stderr=f"{unit} stderr\n") + + monkeypatch.setattr(runner, "_exec", command) + runner.execute(["node0000", "node0001"]) + assert sorted(finished) == ["node0000", "node0001"] + for unit in finished: + for stream in ("stdout", "stderr"): + assert (runner.cell_dir / "logs" / f"{unit}.{stream}.log").read_text() == f"{unit} {stream}\n" + + @pytest.mark.parametrize("interrupt", [signal.SIGINT, signal.SIGTERM]) @pytest.mark.parametrize("child_mode", ["cooperative", "ignore_term", "pipe_descendant"]) @pytest.mark.skipif(os.name != "posix", reason="POSIX transport process groups") From 5bac3f5cdde38a2a921f7ff8402a84da32b3098a Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Sun, 11 Oct 2026 23:47:27 +0800 Subject: [PATCH 04/13] fix: bind runtime memory evidence to independent collection phases Signed-off-by: Yiming Liu --- .../fpm-self-service/implementation.md | 27 +- .../fpm_forward/runtime_observations.py | 53 ++- .../collector/fpm_forward/runtime_probe.py | 22 +- .../support/collection_readiness.py | 58 ++- .../src/aisimulate/support/runtime.py | 5 + .../aisimulate/support/runtime_collection.py | 314 +++++++++++++ .../tests/unit/test_onboard_runtime.py | 26 +- .../unit/test_onboard_runtime_bindings.py | 428 ++++++++++++++++++ 8 files changed, 918 insertions(+), 15 deletions(-) create mode 100644 python/aisimulate/src/aisimulate/support/runtime_collection.py create mode 100644 python/aisimulate/tests/unit/test_onboard_runtime_bindings.py diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index d6af28285..dc1f73274 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -869,6 +869,7 @@ Execution saves `fpm-readiness.json` in the onboarding output directory and retu | Direct-eligible point counts | Formal `fake_fallback` rows are excluded. Zero eligible decode points blocks full collection and direct prediction. Mixed usable/fallback data covers only the reported samples and is not complete query coverage. | | Available execution-worker observations | Inspect observed attention groups, resolved cache settings and graph configuration. Contradictions in these checked observations are blockers. Missing optional observations remain unqualified without requiring completed memory sizing or serving validation before timing collection. Referenced `fpm-memory-*.json` files are unvalidated sources retained for separate runtime import/finalization checks; their presence does not qualify allocation or memory. | | CPU allocation and affinity | For new Slurm campaigns, inspect `execution.cpu_affinity` and the per-node launcher, per-rank worker and scheduler sidecars. The actual step must expose the requested CPU pool; local schedulers must have sufficient distinct CPUs available, and observed threads must stay within that pool. Missing evidence is incomplete; contradictions or insufficient capacity block full collection. A shared pool does not mean dedicated per-rank pinning. | +| Imported runtime memory profile | For a request backed by accepted probe observations, readiness also verifies the smoke or source collection's raw memory evidence against that profile. Missing phase/rank evidence, changed settings or insufficient usable capacity blocks full collection. This uses the same verifier as explicit independent-phase bindings below. | Zero-KV prefill can retain a native `real_prefix` path stamp while its formal seed regime is `n/a`; the stamp alone does not establish that cached tokens were read. Keep the native provenance and use the validated phase/token counts and formal regime when interpreting readiness. @@ -876,7 +877,7 @@ Dense decode can be synthetic by design only when the producer reports enabled w Readiness counts describe the current reaggregation of verified native artifacts. They do not relabel an existing formal pair. Historical publications without the dense classification marker retain their recorded regimes during verification, finalization and validation; any revised publication needs a fresh output tree while preserving the original pair. -`ready_for_full_collection` permits the next collection step only for the verified sampled scope. It does not establish memory readiness, interpolation coverage, repeatability or workload accuracy. Full execution checks matching usable smoke or formal evidence for every current cell before launching new GPU work; it does not treat a completed memory probe, process exit or existing report file as sufficient. Continue the existing collection-quality, memory-finalization, replay and matched-serving steps afterward. +`ready_for_full_collection` permits the next collection step only for the verified sampled scope. For imported runtime profiles, `runtime_compatibility` separately reports whether the actual collection observations preserve accepted memory; requests without those observations retain the timing-only assessment. Readiness does not establish interpolation coverage, repeatability or workload accuracy. Full execution checks matching usable smoke or formal evidence for every current cell before launching new GPU work; it does not treat a completed memory probe, process exit or existing report file as sufficient. Continue the existing collection-quality, memory-finalization, replay and matched-serving steps afterward. If no usable decode exists, inspect the pinned runtime, exact warm-up flags, eligibility/skip reason and worker logs before deciding why. `fake_fallback` alone does not prove unsupported convolution: TP eligibility, prefix settings, capacity and transient execution failures can also matter. Preserve raw results and completed prefill. Record a deterministic capability failure as blocked and stop unchanged automatic retries. Unsupported convolution or other recurrent-state seeding needs a separate Dynamo/runtime implementation and bounded qualification of the resulting runtime; disabling required warm-up or relabeling synthetic state does not provide support. Continue independent configurations under their own checks. @@ -1256,6 +1257,30 @@ Import writes a validation result for every selected configuration and fresh `re After acceptance, continue ordinary planning and formal timing collection. Observed memory/layout does not establish timing availability, query coverage or serving accuracy. Formal collection automatically inherits the imported profile’s verified bundle and launch evidence; no extra public instrumentation flag is needed on `collect-fpm`. Its deployment options must match the accepted probe. After collection, `runtime-compatibility.json` checks the actual runtime, group geometry and usable capacity against that accepted bound. The existing timing publication and accuracy stages still apply. +#### Bind independently collected phases to an accepted profile + +Campaign orchestration may record a different configuration label or collect prefill and decode in separate successful attempts. Use the shared Python API in `aisimulate.support.runtime` to bind those records explicitly. The default `verify_collection_runtime(request, observations, ...)` remains strict: it requires the original configuration and launch, unchanged runtime and instrumentation, and every required phase in one active attempt. Configuration labels are never interpreted as topology or repaired by stripping suffixes. + +The campaign recorder supplies a list with exactly one binding per phase required by the accepted request. Each binding has these fields: + +| Field | Meaning | +| --- | --- | +| `phase` | `prefill` or `decode`, present in the selected frozen plan. Aggregated requests require both, even when their attempts differ. | +| `plan` | `{ "path": ".../collection-plan.json", "sha256": "..." }`. This digest is the producer's semantic plan SHA, independently recomputed from its immutable inputs. The plan supplies the recorded configuration, launch, bundle and unique phase cell. | +| `review` | Required when runtime, instrumentation or deployment differs from the accepted probe. Exact review record described below; no wildcard or mismatch bypass exists. | +| `checkpoint` | `{ "path": ".../fpm_forward.json", "sha256": "..." }`, with the byte SHA of the completed collector checkpoint. | +| `observations` | `{ "path": ".../runtime-observations.json", "sha256": "..." }`, with the byte SHA of the immutable observation index inside that plan's campaign directory. | +| `attempt_id` | The index's active runtime observation attempt, also recorded in the checkpoint. | +| `collector_attempt_id` | The checkpoint's successful native attempt for that phase's planned cell, also recorded in the phase launch manifest. | + +Before GPU work, pass bindings containing `phase`, `plan` and any `review` to `verify_collection_runtime_plans(request, bindings)`. It checks the frozen plan, archived deployment inputs, local model configuration bytes, precision and launch settings without running observers or workers. This preflight does not establish observed memory. After the bounded smoke or source collection completes, add the checkpoint, index and attempt fields and call `verify_collection_runtime_bindings(request, bindings)` before full-grid repeats. Programmatic readiness uses the same contract through `assess_readiness(request, root, checkpoint_root, runtime_bindings=bindings)`, combining raw memory validation with the ordinary native timing checks. Guided `collect-fpm --check-readiness` builds bindings from its own unchanged campaign automatically; custom campaign recorders supply their explicit bindings through the Python API. + +To inspect a changed runtime, `collection_runtime_review(request, binding)` returns its exact accepted/collection bundle hashes, collector revision, generator configuration hash, and complete sorted `changes` of `{field, accepted, collection}` for runtime source/build metadata and deployment settings. Absent values are represented as `{ "missing": true }`. Inspect the actual source changes, including scheduler patches, mounts and CPU policy, and save this record with a nonempty `rationale` in `review`. Computing that record alone is not a compatibility verdict. Verification recomputes it and rejects missing, altered or stale expectations. Equal cache capacity cannot authorize an unreviewed source change, and reviewed source changes cannot override incompatible model, precision, scheduler, graph or cache properties. + +Each phase is independently checked under its own frozen launch and instrumentation, including artifact containment and hashes, worker/scheduler ranks, tensor geometry and physical capacity. The serving role qualifies only after every required phase passes against the same accepted profile. A Hugging Face repository name and its exact local snapshot spelling can compare as the same model only after their immutable revision, configuration hash and sidecar hashes agree and each observed spelling independently passes the checkpoint-path validator. The report retains those spelling differences and original runtime settings. Other effective settings and geometry remain exact; every phase/rank must provide at least the accepted usable capacity. + +The result contains per-phase `provenance`, original `binding` and compatibility details, the minimum compatible capacity, and an absolute `inputs` list of byte hashes for all consulted source, plan, checkpoint, index, launch, raw and instrumentation files. Save the report separately and recheck those inputs before subsequent CPU prediction or publication. Distinct historical attempts remain distinct; do not merge indexes, rewrite old labels or modify accepted values to make a comparison pass. This compatibility report does not change profile acceptance, timing measurements, estimator behavior, repeatability policy or accuracy qualification. + | Diagnostic | Investigate and retain | | --- | --- | | Missing or duplicate ranks | Actual launched worker/DP counts, process failures and per-rank logs. Never fill a missing rank with another rank's record. | diff --git a/python/aisimulate/collector/fpm_forward/runtime_observations.py b/python/aisimulate/collector/fpm_forward/runtime_observations.py index e1bedf1b4..88132ab50 100644 --- a/python/aisimulate/collector/fpm_forward/runtime_observations.py +++ b/python/aisimulate/collector/fpm_forward/runtime_observations.py @@ -590,7 +590,15 @@ def _physical_views( return groups, accounting -def _configuration(root: Path, label: str, saved: dict[str, Any], launch: dict[str, Any]) -> dict[str, Any]: +def _configuration( + root: Path, + label: str, + saved: dict[str, Any], + launch: dict[str, Any], + *, + phase: str | None = None, + attempt_id: str | None = None, +) -> dict[str, Any]: _same(saved.get("launch"), launch, "reviewed configuration launch") ranks = expected_ranks(launch["topology"]) attempts = saved.get("attempts") @@ -602,6 +610,8 @@ def _configuration(root: Path, label: str, saved: dict[str, Any], launch: dict[s active = saved.get("active_attempt_id") if active not in ids: raise ValueError("configuration requires an explicit active attempt identity") + if phase is not None: + _same(active, attempt_id, "selected phase attempt identity") attempt = attempts[ids.index(active)] bundle_path = contained_file(root, attempt["bundle"]["manifest"]) identity = launch["identity"] @@ -614,7 +624,11 @@ def _configuration(root: Path, label: str, saved: dict[str, Any], launch: dict[s raise ValueError("runtime source mapping requires hashed source_notes and runtime.source_files") phases = attempt.get("phases", {}) required_phases = collection_phases(launch.get("worker_type")) - if set(phases) != set(required_phases): + if phase is not None: + if phase not in required_phases or phase not in phases or set(phases) - set(required_phases): + raise ValueError("runtime evidence does not contain the selected serving-role phase") + required_phases = (phase,) + elif set(phases) != set(required_phases): raise ValueError("runtime evidence requires every phase of the selected serving role") canonical_settings, canonical_geometry, canonical_hardware, normalized = None, None, None, None capacities, artifacts, launch_artifacts, runtime_artifacts, import_bindings = [], [], [], [], [] @@ -753,6 +767,41 @@ def _configuration(root: Path, label: str, saved: dict[str, Any], launch: dict[s return {"status": "complete", "resources": resources, "diagnostics": [], "provenance": provenance} +def validate_observation_phase( + index_path: str | Path, + configuration: str, + launch: dict[str, Any], + *, + phase: str, + attempt_id: str, +) -> dict[str, Any]: + """Validate one explicitly bound active phase; does not qualify a serving role. + + The onboarding collection verifier binds this selection to its frozen plan + and passed native attempt, then requires complete role coverage separately. + """ + index_path = Path(index_path).absolute() + if index_path.is_symlink(): + raise ValueError("runtime observation index cannot be a symlink") + index = read_json(index_path) + if index.get("schema_version") != INDEX_SCHEMA or not isinstance(index.get("configurations"), dict): + raise ValueError("unsupported runtime observation index schema") + result = _configuration( + index_path.parent, + configuration, + index["configurations"][configuration], + launch, + phase=phase, + attempt_id=_text(attempt_id, "selected phase attempt identity"), + ) + result["provenance"]["observations_index"] = { + "path": str(index_path), + "sha256": sha256_bytes(index_path.read_bytes()), + } + result["resources"]["runtime_memory"]["provenance"] = canonical_json(result["provenance"]) + return result + + def validate_observations( index_path: str | Path, expected_configurations: dict[str, dict[str, Any]] ) -> dict[str, dict[str, Any]]: diff --git a/python/aisimulate/collector/fpm_forward/runtime_probe.py b/python/aisimulate/collector/fpm_forward/runtime_probe.py index 9eeedfef0..2a1e3ae45 100644 --- a/python/aisimulate/collector/fpm_forward/runtime_probe.py +++ b/python/aisimulate/collector/fpm_forward/runtime_probe.py @@ -347,7 +347,18 @@ def build_runtime_probe_plan(configuration: str, facts: dict[str, Any], bundle: def validate_collection_probe_launch( - launch, bundle, *, model_path, system, backend, backend_version, cells, options, profile, generator_overrides + launch, + bundle, + *, + model_path, + system, + backend, + backend_version, + cells, + options, + profile, + generator_overrides, + phases=None, ): """Bind optional formal instrumentation to the actual collection inputs.""" @@ -363,7 +374,12 @@ def validate_collection_probe_launch( if profile is None or profile.model_revision != identity["model_revision"]: raise ValueError("formal collection model revision differs from the accepted runtime probe") expected_cell = expected.cells[0] - if {cell.workload_kind for cell in cells} != set(expected.options.workload_kinds): + required = set(expected.options.workload_kinds) + if phases is not None: + if not phases or not set(phases) <= required: + raise ValueError("formal collection phases differ from the accepted serving role") + required = set(phases) + if {cell.workload_kind for cell in cells} != required: raise ValueError("formal collection phases differ from the accepted serving role") for cell in cells: fields = ( @@ -423,7 +439,7 @@ def validate_collection_probe_launch( ), } required_limits = (launch["collection"]["max_num_batched_tokens"], launch["collection"]["max_num_seqs"]) - if any(limits[phase] != required_limits for phase in options.workload_kinds): + if any(limits[phase] != required_limits for phase in required): raise ValueError("formal phase scheduler limits differ from the accepted runtime probe") wanted = probe_generator_overrides(launch) actual = copy.deepcopy(generator_overrides) diff --git a/python/aisimulate/src/aisimulate/support/collection_readiness.py b/python/aisimulate/src/aisimulate/support/collection_readiness.py index e33113d95..1ac2b935e 100644 --- a/python/aisimulate/src/aisimulate/support/collection_readiness.py +++ b/python/aisimulate/src/aisimulate/support/collection_readiness.py @@ -225,6 +225,7 @@ def assess_readiness( checkpoint_root: Path, *, expected_plan=None, + runtime_bindings: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: """Inspect saved sources without model fetching, reruns or checkpoint writes. @@ -235,8 +236,13 @@ def assess_readiness( from collector.fpm_forward.repeatability import load_repeatability_deployment, load_repeatability_source from collector.fpm_forward.runner import CHECKPOINT_SCHEMA + from .runtime import runtime_probe_manifest, verify_collection_runtime_bindings + + if runtime_bindings is not None: + return _bound_readiness(request, runtime_bindings) campaigns = [] selected = {} + selected_runtime = {} blockers = [] for smoke in (True, False): checkpoint_path = checkpoint_root / ("fpm_forward_smoke.json" if smoke else "fpm_forward.json") @@ -292,6 +298,16 @@ def assess_readiness( report["cells"].append(cell_report) if cell.cell_id not in selected or cell.cell_id in checkpoint["cells"]: selected[cell.cell_id] = {"campaign": report["kind"], "plan_sha256": plan.sha256, **cell_report} + if runtime_probe_manifest(request) is not None: + index = Path(checkpoint.get("runtime_observations", "")) + selected_runtime[cell.cell_id] = { + "phase": cell.workload_kind, + "plan": {"path": str(campaign / "collection-plan.json"), "sha256": plan.sha256}, + "checkpoint": {key: report["checkpoint"][key] for key in ("path", "sha256")}, + "observations": {key: file_evidence(index)[key] for key in ("path", "sha256")}, + "attempt_id": checkpoint.get("runtime_observation_attempt_id"), + "collector_attempt_id": entry.get("attempt_id") if isinstance(entry, dict) else None, + } except (OSError, TypeError, ValueError, KeyError) as error: report["errors"].append(str(error)) blockers.append(f"{report['kind']}: {error}") @@ -305,15 +321,55 @@ def assess_readiness( blockers.extend( f"{item['cell']['workload_kind']} {item['cell']['cell_id']}: {reason}" for reason in item["blockers"] ) + runtime = None + if runtime_probe_manifest(request) is not None: + try: + runtime = verify_collection_runtime_bindings(request, list(selected_runtime.values())) + except (OSError, TypeError, ValueError, KeyError) as error: + blockers.append(f"runtime memory: {error}") ready = bool(selected) and not blockers and all(item["status"] == "ready" for item in selected.values()) return { "status": "ready" if ready else "blocked" if campaigns else "incomplete", "ready_for_full_collection": ready, - "scope": "sampled native timing readiness only; not complete query coverage, memory or serving accuracy", + "scope": "sampled native timing readiness with runtime memory checks for imported profiles; " + "not complete query coverage, repeatability or serving accuracy", "request_identity": request.identity.model_dump(mode="json"), "blockers": blockers, "selected_cells": list(selected.values()), "campaigns": campaigns, + "runtime_compatibility": runtime, + } + + +def _bound_readiness(request: SupportRequest, bindings: list[dict[str, Any]]) -> dict[str, Any]: + """Apply the same timing and memory gates to explicitly selected phase plans.""" + from collector.fpm_forward.repeatability import load_repeatability_source + + from .runtime import verify_collection_runtime_bindings + + cells, blockers = [], [] + runtime = None + try: + runtime = verify_collection_runtime_bindings(request, bindings) + for binding in bindings: + campaign = Path(binding["plan"]["path"]).parent + plan = load_repeatability_source(campaign) + cell = next(cell for cell in plan.cells if cell.workload_kind == binding["phase"]) + checkpoint = json.loads(Path(binding["checkpoint"]["path"]).read_text()) + result = _cell_report(plan, cell, campaign, checkpoint["cells"][cell.cell_id]) + cells.append({"campaign": str(campaign), "plan_sha256": plan.sha256, **result}) + blockers.extend(f"{binding['phase']}: {message}" for message in result["blockers"]) + except (OSError, ValueError, TypeError, KeyError) as error: + blockers.append(str(error)) + ready = bool(cells) and not blockers and all(item["status"] == "ready" for item in cells) + return { + "status": "ready" if ready else "blocked", + "ready_for_full_collection": ready, + "scope": "sampled native timing and compatible runtime memory; not complete query coverage or serving accuracy", + "request_identity": request.identity.model_dump(mode="json"), + "blockers": blockers, + "selected_cells": cells, + "runtime_compatibility": runtime, } diff --git a/python/aisimulate/src/aisimulate/support/runtime.py b/python/aisimulate/src/aisimulate/support/runtime.py index a04caf010..5deba89e3 100644 --- a/python/aisimulate/src/aisimulate/support/runtime.py +++ b/python/aisimulate/src/aisimulate/support/runtime.py @@ -18,6 +18,11 @@ from aisimulate.fpm_profile import FpmModelProfile from .config_profile import load_model_config, profile_from_observations, validate_overrides +from .runtime_collection import ( # noqa: F401 + collection_runtime_review, + verify_collection_runtime_bindings, + verify_collection_runtime_plans, +) from .schema import ( AGENTX_REFERENCE_CONTEXT, CollectionSpec, diff --git a/python/aisimulate/src/aisimulate/support/runtime_collection.py b/python/aisimulate/src/aisimulate/support/runtime_collection.py new file mode 100644 index 000000000..4dc999ae3 --- /dev/null +++ b/python/aisimulate/src/aisimulate/support/runtime_collection.py @@ -0,0 +1,314 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Explicit frozen-plan bindings for independently collected memory evidence.""" + +from __future__ import annotations + +import hashlib +import json +from copy import deepcopy +from pathlib import Path +from typing import Any + +from .schema import SupportRequest + + +def _file(path: str | Path, *, model_config: bool = False) -> dict[str, str]: + path = Path(path).absolute() + if (path.is_symlink() and not model_config) or not path.is_file(): + raise ValueError(f"runtime evidence must be a regular file: {path}") + return {"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + + +def _read(reference: dict[str, str], *, plan: bool = False) -> tuple[Path, dict[str, Any]]: + from collector.fpm_forward.runtime_instrumentation import read_json, validate_sha256 + from collector.fpm_forward.runtime_memory import validate_saved_plan + + path = Path(reference["path"]).absolute() + actual = _file(path) + validate_sha256(reference["sha256"], "bound collection plan" if plan else "bound runtime evidence") + document = read_json(path) + if plan: + validate_saved_plan(document) + if (document["sha256"] if plan else actual["sha256"]) != reference["sha256"]: + raise ValueError(f"bound {'plan' if plan else 'artifact'} hash mismatch: {path}") + return path, document + + +def _differences(accepted: Any, collection: Any, field: str = "") -> list[dict[str, Any]]: + if accepted == collection: + return [] + if isinstance(accepted, dict) and isinstance(collection, dict): + return [ + difference + for key in sorted(accepted.keys() | collection.keys()) + for difference in _differences( + accepted.get(key, {"missing": True}), collection.get(key, {"missing": True}), field + "/" + key + ) + ] + return [{"field": field, "accepted": accepted, "collection": collection}] + + +def _source(request: SupportRequest) -> dict[str, Any]: + from .runtime import verify_runtime_profile + + source = verify_runtime_profile(request) + if source is None: + raise ValueError("bound runtime comparison requires imported probe evidence") + return source + + +def _plan(request: SupportRequest, binding: dict[str, Any], source: dict[str, Any]): + from collector.fpm_forward.repeatability import load_repeatability_deployment, load_repeatability_source + from collector.fpm_forward.runtime_probe import validate_collection_probe_launch + + path, saved = _read(binding["plan"], plan=True) + if path.name != "collection-plan.json": + raise ValueError("bound runtime plan must be the archived collection-plan.json") + plan = load_repeatability_source(path.parent) + reference = saved.get("runtime_observation") + if ( + not isinstance(reference, dict) + or not isinstance(reference.get("configuration"), str) + or not reference["configuration"].strip() + ): + raise ValueError("bound collection plan lacks a recorded runtime configuration") + launch = plan.runtime_launch + original = source["provenance"]["launch"] + # Paths locate config bytes; their hashes, sidecars and model revision identify + # the checkpoint. All serving, precision and scheduler facts remain exact. + expected, actual = deepcopy(original), deepcopy(launch) + for value in (expected, actual): + value.pop("deployment") + value["model_config"].pop("path") + value["model_config"].setdefault("source_files", {}) + if actual != expected: + raise ValueError( + "collection immutable model/precision/settings differ: " + json.dumps(_differences(expected, actual)) + ) + phase = binding["phase"] + cells = [cell for cell in plan.cells if cell.workload_kind == phase] + if phase not in request.collection_phases or len(cells) != 1: + raise ValueError("bound phase requires exactly one matching frozen collection cell") + bundle = plan.runtime_instrumentation + deployment = load_repeatability_deployment(path.parent) + validate_collection_probe_launch( + launch, + bundle, + model_path=plan.model_path, + system=plan.system, + backend=plan.backend, + backend_version=plan.capability.aic_database_version, + cells=plan.cells, + options=plan.options, + profile=plan.fpm_profile, + generator_overrides=deployment, + phases=tuple(cell.workload_kind for cell in plan.cells), + ) + reviewed = { + "accepted_bundle_sha256": source["provenance"]["bundle_sha256"], + "collection_bundle_sha256": bundle.sha256, + "collector_revision": plan.aic_revision, + "generator_config_sha256": plan.generator_config_sha256, + "changes": _differences( + { + "bundle_sha256": source["provenance"]["bundle_sha256"], + "runtime": source["provenance"]["runtime"], + "deployment": original["deployment"], + }, + {"bundle_sha256": bundle.sha256, "runtime": bundle.manifest["runtime"], "deployment": launch["deployment"]}, + ), + } + return path, plan, cells[0], bundle, reviewed + + +def collection_runtime_review(request: SupportRequest, binding: dict[str, Any]) -> dict[str, Any]: + """Describe exact runtime changes for review; never qualify observations. + + A caller must inspect these changes and add a recorded ``rationale`` before + passing the resulting review in a binding. Settings mismatches still fail. + """ + return _plan(request, binding, _source(request))[-1] + + +def _plans(request: SupportRequest, bindings: list[dict[str, Any]], source: dict[str, Any]): + phases = [item["phase"] for item in bindings] + if len(set(phases)) != len(phases) or set(phases) != set(request.collection_phases): + raise ValueError("runtime bindings require each serving-role phase exactly once") + verified = [] + for binding in bindings: + try: + values = _plan(request, binding, source) + review = binding.get("review") + required = values[-1] + if (required["changes"] or review is not None) and ( + not isinstance(review, dict) + or not isinstance(review.get("rationale"), str) + or not review["rationale"].strip() + or {key: value for key, value in review.items() if key != "rationale"} != required + ): + raise ValueError("exact runtime change review required: " + json.dumps(required, sort_keys=True)) + verified.append((binding, *values)) + except (OSError, ValueError, TypeError, KeyError, AttributeError) as error: + raise ValueError(f"{binding['phase']}: {error}") from error + return verified + + +def verify_collection_runtime_plans(request: SupportRequest, bindings: list[dict[str, Any]]) -> dict[str, Any]: + """Check all phase plans before attempts exist; this is not memory evidence.""" + verified = _plans(request, bindings, _source(request)) + return { + "status": "verified", + "scope": "frozen collection plans and reviewed runtime changes only; live evidence remains required", + "phases": { + binding["phase"]: { + "plan": deepcopy(binding["plan"]), + "configuration": plan.runtime_configuration, + "review": deepcopy(binding.get("review")), + } + for binding, _, plan, _, _, _ in verified + }, + } + + +def _worker(result): + return next(item["evidence"] for item in result["provenance"]["artifacts"] if item["evidence"]["kind"] == "worker") + + +def _geometry(result): + groups = _worker(result)["cache"]["groups"] + return [{key: value for key, value in group.items() if key != "pool_id"} for group in groups] + + +def _evidence(index: Path, result: dict[str, Any]) -> list[dict[str, str]]: + from collector.fpm_forward.runtime_instrumentation import contained_file + + provenance = result["provenance"] + paths = [index] + for field in ("artifacts", "launch_artifacts", "runtime_artifacts"): + paths.extend(contained_file(index.parent, ref["path"]) for ref in provenance[field]) + instrumentation = provenance["instrumentation"] + manifest = contained_file(index.parent, instrumentation["manifest"]) + paths.append(manifest) + paths.extend(contained_file(manifest.parent, name) for name in instrumentation["files"]) + return [_file(path) for path in paths] + + +def verify_collection_runtime_bindings(request: SupportRequest, bindings: list[dict[str, Any]]) -> dict[str, Any]: + """Verify distinct successful phase attempts against unchanged accepted memory. + + Plan references carry the producer's semantic SHA; checkpoint/index references + carry byte hashes. Recorded names, launch and bundle derive only from the plan. + """ + from collector.fpm_forward.runner import CHECKPOINT_SCHEMA + from collector.fpm_forward.runtime_instrumentation import contained_file, read_json + from collector.fpm_forward.runtime_observations import validate_observation_phase + + from .runtime import _resource_values, runtime_probe_manifest + + source = _source(request) + verified = _plans(request, bindings, source) + manifest = runtime_probe_manifest(request) + inputs = [*_evidence(Path(manifest["observations_index"]), source), *manifest["source_artifacts"]] + phases = {} + bound = request.profile_deployment().resources.runtime_memory.kv_cache_bytes + for binding, plan_path, plan, cell, bundle, _ in verified: + phase = binding["phase"] + try: + checkpoint_path, checkpoint = _read(binding["checkpoint"]) + index, document = _read(binding["observations"]) + if not index.resolve().is_relative_to(plan_path.parent.resolve()): + raise ValueError("runtime observation index escapes the frozen collection directory") + if ( + checkpoint.get("schema") != CHECKPOINT_SCHEMA + or checkpoint.get("plan_sha256") != plan.sha256 + or Path(checkpoint.get("runtime_observations", "")).absolute() != index + or checkpoint.get("runtime_observation_attempt_id") != binding["attempt_id"] + or set(checkpoint.get("cells", {})) - {value.cell_id for value in plan.cells} + ): + raise ValueError("runtime checkpoint does not match the bound plan/index/attempt") + saved = checkpoint.get("cells", {}).get(cell.cell_id, {}) + if ( + not isinstance(binding["collector_attempt_id"], str) + or not binding["collector_attempt_id"].strip() + or saved.get("status") != "passed" + or saved.get("attempt_id") != binding["collector_attempt_id"] + or saved.get("cleanup_error") + ): + raise ValueError("runtime phase does not match the successful native cell attempt") + result = validate_observation_phase( + index, plan.runtime_configuration, plan.runtime_launch, phase=phase, attempt_id=binding["attempt_id"] + ) + if result["provenance"]["bundle_sha256"] != bundle.sha256: + raise ValueError("runtime observation bundle differs from the frozen plan") + entry = document["configurations"][plan.runtime_configuration] + attempt = next(value for value in entry["attempts"] if value["attempt_id"] == binding["attempt_id"]) + recorded = attempt["phases"][phase] + context = read_json(contained_file(index.parent, recorded["launch_manifest"]["path"])) + for value in (recorded, context): + if ( + value.get("cell_id") != cell.cell_id + or value.get("collector_attempt_id") != binding["collector_attempt_id"] + ): + raise ValueError("runtime phase does not match the successful native cell attempt") + if recorded.get("status") != "passed": + raise ValueError("runtime phase is not a passed collection attempt") + original_settings, settings = ( + deepcopy(source["provenance"]["runtime_settings"]), + deepcopy(result["provenance"]["runtime_settings"]), + ) + settings_diff = _differences(original_settings, settings, "/runtime_settings") + # Both spellings already passed _matches_model_path and revision/hash + # checks independently. Do not normalize any other effective setting. + settings["model_config"]["model"] = original_settings["model_config"]["model"] + if settings != original_settings: + raise ValueError( + "effective runtime/graph settings differ from accepted probe: " + json.dumps(settings_diff) + ) + if _geometry(result) != _geometry(source): + raise ValueError("runtime cache group geometry differs from the accepted probe") + for field in ("gpu", "device_name", "sm"): + if _worker(result)["hardware"][field] != _worker(source)["hardware"][field]: + raise ValueError("runtime hardware across phases differs from the accepted probe") + expected, actual = _resource_values(source["resources"]), _resource_values(result["resources"]) + expected["runtime_memory"].pop("kv_cache_bytes") + capacity = actual["runtime_memory"].pop("kv_cache_bytes") + if actual != expected: + raise ValueError("runtime cache layout or settings differ from the accepted probe") + if capacity < bound: + raise ValueError(f"runtime usable capacity {capacity} is smaller than the accepted bound {bound}") + result["binding"] = deepcopy(binding) + result["compatibility"] = {"status": "compatible", "runtime_settings_differences": settings_diff} + phases[phase] = result + inputs.extend([_file(plan_path), _file(checkpoint_path), *_evidence(index, result)]) + inputs.extend(_file(bundle.root / name) for name in bundle.files) + inputs.extend([_file(bundle.manifest_path), _file(plan_path.parent / "generator-overrides.json")]) + config = plan.runtime_launch["model_config"] + inputs.append(_file(config["path"], model_config=True)) + inputs.extend( + _file(Path(config["path"]).parent / name, model_config=True) for name in config.get("source_files", {}) + ) + except (OSError, ValueError, TypeError, KeyError, AttributeError, StopIteration) as error: + raise ValueError(f"{phase}: {error}") from error + resources = deepcopy(next(iter(phases.values()))["resources"]) + capacity = min(item["resources"]["runtime_memory"]["kv_cache_bytes"] for item in phases.values()) + provenance = { + "source": "bound_collection_phases", + "phases": {phase: item["provenance"] for phase, item in phases.items()}, + } + resources["runtime_memory"].update(kv_cache_bytes=capacity, provenance=json.dumps(provenance, sort_keys=True)) + return { + "status": "complete", + "resources": resources, + "phases": phases, + "inputs": list({ref["path"]: ref for ref in inputs}.values()), + "compatibility": { + "status": "compatible", + "accepted_capacity_bytes": bound, + "observed_capacity_bytes": capacity, + "source_observations_index": manifest["observations_index"], + "policy": "exact reviewed runtime changes; unchanged model, precision, effective settings and geometry; " + "every phase/rank capacity at least the accepted bound", + }, + } diff --git a/python/aisimulate/tests/unit/test_onboard_runtime.py b/python/aisimulate/tests/unit/test_onboard_runtime.py index 7e03f9f55..2ba269019 100644 --- a/python/aisimulate/tests/unit/test_onboard_runtime.py +++ b/python/aisimulate/tests/unit/test_onboard_runtime.py @@ -29,7 +29,9 @@ def _write(path, value): path.write_text(json.dumps(value)) -def _campaign(tmp_path, *, capacity_multiplier=1, context_length=4096, tp=2, cpu_policy=True, worker_type=None): +def _campaign( + tmp_path, *, capacity_multiplier=1, context_length=4096, tp=2, cpu_policy=True, worker_type=None, model=None +): index, launches = observation_fixture(tmp_path / "probe", dense=True, tp=tp) launch = launches["tp2"] if cpu_policy: @@ -55,7 +57,9 @@ def _campaign(tmp_path, *, capacity_multiplier=1, context_length=4096, tp=2, cpu checkpoint_model = tmp_path / "model-checkpoint" checkpoint_model.mkdir() (checkpoint_model / "config.json").write_bytes(model_path.read_bytes()) - launch["identity"]["model"] = str(checkpoint_model) + launch["identity"]["model"] = model or str(checkpoint_model) + if model is not None: + launch["identity"]["model_revision"] = "4" * 40 launch["model_config"]["sha256"] = hashlib.sha256(model_path.read_bytes()).hexdigest() launch["collection"]["max_model_len"] = context_length if worker_type is not None: @@ -79,7 +83,9 @@ def _campaign(tmp_path, *, capacity_multiplier=1, context_length=4096, tp=2, cpu value["launch"] = launch if ref.get("kind") == "observation": value["identity"] = launch["identity"] - value["resolved_config"]["model_config"]["model"] = str(checkpoint_model) + value["resolved_config"]["model_config"].update( + model=launch["identity"]["model"], revision=launch["identity"]["model_revision"] + ) value["model_config_sha256"] = launch["model_config"]["sha256"] value["resolved_config"]["model_config"]["max_model_len"] = context_length if worker_type is not None: @@ -668,8 +674,10 @@ def test_unknown_geometry_does_not_mask_missing_explicit_topology(tmp_path, caps assert "tensor_parallel" in str(result["configurations"]["tp2"]["diagnostics"]) -def _imported(tmp_path, capsys, *, capacity_multiplier=1, tp=2, worker_type=None): - checkpoint, index, _ = _campaign(tmp_path, capacity_multiplier=capacity_multiplier, tp=tp, worker_type=worker_type) +def _imported(tmp_path, capsys, *, capacity_multiplier=1, tp=2, worker_type=None, model=None): + checkpoint, index, _ = _campaign( + tmp_path, capacity_multiplier=capacity_multiplier, tp=tp, worker_type=worker_type, model=model + ) assert _import(checkpoint, index, tmp_path / "drafts", configurations=["tp2"]) == 0 result = json.loads(capsys.readouterr().out) state = _load(checkpoint) @@ -783,7 +791,7 @@ def _formal_bindings(index, cells=None): for phase, result in attempt["phases"].items(): cell_id = cells[phase] if cells else f"cell-{phase}" attempt_id = f"native-{phase}" - result.update(cell_id=cell_id, collector_attempt_id=attempt_id) + result.update(cell_id=cell_id, collector_attempt_id=attempt_id, status="passed") entries[cell_id] = {"status": "passed", "attempt_id": attempt_id} for ref in [result["launch_manifest"], *result["artifacts"]]: path = index.parent / ref["path"] @@ -795,7 +803,7 @@ def _formal_bindings(index, cells=None): return {"runtime_observation_attempt_id": "formal-parent", "cells": entries} -def _completed_runtime_collection(tmp_path, capsys, *, tp=2, worker_type=None): +def _completed_runtime_collection(tmp_path, capsys, *, tp=2, worker_type=None, model=None): from collector.fpm_forward.cli import _parser from collector.fpm_forward.config import FPMCollectionOptions from collector.fpm_forward.database import aggregate_cell, write_formal_database @@ -807,7 +815,9 @@ def _completed_runtime_collection(tmp_path, capsys, *, tp=2, worker_type=None): from .test_onboard_finalization import _native - checkpoint, _, request, files = _imported(tmp_path, capsys, capacity_multiplier=10, tp=tp, worker_type=worker_type) + checkpoint, _, request, files = _imported( + tmp_path, capsys, capacity_multiplier=10, tp=tp, worker_type=worker_type, model=model + ) state = _load(checkpoint) save_checkpoint(checkpoint, patch={}, expected_revision=state.revision, accept=["tp2"]) root = tmp_path / "collection" diff --git a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py new file mode 100644 index 000000000..75e8e1761 --- /dev/null +++ b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py @@ -0,0 +1,428 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Plan-bound memory verification keeps independently collected phases distinct.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import shutil +from dataclasses import replace +from pathlib import Path + +import pytest +from collector.fpm_forward.cli import _parser +from collector.fpm_forward.entry import _load_generator_overrides +from collector.fpm_forward.repeatability import _subset_plan, load_repeatability_source +from collector.fpm_forward.runtime_instrumentation import load_instrumentation + +from aisimulate.support import runtime + +from .test_onboard_runtime import _completed_runtime_collection + +pytestmark = pytest.mark.unit + + +def _write(path, value): + path.write_text(json.dumps(value)) + + +def _ref(path): + return {"path": str(path), "sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + + +@pytest.fixture +def split_collection(tmp_path, capsys, request, monkeypatch): + if getattr(request, "param", None) is not None: + from aisimulate_core.sdk import fpm_profile + + # The synthetic repo's immutable checkpoint is local to this fixture. + monkeypatch.setattr( + fpm_profile, + "_load_pinned_checkpoint_config", + lambda model, revision: json.loads((tmp_path / "model-checkpoint/config.json").read_text()), + ) + _, request, _, root = _completed_runtime_collection(tmp_path, capsys, model=getattr(request, "param", None)) + original_checkpoint = json.loads((root / "fpm-checkpoint/fpm_forward.json").read_text()) + original_campaign = root / "fpm-artifacts" / original_checkpoint["plan_sha256"][:16] + original_plan = load_repeatability_source(original_campaign) + args = _parser().parse_args(json.loads((root / "commands.json").read_text())["fpm_plan_local"][3:]) + bindings = [] + for phase in ("prefill", "decode"): + campaign = tmp_path / (phase + "-independent") + shutil.copytree(original_campaign, campaign) + observations = campaign / "observations/observations.json" + document = json.loads(observations.read_text()) + entry = document["configurations"].pop("tp2") + name = "tp2-long-context-readiness-full-recollection" + document["configurations"][name] = entry + attempt = entry["attempts"][0] + attempt_id = entry["active_attempt_id"] = attempt["attempt_id"] = phase + "-parent" + attempt["phases"] = {phase: attempt["phases"][phase]} + launch = entry["launch"] + launch["deployment"]["cpus_per_task"] = 32 + bundle_path = observations.parent / attempt["bundle"]["manifest"] + manifest = json.loads(bundle_path.read_text()) + manifest["runtime"]["source_files"]["vllm/v1/core/block_pool.py"] = ("d" if phase == "prefill" else "e") * 64 + _write(bundle_path, manifest) + bundle = load_instrumentation(bundle_path) + attempt["bundle"]["sha256"] = bundle.sha256 + for result in attempt["phases"].values(): + result["status"] = "passed" + for ref in [result["launch_manifest"], *result["artifacts"]]: + path = observations.parent / ref["path"] + record = json.loads(path.read_text()) + record.update(configuration=name, attempt_id=attempt_id, launch=launch, bundle_sha256=bundle.sha256) + if ref.get("kind") == "observation": + record["runtime"] = {**manifest["runtime"], "image": launch["deployment"]["image"]} + _write(path, record) + ref["sha256"] = _ref(path)["sha256"] + _write(observations, document) + cell = next(cell for cell in original_plan.cells if cell.workload_kind == phase) + plan = replace( + original_plan, + aic_revision="reviewed-collector-revision", + options=replace(original_plan.options, slurm_cpus_per_task=32), + _runtime_observation_json=json.dumps( + {"manifest": str(bundle_path), "bundle_sha256": bundle.sha256, "launch": launch, "configuration": name} + ), + ) + plan = _subset_plan(plan, {"cell_id": cell.cell_id, "benchmark_points": {}}, comparison_mode="full_grid") + _write(campaign / "collection-plan.json", plan.to_dict()) + _write(campaign / "generator-overrides.json", _load_generator_overrides(args)) + checkpoint = { + **original_checkpoint, + "plan_sha256": plan.sha256, + "runtime_observation_attempt_id": attempt_id, + "runtime_observations": str(observations), + "cells": {cell.cell_id: original_checkpoint["cells"][cell.cell_id]}, + } + checkpoint_path = campaign / "checkpoint.json" + _write(checkpoint_path, checkpoint) + bindings.append( + { + "phase": phase, + "plan": {"path": str(campaign / "collection-plan.json"), "sha256": plan.sha256}, + "checkpoint": _ref(checkpoint_path), + "observations": _ref(observations), + "attempt_id": attempt_id, + "collector_attempt_id": checkpoint["cells"][cell.cell_id]["attempt_id"], + } + ) + return request, bindings + + +def _review(request, bindings): + for binding in bindings: + binding["review"] = { + **runtime.collection_runtime_review(request, binding), + "rationale": "Synthetic regression: reviewed CPU allocation and phase-specific source changes.", + } + + +def test_renamed_split_collection_requires_exact_review_and_keeps_attempts(split_collection): + request, bindings = split_collection + original_request = request.model_dump(mode="json") + with pytest.raises(ValueError, match="review"): + runtime.verify_collection_runtime_bindings(request, bindings) + _review(request, bindings) + before = { + Path(ref["path"]): Path(ref["path"]).read_bytes() + for item in bindings + for ref in (item["plan"], item["checkpoint"], item["observations"]) + } + planned = [ + {key: copy.deepcopy(value) for key, value in item.items() if key in {"phase", "plan", "review"}} + for item in bindings + ] + assert runtime.verify_collection_runtime_plans(request, planned)["status"] == "verified" + result = runtime.verify_collection_runtime_bindings(request, bindings) + assert result["compatibility"]["status"] == "compatible" + assert {item["provenance"]["attempt_id"] for item in result["phases"].values()} == { + "prefill-parent", + "decode-parent", + } + assert ( + result["resources"]["runtime_memory"]["kv_cache_bytes"] + == request.profile_deployment().resources.runtime_memory.kv_cache_bytes + ) + assert all(path.read_bytes() == contents for path, contents in before.items()) + assert request.model_dump(mode="json") == original_request + inputs = {item["path"]: item["sha256"] for item in result["inputs"]} + assert all(str(path) in inputs for path in before) + assert any("worker-0-0.json" in path for path in inputs) + assert any("manifest.yaml" in path for path in inputs) + assert all(_ref(Path(path))["sha256"] == digest for path, digest in inputs.items()) + with pytest.raises(ValueError, match="incomplete"): + runtime.verify_collection_runtime(request, Path(bindings[0]["observations"]["path"])) + + +def _change_index(binding, change): + path = Path(binding["observations"]["path"]) + document = json.loads(path.read_text()) + entry = next(iter(document["configurations"].values())) + change(path.parent, entry, entry["attempts"][0]["phases"][binding["phase"]]) + _write(path, document) + binding["observations"] = _ref(path) + + +def _change_records(binding, change): + def apply(root, entry, phase): + for ref in phase["artifacts"]: + if ref.get("kind") != "observation": + continue + path = root / ref["path"] + record = json.loads(path.read_text()) + change(record) + _write(path, record) + ref["sha256"] = _ref(path)["sha256"] + + _change_index(binding, apply) + + +@pytest.mark.parametrize( + "change", + [ + "missing-phase", + "duplicate-phase", + "runtime-attempt", + "native-attempt", + "checkpoint-plan", + "plan-hash", + "review-bundle", + "review-diff", + "review-revision", + "review-rationale", + ], +) +def test_explicit_phase_plan_attempt_and_review_bindings_are_required(split_collection, change): + request, bindings = split_collection + _review(request, bindings) + binding = bindings[0] + if change == "missing-phase": + bindings.pop() + elif change == "duplicate-phase": + bindings.append(copy.deepcopy(binding)) + elif change == "runtime-attempt": + binding["attempt_id"] = "another-parent" + elif change == "native-attempt": + binding["collector_attempt_id"] = "another-native-attempt" + elif change == "checkpoint-plan": + path = Path(binding["checkpoint"]["path"]) + checkpoint = json.loads(path.read_text()) + checkpoint["plan_sha256"] = "0" * 64 + _write(path, checkpoint) + binding["checkpoint"] = _ref(path) + elif change == "plan-hash": + binding["plan"]["sha256"] = "0" * 64 + elif change == "review-bundle": + binding["review"]["collection_bundle_sha256"] = "0" * 64 + elif change == "review-diff": + binding["review"]["changes"].pop() + elif change == "review-revision": + binding["review"]["collector_revision"] = "unreviewed-collector" + else: + binding["review"]["rationale"] = "" + with pytest.raises(ValueError): + runtime.verify_collection_runtime_bindings(request, bindings) + + +@pytest.mark.parametrize( + "change", + [ + "missing-rank", + "duplicate-rank", + "artifact-bytes", + "artifact-escape", + "launch-attempt", + "model", + "revision", + "model-config-hash", + "model-sidecar-hash", + "precision", + "scheduler", + "graph", + "settings", + "geometry", + "capacity", + ], +) +def test_raw_evidence_is_independently_validated_under_each_frozen_launch(split_collection, change): + request, bindings = split_collection + _review(request, bindings) + binding = bindings[0] + if change in {"missing-rank", "duplicate-rank", "artifact-bytes", "artifact-escape", "launch-attempt"}: + + def mutate(root, entry, phase): + ref = next(ref for ref in phase["artifacts"] if "worker" in ref["path"]) + path = root / ref["path"] + if change == "missing-rank": + phase["artifacts"].remove(ref) + elif change == "duplicate-rank": + duplicate = root / "duplicate-worker.json" + duplicate.write_bytes(path.read_bytes()) + phase["artifacts"].append({**ref, "path": duplicate.name}) + elif change == "artifact-bytes": + path.write_text(path.read_text() + "\n") + elif change == "artifact-escape": + outside = root.parent / "outside.json" + outside.write_bytes(path.read_bytes()) + ref["path"] = "../outside.json" + else: + context_path = root / phase["launch_manifest"]["path"] + context = json.loads(context_path.read_text()) + context["collector_attempt_id"] = "unrelated-native" + _write(context_path, context) + phase["launch_manifest"]["sha256"] = _ref(context_path)["sha256"] + + _change_index(binding, mutate) + else: + + def mutate(record): + config = record["resolved_config"] + if change == "model": + config["model_config"]["model"] = "other/model" + elif change == "revision": + config["model_config"].update(revision="wrong", loaded_config_commit_hash="wrong") + elif change == "model-config-hash": + record["model_config_sha256"] = "0" * 64 + elif change == "model-sidecar-hash": + record["model_config_source_files"] = {"hf_quant_config.json": "0" * 64} + elif change == "precision": + config["cache_config"]["cache_dtype"] = "fp8" + elif change == "scheduler": + config["scheduler_config"]["max_num_seqs"] += 1 + elif change == "graph": + config["compilation_config"]["cudagraph_capture_sizes"] = [1, 4] + elif change == "settings": + config["model_config"]["trust_remote_code"] = True + elif change == "capacity" and record["kind"] == "scheduler": + record["cache"]["initial_free_blocks"] -= 100 + record["cache"]["reserved_blocks"] += 100 + record["cache"]["permanent_reserved_block_ids"].extend(range(2, 102)) + elif change == "geometry": + cache = record["cache"] + for group in cache["groups"]: + group["layer_names"] = ["changed" if name == "layer0" else name for name in group["layer_names"]] + if "layer_classes" in group and "layer0" in group["layer_classes"]: + group["layer_classes"]["changed"] = group["layer_classes"].pop("layer0") + if record["kind"] == "worker": + cache["layer_tensors"]["changed"] = cache["layer_tensors"].pop("layer0") + for allocation in cache["tensor_allocations"]: + allocation["shared_by"] = [ + "changed" if name == "layer0" else name for name in allocation["shared_by"] + ] + + _change_records(binding, mutate) + with pytest.raises(ValueError, match="prefill:"): + runtime.verify_collection_runtime_bindings(request, bindings) + + +@pytest.mark.parametrize( + ("section", "field", "value"), + [ + ("identity", "model", "other/model"), + ("identity", "model_revision", "another-revision"), + ("topology", "tp", 4), + ("precision", "gemm_quant_mode", "fp8"), + ("collection", "max_num_seqs", 32), + ("collection", "cudagraph_mode", "NONE"), + ("model_config", "sha256", "0" * 64), + ], +) +def test_preflight_rejects_incompatible_frozen_plans_before_attempts(split_collection, section, field, value): + request, bindings = split_collection + _review(request, bindings) + binding = bindings[0] + path = Path(binding["plan"]["path"]) + plan = load_repeatability_source(path.parent) + observation = json.loads(plan._runtime_observation_json) + observation["launch"][section][field] = value + plan = replace(plan, _runtime_observation_json=json.dumps(observation)) + plan = _subset_plan(plan, {"cell_id": plan.cells[0].cell_id, "benchmark_points": {}}, comparison_mode="full_grid") + _write(path, plan.to_dict()) + binding["plan"]["sha256"] = plan.sha256 + with pytest.raises(ValueError, match="immutable model/precision/settings differ"): + runtime.verify_collection_runtime_plans(request, bindings) + + +def test_readiness_uses_phase_bound_memory_and_keeps_native_timing_failures(split_collection, monkeypatch): + from aisimulate.support import collection_readiness + + request, bindings = split_collection + _review(request, bindings) + # Timing assessment is independently covered by native readiness fixtures; + # exercise its composition with real plan/raw-memory verification here. + monkeypatch.setattr( + collection_readiness, + "_cell_report", + lambda plan, cell, campaign, entry: {"status": "ready", "blockers": [], "cell": cell.to_dict()}, + ) + report = collection_readiness.assess_readiness(request, Path("unused"), Path("unused"), runtime_bindings=bindings) + assert report["ready_for_full_collection"] + assert set(report["runtime_compatibility"]["phases"]) == {"prefill", "decode"} + monkeypatch.setattr( + collection_readiness, + "_cell_report", + lambda plan, cell, campaign, entry: { + "status": "blocked", + "blockers": ["native timing rank missing"], + "cell": cell.to_dict(), + }, + ) + report = collection_readiness.assess_readiness(request, Path("unused"), Path("unused"), runtime_bindings=bindings) + assert not report["ready_for_full_collection"] + assert "native timing rank missing" in " ".join(report["blockers"]) + bindings[0]["attempt_id"] = "unrelated" + report = collection_readiness.assess_readiness(request, Path("unused"), Path("unused"), runtime_bindings=bindings) + assert not report["ready_for_full_collection"] + assert "runtime checkpoint" in " ".join(report["blockers"]) + + +@pytest.mark.parametrize("split_collection", ["example/model"], indirect=True) +def test_exact_checkpoint_snapshot_spelling_preserves_full_settings_evidence(split_collection): + request, bindings = split_collection + _review(request, bindings) + snapshot = "/cache/models--example--model/snapshots/" + request.identity.model_revision + for binding in bindings: + _change_records(binding, lambda record: record["resolved_config"]["model_config"].update(model=snapshot)) + result = runtime.verify_collection_runtime_bindings(request, bindings) + assert result["compatibility"]["status"] == "compatible" + for phase in result["phases"].values(): + assert phase["compatibility"]["runtime_settings_differences"] == [ + {"field": "/runtime_settings/model_config/model", "accepted": "example/model", "collection": snapshot} + ] + assert phase["provenance"]["runtime_settings"]["model_config"]["model"] == snapshot + _change_records( + bindings[0], + lambda record: record["resolved_config"]["model_config"].update(model=snapshot.replace("4" * 40, "5" * 40)), + ) + with pytest.raises(ValueError, match="observed model path"): + runtime.verify_collection_runtime_bindings(request, bindings) + + +def test_default_readiness_requires_compatible_raw_memory_without_explicit_bindings(tmp_path, capsys, monkeypatch): + from aisimulate.support import collection_readiness + + _, request, _, root = _completed_runtime_collection(tmp_path, capsys) + checkpoint = json.loads((root / "fpm-checkpoint/fpm_forward.json").read_text()) + campaign = root / "fpm-artifacts" / checkpoint["plan_sha256"][:16] + args = _parser().parse_args(json.loads((root / "commands.json").read_text())["fpm_plan_local"][3:]) + _write(campaign / "generator-overrides.json", _load_generator_overrides(args)) + monkeypatch.setattr( + collection_readiness, + "_cell_report", + lambda plan, cell, campaign, entry: {"status": "ready", "blockers": [], "cell": cell.to_dict()}, + ) + report = collection_readiness.assess_readiness(request, root, root / "fpm-checkpoint") + assert report["ready_for_full_collection"], report["blockers"] + assert report["runtime_compatibility"]["compatibility"]["status"] == "compatible" + binding = {"phase": "prefill", "observations": _ref(Path(checkpoint["runtime_observations"]))} + _change_records( + binding, lambda record: record["runtime"]["source_files"].update({"vllm/v1/core/block_pool.py": "0" * 64}) + ) + report = collection_readiness.assess_readiness(request, root, root / "fpm-checkpoint") + assert not report["ready_for_full_collection"] + assert "runtime build/source" in " ".join(report["blockers"]) From 4c72b33fbb4658364d8b43a4b5daff033ecb0e85 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 12 Oct 2026 00:06:00 +0800 Subject: [PATCH 05/13] fix: reject conflicting readiness plan inputs Signed-off-by: Yiming Liu --- .../fpm-self-service/implementation.md | 2 ++ .../support/collection_readiness.py | 2 ++ .../unit/test_onboard_runtime_bindings.py | 35 +++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index dc1f73274..7b7705df2 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -1275,6 +1275,8 @@ The campaign recorder supplies a list with exactly one binding per phase require Before GPU work, pass bindings containing `phase`, `plan` and any `review` to `verify_collection_runtime_plans(request, bindings)`. It checks the frozen plan, archived deployment inputs, local model configuration bytes, precision and launch settings without running observers or workers. This preflight does not establish observed memory. After the bounded smoke or source collection completes, add the checkpoint, index and attempt fields and call `verify_collection_runtime_bindings(request, bindings)` before full-grid repeats. Programmatic readiness uses the same contract through `assess_readiness(request, root, checkpoint_root, runtime_bindings=bindings)`, combining raw memory validation with the ordinary native timing checks. Guided `collect-fpm --check-readiness` builds bindings from its own unchanged campaign automatically; custom campaign recorders supply their explicit bindings through the Python API. +The `expected_plan` and `runtime_bindings` arguments are mutually exclusive; passing both raises `ValueError`. + To inspect a changed runtime, `collection_runtime_review(request, binding)` returns its exact accepted/collection bundle hashes, collector revision, generator configuration hash, and complete sorted `changes` of `{field, accepted, collection}` for runtime source/build metadata and deployment settings. Absent values are represented as `{ "missing": true }`. Inspect the actual source changes, including scheduler patches, mounts and CPU policy, and save this record with a nonempty `rationale` in `review`. Computing that record alone is not a compatibility verdict. Verification recomputes it and rejects missing, altered or stale expectations. Equal cache capacity cannot authorize an unreviewed source change, and reviewed source changes cannot override incompatible model, precision, scheduler, graph or cache properties. Each phase is independently checked under its own frozen launch and instrumentation, including artifact containment and hashes, worker/scheduler ranks, tensor geometry and physical capacity. The serving role qualifies only after every required phase passes against the same accepted profile. A Hugging Face repository name and its exact local snapshot spelling can compare as the same model only after their immutable revision, configuration hash and sidecar hashes agree and each observed spelling independently passes the checkpoint-path validator. The report retains those spelling differences and original runtime settings. Other effective settings and geometry remain exact; every phase/rank must provide at least the accepted usable capacity. diff --git a/python/aisimulate/src/aisimulate/support/collection_readiness.py b/python/aisimulate/src/aisimulate/support/collection_readiness.py index 1ac2b935e..9ada72269 100644 --- a/python/aisimulate/src/aisimulate/support/collection_readiness.py +++ b/python/aisimulate/src/aisimulate/support/collection_readiness.py @@ -239,6 +239,8 @@ def assess_readiness( from .runtime import runtime_probe_manifest, verify_collection_runtime_bindings if runtime_bindings is not None: + if expected_plan is not None: + raise ValueError("expected_plan and runtime_bindings are mutually exclusive") return _bound_readiness(request, runtime_bindings) campaigns = [] selected = {} diff --git a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py index 75e8e1761..d9fe5e41d 100644 --- a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py +++ b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py @@ -381,6 +381,30 @@ def test_readiness_uses_phase_bound_memory_and_keeps_native_timing_failures(spli assert "runtime checkpoint" in " ".join(report["blockers"]) +@pytest.mark.parametrize("conflicting_plan", [False, True]) +def test_readiness_rejects_expected_plan_with_explicit_bindings(split_collection, monkeypatch, conflicting_plan): + from aisimulate.support import collection_readiness + + request, bindings = split_collection + _review(request, bindings) + expected_plan = load_repeatability_source(Path(bindings[0]["plan"]["path"]).parent) + if conflicting_plan: + expected_plan = replace(expected_plan, sha256="0" * 64) + monkeypatch.setattr( + collection_readiness, + "_cell_report", + lambda plan, cell, campaign, entry: {"status": "ready", "blockers": [], "cell": cell.to_dict()}, + ) + with pytest.raises(ValueError, match="expected_plan and runtime_bindings are mutually exclusive"): + collection_readiness.assess_readiness( + request, + Path("unused"), + Path("unused"), + expected_plan=expected_plan, + runtime_bindings=bindings, + ) + + @pytest.mark.parametrize("split_collection", ["example/model"], indirect=True) def test_exact_checkpoint_snapshot_spelling_preserves_full_settings_evidence(split_collection): request, bindings = split_collection @@ -419,6 +443,17 @@ def test_default_readiness_requires_compatible_raw_memory_without_explicit_bindi report = collection_readiness.assess_readiness(request, root, root / "fpm-checkpoint") assert report["ready_for_full_collection"], report["blockers"] assert report["runtime_compatibility"]["compatibility"]["status"] == "compatible" + plan = load_repeatability_source(campaign) + report = collection_readiness.assess_readiness(request, root, root / "fpm-checkpoint", expected_plan=plan) + assert report["ready_for_full_collection"], report["blockers"] + report = collection_readiness.assess_readiness( + request, + root, + root / "fpm-checkpoint", + expected_plan=replace(plan, sha256="0" * 64), + ) + assert not report["ready_for_full_collection"] + assert "current frozen runtime/launch plan" in " ".join(report["blockers"]) binding = {"phase": "prefill", "observations": _ref(Path(checkpoint["runtime_observations"]))} _change_records( binding, lambda record: record["runtime"]["source_files"].update({"vllm/v1/core/block_pool.py": "0" * 64}) From 0f25427f269225b22acf8035e93e74ef230f2f00 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 12 Oct 2026 00:26:26 +0800 Subject: [PATCH 06/13] fix: review AISimulate source revisions separately from model identity The whole-launch identity comparison incorrectly rejected reviewed collector source updates. Keep only aisimulate_revision outside immutable identity equality and include its exact accepted and collection values in the required review. Signed-off-by: Yiming Liu --- .../fpm-self-service/implementation.md | 4 +- .../aisimulate/support/runtime_collection.py | 10 ++- .../unit/test_onboard_runtime_bindings.py | 66 ++++++++++++++++++- 3 files changed, 76 insertions(+), 4 deletions(-) diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index 7b7705df2..aae4fa89f 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -1267,7 +1267,7 @@ The campaign recorder supplies a list with exactly one binding per phase require | --- | --- | | `phase` | `prefill` or `decode`, present in the selected frozen plan. Aggregated requests require both, even when their attempts differ. | | `plan` | `{ "path": ".../collection-plan.json", "sha256": "..." }`. This digest is the producer's semantic plan SHA, independently recomputed from its immutable inputs. The plan supplies the recorded configuration, launch, bundle and unique phase cell. | -| `review` | Required when runtime, instrumentation or deployment differs from the accepted probe. Exact review record described below; no wildcard or mismatch bypass exists. | +| `review` | Required when runtime, instrumentation, AISimulate source revision or deployment differs from the accepted probe. Exact review record described below; no wildcard or mismatch bypass exists. | | `checkpoint` | `{ "path": ".../fpm_forward.json", "sha256": "..." }`, with the byte SHA of the completed collector checkpoint. | | `observations` | `{ "path": ".../runtime-observations.json", "sha256": "..." }`, with the byte SHA of the immutable observation index inside that plan's campaign directory. | | `attempt_id` | The index's active runtime observation attempt, also recorded in the checkpoint. | @@ -1277,7 +1277,7 @@ Before GPU work, pass bindings containing `phase`, `plan` and any `review` to `v The `expected_plan` and `runtime_bindings` arguments are mutually exclusive; passing both raises `ValueError`. -To inspect a changed runtime, `collection_runtime_review(request, binding)` returns its exact accepted/collection bundle hashes, collector revision, generator configuration hash, and complete sorted `changes` of `{field, accepted, collection}` for runtime source/build metadata and deployment settings. Absent values are represented as `{ "missing": true }`. Inspect the actual source changes, including scheduler patches, mounts and CPU policy, and save this record with a nonempty `rationale` in `review`. Computing that record alone is not a compatibility verdict. Verification recomputes it and rejects missing, altered or stale expectations. Equal cache capacity cannot authorize an unreviewed source change, and reviewed source changes cannot override incompatible model, precision, scheduler, graph or cache properties. +To inspect a changed runtime, `collection_runtime_review(request, binding)` returns its exact accepted/collection bundle hashes, collector revision, generator configuration hash, and complete sorted `changes` of `{field, accepted, collection}` for runtime source/build metadata, launch `identity.aisimulate_revision` and deployment settings. Absent values are represented as `{ "missing": true }`. `collector_revision` remains the frozen plan's `aic_revision`; each launch retains its own recorded AISimulate source revision. Inspect the actual source changes, including scheduler patches, mounts and CPU policy, and save this record with a nonempty `rationale` in `review`. Computing that record alone is not a compatibility verdict. Verification recomputes it and rejects missing, altered or stale expectations. Equal cache capacity cannot authorize an unreviewed source change, and reviewed source changes cannot override incompatible model, precision, scheduler, graph or cache properties. Each phase is independently checked under its own frozen launch and instrumentation, including artifact containment and hashes, worker/scheduler ranks, tensor geometry and physical capacity. The serving role qualifies only after every required phase passes against the same accepted profile. A Hugging Face repository name and its exact local snapshot spelling can compare as the same model only after their immutable revision, configuration hash and sidecar hashes agree and each observed spelling independently passes the checkpoint-path validator. The report retains those spelling differences and original runtime settings. Other effective settings and geometry remain exact; every phase/rank must provide at least the accepted usable capacity. diff --git a/python/aisimulate/src/aisimulate/support/runtime_collection.py b/python/aisimulate/src/aisimulate/support/runtime_collection.py index 4dc999ae3..9af095ee3 100644 --- a/python/aisimulate/src/aisimulate/support/runtime_collection.py +++ b/python/aisimulate/src/aisimulate/support/runtime_collection.py @@ -81,6 +81,8 @@ def _plan(request: SupportRequest, binding: dict[str, Any], source: dict[str, An expected, actual = deepcopy(original), deepcopy(launch) for value in (expected, actual): value.pop("deployment") + # AISimulate source provenance is reviewed below, separately from the checkpoint. + value["identity"].pop("aisimulate_revision", None) value["model_config"].pop("path") value["model_config"].setdefault("source_files", {}) if actual != expected: @@ -116,8 +118,14 @@ def _plan(request: SupportRequest, binding: dict[str, Any], source: dict[str, An "bundle_sha256": source["provenance"]["bundle_sha256"], "runtime": source["provenance"]["runtime"], "deployment": original["deployment"], + "identity": original["identity"], + }, + { + "bundle_sha256": bundle.sha256, + "runtime": bundle.manifest["runtime"], + "deployment": launch["deployment"], + "identity": launch["identity"], }, - {"bundle_sha256": bundle.sha256, "runtime": bundle.manifest["runtime"], "deployment": launch["deployment"]}, ), } return path, plan, cells[0], bundle, reviewed diff --git a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py index d9fe5e41d..f667da904 100644 --- a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py +++ b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py @@ -34,7 +34,24 @@ def _ref(path): @pytest.fixture -def split_collection(tmp_path, capsys, request, monkeypatch): +def source_revisions(request, monkeypatch): + from . import test_onboard_runtime + + revisions = getattr(request, "param", (None, None)) + observation_fixture = test_onboard_runtime.observation_fixture + + def with_revision(*args, **kwargs): + index, launches = observation_fixture(*args, **kwargs) + if revisions[0] is not None: + launches["tp2"]["identity"]["aisimulate_revision"] = revisions[0] + return index, launches + + monkeypatch.setattr(test_onboard_runtime, "observation_fixture", with_revision) + return revisions + + +@pytest.fixture +def split_collection(tmp_path, capsys, request, monkeypatch, source_revisions): if getattr(request, "param", None) is not None: from aisimulate_core.sdk import fpm_profile @@ -63,6 +80,9 @@ def split_collection(tmp_path, capsys, request, monkeypatch): attempt["phases"] = {phase: attempt["phases"][phase]} launch = entry["launch"] launch["deployment"]["cpus_per_task"] = 32 + launch["identity"].pop("aisimulate_revision", None) + if source_revisions[1] is not None: + launch["identity"]["aisimulate_revision"] = source_revisions[1] bundle_path = observations.parent / attempt["bundle"]["manifest"] manifest = json.loads(bundle_path.read_text()) manifest["runtime"]["source_files"]["vllm/v1/core/block_pool.py"] = ("d" if phase == "prefill" else "e") * 64 @@ -76,6 +96,7 @@ def split_collection(tmp_path, capsys, request, monkeypatch): record = json.loads(path.read_text()) record.update(configuration=name, attempt_id=attempt_id, launch=launch, bundle_sha256=bundle.sha256) if ref.get("kind") == "observation": + record["identity"] = launch["identity"] record["runtime"] = {**manifest["runtime"], "image": launch["deployment"]["image"]} _write(path, record) ref["sha256"] = _ref(path)["sha256"] @@ -159,6 +180,48 @@ def test_renamed_split_collection_requires_exact_review_and_keeps_attempts(split runtime.verify_collection_runtime(request, Path(bindings[0]["observations"]["path"])) +@pytest.mark.parametrize( + "source_revisions", + [("1" * 40, "2" * 40), (None, "2" * 40), ("1" * 40, None)], + indirect=True, +) +def test_aisimulate_source_revision_requires_exact_review_and_preserves_provenance(split_collection, source_revisions): + request, bindings = split_collection + original_request = request.model_dump(mode="json") + original_source = runtime.verify_runtime_profile(request) + expected = { + "field": "/identity/aisimulate_revision", + "accepted": source_revisions[0] if source_revisions[0] is not None else {"missing": True}, + "collection": source_revisions[1] if source_revisions[1] is not None else {"missing": True}, + } + for binding in bindings: + review = runtime.collection_runtime_review(request, binding) + assert expected in review["changes"] + assert review["collector_revision"] == "reviewed-collector-revision" + with pytest.raises(ValueError, match="exact runtime change review required"): + runtime.verify_collection_runtime_plans(request, bindings) + _review(request, bindings) + for change in ("missing", "accepted", "collection"): + stale = copy.deepcopy(bindings) + changes = stale[0]["review"]["changes"] + entry = next(item for item in changes if item["field"] == expected["field"]) + if change == "missing": + changes.remove(entry) + else: + entry[change] = "unreviewed-revision" + with pytest.raises(ValueError, match="exact runtime change review required"): + runtime.verify_collection_runtime_bindings(request, stale) + assert runtime.verify_collection_runtime_plans(request, bindings)["status"] == "verified" + result = runtime.verify_collection_runtime_bindings(request, bindings) + assert result["compatibility"]["status"] == "compatible" + for phase in result["phases"].values(): + identity = phase["provenance"]["launch"]["identity"] + assert identity.get("aisimulate_revision", {"missing": True}) == expected["collection"] + assert expected in phase["binding"]["review"]["changes"] + assert request.model_dump(mode="json") == original_request + assert runtime.verify_runtime_profile(request) == original_source + + def _change_index(binding, change): path = Path(binding["observations"]["path"]) document = json.loads(path.read_text()) @@ -325,6 +388,7 @@ def mutate(record): [ ("identity", "model", "other/model"), ("identity", "model_revision", "another-revision"), + ("identity", "framework_version", "0.29.0"), ("topology", "tp", 4), ("precision", "gemm_quant_mode", "fp8"), ("collection", "max_num_seqs", 32), From b96b560afa10d269fea5c797c5f75f0ef9e154ff Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 12 Oct 2026 00:49:28 +0800 Subject: [PATCH 07/13] fix: review archived collection controls for memory compatibility Whole-policy equality rejected archived deadline overrides and explicit empty CLI lists before memory verification. Separate only the supported collection controls for exact review, preserve the frozen plans and hashes, and retain strict serving configuration and raw memory checks. Collection-control review does not establish timing or measurement equivalence. Signed-off-by: Yiming Liu --- .../fpm-self-service/implementation.md | 15 +- .../aisimulate/support/runtime_collection.py | 97 +++++++- .../unit/test_onboard_runtime_bindings.py | 226 +++++++++++++++++- 3 files changed, 333 insertions(+), 5 deletions(-) diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index aae4fa89f..f67a49e85 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -1267,7 +1267,7 @@ The campaign recorder supplies a list with exactly one binding per phase require | --- | --- | | `phase` | `prefill` or `decode`, present in the selected frozen plan. Aggregated requests require both, even when their attempts differ. | | `plan` | `{ "path": ".../collection-plan.json", "sha256": "..." }`. This digest is the producer's semantic plan SHA, independently recomputed from its immutable inputs. The plan supplies the recorded configuration, launch, bundle and unique phase cell. | -| `review` | Required when runtime, instrumentation, AISimulate source revision or deployment differs from the accepted probe. Exact review record described below; no wildcard or mismatch bypass exists. | +| `review` | Required when runtime, instrumentation, AISimulate source revision, deployment or supported collection controls differ from the accepted probe. Exact review record described below; no wildcard or mismatch bypass exists. | | `checkpoint` | `{ "path": ".../fpm_forward.json", "sha256": "..." }`, with the byte SHA of the completed collector checkpoint. | | `observations` | `{ "path": ".../runtime-observations.json", "sha256": "..." }`, with the byte SHA of the immutable observation index inside that plan's campaign directory. | | `attempt_id` | The index's active runtime observation attempt, also recorded in the checkpoint. | @@ -1279,6 +1279,19 @@ The `expected_plan` and `runtime_bindings` arguments are mutually exclusive; pas To inspect a changed runtime, `collection_runtime_review(request, binding)` returns its exact accepted/collection bundle hashes, collector revision, generator configuration hash, and complete sorted `changes` of `{field, accepted, collection}` for runtime source/build metadata, launch `identity.aisimulate_revision` and deployment settings. Absent values are represented as `{ "missing": true }`. `collector_revision` remains the frozen plan's `aic_revision`; each launch retains its own recorded AISimulate source revision. Inspect the actual source changes, including scheduler patches, mounts and CPU policy, and save this record with a nonempty `rationale` in `review`. Computing that record alone is not a compatibility verdict. Verification recomputes it and rejects missing, altered or stale expectations. Equal cache capacity cannot authorize an unreviewed source change, and reviewed source changes cannot override incompatible model, precision, scheduler, graph or cache properties. +Explicit bindings also review these archived collection controls under `/collection_controls`, preserving their exact values and the original plan and generator hashes: + +| Control | Validation and review scope | +| --- | --- | +| Backend-policy `params.agg.extra_cli_args` option `--benchmark-timeout` | One positive integer deadline, in split or `=value` form. The exact argument tokens are recorded; `[]` means the probe declared no override. Duplicate, conflicting or malformed options fail. An explicitly empty argument list is equivalent to no policy override for comparison. | +| `K8sConfig.extra_env.FPM_RESULT_WAIT_TIMEOUT_SECONDS` | One literal integer from 1 through 43,200 seconds for the outer result wait. This checks an archived control; it does not change current rendering or transport limits. | +| `DYN_BENCH_CONTENT_SEED`, `DYN_BENCH_POOL_TAG` | Literal strings, including an empty pool tag. | +| `DYN_BENCH_PREFILL_CONTENT` | `random`, `sharegpt`, `sharegpt_chain` or `zeros`. | +| `DYN_BENCH_GIANT_KV_THRESHOLD`, `DYN_BENCH_GIANT_KV_REPEATS` | Literal nonnegative integers. | +| `DYN_FORWARDPASS_METRIC_PORT` | Literal integer from 1 through 65,535. | + +The environment controls above must each appear at most once as an exact `{name, value}` entry. Their absence from the probe is recorded as `{ "missing": true }`, without inferring a historical default. All remaining backend policy and deployment fields pass the strict launch comparison, including `DYN_BENCH_KV_WARMUP` and `DYN_BENCH_PREFILL_REAL_SEED`; unknown CLI or environment overrides fail. Collection-control review authorizes only a memory compatibility comparison. It does not establish timing or measurement equivalence: preserve the original source artifacts and independently require the ordinary timing, measurement-quality and repeatability checks. No archived policy or request is rewritten. + Each phase is independently checked under its own frozen launch and instrumentation, including artifact containment and hashes, worker/scheduler ranks, tensor geometry and physical capacity. The serving role qualifies only after every required phase passes against the same accepted profile. A Hugging Face repository name and its exact local snapshot spelling can compare as the same model only after their immutable revision, configuration hash and sidecar hashes agree and each observed spelling independently passes the checkpoint-path validator. The report retains those spelling differences and original runtime settings. Other effective settings and geometry remain exact; every phase/rank must provide at least the accepted usable capacity. The result contains per-phase `provenance`, original `binding` and compatibility details, the minimum compatible capacity, and an absolute `inputs` list of byte hashes for all consulted source, plan, checkpoint, index, launch, raw and instrumentation files. Save the report separately and recheck those inputs before subsequent CPU prediction or publication. Distinct historical attempts remain distinct; do not merge indexes, rewrite old labels or modify accepted values to make a comparison pass. This compatibility report does not change profile acceptance, timing measurements, estimator behavior, repeatability policy or accuracy qualification. diff --git a/python/aisimulate/src/aisimulate/support/runtime_collection.py b/python/aisimulate/src/aisimulate/support/runtime_collection.py index 9af095ee3..e51e3b855 100644 --- a/python/aisimulate/src/aisimulate/support/runtime_collection.py +++ b/python/aisimulate/src/aisimulate/support/runtime_collection.py @@ -7,7 +7,9 @@ import hashlib import json +import re from copy import deepcopy +from dataclasses import replace from pathlib import Path from typing import Any @@ -59,6 +61,93 @@ def _source(request: SupportRequest) -> dict[str, Any]: return source +def _collection_controls(cells, deployment): + """Separate recorded collection controls for memory-only review, without editing plans.""" + from collector.fpm_forward.planner import BackendPolicy + + from aisimulate.generator.builders.fpm_builder import _benchmark_wait_timeout_seconds + + compared_cells, changes = [], [] + for cell in cells: + policy = cell.backend_policy.to_dict() + overrides = policy["generator_overrides"] + params = overrides.get("params", {}) + agg = params.get("agg", {}) + args = agg.get("extra_cli_args", []) + if not isinstance(args, list) or any(not isinstance(token, str) for token in args): + raise ValueError("collection policy extra_cli_args must be a list of strings") + remaining, deadline = [], [] + index = 0 + while index < len(args): + token = args[index] + if token == "--benchmark-timeout" or token.startswith("--benchmark-timeout="): + if deadline: + raise ValueError("collection policy contains duplicate --benchmark-timeout controls") + width = 2 if token == "--benchmark-timeout" else 1 + deadline = args[index : index + width] + _benchmark_wait_timeout_seconds(deadline) + index += width + else: + remaining.append(token) + index += 1 + if "extra_cli_args" in agg: + agg["extra_cli_args"] = remaining + if not remaining: + del agg["extra_cli_args"] + if not agg: + params.pop("agg") + if not params: + overrides.pop("params") + compared_cells.append(replace(cell, backend_policy=BackendPolicy(**policy))) + changes.extend(_differences([], deadline, f"/collection_controls/{cell.workload_kind}/benchmark_timeout")) + + deployment = deepcopy(deployment) + k8s = deployment.get("K8sConfig", {}) + environment = k8s.get("extra_env", []) + known = { + "DYN_BENCH_CONTENT_SEED", + "DYN_BENCH_PREFILL_CONTENT", + "DYN_BENCH_POOL_TAG", + "DYN_BENCH_GIANT_KV_THRESHOLD", + "DYN_BENCH_GIANT_KV_REPEATS", + "DYN_FORWARDPASS_METRIC_PORT", + "FPM_RESULT_WAIT_TIMEOUT_SECONDS", + } + retained, seen = [], set() + if not isinstance(environment, list): + raise ValueError("collection environment must be a list") + for item in environment: + if not isinstance(item, dict) or not isinstance(item.get("name"), str): + raise ValueError("collection environment requires named entries") + name = item["name"] + if name not in known: + retained.append(item) + continue + value = item.get("value") + if ( + name in seen + or set(item) != {"name", "value"} + or not isinstance(value, str) + or any(ord(char) < 32 for char in value) + ): + raise ValueError(f"collection control {name} requires one literal string value") + seen.add(name) + if name == "DYN_BENCH_PREFILL_CONTENT" and value not in {"random", "sharegpt", "sharegpt_chain", "zeros"}: + raise ValueError(f"collection control {name} has an unsupported content mode") + if name in {"DYN_BENCH_GIANT_KV_THRESHOLD", "DYN_BENCH_GIANT_KV_REPEATS"} and not re.fullmatch( + r"0|[1-9][0-9]*", value + ): + raise ValueError(f"collection control {name} must be a nonnegative integer") + if name in {"DYN_FORWARDPASS_METRIC_PORT", "FPM_RESULT_WAIT_TIMEOUT_SECONDS"}: + maximum = 65535 if name == "DYN_FORWARDPASS_METRIC_PORT" else 43200 + if not re.fullmatch(r"[1-9][0-9]*", value) or int(value) > maximum: + raise ValueError(f"collection control {name} must be an integer from 1 through {maximum}") + changes.extend(_differences({"missing": True}, value, "/collection_controls/environment/" + name)) + if "extra_env" in k8s: + k8s["extra_env"] = retained + return tuple(compared_cells), deployment, changes + + def _plan(request: SupportRequest, binding: dict[str, Any], source: dict[str, Any]): from collector.fpm_forward.repeatability import load_repeatability_deployment, load_repeatability_source from collector.fpm_forward.runtime_probe import validate_collection_probe_launch @@ -95,6 +184,7 @@ def _plan(request: SupportRequest, binding: dict[str, Any], source: dict[str, An raise ValueError("bound phase requires exactly one matching frozen collection cell") bundle = plan.runtime_instrumentation deployment = load_repeatability_deployment(path.parent) + compared_cells, compared_deployment, controls = _collection_controls(plan.cells, deployment) validate_collection_probe_launch( launch, bundle, @@ -102,10 +192,10 @@ def _plan(request: SupportRequest, binding: dict[str, Any], source: dict[str, An system=plan.system, backend=plan.backend, backend_version=plan.capability.aic_database_version, - cells=plan.cells, + cells=compared_cells, options=plan.options, profile=plan.fpm_profile, - generator_overrides=deployment, + generator_overrides=compared_deployment, phases=tuple(cell.workload_kind for cell in plan.cells), ) reviewed = { @@ -128,6 +218,7 @@ def _plan(request: SupportRequest, binding: dict[str, Any], source: dict[str, An }, ), } + reviewed["changes"] = sorted([*reviewed["changes"], *controls], key=lambda item: item["field"]) return path, plan, cells[0], bundle, reviewed @@ -313,6 +404,8 @@ def verify_collection_runtime_bindings(request: SupportRequest, bindings: list[d "inputs": list({ref["path"]: ref for ref in inputs}.values()), "compatibility": { "status": "compatible", + "scope": "memory compatibility only; collection-control review does not establish " + "timing or measurement equivalence", "accepted_capacity_bytes": bound, "observed_capacity_bytes": capacity, "source_observations_index": manifest["observations_index"], diff --git a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py index f667da904..33e642b8d 100644 --- a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py +++ b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py @@ -14,7 +14,9 @@ import pytest from collector.fpm_forward.cli import _parser +from collector.fpm_forward.config import with_kv_warmup_defaults from collector.fpm_forward.entry import _load_generator_overrides +from collector.fpm_forward.planner import BackendPolicy, _canonical_hash from collector.fpm_forward.repeatability import _subset_plan, load_repeatability_source from collector.fpm_forward.runtime_instrumentation import load_instrumentation @@ -143,6 +145,225 @@ def _review(request, bindings): } +@pytest.fixture +def archived_controls(split_collection, request): + support_request, bindings = split_collection + controls = getattr( + request, + "param", + { + "extra_cli_args": ["--benchmark-timeout", "12600"], + "extra_env": [ + {"name": "DYN_BENCH_CONTENT_SEED", "value": "0"}, + {"name": "DYN_BENCH_PREFILL_CONTENT", "value": "random"}, + {"name": "DYN_BENCH_POOL_TAG", "value": ""}, + {"name": "DYN_BENCH_GIANT_KV_THRESHOLD", "value": "1000000"}, + {"name": "DYN_BENCH_GIANT_KV_REPEATS", "value": "3"}, + {"name": "DYN_FORWARDPASS_METRIC_PORT", "value": "7020"}, + {"name": "FPM_RESULT_WAIT_TIMEOUT_SECONDS", "value": "14100"}, + ], + }, + ) + binding = bindings[0] + path = Path(binding["plan"]["path"]) + plan = load_repeatability_source(path.parent) + policy = plan.cells[0].backend_policy.to_dict() + policy["generator_overrides"].setdefault("params", {}).setdefault("agg", {})["extra_cli_args"] = controls[ + "extra_cli_args" + ] + policy = BackendPolicy(**policy) + deployment_path = path.parent / "generator-overrides.json" + deployment = json.loads(deployment_path.read_text()) + deployment.setdefault("K8sConfig", {}).setdefault("extra_env", []).extend(controls.get("extra_env", [])) + _write(deployment_path, deployment) + plan = replace( + plan, + cells=(replace(plan.cells[0], backend_policy=policy),), + backend_policies=(policy,), + generator_config_sha256=_canonical_hash(with_kv_warmup_defaults(deployment)), + ) + plan = _subset_plan(plan, {"cell_id": plan.cells[0].cell_id, "benchmark_points": {}}, comparison_mode="full_grid") + _write(path, plan.to_dict()) + binding["plan"]["sha256"] = plan.sha256 + checkpoint_path = Path(binding["checkpoint"]["path"]) + checkpoint = json.loads(checkpoint_path.read_text()) + checkpoint["plan_sha256"] = plan.sha256 + _write(checkpoint_path, checkpoint) + binding["checkpoint"] = _ref(checkpoint_path) + # Reload the archived representation through the real hash validation path. + assert load_repeatability_source(path.parent).sha256 == plan.sha256 + return support_request, bindings + + +def test_archived_collection_controls_require_exact_review_and_preserve_plan(archived_controls): + request, bindings = archived_controls + original_request = request.model_dump(mode="json") + original_source = runtime.verify_runtime_profile(request) + archived = { + path: path.read_bytes() + for binding in bindings + for path in Path(binding["plan"]["path"]).parent.rglob("*") + if path.is_file() + } + changes = [ + { + "field": "/collection_controls/prefill/benchmark_timeout", + "accepted": [], + "collection": ["--benchmark-timeout", "12600"], + }, + *[ + { + "field": "/collection_controls/environment/" + name, + "accepted": {"missing": True}, + "collection": value, + } + for name, value in { + "DYN_BENCH_CONTENT_SEED": "0", + "DYN_BENCH_PREFILL_CONTENT": "random", + "DYN_BENCH_POOL_TAG": "", + "DYN_BENCH_GIANT_KV_THRESHOLD": "1000000", + "DYN_BENCH_GIANT_KV_REPEATS": "3", + "DYN_FORWARDPASS_METRIC_PORT": "7020", + "FPM_RESULT_WAIT_TIMEOUT_SECONDS": "14100", + }.items() + ], + ] + review = runtime.collection_runtime_review(request, bindings[0]) + assert all(change in review["changes"] for change in changes) + with pytest.raises(ValueError, match="exact runtime change review required"): + runtime.verify_collection_runtime_plans(request, bindings) + _review(request, bindings) + for field in (change["field"] for change in changes): + for mutation in ("missing", "accepted", "collection"): + stale = copy.deepcopy(bindings) + entries = stale[0]["review"]["changes"] + entry = next(item for item in entries if item["field"] == field) + if mutation == "missing": + entries.remove(entry) + else: + entry[mutation] = ["unreviewed-timeout"] + with pytest.raises(ValueError, match="exact runtime change review required"): + runtime.verify_collection_runtime_bindings(request, stale) + assert runtime.verify_collection_runtime_plans(request, bindings)["status"] == "verified" + result = runtime.verify_collection_runtime_bindings(request, bindings) + assert result["compatibility"]["status"] == "compatible" + assert "timing or measurement equivalence" in result["compatibility"]["scope"] + assert result["phases"]["prefill"]["binding"] == bindings[0] + assert all(path.read_bytes() == contents for path, contents in archived.items()) + assert request.model_dump(mode="json") == original_request + assert runtime.verify_runtime_profile(request) == original_source + + +@pytest.mark.parametrize( + "archived_controls", + [{"extra_cli_args": []}, {"extra_cli_args": ["--benchmark-timeout=12600"]}], + indirect=True, +) +def test_archived_empty_policy_and_equals_deadline_are_explicitly_verified(archived_controls): + request, bindings = archived_controls + _review(request, bindings) + assert runtime.verify_collection_runtime_bindings(request, bindings)["status"] == "complete" + + +@pytest.mark.parametrize( + "archived_controls", + [ + {"extra_cli_args": args} + for args in ( + ["--benchmark-timeout"], + ["--benchmark-timeout", ""], + ["--benchmark-timeout", "0"], + ["--benchmark-timeout=-1"], + ["--benchmark-timeout", "1.5"], + ["--benchmark-timeout", 12600], + ["--benchmark-timeout 12600"], + ["--benchmark-timeout", "12600", "--benchmark-timeout=12600"], + ["--benchmark-timeout", "12600", "--benchmark-timeout=10800"], + ["--benchmark-timeout", "12600", "--unknown-option", "1"], + ["--benchmark-timeout", "12600", "--max-num-seqs", "32"], + ["--benchmark-timeout", "12600", "--enforce-eager"], + ["--benchmark-timeout", "12600", "--kv-cache-dtype", "fp8"], + ) + ] + + [ + { + "extra_cli_args": ["--benchmark-timeout", "12600"], + "extra_env": [{"name": "FPM_RESULT_WAIT_TIMEOUT_SECONDS", "value": value}], + } + for value in (None, 14100, True, "", "0", "43201", "1.5", "-1", " 14100", "${TIMEOUT}") + ] + + [ + { + "extra_cli_args": ["--benchmark-timeout", "12600"], + "extra_env": [ + {"name": "FPM_RESULT_WAIT_TIMEOUT_SECONDS", "value": "14100"}, + {"name": "FPM_RESULT_WAIT_TIMEOUT_SECONDS", "value": value}, + ], + } + for value in ("14100", "14000") + ] + + [ + {"extra_cli_args": [], "extra_env": [{"name": name, "value": "off"}]} + for name in ( + "UNKNOWN_CONTROL", + "DYN_BENCH_PREFILL_CONTENT", + "DYN_BENCH_KV_WARMUP", + "DYN_BENCH_PREFILL_REAL_SEED", + "VLLM_ATTENTION_BACKEND", + ) + ] + + [ + {"extra_cli_args": [], "extra_env": [{"name": name, "value": value}]} + for name, value in ( + ("DYN_BENCH_CONTENT_SEED", "seed\nchanged"), + ("DYN_BENCH_POOL_TAG", None), + ("DYN_BENCH_GIANT_KV_THRESHOLD", "-1"), + ("DYN_BENCH_GIANT_KV_REPEATS", "1.5"), + ("DYN_FORWARDPASS_METRIC_PORT", "0"), + ("DYN_FORWARDPASS_METRIC_PORT", "65536"), + ) + ] + + [ + { + "extra_cli_args": [], + "extra_env": [{"name": "DYN_BENCH_CONTENT_SEED", "value": "0"}] * 2, + }, + { + "extra_cli_args": [], + "extra_env": [{"name": "DYN_BENCH_POOL_TAG", "value": "", "valueFrom": "unreviewed"}], + }, + ], + indirect=True, +) +def test_archived_controls_reject_malformed_deadlines_and_serving_changes(archived_controls): + request, bindings = archived_controls + with pytest.raises(ValueError): + runtime.collection_runtime_review(request, bindings[0]) + + +def test_legacy_launch_validation_remains_strict_on_collection_controls(archived_controls): + from collector.fpm_forward.repeatability import load_repeatability_deployment + from collector.fpm_forward.runtime_probe import validate_collection_probe_launch + + _, bindings = archived_controls + directory = Path(bindings[0]["plan"]["path"]).parent + plan = load_repeatability_source(directory) + with pytest.raises(ValueError, match="topology or precision differs"): + validate_collection_probe_launch( + plan.runtime_launch, + plan.runtime_instrumentation, + model_path=plan.model_path, + system=plan.system, + backend=plan.backend, + backend_version=plan.capability.aic_database_version, + cells=plan.cells, + options=plan.options, + profile=plan.fpm_profile, + generator_overrides=load_repeatability_deployment(directory), + phases=("prefill",), + ) + + def test_renamed_split_collection_requires_exact_review_and_keeps_attempts(split_collection): request, bindings = split_collection original_request = request.model_dump(mode="json") @@ -292,6 +513,7 @@ def test_explicit_phase_plan_attempt_and_review_bindings_are_required(split_coll runtime.verify_collection_runtime_bindings(request, bindings) +@pytest.mark.parametrize("collection", ["split_collection", "archived_controls"]) @pytest.mark.parametrize( "change", [ @@ -312,8 +534,8 @@ def test_explicit_phase_plan_attempt_and_review_bindings_are_required(split_coll "capacity", ], ) -def test_raw_evidence_is_independently_validated_under_each_frozen_launch(split_collection, change): - request, bindings = split_collection +def test_raw_evidence_is_independently_validated_under_each_frozen_launch(request, collection, change): + request, bindings = request.getfixturevalue(collection) _review(request, bindings) binding = bindings[0] if change in {"missing-rank", "duplicate-rank", "artifact-bytes", "artifact-escape", "launch-attempt"}: From 5ed50924484c9fb6bdfc04ea2df23914117d4073 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 12 Oct 2026 10:23:17 +0800 Subject: [PATCH 08/13] fix: add KV-first fallback for direct FPM prefill Signed-off-by: Yiming Liu --- .../src/perfmodel/operators/fpm_forward.rs | 221 +++++++++++++++++- docs/perf-model/api/python.md | 2 +- .../fpm-self-service/implementation.md | 2 +- docs/perf-model/methods/whole-forward.md | 4 + .../tests/unit/sdk/test_fpm_profile.py | 82 ++++++- 5 files changed, 298 insertions(+), 13 deletions(-) diff --git a/crates/core/src/perfmodel/operators/fpm_forward.rs b/crates/core/src/perfmodel/operators/fpm_forward.rs index dfc264480..a8d3c770b 100644 --- a/crates/core/src/perfmodel/operators/fpm_forward.rs +++ b/crates/core/src/perfmodel/operators/fpm_forward.rs @@ -28,8 +28,10 @@ //! `interpolation="direct"` instead uses raw linear interpolation between //! genuine measured curves. Prefill uses the nearest same-batch lower and //! upper KV sites whose token curves cover the query, without a KV distance -//! limit; decode stays inside its inferred capture regime. It never -//! evaluates `sol_ops`, clamps an unsupported batch, or uses fabricated KV. +//! limit. If those sites cannot bracket the query, it interpolates KV at fixed +//! measured token counts, then brackets tokens. Decode stays inside its inferred +//! capture regime. It never evaluates `sol_ops`, clamps an unsupported batch, +//! or uses fabricated KV. //! //! The canonical `ForwardPassPerfModel` constructor selects this native //! operator for `estimation_mode="fpm_interpolation"` and pins its method. @@ -630,11 +632,7 @@ impl FpmForwardOp { .find(|&&(site_kv, _)| (site_kv as f64) < kv); let upper = covered.iter().find(|&&(site_kv, _)| (site_kv as f64) > kv); let Some((&(lo, lo_ms), &(hi, hi_ms))) = lower.zip(upper) else { - return Err(self.direct_coverage_err( - cell, - coords, - "no same-batch, two-sided KV bracket whose token curves cover this query", - )); + return self.resolve_direct_prefill_kv_first(cell, coords); }; let weight = (kv - lo as f64) / (hi as f64 - lo as f64); let latency = lo_ms + (hi_ms - lo_ms) * weight; @@ -663,6 +661,66 @@ impl FpmForwardOp { ) } + fn resolve_direct_prefill_kv_first( + &self, + cell: &FpmForwardCell, + coords: &[f64], + ) -> Result<(PerformanceResult, DirectFpmResolution), AicError> { + let (batch, tokens, kv) = (coords[0], coords[1], coords[2]); + // A collection's maximum KV can differ at each measured token count. + // Transpose only after the usual path fails, retaining the same batch + // and genuine measurements; supported queries keep their old answer. + let mut kv_curves: BTreeMap> = BTreeMap::new(); + for (&(_, site_kv), curve) in cell + .direct_prefill + .range((batch as u32, 0)..=(batch as u32, u32::MAX)) + { + for (&site_tokens, &latency) in curve { + kv_curves + .entry(site_tokens) + .or_default() + .insert(site_kv, latency); + } + } + let token_curve: BTreeMap = kv_curves + .iter() + .filter_map(|(&site_tokens, curve)| { + direct_curve_value(curve, kv).map(|latency| (site_tokens, latency)) + }) + .collect(); + let latency = direct_curve_value(&token_curve, tokens).ok_or_else(|| { + self.direct_coverage_err( + cell, + coords, + "no same-batch, two-sided measured support in either token-first or KV-first interpolation", + ) + })?; + let support = direct_curve_support(&token_curve, tokens, |token| { + fpm_coordinates(self.phase, &[batch, token, kv]) + }) + .into_iter() + .flat_map(|token_point| { + let token = token_point.coordinates.total_prefill_tokens.unwrap(); + direct_curve_support(&kv_curves[&(token as u32)], kv, |site_kv| { + fpm_coordinates(self.phase, &[batch, token, site_kv]) + }) + .into_iter() + .map(move |mut point| { + point.weight *= token_point.weight; + point + }) + }) + .collect(); + self.direct_result( + cell, + coords, + latency, + FpmQueryResolution::CrossKvInterpolation, + support, + false, + ) + } + fn direct_result( &self, cell: &FpmForwardCell, @@ -1289,6 +1347,99 @@ mod tests { .latency_ms, 27.5 ); + // KV-first could use the 999 ms points at KV=80/112. Keep both the + // old answer and original support when token-first already succeeds. + let result = direct.query_totals(&db, &[1.0, 150.0, 96.0]).unwrap(); + let support = &result.fpm_queries.as_ref().unwrap()[0].support; + assert_eq!( + support + .iter() + .map(|point| ( + point.coordinates.total_prefill_tokens, + point.coordinates.total_kv_read_tokens, + point.weight, + )) + .collect::>(), + vec![ + (Some(100.0), 64.0, 0.25), + (Some(200.0), 64.0, 0.25), + (Some(100.0), 128.0, 0.25), + (Some(200.0), 128.0, 0.25), + ] + ); + } + + #[test] + fn direct_prefill_kv_first_uses_nearest_covered_tokens_and_original_measurements() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let rows = [ + (100, 0, 10.0), + (100, 64, 30.0), + (150, 0, 900.0), // Nearer tokens, but no two-sided KV support. + (200, 0, 20.0), + (200, 128, 60.0), + (400, 0, 999.0), // Covered tokens, but farther from the query. + (400, 80, 999.0), + ] + .into_iter() + .map(|(tokens, kv, latency)| RowSpec { + workload_kind: "prefill", + batch_size: 1, + total_prefill_tokens: tokens, + total_kv_read_tokens: kv, + latency_ms: latency, + ..RowSpec::default() + }) + .collect::>(); + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let db = db_with_pair(tmp.path()); + let mut direct = op(FpmPhase::Prefill); + direct.interpolation = FpmInterpolation::Direct; + // At KV=32 the token=100/200 curves yield 20/30 ms. Token=175 is + // three quarters between them: 27.5 ms. Flattening gives 1/8,1/8,9/16,3/16. + let result = direct.query_totals(&db, &[1.0, 175.0, 32.0]).unwrap(); + assert_eq!(result.latency_ms, 27.5); + assert_eq!(result.source, Source::Silicon); + let evidence = &result.fpm_queries.as_ref().unwrap()[0]; + assert_eq!( + evidence.resolution, + FpmQueryResolution::CrossKvInterpolation + ); + assert_eq!( + evidence + .support + .iter() + .map(|point| ( + point.coordinates.total_prefill_tokens, + point.coordinates.total_kv_read_tokens, + point.latency_ms, + point.weight, + )) + .collect::>(), + vec![ + (Some(100.0), 0.0, 10.0, 0.125), + (Some(100.0), 64.0, 30.0, 0.125), + (Some(200.0), 0.0, 20.0, 0.5625), + (Some(200.0), 128.0, 60.0, 0.1875), + ] + ); + // An exact KV endpoint needs only one original row for that token. + let result = direct.query_totals(&db, &[1.0, 175.0, 64.0]).unwrap(); + assert_eq!(result.latency_ms, 37.5); // 1/4 * 30 + 3/4 * 40 ms. + assert_eq!(result.fpm_queries.as_ref().unwrap()[0].support.len(), 3); + // Bounding-box membership alone is insufficient; both token neighbors + // must themselves cover KV. Neither axis may hold a one-sided value. + for coords in [ + [1.0, 50.0, 50.0], + [1.0, 500.0, 50.0], + [1.0, 175.0, 110.0], + [1.0, 175.0, 121.0], + [1.0, 175.0, -1.0], + [2.0, 175.0, 50.0], + ] { + assert!(direct.query_totals(&db, &coords).is_err(), "{coords:?}"); + } } #[test] @@ -1340,6 +1491,62 @@ mod tests { ); } + #[test] + fn direct_prefill_kv_first_cannot_borrow_fake_rows_other_batches_or_cells() { + use crate::perf_database::fpm_forward::tests::RowSpec; + let mut rows = [ + (100, 0, 10.0, "real_kv"), + (100, 16, 15.0, "real_kv"), + (100, 64, 30.0, "fake_fallback"), + (200, 0, 20.0, "real_kv"), + (200, 128, 60.0, "real_kv"), + ] + .into_iter() + .map(|(tokens, kv, latency, seed)| RowSpec { + workload_kind: "prefill", + batch_size: 1, + total_prefill_tokens: tokens, + total_kv_read_tokens: kv, + latency_ms: latency, + kv_seed_regime: Some(seed), + ..RowSpec::default() + }) + .collect::>(); + for (batch, model, cell_id) in [ + (2, "org/model-a", "other-batch"), + (1, "org/model-b", "other-model"), + ] { + rows.push(RowSpec { + workload_kind: "prefill", + batch_size: batch, + total_prefill_tokens: 175, + total_kv_read_tokens: 32, + latency_ms: 1.0, + model_path: model, + cell_id: Some(cell_id), + ..RowSpec::default() + }); + } + let tmp = tempfile::tempdir().unwrap(); + write_pair(tmp.path(), &rows); + let mut direct = op(FpmPhase::Prefill); + direct.interpolation = FpmInterpolation::Direct; + for replace in [false, true] { + let mut db = db_with_pair(tmp.path()); + db.set_fpm_forward_for_test( + crate::perf_database::FpmForwardTable::new_with_replacement( + tmp.path().to_path_buf(), + "b200_sxm", + "vllm", + "0.25.1", + replace, + ), + ); + let error = direct.query_totals(&db, &[1.0, 175.0, 32.0]).unwrap_err(); + assert!(error.to_string().contains("genuine FPM points"), "{error}"); + } + } + #[test] fn direct_decode_excludes_healed_fake_fallback_queries_and_baselines() { use crate::perf_database::fpm_forward::tests::RowSpec; diff --git a/docs/perf-model/api/python.md b/docs/perf-model/api/python.md index a73dfe13e..5f5ab9b2c 100644 --- a/docs/perf-model/api/python.md +++ b/docs/perf-model/api/python.md @@ -124,7 +124,7 @@ for rank in result["ranks"]: print(query["resolution"], query["query"], query["support"]) ``` -Each direct lookup records `exact_lookup`, `within_curve_interpolation`, `cross_kv_interpolation`, or `cross_batch_interpolation`, its phase and model, requested coordinates, raw latency, and supporting measured coordinates, latencies and weights. Coordinates are iteration totals: `batch_size`, `total_prefill_tokens` (null for decode), and `total_kv_read_tokens`. Nested interpolation weights are multiplied for each supporting measurement. This evidence preserves `source="silicon"`, whose meaning includes interpolation; it does not establish accuracy or observed CUDA graph dispatch. +Each direct lookup records `exact_lookup`, `within_curve_interpolation`, `cross_kv_interpolation`, or `cross_batch_interpolation`, its phase and model, requested coordinates, raw latency, and supporting measured coordinates, latencies and weights. Coordinates are iteration totals: `batch_size`, `total_prefill_tokens` (null for decode), and `total_kv_read_tokens`. Nested interpolation weights are multiplied for each supporting measurement. Prefill's `cross_kv_interpolation` includes both the usual token-first order and its KV-first fallback; support always lists original measurements rather than intermediate estimates. This evidence preserves `source="silicon"`, whose meaning includes interpolation; it does not establish accuracy or observed CUDA graph dispatch. Weights describe a raw lookup, before later composition. Each rank records prefill, decode and mixed-pass baseline latencies, plus `marginal_decode_ms = max(decode_ms - decode_baseline_ms, 0)`. A baseline lookup has `decode_baseline=true`: its query is the paired decode request, but its support contains each selected row's actual measured minimum-KV point. `max_rank` is the first zero-based input index attaining the positive native maximum. The result separately records `native_latency_ms`, `correction_factor`, and final `latency_ms`. Empty work has no query evidence. Other estimators also have no direct lookup evidence; an unready regression retains `latency_ms=null`. diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index d6af28285..65ddc85b8 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -1748,7 +1748,7 @@ Within `estimator_config.fpm_interpolation`, `method: auto` retains SOL for a re Direct readiness requires genuine measurements for each operation's phase. Prefill rows cannot make an operation with no genuine decode rows ready, and vice versa. Construction tries later configured systems roots when a root lacks a required phase; if none is usable, it reports the missing phase. An operation that needs only prefill does not require decode measurements, and vice versa. Readiness does not guarantee coverage of every subsequent query. -Direct timing first uses an exact point or interpolation within a measured curve. For cross-KV prefill interpolation, it selects the nearest measured KV curve on each side at the same batch size, considering only curves that cover the requested total new-token count. This direct bracket has no KV distance limit; SOL's site-distance guard is separate. Decode interpolation respects the measured batch/capture domain. Both phases exclude synthetic `fake_fallback` rows, including healed/extrapolated values. Missing two-sided support, unmeasured prefill batches and out-of-domain queries fail explicitly. The direct route does not apply SOL-dependent prefill batch clamping or general extrapolation; 2D interpolation remains experimental. +Direct timing first uses an exact point or interpolation within a measured curve. For cross-KV prefill interpolation, it selects the nearest measured KV curve on each side at the same batch size, considering only curves that cover the requested total new-token count. If that cannot bracket the query, it first interpolates KV at fixed measured token counts, then interpolates between the nearest token counts that cover the query's KV. This accommodates irregular collection boundaries without changing previously supported results. Both orders require measured bounds at the same batch and cell; neither has a KV distance limit, and SOL's site-distance guard is separate. Decode interpolation respects the measured batch/capture domain. Both phases exclude synthetic `fake_fallback` rows, including healed/extrapolated values. Missing two-sided support, unmeasured prefill batches and out-of-domain queries fail explicitly. The direct route does not apply SOL-dependent prefill batch clamping or general extrapolation; general scattered 2D interpolation remains experimental. Use `predict --detail source --format json` to inspect executed direct lookups and their measured support, or inspect the saved `prediction.json` after requesting source details. The default text output shows operation sources and fallbacks. The canonical model's `estimate_forward_pass_detailed(metrics)` provides the same lookup evidence with rank reduction, mixed-pass baseline and online-correction context. Replay and recommendation APIs accept `ReplayOutputRequirements(capture_performance_diagnostics=True)` to retain counted query records in their returned metadata. Ordinary runs keep no query history; requested capture retains every distinct query without truncation. See [Direct FPM query evidence](../api/python.md#direct-fpm-query-evidence) for the result fields and recommendation API. Exact and interpolated values keep the existing `silicon` source tag. Provenance alone does not qualify interpolation accuracy or prove which CUDA graph a measured forward used; accuracy gates and graph-aware interpolation remain deferred until new measurements establish the relevant behavior. diff --git a/docs/perf-model/methods/whole-forward.md b/docs/perf-model/methods/whole-forward.md index c9aba5279..181ac0e32 100644 --- a/docs/perf-model/methods/whole-forward.md +++ b/docs/perf-model/methods/whole-forward.md @@ -25,6 +25,10 @@ owns artifact validation, publication, checkpoints, and recovery. profile with verified but unregistered architecture metadata. Without a profile it retains SOL. +Direct prefill first uses an exact measurement or interpolates new-token counts at fixed KV, then brackets KV using curves that cover the requested token count. If this cannot bracket the query, it interpolates KV at each fixed measured token count and then uses the nearest covered token counts on either side of the query. The second order can cover irregular collection boundaries where different prompt lengths have different maximum KV coordinates. It preserves every result and supporting measurement from the first order when that order succeeds. + +Both orders stay within measured bounds at the same batch size and selected cell. Neither uses SOL, synthetic `fake_fallback` rows, or extrapolation. The fallback is part of `method: direct` and needs no extra setting; decode interpolation is unchanged. Detailed evidence reports `cross_kv_interpolation` for either cross-KV order, with weights flattened to original measurements. Additional query coverage does not establish interpolation accuracy or matching CUDA graph execution regimes. + Readiness is phase-specific: prefill needs real prefill rows, decode needs real decode rows, and aggregated needs both. A ready table still can lack an individual query's measured domain. Missing timing and pending [resource memory](../memory.md) diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py index 14770507c..7c113c06b 100644 --- a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py +++ b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py @@ -1192,8 +1192,8 @@ def test_saved_profile_compiler_controls_merge_with_caller_options(native_profil assert controls["features"]["attention_kv_weight"] == 7 -def _write_external_profile_pair(profile, directory): - """Generate routing/identity evidence; these 2/3 ms rows are not silicon data.""" +def _write_external_profile_pair(profile, directory, *, prefill_points=None): + """Generate routing/identity evidence; these synthetic rows are not silicon data.""" import pyarrow as pa import pyarrow.parquet as pq @@ -1208,12 +1208,16 @@ def _write_external_profile_pair(profile, directory): "workload_kind": phase, "partition_policy": "balanced_v1", "batch_size": 1, - "total_prefill_tokens": 1 if phase == "prefill" else 0, + "total_prefill_tokens": tokens, "total_kv_read_tokens": kv, "latency_ms": latency, "kv_seed_regime": "real_kv", } - for phase, kv, latency in [("prefill", 0, 2.0), ("decode", 1, 3.0), ("decode", 64, 3.0)] + for phase, tokens, kv, latency in [ + *[("prefill", *point) for point in (prefill_points or [(1, 0, 2.0)])], + ("decode", 0, 1, 3.0), + ("decode", 0, 64, 3.0), + ] ] path = directory / "synthetic-profile.parquet" pq.write_table(pa.Table.from_pylist(rows), path) @@ -1235,6 +1239,76 @@ def _write_external_profile_pair(profile, directory): return path +def test_direct_prefill_kv_first_preserves_public_evidence_and_saved_config(profile_dict, monkeypatch, tmp_path): + # An irregular collection boundary: each prompt length has its own upper + # KV coordinate. Synthetic latency is P/256 + K/8192 ms, so the query is + # 10.5 + 18 = 28.5 ms; these values are not an accuracy qualification. + points = [ + (2048, 131072, 24.0), + (2048, 253920, 38.99609375), + (4096, 131072, 32.0), + (4096, 251872, 46.74609375), + ] + profile_dict["context_length"] = 256000 + profile_dict["deployments"][0]["system"] = "h200_sxm" + monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") + monkeypatch.setattr(engine, "get_model", _fail_graph) + monkeypatch.setattr(engine, "build_model_config", _fail_graph) + monkeypatch.setattr(engine, "_maybe_load_database", _fail_graph) + path = _write_external_profile_pair(profile_dict, tmp_path, prefill_points=points) + config = _request( + profile_dict, + estimator_config={ + "fpm_interpolation": {"method": "direct", "fpm_parquet_path": str(path), "collect_coverage": True}, + "correction": {"enabled": False}, + }, + ) + metrics = { + "scheduled_requests": { + "num_prefill_requests": 1, + "sum_prefill_tokens": 2688, + "sum_prefill_kv_tokens": 147456, + } + } + model = RustForwardPassPerfModel.best_available(config) + try: + detailed = model.estimate_forward_pass_detailed(metrics) + assert detailed["latency_ms"] == pytest.approx(28.5) + query = detailed["ranks"][0]["queries"][0] + assert query["resolution"] == "cross_kv_interpolation" + assert query["query"] == { + "batch_size": 1.0, + "total_prefill_tokens": 2688.0, + "total_kv_read_tokens": 147456.0, + } + assert [ + ( + point["coordinates"]["total_prefill_tokens"], + point["coordinates"]["total_kv_read_tokens"], + point["latency_ms"], + ) + for point in query["support"] + ] == points + weights = [point["weight"] for point in query["support"]] + # Token weights are 11/16 and 5/16; the KV upper weights are + # 512/3839 and 512/3775 respectively. These are their leaf products. + assert weights == pytest.approx([36597 / 61424, 5632 / 61424, 16315 / 60400, 2560 / 60400]) + assert all(0.0 <= weight <= 1.0 for weight in weights) + assert sum(weights) == pytest.approx(1.0) + assert sum(point["latency_ms"] * point["weight"] for point in query["support"]) == pytest.approx(28.5) + assert model.fpm_query_coverage()["queries"] == {"measured": 0, "interpolated": 1, "unsupported": 0} + saved = json.loads(json.dumps(model.diagnostics()["provenance"]["config"])) + assert saved["estimator_config"]["fpm_interpolation"]["method"] == "direct" + finally: + model.close() + reloaded = RustForwardPassPerfModel.best_available(saved) + try: + assert reloaded.estimate_forward_pass_detailed(metrics) == detailed + assert reloaded.estimate_forward_pass_time_ms(metrics) == pytest.approx(28.5) + finally: + reloaded.close() + + @pytest.mark.parametrize( "architecture,method", [("UnregisteredDecoderForCausalLM", "auto"), ("LlamaForCausalLM", "direct")], From c869bb1bc88c7693448b16fd6f1548b4f638dfa2 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 12 Oct 2026 11:11:41 +0800 Subject: [PATCH 09/13] fix: preserve readiness reports for unavailable runtime indexes Signed-off-by: Yiming Liu --- .../fpm-self-service/implementation.md | 2 + .../support/collection_readiness.py | 31 ++- .../aisimulate/support/runtime_collection.py | 14 +- .../unit/test_onboard_runtime_bindings.py | 213 ++++++++++++++++++ 4 files changed, 252 insertions(+), 8 deletions(-) diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index aa7fb2c89..5454952e9 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -1275,6 +1275,8 @@ The campaign recorder supplies a list with exactly one binding per phase require Before GPU work, pass bindings containing `phase`, `plan` and any `review` to `verify_collection_runtime_plans(request, bindings)`. It checks the frozen plan, archived deployment inputs, local model configuration bytes, precision and launch settings without running observers or workers. This preflight does not establish observed memory. After the bounded smoke or source collection completes, add the checkpoint, index and attempt fields and call `verify_collection_runtime_bindings(request, bindings)` before full-grid repeats. Programmatic readiness uses the same contract through `assess_readiness(request, root, checkpoint_root, runtime_bindings=bindings)`, combining raw memory validation with the ordinary native timing checks. Guided `collect-fpm --check-readiness` builds bindings from its own unchanged campaign automatically; custom campaign recorders supply their explicit bindings through the Python API. +Use absolute paths in binding references. The collector writes an absolute `runtime_observations` path in its checkpoint; a custom recorder's relative value is resolved against the checkpoint file's resolved parent directory, independently of the working directory. Directory symlinks are resolved before comparing index paths, while the index itself must remain a regular file inside the frozen campaign directory. Hash and attempt checks still apply. A missing or unusable index blocks runtime-memory readiness while preserving all per-cell timing reports. Once a formal cell attempt is recorded, its missing or invalid memory evidence cannot fall back to an older smoke attempt. + The `expected_plan` and `runtime_bindings` arguments are mutually exclusive; passing both raises `ValueError`. To inspect a changed runtime, `collection_runtime_review(request, binding)` returns its exact accepted/collection bundle hashes, collector revision, generator configuration hash, and complete sorted `changes` of `{field, accepted, collection}` for runtime source/build metadata, launch `identity.aisimulate_revision` and deployment settings. Absent values are represented as `{ "missing": true }`. `collector_revision` remains the frozen plan's `aic_revision`; each launch retains its own recorded AISimulate source revision. Inspect the actual source changes, including scheduler patches, mounts and CPU policy, and save this record with a nonempty `rationale` in `review`. Computing that record alone is not a compatibility verdict. Verification recomputes it and rejects missing, altered or stale expectations. Equal cache capacity cannot authorize an unreviewed source change, and reviewed source changes cannot override incompatible model, precision, scheduler, graph or cache properties. diff --git a/python/aisimulate/src/aisimulate/support/collection_readiness.py b/python/aisimulate/src/aisimulate/support/collection_readiness.py index 9ada72269..95db88614 100644 --- a/python/aisimulate/src/aisimulate/support/collection_readiness.py +++ b/python/aisimulate/src/aisimulate/support/collection_readiness.py @@ -237,6 +237,7 @@ def assess_readiness( from collector.fpm_forward.runner import CHECKPOINT_SCHEMA from .runtime import runtime_probe_manifest, verify_collection_runtime_bindings + from .runtime_collection import _checkpoint_observation_index if runtime_bindings is not None: if expected_plan is not None: @@ -246,6 +247,7 @@ def assess_readiness( selected = {} selected_runtime = {} blockers = [] + requires_runtime = runtime_probe_manifest(request) is not None for smoke in (True, False): checkpoint_path = checkpoint_root / ("fpm_forward_smoke.json" if smoke else "fpm_forward.json") if not checkpoint_path.exists(): @@ -294,20 +296,35 @@ def assess_readiness( "scope": "frozen requested identity; observed backend and launch evidence are reported per cell", }, ) + runtime_binding = None + if requires_runtime and any( + cell.cell_id not in selected or cell.cell_id in checkpoint["cells"] for cell in plan.cells + ): + try: + index = _checkpoint_observation_index(checkpoint_path, checkpoint) + evidence = file_evidence(index) + runtime_binding = { + "plan": {"path": str(campaign / "collection-plan.json"), "sha256": plan.sha256}, + "checkpoint": {key: report["checkpoint"][key] for key in ("path", "sha256")}, + "observations": {key: evidence[key] for key in ("path", "sha256")}, + "attempt_id": checkpoint.get("runtime_observation_attempt_id"), + } + except (OSError, TypeError, ValueError, KeyError) as error: + report["errors"].append(f"runtime memory: {error}") + blockers.append(f"{report['kind']}: runtime memory: {error}") for cell in plan.cells: entry = checkpoint["cells"].get(cell.cell_id) cell_report = _cell_report(plan, cell, campaign, entry) report["cells"].append(cell_report) if cell.cell_id not in selected or cell.cell_id in checkpoint["cells"]: selected[cell.cell_id] = {"campaign": report["kind"], "plan_sha256": plan.sha256, **cell_report} - if runtime_probe_manifest(request) is not None: - index = Path(checkpoint.get("runtime_observations", "")) + # A later formal attempt supersedes its smoke memory evidence + # even when its observation index is missing or unreadable. + selected_runtime.pop(cell.cell_id, None) + if runtime_binding is not None: selected_runtime[cell.cell_id] = { + **runtime_binding, "phase": cell.workload_kind, - "plan": {"path": str(campaign / "collection-plan.json"), "sha256": plan.sha256}, - "checkpoint": {key: report["checkpoint"][key] for key in ("path", "sha256")}, - "observations": {key: file_evidence(index)[key] for key in ("path", "sha256")}, - "attempt_id": checkpoint.get("runtime_observation_attempt_id"), "collector_attempt_id": entry.get("attempt_id") if isinstance(entry, dict) else None, } except (OSError, TypeError, ValueError, KeyError) as error: @@ -324,7 +341,7 @@ def assess_readiness( f"{item['cell']['workload_kind']} {item['cell']['cell_id']}: {reason}" for reason in item["blockers"] ) runtime = None - if runtime_probe_manifest(request) is not None: + if requires_runtime: try: runtime = verify_collection_runtime_bindings(request, list(selected_runtime.values())) except (OSError, TypeError, ValueError, KeyError) as error: diff --git a/python/aisimulate/src/aisimulate/support/runtime_collection.py b/python/aisimulate/src/aisimulate/support/runtime_collection.py index e51e3b855..e3b2e0f02 100644 --- a/python/aisimulate/src/aisimulate/support/runtime_collection.py +++ b/python/aisimulate/src/aisimulate/support/runtime_collection.py @@ -38,6 +38,18 @@ def _read(reference: dict[str, str], *, plan: bool = False) -> tuple[Path, dict[ return path, document +def _checkpoint_observation_index(checkpoint_path: Path, checkpoint: dict[str, Any]) -> Path: + recorded = checkpoint.get("runtime_observations") + if not isinstance(recorded, str) or not recorded.strip(): + raise ValueError("collector checkpoint records no runtime observation index") + index = Path(recorded) + if not index.is_absolute(): + index = checkpoint_path.resolve().parent / index + if index.is_symlink() or not index.is_file(): + raise ValueError(f"runtime observation index must be a regular file: {index}") + return index.resolve() + + def _differences(accepted: Any, collection: Any, field: str = "") -> list[dict[str, Any]]: if accepted == collection: return [] @@ -322,7 +334,7 @@ def verify_collection_runtime_bindings(request: SupportRequest, bindings: list[d if ( checkpoint.get("schema") != CHECKPOINT_SCHEMA or checkpoint.get("plan_sha256") != plan.sha256 - or Path(checkpoint.get("runtime_observations", "")).absolute() != index + or _checkpoint_observation_index(checkpoint_path, checkpoint) != index.resolve() or checkpoint.get("runtime_observation_attempt_id") != binding["attempt_id"] or set(checkpoint.get("cells", {})) - {value.cell_id for value in plan.cells} ): diff --git a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py index 33e642b8d..68c819dd5 100644 --- a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py +++ b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py @@ -747,3 +747,216 @@ def test_default_readiness_requires_compatible_raw_memory_without_explicit_bindi report = collection_readiness.assess_readiness(request, root, root / "fpm-checkpoint") assert not report["ready_for_full_collection"] assert "runtime build/source" in " ".join(report["blockers"]) + + +@pytest.fixture +def default_collection(tmp_path, capsys): + _, request, _, root = _completed_runtime_collection(tmp_path, capsys) + checkpoint_path = root / "fpm-checkpoint/fpm_forward.json" + checkpoint = json.loads(checkpoint_path.read_text()) + campaign = root / "fpm-artifacts" / checkpoint["plan_sha256"][:16] + args = _parser().parse_args(json.loads((root / "commands.json").read_text())["fpm_plan_local"][3:]) + _write(campaign / "generator-overrides.json", _load_generator_overrides(args)) + return request, root, checkpoint_path + + +def test_missing_runtime_index_preserves_all_cell_reports(default_collection): + from aisimulate.support.collection_readiness import assess_readiness + + request, root, checkpoint_path = default_collection + checkpoint = json.loads(checkpoint_path.read_text()) + original = assess_readiness(request, root, checkpoint_path.parent) + assert original["runtime_compatibility"]["status"] == "complete" + assert len(original["selected_cells"]) == 2 + checkpoint.pop("runtime_observations") + _write(checkpoint_path, checkpoint) + + report = assess_readiness(request, root, checkpoint_path.parent) + + assert report["selected_cells"] == original["selected_cells"] + assert report["campaigns"][0]["cells"] == original["campaigns"][0]["cells"] + assert not report["ready_for_full_collection"] + assert report["runtime_compatibility"] is None + assert "collector checkpoint records no runtime observation index" in " ".join(report["blockers"]) + + +@pytest.mark.parametrize("path_kind", ["relative", "symlinked_directory"]) +def test_default_readiness_resolves_index_from_checkpoint(default_collection, tmp_path, monkeypatch, path_kind): + from aisimulate.support.collection_readiness import assess_readiness + + request, root, checkpoint_path = default_collection + checkpoint = json.loads(checkpoint_path.read_text()) + index = Path(checkpoint["runtime_observations"]) + if path_kind == "relative": + checkpoint["runtime_observations"] = str(Path("..") / index.relative_to(root)) + else: + alias = tmp_path / "campaign-alias" + alias.symlink_to(index.parent, target_is_directory=True) + checkpoint["runtime_observations"] = str(alias / index.name) + _write(checkpoint_path, checkpoint) + elsewhere = tmp_path / "unrelated-working-directory" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + + report = assess_readiness(request, root, checkpoint_path.parent) + + assert report["runtime_compatibility"] is not None, report["blockers"] + assert report["runtime_compatibility"]["status"] == "complete" + assert len(report["selected_cells"]) == 2 + for phase in report["runtime_compatibility"]["phases"].values(): + assert phase["binding"]["observations"] == _ref(index.resolve()) + + +@pytest.mark.parametrize("index_state", ["empty", "null", "non_string", "directory", "missing_file", "invalid_json"]) +def test_unusable_runtime_index_preserves_all_cell_reports(default_collection, index_state): + from aisimulate.support.collection_readiness import assess_readiness + + request, root, checkpoint_path = default_collection + original = assess_readiness(request, root, checkpoint_path.parent) + checkpoint = json.loads(checkpoint_path.read_text()) + index = Path(checkpoint["runtime_observations"]) + if index_state == "invalid_json": + index.write_text("{") + else: + checkpoint["runtime_observations"] = { + "empty": "", + "null": None, + "non_string": 17, + "directory": str(index.parent), + "missing_file": str(index.parent / "missing.json"), + }[index_state] + _write(checkpoint_path, checkpoint) + + report = assess_readiness(request, root, checkpoint_path.parent) + + assert report["selected_cells"] == original["selected_cells"] + assert report["campaigns"][0]["cells"] == original["campaigns"][0]["cells"] + assert not report["ready_for_full_collection"] + assert report["runtime_compatibility"] is None + assert any("runtime memory:" in reason for reason in report["blockers"]) + + +def test_unreadable_runtime_index_preserves_all_cell_reports(default_collection, monkeypatch): + from aisimulate.support.collection_readiness import assess_readiness + + request, root, checkpoint_path = default_collection + original = assess_readiness(request, root, checkpoint_path.parent) + index = Path(json.loads(checkpoint_path.read_text())["runtime_observations"]).resolve() + read_bytes = Path.read_bytes + + def denied_index(path): + if path.resolve() == index: + raise PermissionError("runtime observation index is not readable") + return read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", denied_index) + report = assess_readiness(request, root, checkpoint_path.parent) + + assert report["selected_cells"] == original["selected_cells"] + assert report["campaigns"][0]["cells"] == original["campaigns"][0]["cells"] + assert not report["ready_for_full_collection"] + assert report["runtime_compatibility"] is None + assert "runtime observation index is not readable" in " ".join(report["blockers"]) + + +@pytest.mark.parametrize("formal_has_cells", [False, True]) +def test_formal_runtime_evidence_supersedes_only_attempted_smoke_cells(default_collection, formal_has_cells): + from aisimulate.support.collection_readiness import assess_readiness + + request, root, checkpoint_path = default_collection + formal = json.loads(checkpoint_path.read_text()) + campaign = root / "fpm-artifacts" / formal["plan_sha256"][:16] + smoke = campaign / "smoke" + shutil.copytree(campaign, smoke, ignore=shutil.ignore_patterns("smoke")) + smoke_checkpoint = copy.deepcopy(formal) + smoke_checkpoint["runtime_observations"] = str(smoke / Path(formal["runtime_observations"]).relative_to(campaign)) + _write(checkpoint_path.with_name("fpm_forward_smoke.json"), smoke_checkpoint) + checkpoint_path.unlink() + smoke_report = assess_readiness(request, root, checkpoint_path.parent) + assert smoke_report["runtime_compatibility"]["status"] == "complete" + assert {item["campaign"] for item in smoke_report["selected_cells"]} == {"smoke"} + + formal.pop("runtime_observations") + if not formal_has_cells: + formal["cells"] = {} + _write(checkpoint_path, formal) + report = assess_readiness(request, root, checkpoint_path.parent) + + assert not report["ready_for_full_collection"] + if formal_has_cells: + assert report["runtime_compatibility"] is None + assert "collector checkpoint records no runtime observation index" in " ".join(report["blockers"]) + else: + assert report["runtime_compatibility"]["status"] == "complete" + assert not any("runtime memory:" in reason for reason in report["blockers"]) + assert {item["campaign"] for item in report["selected_cells"]} == {"formal" if formal_has_cells else "smoke"} + assert all(len(campaign["cells"]) == 2 for campaign in report["campaigns"]) + assert {item["cell"]["workload_kind"] for item in report["selected_cells"]} == {"prefill", "decode"} + + +@pytest.mark.parametrize("path_kind", ["relative", "symlinked_directory"]) +def test_explicit_bindings_resolve_checkpoint_index_paths(split_collection, tmp_path, monkeypatch, path_kind): + request, bindings = split_collection + _review(request, bindings) + for binding in bindings: + checkpoint_path = Path(binding["checkpoint"]["path"]) + checkpoint = json.loads(checkpoint_path.read_text()) + if path_kind == "relative": + checkpoint["runtime_observations"] = str( + Path(checkpoint["runtime_observations"]).relative_to(checkpoint_path.parent) + ) + _write(checkpoint_path, checkpoint) + binding["checkpoint"] = _ref(checkpoint_path) + else: + alias = tmp_path / (binding["phase"] + "-alias") + alias.symlink_to(checkpoint_path.parent, target_is_directory=True) + for field in ("plan", "checkpoint", "observations"): + original = Path(binding[field]["path"]) + binding[field]["path"] = str(alias / original.relative_to(checkpoint_path.parent)) + elsewhere = tmp_path / "unrelated-working-directory" + elsewhere.mkdir() + monkeypatch.chdir(elsewhere) + + result = runtime.verify_collection_runtime_bindings(request, bindings) + + assert result["status"] == "complete" + assert set(result["phases"]) == {"prefill", "decode"} + + +@pytest.mark.parametrize("change", ["mismatch", "escape", "symlink_escape", "symlink_file", "hash"]) +def test_runtime_index_path_resolution_preserves_binding_checks(split_collection, tmp_path, change): + request, bindings = split_collection + _review(request, bindings) + binding = bindings[0] + checkpoint_path = Path(binding["checkpoint"]["path"]) + checkpoint = json.loads(checkpoint_path.read_text()) + index = Path(binding["observations"]["path"]) + if change == "mismatch": + another = index.with_name("another-index.json") + another.write_bytes(index.read_bytes()) + checkpoint["runtime_observations"] = str(another) + expected = "runtime checkpoint does not match" + elif change in {"escape", "symlink_escape"}: + outside = tmp_path / "outside-collection" + outside.mkdir() + (outside / index.name).write_bytes(index.read_bytes()) + if change == "symlink_escape": + alias = checkpoint_path.parent / "escaped-observations" + alias.symlink_to(outside, target_is_directory=True) + outside = alias + binding["observations"] = _ref(outside / index.name) + checkpoint["runtime_observations"] = str(outside / index.name) + expected = "index escapes the frozen collection directory" + elif change == "symlink_file": + alias = index.with_name("symlink-index.json") + alias.symlink_to(index) + checkpoint["runtime_observations"] = str(alias) + expected = "runtime observation index must be a regular file" + else: + binding["observations"]["sha256"] = "0" * 64 + expected = "bound artifact hash mismatch" + _write(checkpoint_path, checkpoint) + binding["checkpoint"] = _ref(checkpoint_path) + + with pytest.raises(ValueError, match=expected): + runtime.verify_collection_runtime_bindings(request, bindings) From 75b00beabef231a21972324ed951db5707746781 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 12 Oct 2026 11:23:56 +0800 Subject: [PATCH 10/13] fix: qualify readiness errors against selected evidence Signed-off-by: Yiming Liu --- .../fpm-self-service/implementation.md | 2 +- .../support/collection_readiness.py | 8 ++- .../unit/test_onboard_runtime_bindings.py | 61 +++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index 5454952e9..704b01abd 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -1275,7 +1275,7 @@ The campaign recorder supplies a list with exactly one binding per phase require Before GPU work, pass bindings containing `phase`, `plan` and any `review` to `verify_collection_runtime_plans(request, bindings)`. It checks the frozen plan, archived deployment inputs, local model configuration bytes, precision and launch settings without running observers or workers. This preflight does not establish observed memory. After the bounded smoke or source collection completes, add the checkpoint, index and attempt fields and call `verify_collection_runtime_bindings(request, bindings)` before full-grid repeats. Programmatic readiness uses the same contract through `assess_readiness(request, root, checkpoint_root, runtime_bindings=bindings)`, combining raw memory validation with the ordinary native timing checks. Guided `collect-fpm --check-readiness` builds bindings from its own unchanged campaign automatically; custom campaign recorders supply their explicit bindings through the Python API. -Use absolute paths in binding references. The collector writes an absolute `runtime_observations` path in its checkpoint; a custom recorder's relative value is resolved against the checkpoint file's resolved parent directory, independently of the working directory. Directory symlinks are resolved before comparing index paths, while the index itself must remain a regular file inside the frozen campaign directory. Hash and attempt checks still apply. A missing or unusable index blocks runtime-memory readiness while preserving all per-cell timing reports. Once a formal cell attempt is recorded, its missing or invalid memory evidence cannot fall back to an older smoke attempt. +Use absolute paths in binding references. The collector writes an absolute `runtime_observations` path in its checkpoint; a custom recorder's relative value is resolved against the checkpoint file's resolved parent directory, independently of the working directory. Directory symlinks are resolved before comparing index paths, while the index itself must remain a regular file inside the frozen campaign directory. Hash and attempt checks still apply. Guided `collect-fpm --check-readiness` preserves all per-cell timing reports when an observation index is missing or unusable. It retains historical index errors in campaign diagnostics; an index error blocks runtime-memory readiness only while the affected campaign supplies a selected cell. Once a formal cell attempt is recorded, its missing or invalid memory evidence cannot fall back to an older smoke attempt. The `expected_plan` and `runtime_bindings` arguments are mutually exclusive; passing both raises `ValueError`. diff --git a/python/aisimulate/src/aisimulate/support/collection_readiness.py b/python/aisimulate/src/aisimulate/support/collection_readiness.py index 95db88614..98f9a54e7 100644 --- a/python/aisimulate/src/aisimulate/support/collection_readiness.py +++ b/python/aisimulate/src/aisimulate/support/collection_readiness.py @@ -246,6 +246,7 @@ def assess_readiness( campaigns = [] selected = {} selected_runtime = {} + selected_runtime_errors = {} blockers = [] requires_runtime = runtime_probe_manifest(request) is not None for smoke in (True, False): @@ -297,6 +298,7 @@ def assess_readiness( }, ) runtime_binding = None + runtime_error = None if requires_runtime and any( cell.cell_id not in selected or cell.cell_id in checkpoint["cells"] for cell in plan.cells ): @@ -311,7 +313,7 @@ def assess_readiness( } except (OSError, TypeError, ValueError, KeyError) as error: report["errors"].append(f"runtime memory: {error}") - blockers.append(f"{report['kind']}: runtime memory: {error}") + runtime_error = f"{report['kind']}: runtime memory: {error}" for cell in plan.cells: entry = checkpoint["cells"].get(cell.cell_id) cell_report = _cell_report(plan, cell, campaign, entry) @@ -321,6 +323,9 @@ def assess_readiness( # A later formal attempt supersedes its smoke memory evidence # even when its observation index is missing or unreadable. selected_runtime.pop(cell.cell_id, None) + selected_runtime_errors.pop(cell.cell_id, None) + if runtime_error is not None: + selected_runtime_errors[cell.cell_id] = runtime_error if runtime_binding is not None: selected_runtime[cell.cell_id] = { **runtime_binding, @@ -342,6 +347,7 @@ def assess_readiness( ) runtime = None if requires_runtime: + blockers.extend(dict.fromkeys(selected_runtime_errors.values())) try: runtime = verify_collection_runtime_bindings(request, list(selected_runtime.values())) except (OSError, TypeError, ValueError, KeyError) as error: diff --git a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py index 68c819dd5..3fa8f95ed 100644 --- a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py +++ b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py @@ -859,6 +859,67 @@ def denied_index(path): assert "runtime observation index is not readable" in " ".join(report["blockers"]) +@pytest.mark.parametrize("formal_phases", [(), ("prefill",), ("prefill", "decode")]) +@pytest.mark.parametrize("index_state", ["missing", "missing_file", "directory", "unreadable"]) +def test_only_selected_smoke_index_errors_block_readiness(default_collection, monkeypatch, formal_phases, index_state): + from aisimulate.support import collection_readiness + + request, root, checkpoint_path = default_collection + monkeypatch.setattr( + collection_readiness, + "_cell_report", + lambda plan, cell, campaign, entry: {"status": "ready", "blockers": [], "cell": cell.to_dict()}, + ) + baseline = collection_readiness.assess_readiness(request, root, checkpoint_path.parent) + assert baseline["ready_for_full_collection"], baseline["blockers"] + formal = json.loads(checkpoint_path.read_text()) + campaign = root / "fpm-artifacts" / formal["plan_sha256"][:16] + smoke = campaign / "smoke" + shutil.copytree(campaign, smoke, ignore=shutil.ignore_patterns("smoke")) + smoke_checkpoint = copy.deepcopy(formal) + index = smoke / Path(formal["runtime_observations"]).relative_to(campaign) + smoke_checkpoint["runtime_observations"] = str(index) + if index_state == "missing": + smoke_checkpoint.pop("runtime_observations") + elif index_state == "missing_file": + index.unlink() + elif index_state == "directory": + smoke_checkpoint["runtime_observations"] = str(index.parent) + else: + read_bytes = Path.read_bytes + + def denied_smoke_index(path): + if path.resolve() == index.resolve(): + raise PermissionError("smoke runtime observation index is not readable") + return read_bytes(path) + + monkeypatch.setattr(Path, "read_bytes", denied_smoke_index) + _write(checkpoint_path.with_name("fpm_forward_smoke.json"), smoke_checkpoint) + cells_by_phase = {item["cell"]["workload_kind"]: item["cell"]["cell_id"] for item in baseline["selected_cells"]} + formal["cells"] = {cells_by_phase[phase]: formal["cells"][cells_by_phase[phase]] for phase in formal_phases} + _write(checkpoint_path, formal) + + report = collection_readiness.assess_readiness(request, root, checkpoint_path.parent) + + assert report["campaigns"][0]["kind"] == "smoke" + assert report["campaigns"][0]["errors"] + assert all("runtime memory:" in error for error in report["campaigns"][0]["errors"]) + assert report["campaigns"][1]["errors"] == [] + assert all(len(item["cells"]) == 2 for item in report["campaigns"]) + assert {item["cell"]["workload_kind"]: item["campaign"] for item in report["selected_cells"]} == { + phase: "formal" if phase in formal_phases else "smoke" for phase in ("prefill", "decode") + } + if len(formal_phases) == 2: + assert report["ready_for_full_collection"], report["blockers"] + assert report["blockers"] == [] + assert report["runtime_compatibility"]["status"] == "complete" + assert set(report["runtime_compatibility"]["phases"]) == {"prefill", "decode"} + else: + assert not report["ready_for_full_collection"] + assert report["runtime_compatibility"] is None + assert any("smoke: runtime memory:" in reason for reason in report["blockers"]) + + @pytest.mark.parametrize("formal_has_cells", [False, True]) def test_formal_runtime_evidence_supersedes_only_attempted_smoke_cells(default_collection, formal_has_cells): from aisimulate.support.collection_readiness import assess_readiness From 20e3eb9d52325a3e5a24e34c54dd09da130346fd Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Mon, 12 Oct 2026 16:42:46 +0800 Subject: [PATCH 11/13] fix: load aggregated FPM full-grid medians Signed-off-by: Yiming Liu --- .../perfmodel/perf_database/fpm_forward.rs | 139 +++++++++++++++--- docs/perf-model/methods/whole-forward.md | 16 ++ .../tests/unit/sdk/test_fpm_profile.py | 88 +++++++++++ 3 files changed, 224 insertions(+), 19 deletions(-) diff --git a/crates/core/src/perfmodel/perf_database/fpm_forward.rs b/crates/core/src/perfmodel/perf_database/fpm_forward.rs index 6d2c445ed..6d940a922 100644 --- a/crates/core/src/perfmodel/perf_database/fpm_forward.rs +++ b/crates/core/src/perfmodel/perf_database/fpm_forward.rs @@ -61,6 +61,7 @@ pub const FPM_FORWARD_PARTITION_POLICY: &str = "balanced_v1"; /// Legacy collector policy; self-benchmark repeats use a per-row contract. pub const FPM_FORWARD_MEASUREMENT_POLICY: &str = "dynamo_native_single_sample_v1"; const FPM_ROW_MEASUREMENT_POLICY: &str = "per_row_single_sample_or_median_of_3"; +const FPM_FULL_GRID_MEASUREMENT_POLICY: &str = "per_row_single_sample_or_full_grid_median_v1"; /// `kv_seed_regime` marker for rows whose KV state the collector could not /// reach through the real kvwarm chain and fabricated instead. Such rows /// are measurements of the wrong regime (observed 2-3.7x inflated on 4-GPU @@ -428,7 +429,7 @@ fn sha256_file(path: &Path) -> Result { struct SidecarContract { schema_version: u64, row_count: Option, - per_row_policy: bool, + row_median_policy: Option<&'static str>, selector_dcp: Option>, } fn validate_sidecar( @@ -486,16 +487,18 @@ fn validate_sidecar( ))); } let measurement_policy = metadata.get("measurement_policy").and_then(|v| v.as_str()); - if !matches!( - measurement_policy, - Some(FPM_FORWARD_MEASUREMENT_POLICY | FPM_ROW_MEASUREMENT_POLICY) - ) { - return Err(structural(format!( - "unsupported FPM measurement_policy={:?} (expected {FPM_FORWARD_MEASUREMENT_POLICY:?} or {FPM_ROW_MEASUREMENT_POLICY:?}): {}", - metadata.get("measurement_policy"), - metadata_path.display() - ))); - } + let row_median_policy = match measurement_policy { + Some(FPM_FORWARD_MEASUREMENT_POLICY) => None, + Some(FPM_ROW_MEASUREMENT_POLICY) => Some("kvwarm_median_of_3"), + Some(FPM_FULL_GRID_MEASUREMENT_POLICY) => Some("independent_full_grid_median_of_3"), + _ => { + return Err(structural(format!( + "unsupported FPM measurement_policy={:?} (expected {FPM_FORWARD_MEASUREMENT_POLICY:?}, {FPM_ROW_MEASUREMENT_POLICY:?} or {FPM_FULL_GRID_MEASUREMENT_POLICY:?}): {}", + metadata.get("measurement_policy"), + metadata_path.display() + ))); + } + }; // The commit record names the database identity it was published for; // contradictory metadata (a pair copied into the wrong tree with its // sidecar) is rejected here, before any row is read. @@ -544,7 +547,7 @@ fn validate_sidecar( Ok(SidecarContract { schema_version: schema_version.unwrap(), row_count: json_uint(metadata.get("row_count")), - per_row_policy: measurement_policy == Some(FPM_ROW_MEASUREMENT_POLICY), + row_median_policy, selector_dcp, }) } @@ -661,7 +664,7 @@ fn load_pair( let dcp_col = reader.col_optional("dcp"); let row_policy_col = reader.col_optional("measurement_policy"); let repeats_col = reader.col_optional("measurement_repeats"); - if sidecar.per_row_policy && (row_policy_col.is_none() || repeats_col.is_none()) { + if sidecar.row_median_policy.is_some() && (row_policy_col.is_none() || repeats_col.is_none()) { return Err(structural("FPM per-row measurement policy requires measurement_policy and measurement_repeats columns".into())); } @@ -821,14 +824,12 @@ fn load_pair( "FPM row {index} dcp disagrees with the sidecar selector" ))); } - if sidecar.per_row_policy { + if let Some(median_policy) = sidecar.row_median_policy { let policy = row.str_optional(row_policy_col)?; let repeats = row.u32_optional(repeats_col)?; - if !matches!( - (policy, repeats), - (Some(FPM_FORWARD_MEASUREMENT_POLICY), Some(1)) - | (Some("kvwarm_median_of_3"), Some(3)) - ) { + if (policy, repeats) != (Some(FPM_FORWARD_MEASUREMENT_POLICY), Some(1)) + && (policy, repeats) != (Some(median_policy), Some(3)) + { return Err(structural(format!( "FPM row {index} measurement policy/repeats are inconsistent" ))); @@ -1932,6 +1933,106 @@ pub(crate) mod tests { ); } + #[test] + fn full_grid_median_policy_preserves_aggregate_latencies() { + let tmp = tempfile::tempdir().unwrap(); + let rows = [ + RowSpec { + workload_kind: "prefill", + batch_size: 1, + total_prefill_tokens: 128, + total_kv_read_tokens: 0, + latency_ms: 7.25, + measurement: Some(("independent_full_grid_median_of_3", 3)), + ..RowSpec::default() + }, + RowSpec { + workload_kind: "decode", + batch_size: 1, + total_prefill_tokens: 0, + total_kv_read_tokens: 128, + latency_ms: 3.125, + measurement: Some((FPM_FORWARD_MEASUREMENT_POLICY, 1)), + ..RowSpec::default() + }, + ]; + write_pair_with(tmp.path(), &rows, |meta| { + meta.insert( + "measurement_policy".into(), + "per_row_single_sample_or_full_grid_median_v1".into(), + ); + }); + let table = loaded_table(tmp.path()); + let cells = table + .cells() + .expect("aggregate pair must load without campaign files"); + assert_eq!(cells.len(), 1); + // Independent oracle: stored fixture latencies are already aggregated. + assert_eq!(cells[0].direct_prefill[&(1, 0)][&128], 7.25); + assert_eq!(cells[0].direct_decode[&1][&128], 3.125); + } + + #[test] + fn full_grid_median_policy_rejects_inconsistent_records() { + for (sidecar_policy, row_policy, repeats) in [ + ( + FPM_FULL_GRID_MEASUREMENT_POLICY, + "independent_full_grid_median_of_3", + 0, + ), + ( + FPM_FULL_GRID_MEASUREMENT_POLICY, + "independent_full_grid_median_of_3", + 1, + ), + ( + FPM_FULL_GRID_MEASUREMENT_POLICY, + "independent_full_grid_median_of_3", + 6, + ), + ( + FPM_FULL_GRID_MEASUREMENT_POLICY, + FPM_FORWARD_MEASUREMENT_POLICY, + 3, + ), + (FPM_FULL_GRID_MEASUREMENT_POLICY, "kvwarm_median_of_3", 3), + ( + FPM_FULL_GRID_MEASUREMENT_POLICY, + "independent_full_grid_median_of_6", + 6, + ), + (FPM_FULL_GRID_MEASUREMENT_POLICY, "unknown", 3), + ( + FPM_ROW_MEASUREMENT_POLICY, + "independent_full_grid_median_of_3", + 3, + ), + ] { + let tmp = tempfile::tempdir().unwrap(); + let rows = [RowSpec { + measurement: Some((row_policy, repeats)), + ..RowSpec::default() + }]; + write_pair_with(tmp.path(), &rows, |meta| { + meta.insert("measurement_policy".into(), sidecar_policy.into()); + }); + let error = loaded_table(tmp.path()).cells().unwrap_err(); + assert!(error.to_string().contains("policy/repeats"), "{error}"); + } + let tmp = tempfile::tempdir().unwrap(); + write_pair_with(tmp.path(), &default_rows(), |meta| { + meta.insert( + "measurement_policy".into(), + FPM_FULL_GRID_MEASUREMENT_POLICY.into(), + ); + }); + let error = loaded_table(tmp.path()).cells().unwrap_err(); + assert!( + error.to_string().contains("requires measurement_policy"), + "{error}" + ); + } + #[test] fn absent_parquet_is_soft_not_collected() { let tmp = tempfile::tempdir().expect("tmpdir"); diff --git a/docs/perf-model/methods/whole-forward.md b/docs/perf-model/methods/whole-forward.md index 181ac0e32..ca692e100 100644 --- a/docs/perf-model/methods/whole-forward.md +++ b/docs/perf-model/methods/whole-forward.md @@ -144,6 +144,22 @@ every row declares a consistent `measurement_policy`/`measurement_repeats` pair: `dynamo_native_single_sample_v1`/1 or `kvwarm_median_of_3`/3. Values are already aggregated by the producer; the loader preserves their latency. +For independent full-grid repetitions, use the existing +`per_row_single_sample_or_full_grid_median_v1` sidecar policy. Every row must +declare `dynamo_native_single_sample_v1`/1 or +`independent_full_grid_median_of_3`/3. The latter records the producer's median +of three independent full-grid measurements; it is a different procedure from +`kvwarm_median_of_3`, and the two per-row sidecar policies cannot be exchanged. +Other repetition counts and unknown row policies are rejected. + +Prediction needs only the aggregated Parquet table and its metadata sidecar. +The loader preserves the stored latencies and checks the pair's digest, row +count, identity, workload and duplicate coordinates, as well as the declared +row policy/count pairs. It does not open raw campaign files, recompute medians +or assess repeatability and serving accuracy. The publisher must validate +aggregation and quality evidence before publication; successful loading does +not qualify the dataset's accuracy. + For a Kimi K3 text profile, an explicit configuration can be: ```python diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py index 7c113c06b..e05ff66ff 100644 --- a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py +++ b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py @@ -1239,6 +1239,94 @@ def _write_external_profile_pair(profile, directory, *, prefill_points=None): return path +@pytest.fixture +def full_grid_median_profile(profile_dict, monkeypatch, tmp_path): + import pyarrow as pa + import pyarrow.parquet as pq + + profile_dict["deployments"][0]["system"] = "h200_sxm" + monkeypatch.setenv("AIC_ALLOW_UNLISTED_VERSIONS", "1") + monkeypatch.setattr(engine, "get_model", _fail_graph) + monkeypatch.setattr(engine, "build_model_config", _fail_graph) + monkeypatch.setattr(engine, "_maybe_load_database", _fail_graph) + path = _write_external_profile_pair(profile_dict, tmp_path) + rows = pq.read_table(path).to_pylist() + for row in rows: + row.update(measurement_policy="independent_full_grid_median_of_3", measurement_repeats=3) + pq.write_table(pa.Table.from_pylist(rows), path) + metadata = json.loads(path.with_suffix(".metadata.json").read_text()) + metadata.update( + measurement_policy="per_row_single_sample_or_full_grid_median_v1", + parquet_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), + ) + path.with_suffix(".metadata.json").write_text(json.dumps(metadata)) + config = _request( + profile_dict, + estimator_config={ + "fpm_interpolation": {"method": "direct", "fpm_parquet_path": str(path), "collect_coverage": True}, + "correction": {"enabled": False}, + }, + ) + return config, path + + +def test_full_grid_medians_query_and_reload_without_campaign_files(full_grid_median_profile, tmp_path): + config, path = full_grid_median_profile + originals = {file: file.read_bytes() for file in (path, path.with_suffix(".metadata.json"))} + # Only the aggregate pair is distributed: these are synthetic stored medians, + # not a test of the publisher's aggregation or quality qualification. + assert set(tmp_path.iterdir()) == set(originals) + queries = [ + ({"scheduled_requests": {"num_prefill_requests": 1, "sum_prefill_tokens": 1}}, 2.0), + ({"scheduled_requests": {"num_decode_requests": 1, "sum_decode_kv_tokens": 1}}, 3.0), + ] + model = RustForwardPassPerfModel.best_available(config) + try: + for query, expected in queries: + assert model.estimate_forward_pass_time_ms(query) == expected + assert model.fpm_query_coverage()["queries"] == {"measured": 2, "interpolated": 0, "unsupported": 0} + saved_path = tmp_path / "saved-config.json" + saved_path.write_text(json.dumps(model.diagnostics()["provenance"]["config"])) + finally: + model.close() + restored = RustForwardPassPerfModel.best_available(json.loads(saved_path.read_text())) + try: + assert restored.diagnostics()["provenance"]["config"]["estimator_config"]["fpm_interpolation"][ + "fpm_parquet_path" + ] == str(path) + for query, expected in queries: + assert restored.estimate_forward_pass_time_ms(query) == expected + finally: + restored.close() + assert {file: file.read_bytes() for file in originals} == originals + + +@pytest.mark.parametrize( + "row_changes,sidecar_policy", + [ + ({"measurement_repeats": 1}, "per_row_single_sample_or_full_grid_median_v1"), + ({"measurement_repeats": 6}, "per_row_single_sample_or_full_grid_median_v1"), + ({"measurement_repeats": None}, "per_row_single_sample_or_full_grid_median_v1"), + ({"measurement_policy": "unknown"}, "per_row_single_sample_or_full_grid_median_v1"), + ({"measurement_policy": "kvwarm_median_of_3"}, "per_row_single_sample_or_full_grid_median_v1"), + ({}, "per_row_single_sample_or_median_of_3"), + ], +) +def test_full_grid_medians_reject_inconsistent_provenance(full_grid_median_profile, row_changes, sidecar_policy): + import pyarrow as pa + import pyarrow.parquet as pq + + config, path = full_grid_median_profile + rows = pq.read_table(path).to_pylist() + rows[-1].update(row_changes) + pq.write_table(pa.Table.from_pylist(rows), path) + metadata = json.loads(path.with_suffix(".metadata.json").read_text()) + metadata.update(measurement_policy=sidecar_policy, parquet_sha256=hashlib.sha256(path.read_bytes()).hexdigest()) + path.with_suffix(".metadata.json").write_text(json.dumps(metadata)) + with pytest.raises(PerfDataNotAvailableError, match="policy/repeats"): + RustForwardPassPerfModel.best_available(config) + + def test_direct_prefill_kv_first_preserves_public_evidence_and_saved_config(profile_dict, monkeypatch, tmp_path): # An irregular collection boundary: each prompt length has its own upper # KV coordinate. Synthetic latency is P/256 + K/8192 ms, so the query is From fe8cebb6adbdd1e1b90c3e23039b305e7c381df0 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Tue, 13 Oct 2026 22:44:29 +0800 Subject: [PATCH 12/13] fix: forward FPM finalization timeout to collector startup Signed-off-by: Yiming Liu --- docs/aic-backward-compatibility/cli.md | 8 ++ .../collector/fpm_forward/runner.py | 6 +- .../aisimulate/src/aisimulate/fpm_contract.py | 14 ++- .../tests/unit/collector/test_fpm_exec.py | 6 +- .../tests/unit/collector/test_fpm_runner.py | 114 ++++++++++++++---- 5 files changed, 120 insertions(+), 28 deletions(-) diff --git a/docs/aic-backward-compatibility/cli.md b/docs/aic-backward-compatibility/cli.md index d8500506b..95ac77bb1 100644 --- a/docs/aic-backward-compatibility/cli.md +++ b/docs/aic-backward-compatibility/cli.md @@ -754,6 +754,14 @@ K8sConfig: `Workers.agg.extra_cli_args` must be a `list[str]`, and the final command must include `--benchmark-mode` with one of `agg`, `prefill`, or `decode`. This runtime option selects the FPM collection phase; it does not change the required single aggregated-worker deployment topology. `K8sConfig.extra_env` accepts concrete `{name, value}` entries only; `valueFrom`, `envFrom`, and Secret-derived values are not supported. The normal rule, mapping, and versioned-template pipeline still builds the base command before the extra arguments are appended. If both `--benchmark-output-path` and `DYN_FPM_BENCHMARK_OUTPUT_PATH` are supplied, they must be identical. If neither is supplied, the generator adds `/results/benchmark.json` to both the command and environment. +When the FPM collector drives these artifacts, its runtime also accepts this `K8sConfig.extra_env` entry: + +| Environment entry name | Default | Accepted value and behavior | +| --- | --- | --- | +| `FPM_FINALIZATION_TIMEOUT_SECONDS` | `120` seconds | A literal string containing an integer from `1` through `3600`, such as `{name: FPM_FINALIZATION_TIMEOUT_SECONDS, value: "600"}`. Invalid values fail before engine startup. | + +The collector stages this setting in `collector-runtime-env.sh` for both Kubernetes and Slurm so the parent collection process receives it before launching `run.sh`. The deadline starts after all local rank timing files are complete and bounds waiting for the matching merged artifact and any advertised worker-probe sidecar. It cannot extend the overall collection deadline. On expiry, collection fails and retains raw timings for recovery. Running the generated engine script alone does not enforce this collector-owned deadline. + `K8sConfig.fpm_shared_memory_size` controls the generated `/dev/shm` `emptyDir` limit, while `K8sConfig.fpm_resource_labels` adds labels to the workload and its Pods. Container memory, ephemeral-storage, and other requests or limits can be supplied under `K8sConfig.worker_extra_pod_spec.mainContainer.resources`; the Generator-resolved per-node GPU count cannot be changed there. Grove does not inject a scheduler or queue: clusters that use KAI Scheduler can set `worker_extra_pod_spec.schedulerName` and the `kai.scheduler/queue` resource label explicitly, while other clusters can supply their own scheduler settings through the same generic fields. Matching user-provided `results` or `dshm` volume-and-mount pairs are preserved instead of replaced. The generated artifact directory contains only: diff --git a/python/aisimulate/collector/fpm_forward/runner.py b/python/aisimulate/collector/fpm_forward/runner.py index 3c8f80f0a..5b14ab95c 100644 --- a/python/aisimulate/collector/fpm_forward/runner.py +++ b/python/aisimulate/collector/fpm_forward/runner.py @@ -38,6 +38,7 @@ FPM_CELL_LABEL, FPM_ENGINE_BENCHMARK_OUTPUT_ENV, FPM_ENV_FILENAME, + FPM_FINALIZATION_TIMEOUT_ENV, FPM_MANIFEST_FILENAME, FPM_RESULTS_DIR, FPM_RUN_ID_ENV, @@ -1598,11 +1599,12 @@ def _configured_sampling_metadata( def _write_runtime_environment(cell_dir: Path, overrides: dict[str, Any]) -> None: # Generator owns run.sh and fpm_env.sh. Mirror only startup inputs through - # this Collector-owned file because Slurm does not start a Kubernetes Pod - # with extra_env, and run.sh's exports happen after the preflight process. + # this Collector-owned file: run.sh cannot export settings back to its + # parent runtime, and Slurm does not inherit a Kubernetes Pod environment. names = { *FPM_KV_WARMUP_DEFAULTS, READINESS_TIMEOUT_ENV, + FPM_FINALIZATION_TIMEOUT_ENV, "PYTHONPATH", "DYN_FPM_DSV41_REAL_KV", "DYN_FPM_INPUT_TEXT", diff --git a/python/aisimulate/src/aisimulate/fpm_contract.py b/python/aisimulate/src/aisimulate/fpm_contract.py index 912bd6c62..7c7c6c066 100644 --- a/python/aisimulate/src/aisimulate/fpm_contract.py +++ b/python/aisimulate/src/aisimulate/fpm_contract.py @@ -39,8 +39,12 @@ :data:`FPM_ENV_EXPORTED_VARS` and :data:`FPM_RUN_ID_ENV`; * collector tests assert ``fpm_exec.sh`` consumes only :data:`FPM_ENV_EXPORTED_VARS` (plus the :data:`FPM_BARRIER_TIMEOUT_ENV` - tunable it defaults in-script) and that its result-naming shell function - matches :func:`fpm_benchmark_result_name` on a shared vector set. + tunable it defaults in-script and Collector-owned startup settings) and + that its result-naming shell function matches + :func:`fpm_benchmark_result_name` on a shared vector set. Generated-path + tests verify :data:`FPM_FINALIZATION_TIMEOUT_ENV` reaches the parent + runtime through the Collector's staged ``collector-runtime-env.sh`` in + both Kubernetes and Slurm, before the child ``run.sh`` executes. """ from __future__ import annotations @@ -110,6 +114,12 @@ # while the collection runtime keeps the originals. FPM_BARRIER_TIMEOUT_ENV = "FPM_COMPLETION_BARRIER_TIMEOUT_SECONDS" +# Collector-owned finalization deadline, supplied through K8sConfig.extra_env. +# Unlike the Generator-owned barrier export, the Collector forwards this in +# collector-runtime-env.sh so the parent runtime sees it before engine launch. +# fpm_exec.sh defaults to 120 seconds and rejects values outside 1..3600. +FPM_FINALIZATION_TIMEOUT_ENV = "FPM_FINALIZATION_TIMEOUT_SECONDS" + # In-pod results directory: the Generator mounts it and defaults the engine's # benchmark output under it; the collector's runtime and transfer own its # contents. Shell/in-pod replicas of this path are pinned by tests. diff --git a/python/aisimulate/tests/unit/collector/test_fpm_exec.py b/python/aisimulate/tests/unit/collector/test_fpm_exec.py index e77523626..0d074b8ef 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_exec.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_exec.py @@ -28,6 +28,7 @@ from aisimulate.fpm_contract import ( FPM_ENV_EXPORTED_VARS, + FPM_FINALIZATION_TIMEOUT_ENV, FPM_NATIVE_BENCHMARK_RESULT_SCHEMA_VERSION, fpm_expected_result_paths, ) @@ -1230,7 +1231,8 @@ def stop(*_): def test_fpm_exec_consumes_only_contract_environment(): """Everything fpm_exec.sh needs from the Generator travels through the fpm_env.sh exports. The Collector separately stages the configurable - readiness budget; completion-barrier timeout remains operator-tunable.""" + readiness and finalization budgets; completion-barrier timeout remains + operator-tunable.""" script = FPM_EXEC.read_text() # Match complete shell identifiers. DYN_FPM_* exports configure the native @@ -1240,7 +1242,7 @@ def test_fpm_exec_consumes_only_contract_environment(): assert {"FPM_NODE_RANK", "FPM_MASTER_ADDR", "FPM_BENCHMARK_OUTPUT_PATH"} <= consumed allowed = set(FPM_ENV_EXPORTED_VARS) | { "FPM_COMPLETION_BARRIER_TIMEOUT_SECONDS", - "FPM_FINALIZATION_TIMEOUT_SECONDS", + FPM_FINALIZATION_TIMEOUT_ENV, "FPM_READINESS_TIMEOUT_SECONDS", "FPM_SLURM_CPUS_PER_TASK", "FPM_SLURM_CPU_BIND", diff --git a/python/aisimulate/tests/unit/collector/test_fpm_runner.py b/python/aisimulate/tests/unit/collector/test_fpm_runner.py index 38743c535..8eb14655b 100644 --- a/python/aisimulate/tests/unit/collector/test_fpm_runner.py +++ b/python/aisimulate/tests/unit/collector/test_fpm_runner.py @@ -34,6 +34,7 @@ _validate_runtime_collection, run_collection, ) +from collector.fpm_forward.slurm import SlurmCellRunner from collector.fpm_forward.types import ParallelTopology from aisimulate.fpm_contract import FPM_CELL_LABEL @@ -3332,25 +3333,94 @@ def test_runtime_default_path_is_adapter_owned(): assert env["FPM_READINESS_TIMEOUT_SECONDS"] == "900" -def test_kubernetes_stages_exact_startup_configuration(tmp_path): - runner = _runner(tmp_path) - source = tmp_path / "collector-runtime-env.sh" - source.write_text("export FPM_READINESS_TIMEOUT_SECONDS=600\n") - remote = tmp_path / "remote" - remote.mkdir() - - def kubectl(*args, **kwargs): - assert args[0] == "cp" - shutil.copy2(args[1], remote / Path(args[1]).name) - return subprocess.CompletedProcess(args, 0, stdout="", stderr="") - - def execute(pod, command, *, timeout): - if command[0] == "mkdir": - return subprocess.CompletedProcess(command, 0, stdout="", stderr="") - rewritten = [arg.replace(REMOTE_WORKDIR, str(remote)) for arg in command] - return subprocess.run(rewritten, check=True, capture_output=True, text=True, timeout=timeout) - - runner._kubectl = kubectl - runner._exec_checked = execute - runner.stage(["pod-0"], [source]) - assert (remote / source.name).read_bytes() == source.read_bytes() +@pytest.mark.parametrize("transport", ["kubernetes", "slurm"]) +@pytest.mark.parametrize("phase", ["prefill", "decode"]) +@pytest.mark.parametrize( + ("budget", "inherited"), + [ + (None, None), + (None, "600"), + ("600", "45"), + ("1", None), + ("3600", None), + ("", None), + ("0", None), + ("3601", None), + ("invalid", None), + ], +) +def test_generated_finalization_budget_reaches_staged_collector( + tmp_path, monkeypatch, transport, phase, budget, inherited +): + cell = _cell(phase=phase) + base = {"K8sConfig": {"k8s_image": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:test"}} + if budget is not None: + base["K8sConfig"]["extra_env"] = [{"name": "FPM_FINALIZATION_TIMEOUT_SECONDS", "value": budget}] + _render_cell(_plan(cell), cell, tmp_path, base) + files = [tmp_path / name for name in ("fpm_env.sh", "collector-runtime-env.sh", "run.sh")] + + if transport == "kubernetes": + runner = _runner(tmp_path) + remote = tmp_path / "remote" + remote.mkdir() + + def kubectl(*args, **kwargs): + assert args[0] == "cp" + shutil.copy2(args[1], remote / Path(args[1]).name) + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + + def execute(pod, command, *, timeout): + if command[0] == "mkdir": + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + rewritten = [arg.replace(REMOTE_WORKDIR, str(remote)) for arg in command] + return subprocess.run(rewritten, check=True, capture_output=True, text=True, timeout=timeout) + + runner._kubectl = kubectl + runner._exec_checked = execute + runner.stage(["pod-0"], files) + else: + monkeypatch.setenv("SLURM_JOB_ID", "1234") + runner = SlurmCellRunner( + tmp_path / "k8s_deploy.yaml", + tmp_path, + image=base["K8sConfig"]["k8s_image"], + mounts=(), + total_gpus=cell.topology.total_gpus, + cpus_per_task=16, + cpu_bind="cores", + ) + runner.stage(["node0000"], files) + remote = tmp_path / "slurm-runtime" + for source in files: + assert (remote / source.name).read_bytes() == source.read_bytes() + + # Execute the production parent's startup and deadline parsing unchanged; + # stop before etcd/preflight/engine startup. A child run.sh export cannot + # satisfy this check, and neither transport inherits the configured value. + runtime = Path(fpm_runner.__file__).parent / "runtime/fpm_exec.sh" + prefix, marker, _ = runtime.read_text().partition('\nif [[ -f "${workdir}/fpm_memory_worker.py" ]]; then') + assert marker + assert prefix.count("workdir=/tmp/fpm-bench") == 1 + prefix = prefix.replace("workdir=/tmp/fpm-bench", 'workdir="$1"') + script = tmp_path / "collector-startup.sh" + script.write_text(prefix + '\nprintf "%s\\n" "$finalization_timeout_seconds"\n') + environment = { + name: value for name, value in os.environ.items() if not name.startswith(("FPM_", "SLURM_", "LWS_", "GROVE_")) + } + if inherited is not None: + environment["FPM_FINALIZATION_TIMEOUT_SECONDS"] = inherited + completed = subprocess.run( + ["bash", str(script), str(remote)], + env=environment, + text=True, + capture_output=True, + timeout=10, + check=False, + ) + if budget in ("", "0", "3601", "invalid"): + assert completed.returncode == 2, completed.stderr + assert "FPM_FINALIZATION_TIMEOUT_SECONDS must be an integer from 1 through 3600" in completed.stderr + assert not completed.stdout + else: + assert completed.returncode == 0, completed.stderr + assert completed.stdout.strip() == (inherited or "120" if budget is None else budget) From 276156fbbbd8fe7ee389d4a5d495f18294ad8051 Mon Sep 17 00:00:00 2001 From: Yiming Liu Date: Thu, 15 Oct 2026 11:02:23 +0800 Subject: [PATCH 13/13] fix: address FPM review controls and interpolation evidence Signed-off-by: Yiming Liu --- crates/core/src/perfmodel/fpm/evidence.rs | 45 ++++++++++ .../src/perfmodel/operators/fpm_forward.rs | 11 ++- docs/perf-model/api/python.md | 2 +- .../fpm-self-service/implementation.md | 1 + docs/perf-model/methods/whole-forward.md | 2 +- .../schemas/prediction-details.schema.json | 3 +- python/aisimulate/src/aisimulate/detail.py | 8 +- .../support/collection_readiness.py | 4 +- .../src/aisimulate/support/runtime.py | 8 +- .../aisimulate/support/runtime_collection.py | 22 +++-- .../tests/unit/sdk/test_fpm_profile.py | 2 +- .../unit/test_onboard_runtime_bindings.py | 84 +++++++++++++++++++ tests/test_fpm_query_evidence.py | 80 ++++++++++++++++++ 13 files changed, 252 insertions(+), 20 deletions(-) diff --git a/crates/core/src/perfmodel/fpm/evidence.rs b/crates/core/src/perfmodel/fpm/evidence.rs index 6f8cce55b..df9c49fe3 100644 --- a/crates/core/src/perfmodel/fpm/evidence.rs +++ b/crates/core/src/perfmodel/fpm/evidence.rs @@ -26,6 +26,8 @@ pub enum FpmQueryResolution { WithinCurveInterpolation, CrossKvInterpolation, CrossBatchInterpolation, + /// Prefill fallback that interpolates KV at each supported token count first. + KvFirstInterpolation, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -95,3 +97,46 @@ pub(crate) fn accumulate_fpm_estimates( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::FpmQueryResolution; + + #[test] + fn query_resolution_wire_values_preserve_existing_indices() { + for (resolution, name, index) in [ + (FpmQueryResolution::ExactLookup, "exact_lookup", 0u32), + ( + FpmQueryResolution::WithinCurveInterpolation, + "within_curve_interpolation", + 1, + ), + ( + FpmQueryResolution::CrossKvInterpolation, + "cross_kv_interpolation", + 2, + ), + ( + FpmQueryResolution::CrossBatchInterpolation, + "cross_batch_interpolation", + 3, + ), + ( + FpmQueryResolution::KvFirstInterpolation, + "kv_first_interpolation", + 4, + ), + ] { + let json = serde_json::to_string(&resolution).unwrap(); + assert_eq!(json, format!("\"{name}\"")); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + resolution + ); + assert_eq!( + bincode::serialize(&resolution).unwrap(), + index.to_le_bytes() + ); + } + } +} diff --git a/crates/core/src/perfmodel/operators/fpm_forward.rs b/crates/core/src/perfmodel/operators/fpm_forward.rs index 46dc58f7e..6d1a67e73 100644 --- a/crates/core/src/perfmodel/operators/fpm_forward.rs +++ b/crates/core/src/perfmodel/operators/fpm_forward.rs @@ -715,7 +715,7 @@ impl FpmForwardOp { cell, coords, latency, - FpmQueryResolution::CrossKvInterpolation, + FpmQueryResolution::KvFirstInterpolation, support, false, ) @@ -1390,7 +1390,12 @@ mod tests { // KV-first could use the 999 ms points at KV=80/112. Keep both the // old answer and original support when token-first already succeeds. let result = direct.query_totals(&db, &[1.0, 150.0, 96.0]).unwrap(); - let support = &result.fpm_queries.as_ref().unwrap()[0].support; + let evidence = &result.fpm_queries.as_ref().unwrap()[0]; + assert_eq!( + evidence.resolution, + FpmQueryResolution::CrossKvInterpolation + ); + let support = &evidence.support; assert_eq!( support .iter() @@ -1444,7 +1449,7 @@ mod tests { let evidence = &result.fpm_queries.as_ref().unwrap()[0]; assert_eq!( evidence.resolution, - FpmQueryResolution::CrossKvInterpolation + FpmQueryResolution::KvFirstInterpolation ); assert_eq!( evidence diff --git a/docs/perf-model/api/python.md b/docs/perf-model/api/python.md index 5f5ab9b2c..fec08cc4d 100644 --- a/docs/perf-model/api/python.md +++ b/docs/perf-model/api/python.md @@ -124,7 +124,7 @@ for rank in result["ranks"]: print(query["resolution"], query["query"], query["support"]) ``` -Each direct lookup records `exact_lookup`, `within_curve_interpolation`, `cross_kv_interpolation`, or `cross_batch_interpolation`, its phase and model, requested coordinates, raw latency, and supporting measured coordinates, latencies and weights. Coordinates are iteration totals: `batch_size`, `total_prefill_tokens` (null for decode), and `total_kv_read_tokens`. Nested interpolation weights are multiplied for each supporting measurement. Prefill's `cross_kv_interpolation` includes both the usual token-first order and its KV-first fallback; support always lists original measurements rather than intermediate estimates. This evidence preserves `source="silicon"`, whose meaning includes interpolation; it does not establish accuracy or observed CUDA graph dispatch. +Each direct lookup records `exact_lookup`, `within_curve_interpolation`, `cross_kv_interpolation`, `cross_batch_interpolation`, or `kv_first_interpolation`, its phase and model, requested coordinates, raw latency, and supporting measured coordinates, latencies and weights. Coordinates are iteration totals: `batch_size`, `total_prefill_tokens` (null for decode), and `total_kv_read_tokens`. Nested interpolation weights are multiplied for each supporting measurement. Prefill's `cross_kv_interpolation` identifies the usual token-first order; `kv_first_interpolation` identifies its KV-first fallback. Support always lists original measurements rather than intermediate estimates. This evidence preserves `source="silicon"`, whose meaning includes interpolation; it does not establish accuracy or observed CUDA graph dispatch. Weights describe a raw lookup, before later composition. Each rank records prefill, decode and mixed-pass baseline latencies, plus `marginal_decode_ms = max(decode_ms - decode_baseline_ms, 0)`. A baseline lookup has `decode_baseline=true`: its query is the paired decode request, but its support contains each selected row's actual measured minimum-KV point. `max_rank` is the first zero-based input index attaining the positive native maximum. The result separately records `native_latency_ms`, `correction_factor`, and final `latency_ms`. Empty work has no query evidence. Other estimators also have no direct lookup evidence; an unready regression retains `latency_ms=null`. diff --git a/docs/perf-model/fpm-self-service/implementation.md b/docs/perf-model/fpm-self-service/implementation.md index 704b01abd..c9c960980 100644 --- a/docs/perf-model/fpm-self-service/implementation.md +++ b/docs/perf-model/fpm-self-service/implementation.md @@ -1287,6 +1287,7 @@ Explicit bindings also review these archived collection controls under `/collect | --- | --- | | Backend-policy `params.agg.extra_cli_args` option `--benchmark-timeout` | One positive integer deadline, in split or `=value` form. The exact argument tokens are recorded; `[]` means the probe declared no override. Duplicate, conflicting or malformed options fail. An explicitly empty argument list is equivalent to no policy override for comparison. | | `K8sConfig.extra_env.FPM_RESULT_WAIT_TIMEOUT_SECONDS` | One literal integer from 1 through 43,200 seconds for the outer result wait. This checks an archived control; it does not change current rendering or transport limits. | +| `K8sConfig.extra_env.FPM_FINALIZATION_TIMEOUT_SECONDS` | One literal ASCII integer from 1 through 3,600 seconds for artifact finalization after local rank outputs complete. Leading zeros, signs and whitespace are invalid, matching the execution harness. The collection deadline still applies. | | `DYN_BENCH_CONTENT_SEED`, `DYN_BENCH_POOL_TAG` | Literal strings, including an empty pool tag. | | `DYN_BENCH_PREFILL_CONTENT` | `random`, `sharegpt`, `sharegpt_chain` or `zeros`. | | `DYN_BENCH_GIANT_KV_THRESHOLD`, `DYN_BENCH_GIANT_KV_REPEATS` | Literal nonnegative integers. | diff --git a/docs/perf-model/methods/whole-forward.md b/docs/perf-model/methods/whole-forward.md index 054654371..9792a6e6d 100644 --- a/docs/perf-model/methods/whole-forward.md +++ b/docs/perf-model/methods/whole-forward.md @@ -27,7 +27,7 @@ owns artifact validation, publication, checkpoints, and recovery. Direct prefill first uses an exact measurement or interpolates new-token counts at fixed KV, then brackets KV using curves that cover the requested token count. If this cannot bracket the query, it interpolates KV at each fixed measured token count and then uses the nearest covered token counts on either side of the query. The second order can cover irregular collection boundaries where different prompt lengths have different maximum KV coordinates. It preserves every result and supporting measurement from the first order when that order succeeds. -Both orders stay within measured bounds at the same batch size and selected cell. Neither uses SOL, synthetic `fake_fallback` rows, or extrapolation. The fallback is part of `method: direct` and needs no extra setting; decode interpolation is unchanged. Detailed evidence reports `cross_kv_interpolation` for either cross-KV order, with weights flattened to original measurements. Additional query coverage does not establish interpolation accuracy or matching CUDA graph execution regimes. +Both orders stay within measured bounds at the same batch size and selected cell. Neither uses SOL, synthetic `fake_fallback` rows, or extrapolation. The fallback is part of `method: direct` and needs no extra setting; decode interpolation is unchanged. Detailed evidence reports `cross_kv_interpolation` for the usual token-first order and `kv_first_interpolation` for the fallback, with weights flattened to original measurements. Additional query coverage does not establish interpolation accuracy or matching CUDA graph execution regimes. Readiness is phase-specific: prefill needs real prefill rows, decode needs real decode rows, and aggregated needs both. A ready table still can lack an individual diff --git a/docs/reference/schemas/prediction-details.schema.json b/docs/reference/schemas/prediction-details.schema.json index 8e6cbee14..c111e00ee 100644 --- a/docs/reference/schemas/prediction-details.schema.json +++ b/docs/reference/schemas/prediction-details.schema.json @@ -1412,7 +1412,8 @@ "exact_lookup", "within_curve_interpolation", "cross_kv_interpolation", - "cross_batch_interpolation" + "cross_batch_interpolation", + "kv_first_interpolation" ] }, "latency_ms": { diff --git a/python/aisimulate/src/aisimulate/detail.py b/python/aisimulate/src/aisimulate/detail.py index f1e58c2cd..00e985e9c 100644 --- a/python/aisimulate/src/aisimulate/detail.py +++ b/python/aisimulate/src/aisimulate/detail.py @@ -150,7 +150,13 @@ def fpm_query(value: Any, path: str) -> None: fpm_coordinates(value["query"], f"{path}.query") require( value["resolution"] - in ("exact_lookup", "within_curve_interpolation", "cross_kv_interpolation", "cross_batch_interpolation"), + in ( + "exact_lookup", + "within_curve_interpolation", + "cross_kv_interpolation", + "cross_batch_interpolation", + "kv_first_interpolation", + ), f"{path}.resolution", "a supported direct resolution", ) diff --git a/python/aisimulate/src/aisimulate/support/collection_readiness.py b/python/aisimulate/src/aisimulate/support/collection_readiness.py index 98f9a54e7..ccae04809 100644 --- a/python/aisimulate/src/aisimulate/support/collection_readiness.py +++ b/python/aisimulate/src/aisimulate/support/collection_readiness.py @@ -237,7 +237,7 @@ def assess_readiness( from collector.fpm_forward.runner import CHECKPOINT_SCHEMA from .runtime import runtime_probe_manifest, verify_collection_runtime_bindings - from .runtime_collection import _checkpoint_observation_index + from .runtime_collection import checkpoint_observation_index if runtime_bindings is not None: if expected_plan is not None: @@ -303,7 +303,7 @@ def assess_readiness( cell.cell_id not in selected or cell.cell_id in checkpoint["cells"] for cell in plan.cells ): try: - index = _checkpoint_observation_index(checkpoint_path, checkpoint) + index = checkpoint_observation_index(checkpoint_path, checkpoint) evidence = file_evidence(index) runtime_binding = { "plan": {"path": str(campaign / "collection-plan.json"), "sha256": plan.sha256}, diff --git a/python/aisimulate/src/aisimulate/support/runtime.py b/python/aisimulate/src/aisimulate/support/runtime.py index 5deba89e3..2724098a9 100644 --- a/python/aisimulate/src/aisimulate/support/runtime.py +++ b/python/aisimulate/src/aisimulate/support/runtime.py @@ -18,11 +18,9 @@ from aisimulate.fpm_profile import FpmModelProfile from .config_profile import load_model_config, profile_from_observations, validate_overrides -from .runtime_collection import ( # noqa: F401 - collection_runtime_review, - verify_collection_runtime_bindings, - verify_collection_runtime_plans, -) +from .runtime_collection import collection_runtime_review as collection_runtime_review +from .runtime_collection import verify_collection_runtime_bindings as verify_collection_runtime_bindings +from .runtime_collection import verify_collection_runtime_plans as verify_collection_runtime_plans from .schema import ( AGENTX_REFERENCE_CONTEXT, CollectionSpec, diff --git a/python/aisimulate/src/aisimulate/support/runtime_collection.py b/python/aisimulate/src/aisimulate/support/runtime_collection.py index e3b2e0f02..da4629124 100644 --- a/python/aisimulate/src/aisimulate/support/runtime_collection.py +++ b/python/aisimulate/src/aisimulate/support/runtime_collection.py @@ -13,6 +13,8 @@ from pathlib import Path from typing import Any +from aisimulate.fpm_contract import FPM_FINALIZATION_TIMEOUT_ENV + from .schema import SupportRequest @@ -38,7 +40,12 @@ def _read(reference: dict[str, str], *, plan: bool = False) -> tuple[Path, dict[ return path, document -def _checkpoint_observation_index(checkpoint_path: Path, checkpoint: dict[str, Any]) -> Path: +def checkpoint_observation_index(checkpoint_path: Path, checkpoint: dict[str, Any]) -> Path: + """Resolve a checkpoint's recorded observation index to a regular file. + + Relative paths are anchored to the checkpoint directory. Reject a symlink + at the index itself; callers separately verify containment and byte hashes. + """ recorded = checkpoint.get("runtime_observations") if not isinstance(recorded, str) or not recorded.strip(): raise ValueError("collector checkpoint records no runtime observation index") @@ -124,6 +131,7 @@ def _collection_controls(cells, deployment): "DYN_BENCH_GIANT_KV_REPEATS", "DYN_FORWARDPASS_METRIC_PORT", "FPM_RESULT_WAIT_TIMEOUT_SECONDS", + FPM_FINALIZATION_TIMEOUT_ENV, } retained, seen = [], set() if not isinstance(environment, list): @@ -150,9 +158,13 @@ def _collection_controls(cells, deployment): r"0|[1-9][0-9]*", value ): raise ValueError(f"collection control {name} must be a nonnegative integer") - if name in {"DYN_FORWARDPASS_METRIC_PORT", "FPM_RESULT_WAIT_TIMEOUT_SECONDS"}: - maximum = 65535 if name == "DYN_FORWARDPASS_METRIC_PORT" else 43200 - if not re.fullmatch(r"[1-9][0-9]*", value) or int(value) > maximum: + if name in {"DYN_FORWARDPASS_METRIC_PORT", "FPM_RESULT_WAIT_TIMEOUT_SECONDS", FPM_FINALIZATION_TIMEOUT_ENV}: + maximum = { + "DYN_FORWARDPASS_METRIC_PORT": 65535, + "FPM_RESULT_WAIT_TIMEOUT_SECONDS": 43200, + FPM_FINALIZATION_TIMEOUT_ENV: 3600, + }[name] + if not re.fullmatch(r"[1-9][0-9]*", value) or len(value) > len(str(maximum)) or int(value) > maximum: raise ValueError(f"collection control {name} must be an integer from 1 through {maximum}") changes.extend(_differences({"missing": True}, value, "/collection_controls/environment/" + name)) if "extra_env" in k8s: @@ -334,7 +346,7 @@ def verify_collection_runtime_bindings(request: SupportRequest, bindings: list[d if ( checkpoint.get("schema") != CHECKPOINT_SCHEMA or checkpoint.get("plan_sha256") != plan.sha256 - or _checkpoint_observation_index(checkpoint_path, checkpoint) != index.resolve() + or checkpoint_observation_index(checkpoint_path, checkpoint) != index.resolve() or checkpoint.get("runtime_observation_attempt_id") != binding["attempt_id"] or set(checkpoint.get("cells", {})) - {value.cell_id for value in plan.cells} ): diff --git a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py index 0213e19c1..ee7eed011 100644 --- a/python/aisimulate/tests/unit/sdk/test_fpm_profile.py +++ b/python/aisimulate/tests/unit/sdk/test_fpm_profile.py @@ -1366,7 +1366,7 @@ def test_direct_prefill_kv_first_preserves_public_evidence_and_saved_config(prof detailed = model.estimate_forward_pass_detailed(metrics) assert detailed["latency_ms"] == pytest.approx(28.5) query = detailed["ranks"][0]["queries"][0] - assert query["resolution"] == "cross_kv_interpolation" + assert query["resolution"] == "kv_first_interpolation" assert query["query"] == { "batch_size": 1.0, "total_prefill_tokens": 2688.0, diff --git a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py index 3fa8f95ed..4d7528418 100644 --- a/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py +++ b/python/aisimulate/tests/unit/test_onboard_runtime_bindings.py @@ -161,6 +161,7 @@ def archived_controls(split_collection, request): {"name": "DYN_BENCH_GIANT_KV_REPEATS", "value": "3"}, {"name": "DYN_FORWARDPASS_METRIC_PORT", "value": "7020"}, {"name": "FPM_RESULT_WAIT_TIMEOUT_SECONDS", "value": "14100"}, + {"name": "FPM_FINALIZATION_TIMEOUT_SECONDS", "value": "600"}, ], }, ) @@ -225,6 +226,7 @@ def test_archived_collection_controls_require_exact_review_and_preserve_plan(arc "DYN_BENCH_GIANT_KV_REPEATS": "3", "DYN_FORWARDPASS_METRIC_PORT": "7020", "FPM_RESULT_WAIT_TIMEOUT_SECONDS": "14100", + "FPM_FINALIZATION_TIMEOUT_SECONDS": "600", }.items() ], ] @@ -254,6 +256,45 @@ def test_archived_collection_controls_require_exact_review_and_preserve_plan(arc assert runtime.verify_runtime_profile(request) == original_source +@pytest.mark.parametrize( + "archived_controls", + [ + { + "extra_cli_args": [], + "extra_env": [{"name": "FPM_FINALIZATION_TIMEOUT_SECONDS", "value": value}], + } + for value in ("1", "600", "3600") + ], + indirect=True, +) +def test_archived_finalization_budget_requires_review_before_memory_binding(archived_controls): + request, bindings = archived_controls + originals = { + path: path.read_bytes() + for binding in bindings + for path in Path(binding["plan"]["path"]).parent.rglob("*") + if path.is_file() + } + deployment = json.loads((Path(bindings[0]["plan"]["path"]).parent / "generator-overrides.json").read_text()) + budget = next( + item["value"] + for item in deployment["K8sConfig"]["extra_env"] + if item["name"] == "FPM_FINALIZATION_TIMEOUT_SECONDS" + ) + review = runtime.collection_runtime_review(request, bindings[0]) + assert { + "field": "/collection_controls/environment/FPM_FINALIZATION_TIMEOUT_SECONDS", + "accepted": {"missing": True}, + "collection": budget, + } in review["changes"] + with pytest.raises(ValueError, match="exact runtime change review required"): + runtime.verify_collection_runtime_bindings(request, bindings) + _review(request, bindings) + assert runtime.verify_collection_runtime_plans(request, bindings)["status"] == "verified" + assert runtime.verify_collection_runtime_bindings(request, bindings)["compatibility"]["status"] == "compatible" + assert all(path.read_bytes() == contents for path, contents in originals.items()) + + @pytest.mark.parametrize( "archived_controls", [{"extra_cli_args": []}, {"extra_cli_args": ["--benchmark-timeout=12600"]}], @@ -285,6 +326,49 @@ def test_archived_empty_policy_and_equals_deadline_are_explicitly_verified(archi ["--benchmark-timeout", "12600", "--kv-cache-dtype", "fp8"], ) ] + + [ + { + "extra_cli_args": [], + "extra_env": [{"name": "FPM_FINALIZATION_TIMEOUT_SECONDS", "value": value}], + } + for value in ( + None, + 600, + True, + "", + "0", + "3601", + "1.5", + "-1", + "+1", + "01", + " 600", + "600 ", + "600\n", + "600\x00", + "٦٠٠", + "\uff16\uff10\uff10", + "${TIMEOUT}", + "9" * 5000, + ) + ] + + [ + { + "extra_cli_args": [], + "extra_env": [ + {"name": "FPM_FINALIZATION_TIMEOUT_SECONDS", "value": "600"}, + {"name": "FPM_FINALIZATION_TIMEOUT_SECONDS", "value": value}, + ], + } + for value in ("600", "1200") + ] + + [ + { + "extra_cli_args": [], + "extra_env": [{"name": "FPM_FINALIZATION_TIMEOUT_SECONDS", **value}], + } + for value in ({"valueFrom": {"fieldRef": {"fieldPath": "metadata.name"}}}, {"value": "600", "valueFrom": {}}) + ] + [ { "extra_cli_args": ["--benchmark-timeout", "12600"], diff --git a/tests/test_fpm_query_evidence.py b/tests/test_fpm_query_evidence.py index 4a92b2479..783598044 100644 --- a/tests/test_fpm_query_evidence.py +++ b/tests/test_fpm_query_evidence.py @@ -74,6 +74,29 @@ def evidence_systems(timing_systems): return timing_systems +@pytest.fixture +def kv_first_systems(evidence_systems): + path = Path(evidence_systems) / "data/h200_sxm/vllm/0.25.1/fpm_forward_perf.parquet" + original = pq.read_table(path).to_pylist() + templates = [row for row in original if row["workload_kind"] == "prefill" and row["total_prefill_tokens"] == 1] + rows = [row for row in original if row["workload_kind"] == "decode"] + # Replay caps inferred capacity at decode-table coverage, including for a + # prefill-only request. Supply enough batch-1 coverage for both chunks. + rows.extend( + {**row, "total_kv_read_tokens": 8192} + for row in original + if row["workload_kind"] == "decode" and row["batch_size"] == 1 and row["total_kv_read_tokens"] == 64 + ) + for template in templates: + for tokens, kv, latency in ((1024, 0, 10.0), (1024, 3584, 30.0), (2048, 0, 20.0), (2048, 7168, 60.0)): + rows.append({**template, "total_prefill_tokens": tokens, "total_kv_read_tokens": kv, "latency_ms": latency}) + pq.write_table(pa.Table.from_pylist(rows), path) + metadata = json.loads(path.with_suffix(".metadata.json").read_text()) + metadata.update(row_count=len(rows), parquet_sha256=hashlib.sha256(path.read_bytes()).hexdigest()) + path.with_suffix(".metadata.json").write_text(json.dumps(metadata)) + return evidence_systems + + def canonical(profile, systems): return { "model": profile["model"], @@ -267,6 +290,63 @@ def records(evidence): ] +def test_kv_first_marker_survives_saved_config_and_chunked_replay(profile, kv_first_systems, tmp_path): + import jsonschema + + raw = prediction(profile, kv_first_systems) + raw["engine"]["enable_chunked_prefill"] = True + raw["traffic"]["source"].update(input_tokens=3584, output_tokens=1) + raw["traffic"]["stop"]["requests"] = 1 + worker = raw["engine"]["workers"]["aggregated"] + worker["scheduler"]["max_batched_tokens"] = 1792 + worker["timing"]["estimator_config"]["fpm_interpolation"]["collect_coverage"] = True + saved = tmp_path / "prediction.json" + saved.write_text(CorePredictionConfig.model_validate(raw).model_dump_json()) + reports = [] + for _ in range(2): + spec = prediction_to_replay_spec(CorePredictionConfig.model_validate_json(saved.read_text())) + runner = EngineReplayRunnerFactory().create(0) + try: + reports.append( + runner.run(spec, output_requirements=ReplayOutputRequirements(capture_performance_diagnostics=True)) + ) + finally: + runner.close() + assert reports[0].metadata["fpm_query_evidence"] == reports[1].metadata["fpm_query_evidence"] + queries = [ + query + for record in records(reports[0].metadata["fpm_query_evidence"]) + for rank in record["estimate"]["ranks"] + for query in rank["queries"] + ] + assert reports[0].metrics["completed_requests"] == 1 + # Two 1792-token chunks: the first interpolates at KV=0 to 17.5 ms. + # At KV=1792, the token=1024/2048 curves yield 20/30 ms. Token=1792 + # lies 3/4 between them, giving 27.5 ms and leaf weights 1/8,1/8,9/16,3/16. + assert {(query["resolution"], query["latency_ms"]) for query in queries} == { + ("within_curve_interpolation", 17.5), + ("kv_first_interpolation", 27.5), + } + fallback = next(query for query in queries if query["resolution"] == "kv_first_interpolation") + assert [point["weight"] for point in fallback["support"]] == [0.125, 0.125, 0.5625, 0.1875] + native = reports[0].metadata["native_report"] + assert native["fpm_query_coverage"]["queries"] == {"measured": 0, "interpolated": 2, "unsupported": 0} + details = build_prediction_details(native, ("source", "time")) + schema = json.loads( + (Path(__file__).resolve().parents[1] / "docs/reference/schemas/prediction-details.schema.json").read_text() + ) + jsonschema.validate(details, schema) + for section in (details["sections"]["source"], details["sections"]["time"]["diagnostics"]): + assert fallback in [ + query + for phase in section["phases"] + for operation in phase["operations"] + for record in operation.get("fpm_estimates", []) + for rank in record["estimate"]["ranks"] + for query in rank["queries"] + ] + + @pytest.mark.parametrize("collect_coverage", [False, True]) def test_native_prediction_replay_and_source_detail_roundtrip(profile, evidence_systems, collect_coverage): import jsonschema