Skip to content
1 change: 0 additions & 1 deletion .github/prediction-numerical-sentinels.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
{
"schema_version": 1,
"baseline_source_sha": "fc6e26a82dea7fe49317d48ae97523323ee940b7",
"interpretation": "Native prediction stability sentinels; not measured-hardware accuracy. Two percent relative and 0.0001 ms absolute tolerance permit small numerical changes; baseline changes require explained review.",
"cases": [
{
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -692,7 +692,6 @@ jobs:
run: |
python -m uv pip install --system "${AISIMULATE_TEST_WHEEL}[dev]" --group scripts/pyproject.toml:ci --group scripts/pyproject.toml:fpm-accuracy --group scripts/pyproject.toml:e2e-accuracy
python -m pip check
python scripts/prediction_regression/check_prediction_numerics.py --fetch-baseline-only
python scripts/ci/check_application_test_inventory.py --output application-test-inventory.json
python -m pytest -q \
python/aisimulate/tests/unit/cli/test_aisimulate_compatibility.py \
Expand Down Expand Up @@ -862,7 +861,6 @@ jobs:
crates/core/parity_tests/perfmodel/test_afd_execution_plan_parity.py
- name: Check native prediction numerical sentinels
run: |
python scripts/prediction_regression/check_prediction_numerics.py --fetch-baseline-only
python scripts/prediction_regression/check_prediction_numerics.py --output numerical-sentinels.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
if: always()
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/fast-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,6 @@ jobs:
fi
- name: Check active workflow contracts
run: |
python scripts/prediction_regression/check_prediction_numerics.py --fetch-baseline-only
python -m pytest -c /dev/null tests/test_ci_workflow_contracts.py tests/test_ci_qualification.py tests/test_release_fpe.py -q \
-p no:cacheprovider \
--override-ini="addopts=" \
Expand Down
15 changes: 5 additions & 10 deletions docs/ci/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,16 +252,11 @@ marked it N/A.
MoE MiniMax-M2.5, plus four Qwen3-32B queries each for TRT-LLM 1.3.0rc20 and
SGLang 0.5.14. Every backend covers prefill/decode and short/long sequences.
The [manifest](../../.github/prediction-numerical-sentinels.json)
records a full baseline commit that must resolve in the checkout. Fast CI and
the numerical-check job explicitly fetch that SHA from `origin` if missing;
full branch history alone can omit a baseline from a squashed PR. Fetch or
commit-validation failures remain errors. Fetch-only mode does not load the
AISimulate runtime; the subsequent checks validate the complete manifest.
To prepare a checkout locally, run:

```sh
python scripts/prediction_regression/check_prediction_numerics.py --fetch-baseline-only
```
records each query's inputs and expected value. The expected values come from
the code of the commit that last changed the manifest; find it with
`git log -1 -- .github/prediction-numerical-sentinels.json`. That commit is on
the main branch of every clone and mirror, so the manifest records no separate
commit: a commit from a squashed PR branch would not be.

Tolerances are 2% relative and 0.0001 ms absolute. Missing, duplicate, failed, nonfinite,
nonpositive, or out-of-tolerance results fail. Intentional modeling changes
Expand Down
44 changes: 5 additions & 39 deletions scripts/prediction_regression/check_prediction_numerics.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@

This is numerical stability coverage, not predictive accuracy against hardware.
The broad old/new modeling report remains advisory for numerical drift.

The expected values come from the code of the commit that last changed the
manifest (`git log -1 -- .github/prediction-numerical-sentinels.json`). That
commit is on the main branch of every clone, so no extra commit is recorded.
"""

from __future__ import annotations
Expand All @@ -15,40 +19,10 @@
import json
import math
import os
import re
import subprocess
from pathlib import Path


def resolve_baseline(manifest: dict, *, fetch: bool = False, repository_root: Path | None = None) -> str:
baseline = manifest.get("baseline_source_sha")
if not isinstance(baseline, str) or not re.fullmatch(r"[0-9a-f]{40}", baseline):
raise ValueError("baseline_source_sha must be a full commit SHA")
root = repository_root or Path(__file__).resolve().parents[2]
command = ["git", "cat-file", "-e", f"{baseline}^{{commit}}"]
resolved = subprocess.run(
command,
cwd=root,
capture_output=True,
timeout=10,
)
if resolved.returncode and fetch:
# Squashed PR commits may be absent even after fetching all branch history.
# Fetch only the recorded source; never replace it or refresh expected values.
subprocess.run(
["git", "fetch", "--no-tags", "--depth=1", "--no-write-fetch-head", "origin", baseline],
cwd=root,
check=True,
timeout=60,
)
resolved = subprocess.run(command, cwd=root, capture_output=True, timeout=10)
if resolved.returncode:
raise ValueError(f"baseline_source_sha does not resolve to a commit: {baseline}")
return baseline


def validate_cases(manifest: dict) -> list[dict]:
resolve_baseline(manifest)
cases = manifest["cases"]
if manifest["schema_version"] != 1 or not cases:
raise ValueError("sentinel manifest must have schema 1 and nonempty cases")
Expand Down Expand Up @@ -121,22 +95,14 @@ def main() -> int:
type=Path,
default=Path(__file__).resolve().parents[2] / ".github/prediction-numerical-sentinels.json",
)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--output", type=Path)
mode.add_argument(
"--fetch-baseline-only", action="store_true", help="Fetch the recorded commit without running predictions"
)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
manifest = json.loads(args.manifest.read_text())
if args.fetch_baseline_only:
print(f"Numerical baseline available: {resolve_baseline(manifest, fetch=True)}")
return 0
cases = validate_cases(manifest)
rows = collect(cases)
failures = check_results(cases, rows)
result = {
"source_sha": os.environ.get("GITHUB_SHA"),
"baseline_source_sha": manifest["baseline_source_sha"],
"manifest_sha256": hashlib.sha256(args.manifest.read_bytes()).hexdigest(),
"qualification": "native_prediction_stability",
"results": rows,
Expand Down
94 changes: 10 additions & 84 deletions tests/test_ci_qualification.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,9 @@

import pytest

from scripts.prediction_regression.check_prediction_numerics import check_results, resolve_baseline, validate_cases
from scripts.prediction_regression.check_prediction_numerics import check_results, validate_cases

ROOT = Path(__file__).resolve().parents[1]
BASELINE_SHA = json.loads((ROOT / ".github/prediction-numerical-sentinels.json").read_text())["baseline_source_sha"]


@pytest.fixture
Expand Down Expand Up @@ -52,100 +51,27 @@ def test_missing_duplicate_and_skipped_sentinels_fail(case):
def test_invalid_tolerances_or_query_rejected(case, field, value):
case[field] = value
with pytest.raises(ValueError):
validate_cases({"schema_version": 1, "baseline_source_sha": BASELINE_SHA, "cases": [case]})
validate_cases({"schema_version": 1, "cases": [case]})


@pytest.mark.parametrize("baseline", [None, "", "main", "a" * 39, "z" * 40, "0" * 40])
def test_invalid_or_unresolved_baseline_commit_fails(case, baseline):
with pytest.raises(ValueError, match="baseline_source_sha"):
validate_cases({"schema_version": 1, "baseline_source_sha": baseline, "cases": [case]})
def test_valid_manifest_is_accepted(case):
assert validate_cases({"schema_version": 1, "cases": [case]}) == [case]


def test_valid_baseline_commit_is_accepted(case):
assert validate_cases({"schema_version": 1, "baseline_source_sha": BASELINE_SHA, "cases": [case]}) == [case]


def test_fetch_historical_baseline_preserves_checkout_and_works_offline_afterward(tmp_path):
upstream = tmp_path / "upstream"
checkout = tmp_path / "checkout"
upstream.mkdir()

def git(root, *args):
return subprocess.run(["git", *args], cwd=root, check=True, capture_output=True, text=True).stdout.strip()

git(upstream, "init", "-q", "-b", "main")
git(upstream, "-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "--allow-empty", "-qm", "main")
git(upstream, "checkout", "-qb", "historical-baseline")
git(
upstream,
"-c",
"user.name=Test",
"-c",
"user.email=test@example.com",
"commit",
"--allow-empty",
"-qm",
"baseline",
)
baseline = git(upstream, "rev-parse", "HEAD")
git(upstream, "checkout", "main")
git(tmp_path, "clone", "--depth=1", "--single-branch", upstream.as_uri(), str(checkout))
head = git(checkout, "rev-parse", "HEAD")
manifest = {"baseline_source_sha": baseline}
with pytest.raises(ValueError, match="does not resolve"):
resolve_baseline(manifest, repository_root=checkout)
assert resolve_baseline(manifest, fetch=True, repository_root=checkout) == baseline
assert git(checkout, "rev-parse", "HEAD") == head
assert git(checkout, "branch", "--show-current") == "main"
git(checkout, "remote", "remove", "origin")
assert resolve_baseline(manifest, fetch=True, repository_root=checkout) == baseline
assert manifest == {"baseline_source_sha": baseline}


@pytest.mark.parametrize("baseline", [None, "main", "--upload-pack=invalid", "a" * 39, "z" * 40])
def test_fetch_baseline_rejects_non_sha_before_git(tmp_path, baseline):
with pytest.raises(ValueError, match="full commit SHA"):
resolve_baseline({"baseline_source_sha": baseline}, fetch=True, repository_root=tmp_path)


def test_fetch_baseline_fails_when_origin_cannot_supply_commit(tmp_path):
subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)
subprocess.run(["git", "remote", "add", "origin", str(tmp_path / "missing")], cwd=tmp_path, check=True)
with pytest.raises(subprocess.CalledProcessError):
resolve_baseline({"baseline_source_sha": "0" * 40}, fetch=True, repository_root=tmp_path)


def test_fetch_baseline_cli_does_not_require_prediction_cases_or_write_results(tmp_path):
baseline = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()
manifest = tmp_path / "manifest.json"
manifest.write_text(json.dumps({"baseline_source_sha": baseline}))
result = subprocess.run(
[
sys.executable,
str(ROOT / "scripts/prediction_regression/check_prediction_numerics.py"),
"--manifest",
str(manifest),
"--fetch-baseline-only",
],
cwd=tmp_path,
capture_output=True,
text=True,
check=True,
)
assert result.stdout.strip() == f"Numerical baseline available: {baseline}"
assert set(tmp_path.iterdir()) == {manifest}
def test_manifest_records_no_baseline_commit():
# The commit that last changed the manifest is the source of its values.
manifest = json.loads((ROOT / ".github/prediction-numerical-sentinels.json").read_text())
assert "baseline_source_sha" not in manifest


@pytest.mark.parametrize("arguments", [[], ["--fetch-baseline-only", "--output", "results.json"]])
def test_numerical_cli_requires_exactly_one_mode(tmp_path, arguments):
def test_numerical_cli_requires_output(tmp_path):
result = subprocess.run(
[sys.executable, str(ROOT / "scripts/prediction_regression/check_prediction_numerics.py"), *arguments],
[sys.executable, str(ROOT / "scripts/prediction_regression/check_prediction_numerics.py")],
cwd=tmp_path,
capture_output=True,
text=True,
)
assert result.returncode == 2
assert "--fetch-baseline-only" in result.stderr
assert "--output" in result.stderr
assert not list(tmp_path.iterdir())

Expand Down
10 changes: 4 additions & 6 deletions tests/test_ci_workflow_contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -758,12 +758,10 @@ def test_full_ci_owns_migrated_expensive_suites() -> None:
assert "test_engine_step_parity.py" in regression_commands
assert "test_compile_engine_parity.py" in regression_commands
assert regression_commands.count("-c python/aisimulate/pytest.ini") == 2
assert regression_commands.index("check_prediction_numerics.py --fetch-baseline-only") < regression_commands.index(
"check_prediction_numerics.py --output"
)
policy_commands = _run_commands(_workflow("fast-ci.yml")["jobs"]["policy"])
fetch = "check_prediction_numerics.py --fetch-baseline-only"
assert policy_commands.index(fetch) < policy_commands.index("tests/test_ci_qualification.py")
assert "check_prediction_numerics.py --output" in regression_commands
# The sentinel values need no extra commit: no job fetches one.
for workflow in ("ci.yml", "fast-ci.yml"):
assert "--fetch-baseline-only" not in (WORKFLOW_ROOT / workflow).read_text()

feature_mode_commands = _run_commands(jobs["rust-feature-modes"])
assert "cargo test --workspace --features embed-python,replay-bench" in feature_mode_commands
Expand Down
Loading