From dd536f5bb1dff99bd3f00c7203abab2e66e07c43 Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Wed, 30 Sep 2026 11:44:39 -0700 Subject: [PATCH 1/3] fix: support release namespaces in E2E accuracy campaigns Signed-off-by: Simone Chen --- .github/workflows/e2e-accuracy-branch.yml | 8 ++++ docs/ci.md | 3 ++ scripts/run_e2e_accuracy.py | 31 ++++++++++----- tests/test_e2e_accuracy_nightly.py | 47 +++++++++++++++++++++++ 4 files changed, 80 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-accuracy-branch.yml b/.github/workflows/e2e-accuracy-branch.yml index 042ea2fe6..b56224db6 100644 --- a/.github/workflows/e2e-accuracy-branch.yml +++ b/.github/workflows/e2e-accuracy-branch.yml @@ -157,6 +157,14 @@ jobs: echo "${WHEEL_SHA256} $(echo "${RUNNER_TEMP}"/accuracy-wheel/*.whl)" | sha256sum --check --strict python -m pip install "${RUNNER_TEMP}"/accuracy-wheel/*.whl python -m pip check + python - <<'PYCODE' + import os, sys + from pathlib import Path + sys.path.insert(0, 'scripts') + from run_e2e_accuracy import wheel_identity + wheel = next((Path(os.environ['RUNNER_TEMP']) / 'accuracy-wheel').glob('*.whl')) + print(wheel_identity(wheel)) + PYCODE - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: e2e-accuracy-dataset diff --git a/docs/ci.md b/docs/ci.md index 28d4df44a..93b682ac2 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -523,6 +523,9 @@ Campaign jobs use the configured `CI_JOB_CONTAINER_IMAGE`, Python 3.12, and `sudo` to install the PostgreSQL 18 client from the official PGDG repository, plus `zstd` and `libgomp1`. The runner executes container steps as a non-root user, so a bare PostgreSQL image cannot install these dependencies with `apt-get`. +The evaluated wheel's baseline CLI and config adapter are selected from its own +package layout: `aiconfigurator.*` on 0.12 releases or `aisimulate.*` after the +namespace migration. Wheel byte checks and imports run before downloading measurements. The client reads the dump without starting a database server. Artifact upload uses the output directory so the runner container hook remaps the full path. diff --git a/scripts/run_e2e_accuracy.py b/scripts/run_e2e_accuracy.py index 92dab51df..0a9046057 100644 --- a/scripts/run_e2e_accuracy.py +++ b/scripts/run_e2e_accuracy.py @@ -154,11 +154,26 @@ def eligibility_exclusion(bench): } +def predictor_module_names(files) -> tuple[str, str]: + """Select the namespace shipped by the evaluated wheel, not the evaluator.""" + members = {str(path) for path in files} + for api, adapter in ( + ("aisimulate.legacy_cli.api", "aisimulate.sdk.config_adapter"), + ("aiconfigurator.cli.api", "aiconfigurator.sdk.config_adapter"), + ): + if api.replace(".", "/") + ".py" in members: + if adapter.replace(".", "/") + "/__init__.py" not in members: + raise ValueError("wheel is missing its matching config adapter") + return api, adapter + raise ValueError("wheel has no supported baseline CLI layout") + + def wheel_identity(wheel: Path) -> dict: """Check installed runtime AND legacy CLI bytes against the single wheel.""" dist = importlib.metadata.distribution("aisimulate") checked = 0 with zipfile.ZipFile(wheel) as archive: + api, adapter = predictor_module_names(archive.namelist()) for member in archive.namelist(): if member.endswith("/") or not member.startswith(("aisimulate/", "aiconfigurator/", "aisimulate_core/")): continue @@ -168,7 +183,7 @@ def wheel_identity(wheel: Path) -> dict: checked += 1 if checked < 3: raise ValueError("wheel does not contain the unified package") - for name in ("aisimulate._runtime", "aisimulate.runner", "aisimulate.legacy_cli.api"): + for name in ("aisimulate._runtime", "aisimulate.runner", api, adapter): module = importlib.import_module(name) if not Path(module.__file__).resolve().is_relative_to(Path(dist.locate_file("")).resolve()): raise ValueError("import resolved outside installed wheel") @@ -240,13 +255,11 @@ def engine(worker, system): def predict_point(point: dict) -> dict: - from aisimulate.legacy_cli.api import cli_estimate from aisimulate.runner import EngineReplayRunnerFactory - from aisimulate.sdk.config_adapter import ( - InferenceXSource, - adapt_config, - to_cli_estimate_kwargs, - ) + + api_name, adapter_name = predictor_module_names(importlib.metadata.distribution("aisimulate").files or ()) + cli_estimate = importlib.import_module(api_name).cli_estimate + adapter = importlib.import_module(adapter_name) config, bench = point["config"], point["benchmark"] # A fingerprint alone cannot reconstruct non-normalized recipe knobs. @@ -258,7 +271,7 @@ def predict_point(point: dict) -> dict: "outcome": "unsupported", "reason": "recipe_required", } - adaptation = adapt_config(InferenceXSource(config=config, benchmark=bench)) + adaptation = adapter.adapt_config(adapter.InferenceXSource(config=config, benchmark=bench)) if len(adaptation.requests) != 1: return { "id": point["id"], @@ -267,7 +280,7 @@ def predict_point(point: dict) -> dict: } request = adaptation.requests[0] try: - baseline = cli_estimate(**to_cli_estimate_kwargs(request)) + baseline = cli_estimate(**adapter.to_cli_estimate_kwargs(request)) if not all(positive(value) for value in (baseline.ttft, baseline.tpot)): raise ValueError("invalid baseline latency") except Exception: diff --git a/tests/test_e2e_accuracy_nightly.py b/tests/test_e2e_accuracy_nightly.py index 2737efb99..2c13ea6fe 100644 --- a/tests/test_e2e_accuracy_nightly.py +++ b/tests/test_e2e_accuracy_nightly.py @@ -1091,3 +1091,50 @@ def download(url, timeout): fetch.download_part("https://example.invalid/part", {"name": "part00", "size": 7, "sha256": "a" * 64}, target) assert len(calls) == 3 assert target.getvalue() == b"previous part" + + +@pytest.mark.parametrize( + "api,adapter", + [ + ("aisimulate.legacy_cli.api", "aisimulate.sdk.config_adapter"), + ("aiconfigurator.cli.api", "aiconfigurator.sdk.config_adapter"), + ], +) +def test_wheel_identity_accepts_both_packaged_predictor_layouts(tmp_path, monkeypatch, api, adapter): + members = { + "aisimulate/_runtime.py": b"runtime", + "aisimulate/runner.py": b"runner", + api.replace(".", "/") + ".py": b"baseline", + adapter.replace(".", "/") + "/__init__.py": b"adapter", + } + wheel = tmp_path / "test.whl" + with zipfile.ZipFile(wheel, "w") as archive: + for name, content in members.items(): + archive.writestr(name, content) + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + dist = SimpleNamespace(locate_file=lambda name: tmp_path / name, version="test", files=list(members)) + monkeypatch.setattr(campaign.importlib.metadata, "distribution", lambda _: dist) + imported = [] + + def load(name): + imported.append(name) + path = name.replace(".", "/") + ("/__init__.py" if name == adapter else ".py") + return SimpleNamespace(__file__=str(tmp_path / path)) + + monkeypatch.setattr(campaign.importlib, "import_module", load) + assert campaign.wheel_identity(wheel)["packages"] == {"aisimulate": "test"} + assert imported == ["aisimulate._runtime", "aisimulate.runner", api, adapter] + (tmp_path / (api.replace(".", "/") + ".py")).write_bytes(b"tampered") + with pytest.raises(ValueError, match="differs from qualified wheel"): + campaign.wheel_identity(wheel) + + +def test_predictor_layout_rejects_missing_or_mixed_namespaces(): + with pytest.raises(ValueError, match="no supported baseline"): + campaign.predictor_module_names([]) + with pytest.raises(ValueError, match="matching config adapter"): + campaign.predictor_module_names( + ["aisimulate/legacy_cli/api.py", "aiconfigurator/sdk/config_adapter/__init__.py"] + ) From 045050497ae4898d8e8e5f6e49beb015b58cdc39 Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Wed, 30 Sep 2026 15:00:48 -0700 Subject: [PATCH 2/3] fix: reject ambiguous predictors and preserve baseline provenance Signed-off-by: Simone Chen --- docs/ci.md | 4 +- pages/e2e-accuracy/README.md | 5 +- scripts/build_e2e_accuracy_overview.py | 14 +++++- scripts/prepare_e2e_accuracy_pages.py | 7 ++- scripts/run_e2e_accuracy.py | 36 ++++++++++---- tests/test_e2e_accuracy_nightly.py | 69 ++++++++++++++++++++++++-- tests/test_e2e_accuracy_overview.py | 19 +++++++ 7 files changed, 135 insertions(+), 19 deletions(-) diff --git a/docs/ci.md b/docs/ci.md index 93b682ac2..ad80dc7ef 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -525,7 +525,9 @@ plus `zstd` and `libgomp1`. The runner executes container steps as a non-root user, so a bare PostgreSQL image cannot install these dependencies with `apt-get`. The evaluated wheel's baseline CLI and config adapter are selected from its own package layout: `aiconfigurator.*` on 0.12 releases or `aisimulate.*` after the -namespace migration. Wheel byte checks and imports run before downloading measurements. +namespace migration. Ambiguous wheels with both baseline APIs are rejected. +Baseline provenance records the selected API, adapter, and actual console entry point; +the public summary retains that entry point. Wheel byte checks and imports run before downloading measurements. The client reads the dump without starting a database server. Artifact upload uses the output directory so the runner container hook remaps the full path. diff --git a/pages/e2e-accuracy/README.md b/pages/e2e-accuracy/README.md index 2bf790fa5..a2a8cbf2a 100644 --- a/pages/e2e-accuracy/README.md +++ b/pages/e2e-accuracy/README.md @@ -242,7 +242,10 @@ metadata, and a coverage report. For a branch-qualified snapshot, both All three producer documents must also carry the same completed `aic_run`. Its `runtime.source_checkout` records the same branch, full commit SHA, and `clean: true`, plus `repository: "https://github.com/ai-dynamo/aisimulate"`. -Its `runtime.cli_entry_point` is `"aisimulate.legacy_cli.entrypoint:main"`, and its `status` +Its `runtime.cli_entry_point` is `"aisimulate.legacy_cli.entrypoint:main"` or +`"aiconfigurator.main:main"` for 0.12 wheels. New producer records include the +matching `baseline_api` and `config_adapter`; the public `aic_source` retains +`cli_entry_point`. Historical summaries without this field remain readable. Its `status` is `"complete"`. The producer's `aic_commit_sha` identifies that AISimulate commit. Branch publication rejects a baseline from another repository or revision, an incomplete baseline, or inconsistent producer documents. diff --git a/scripts/build_e2e_accuracy_overview.py b/scripts/build_e2e_accuracy_overview.py index 5c5b0b01f..11b5a712a 100644 --- a/scripts/build_e2e_accuracy_overview.py +++ b/scripts/build_e2e_accuracy_overview.py @@ -336,12 +336,22 @@ def _aic_source( or not isinstance(source.get("commit_sha"), str) or not re.fullmatch(r"[0-9a-f]{40}", source["commit_sha"]) or source["commit_sha"] != predictions.get("aic_commit_sha") - or runtime.get("cli_entry_point") != "aisimulate.legacy_cli.entrypoint:main" + or runtime.get("cli_entry_point") not in {"aisimulate.legacy_cli.entrypoint:main", "aiconfigurator.main:main"} ): raise SnapshotError("AIC baseline must use the legacy CLI bundled in a clean AISimulate checkout") if revision and any(source[key] != revision[key] for key in ("branch", "commit_sha")): raise SnapshotError("AIC baseline and AISimulate replay must evaluate the same branch and commit") - return {key: source[key] for key in ("repository", "branch", "commit_sha")} + result = {key: source[key] for key in ("repository", "branch", "commit_sha")} + if "baseline_api" in runtime or "config_adapter" in runtime: + expected = ( + ("aiconfigurator.cli.api", "aiconfigurator.sdk.config_adapter") + if runtime["cli_entry_point"] == "aiconfigurator.main:main" + else ("aisimulate.legacy_cli.api", "aisimulate.sdk.config_adapter") + ) + if (runtime.get("baseline_api"), runtime.get("config_adapter")) != expected: + raise SnapshotError("baseline entry point and predictor modules disagree") + result["cli_entry_point"] = runtime["cli_entry_point"] + return result def _model_summary(model: str, rows: list[dict[str, Any]]) -> dict[str, Any]: diff --git a/scripts/prepare_e2e_accuracy_pages.py b/scripts/prepare_e2e_accuracy_pages.py index 078097f31..2ae0ebb78 100644 --- a/scripts/prepare_e2e_accuracy_pages.py +++ b/scripts/prepare_e2e_accuracy_pages.py @@ -114,7 +114,12 @@ def public_contract(summary): }, ) keys(snapshot["evaluated_revision"], {"branch", "commit_sha"}) - keys(snapshot["aic_source"], {"repository", "branch", "commit_sha"}) + keys(snapshot["aic_source"], {"repository", "branch", "commit_sha", "cli_entry_point"}) + if "cli_entry_point" in snapshot["aic_source"] and snapshot["aic_source"]["cli_entry_point"] not in { + "aiconfigurator.main:main", + "aisimulate.legacy_cli.entrypoint:main", + }: + raise ValueError("unsupported baseline entry point") keys(snapshot["aisimulate_packages"], {"aisimulate"}) if snapshot["corrections"] != []: raise ValueError("nightly campaign cannot contain unreviewed corrections") diff --git a/scripts/run_e2e_accuracy.py b/scripts/run_e2e_accuracy.py index 0a9046057..86a9c3918 100644 --- a/scripts/run_e2e_accuracy.py +++ b/scripts/run_e2e_accuracy.py @@ -157,15 +157,22 @@ def eligibility_exclusion(bench): def predictor_module_names(files) -> tuple[str, str]: """Select the namespace shipped by the evaluated wheel, not the evaluator.""" members = {str(path) for path in files} - for api, adapter in ( - ("aisimulate.legacy_cli.api", "aisimulate.sdk.config_adapter"), - ("aiconfigurator.cli.api", "aiconfigurator.sdk.config_adapter"), - ): - if api.replace(".", "/") + ".py" in members: - if adapter.replace(".", "/") + "/__init__.py" not in members: - raise ValueError("wheel is missing its matching config adapter") - return api, adapter - raise ValueError("wheel has no supported baseline CLI layout") + layouts = [ + (api, adapter) + for api, adapter in ( + ("aisimulate.legacy_cli.api", "aisimulate.sdk.config_adapter"), + ("aiconfigurator.cli.api", "aiconfigurator.sdk.config_adapter"), + ) + if api.replace(".", "/") + ".py" in members + ] + if not layouts: + raise ValueError("wheel has no supported baseline CLI layout") + if len(layouts) != 1: + raise ValueError("wheel contains ambiguous baseline CLI layouts") + api, adapter = layouts[0] + if adapter.replace(".", "/") + "/__init__.py" not in members: + raise ValueError("wheel is missing its matching config adapter") + return api, adapter def wheel_identity(wheel: Path) -> dict: @@ -187,7 +194,15 @@ def wheel_identity(wheel: Path) -> dict: module = importlib.import_module(name) if not Path(module.__file__).resolve().is_relative_to(Path(dist.locate_file("")).resolve()): raise ValueError("import resolved outside installed wheel") - return {"wheel_sha256": digest(wheel), "packages": {"aisimulate": dist.version}} + return { + "wheel_sha256": digest(wheel), + "packages": {"aisimulate": dist.version}, + "baseline_api": api, + "config_adapter": adapter, + "cli_entry_point": "aiconfigurator.main:main" + if api.startswith("aiconfigurator.") + else "aisimulate.legacy_cli.entrypoint:main", + } def replay_spec(request, backend_version: str): @@ -417,7 +432,6 @@ def campaign(args) -> None: "runtime": { **identity, "source_checkout": {**source, "repository": REPOSITORY}, - "cli_entry_point": "aisimulate.legacy_cli.entrypoint:main", }, } common = { diff --git a/tests/test_e2e_accuracy_nightly.py b/tests/test_e2e_accuracy_nightly.py index 2c13ea6fe..09fccf904 100644 --- a/tests/test_e2e_accuracy_nightly.py +++ b/tests/test_e2e_accuracy_nightly.py @@ -337,8 +337,8 @@ def timed_out(*args, **kwargs): } -@pytest.fixture -def artifact(tmp_path, monkeypatch): +@pytest.fixture(params=["aisimulate", "aiconfigurator"]) +def artifact(tmp_path, monkeypatch, request): source = tmp_path / "tables.json" source.write_text(json.dumps(tables())) manifest = tmp_path / "manifest.json" @@ -346,7 +346,17 @@ def artifact(tmp_path, monkeypatch): monkeypatch.setattr( campaign, "wheel_identity", - lambda path: {"wheel_sha256": "a" * 64, "packages": {"aisimulate": "0.12.0"}}, + lambda path: { + "wheel_sha256": "a" * 64, + "packages": {"aisimulate": "0.12.0"}, + "baseline_api": "aiconfigurator.cli.api" + if request.param == "aiconfigurator" + else "aisimulate.legacy_cli.api", + "config_adapter": request.param + ".sdk.config_adapter", + "cli_entry_point": "aiconfigurator.main:main" + if request.param == "aiconfigurator" + else "aisimulate.legacy_cli.entrypoint:main", + }, ) def predictor(point, timeout): @@ -1138,3 +1148,56 @@ def test_predictor_layout_rejects_missing_or_mixed_namespaces(): campaign.predictor_module_names( ["aisimulate/legacy_cli/api.py", "aiconfigurator/sdk/config_adapter/__init__.py"] ) + + +@pytest.mark.parametrize("adapters", [[], ["aisimulate"], ["aiconfigurator"], ["aisimulate", "aiconfigurator"]]) +def test_predictor_layout_rejects_multiple_baseline_apis(adapters): + files = ["aisimulate/legacy_cli/api.py", "aiconfigurator/cli/api.py"] + files += [name + "/sdk/config_adapter/__init__.py" for name in adapters] + with pytest.raises(ValueError, match="ambiguous"): + campaign.predictor_module_names(files) + + +@pytest.mark.parametrize( + "api,adapter_name", + [ + ("aisimulate.legacy_cli.api", "aisimulate.sdk.config_adapter"), + ("aiconfigurator.cli.api", "aiconfigurator.sdk.config_adapter"), + ], +) +def test_predict_point_calls_selected_api_and_adapter(monkeypatch, api, adapter_name): + files = [api.replace(".", "/") + ".py", adapter_name.replace(".", "/") + "/__init__.py"] + monkeypatch.setattr(campaign.importlib.metadata, "distribution", lambda _: SimpleNamespace(files=files)) + monkeypatch.setitem(sys.modules, "aisimulate.runner", SimpleNamespace(EngineReplayRunnerFactory=object)) + calls = [] + request = object() + + def adapt(source): + calls.append(("adapt", source)) + return SimpleNamespace(requests=[request]) + + def kwargs(value): + assert value is request + return {"test_input": 42} + + def estimate(**values): + calls.append(("estimate", values)) + raise ValueError("test baseline failure") + + modules = { + api: SimpleNamespace(cli_estimate=estimate), + adapter_name: SimpleNamespace( + InferenceXSource=lambda **values: values, adapt_config=adapt, to_cli_estimate_kwargs=kwargs + ), + } + monkeypatch.setattr(campaign.importlib, "import_module", modules.__getitem__) + result = campaign.predict_point({"id": "point", "config": {}, "benchmark": {}}) + assert result["outcome"] == "baseline_failed" + assert calls == [("adapt", {"config": {}, "benchmark": {}}), ("estimate", {"test_input": 42})] + + +def test_publication_rejects_unknown_baseline_entry_point(artifact): + summary, run = artifact + summary["snapshot"]["aic_source"]["cli_entry_point"] = "foreign.main:main" + with pytest.raises(ValueError, match="baseline entry point"): + publish.validate_artifact(archive(summary), run) diff --git a/tests/test_e2e_accuracy_overview.py b/tests/test_e2e_accuracy_overview.py index f5d9fdd95..0e0f86f7f 100644 --- a/tests/test_e2e_accuracy_overview.py +++ b/tests/test_e2e_accuracy_overview.py @@ -560,3 +560,22 @@ def test_page_explains_cli_migration_and_aic_deprecation() -> None: assert "confidence" in page assert "deprecate the AIC CLI" in page assert "different prediction coverage" in page + + +@pytest.mark.parametrize( + "entry,api,adapter", + [ + ("aisimulate.legacy_cli.entrypoint:main", "aisimulate.legacy_cli.api", "aisimulate.sdk.config_adapter"), + ("aiconfigurator.main:main", "aiconfigurator.cli.api", "aiconfigurator.sdk.config_adapter"), + ], +) +def test_baseline_provenance_matches_selected_modules(entry, api, adapter): + predictions, metadata, coverage = _qualified_inputs() + for document in (predictions, metadata, coverage): + document["aic_run"]["runtime"].update(cli_entry_point=entry, baseline_api=api, config_adapter=adapter) + source = OVERVIEW._aic_source(predictions, metadata, coverage, None) + assert source["cli_entry_point"] == entry + for document in (predictions, metadata, coverage): + document["aic_run"]["runtime"]["config_adapter"] = "foreign.adapter" + with pytest.raises(OVERVIEW.SnapshotError, match="modules disagree"): + OVERVIEW._aic_source(predictions, metadata, coverage, None) From de04ef87a44a9839bb52435d6d93cefa20607535 Mon Sep 17 00:00:00 2001 From: Simone Chen Date: Wed, 30 Sep 2026 15:25:26 -0700 Subject: [PATCH 3/3] fix: validate and display baseline provenance in Pages Signed-off-by: Simone Chen --- pages/e2e-accuracy/README.md | 4 +++- pages/e2e-accuracy/app.js | 4 +++- scripts/build_pages_site.py | 5 +++++ tests/test_e2e_accuracy_nightly.py | 18 ++++++++++++++++++ tests/test_e2e_accuracy_ui.mjs | 17 +++++++++++++++++ 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/pages/e2e-accuracy/README.md b/pages/e2e-accuracy/README.md index a2a8cbf2a..2f7d73911 100644 --- a/pages/e2e-accuracy/README.md +++ b/pages/e2e-accuracy/README.md @@ -245,7 +245,9 @@ Its `runtime.source_checkout` records the same branch, full commit SHA, and Its `runtime.cli_entry_point` is `"aisimulate.legacy_cli.entrypoint:main"` or `"aiconfigurator.main:main"` for 0.12 wheels. New producer records include the matching `baseline_api` and `config_adapter`; the public `aic_source` retains -`cli_entry_point`. Historical summaries without this field remain readable. Its `status` +`cli_entry_point`. Both the site builder and browser validate the supported +entry points, and the provenance panel displays the recorded value. Historical +summaries without this field remain readable. Its `status` is `"complete"`. The producer's `aic_commit_sha` identifies that AISimulate commit. Branch publication rejects a baseline from another repository or revision, an incomplete baseline, or inconsistent producer documents. diff --git a/pages/e2e-accuracy/app.js b/pages/e2e-accuracy/app.js index 0220b0b7f..7afb4ee9f 100644 --- a/pages/e2e-accuracy/app.js +++ b/pages/e2e-accuracy/app.js @@ -142,7 +142,7 @@ function renderSnapshot() { ? `${escapeHtml(snapshot.evaluated_revision.branch)} @ ${escapeHtml(snapshot.evaluated_revision.commit_sha.slice(0, 12))}` : "Not recorded in this historical snapshot"}

Legacy AIC CLI source: ${snapshot.aic_source - ? `AISimulate ${escapeHtml(snapshot.aic_source.branch)} @ ${snapshot.aic_source.commit_sha.slice(0, 12)} (bundled aiconfigurator CLI)` + ? `AISimulate ${escapeHtml(snapshot.aic_source.branch)} @ ${snapshot.aic_source.commit_sha.slice(0, 12)} (${escapeHtml(snapshot.aic_source.cli_entry_point ?? "bundled aiconfigurator CLI")})` : "Repository provenance was not recorded in this historical snapshot"}

${snapshot.campaign ? `

Accuracy campaign: GitHub Actions run (advisory)
Selected operating points: ${escapeHtml(snapshot.campaign.selected)}; published comparison points: ${escapeHtml(snapshot.campaign.published)}.
@@ -555,6 +555,8 @@ function validateSummary(data) { aicSource.repository !== "https://github.com/ai-dynamo/aisimulate" || typeof aicSource.commit_sha !== "string" || !/^[0-9a-f]{40}$/.test(aicSource.commit_sha) || typeof aicSource.branch !== "string" || aicSource.commit_sha !== data.snapshot.aic_commit_sha || + (Object.hasOwn(aicSource, "cli_entry_point") && + !["aiconfigurator.main:main", "aisimulate.legacy_cli.entrypoint:main"].includes(aicSource.cli_entry_point)) || (revision && (aicSource.branch !== revision.branch || aicSource.commit_sha !== revision.commit_sha)))) { throw new Error("invalid legacy AIC CLI source"); } diff --git a/scripts/build_pages_site.py b/scripts/build_pages_site.py index d8f53d1ee..00e75d4a4 100755 --- a/scripts/build_pages_site.py +++ b/scripts/build_pages_site.py @@ -288,6 +288,11 @@ def topology(item: dict) -> None: and isinstance(aic_source.get("commit_sha"), str) and bool(re.fullmatch(r"[0-9a-f]{40}", aic_source["commit_sha"])) and aic_source["commit_sha"] == snapshot.get("aic_commit_sha") + and ( + "cli_entry_point" not in aic_source + or aic_source["cli_entry_point"] + in ("aiconfigurator.main:main", "aisimulate.legacy_cli.entrypoint:main") + ) and (revision is None or all(aic_source[key] == revision[key] for key in ("branch", "commit_sha"))), "legacy AIC CLI source", ) diff --git a/tests/test_e2e_accuracy_nightly.py b/tests/test_e2e_accuracy_nightly.py index 09fccf904..d395e2ef8 100644 --- a/tests/test_e2e_accuracy_nightly.py +++ b/tests/test_e2e_accuracy_nightly.py @@ -1201,3 +1201,21 @@ def test_publication_rejects_unknown_baseline_entry_point(artifact): summary["snapshot"]["aic_source"]["cli_entry_point"] = "foreign.main:main" with pytest.raises(ValueError, match="baseline entry point"): publish.validate_artifact(archive(summary), run) + + +@pytest.mark.parametrize( + "entry", ["aiconfigurator.main:main", "aisimulate.legacy_cli.entrypoint:main", None, "foreign.main:main", 42] +) +def test_site_builder_validates_baseline_entry_point(artifact, tmp_path, entry): + summary, _ = artifact + summary["snapshot"]["aic_source"]["cli_entry_point"] = entry + path = tmp_path / "summary.json" + path.write_text(json.dumps(summary)) + if entry in ("aiconfigurator.main:main", "aisimulate.legacy_cli.entrypoint:main"): + pages._accuracy_summary(path.read_text()) + else: + with pytest.raises(pages.PagesBuildError, match="legacy AIC CLI source"): + pages._accuracy_summary(path.read_text()) + del summary["snapshot"]["aic_source"]["cli_entry_point"] + path.write_text(json.dumps(summary)) + pages._accuracy_summary(path.read_text()) diff --git a/tests/test_e2e_accuracy_ui.mjs b/tests/test_e2e_accuracy_ui.mjs index 745bcf97b..23ff5e4a0 100644 --- a/tests/test_e2e_accuracy_ui.mjs +++ b/tests/test_e2e_accuracy_ui.mjs @@ -689,3 +689,20 @@ test("evaluated snapshots require matching legacy CLI provenance", () => { app.set("historicalOnly", data); assert.doesNotThrow(() => app.run("validateSummary(historicalOnly)")); }); + +test("baseline entry point is validated and displayed for both layouts", async () => { + for (const entry of ["aiconfigurator.main:main", "aisimulate.legacy_cli.entrypoint:main"]) { + const data = withEvaluation(); + data.snapshot.aic_source.cli_entry_point = entry; + const app = setup(async () => response(data)); + app.set("revisionFixture", data.snapshot.evaluated_revision); + app.run('Object.assign(state.catalog.branches[0], {status: "evaluated", evaluated_revision: revisionFixture})'); + await app.run('loadBranch("main")'); + assert.ok(app.element("provenance-content").innerHTML.includes(entry)); + for (const invalid of ["foreign.main:main", "", null, 42]) { + data.snapshot.aic_source.cli_entry_point = invalid; + app.set("invalid", data); + assert.throws(() => app.run("validateSummary(invalid)"), /legacy AIC CLI source/); + } + } +});