From 3f97ce0f5affd4c4b5d5f354c9685b47f834129b Mon Sep 17 00:00:00 2001 From: Ytallo Layon Date: Thu, 30 Jul 2026 20:02:24 -0300 Subject: [PATCH] (MOT-4277) feat(harness): preview local E2E results --- .github/benchmark-site/README.md | 22 +- .github/benchmark-site/execution-data.js | 1 + .../benchmark-site/execution-data.test.cjs | 9 + .github/benchmark-site/execution.js | 1 + .github/benchmark-site/index.html | 7 +- .github/benchmark-site/overview.js | 19 +- .../scripts/publish_harness_e2e_dashboard.py | 10 + .../scripts/serve_harness_e2e_dashboard.py | 460 ++++++++++++++++++ .../tests/test_serve_harness_e2e_dashboard.py | 207 ++++++++ harness/tests/e2e/README.md | 11 + 10 files changed, 740 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/serve_harness_e2e_dashboard.py create mode 100644 .github/scripts/tests/test_serve_harness_e2e_dashboard.py diff --git a/.github/benchmark-site/README.md b/.github/benchmark-site/README.md index 52335c4c6..b83c178a6 100644 --- a/.github/benchmark-site/README.md +++ b/.github/benchmark-site/README.md @@ -5,7 +5,27 @@ This static shell replaces the generic benchmark-action index at for metric trends. `executions.js` indexes workflow attempts, and `runs/.json` supplies the complete retained reports. -Open a local preview from the repository root: +Import the default local report and serve the real dashboard from the repository +root: + +```bash +python3 .github/scripts/serve_harness_e2e_dashboard.py +``` + +Pass files or directories to import other local executions: + +```bash +python3 .github/scripts/serve_harness_e2e_dashboard.py \ + harness/target/e2e-reactive-fix/results.json \ + target/harness-e2e-glm-5.2 +``` + +Imports accumulate in `target/harness-e2e-dashboard-local`. Reimporting the same +report is idempotent. Use `--reset` to start a new local history, or `--host` +and `--port` to change the default `127.0.0.1:4173` listener. The command only +reads existing reports; it does not run E2E scenarios. + +To preview the sample fixtures instead, serve `.github/benchmark-site` directly: ```bash python3 -m http.server 4173 --directory .github/benchmark-site diff --git a/.github/benchmark-site/execution-data.js b/.github/benchmark-site/execution-data.js index 21b28847e..5ec8c1b7c 100644 --- a/.github/benchmark-site/execution-data.js +++ b/.github/benchmark-site/execution-data.js @@ -307,6 +307,7 @@ ); return { schemaVersion: Number(raw.schema_version) || 1, + mode: raw.mode === "local" ? "local" : "published", lastUpdate: raw.last_update || benchmarkData?.lastUpdate || "", repoUrl: raw.repo_url || benchmarkData?.repoUrl || "", preview: Boolean(globalThis.HARNESS_BENCHMARK_PREVIEW), diff --git a/.github/benchmark-site/execution-data.test.cjs b/.github/benchmark-site/execution-data.test.cjs index e3f49607d..ea1997012 100644 --- a/.github/benchmark-site/execution-data.test.cjs +++ b/.github/benchmark-site/execution-data.test.cjs @@ -68,6 +68,7 @@ test("merges manifest executions and finds a retained detail", () => { const history = mergeExecutionHistory( { schema_version: 2, + mode: "local", last_update: "2026-07-29T06:10:00Z", executions: [execution()], }, @@ -75,9 +76,17 @@ test("merges manifest executions and finds a retained detail", () => { ); assert.equal(history.executions.length, 1); + assert.equal(history.mode, "local"); assert.equal(findExecution(history, "123-1").detail_path, "runs/123-1.json"); }); +test("defaults execution history to published mode", () => { + assert.equal( + mergeExecutionHistory({ executions: [] }, { snapshots: [] }).mode, + "published", + ); +}); + test("keeps workflow attempts distinct and newest first", () => { const history = mergeExecutionHistory( { diff --git a/.github/benchmark-site/execution.js b/.github/benchmark-site/execution.js index f7482b5f0..7e5b7f0ac 100644 --- a/.github/benchmark-site/execution.js +++ b/.github/benchmark-site/execution.js @@ -181,6 +181,7 @@ document.title = `Run ${runLabel} · Harness E2E`; const workflowUrl = safeUrl(execution.workflow_url); + elements.workflowLink.hidden = !workflowUrl; if (workflowUrl) elements.workflowLink.href = workflowUrl; const commit = execution.source?.sha || ""; const repo = history.repoUrl.replace(/\/$/, ""); diff --git a/.github/benchmark-site/index.html b/.github/benchmark-site/index.html index 9a4c9bc22..5bfe56027 100644 --- a/.github/benchmark-site/index.html +++ b/.github/benchmark-site/index.html @@ -42,15 +42,15 @@

Execution dashboard

Workflow-level health with scenario results and complete run diagnostics.

- Last published + Last published
@@ -212,6 +212,7 @@

All executions

+
diff --git a/.github/benchmark-site/overview.js b/.github/benchmark-site/overview.js index bfb329eb3..3157bd7db 100644 --- a/.github/benchmark-site/overview.js +++ b/.github/benchmark-site/overview.js @@ -79,6 +79,8 @@ content: document.querySelector("#overview-content"), count: document.querySelector("#execution-count"), empty: document.querySelector("#empty-state"), + emptyDescription: document.querySelector("#empty-description"), + emptyTitle: document.querySelector("#empty-title"), efficiencyBody: document.querySelector("#efficiency-body"), efficiencyCost: document.querySelector("#efficiency-cost"), efficiencyCostDelta: document.querySelector("#efficiency-cost-delta"), @@ -125,6 +127,7 @@ ), scenarioHistoryDialog: document.querySelector("#scenario-history-dialog"), scenarioHistoryTitle: document.querySelector("#scenario-history-title"), + syncLabel: document.querySelector("#sync-label"), status: document.querySelector("#status-filter"), }; @@ -1086,7 +1089,16 @@ } async function initialize() { - elements.preview.hidden = !history.preview; + const isLocal = history.mode === "local"; + elements.preview.hidden = !(history.preview || isLocal); + elements.preview.textContent = isLocal ? "Local data" : "Preview data"; + if (isLocal) { + elements.syncLabel.textContent = "Last imported"; + elements.emptyTitle.textContent = "No local executions imported"; + elements.emptyDescription.textContent = + "Import a results.json file to populate this dashboard."; + elements.actionsLink.textContent = "View repository ↗"; + } const lastUpdate = history.lastUpdate || history.executions[0]?.completed_at; if (lastUpdate) { elements.lastUpdate.dateTime = new Date(lastUpdate).toISOString(); @@ -1095,8 +1107,9 @@ if (history.repoUrl) { const repo = safeUrl(history.repoUrl); if (repo) { - elements.actionsLink.href = - `${repo.replace(/\/$/, "")}/actions/workflows/harness-e2e-daily.yml`; + elements.actionsLink.href = isLocal + ? repo + : `${repo.replace(/\/$/, "")}/actions/workflows/harness-e2e-daily.yml`; } } elements.scenarioHistoryClose.addEventListener("click", () => { diff --git a/.github/scripts/publish_harness_e2e_dashboard.py b/.github/scripts/publish_harness_e2e_dashboard.py index 1c97444ae..21c86bd9b 100644 --- a/.github/scripts/publish_harness_e2e_dashboard.py +++ b/.github/scripts/publish_harness_e2e_dashboard.py @@ -377,9 +377,12 @@ def publish( repo_url: str, max_summaries: int, max_details: int, + site_mode: str = "published", ) -> dict[str, Any]: if max_summaries < 1 or max_details < 0 or max_details > max_summaries: raise PublishError("retention must satisfy 0 <= details <= summaries") + if site_mode not in {"local", "published"}: + raise PublishError("site mode must be local or published") site_dir.mkdir(parents=True, exist_ok=True) runs_dir = site_dir / "runs" runs_dir.mkdir(parents=True, exist_ok=True) @@ -456,6 +459,7 @@ def publish( updated = { "schema_version": SCHEMA_VERSION, + "mode": site_mode, "last_update": metadata["completed_at"] or metadata["started_at"], "repo_url": repo_url, "retention": { @@ -492,6 +496,11 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--repo-url", required=True) parser.add_argument("--max-summaries", type=int, default=100) parser.add_argument("--max-details", type=int, default=30) + parser.add_argument( + "--site-mode", + choices=("local", "published"), + default="published", + ) return parser @@ -522,6 +531,7 @@ def main(argv: list[str] | None = None) -> int: repo_url=args.repo_url, max_summaries=args.max_summaries, max_details=args.max_details, + site_mode=args.site_mode, ) print( json.dumps( diff --git a/.github/scripts/serve_harness_e2e_dashboard.py b/.github/scripts/serve_harness_e2e_dashboard.py new file mode 100644 index 000000000..8d10e802d --- /dev/null +++ b/.github/scripts/serve_harness_e2e_dashboard.py @@ -0,0 +1,460 @@ +#!/usr/bin/env python3 +"""Import local Harness E2E reports and serve the benchmark dashboard.""" + +from __future__ import annotations + +import argparse +import functools +import getpass +import hashlib +import http.server +import json +import re +import shutil +import subprocess +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from collect_harness_e2e_benchmarks import ( + CollectionConfig, + CollectionError, + collect, + write_outputs, +) +from publish_harness_e2e_dashboard import ( + MANIFEST_PREFIX, + PublishError, + load_manifest, + publish, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +DASHBOARD_SOURCE = REPO_ROOT / ".github" / "benchmark-site" +DEFAULT_RESULTS = REPO_ROOT / "harness" / "target" / "e2e" / "results.json" +DEFAULT_SITE_DIR = REPO_ROOT / "target" / "harness-e2e-dashboard-local" +LOCAL_SITE_MARKER = ".harness-e2e-local-dashboard" +BENCHMARK_DATA_PREFIX = "window.BENCHMARK_DATA = " + + +class LocalDashboardError(ValueError): + """Raised when local reports cannot be rendered safely.""" + + +def load_report(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise LocalDashboardError(f"cannot decode {path}: {exc}") from exc + if not isinstance(value, dict): + raise LocalDashboardError(f"{path} must contain a JSON object") + subject = value.get("subject") + if not isinstance(subject, dict): + raise LocalDashboardError(f"{path}: subject must be an object") + for field in ("model", "provider"): + if not isinstance(subject.get(field), str) or not subject[field]: + raise LocalDashboardError(f"{path}: subject.{field} is required") + judge = value.get("judge") + if judge is not None: + if not isinstance(judge, dict): + raise LocalDashboardError(f"{path}: judge must be an object or null") + for field in ("model", "provider"): + if not isinstance(judge.get(field), str) or not judge[field]: + raise LocalDashboardError(f"{path}: judge.{field} is required") + scenarios = value.get("scenarios") + if not isinstance(scenarios, list) or not scenarios: + raise LocalDashboardError(f"{path}: scenarios must be a non-empty array") + scenario_ids = [ + scenario.get("scenario_id") if isinstance(scenario, dict) else None + for scenario in scenarios + ] + if any( + not isinstance(scenario_id, str) or not scenario_id + for scenario_id in scenario_ids + ): + raise LocalDashboardError(f"{path}: every scenario must have a scenario_id") + if len(set(scenario_ids)) != len(scenario_ids): + raise LocalDashboardError(f"{path}: scenario ids must be unique") + if not isinstance(value.get("passed"), bool): + value["passed"] = all( + bool(scenario.get("passed")) + for scenario in scenarios + if isinstance(scenario, dict) + ) + return value + + +def discover_results( + inputs: list[Path], + *, + site_dir: Path, +) -> list[Path]: + candidates = inputs or [DEFAULT_RESULTS] + site_root = site_dir.resolve() + discovered: list[Path] = [] + seen: set[Path] = set() + + for candidate in candidates: + path = candidate.expanduser().resolve() + if path.is_file(): + matches = [path] + elif path.is_dir(): + direct = path / "results.json" + matches = ( + [direct] if direct.is_file() else sorted(path.rglob("results.json")) + ) + else: + raise LocalDashboardError(f"results path does not exist: {path}") + + matches = [ + match.resolve() + for match in matches + if not match.resolve().is_relative_to(site_root) + ] + if not matches: + raise LocalDashboardError(f"no results.json found under {path}") + for match in matches: + if match not in seen: + seen.add(match) + discovered.append(match) + + return sorted(discovered, key=lambda path: (path.stat().st_mtime_ns, str(path))) + + +def report_digest(report: dict[str, Any]) -> str: + canonical = json.dumps( + report, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode() + return hashlib.sha256(canonical).hexdigest() + + +def slug(value: str) -> str: + normalized = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + return normalized[:80] or "subject" + + +def git_value(*arguments: str) -> str: + try: + result = subprocess.run( + ["git", "-C", str(REPO_ROOT), *arguments], + check=True, + capture_output=True, + text=True, + ) + except (OSError, subprocess.CalledProcessError): + return "" + return result.stdout.strip() + + +def repository_url() -> str: + remote = git_value("remote", "get-url", "origin") + ssh_match = re.fullmatch(r"git@github\.com:(.+?)(?:\.git)?", remote) + if ssh_match: + return f"https://github.com/{ssh_match.group(1)}" + if remote.startswith(("https://", "http://")): + return remote.removesuffix(".git") + return "https://github.com/iii-hq/workers" + + +def repository_name(repo_url: str) -> str: + match = re.search(r"github\.com/([^/]+/[^/]+?)(?:\.git)?$", repo_url) + return match.group(1) if match else "iii-hq/workers" + + +def report_wall_time(report: dict[str, Any]) -> float: + total_ms = 0.0 + for scenario in report["scenarios"]: + runs = scenario.get("runs", []) + if not isinstance(runs, list): + continue + for run in runs: + wall_time_ms = run.get("wall_time_ms") if isinstance(run, dict) else None + if isinstance(wall_time_ms, (int, float)) and not isinstance( + wall_time_ms, bool + ): + total_ms += max(0.0, float(wall_time_ms)) + return total_ms / 1000 + + +def requested_runs(report: dict[str, Any]) -> int: + values = [] + for scenario in report["scenarios"]: + aggregate = scenario.get("aggregate", {}) + aggregate_runs = ( + aggregate.get("runs") if isinstance(aggregate, dict) else None + ) + runs = scenario.get("runs", []) + if ( + isinstance(aggregate_runs, int) + and not isinstance(aggregate_runs, bool) + and aggregate_runs > 0 + ): + values.append(aggregate_runs) + elif isinstance(runs, list) and runs: + values.append(len(runs)) + return max(values, default=1) + + +def prepare_site(site_dir: Path, *, reset: bool) -> None: + if reset and site_dir.exists(): + marker = site_dir / LOCAL_SITE_MARKER + if not marker.is_file(): + raise LocalDashboardError( + f"refusing to reset unmarked directory: {site_dir}" + ) + shutil.rmtree(site_dir) + if site_dir.exists() and not site_dir.is_dir(): + raise LocalDashboardError(f"site path is not a directory: {site_dir}") + site_dir.mkdir(parents=True, exist_ok=True) + shutil.copytree(DASHBOARD_SOURCE, site_dir, dirs_exist_ok=True) + (site_dir / LOCAL_SITE_MARKER).write_text("Harness E2E local dashboard\n") + + +def write_benchmark_data(site_dir: Path, repo_url: str, last_update: str) -> None: + timestamp = 0 + if last_update: + try: + timestamp = int(datetime.fromisoformat(last_update).timestamp() * 1000) + except ValueError: + timestamp = 0 + payload = { + "entries": {}, + "lastUpdate": timestamp, + "repoUrl": repo_url, + } + (site_dir / "data.js").write_text( + BENCHMARK_DATA_PREFIX + + json.dumps(payload, indent=2, sort_keys=True) + + ";\n" + ) + + +def stage_report( + report: dict[str, Any], + reports_root: Path, + subject_id: str, +) -> list[str]: + scenario_ids = [] + for scenario in report["scenarios"]: + scenario_id = scenario["scenario_id"] + scenario_ids.append(scenario_id) + directory = reports_root / f"{subject_id}-{slug(scenario_id)}" + directory.mkdir(parents=True) + (directory / "benchmark-context.json").write_text( + json.dumps( + {"subject_id": subject_id, "scenario_id": scenario_id}, + sort_keys=True, + ) + + "\n" + ) + scenario_report = { + **report, + "passed": bool(scenario.get("passed")), + "scenarios": [scenario], + } + (directory / "results.json").write_text( + json.dumps(scenario_report, indent=2, sort_keys=True) + "\n" + ) + return scenario_ids + + +def import_report( + results_path: Path, + *, + site_dir: Path, + repo_url: str, + repo_name: str, + source_sha: str, + source_ref: str, +) -> str: + report = load_report(results_path) + digest = report_digest(report) + run_id = f"local-{digest[:12]}" + subject = report["subject"] + subject_id = slug(f"{subject['provider']}-{subject['model']}") + judge = report.get("judge") + if not isinstance(judge, dict): + judge = subject + + completed = datetime.fromtimestamp( + results_path.stat().st_mtime, + timezone.utc, + ) + started = completed - timedelta(seconds=report_wall_time(report)) + completed_at = completed.isoformat() + started_at = started.isoformat() + + with tempfile.TemporaryDirectory(prefix="harness-e2e-local-dashboard-") as temp: + temp_root = Path(temp) + reports_root = temp_root / "reports" + scenarios = stage_report(report, reports_root, subject_id) + output_dir = temp_root / "output" + config = CollectionConfig( + reports_root=reports_root, + output_dir=output_dir, + subjects=[ + { + "id": subject_id, + "model": subject["model"], + "provider": subject["provider"], + } + ], + scenarios=scenarios, + lane="local", + requested_runs=requested_runs(report), + source_sha=source_sha, + source_ref=source_ref, + repository=repo_name, + workflow_url="", + release_tag="", + release_worker="", + release_version="", + release_url="", + registry_tag="local", + judge_model=str(judge["model"]), + judge_provider=str(judge["provider"]), + execution_run_id=run_id, + execution_attempt=1, + execution_event="local", + execution_actor=getpass.getuser(), + generated_at=completed_at, + ) + quality, efficiency, snapshot, execution = collect(config) + if report.get("judge") is None: + snapshot["subjects"][0]["judge"] = {} + write_outputs(output_dir, quality, efficiency, snapshot, execution) + publish( + site_dir, + snapshot_path=output_dir / "snapshot.json", + detail_path=output_dir / "execution.json", + metadata={ + "id": f"{run_id}-1", + "run_id": run_id, + "attempt": 1, + "workflow_name": "Harness E2E Local", + "workflow_url": "", + "event": "local", + "actor": getpass.getuser(), + "started_at": started_at, + "completed_at": completed_at, + "conclusion": "success" if bool(report.get("passed")) else "failure", + "head_sha": source_sha, + "head_branch": source_ref, + "repository": repo_name, + }, + repo_url=repo_url, + max_summaries=100, + max_details=30, + site_mode="local", + ) + return f"{run_id}-1" + + +def build_local_dashboard( + results_paths: list[Path], + *, + site_dir: Path, + reset: bool = False, +) -> list[str]: + site_dir = site_dir.expanduser().resolve() + paths = discover_results(results_paths, site_dir=site_dir) + prepare_site(site_dir, reset=reset) + repo_url = repository_url() + repo_name = repository_name(repo_url) + source_sha = git_value("rev-parse", "HEAD") or "local" + source_ref = git_value("branch", "--show-current") or "local" + execution_ids = [ + import_report( + path, + site_dir=site_dir, + repo_url=repo_url, + repo_name=repo_name, + source_sha=source_sha, + source_ref=source_ref, + ) + for path in paths + ] + imported_at = datetime.now(timezone.utc).isoformat() + manifest_path = site_dir / "executions.js" + manifest = load_manifest(manifest_path) + manifest["mode"] = "local" + manifest["last_update"] = imported_at + manifest_path.write_text( + MANIFEST_PREFIX + + json.dumps(manifest, indent=2, sort_keys=True, ensure_ascii=False) + + ";\n" + ) + write_benchmark_data(site_dir, repo_url, imported_at) + return execution_ids + + +def serve(site_dir: Path, host: str, port: int) -> None: + handler = functools.partial( + http.server.SimpleHTTPRequestHandler, + directory=str(site_dir), + ) + try: + server = http.server.ThreadingHTTPServer((host, port), handler) + except OSError as exc: + raise LocalDashboardError( + f"cannot serve dashboard on {host}:{port}: {exc}" + ) from exc + visible_host = "127.0.0.1" if host in {"0.0.0.0", "::"} else host + print(f"dashboard: http://{visible_host}:{server.server_port}/index.html") + print("press Ctrl+C to stop") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nstopping dashboard") + finally: + server.server_close() + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Import local Harness E2E results and serve the dashboard." + ) + parser.add_argument( + "results", + nargs="*", + type=Path, + help="results.json files or directories containing local reports", + ) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=4173) + parser.add_argument("--site-dir", type=Path, default=DEFAULT_SITE_DIR) + parser.add_argument( + "--reset", + action="store_true", + help="clear previously imported local executions before importing", + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + if not 1 <= args.port <= 65535: + parser.error("--port must be between 1 and 65535") + try: + execution_ids = build_local_dashboard( + args.results, + site_dir=args.site_dir, + reset=args.reset, + ) + print( + f"imported {len(execution_ids)} execution" + f"{'' if len(execution_ids) == 1 else 's'}" + ) + serve(args.site_dir.expanduser().resolve(), args.host, args.port) + except (CollectionError, LocalDashboardError, PublishError) as exc: + parser.error(str(exc)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/tests/test_serve_harness_e2e_dashboard.py b/.github/scripts/tests/test_serve_harness_e2e_dashboard.py new file mode 100644 index 000000000..70aaed921 --- /dev/null +++ b/.github/scripts/tests/test_serve_harness_e2e_dashboard.py @@ -0,0 +1,207 @@ +"""Tests for importing local Harness E2E reports into the static dashboard.""" + +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from publish_harness_e2e_dashboard import load_manifest +from serve_harness_e2e_dashboard import ( + build_local_dashboard, + discover_results, +) + + +def scenario(scenario_id: str, score: int = 90, *, passed: bool = True) -> dict: + return { + "scenario_id": scenario_id, + "threshold": 50, + "execution_policy": {"max_turns": 4}, + "aggregate": { + "runs": 1, + "scored_runs": 1, + "passed_runs": int(passed), + "required_passes": 1, + "pass_rate": float(passed), + "median_score": score, + "hard_gate_failures": 0 if passed else 1, + "technical_failures": 0, + "cost": { + "subject_usd": 0.2, + "judge_usd": 0.1, + "total_usd": 0.3, + }, + }, + "passed": passed, + "runs": [ + { + "run_id": f"{scenario_id}-run", + "session_id": f"{scenario_id}-session", + "prompt": f"Run {scenario_id}.", + "wall_time_ms": 10_000, + "score": score, + "status": "passed" if passed else "hard_gate_failed", + "hard_gates": [], + "criteria": [], + "transcript": {"messages": []}, + "metrics": { + "totals": { + "input_tokens": 100, + "output_tokens": 20, + "function_calls": 2, + "function_call_errors": 0, + "sessions": 1, + "turns": 2, + } + }, + "cost": { + "subject_usd": 0.2, + "judge_usd": 0.1, + "total_usd": 0.3, + }, + "retry_attempts": [], + "failures": [], + } + ], + } + + +def report(*scenarios: dict, passed: bool = True) -> dict: + return { + "subject": {"model": "glm-5.2", "provider": "zai"}, + "judge": {"model": "glm-5.2", "provider": "zai"}, + "passed": passed, + "scenarios": list(scenarios), + } + + +def write_report(path: Path, value: dict, timestamp: int = 1_750_000_000) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + os.utime(path, (timestamp, timestamp)) + return path + + +def fake_git_value(*arguments: str) -> str: + return "a" * 40 if arguments[-1] == "HEAD" else "main" + + +class LocalDashboardTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.root = Path(self.temporary_directory.name) + + def tearDown(self) -> None: + self.temporary_directory.cleanup() + + def test_discovers_direct_and_recursive_results_without_duplicates( + self, + ) -> None: + direct = write_report( + self.root / "direct/results.json", + report(scenario("direct_answer")), + ) + nested = write_report( + self.root / "nested/run/results.json", + report(scenario("security_review")), + timestamp=1_750_000_001, + ) + + self.assertEqual( + discover_results( + [direct.parent, self.root / "nested", direct], + site_dir=self.root / "site", + ), + [direct, nested], + ) + + @patch( + "serve_harness_e2e_dashboard.repository_url", + return_value="https://github.com/iii-hq/workers", + ) + @patch( + "serve_harness_e2e_dashboard.git_value", + side_effect=fake_git_value, + ) + def test_builds_idempotent_local_history_with_full_multi_scenario_detail( + self, + _git_value, + _repository_url, + ) -> None: + results = write_report( + self.root / "reports/results.json", + report( + scenario("direct_answer"), + scenario("security_review", score=75), + ), + ) + site = self.root / "site" + + first_ids = build_local_dashboard([results], site_dir=site) + second_ids = build_local_dashboard([results], site_dir=site) + + self.assertEqual(first_ids, second_ids) + manifest = load_manifest(site / "executions.js") + self.assertEqual(manifest["mode"], "local") + self.assertEqual(len(manifest["executions"]), 1) + execution = manifest["executions"][0] + self.assertEqual(execution["event"], "local") + self.assertEqual(execution["status"], "passed") + self.assertEqual(execution["totals"]["total_tokens"], 240) + detail = json.loads((site / execution["detail_path"]).read_text()) + self.assertEqual( + [entry["scenario_id"] for entry in detail["reports"]], + ["direct_answer", "security_review"], + ) + self.assertTrue( + (site / "data.js").read_text().startswith( + "window.BENCHMARK_DATA = " + ) + ) + self.assertNotIn( + "HARNESS_BENCHMARK_PREVIEW", + (site / "data.js").read_text(), + ) + + @patch( + "serve_harness_e2e_dashboard.repository_url", + return_value="https://github.com/iii-hq/workers", + ) + @patch( + "serve_harness_e2e_dashboard.git_value", + side_effect=fake_git_value, + ) + def test_accumulates_changed_reports_and_preserves_failures( + self, + _git_value, + _repository_url, + ) -> None: + first = write_report( + self.root / "first/results.json", + report(scenario("direct_answer")), + ) + second = write_report( + self.root / "second/results.json", + report( + scenario("direct_answer", score=40, passed=False), + passed=False, + ), + timestamp=1_750_000_001, + ) + site = self.root / "site" + + build_local_dashboard([first], site_dir=site) + build_local_dashboard([second], site_dir=site) + + manifest = load_manifest(site / "executions.js") + self.assertEqual(len(manifest["executions"]), 2) + self.assertEqual(manifest["executions"][0]["status"], "failed") + self.assertEqual(manifest["executions"][1]["status"], "passed") + + +if __name__ == "__main__": + unittest.main() diff --git a/harness/tests/e2e/README.md b/harness/tests/e2e/README.md index 0bd6e24c0..48bb91718 100644 --- a/harness/tests/e2e/README.md +++ b/harness/tests/e2e/README.md @@ -165,6 +165,17 @@ cargo run -p harness-e2e -- report target/e2e cargo run -p harness-e2e -- report target/e2e/results.json --verbose ``` +From the repository root, import that report into the benchmark dashboard and +serve it locally: + +```bash +python3 .github/scripts/serve_harness_e2e_dashboard.py +``` + +The page is available at . Pass additional +`results.json` files or directories to build a local execution history. The +generated site stays under `target/` and is not committed. + The runner writes `results.json` with: - exact catalog-resolved subject and judge model identity and capabilities;