ci(next): retain exact frame pacing receipts

.github/workflows/cmux-next-frame-pacing.yml

@@ -193,6 +193,10 @@ jobs:
     timeout-minutes: 45
     env:
       CMUX_TAG: ${{ needs.gate.outputs.tag }}
+      # Keep the receipt bound to the exact app source and workflow revision
+      # that produced it; the summary remains warning-only for now.
+      CMUX_PERF_SOURCE_SHA: ${{ needs.resolve-ref.outputs.sha }}
+      CMUX_PERF_WORKFLOW_SHA: ${{ github.workflow_sha }}
     steps:
       # The bench draws in the console session, which every GUI job on the mini
       # shares: hold glaeda's gui token for the rest of the job, as test-e2e.yml
@@ -205,9 +209,13 @@ jobs:
           status=0
           [ -x "$helper" ] && { "$helper" take-gui --wait 900 || status=$?; }
           case "$status" in
-            0|2) echo "taken=true" >> "$GITHUB_OUTPUT" ;;
+            0|2)
+              echo "taken=true" >> "$GITHUB_OUTPUT"
+              echo "reason=gui token acquired" >> "$GITHUB_OUTPUT"
+              ;;
             *)
               echo "taken=false" >> "$GITHUB_OUTPUT"
+              echo "reason=the console session stayed busy (take-gui exited $status)" >> "$GITHUB_OUTPUT"
               echo "::notice title=Agent pane frame pacing::Skipped: the console session stayed busy (take-gui exited $status)."
               printf '## Agent pane frame pacing\n\nSkipped: the console session stayed busy (take-gui exited %s).\n' "$status" >> "$GITHUB_STEP_SUMMARY" ;;
           esac
@@ -280,13 +288,70 @@ jobs:
 
       - name: Summary
         if: always() && steps.gui.outputs.taken == 'true' && contains(env.SUITES, 'fling')
-        run: python3 scripts/ci/frame-pacing/summarize.py "$RUNNER_TEMP/frame-pacing/out" >> "$GITHUB_STEP_SUMMARY"
+        run: |
+          python3 scripts/ci/frame-pacing/summarize.py \
+            "$RUNNER_TEMP/frame-pacing/out" \
+            --receipt "$RUNNER_TEMP/frame-pacing/out/summary.json" \
+            >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Receipt for a skipped bench
+        if: always() && steps.gui.outputs.taken != 'true'
+        env:
+          SKIP_REASON: ${{ steps.gui.outputs.reason || 'the GUI token was not acquired' }}
+        run: |
+          mkdir -p "$RUNNER_TEMP/frame-pacing/out"
+          python3 scripts/ci/frame-pacing/summarize.py \
+            "$RUNNER_TEMP/frame-pacing/out" \
+            --receipt "$RUNNER_TEMP/frame-pacing/out/summary.json" \
+            --skip-reason "$SKIP_REASON"
 
       - name: Upload the bench results
-        if: always() && steps.gui.outputs.taken == 'true'
+        if: always()
         continue-on-error: true
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: frame-pacing-${{ github.run_id }}
           path: ${{ runner.temp }}/frame-pacing/out
           retention-days: 14
+
+  # A gate skip prevents the bench job from starting, so keep a receipt for
+  # that outcome too. This makes skipped nights distinguishable from missing
+  # artifacts without turning the warning-only lane into a required check.
+  skipped-receipt:
+    needs: [resolve-ref, gate, bench]
+    if: always() && needs.gate.outputs.run != 'true'
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 5
+    env:
+      CMUX_PERF_SOURCE_SHA: ${{ needs.resolve-ref.outputs.sha || github.sha }}
+      CMUX_PERF_WORKFLOW_SHA: ${{ github.workflow_sha }}
+      SKIP_REASON: frame-pacing gate did not authorize a bench (gate=${{ needs.gate.result }}, run=${{ needs.gate.outputs.run || 'unset' }})
+    steps:
+      - name: Write gate-skip receipt
+        run: |
+          mkdir -p "$RUNNER_TEMP/frame-pacing/out"
+          python3 - <<'PY'
+          import json
+          import os
+          from datetime import datetime, timezone
+          from pathlib import Path
+
+          path = Path(os.environ["RUNNER_TEMP"]) / "frame-pacing" / "out" / "summary.json"
+          path.write_text(json.dumps({
+              "schema": "cmux.next.frame-pacing.v1",
+              "recorded_at": datetime.now(timezone.utc).isoformat(),
+              "source_sha": os.environ.get("CMUX_PERF_SOURCE_SHA"),
+              "workflow_sha": os.environ.get("CMUX_PERF_WORKFLOW_SHA"),
+              "status": "skipped",
+              "reason": os.environ["SKIP_REASON"],
+              "thresholds_enforced": False,
+              "modes": {},
+          }, indent=2, sort_keys=True) + "\n")
+          PY
+
+      - name: Upload gate-skip receipt
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: frame-pacing-${{ github.run_id }}
+          path: ${{ runner.temp }}/frame-pacing/out
+          retention-days: 14

scripts/ci/frame-pacing/summarize.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """Summarize the agent pane fling bench (bench.sh) as markdown for the run summary.
 
-    summarize.py OUT_DIR
+    summarize.py OUT_DIR [--receipt PATH]
 
 Reads OUT_DIR/display.txt and OUT_DIR/<mode>-<n>.json (mode 0 is the capped
 render rate, 1 is full rate) and prints one table row per mode: flings, frames
@@ -14,8 +14,11 @@
 from __future__ import annotations
 
 import json
+import os
 import statistics
 import sys
+from datetime import datetime, timezone
+from argparse import ArgumentParser
 from dataclasses import dataclass
 from pathlib import Path
 
@@ -45,6 +48,24 @@ def regressions(self) -> list[str]:
             found.append(f"dropped {self.dropped}/{self.total_frames} > {DROPPED_LIMIT:.0%}")
         return found
 
+    def receipt(self) -> dict[str, object]:
+        """Return stable, reviewable measurements for CI artifacts."""
+        problems = self.regressions()
+        return {
+            "status": "regression" if problems else "ok",
+            "flings": self.flings,
+            "frames_per_fling_median": self.frames,
+            "p50_ms_median": self.p50_ms,
+            "p95_ms_median": self.p95_ms,
+            "dropped_frames": self.dropped,
+            "total_frames": self.total_frames,
+            "regressions": problems,
+            "thresholds": {
+                "p50_ms": P50_LIMIT_MS[self.mode],
+                "dropped_fraction": DROPPED_LIMIT,
+            },
+        }
+
 
 def load_mode(out: Path, mode: str) -> ModeResult | None:
     flings = []
@@ -99,14 +120,69 @@ def summarize(out: Path) -> tuple[str, list[str]]:
     return "\n".join(lines) + "\n", warnings
 
 
+def write_receipt(out: Path, path: Path, skip_reason: str | None = None) -> None:
+    """Persist the same measurements shown in the run summary.
+
+    The frame-pacing job is intentionally warning-only today. Keeping a
+    machine-readable receipt alongside the raw fling files makes each run
+    reviewable and gives a later ratchet a stable input without changing the
+    job's existing failure policy.
+    """
+    display = (out / "display.txt").read_text().strip() if (out / "display.txt").exists() else ""
+    payload: dict[str, object] = {
+        "schema": "cmux.next.frame-pacing.v1",
+        "recorded_at": datetime.now(timezone.utc).isoformat(),
+        "source_sha": os.environ.get("CMUX_PERF_SOURCE_SHA") or os.environ.get("GITHUB_SHA"),
+        "workflow_sha": os.environ.get("CMUX_PERF_WORKFLOW_SHA") or os.environ.get("GITHUB_WORKFLOW_SHA"),
+        "display": display,
+        "thresholds_enforced": False,
+        "modes": {},
+    }
+    if skip_reason:
+        payload["status"] = "skipped"
+        payload["reason"] = skip_reason
+    elif not display.startswith("ready"):
+        payload["status"] = "skipped"
+        payload["reason"] = display or "no display report"
+    else:
+        payload["status"] = "ok"
+        unmeasured_modes = []
+        for mode, name in MODES.items():
+            result = load_mode(out, mode)
+            if result is None:
+                error = out / f"{mode}-error.txt"
+                payload["modes"][mode] = {
+                    "name": name,
+                    "status": "not_measured",
+                    "reason": error.read_text().strip() if error.exists() else "no fling drew a frame",
+                }
+                unmeasured_modes.append(mode)
+                continue
+            data = result.receipt()
+            data["name"] = name
+            payload["modes"][mode] = data
+            if data["status"] == "regression":
+                payload["status"] = "regression"
+        if unmeasured_modes and payload["status"] == "ok":
+            payload["status"] = "partial"
+        if unmeasured_modes:
+            payload["unmeasured_modes"] = unmeasured_modes
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+
+
 def main(argv: list[str]) -> int:
-    if len(argv) != 2:
-        print(__doc__.strip().splitlines()[2].strip(), file=sys.stderr)
-        return 2
-    markdown, warnings = summarize(Path(argv[1]))
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument("out", type=Path)
+    parser.add_argument("--receipt", type=Path)
+    parser.add_argument("--skip-reason")
+    args = parser.parse_args(argv[1:])
+    markdown, warnings = summarize(args.out)
     sys.stdout.write(markdown)
     for warning in warnings:
         print(f"::warning title=Agent pane frame pacing::{warning}", file=sys.stderr)
+    if args.receipt:
+        write_receipt(args.out, args.receipt, args.skip_reason)
     return 0
 
 

tests/test_ci_frame_pacing_summary.py

@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 import json
+import os
 import subprocess
 import sys
 import tempfile
@@ -26,8 +27,19 @@ def write_run(out: Path, display: str, flings: dict[str, list[tuple[int, float,
             (out / f"{mode}-{n}.json").write_text(json.dumps({"fling": fling, "perf": {}}))
 
 
-def run(out: Path) -> subprocess.CompletedProcess[str]:
-    return subprocess.run([sys.executable, str(SCRIPT), str(out)], capture_output=True, text=True, check=False)
+def run(
+    out: Path,
+    receipt: Path | None = None,
+    env: dict[str, str] | None = None,
+    skip_reason: str | None = None,
+) -> subprocess.CompletedProcess[str]:
+    command = [sys.executable, str(SCRIPT), str(out)]
+    if receipt is not None:
+        command += ["--receipt", str(receipt)]
+    if skip_reason is not None:
+        command += ["--skip-reason", skip_reason]
+    process_env = None if env is None else {**os.environ, **env}
+    return subprocess.run(command, capture_output=True, text=True, check=False, env=process_env)
 
 
 def test_a_healthy_night_reports_both_modes_without_warnings() -> None:
@@ -98,12 +110,72 @@ def test_a_fling_whose_rpc_replied_with_an_error_is_left_out() -> None:
         assert "::warning" not in result.stderr, result.stderr
 
 
+def test_receipt_captures_exact_source_and_reviewable_measurements() -> None:
+    with tempfile.TemporaryDirectory() as temp:
+        out = Path(temp)
+        receipt = out / "summary.json"
+        write_run(out, "ready 17 8.33 main=17", {
+            "0": [(181, 17, 18, 0)],
+            "1": [(360, 8, 9, 1)],
+        })
+        result = run(out, receipt, {"CMUX_PERF_SOURCE_SHA": "source-sha", "CMUX_PERF_WORKFLOW_SHA": "workflow-sha"})
+        assert result.returncode == 0, result.stderr
+        data = json.loads(receipt.read_text())
+        assert data["schema"] == "cmux.next.frame-pacing.v1"
+        assert data["source_sha"] == "source-sha"
+        assert data["workflow_sha"] == "workflow-sha"
+        assert data["status"] == "ok"
+        assert data["thresholds_enforced"] is False
+        assert data["modes"]["1"]["p50_ms_median"] == 8.0
+        assert data["modes"]["1"]["dropped_frames"] == 1
+
+
+def test_receipt_preserves_skip_reason() -> None:
+    with tempfile.TemporaryDirectory() as temp:
+        out = Path(temp)
+        receipt = out / "summary.json"
+        write_run(out, "error: CGVirtualDisplay unavailable", {})
+        result = run(out, receipt)
+        assert result.returncode == 0, result.stderr
+        data = json.loads(receipt.read_text())
+        assert data["status"] == "skipped"
+        assert data["reason"] == "error: CGVirtualDisplay unavailable"
+
+
+def test_receipt_marks_unmeasured_mode_as_partial() -> None:
+    with tempfile.TemporaryDirectory() as temp:
+        out = Path(temp)
+        receipt = out / "summary.json"
+        write_run(out, "ready 17 8.33 main=17", {"0": [(181, 17, 18, 0)]})
+        result = run(out, receipt)
+        assert result.returncode == 0, result.stderr
+        data = json.loads(receipt.read_text())
+        assert data["status"] == "partial"
+        assert data["unmeasured_modes"] == ["1"]
+        assert data["modes"]["1"]["status"] == "not_measured"
+
+
+def test_receipt_can_record_a_prebench_skip() -> None:
+    with tempfile.TemporaryDirectory() as temp:
+        out = Path(temp) / "out"
+        receipt = out / "summary.json"
+        result = run(out, receipt, skip_reason="no owned Mac was available")
+        assert result.returncode == 0, result.stderr
+        data = json.loads(receipt.read_text())
+        assert data["status"] == "skipped"
+        assert data["reason"] == "no owned Mac was available"
+
+
 def main() -> int:
     test_a_healthy_night_reports_both_modes_without_warnings()
     test_a_regression_is_flagged_and_warned_but_exits_zero()
     test_no_virtual_display_skips_the_table()
     test_a_mode_that_never_drew_says_why_instead_of_flagging()
     test_a_fling_whose_rpc_replied_with_an_error_is_left_out()
+    test_receipt_captures_exact_source_and_reviewable_measurements()
+    test_receipt_preserves_skip_reason()
+    test_receipt_marks_unmeasured_mode_as_partial()
+    test_receipt_can_record_a_prebench_skip()
     print("PASS: frame pacing summary reports numbers and flags regressions without failing")
     return 0