ci: webview flakes on the base only warn; split accent-glass; name new webview reds

.github/workflows/ci-web.yml

@@ -505,7 +505,9 @@ jobs:
           # Replay uses this exact runner and result parser, with the base's tests and bun.
           # Different deadline policies can otherwise turn an old named failure into a new timeout.
           mkdir -p "$RUNNER_TEMP/webviews-head-results/runner"
+          # The comparison and the flake history are the head's too: the base's copies may predate --flaky.
           cp ../scripts/ci/run-webviews-tests.sh ../scripts/ci/webviews_test_results.py ../scripts/ci/webviews_test_shards.py \
+            ../scripts/ci/guards_new_reds.py ../scripts/ci/webviews_test_history.py \
             "$RUNNER_TEMP/webviews-head-results/runner/"
           ../scripts/ci/webviews-tests-new-reds.sh "$BASE_REF"
 
@@ -521,6 +523,22 @@ jobs:
           if-no-files-found: error
           retention-days: 14
 
+      # A push's verdicts feed the base's flake history (the webviews-test-history job), which a
+      # pull request's base replay reads so a test that flips by itself on the base only warns.
+      # Optional upkeep, like the durations below: it never fails the shard.
+      - name: Upload this push's webview results
+        id: history-upload
+        if: ${{ !cancelled() && github.event_name == 'push' }}
+        continue-on-error: true
+        timeout-minutes: 2
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: webviews-test-history-shard${{ matrix.shard }}
+          overwrite: true
+          path: ${{ runner.temp }}/webviews-head-results/webviews-head-results.json
+          if-no-files-found: ignore
+          retention-days: 1
+
       # Every event: push runs on feat-cmux-next keep the manifest fresh for everyone.
       # Optional upkeep: an artifact-service failure or stall never fails the
       # shard; the gate step below reports it as a warning.
@@ -552,6 +570,7 @@ jobs:
           PR_SCOPED_RULES: ${{ steps.pr-scoped-rules.outcome }}
           # Warn-only: the next run's shard split, never this run's verdict.
           DURATIONS_UPLOAD: ${{ steps.durations-upload.outcome }}
+          HISTORY_UPLOAD: ${{ steps.history-upload.outcome }}
           # The eager JS budget (a size cap) and the React Compiler coverage are
           # structural: for feat-cmux-next they warn (Leo 2026-10-19).
           CMUX_NEXT_STYLE_GUARDS: ${{ (github.base_ref == 'feat-cmux-next' || github.ref == 'refs/heads/feat-cmux-next') && 'warn' || 'fail' }}
@@ -561,6 +580,9 @@ jobs:
           if [ "$DURATIONS_UPLOAD" = failure ]; then
             echo "::warning::This shard's test durations did not upload; the next run's shard split misses them."
           fi
+          if [ "$HISTORY_UPLOAD" = failure ]; then
+            echo "::warning::This push's webview results did not upload; the base's flake history misses this run."
+          fi
           for gate in \
             "SOURCE_BUILD|Verify webviews source checks" \
             "BUNDLES|Build the web bundles" \
@@ -662,6 +684,19 @@ jobs:
         working-directory: webviews
         run: ./node_modules/.bin/playwright install --with-deps --only-shell chromium webkit
 
+      # The base's recent push verdicts (webviews-test-history), newest first. A test whose
+      # verdict flipped by itself there passed this replay by chance as often as not, so it
+      # only warns: agent-pane-suggested-tasks was charged to 9 of 40 sampled red pull
+      # request runs (2026-10-30). Read-only and optional: no history means no flakes.
+      - name: Restore the base's webview test history
+        continue-on-error: true
+        timeout-minutes: 2
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ runner.temp }}/webviews-test-history/history.json
+          key: webviews-test-history-${{ github.base_ref }}-none
+          restore-keys: webviews-test-history-${{ github.base_ref }}-
+
       - name: Run base reds and compare
         working-directory: webviews
         env:
@@ -690,7 +725,12 @@ jobs:
               env -u CMUX_WEB_TEST_SHARD -u CMUX_WEB_TEST_SHARD_COUNT -u CMUX_PR_CHANGED_FILES \
               bash "$RUNNER_TEMP/head-results/runner/run-webviews-tests.sh" || true
           fi
-          python3 ../scripts/ci/guards_new_reds.py --label "Webview test" "$HEAD_RESULTS" "$BASE_RESULTS"
+          runner="$RUNNER_TEMP/head-results/runner"
+          flaky="$RUNNER_TEMP/flaky-on-base"
+          python3 "$runner/webviews_test_history.py" flaky "$RUNNER_TEMP/webviews-test-history/history.json" \
+            --names "$HEAD_RESULTS" > "$flaky" || : > "$flaky"
+          if [ -s "$flaky" ]; then echo "flipped by themselves on recent base pushes:"; sed 's/^/  /' "$flaky"; fi
+          python3 "$runner/guards_new_reds.py" --label "Webview test" --flaky "$flaky" "$HEAD_RESULTS" "$BASE_RESULTS"
 
   # The runner capability map (.github/runners.json), for jobs that pick a
   # size from it. Resolves in parallel with web-subarea-scope.
@@ -820,6 +860,68 @@ jobs:
           path: ${{ runner.temp }}/webviews-durations/manifest.json
           key: webviews-test-durations-${{ github.run_id }}-${{ github.run_attempt }}
 
+  # The base's flake history: each feat-cmux-next push's webview verdicts (the shards'
+  # results), appended to the newest saved history and saved again under this run's key.
+  # react-apps-base-check reads it (webviews_test_history.py flaky). Pushes only: a pull
+  # request saves nothing, so its restore falls through to its base branch's history.
+  # Optional upkeep outside the required gate, like webviews-test-durations; two pushes
+  # finishing together may drop one of their runs, which costs one data point.
+  webviews-test-history:
+    needs: [web-subarea-scope, react-apps-check-shard]
+    if: ${{ always() && github.event_name == 'push' && inputs.web == 'true' && needs.web-subarea-scope.outputs.react_apps == 'true' && needs.react-apps-check-shard.result != 'skipped' }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
+    timeout-minutes: 5
+    steps:
+      - name: Checkout the history helper
+        id: history-checkout
+        continue-on-error: true
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          sparse-checkout: |
+            scripts/ci/webviews_test_history.py
+            scripts/ci/web_subareas.py
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+
+      - name: Download this push's results
+        id: history-download
+        if: ${{ steps.history-checkout.outcome == 'success' }}
+        continue-on-error: true
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          pattern: webviews-test-history-shard*
+          path: ${{ runner.temp }}/history-results
+
+      - name: Restore the history
+        if: ${{ steps.history-download.outcome == 'success' }}
+        continue-on-error: true
+        uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ runner.temp }}/webviews-test-history/history.json
+          key: webviews-test-history-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
+          restore-keys: webviews-test-history-${{ github.ref_name }}-
+
+      - name: Add this push to the history
+        id: history-record
+        if: ${{ steps.history-download.outcome == 'success' }}
+        continue-on-error: true
+        run: |
+          shards=("$RUNNER_TEMP"/history-results/webviews-test-history-shard*/webviews-head-results.json)
+          [ -e "${shards[0]}" ] || { echo "no shard recorded results"; exit 1; }
+          history="$RUNNER_TEMP/webviews-test-history/history.json"
+          python3 scripts/ci/webviews_test_history.py record "$history" "$history.new" \
+            --sha "$GITHUB_SHA" --attempt "$GITHUB_RUN_ATTEMPT" --time "$(git log -1 --format=%ct)" \
+            --inputs "$(python3 scripts/ci/webviews_test_history.py inputs)" "${shards[@]}"
+          mv "$history.new" "$history"
+
+      - name: Save the history
+        if: ${{ steps.history-record.outcome == 'success' }}
+        continue-on-error: true
+        uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+        with:
+          path: ${{ runner.temp }}/webviews-test-history/history.json
+          key: webviews-test-history-${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_attempt }}
+
   diff-sidecar-check:
     # Generated protocol checks and the sidecar tests only run for their owned inputs.
     needs: web-subarea-scope

.github/workflows/cmux-next.yml

@@ -1743,6 +1743,7 @@ jobs:
             # ci-guards.yml's `ci` group runs for feat-cmux-next too (cmux-next-guards.yml).
             python3 tests/test_feat_ci_guard_group.py
             python3 tests/test_guards_new_reds.py
+            python3 tests/test_webviews_test_history.py
             python3 tests/test_app_symbol_map.py
             bash scripts/cmux-next/tests/pin-cmux-tui.test.sh
             bash scripts/cmux-next/tests/bundle-acpmux-alias.test.sh

scripts/ci/guards_new_reds.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """Fail only on the guard steps a change turns red; report the base's own reds as warnings.
 
-    guards_new_reds.py [--label NAME] HEAD_RESULTS BASE_RESULTS
+    guards_new_reds.py [--label NAME] [--flaky FILE] HEAD_RESULTS BASE_RESULTS
 
 Each file is run_ci_guards.py --results output ({step name: passed}) from one runner: the head's
 checkout run on the head, then on the base (--root). A step red on the head that passed on the
@@ -13,6 +13,11 @@
 
 --label names what the annotations count (default: Guard); the webview tests use it with
 webviews_test_results.py's per-test results.
+
+--flaky names a file with one step a line (webviews_test_history.py flaky: the webview tests
+whose verdict flipped by itself on recent base pushes). A red the base replay passed that is in
+it is a warning, never the change's: one passing replay on the base proves nothing about a test
+that fails now and then there. A missing or unreadable file lists nothing.
 """
 from __future__ import annotations
 
@@ -29,10 +34,21 @@ def load(path: str) -> dict[str, bool] | None:
     return {str(k): bool(v) for k, v in found.items()} if isinstance(found, dict) else None
 
 
+def flaky_names(path: str) -> set[str]:
+    try:
+        return {line.strip() for line in Path(path).read_text(encoding="utf-8").splitlines() if line.strip()}
+    except OSError:
+        return set()
+
+
 def main(argv: list[str]) -> int:
-    label = "Guard"
-    if len(argv) == 4 and argv[0] == "--label":
-        label, argv = argv[1], argv[2:]
+    label, flaky = "Guard", set()
+    while len(argv) > 2 and argv[0] in ("--label", "--flaky"):
+        if argv[0] == "--label":
+            label = argv[1]
+        else:
+            flaky = flaky_names(argv[1])
+        argv = argv[2:]
     if len(argv) != 2:
         print(__doc__, file=sys.stderr)
         return 2
@@ -43,16 +59,21 @@ def main(argv: list[str]) -> int:
     if base is None:
         print(f"no base results in {argv[1]}: every red step counts as this change's")
         base = {}
-    new = sorted(step for step, ok in head.items() if not ok and base.get(step, True))
+    red = sorted(step for step, ok in head.items() if not ok and base.get(step, True))
+    new = [step for step in red if step not in flaky]
+    flaked = [step for step in red if step in flaky]
     old = sorted(step for step, ok in head.items() if not ok and base.get(step) is False)
     fixed = sorted(step for step, ok in head.items() if ok and base.get(step) is False)
     for step in old:
         print(f"::warning title={label} red on the base too::{step}")
+    for step in flaked:
+        print(f"::warning title={label} flaky on the base::{step}")
     for step in new:
         print(f"::error title={label} red on this change::{step}")
     if fixed:
         print("fixed by this change: " + "; ".join(fixed))
-    print(f"{len(new)} {label.lower()} steps red on this change, {len(old)} red on the base too, {len(fixed)} fixed")
+    print(f"{len(new)} {label.lower()} steps red on this change, {len(old)} red on the base too, "
+          f"{len(flaked)} flaky on the base, {len(fixed)} fixed")
     return 1 if new else 0
 
 

scripts/ci/pr_red_checks_comment.py

@@ -20,7 +20,19 @@
 its failure signature is its failed step names plus its failure annotations (minus "Process
 completed with exit code N"). Anything in the pull request's signature that the base's lacks is
 NEW and blocks; the same signature is a base red and does not; a base with no finished run of the
-check cannot excuse it. A new advisory check is one more entry
+check cannot excuse it.
+
+Two kinds of check are never compared with the base that way (2026-10-30: 22 pull requests merged webview
+test reds that broke the base, and this line called most of them base reds):
+  - web / react-apps-base-check (N) is already the base comparison: it replays the head's red webview test
+    files on the merge parent and fails only on tests red on the change alone. A push never runs it, so the
+    base has no run of it. Its red line names those tests, read from its "Webview test red on this
+    change" annotations.
+  - web / react-apps-check and web / Web status only gather other jobs. Their one failed step ("Check
+    webview test shards", "Check routed web jobs") failed the same way on a red base for any reason, so a
+    signature match said nothing. Each follows the lines of the red jobs it gathers (AGGREGATES): it blocks
+    when one of them blocks, when a job it gathers was cancelled, or when none of them is red. A red check
+    it does not gather (the size cap) never excuses it. A new advisory check is one more entry
 in TRACKED. The comment this used to keep on its own (LEGACY_MARKER) is deleted on the first write.
 
 OWN_CHECKS_GREEN=true (the caller's tracked jobs all passed or were skipped) reads only the status
@@ -55,6 +67,19 @@
     ("name", "cmux-next checks (god files, concurrency, crash safety, l10n)"),  # cmux-next.yml checks job
 )
 RED = {"failure", "timed_out", "action_required", "startup_failure"}
+# The webview tests' own base comparison (ci-web.yml), and its annotations' title for each new red test.
+BASE_CHECK = re.compile(r"^web / react-apps-base-check \(\d+\)$")
+NEW_TEST_TITLE = "Webview test red on this change"
+NAMED = 3  # new red tests a line names before "(+N more)"
+# Jobs that only gather other jobs' verdicts, each with the jobs it gathers (its `needs`, as check names): never
+# compared with the base on their own. react-apps-check comes first: Web status gathers it too.
+AGGREGATES = {
+    "web / react-apps-check": re.compile(r"^web / (?:react-apps-(?:base-)?check \(\d+\)|react-doctor)$"),
+    "web / Web status": re.compile(r"^web / (?!Web status$)"),  # it needs every routed web job
+}
+# A gathered job that ended with no verdict: the aggregate failed on it, and no base red explains that. Skipped is
+# not one: a replay is skipped after its shard fails (that shard's line decides), and an unrouted job skips.
+NO_VERDICT = {"cancelled", "stale"}
 GREEN_LINE = "web and size checks: green"
 BLOCK = "do not merge until green; see lane-rules"
 EXIT_CODE = re.compile(r"^Process completed with exit code \d+\.?$")
@@ -151,10 +176,41 @@ def pages(self, path, key=None):
         return out
 
 
+def annotations(gh, run):
+    return gh.call("GET", f"/repos/{gh.repo}/check-runs/{run['id']}/annotations?per_page=100") or []
+
+
 def run_signature(gh, run):
     steps = gh.call("GET", f"/repos/{gh.repo}/actions/jobs/{run['id']}").get("steps") or []
-    annotations = gh.call("GET", f"/repos/{gh.repo}/check-runs/{run['id']}/annotations?per_page=100") or []
-    return signature(steps, annotations)
+    return signature(steps, annotations(gh, run))
+
+
+def new_tests(found):
+    """The tests a base replay charged to the change, from its failure annotations' messages."""
+    names = [" ".join((a.get("message") or "").split()) for a in found
+             if a.get("annotation_level") == "failure" and a.get("title") == NEW_TEST_TITLE]
+    return list(dict.fromkeys(n for n in names if n))
+
+
+def base_check_line(name, tests):
+    """The line for a red base replay: the tests it charged to the change. It always blocks."""
+    if not tests:
+        return f"red: {name}; it named no new red test (setup or base replay failed; see its log); {BLOCK}"
+    more = f" (+{len(tests) - NAMED} more)" if len(tests) > NAMED else ""
+    return f"red: {name}; NEW webview test reds: {'; '.join(tests[:NAMED])}{more}; {BLOCK}"
+
+
+def aggregate_line(name, gathered, unfinished=()):
+    """An aggregate's line, from the lines of the red jobs it gathers: never a base red of its own. It blocks when one
+    of them blocks, when a job it gathers was cancelled or skipped (its own failure, which no base red explains), or
+    when none of them is red."""
+    if any(BLOCK in line for line in gathered):
+        return f"red: {name}; a job it gathers is red on this change (its line here); {BLOCK}"
+    if unfinished:
+        return f"red: {name}; a job it gathers ended without a verdict ({unfinished[0]}); {BLOCK}"
+    if not gathered:
+        return f"red: {name}; none of the jobs it gathers is red (one was skipped or cancelled); {BLOCK}"
+    return f"red: {name}; follows the jobs it gathers, whose reds are all on the base too; does not block"
 
 
 def base_lines(gh, runs, base_sha):
@@ -170,14 +226,27 @@ def base_lines(gh, runs, base_sha):
     mine = newest_runs(runs)
     lines = {}
     for name in red:
-        base_run = base.get(name)
+        if name in AGGREGATES:
+            continue
         try:
+            if BASE_CHECK.match(name):
+                lines[name] = base_check_line(name, new_tests(annotations(gh, mine[name])))
+                continue
+            base_run = base.get(name)
             pr_sig = run_signature(gh, mine[name])
             base_sig = run_signature(gh, base_run) if is_red(base_run) else []
         except (urllib.error.URLError, TimeoutError) as err:
             print(f"pr-red-checks: {name}: no signature ({err}); it blocks", file=sys.stderr)
             continue
         lines[name] = compare(name, pr_sig, base_run, base_sig, base_sha)
+    for name, gathers in AGGREGATES.items():
+        if name not in red:
+            continue
+        jobs = [n for n in red if n != name and gathers.match(n)]
+        unfinished = sorted(f"{n}: {r.get('conclusion')}" for n, r in mine.items()
+                            if n != name and gathers.match(n)
+                            and r.get("status") == "completed" and r.get("conclusion") in NO_VERDICT)
+        lines[name] = aggregate_line(name, [lines.get(n) or f"red: {n}; {BLOCK}" for n in jobs], unfinished)
     return lines
 
 

scripts/ci/web_subareas.py

@@ -185,6 +185,8 @@ def test_path_requires_db(path: str, repo_root: Path) -> bool:
     "scripts/ci/webviews-tests-new-reds.sh",
     "scripts/ci/webviews_test_results.py",
     "scripts/ci/guards_new_reds.py",
+    # and the base's flake history that keeps a test flipping there a warning.
+    "scripts/ci/webviews_test_history.py",
     # It also runs the test of the PR scoping of ui-rules and gallery coverage.
     "scripts/tests/test_pr_scoped_webviews_checks.py",
     "scripts/check-webviews-bun-version.sh",