diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index a935ca793104..67ac96b619c6 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -372,6 +372,10 @@ jobs: if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_merge_receipt.py + - name: Validate main full-suite failure reruns and bisects + if: ${{ matrix.group == 'ci' }} + run: python3 tests/test_ci_main_regression_bisect.py + - name: Validate CI queue janitor policy if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_queue_janitor.py diff --git a/.github/workflows/main-regression-bisect.yml b/.github/workflows/main-regression-bisect.yml new file mode 100644 index 000000000000..b5b9405ae7c7 --- /dev/null +++ b/.github/workflows/main-regression-bisect.yml @@ -0,0 +1,78 @@ +name: Main regression bisect + +# Checks what main_regression_attribution.py reported on the main full-suite +# issue against real runs: each new failure is rerun alone at the red run's +# head (a pass marks it flaky), and a reproduced failure with no single +# suspect is bisected over the commits in its range. A focused run takes 10 to +# 30 minutes, so each invocation advances every open check by one step and +# keeps its state in a comment on the issue; see +# scripts/ci/main_regression_bisect.py. MAIN_REGRESSION_BISECT_DISPATCHES_PER_DAY +# caps the focused runs dispatched in any 24 hours (default 24; 0 stops new +# ones). +on: + schedule: + - cron: "*/15 * * * *" + # Right after the report job posts a red run's new failures. + workflow_run: + workflows: [CI main full suite] + types: [completed] + branches: [main] + workflow_dispatch: + +permissions: {} + +env: + # Keep the display-name trigger checked against its stable workflow identity. + SOURCE_WORKFLOW_PATHS: .github/workflows/ci-main-full-suite.yml + +jobs: + advance: + name: Advance flake checks and bisects + # A CI main full suite run started by a push only dispatches; the one a + # completed CI run starts is the one that reports. + if: ${{ github.repository == 'manaflow-ai/cmux' && github.ref == 'refs/heads/main' && (github.event_name != 'workflow_run' || (github.event.workflow_run.event == 'workflow_run' && github.event.workflow_run.path == '.github/workflows/ci-main-full-suite.yml')) }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + # At most four dispatches, each bounded at eight minutes, plus reads. + timeout-minutes: 45 + concurrency: + group: main-regression-bisect + cancel-in-progress: false + permissions: + # Dispatches app-host-test-rerun.yml and test-e2e.yml. + actions: write + contents: read + issues: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 1 + persist-credentials: false + # dispatch-focused-test.py reads the test sources for selectors and + # test-e2e.yml for its default runner. + sparse-checkout: | + scripts/ci + cmuxTests + .github/workflows + + - name: Advance + env: + GH_TOKEN: ${{ github.token }} + # The job token cannot list repository variables; the dispatcher + # reads the pool choice from these instead. + CMUX_MACOS_RUNNER_TESTS: ${{ vars.MACOS_RUNNER_TESTS }} + CMUX_CI_E2E_LARGE_POOL_OVERFLOW: ${{ vars.CI_E2E_LARGE_POOL_OVERFLOW }} + CMUX_CI_PR_POOL_ORDER: ${{ vars.CI_PR_POOL_ORDER }} + CMUX_CI_PR_POOL_MAX_QUEUED: ${{ vars.CI_PR_POOL_MAX_QUEUED }} + CMUX_CI_PR_POOL_OWNED: ${{ vars.CI_PR_POOL_OWNED }} + CMUX_CI_OWNED_POOL_SLOTS: ${{ vars.CI_OWNED_POOL_SLOTS }} + CMUX_CMUX_CI_XCODE_APP_PR: ${{ vars.CMUX_CI_XCODE_APP_PR }} + CMUX_CI_E2E_OWNED_UI: ${{ vars.CI_E2E_OWNED_UI }} + PER_DAY: ${{ vars.MAIN_REGRESSION_BISECT_DISPATCHES_PER_DAY || '24' }} + run: | + set -euo pipefail + # Main's recent history without blobs: the dispatcher looks for an + # ancestor CI already compiled products for. + git fetch --no-tags --filter=blob:none --depth=1000 origin main + python3 scripts/ci/main_regression_bisect.py advance --max-dispatches-per-day "$PER_DAY" diff --git a/scripts/ci/main_regression_attribution.py b/scripts/ci/main_regression_attribution.py index 94e40cac770f..24af2ca3066a 100644 --- a/scripts/ci/main_regression_attribution.py +++ b/scripts/ci/main_regression_attribution.py @@ -27,8 +27,11 @@ by main_full_suite.py report --extra-section) and comments once on each suspect pull request, idempotent through a hidden marker keyed on the pull request and its failing test set, and once per commit range. A test tied between more than -MAX_PINGED_SUSPECTS pull requests is listed in the issue only. Nothing is -reverted or re-run here. +MAX_PINGED_SUSPECTS pull requests is listed in the issue only. The section +ends with a hidden data marker (DATA_PREFIX): the failures, their suspects, +and the commits in the range that can change an app-host test's outcome. +main_regression_bisect.py reads it to rerun and bisect each failure. Nothing +is reverted or re-run here. """ from __future__ import annotations @@ -47,6 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import main_full_suite as suite_run # noqa: E402 from merge_receipt import LABEL as UNVERIFIED_LABEL # noqa: E402 +from app_host_test_rerun import OUTSIDE_THE_APP, TEST_ROOT # noqa: E402 APP_HOST_JOB_RE = re.compile(r"app-host unit tests \((\d+)/\d+\)") ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") @@ -81,6 +85,11 @@ CATALOG = Path(__file__).resolve().parent / "app-host-known-failures.json" MARKER_PREFIX = "") +# One line of JSON main_regression_bisect.py reads back from the issue. +DATA_PREFIX = "" + + def issue_section( *, repo: str, @@ -303,6 +363,7 @@ def issue_section( prs: list[PullRequest], direct: list[str], no_baseline: Iterable[str] = (), + commits: list[str] | None = None, ) -> str: if previous is None: return "### New failures\n\nNo earlier full-suite run with every app-host shard finished to compare against." @@ -337,6 +398,10 @@ def issue_section( lines += ["", f"Pull requests merged in the range: " + (", ".join(f"#{pr.number}" for pr in prs) or "none")] if direct and not prs: lines.append("Commits without a merged pull request: " + ", ".join(short(sha) for sha in direct[:10])) + if commits is not None: + lines += ["", data_marker( + run=run, previous=previous, failures=failures, attributions=attributions, prs=prs, commits=commits, + )] return "\n".join(lines) @@ -607,16 +672,22 @@ def command_report(args: argparse.Namespace) -> int: prs: list[PullRequest] = [] direct: list[str] = [] attributions: dict[str, tuple[list[PullRequest], str]] = {} + commits: list[str] | None = None if previous and failures: - shas = git(args.root, "rev-list", f"{previous['head_sha']}..{run['head_sha']}").split() + range_ = f"{previous['head_sha']}..{run['head_sha']}" + shas = git(args.root, "rev-list", range_).split() prs, direct = merged_prs(shas, associated_prs(args.repo, shas), args.branch) if len(prs) > 1 or (prs and direct): rank_inputs(args.root, prs) attributions = {test: suspects_for(test, prs, direct) for test in failures} + commits = outcome_commits(git( + args.root, "log", "--first-parent", "--reverse", "--diff-merges=first-parent", + "--name-only", "--format=%x00%H", range_, + )) section = issue_section( repo=args.repo, run=run, previous=previous, failures=failures, - attributions=attributions, prs=prs, direct=direct, no_baseline=no_baseline, + attributions=attributions, prs=prs, direct=direct, no_baseline=no_baseline, commits=commits, ) print(section) if args.section_output: diff --git a/scripts/ci/main_regression_bisect.py b/scripts/ci/main_regression_bisect.py new file mode 100644 index 000000000000..5b3a8ee90a27 --- /dev/null +++ b/scripts/ci/main_regression_bisect.py @@ -0,0 +1,766 @@ +#!/usr/bin/env python3 +"""Rerun each new main failure, and bisect the ones attribution could not pin. + +main_regression_attribution.py names the pull requests suspected of each test +that newly fails in main's full suite, from the diff alone. That guess is +wrong for a flaky test, and gives no single answer for a tie or a failure no +diff explains. This checks it against real runs, one step per invocation, +because a dispatched focused run takes 10 to 30 minutes and a job must not sit +open for hours: + +1. Flake check. The first MAX_CHECKS_PER_RUN new failures of each red run + are run alone at that run's head with dispatch-focused-test.py, which reuses + the app-host products the red run compiled when it can. A pass marks the + failure flaky: the issue's row says so, and the suspect pull requests' + comments say it did not reproduce, so their authors are not blamed. +2. Bisect. A reproduced failure with no suspect or several is bisected over + the commits between the two full-suite runs that can change an app-host + test (the section's data marker lists them): the test runs at the midpoint, + the window halves, and so on until one commit is left. A commit where the + test does not exist yet counts as passing. A failure with one suspect is + left at "reproduced". At most MAX_ACTIVE_BISECTS run at once. +3. Verdict. The last commit left is itself probed unless it is the red run's + head, so a failure that only reproduces there (another runner pool, or a + cause outside the listed commits) ends unresolved instead of blaming it. + Then the issue row and the pull request's comment say "confirmed", with + the passing and failing run links; other suspects' comments say the bisect + cleared them. + +State lives in one comment on the tracking issue, as a hidden JSON marker the +next invocation resumes from. Every dispatch is recorded there, and at most +the --max-dispatches-per-day budget is spent in any 24 hours. Only comments +github-actions wrote are read, so no one else's comment can steer a dispatch. +Nothing is reverted. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import main_full_suite as suite_run # noqa: E402 +import main_regression_attribution as attribution # noqa: E402 + +DISPATCH_SCRIPT = Path(__file__).resolve().parent / "dispatch-focused-test.py" +STATE_PREFIX = "") +BOT_LOGINS = frozenset({"github-actions", "github-actions[bot]"}) +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +# What dispatch-focused-test.py accepts after cmuxTests/. +TEST_RE = re.compile(r"^[A-Za-z_]\w*/[A-Za-z_]\w*(?:\((?:[A-Za-z_]\w*:)*\))?$") +RUN_URL_RE = re.compile(r"^Run: (https://github\.com/[^\s]+/actions/runs/(\d+))\s*$", re.M) + +# Bounds on what one red streak can spend. +MAX_CHECKS_PER_RUN = 5 +MAX_ACTIVE_BISECTS = 2 +MAX_DISPATCHES_PER_INVOCATION = 4 +DEFAULT_MAX_DISPATCHES_PER_DAY = 24 +# A probe that errored (a runner or infrastructure failure, not the test) is +# retried once before the item gives up; any other result resets the count. +MAX_PROBE_ERRORS = 2 +# An item still open this long after it was queued is dropped. +ITEM_TTL = timedelta(days=3) +# Bounds on the state comment, which GitHub caps at 65,536 characters: a +# red run arriving while this many checks are open is not checked, and only +# the newest finished items are kept for the status table. +MAX_OPEN_ITEMS = 10 +MAX_FINISHED_ITEMS = 20 +KEEP_SEEN_RUNS = 200 +DISPATCH_TIMEOUT_SECONDS = 8 * 60 +# The step that runs the selected tests, in app-host-test-rerun.yml and +# test-e2e.yml. A failure anywhere else is not the test's verdict... +TEST_STEP = "Run selected tests" +# ...except test-e2e.yml's selector resolution, which fails when the built +# tests do not include the selector: the test does not exist at that commit. +RESOLVE_STEP = "Resolve selectors against the built tests" +# Kept under GitHub's 65,536-character comment limit with room for one more +# dispatch's worth of state. +STATE_LIMIT = 60_000 +FLAKY_TEXT = "likely flaky, not this pull request" + +OPEN_STATES = frozenset({"queued", "flake-check", "bisect-wait", "bisecting"}) +UPDATE_SEPARATOR = " · " +PR_UPDATE = " **Update:** " + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def stamp(moment: datetime) -> str: + return moment.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_stamp(text: str) -> datetime: + return datetime.strptime(text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + + +def short(sha: str) -> str: + return sha[:10] + + +def test_digest(test: str) -> str: + return hashlib.sha256(test.encode()).hexdigest()[:16] + + +# ---- markers ------------------------------------------------------------------------------ + + +def hidden_json(body: str, prefix: str) -> list[dict]: + """Every JSON object a `{...} -->` marker in the body carries.""" + found = [] + for line in (body or "").splitlines(): + line = line.strip() + if line.startswith(prefix) and line.endswith("-->"): + try: + value = json.loads(line[len(prefix):-3].strip()) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + found.append(value) + return found + + +def valid_data(data: Mapping[str, object]) -> bool: + """A data marker whose shas are safe to dispatch; each test is checked on its own.""" + commits = data.get("commits") + shas = [data.get("head"), data.get("prev"), *(commits or [])] + return ( + isinstance(data.get("run_id"), int) + and (commits is None or isinstance(commits, list)) + and isinstance(data.get("tests"), list) + and all(isinstance(sha, str) and SHA_RE.match(sha) for sha in shas) + ) + + +def dispatchable(entry: object) -> bool: + """A test entry dispatch-focused-test.py can run: Suite/method, not a nested suite.""" + return isinstance(entry, dict) and isinstance(entry.get("test"), str) and bool(TEST_RE.match(entry["test"])) + + +def empty_state() -> dict: + return {"v": 1, "items": [], "seen": [], "dispatches": []} + + +def render_state(state: Mapping[str, object], budget: int) -> str: + lines = [ + f"{STATE_PREFIX}{json.dumps(state, separators=(',', ':'))} -->", + "### Reruns and bisects of new failures", + "", + "Each new failure is run again alone at the red run's head; one that does not reproduce is " + "marked flaky. A reproduced failure with no single suspect is bisected over the commits in its " + "range. This comment is rewritten as runs finish.", + "", + ] + items = list(state.get("items") or []) + if items: + lines += ["Test | Red run | State", "--- | --- | ---"] + for item in items: + lines.append(f"`{item['test']}` | [{item['run']}]({item.get('run_url', '')}) | {item.get('note') or item['state']}") + else: + lines.append("Nothing checked yet.") + lines += ["", f"Dispatches in the last 24 hours: {len(state.get('dispatches') or [])} of {budget}."] + return "\n".join(lines) + + +# ---- edits to the issue section and pull request comments ---------------------------------- + + +def annotate_row(body: str, test: str, note: str) -> str: + """The "New since" table row for `test` with its suspect cell's update replaced by `note`.""" + lines = body.split("\n") + for index, line in enumerate(lines): + if not line.startswith(f"`{test}` | "): + continue + cells = line.split(" | ", 2) + if len(cells) != 3: + continue + cells[1] = cells[1].split(UPDATE_SEPARATOR, 1)[0] + UPDATE_SEPARATOR + note + lines[index] = " | ".join(cells) + return "\n".join(lines) + + +def annotate_pr_comment(body: str, test: str, note: str, header: str = "") -> str: + """A suspect comment with `test`'s line updated, and `header` placed under the marker.""" + lines = body.split("\n") + for index, line in enumerate(lines): + if line.startswith(f"- `{test}` "): + lines[index] = line.split(PR_UPDATE, 1)[0] + PR_UPDATE + note + if header: + top = [line for line in lines[1:] if line.startswith("**Update:** ")] + lines = [lines[0]] + [line for line in lines[1:] if line not in top] + lines.insert(1, f"**Update:** {header}") + return "\n".join(lines) + + +def comment_tests(body: str) -> list[str]: + return re.findall(r"^- `([^`]+)` ", body, re.M) + + +def suspect_comment_range(data: Mapping[str, object]) -> str: + return f"{short(str(data['prev']))}..{short(str(data['head']))}" + + +def is_suspect_comment(body: str, pr: int, range_: str) -> bool: + return any( + int(number) == pr and seen_range == range_ + for number, _, seen_range in attribution.MARKER_RE.findall(body or "") + ) + + +# ---- the state machine --------------------------------------------------------------------- + + +def new_items(state: dict, runs: Mapping[int, Mapping[str, object]], now: datetime) -> None: + """Queue a flake check for the first new failures of each red run not seen before.""" + seen = set(state["seen"]) + for run_id, data in sorted(runs.items()): + if run_id in seen: + continue + state["seen"].append(run_id) + room = MAX_OPEN_ITEMS - sum(1 for item in state["items"] if item["state"] in OPEN_STATES) + tests = [entry for entry in data.get("tests") or [] if dispatchable(entry)] + for entry in tests[:max(0, min(room, MAX_CHECKS_PER_RUN))]: + state["items"].append({ + "run": run_id, + "run_url": data.get("run_url"), + "test": entry["test"], + "suspects": list(entry.get("suspects") or []), + "state": "queued", + "created": stamp(now), + "probes": {}, + "errors": 0, + }) + state["seen"] = state["seen"][-KEEP_SEEN_RUNS:] + + +def points(data: Mapping[str, object]) -> list[str]: + """The known-good previous head, then each commit that can change the outcome, oldest first.""" + return [str(data["prev"]), *[str(sha) for sha in data.get("commits") or []]] + + +@dataclass +class Event: + """Something the issue and pull requests should now say about one item.""" + + kind: str # flaky, reproduced, bisecting, confirmed, unresolved, error + item: dict + + +def culprit_checked(item: dict, data: Mapping[str, object]) -> bool: + """Whether the window's last commit is known to fail: probed there, or it is the red run's head.""" + culprit = points(data)[item["hi"]] + return culprit == data["head"] or (item["probes"].get(culprit) or {}).get("result") == "fail" + + +def conclude(item: dict, data: Mapping[str, object]) -> Event: + """The verdict once the window is one commit wide and its last commit fails.""" + window = points(data) + culprit = window[item["hi"]] + good = window[item["lo"]] + culprit_probe = item["probes"].get(culprit) or item["probes"][str(data["head"])] + item["culprit"] = { + "sha": culprit, + "pr": (data.get("prs") or {}).get(culprit), + "pass_sha": good, + "pass_url": (item["probes"].get(good) or {}).get("url") or data.get("prev_run_url"), + "fail_url": culprit_probe.get("url"), + } + item["state"] = "confirmed" + pr = item["culprit"]["pr"] + who = f"#{pr}" if pr else f"direct commit `{short(culprit)}`" + item["note"] = ( + f"confirmed: {who} (passes at `{short(good)}` [run]({item['culprit']['pass_url']}), " + f"fails at `{short(culprit)}` [run]({item['culprit']['fail_url']}))" + ) + return Event("confirmed", item) + + +def settle(item: dict, data: Mapping[str, object]) -> Event | None: + if item["hi"] - item["lo"] <= 1 and culprit_checked(item, data): + return conclude(item, data) + item["note"] = f"bisecting: {item['hi'] - item['lo']} commits left" + return None + + +def after_reproduced(item: dict, data: Mapping[str, object]) -> Event: + """Decide what a reproduced failure needs next.""" + window = points(data) + head_url = item["rerun_url"] + if data.get("commits") is None: + item["state"] = "reproduced" + item["note"] = f"reproduced ([rerun]({head_url})); too many commits in the range to bisect" + return Event("reproduced", item) + if len(window) < 2: + item["state"] = "unresolved" + item["note"] = ( + f"reproduced ([rerun]({head_url})), but no commit in the range changes the app or its tests" + ) + return Event("unresolved", item) + item["lo"], item["hi"] = 0, len(window) - 1 + if len(window) == 2 and culprit_checked(item, data): + return conclude(item, data) + if len(item["suspects"]) == 1 and len(window) > 2: + item["state"] = "reproduced" + item["note"] = f"reproduced ([rerun]({head_url}))" + return Event("reproduced", item) + item["state"] = "bisect-wait" + item["note"] = f"reproduced ([rerun]({head_url})); waiting to bisect {len(window) - 1} commit(s)" + return Event("reproduced", item) + + +def record_result(item: dict, data: Mapping[str, object], sha: str, result: str) -> Event | None: + """Apply one finished probe to the item.""" + probe = item["probes"][sha] + if result == "absent": + # The test does not exist at this commit, so it is not failing there. + # At the red run's head that cannot be, so it is an error. + result = "pass" if item["state"] == "bisecting" else "error" + probe["result"] = result + item.pop("pending", None) + if result == "error": + item["errors"] = item.get("errors", 0) + 1 + if item["errors"] >= MAX_PROBE_ERRORS: + item["state"] = "error" + item["note"] = f"could not be rerun ([last run]({probe['url']}))" + return Event("error", item) + return None + item["errors"] = 0 + if item["state"] == "flake-check": + item["rerun_url"] = probe["url"] + if result == "pass": + item["state"] = "flaky" + item["note"] = f"did not reproduce on a rerun at the same commit ([run]({probe['url']})); likely flaky" + return Event("flaky", item) + return after_reproduced(item, data) + window = points(data) + index = window.index(sha) + if result == "fail": + item["hi"] = index + elif index == item["hi"]: + item["state"] = "unresolved" + item["note"] = ( + f"fails at the red run's head ([rerun]({item['rerun_url']})) but passes at `{short(sha)}` " + f"([run]({probe['url']})), the last commit in the range that changes the app or its tests; " + "the cause is outside those commits, or it only fails on the red run's runner" + ) + return Event("unresolved", item) + else: + item["lo"] = index + return settle(item, data) + + +def next_probe(item: dict, data: Mapping[str, object]) -> str | None: + """The commit this item needs a run at now, or None.""" + if item.get("pending"): + return None + if item["state"] in ("queued", "flake-check"): + return str(data["head"]) + if item["state"] == "bisecting": + window = points(data) + if item["hi"] - item["lo"] > 1: + return window[(item["lo"] + item["hi"]) // 2] + return window[item["hi"]] + return None + + +class Budget: + """Dispatches left in the rolling day and in this invocation.""" + + def __init__(self, state: dict, per_day: int, now: datetime): + cutoff = now - timedelta(days=1) + state["dispatches"] = [text for text in state.get("dispatches") or [] if parse_stamp(text) > cutoff] + self.state, self.per_day, self.now = state, per_day, now + self.this_invocation = 0 + + def available(self) -> bool: + return ( + len(self.state["dispatches"]) < self.per_day + and self.this_invocation < MAX_DISPATCHES_PER_INVOCATION + ) + + def spend(self) -> None: + self.state["dispatches"].append(stamp(self.now)) + self.this_invocation += 1 + + +def advance( + state: dict, + runs: Mapping[int, Mapping[str, object]], + *, + poll: Callable[[int], str], + dispatch: Callable[[str, str], tuple[int, str] | None], + per_day: int, + now: datetime, +) -> list[Event]: + """One step for every open item: collect finished probes, start bisects, dispatch the next runs. + + `poll(run_id)` answers pending, pass, fail, absent or error. + `dispatch(test, sha)` returns (run id, url), or None when it failed. + """ + events: list[Event] = [] + new_items(state, runs, now) + budget = Budget(state, per_day, now) + for item in state["items"]: + if item["state"] not in OPEN_STATES: + continue + data = runs.get(item["run"]) + if data is None or now - parse_stamp(item["created"]) > ITEM_TTL: + item["state"] = "expired" + item["note"] = "dropped: its red run's report is gone or the check ran too long" + continue + pending = item.get("pending") + if pending: + result = poll(int(item["probes"][pending]["run_id"])) + if result != "pending": + event = record_result(item, data, pending, result) + if event: + events.append(event) + active = sum(1 for item in state["items"] if item["state"] == "bisecting") + for item in state["items"]: + if item["state"] == "bisect-wait" and active < MAX_ACTIVE_BISECTS: + item["state"] = "bisecting" + item["note"] = f"bisecting: {item['hi'] - item['lo']} commits left" + active += 1 + events.append(Event("bisecting", item)) + for item in state["items"]: + if item["state"] not in OPEN_STATES or item["run"] not in runs: + continue + sha = next_probe(item, runs[item["run"]]) + if sha is None or not budget.available(): + continue + if len(render_state(state, per_day)) > STATE_LIMIT: + print("::warning::The state comment is near GitHub's size limit; dispatching nothing more.", file=sys.stderr) + break + budget.spend() + started = dispatch(item["test"], sha) + if started is None: + item["errors"] = item.get("errors", 0) + 1 + if item["errors"] >= MAX_PROBE_ERRORS: + item["state"] = "error" + item["note"] = "could not be dispatched; see the bisect job's log" + events.append(Event("error", item)) + continue + run_id, url = started + if item["state"] == "queued": + item["state"] = "flake-check" + item["note"] = f"rerunning at the red run's head ([run]({url}))" + item["probes"][sha] = {"run_id": run_id, "url": url, "result": "pending"} + item["pending"] = sha + finished = [item for item in state["items"] if item["state"] not in OPEN_STATES] + for item in finished: + # The note and culprit carry every link a finished item still shows. + for key in ("probes", "pending", "lo", "hi", "errors"): + item.pop(key, None) + keep = {id(item) for item in finished[-MAX_FINISHED_ITEMS:]} + state["items"] = [item for item in state["items"] if item["state"] in OPEN_STATES or id(item) in keep] + return events + + +def classify(run: Mapping[str, object], failed_steps: Callable[[], list[str]]) -> str: + """pending, pass, fail (the test step failed), absent (no such test built) or error.""" + if run.get("status") != "completed": + return "pending" + if run.get("conclusion") == "success": + return "pass" + if run.get("conclusion") == "failure": + failed = failed_steps() + if TEST_STEP in failed: + return "fail" + if RESOLVE_STEP in failed: + return "absent" + return "error" + + +# ---- what each event changes --------------------------------------------------------------- + + +@dataclass +class Comment: + """A comment github-actions wrote, or the issue body (id None, kind issue).""" + + id: int | None + body: str + kind: str = "comment" + + +def section_edits( + events: Iterable[Event], runs: Mapping[int, Mapping[str, object]], nodes: list[Comment], +) -> dict[int | None, str]: + """New bodies for the issue comments whose "New since" rows changed, by comment id.""" + edits: dict[int | None, str] = {} + for event in events: + data = runs.get(event.item["run"]) + if not data: + continue + for node in nodes: + body = edits.get(node.id, node.body) + if not any(found.get("run_id") == event.item["run"] for found in hidden_json(body, attribution.DATA_PREFIX)): + continue + changed = annotate_row(body, event.item["test"], event.item["note"]) + if changed != body: + edits[node.id] = changed + return edits + + +def pr_updates( + event: Event, data: Mapping[str, object], comments: Callable[[int], list[Comment]], +) -> tuple[dict[int, str], list[tuple[int, str]]]: + """(comment id -> new body, (pull request, new comment) to post) for one event.""" + item = event.item + test = item["test"] + range_ = suspect_comment_range(data) + edits: dict[int, str] = {} + posts: list[tuple[int, str]] = [] + + def edit_suspects(numbers: Iterable[int], note: str, header_when_all: str = "") -> int: + """Update `test`'s line in each suspect comment that lists it; returns how many list it.""" + listed = 0 + for number in numbers: + for comment in comments(number): + if comment.id is None or not is_suspect_comment(comment.body, number, range_): + continue + if test not in comment_tests(comment.body): + continue + listed += 1 + body = annotate_pr_comment(comment.body, test, note) + if header_when_all and all( + FLAKY_TEXT in line for line in body.split("\n") if line.startswith("- `") + ): + body = annotate_pr_comment(body, test, note, header=header_when_all) + if body != comment.body: + edits[comment.id] = body + return listed + + if event.kind == "flaky": + edit_suspects( + item["suspects"], + f"did not reproduce on a rerun at the same commit ([run]({item['rerun_url']})); {FLAKY_TEXT}", + header_when_all="none of these failures reproduced on a rerun, so they look flaky. " + "Nothing to do here unless you know otherwise.", + ) + elif event.kind == "reproduced" and item["state"] == "reproduced": + edit_suspects( + item["suspects"], + f"reproduced on a rerun at the same commit ([run]({item['rerun_url']}))", + ) + elif event.kind == "confirmed": + culprit = item["culprit"] + pr = culprit.get("pr") + links = ( + f"passes at `{short(culprit['pass_sha'])}` ([run]({culprit['pass_url']})), " + f"fails at `{short(culprit['sha'])}` ([run]({culprit['fail_url']}))" + ) + if pr: + if not edit_suspects([pr], f"**confirmed** by bisect: {links}"): + marker = f"" + if not any(marker in comment.body for comment in comments(pr)): + posts.append((pr, "\n".join([ + marker, + f"`{test}` newly fails in [main's full suite]({data.get('run_url')}), and a bisect over " + f"the commits since [the previous full-suite run]({data.get('prev_run_url')}) " + f"points at this pull request: the test {links}.", + "", + "Please fix forward or revert. If this looks wrong, say so here; a flaky test " + "can mislead a bisect.", + ]))) + cleared = [number for number in item["suspects"] if number != pr] + who = f"#{pr}" if pr else f"commit `{short(culprit['sha'])}`" + edit_suspects(cleared, f"a bisect points at {who} instead ({links}); this pull request is cleared") + return edits, posts + + +# ---- I/O --------------------------------------------------------------------------------- + + +def gh(args: list[str]) -> str: + return subprocess.run(["gh", *args], check=True, capture_output=True, text=True).stdout + + +def graphql(query: str, **variables: object) -> dict: + args = ["api", "graphql", "-f", f"query={query}"] + for key, value in variables.items(): + if value is None: + continue + args += ["-F" if isinstance(value, int) else "-f", f"{key}={value}"] + return json.loads(gh(args))["data"] + + +def is_bot(node: Mapping[str, object]) -> bool: + return str(((node.get("author") or {}) or {}).get("login") or "") in BOT_LOGINS + + +def issue_comments(repo: str, number: int) -> tuple[Comment | None, list[Comment]]: + """(the issue body if github-actions wrote it, its github-actions comments), oldest first.""" + owner, name = repo.split("/", 1) + query = ( + "query($owner: String!, $name: String!, $number: Int!, $after: String) { repository(owner: $owner, name: $name) " + "{ issue(number: $number) { body author { login } comments(first: 100, after: $after) " + "{ pageInfo { hasNextPage endCursor } nodes { databaseId body author { login } } } } } }" + ) + after = None + body: Comment | None = None + found: list[Comment] = [] + while True: + issue = graphql(query, owner=owner, name=name, number=number, after=after)["repository"]["issue"] + if body is None and is_bot(issue): + body = Comment(None, str(issue.get("body") or ""), "issue") + page = issue["comments"] + found += [Comment(int(node["databaseId"]), str(node.get("body") or "")) for node in page["nodes"] if is_bot(node)] + if not page["pageInfo"]["hasNextPage"]: + return body, found + after = page["pageInfo"]["endCursor"] + + +def pr_comments(repo: str, number: int) -> list[Comment]: + owner, name = repo.split("/", 1) + query = ( + "query($owner: String!, $name: String!, $number: Int!) { repository(owner: $owner, name: $name) " + "{ pullRequest(number: $number) { comments(last: 100) { nodes { databaseId body author { login } } } } } }" + ) + data = graphql(query, owner=owner, name=name, number=number) + nodes = ((data["repository"].get("pullRequest") or {}).get("comments") or {}).get("nodes") or [] + return [Comment(int(node["databaseId"]), str(node.get("body") or "")) for node in nodes if is_bot(node)] + + +def poll_run(repo: str, run_id: int) -> str: + try: + run = json.loads(gh(["api", f"repos/{repo}/actions/runs/{run_id}"])) + + def failed_steps() -> list[str]: + jobs = json.loads(gh(["api", f"repos/{repo}/actions/runs/{run_id}/jobs?filter=latest&per_page=100"])) + return [ + str(step.get("name")) + for job in jobs.get("jobs") or [] + for step in job.get("steps") or [] + if step.get("conclusion") == "failure" + ] + + return classify(run, failed_steps) + except (subprocess.CalledProcessError, json.JSONDecodeError) as error: + print(f"::warning::Could not read run {run_id}: {error}", file=sys.stderr) + return "pending" + + +def dispatch_run(test: str, sha: str) -> tuple[int, str] | None: + # --force: the dispatcher refuses a selector that already failed at a + # commit, which is the question asked here; the state already keeps one + # run per item in flight. + command = [sys.executable, str(DISPATCH_SCRIPT), f"cmuxTests/{test}", "--ref", sha, "--force"] + print(f"Dispatching {test} at {sha}", flush=True) + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=DISPATCH_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + print(f"::warning::Dispatching {test} at {sha} timed out", file=sys.stderr) + return None + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + match = RUN_URL_RE.search(result.stdout) + if result.returncode != 0 or not match: + print(f"::warning::Could not dispatch {test} at {sha} (exit {result.returncode})", file=sys.stderr) + return None + return int(match.group(2)), match.group(1) + + +def command_advance(args: argparse.Namespace) -> int: + issue = suite_run.open_issue(args.repo) + if issue is None: + print("Main's full suite is not red; nothing to check.") + return 0 + number = int(issue["number"]) + body, comments = issue_comments(args.repo, number) + nodes = ([body] if body else []) + comments + runs: dict[int, dict] = {} + for node in nodes: + for data in hidden_json(node.body, attribution.DATA_PREFIX): + if valid_data(data): + runs[int(data["run_id"])] = data + state_node = next((node for node in comments if hidden_json(node.body, STATE_PREFIX)), None) + state = hidden_json(state_node.body, STATE_PREFIX)[0] if state_node else empty_state() + for key, value in empty_state().items(): + state.setdefault(key, value) + + events = advance( + state, runs, + poll=lambda run_id: poll_run(args.repo, run_id), + dispatch=dispatch_run if not args.dry_run else lambda test, sha: None, + per_day=args.max_dispatches_per_day, + now=now_utc(), + ) + for event in events: + print(f"{event.item['test']} ({event.item['run']}): {event.kind}: {event.item.get('note', '')}") + + state_body = render_state(state, args.max_dispatches_per_day) + cached: dict[int, list[Comment]] = {} + + def comments_of(pr: int) -> list[Comment]: + if pr not in cached: + cached[pr] = pr_comments(args.repo, pr) + return cached[pr] + + # The state goes first: it records the runs just dispatched, and losing it + # would dispatch them again. The edits after it are idempotent. + writes: list[list[str]] = [] + if state_node: + if state_node.body != state_body: + writes.append(["api", "-X", "PATCH", f"repos/{args.repo}/issues/comments/{state_node.id}", "-f", f"body={state_body}"]) + elif state["items"]: + writes.append(["api", f"repos/{args.repo}/issues/{number}/comments", "-f", f"body={state_body}"]) + for comment_id, text in section_edits(events, runs, nodes).items(): + if comment_id is None: + writes.append(["api", "-X", "PATCH", f"repos/{args.repo}/issues/{number}", "-f", f"body={text}"]) + else: + writes.append(["api", "-X", "PATCH", f"repos/{args.repo}/issues/comments/{comment_id}", "-f", f"body={text}"]) + for event in events: + data = runs.get(event.item["run"]) + if not data: + continue + edits, posts = pr_updates(event, data, comments_of) + for comment_id, text in edits.items(): + writes.append(["api", "-X", "PATCH", f"repos/{args.repo}/issues/comments/{comment_id}", "-f", f"body={text}"]) + for pr, text in posts: + writes.append(["api", f"repos/{args.repo}/issues/{pr}/comments", "-f", f"body={text}"]) + + if args.dry_run: + for write in writes: + print("--- would run gh " + " ".join(write[:4])) + return 0 + failed = False + for write in writes: + try: + gh(write) + except subprocess.CalledProcessError as error: + failed = True + print(f"::warning::gh {' '.join(write[:4])} failed: {(error.stderr or '').strip()}", file=sys.stderr) + return 1 if failed else 0 + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--repo", default=os.environ.get("GITHUB_REPOSITORY", "")) + commands = parser.add_subparsers(dest="command", required=True) + step = commands.add_parser("advance", help="one step of every open flake check and bisect") + step.add_argument( + "--max-dispatches-per-day", type=int, default=DEFAULT_MAX_DISPATCHES_PER_DAY, + help="focused runs dispatched in any 24 hours; 0 stops new dispatches", + ) + step.add_argument("--dry-run", action="store_true", help="dispatch and write nothing") + step.set_defaults(handler=command_advance) + args = parser.parse_args(argv) + if not args.repo: + parser.error("--repo or GITHUB_REPOSITORY is required") + return args.handler(args) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 466e7f5511c0..0b64c1bfcd5b 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -1233,3 +1233,7 @@ lane = "linux-guard" [[test]] path = "tests/test_ci_merge_receipt.py" lane = "linux-guard" + +[[test]] +path = "tests/test_ci_main_regression_bisect.py" +lane = "linux-guard" diff --git a/tests/test_ci_main_regression_attribution.py b/tests/test_ci_main_regression_attribution.py index c02bc3b6da14..8ac099dd42d4 100644 --- a/tests/test_ci_main_regression_attribution.py +++ b/tests/test_ci_main_regression_attribution.py @@ -1,6 +1,7 @@ """New failures on main's full suite, their suspect pull requests, and the report text.""" import importlib.util +import json import pathlib import sys import unittest @@ -183,6 +184,16 @@ def test_a_direct_push_in_the_range_needs_the_pull_request_to_reach_the_suite(se reaches = pr(1, reached={"Suite"}) self.assertEqual(MODULE.suspects_for("Suite/test()", [reaches], ["abc"])[0], [reaches]) + def test_only_commits_that_can_change_an_app_host_test_are_bisected(self): + log = "\0".join([ + "", f"{'a' * 40}\n\nSources/App.swift\ndocs/x.md\n", + f"{'b' * 40}\n\ndocs/readme.md\nweb/app/page.tsx\nscripts/ci/foo.py\n", + f"{'c' * 40}\n\ncmuxTests/FooTests.swift\n", + f"{'d' * 40}\n\n", + f"{'e' * 40}\n\nPackages/macOS/Kit/Sources/K.swift\n", + ]) + self.assertEqual(MODULE.outcome_commits(log), ["a" * 40, "c" * 40, "e" * 40]) + def test_overlay_reads_changed_files_at_the_merge(self): files = {"Sources/A.swift": "old", "Sources/B.swift": "b", "cmuxTests/T.swift": "t"} self.assertEqual( @@ -242,6 +253,37 @@ def test_issue_section_lists_each_new_failure_with_suspects_and_jobs(self): self.assertIn("`T/y()` | #2 (changes code the suite names)", text) self.assertIn("`U/z()` | unattributed", text) + def test_issue_section_carries_the_data_the_bisect_reads(self): + text = MODULE.issue_section( + repo=REPO, run=run(), previous=self.previous, failures=self.failures, + attributions=self.attributions, prs=[self.a, self.b], direct=[], commits=["c" * 40], + ) + marker = [line for line in text.splitlines() if line.startswith(MODULE.DATA_PREFIX)] + self.assertEqual(len(marker), 1) + data = json.loads(marker[0][len(MODULE.DATA_PREFIX):-3]) + self.assertEqual((data["run_id"], data["head"], data["prev"]), (2, HEAD, PREV)) + self.assertEqual(data["commits"], ["c" * 40]) + self.assertEqual(data["tests"][0], {"test": "S/x()", "suspects": [1, 2], "how": "changes code the suite names"}) + self.assertEqual(data["tests"][2]["suspects"], []) + self.assertEqual(data["prs"], {}) # neither merge commit is one the bisect probes + merged = [pr(n) for n in range(MODULE.MAX_BISECT_COMMITS + 1)] + for each in merged: + each.merge_sha = f"{each.number:040x}" + long = MODULE.issue_section( + repo=REPO, run=run(), previous=self.previous, failures=self.failures, + attributions=self.attributions, prs=merged, direct=[], + commits=[each.merge_sha for each in merged], + ) + marker = [line for line in long.splitlines() if line.startswith(MODULE.DATA_PREFIX)][0] + long_data = json.loads(marker[len(MODULE.DATA_PREFIX):-3]) + self.assertIsNone(long_data["commits"]) + self.assertEqual(long_data["prs"], {}) + without = MODULE.issue_section( + repo=REPO, run=run(), previous=self.previous, failures=self.failures, + attributions=self.attributions, prs=[self.a, self.b], direct=[], + ) + self.assertNotIn(MODULE.DATA_PREFIX, without) + def test_issue_section_lists_failures_without_a_baseline(self): text = MODULE.issue_section( repo=REPO, run=run(), previous=self.previous, failures={}, attributions={}, prs=[], direct=[], diff --git a/tests/test_ci_main_regression_bisect.py b/tests/test_ci_main_regression_bisect.py new file mode 100644 index 000000000000..1fa0a5a14800 --- /dev/null +++ b/tests/test_ci_main_regression_bisect.py @@ -0,0 +1,432 @@ +"""Flake checks and bisects of new main full-suite failures, driven by fixtures (no network).""" + +import importlib.util +import pathlib +import sys +import unittest +from datetime import datetime, timedelta, timezone + +ROOT = pathlib.Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/ci/main_regression_bisect.py" +WORKFLOW = ROOT / ".github/workflows/main-regression-bisect.yml" +sys.path.insert(0, str(SCRIPT.parent)) +SPEC = importlib.util.spec_from_file_location("main_regression_bisect", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) +attribution = MODULE.attribution + +NOW = datetime(2026, 9, 25, 12, 0, tzinfo=timezone.utc) +PREV = "0" * 40 +HEAD = "f" * 40 +C = [f"{index:x}" * 40 for index in range(1, 8)] # seven commits between PREV and HEAD + + +def data(run_id=7, tests=(("S/x()", [1, 2]),), commits=None, prs=None): + commits = list(C if commits is None else commits) + return { + "v": 1, "run_id": run_id, "run_url": f"https://run/{run_id}", "head": HEAD, "prev": PREV, + "prev_run_url": "https://run/prev", + "tests": [{"test": test, "suspects": suspects, "how": "edits the suite"} for test, suspects in tests], + "prs": prs if prs is not None else {sha: 100 + index for index, sha in enumerate(commits)}, + "commits": commits, + } + + +class Harness: + """Fake GitHub: `outcome(sha)` decides what a dispatched run at a commit reports.""" + + def __init__(self, outcome): + self.outcome = outcome + self.dispatched = [] + self.runs = {} + + def dispatch(self, test, sha): + run_id = 1000 + len(self.dispatched) + self.dispatched.append((test, sha)) + self.runs[run_id] = sha + return run_id, f"https://probe/{run_id}" + + def poll(self, run_id): + return self.outcome(self.runs[run_id]) + + def step(self, state, runs, now=NOW, per_day=24): + return MODULE.advance(state, runs, poll=self.poll, dispatch=self.dispatch, per_day=per_day, now=now) + + +def drive(harness, state, runs, steps=12): + events = [] + for _ in range(steps): + events += harness.step(state, runs) + return events + + +class StateMachineTests(unittest.TestCase): + def test_a_failure_that_passes_on_rerun_is_flaky(self): + harness = Harness(lambda sha: "pass") + state, runs = MODULE.empty_state(), {7: data()} + events = drive(harness, state, runs) + self.assertEqual(harness.dispatched, [("S/x()", HEAD)]) + self.assertEqual([event.kind for event in events], ["flaky"]) + self.assertEqual(state["items"][0]["state"], "flaky") + self.assertIn("did not reproduce", state["items"][0]["note"]) + + def test_a_tied_failure_is_bisected_to_the_first_bad_commit(self): + first_bad = 4 # C[4] broke it + harness = Harness(lambda sha: "fail" if sha == HEAD or (sha in C and C.index(sha) >= first_bad) else "pass") + state, runs = MODULE.empty_state(), {7: data()} + events = drive(harness, state, runs) + item = state["items"][0] + self.assertEqual(item["state"], "confirmed") + self.assertEqual(item["culprit"]["sha"], C[first_bad]) + self.assertEqual(item["culprit"]["pr"], 104) + self.assertEqual(item["culprit"]["pass_sha"], C[first_bad - 1]) + self.assertTrue(item["culprit"]["pass_url"].startswith("https://probe/")) + self.assertEqual([event.kind for event in events], ["reproduced", "bisecting", "confirmed"]) + # One flake check, then log2(8) probes, never the endpoints. + probed = [sha for _, sha in harness.dispatched] + self.assertEqual(probed[0], HEAD) + self.assertEqual(len(probed), 4) + self.assertNotIn(PREV, probed) + + def test_the_first_commit_in_the_range_links_the_baseline_run_as_passing(self): + harness = Harness(lambda sha: "fail") + state, runs = MODULE.empty_state(), {7: data()} + drive(harness, state, runs) + item = state["items"][0] + self.assertEqual(item["culprit"]["sha"], C[0]) + self.assertEqual(item["culprit"]["pass_url"], "https://run/prev") + + def test_a_range_whose_only_commit_is_the_head_is_confirmed_without_a_bisect(self): + harness = Harness(lambda sha: "fail") + state, runs = MODULE.empty_state(), {7: data(tests=(("S/x()", [5]),), commits=[HEAD])} + events = drive(harness, state, runs) + self.assertEqual([event.kind for event in events], ["confirmed"]) + self.assertEqual(len(harness.dispatched), 1) + culprit = state["items"][0]["culprit"] + self.assertEqual((culprit["sha"], culprit["pr"]), (HEAD, 100)) + self.assertEqual(culprit["fail_url"], "https://probe/1000") # the rerun at head + + def test_a_single_commit_other_than_the_head_is_probed_before_it_is_confirmed(self): + harness = Harness(lambda sha: "fail") + state, runs = MODULE.empty_state(), {7: data(tests=(("S/x()", [5]),), commits=[C[2]])} + events = drive(harness, state, runs) + self.assertEqual([sha for _, sha in harness.dispatched], [HEAD, C[2]]) + self.assertEqual(events[-1].kind, "confirmed") + self.assertEqual(state["items"][0]["culprit"]["fail_url"], "https://probe/1001") + + def test_a_failure_only_the_red_runs_head_shows_is_unresolved_not_blamed(self): + # Fails at the head (another pool, or a cause outside the listed + # commits) and passes at every listed commit, including the last. + harness = Harness(lambda sha: "fail" if sha == HEAD else "pass") + state, runs = MODULE.empty_state(), {7: data()} + events = drive(harness, state, runs) + self.assertEqual(events[-1].kind, "unresolved") + self.assertIn(C[-1], [sha for _, sha in harness.dispatched]) + self.assertNotIn("culprit", state["items"][0]) + + def test_a_commit_without_the_test_counts_as_passing(self): + # The test was added at C[2] and broken at C[5]. + def outcome(sha): + if sha in C and C.index(sha) < 2: + return "absent" + return "fail" if sha == HEAD or (sha in C and C.index(sha) >= 5) else "pass" + harness = Harness(outcome) + state, runs = MODULE.empty_state(), {7: data()} + drive(harness, state, runs, steps=20) + self.assertEqual(state["items"][0]["culprit"]["sha"], C[5]) + + def test_a_missing_test_at_the_head_is_an_error(self): + harness = Harness(lambda sha: "absent") + state, runs = MODULE.empty_state(), {7: data()} + self.assertEqual([event.kind for event in drive(harness, state, runs)], ["error"]) + + def test_a_range_too_long_to_list_is_not_bisected(self): + harness = Harness(lambda sha: "fail") + runs = {7: {**data(), "commits": None}} + state = MODULE.empty_state() + events = drive(harness, state, runs) + self.assertEqual([event.kind for event in events], ["reproduced"]) + self.assertIn("too many commits", state["items"][0]["note"]) + + def test_a_nested_suite_is_skipped_without_dropping_the_run(self): + harness = Harness(lambda sha: "pending") + state, runs = MODULE.empty_state(), {7: data(tests=(("Outer/Inner/t()", [1]), ("S/x()", [1])))} + harness.step(state, runs) + self.assertTrue(MODULE.valid_data(runs[7])) + self.assertEqual([item["test"] for item in state["items"]], ["S/x()"]) + + def test_a_single_suspect_is_left_at_reproduced(self): + harness = Harness(lambda sha: "fail") + state, runs = MODULE.empty_state(), {7: data(tests=(("S/x()", [3]),))} + events = drive(harness, state, runs) + self.assertEqual([event.kind for event in events], ["reproduced"]) + self.assertEqual(state["items"][0]["state"], "reproduced") + self.assertEqual(len(harness.dispatched), 1) + + def test_no_commit_that_matters_leaves_it_unresolved(self): + harness = Harness(lambda sha: "fail") + state, runs = MODULE.empty_state(), {7: data(commits=[])} + events = drive(harness, state, runs) + self.assertEqual([event.kind for event in events], ["unresolved"]) + + def test_an_errored_probe_is_retried_once_then_given_up(self): + harness = Harness(lambda sha: "error") + state, runs = MODULE.empty_state(), {7: data()} + events = drive(harness, state, runs) + self.assertEqual(len(harness.dispatched), 2) + self.assertEqual([event.kind for event in events], ["error"]) + + def test_one_error_per_probe_does_not_end_a_bisect(self): + errored = set() + + def outcome(sha): + if sha != HEAD and sha not in errored: + errored.add(sha) + return "error" + return "fail" if sha == HEAD or C.index(sha) >= 3 else "pass" + harness = Harness(outcome) + state, runs = MODULE.empty_state(), {7: data()} + drive(harness, state, runs, steps=30) + self.assertEqual(state["items"][0]["state"], "confirmed") + self.assertEqual(state["items"][0]["culprit"]["sha"], C[3]) + + def test_pending_runs_hold_the_item(self): + harness = Harness(lambda sha: "pending") + state, runs = MODULE.empty_state(), {7: data()} + drive(harness, state, runs, steps=5) + self.assertEqual(len(harness.dispatched), 1) + self.assertEqual(state["items"][0]["state"], "flake-check") + + def test_checks_per_run_are_capped_and_runs_are_queued_once(self): + tests = tuple((f"S/t{index}()", [1]) for index in range(MODULE.MAX_CHECKS_PER_RUN + 3)) + harness = Harness(lambda sha: "pending") + state, runs = MODULE.empty_state(), {7: data(tests=tests)} + harness.step(state, runs) + harness.step(state, runs) + self.assertEqual(len(state["items"]), MODULE.MAX_CHECKS_PER_RUN) + self.assertEqual(state["seen"], [7]) + + def test_the_state_stays_bounded_over_a_long_red_streak(self): + tests = tuple((f"S/t{index}()", [1]) for index in range(MODULE.MAX_CHECKS_PER_RUN)) + outcome = ["pending"] + harness = Harness(lambda sha: outcome[0]) + state = MODULE.empty_state() + runs = {run_id: data(run_id=run_id, tests=tests) for run_id in range(1, 5)} + harness.step(state, runs) + self.assertEqual(len(state["items"]), MODULE.MAX_OPEN_ITEMS) + self.assertEqual(state["seen"], [1, 2, 3, 4]) + outcome[0] = "pass" + for run_id in range(5, 60): + runs[run_id] = data(run_id=run_id, tests=tests) + for _ in range(3): + harness.step(state, runs, per_day=10_000) + self.assertLessEqual(len(state["items"]), MODULE.MAX_OPEN_ITEMS + MODULE.MAX_FINISHED_ITEMS) + self.assertLess(len(MODULE.render_state(state, 24)), 65_536 // 2) + + def test_dispatches_are_capped_per_invocation_and_per_day(self): + tests = tuple((f"S/t{index}()", [1]) for index in range(5)) + harness = Harness(lambda sha: "pending") + state = MODULE.empty_state() + runs = {7: data(tests=tests), 8: data(run_id=8, tests=tests)} + harness.step(state, runs) + self.assertEqual(len(harness.dispatched), MODULE.MAX_DISPATCHES_PER_INVOCATION) + harness.step(state, runs, per_day=6) + self.assertEqual(len(harness.dispatched), 6) + harness.step(state, runs, per_day=6) + self.assertEqual(len(harness.dispatched), 6) + # A day later the budget is back. + harness.step(state, runs, now=NOW + timedelta(days=1, minutes=1), per_day=6) + self.assertEqual(len(harness.dispatched), 10) + + def test_active_bisects_are_capped(self): + tests = tuple((f"S/t{index}()", [1, 2]) for index in range(MODULE.MAX_ACTIVE_BISECTS + 1)) + harness = Harness(lambda sha: "fail" if sha == HEAD else "pending") + state, runs = MODULE.empty_state(), {7: data(tests=tests)} + drive(harness, state, runs, steps=4) + states = [item["state"] for item in state["items"]] + self.assertEqual(states.count("bisecting"), MODULE.MAX_ACTIVE_BISECTS) + self.assertEqual(states.count("bisect-wait"), 1) + + def test_stale_or_orphaned_items_expire(self): + harness = Harness(lambda sha: "pending") + state, runs = MODULE.empty_state(), {7: data()} + harness.step(state, runs) + harness.step(state, {}, now=NOW + timedelta(minutes=15)) + self.assertEqual(state["items"][0]["state"], "expired") + state, runs = MODULE.empty_state(), {7: data()} + harness.step(state, runs) + harness.step(state, runs, now=NOW + MODULE.ITEM_TTL + timedelta(hours=1)) + self.assertEqual(state["items"][0]["state"], "expired") + + def test_a_failed_dispatch_spends_an_attempt(self): + state, runs = MODULE.empty_state(), {7: data()} + for _ in range(3): + MODULE.advance(state, runs, poll=lambda run_id: "pending", dispatch=lambda test, sha: None, per_day=24, now=NOW) + self.assertEqual(state["items"][0]["state"], "error") + + +class ClassifyTests(unittest.TestCase): + def test_only_a_failed_test_step_counts_as_a_failure(self): + steps = lambda names: (lambda: names) # noqa: E731 + self.assertEqual(MODULE.classify({"status": "in_progress"}, steps([])), "pending") + self.assertEqual(MODULE.classify({"status": "completed", "conclusion": "success"}, steps([])), "pass") + failed = {"status": "completed", "conclusion": "failure"} + self.assertEqual(MODULE.classify(failed, steps(["Run selected tests"])), "fail") + self.assertEqual(MODULE.classify(failed, steps(["Build the app-host and UI test product"])), "error") + self.assertEqual(MODULE.classify(failed, steps(["Resolve selectors against the built tests"])), "absent") + self.assertEqual(MODULE.classify({"status": "completed", "conclusion": "cancelled"}, steps([])), "error") + + +class MarkerTests(unittest.TestCase): + def test_the_attribution_data_marker_round_trips(self): + pr = attribution.PullRequest(number=5, title="t", url="u", merge_sha=C[0]) + marker = attribution.data_marker( + run={"id": 7, "html_url": "https://run/7", "head_sha": HEAD}, + previous={"head_sha": PREV, "html_url": "https://run/prev"}, + failures={"S/x()": ["https://job"]}, + attributions={"S/x()": ([pr], "only pull request in the range")}, + prs=[pr], commits=[C[0]], + ) + [parsed] = MODULE.hidden_json(f"table\n{marker}\nmore", attribution.DATA_PREFIX) + self.assertTrue(MODULE.valid_data(parsed)) + self.assertEqual(parsed["tests"], [{"test": "S/x()", "suspects": [5], "how": "only pull request in the range"}]) + self.assertEqual(parsed["prs"], {C[0]: 5}) + + def test_unsafe_data_is_refused(self): + self.assertTrue(MODULE.valid_data(data())) + self.assertFalse(MODULE.valid_data({**data(), "head": "main; rm -rf /"})) + self.assertFalse(MODULE.valid_data({**data(), "commits": ["--force"]})) + self.assertFalse(MODULE.valid_data({**data(), "run_id": "7"})) + self.assertFalse(MODULE.dispatchable({"test": "S/x() --force"})) + self.assertFalse(MODULE.dispatchable({"test": "-S/x()"})) + self.assertTrue(MODULE.dispatchable({"test": "S/x(label:)"})) + + def test_state_renders_and_parses_back(self): + harness = Harness(lambda sha: "pending") + state, runs = MODULE.empty_state(), {7: data()} + harness.step(state, runs) + body = MODULE.render_state(state, 24) + self.assertEqual(MODULE.hidden_json(body, MODULE.STATE_PREFIX), [state]) + self.assertIn("`S/x()` | [7](https://run/7) | rerunning", body) + self.assertIn("Dispatches in the last 24 hours: 1 of 24.", body) + self.assertNotIn("—", body) + + +SECTION = "\n".join([ + "### New since `0000000000`", + "", + "Test | Suspect | Jobs", + "--- | --- | ---", + "`S/x()` | #1, #2 (edits the suite) | [job](https://job/1)", + "`T/y()` | unattributed (no pull request in the range reaches this suite) | [job](https://job/2)", + "", + f"{attribution.DATA_PREFIX}{{\"run_id\":7}} -->", +]) + + +def suspect_comment(pr, tests): + return "\n".join([ + attribution.marker(pr, tests, f"{PREV[:10]}..{HEAD[:10]}"), + "These app-host tests newly fail...", + "", + *[f"- `{test}` (edits the suite) [job](https://job/1)" for test in tests], + "", + "Commits in the range: ...", + ]) + + +class EditTests(unittest.TestCase): + def test_rows_take_the_latest_update(self): + once = MODULE.annotate_row(SECTION, "S/x()", "reproduced") + self.assertIn("`S/x()` | #1, #2 (edits the suite) · reproduced | [job](https://job/1)", once) + twice = MODULE.annotate_row(once, "S/x()", "confirmed: #2") + self.assertIn("`S/x()` | #1, #2 (edits the suite) · confirmed: #2 | [job](https://job/1)", twice) + self.assertIn("`T/y()` | unattributed (no pull request", twice) + + def test_section_edits_only_touch_the_red_runs_own_section(self): + other = SECTION.replace('"run_id":7', '"run_id":9') + nodes = [MODULE.Comment(None, other, "issue"), MODULE.Comment(11, SECTION)] + item = {"run": 7, "test": "S/x()", "note": "flaky"} + edits = MODULE.section_edits([MODULE.Event("flaky", item)], {7: data()}, nodes) + self.assertEqual(list(edits), [11]) + self.assertIn("· flaky", edits[11]) + + def flaky_event(self, test="S/x()", suspects=(1, 2)): + item = {"run": 7, "test": test, "suspects": list(suspects), "note": "", "rerun_url": f"https://probe/{test}"} + return MODULE.Event("flaky", item) + + def test_the_all_flaky_header_waits_for_every_test_in_the_comment(self): + comments = {2: [MODULE.Comment(22, suspect_comment(2, ["S/x()", "T/y()"]))]} + edits, _ = MODULE.pr_updates(self.flaky_event(suspects=(2,)), data(), lambda pr: comments[pr]) + self.assertNotIn("look flaky", edits[22]) + comments[2] = [MODULE.Comment(22, edits[22])] + edits, _ = MODULE.pr_updates(self.flaky_event("T/y()", suspects=(2,)), data(), lambda pr: comments[pr]) + self.assertIn("look flaky", edits[22].split("\n")[1]) + self.assertEqual(edits[22].count("look flaky"), 1) + + def test_a_flaky_failure_clears_the_suspects_comments(self): + comments = { + 1: [MODULE.Comment(21, suspect_comment(1, ["S/x()"]))], + 2: [MODULE.Comment(22, suspect_comment(2, ["S/x()", "T/y()"])), + MODULE.Comment(23, suspect_comment(2, ["S/x()"]).replace(PREV[:10], "1111111111"))], + } + edits, posts = MODULE.pr_updates(self.flaky_event(), data(), lambda pr: comments[pr]) + self.assertEqual(posts, []) + self.assertEqual(sorted(edits), [21, 22]) # 23 is another range's comment + self.assertIn("- `S/x()` (edits the suite) [job](https://job/1) **Update:** did not reproduce", edits[21]) + # Every test in #1's comment is flaky, so it says so up top; #2 still has T/y(). + self.assertEqual(edits[21].split("\n")[1].split(" ")[0], "**Update:**") + self.assertNotIn("look flaky", edits[22]) + + def confirmed_event(self, pr=2, suspects=(1, 2)): + item = {"run": 7, "test": "S/x()", "suspects": list(suspects), "note": "", + "culprit": {"sha": C[3], "pr": pr, "pass_sha": C[2], + "pass_url": "https://probe/pass", "fail_url": "https://probe/fail"}} + return MODULE.Event("confirmed", item) + + def test_a_confirmed_culprit_is_told_and_the_others_cleared(self): + comments = {1: [MODULE.Comment(21, suspect_comment(1, ["S/x()"]))], + 2: [MODULE.Comment(22, suspect_comment(2, ["S/x()"]))]} + edits, posts = MODULE.pr_updates(self.confirmed_event(), data(), lambda pr: comments[pr]) + self.assertEqual(posts, []) + self.assertIn("**Update:** **confirmed** by bisect: passes at", edits[22]) + self.assertIn("[run](https://probe/fail)", edits[22]) + self.assertIn("a bisect points at #2 instead", edits[21]) + + def test_a_culprit_nobody_suspected_gets_one_comment(self): + comments = {1: [MODULE.Comment(21, suspect_comment(1, ["S/x()"]))], 9: []} + event = self.confirmed_event(pr=9, suspects=(1,)) + edits, posts = MODULE.pr_updates(event, data(), lambda pr: comments[pr]) + self.assertEqual([pr for pr, _ in posts], [9]) + self.assertTrue(posts[0][1].startswith("