diff --git a/scripts/ci/delivery_workflows.py b/scripts/ci/delivery_workflows.py index eb6feed6ffc8..6b4424841b2d 100644 --- a/scripts/ci/delivery_workflows.py +++ b/scripts/ci/delivery_workflows.py @@ -392,6 +392,100 @@ def superseded_unstarted(run: Mapping[str, Any], jobs: Sequence[Mapping[str, Any f"({why_not})") +# --- DEV publish runs a newer publish already covered --------------------------- +# cmux-next-dev-daily.yml's publish runs ("cmux NEXT DEV daily · publish ") all run on main, at +# whatever main's head was when the dispatcher asked, so superseded_unstarted() can never tell them apart by head; +# and their Linux sign jobs run within minutes, so build_started() counts every one as started. On 2026-10-31 that +# left 115 of them queued (the oldest 37 hours), each holding a Verify and a tip DMG job on a Mac label (106 on +# blacksmith-12vcpu-macos-26, dead since 10-30, the rest on glaeda-std-xcode-26.6). +# +# Such a run can never publish once a newer publish run has published: its Publish job's not_newer() skips a +# source commit the publish record already contains, and the tip feed refuses a build that is not newer than the +# one it has. So it is certain not to deliver, the "certain to fail" exception of the standing rule, when: +# 1. it is a DEV publish run, in flight, no job of its current attempt is running (in progress, or queued with a +# runner assigned), and its Publish job has not run: what is left only waits for a runner. A tip DMG or tip +# feed job that already finished changes nothing, since the tip feed refuses a build that is not newer; +# 2. nothing in it has moved for STALE_WAIT; +# 3. a newer DEV publish run (a later source run id) completed in success with its Publish job succeeding; and +# 4. that run's source commit contains this run's source commit (`contains(old, new)` is True). +# Anything unread is a keep. +DEV_PUBLISH_WORKFLOW = "cmux-next-dev-daily.yml" +DEV_PUBLISH_TITLE = re.compile(r"^cmux NEXT DEV daily · publish ([0-9]+)$") +DEV_PUBLISH_JOB = "Publish cmux NEXT DEV" + + +def dev_publish_source(run: Mapping[str, Any]) -> int | None: + """The source run id a DEV publish run publishes (from its title), or None when it is not one.""" + if _file_name(run.get("path")) != DEV_PUBLISH_WORKFLOW: + return None + for title in (run.get("name"), run.get("display_title")): + match = DEV_PUBLISH_TITLE.match(str(title or "")) + if match: + return int(match.group(1)) + return None + + +def dev_publish_published(run: Mapping[str, Any], jobs: Sequence[Mapping[str, Any]] | None) -> bool: + """Whether a DEV publish run finished in success with its Publish job succeeding (its source, or a newer one, + is then in the publish record).""" + if dev_publish_source(run) is None or run.get("status") != "completed" or run.get("conclusion") != "success": + return False + mine = attempt_jobs(run, jobs) if jobs is not None else None + return bool(mine) and any(_base_name(str(job.get("name") or "")) == DEV_PUBLISH_JOB + and job.get("status") == "completed" and job.get("conclusion") == "success" + for job in mine) + + +def dev_publish_superseded(run: Mapping[str, Any], jobs: Sequence[Mapping[str, Any]] | None, *, + source_sha: str | None, newer: Mapping[str, Any] | None, + newer_jobs: Sequence[Mapping[str, Any]] | None, newer_source_sha: str | None, + contains: Contains, now: _dt.datetime | None = None, + stale_wait: _dt.timedelta = STALE_WAIT) -> tuple[bool, str]: + """(True, why) when a DEV publish run can never publish because `newer` (a later DEV publish run) already + published a source commit that contains this run's; (False, why not) otherwise or when anything is unread.""" + source = dev_publish_source(run) + if source is None: + return False, "it is not a DEV publish run" + if run.get("status") not in WAITING_JOB_STATUSES and run.get("status") != "in_progress": + return False, f"it is {run.get('status') or 'in an unknown state'}" + mine = attempt_jobs(run, jobs) if jobs is not None else None + if mine is None: + return False, "its jobs, or which attempt they belong to, could not be read" + for job in mine: + name, status = str(job.get("name") or ""), job.get("status") + if status == "completed" and job.get("conclusion") not in KNOWN_CONCLUSIONS | {"skipped"}: + return False, f"its job {name!r} finished with an unknown conclusion ({job.get('conclusion')!r})" + waiting = status in WAITING_JOB_STATUSES and not job.get("runner_name") + if status != "completed" and not waiting: + return False, f"condition 1 fails: its job {name!r} is running ({status}, runner {job.get('runner_name')!r})" + if (_base_name(name) == DEV_PUBLISH_JOB and not waiting + and not (status == "completed" and job.get("conclusion") == "skipped")): + return False, f"condition 1 fails: its Publish job has run ({job.get('conclusion')})" + since = waited_since(run, mine) + if since is None: + return False, "how long it has waited could not be read" + waited = (now or _dt.datetime.now(_dt.timezone.utc)) - since + if waited < stale_wait: + return False, (f"condition 2 fails: it has waited {int(waited.total_seconds() // 60)} min since it last " + f"moved, under {int(stale_wait.total_seconds() // 60)} min") + if newer is None: + return False, "condition 3 fails: no newer DEV publish run has published" + newer_source = dev_publish_source(newer) + if newer_source is None or newer_source <= source: + return False, f"condition 3 fails: run {newer.get('id')} does not publish a later source run than {source}" + if not dev_publish_published(newer, newer_jobs): + return False, f"condition 3 fails: run {newer.get('id')} has not published (Publish job not a success)" + old, new = str(source_sha or "").lower(), str(newer_source_sha or "").lower() + if not _SHA.fullmatch(old) or not _SHA.fullmatch(new): + return False, "condition 4 fails: a source commit could not be read" + if old != new and contains(old, new) is not True: + return False, f"condition 4 fails: {new[:12]} could not be shown to contain {old[:12]}" + return True, (f"certain not to publish: run {newer.get('id')} published source run {newer_source} " + f"({new[:12]}), which contains this run's source {old[:12]}; the publisher and the tip feed refuse " + f"a commit that is not newer (it waited {int(waited.total_seconds() // 60)} min with no job " + "running)") + + # --- runs that can never deliver ---------------------------------------------- # Two more ways a started nightly run may be cancelled, both read by the # agents' cancel guard (never by the queue janitor, which only skips runs that diff --git a/scripts/ci/queue_janitor.py b/scripts/ci/queue_janitor.py index d031287fb6a4..2e5de22648b4 100644 --- a/scripts/ci/queue_janitor.py +++ b/scripts/ci/queue_janitor.py @@ -227,6 +227,7 @@ from delivery_workflows import REPLACEMENT_JOB_READS, attempt_jobs, build_started, is_delivery_run # noqa: E402 from delivery_workflows import delivers, is_delivery_event, is_delivery_workflow # noqa: E402 from delivery_workflows import newer_head_runs, superseded_unstarted # noqa: E402 +from delivery_workflows import dev_publish_published, dev_publish_source, dev_publish_superseded # noqa: E402 API = "https://api.github.com" @@ -2891,6 +2892,124 @@ def tip_of(branch: str) -> str: return decisions, failures +# --- DEV publish runs a newer publish already covered --------------------------------------------------------- +# delivery_workflows.dev_publish_superseded() is the rule. Reads a sweep: one listing of the publisher's +# successful runs, the jobs of up to DEV_PUBLISH_WINNER_READS of them, then per candidate (oldest first, at most +# DEV_PUBLISH_JOB_READS) its jobs, its source run and one compare; a cancel re-reads the run and its jobs first. +DEV_PUBLISH_MAX_CANCELS = 20 +DEV_PUBLISH_JOB_READS = 25 +DEV_PUBLISH_WINNER_READS = 3 + + +def skip_superseded_dev_publish( + github: "GitHub", + runs: Sequence[Mapping[str, Any]], + jobs_by_run: Mapping[int, Sequence[Mapping[str, Any]]], + *, + dry_run: bool, + max_cancels: int = DEV_PUBLISH_MAX_CANCELS, + max_reads: int = DEV_PUBLISH_JOB_READS, + now: dt.datetime | None = None, +) -> tuple[list[SupersededDecision], int]: + """Cancel each in-flight DEV publish run a newer published one already covers (dev_publish_superseded()).""" + candidates = sorted((run for run in runs if run.get("status") in IN_FLIGHT_RUN_STATUSES + and dev_publish_source(run) is not None), + key=lambda run: (dev_publish_source(run) or 0, run["id"])) + if not candidates: + return [], 0 + decisions: list[SupersededDecision] = [] + winner: Mapping[str, Any] | None = None + winner_jobs: Sequence[Mapping[str, Any]] | None = None + try: + published = sorted((run for run in github.dev_publish_runs() if dev_publish_source(run) is not None), + key=lambda run: dev_publish_source(run) or 0, reverse=True) + for run in published[:DEV_PUBLISH_WINNER_READS]: + jobs = complete_jobs(github.jobs(run["id"])) + if dev_publish_published(run, jobs): + winner, winner_jobs = run, jobs + break + winner_source = github.run(dev_publish_source(winner)).get("head_sha") if winner else None + except RateLimited: + raise + except RuntimeError as error: + print(f"queue-janitor: DEV publish runs: {error}", file=sys.stderr) + return [SupersededDecision(run, "keep", f"the published DEV runs could not be read: {error}") + for run in candidates], 1 + failures = cancels = reads = 0 + compares: dict[str, bool | None] = {} + + def contains(old: str, new: str) -> bool | None: + if old not in compares: + try: + compares[old] = github.compare_status(old, new) in ("ahead", "identical") + except RateLimited: + raise + except RuntimeError as error: + print(f"queue-janitor: compare {old[:12]}...{new[:12]}: {error}", file=sys.stderr) + compares[old] = None + return compares[old] + + for run in candidates: + if winner is None: + decisions.append(SupersededDecision(run, "keep", "no newer DEV publish run has published")) + continue + if (dev_publish_source(run) or 0) >= (dev_publish_source(winner) or 0): + decisions.append(SupersededDecision(run, "keep", f"it publishes a source run no older than run " + f"{winner.get('id')}'s")) + continue + if cancels >= max_cancels: + decisions.append(SupersededDecision(run, "keep", f"cap of {max_cancels} reached")) + continue + if reads >= max_reads: + decisions.append(SupersededDecision(run, "keep", f"read budget of {max_reads} runs spent")) + continue + reads += 1 + try: + jobs = complete_jobs(jobs_by_run[run["id"]]) if run["id"] in jobs_by_run else complete_jobs( + github.jobs(run["id"])) + source_sha = github.run(dev_publish_source(run)).get("head_sha") + except RateLimited: + raise + except RuntimeError as error: + failures += 1 + decisions.append(SupersededDecision(run, "keep", f"failed to read: {error}")) + continue + ok, why = dev_publish_superseded(run, jobs, source_sha=source_sha, newer=winner, newer_jobs=winner_jobs, + newer_source_sha=winner_source, contains=contains, now=now) + if not ok: + decisions.append(SupersededDecision(run, "keep", why)) + continue + if dry_run: + decisions.append(SupersededDecision(run, "cancel", f"would cancel: {why}")) + cancels += 1 + continue + # Right before the cancel the run and its jobs are read again: one whose Mac job started since is kept. + try: + current, fresh = github.run(run["id"]), complete_jobs(github.jobs(run["id"])) + except RateLimited: + raise + except RuntimeError as error: + failures += 1 + decisions.append(SupersededDecision(run, "keep", f"failed to re-read: {error}")) + continue + ok, why = dev_publish_superseded(current, fresh, source_sha=source_sha, newer=winner, + newer_jobs=winner_jobs, newer_source_sha=winner_source, + contains=contains, now=now) + if not ok: + decisions.append(SupersededDecision(run, "keep", f"on re-read: {why}")) + continue + try: + github.cancel(run["id"]) + except RuntimeError as error: + if not _cancel_refused(error): + failures += 1 + decisions.append(SupersededDecision(run, "keep", f"cancel failed: {error}")) + continue + decisions.append(SupersededDecision(run, "cancel", f"cancelled: {why}")) + cancels += 1 + return decisions, failures + + def render_superseded_summary(decisions: Sequence[SupersededDecision]) -> str: lines = ["", "### Superseded nightly, DEV and promote runs", "", "A stale delivery run (every job but the light control and decide ones still queued for 20 min or more, " @@ -3520,6 +3639,17 @@ def branch_head(self, branch: str) -> str: raise return str(((payload or {}).get("commit") or {}).get("sha") or "") + def compare_status(self, base: str, head: str) -> str: + """GitHub's compare status of base...head (ahead, behind, identical or diverged).""" + payload = self.request("GET", f"/repos/{self.repo}/compare/{base}...{head}?per_page=1") + return str((payload or {}).get("status") or "") + + def dev_publish_runs(self) -> list[dict[str, Any]]: + """One page of the newest successful runs of the DEV publisher (skip_superseded_dev_publish()).""" + query = urllib.parse.urlencode({"status": "success", "event": "workflow_dispatch", "per_page": 20}) + payload = self.request("GET", f"/repos/{self.repo}/actions/workflows/cmux-next-dev-daily.yml/runs?{query}") + return [run for run in payload.get("workflow_runs") or [] if isinstance(run, Mapping)] + def cancel(self, run_id: int) -> None: self.request("POST", f"/repos/{self.repo}/actions/runs/{run_id}/cancel") @@ -4063,6 +4193,13 @@ def save_lost_reruns(entries: Sequence[Mapping[str, Any]]) -> None: print(f"queue-janitor: superseded delivery runs: {error}", file=sys.stderr) superseded_decisions, superseded_failures = [], 1 failures += superseded_failures + try: + dev_decisions, dev_failures = skip_superseded_dev_publish(github, runs, delivery_jobs, dry_run=args.dry_run) + except RuntimeError as error: # a rate limit: the next sweep reads them again + print(f"queue-janitor: superseded DEV publish runs: {error}", file=sys.stderr) + dev_decisions, dev_failures = [], 1 + superseded_decisions = [*superseded_decisions, *dev_decisions] + failures += dev_failures summary = render_summary(plan, dry_run=args.dry_run, now=now, results=results) summary += render_open_head_reruns(open_head_decisions, open_head_results, freed=open_head_freed) diff --git a/tests/test_ci_queue_janitor.py b/tests/test_ci_queue_janitor.py index b34765a8aaed..238f8acdb9ac 100644 --- a/tests/test_ci_queue_janitor.py +++ b/tests/test_ci_queue_janitor.py @@ -4487,5 +4487,121 @@ def test_the_workflow_hands_the_janitor_cmux_bots_token_twice(self): self.assertTrue(step["env"]["GH_TOKEN"].startswith("${{ steps.cmux-bot-token.outputs.token ||")) +class DevPublishStub: + """The janitor client calls skip_superseded_dev_publish() makes, in memory.""" + + def __init__(self, runs, jobs, heads, compare="ahead"): + self.runs, self.job_map, self.heads, self.compare = runs, jobs, heads, compare + self.cancelled, self.calls = [], [] + + def dev_publish_runs(self): + self.calls.append("list") + return [r for r in self.runs.values() if r["status"] == "completed" and r["conclusion"] == "success"] + + def jobs(self, run_id): + self.calls.append(("jobs", run_id)) + return self.job_map[run_id] + + def run(self, run_id): + self.calls.append(("run", run_id)) + return self.runs.get(run_id) or {"id": run_id, "head_sha": self.heads[run_id]} + + def compare_status(self, base, head): + self.calls.append(("compare", base, head)) + return self.compare + + def cancel(self, run_id): + self.cancelled.append(run_id) + + +def dev_publish(source, ident, *, status="queued", conclusion=None): + title = f"cmux NEXT DEV daily · publish {source}" + return {"id": ident, "path": ".github/workflows/cmux-next-dev-daily.yml", "name": title, "display_title": title, + "status": status, "conclusion": conclusion, "head_sha": "m" * 40, "head_branch": "main", + "event": "workflow_dispatch", "run_attempt": 1, "created_at": iso(600)} + + +def dev_jobs(*, published=False, mac_status="queued"): + def j(name, status="completed", conclusion="success", label="blacksmith-4vcpu-ubuntu-2404", runner="r"): + return {"name": name, "status": status, "conclusion": conclusion if status == "completed" else None, + "labels": [label], "runner_name": runner if status != "queued" else None, + "started_at": iso(590), "completed_at": iso(589) if status == "completed" else None} + jobs = [j("Build the signer"), j("Sign cmux NEXT DEV"), j("runner")] + if published: + return [*jobs, j("Verify signed cmux NEXT DEV", label="glaeda-std-xcode-26.6"), j("Publish cmux NEXT DEV")] + return [*jobs, j("Verify signed cmux NEXT DEV", mac_status, label="blacksmith-12vcpu-macos-26"), + j("Build the unsigned cmux NEXT DEV tip DMG", "queued", label="blacksmith-12vcpu-macos-26")] + + +class SupersededDevPublishTests(unittest.TestCase): + def setUp(self): + self.runs = {1: dev_publish(100, 1), 2: dev_publish(110, 2), + 3: dev_publish(200, 3, status="completed", conclusion="success"), 4: dev_publish(300, 4)} + self.jobs = {1: dev_jobs(), 2: dev_jobs(mac_status="in_progress"), 3: dev_jobs(published=True), 4: dev_jobs()} + self.heads = {100: "a" * 40, 110: "b" * 40, 200: "c" * 40, 300: "d" * 40} + + def sweep(self, stub, dry_run=False, **kw): + inflight = [r for r in stub.runs.values() if r["status"] != "completed"] + return janitor.skip_superseded_dev_publish(stub, inflight, {}, dry_run=dry_run, now=NOW, **kw) + + def test_cancels_only_the_older_waiting_run_and_keeps_started_and_newer_ones(self): + stub = DevPublishStub(self.runs, self.jobs, self.heads) + decisions, failures = self.sweep(stub) + self.assertEqual(failures, 0) + self.assertEqual(stub.cancelled, [1]) + notes = {d.run["id"]: d.note for d in decisions} + self.assertIn("running", notes[2]) + self.assertIn("no older than", notes[4]) + + def test_dry_run_cancels_nothing(self): + stub = DevPublishStub(self.runs, self.jobs, self.heads) + decisions, _ = self.sweep(stub, dry_run=True) + self.assertEqual(stub.cancelled, []) + self.assertEqual([d.run["id"] for d in decisions if d.action == "cancel"], [1]) + + def test_a_source_the_winner_does_not_contain_is_kept(self): + stub = DevPublishStub(self.runs, self.jobs, self.heads, compare="diverged") + self.sweep(stub) + self.assertEqual(stub.cancelled, []) + + def test_no_published_run_keeps_everything(self): + self.runs[3]["conclusion"] = "failure" + stub = DevPublishStub(self.runs, self.jobs, self.heads) + decisions, _ = self.sweep(stub) + self.assertEqual(stub.cancelled, []) + self.assertTrue(all(d.action == "keep" for d in decisions)) + + def test_a_mac_job_that_started_between_the_reads_is_kept(self): + stub = DevPublishStub(self.runs, self.jobs, self.heads) + reads = {"n": 0} + original = stub.jobs + + def jobs(run_id): + if run_id == 1: + reads["n"] += 1 + if reads["n"] == 2: + return dev_jobs(mac_status="in_progress") + return original(run_id) + stub.jobs = jobs + decisions, _ = self.sweep(stub) + self.assertEqual(stub.cancelled, []) + self.assertIn("on re-read", {d.run["id"]: d.note for d in decisions}[1]) + + def test_the_cap_and_read_budget_bound_a_sweep(self): + for ident in range(10, 16): + self.runs[ident] = dev_publish(50 + ident, ident) + self.jobs[ident] = dev_jobs() + self.heads[50 + ident] = f"{ident:x}".ljust(40, "e") + stub = DevPublishStub(self.runs, self.jobs, self.heads) + self.sweep(stub, max_cancels=2) + self.assertEqual(len(stub.cancelled), 2) + stub = DevPublishStub(self.runs, self.jobs, self.heads) + self.sweep(stub, max_reads=3) + self.assertEqual(len(stub.cancelled), 3) + + def test_the_sweep_runs_the_pass(self): + self.assertIn("skip_superseded_dev_publish(github, runs, delivery_jobs", SCRIPT.read_text()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_delivery_workflows_cancel.py b/tests/test_delivery_workflows_cancel.py index ddc629de2cf6..cc6b625aab6d 100644 --- a/tests/test_delivery_workflows_cancel.py +++ b/tests/test_delivery_workflows_cancel.py @@ -483,5 +483,126 @@ def test_replay_40752120475(self): self.assertIn("waited 24 min", why) + +# --- DEV publish runs a newer publish covered (2026-10-31: 115 queued, oldest 37 h) ---------------------------- +DEV_NOW = dw._dt.datetime(2026, 10, 31, 5, 0, tzinfo=dw._dt.timezone.utc) +OLD_SRC, NEW_SRC = sha("a"), sha("b") + + +def dev_run(source, *, ident=10, status="queued", conclusion=None, created="2026-10-29T15:34:57Z", attempt=1): + return {"id": ident, "path": ".github/workflows/cmux-next-dev-daily.yml", + "name": f"cmux NEXT DEV daily · publish {source}", "display_title": f"cmux NEXT DEV daily · publish {source}", + "status": status, "conclusion": conclusion, "head_sha": sha("m"), "head_branch": "main", + "event": "workflow_dispatch", "run_attempt": attempt, "created_at": created} + + +def dev_waiting_jobs(mac="blacksmith-12vcpu-macos-26"): + early = "2026-10-29T15:40:00Z" + return [job("Build the signer", start=early, end=early), job("Sign cmux NEXT DEV", start=early, end=early), + job("runner", start=early, end=early), + job("Verify signed cmux NEXT DEV", "queued", labels=(mac,), start=early), + job("Build the unsigned cmux NEXT DEV tip DMG", "queued", labels=(mac,), start=early)] + + +def dev_published_jobs(): + return [*[job(n) for n in ("Build the signer", "Sign cmux NEXT DEV", "runner", "Verify signed cmux NEXT DEV", + "Build the unsigned cmux NEXT DEV tip DMG", "Publish cmux NEXT DEV")], + job("Publish the cmux NEXT DEV tip feed", "completed", "skipped")] + + +class DevPublishSupersededTests(unittest.TestCase): + def decide(self, run=None, jobs=None, *, newer=None, newer_jobs=None, old=OLD_SRC, new=NEW_SRC, contains=True): + run = run or dev_run(100) + newer = newer or dev_run(200, ident=20, status="completed", conclusion="success") + return dw.dev_publish_superseded( + run, dev_waiting_jobs() if jobs is None else jobs, source_sha=old, newer=newer, + newer_jobs=dev_published_jobs() if newer_jobs is None else newer_jobs, newer_source_sha=new, + contains=lambda a, b: contains, now=DEV_NOW) + + def test_a_publish_run_waiting_on_a_mac_behind_a_newer_published_source_is_superseded(self): + ok, why = self.decide() + self.assertTrue(ok, why) + self.assertIn("certain not to publish", why) + + def test_the_source_id_comes_from_the_title_only_for_the_dev_publisher(self): + self.assertEqual(dw.dev_publish_source(dev_run(41153639573)), 41153639573) + self.assertIsNone(dw.dev_publish_source({**dev_run(5), "path": ".github/workflows/nightly.yml"})) + self.assertIsNone(dw.dev_publish_source({**dev_run(5), "name": "cmux NEXT DEV daily", "display_title": ""})) + + def test_a_started_mac_job_keeps_it(self): + jobs = dev_waiting_jobs() + jobs[3] = job("Verify signed cmux NEXT DEV", "in_progress", labels=("glaeda-std-xcode-26.6",), + start="2026-10-29T15:40:00Z") + ok, why = self.decide(jobs=jobs) + self.assertFalse(ok) + self.assertIn("running", why) + + def test_a_queued_mac_job_a_runner_took_keeps_it(self): + jobs = dev_waiting_jobs() + jobs[3]["runner_name"] = "cmux9s-mac-mini-glaeda" + self.assertFalse(self.decide(jobs=jobs)[0]) + + def test_a_running_or_finished_publish_job_keeps_it(self): + jobs = [*dev_waiting_jobs(), job("Publish cmux NEXT DEV", "in_progress", start="2026-10-29T15:40:00Z")] + ok, why = self.decide(jobs=jobs) + self.assertFalse(ok) + self.assertIn("running", why) + early = "2026-10-29T15:40:00Z" + jobs = [*dev_waiting_jobs(), job("Publish cmux NEXT DEV", "completed", "failure", start=early, end=early)] + ok, why = self.decide(jobs=jobs) + self.assertFalse(ok) + self.assertIn("Publish job has run", why) + + def test_a_finished_tip_dmg_with_verify_still_waiting_is_superseded(self): + # Run 40946510427 on 2026-10-31: its tip DMG built on a Mac; Verify had waited on a Mac since. + early = "2026-10-29T15:40:00Z" + jobs = dev_waiting_jobs() + jobs[4] = job("Build the unsigned cmux NEXT DEV tip DMG", labels=("blacksmith-6vcpu-macos-26",), + start=early, end=early) + jobs.append(job("Sign, notarize and staple the cmux NEXT DEV tip DMG", start=early, end=early)) + ok, why = self.decide(jobs=jobs) + self.assertTrue(ok, why) + + def test_a_recent_run_keeps_it(self): + jobs = [job(j["name"], j["status"], j["conclusion"] or "success", labels=j["labels"], + start="2026-10-31T04:50:00Z", end="2026-10-31T04:50:00Z") for j in dev_waiting_jobs()] + ok, why = self.decide(run=dev_run(100, created="2026-10-31T04:45:00Z"), jobs=jobs) + self.assertFalse(ok) + self.assertIn("condition 2", why) + + def test_a_newer_run_that_did_not_publish_keeps_it(self): + jobs = dev_published_jobs() + jobs[5] = job("Publish cmux NEXT DEV", "completed", "skipped") + self.assertFalse(self.decide(newer_jobs=jobs)[0]) + failed = dev_run(200, ident=20, status="completed", conclusion="failure") + self.assertFalse(self.decide(newer=failed)[0]) + + def test_an_older_or_equal_winner_keeps_it(self): + ok, why = self.decide(run=dev_run(300)) + self.assertFalse(ok) + self.assertIn("later source run", why) + + def test_unread_or_unrelated_sources_keep_it(self): + self.assertFalse(self.decide(contains=None)[0]) + self.assertFalse(self.decide(contains=False)[0]) + self.assertFalse(self.decide(old="")[0]) + self.assertTrue(self.decide(old=NEW_SRC, new=NEW_SRC, contains=None)[0]) # the same commit + + def test_unread_jobs_or_a_finished_run_keep_it(self): + run = dev_run(100) + self.assertFalse(dw.dev_publish_superseded(run, None, source_sha=OLD_SRC, newer=None, newer_jobs=None, + newer_source_sha=None, contains=lambda a, b: True, now=DEV_NOW)[0]) + self.assertFalse(self.decide(run=dev_run(100, status="completed", conclusion="cancelled"))[0]) + self.assertFalse(dw.dev_publish_superseded(run, dev_waiting_jobs(), source_sha=OLD_SRC, newer=None, + newer_jobs=None, newer_source_sha=NEW_SRC, + contains=lambda a, b: True, now=DEV_NOW)[0]) + + def test_the_publish_job_and_title_match_the_workflow(self): + text = (ROOT / ".github/workflows/cmux-next-dev-daily.yml").read_text() + names = set(re.findall(r"^ name: (.+)$", text, re.MULTILINE)) + self.assertIn(dw.DEV_PUBLISH_JOB, names) + self.assertRegex(text, r"cmux NEXT DEV daily · publish \{0\}") + + if __name__ == "__main__": unittest.main()