fix(scheduler): recheck cancellation races

scripts/ci/current_head_run_coalescer.py

@@ -373,7 +373,23 @@ def _fetch_run(repo: str, run_id: int) -> dict[str, Any]:
 
 def _cancel_run(repo: str, run_id: int) -> None:
     """Cancel one run and prove GitHub reached its terminal cancelled state."""
-    _run_json(["gh", "api", "-X", "POST", f"repos/{repo}/actions/runs/{run_id}/cancel"])
+    cancel_args = ["gh", "api", "-X", "POST", f"repos/{repo}/actions/runs/{run_id}/cancel"]
+    try:
+        _run_json(cancel_args)
+    except RuntimeError as exc:
+        # GitHub can race a queued run into startup between the candidate
+        # fetch and POST, returning HTTP 409 instead of accepting cancel.
+        # Re-read the authoritative run state before deciding whether a
+        # single retry is safe; never turn an unknown cancellation error into
+        # a successful result.
+        if "not been queued yet" not in str(exc):
+            raise
+        current = _fetch_run(repo, run_id)
+        if current.get("status") == "completed" and current.get("conclusion") == "cancelled":
+            return
+        if current.get("status") not in ACTIVE_STATUSES:
+            raise RuntimeError(f"workflow run {run_id} is no longer cancellable after HTTP 409") from exc
+        _run_json(cancel_args)
     for attempt in range(CANCELLATION_POLL_ATTEMPTS):
         run_data = _fetch_run(repo, run_id)
         if run_data.get("status") == "completed" and run_data.get("conclusion") == "cancelled":

tests/test_current_head_run_coalescer.py

@@ -473,6 +473,34 @@ def test_cancel_run_uses_explicit_transport_and_ordinary_endpoint(monkeypatch) -
     assert sleeps == [module.CANCELLATION_POLL_INTERVAL_SECONDS]
 
 
+def test_cancel_run_rechecks_and_retries_when_queued_run_start_races(monkeypatch) -> None:
+    """A startup race gets one authoritative retry instead of failing the scheduler."""
+    module = load_module()
+    cancel_calls = 0
+    states = iter(
+        [
+            {"status": "in_progress", "conclusion": None},
+            {"status": "completed", "conclusion": "cancelled"},
+        ]
+    )
+
+    def run_json(args):
+        nonlocal cancel_calls
+        if args[-1].endswith("/cancel"):
+            cancel_calls += 1
+            if cancel_calls == 1:
+                raise RuntimeError("gh: Cannot cancel a workflow run that has not been queued yet. (HTTP409)")
+            return {}
+        raise AssertionError(args)
+
+    monkeypatch.setattr(module, "_run_json", run_json)
+    monkeypatch.setattr(module, "_fetch_run", lambda _repo, _run_id: next(states))
+    monkeypatch.setattr(module.time, "sleep", lambda _seconds: None)
+
+    module._cancel_run("o/r", 123)
+    assert cancel_calls == 2
+
+
 def test_cancel_run_fails_when_terminal_cancellation_is_unproven(monkeypatch) -> None:
     """An accepted cancellation is not reported complete while GitHub stays active."""
     module = load_module()