diff --git a/.github/workflows/ci-manual-dispatch-guard.yml b/.github/workflows/ci-manual-dispatch-guard.yml index c4a87cad0072..939417d54bff 100644 --- a/.github/workflows/ci-manual-dispatch-guard.yml +++ b/.github/workflows/ci-manual-dispatch-guard.yml @@ -35,5 +35,6 @@ jobs: SOURCE_REF_NAME: ${{ github.event.workflow_run.head_branch }} SOURCE_SHA: ${{ github.event.workflow_run.head_sha }} SOURCE_RUN_ID: ${{ github.event.workflow_run.id }} + SOURCE_COVERAGE_FINGERPRINT: ${{ github.event.workflow_run.display_title }} SOURCE_WORKFLOW_PATH: ${{ github.event.workflow_run.path }} run: python3 scripts/ci/manual_dispatch_guard.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b48d87abf24d..bf7faf0817de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,10 @@ name: CI +# The guard watcher receives this title before dispatch jobs start. Keep a +# compact, versioned fingerprint here so it can distinguish equivalent PR +# coverage from a fuller manual request. +run-name: ${{ format('v1;sha={0};ref={1};cache={2};release={3};coverage={4}', github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha, github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref_name, inputs.cache_backend != 'default' && inputs.cache_backend || vars.CI_CACHE_BACKEND || 'r2', inputs.release_archs != 'default' && inputs.release_archs || vars.CI_RELEASE_BUILD_ARCHS || 'universal', github.event_name != 'pull_request' && 'full' || ((vars.CI_PULL_REQUEST_SUITE || 'compile-only') != 'compile-only' || contains(github.event.pull_request.labels.*.name, 'full-ci')) && 'full' || 'compile-only') }} + on: # Run the router for every pull request. The Linux layer is cheap and the # change classifier below keeps unrelated macOS jobs skipped. A workflow-level @@ -88,6 +93,7 @@ jobs: # shard (shard 8), which pr_runner_pool.py still plans for. unit_in_admission: ${{ steps.suite.outputs.unit_in_admission }} coverage_gap: ${{ steps.suite.outputs.coverage_gap }} + coverage_fingerprint: ${{ steps.coverage-fingerprint.outputs.fingerprint }} ui_selectors: ${{ steps.suite.outputs.ui_selectors }} compile_admitted: ${{ steps.unchanged_inputs.outputs.compile_admitted == 'true' && 'true' || steps.admitted.outputs.compile_admitted }} source_parent1: ${{ steps.source-identity.outputs.parent1 }} @@ -164,6 +170,7 @@ jobs: if: github.event_name == 'workflow_dispatch' env: GH_TOKEN: ${{ github.token }} + SOURCE_COVERAGE_FINGERPRINT: ${{ format('v1;sha={0};ref={1};cache={2};release={3};coverage={4}', github.sha, github.ref_name, inputs.cache_backend != 'default' && inputs.cache_backend || vars.CI_CACHE_BACKEND || 'r2', inputs.release_archs != 'default' && inputs.release_archs || vars.CI_RELEASE_BUILD_ARCHS || 'universal', 'full') }} run: python3 scripts/ci/manual_dispatch_guard.py --check-only - name: Record GitHub-selected source identity @@ -690,6 +697,19 @@ jobs: ${files_args[@]+"${files_args[@]}"} \ --github-output "$GITHUB_OUTPUT" + - name: Record coverage fingerprint + id: coverage-fingerprint + env: + SOURCE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + SOURCE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref_name }} + CACHE_BACKEND: ${{ inputs.cache_backend != 'default' && inputs.cache_backend || vars.CI_CACHE_BACKEND || 'r2' }} + RELEASE_ARCHS: ${{ inputs.release_archs != 'default' && inputs.release_archs || vars.CI_RELEASE_BUILD_ARCHS || 'universal' }} + COVERAGE_POLICY: ${{ steps.suite.outputs.full_suite == 'true' && 'full' || 'compile-only' }} + run: | + set -euo pipefail + fingerprint="v1;sha=$SOURCE_SHA;ref=$SOURCE_REF;cache=$CACHE_BACKEND;release=$RELEASE_ARCHS;coverage=$COVERAGE_POLICY" + echo "fingerprint=$fingerprint" >> "$GITHUB_OUTPUT" + - name: Mint the owned-pool routing token id: route-token # The org's manaflow-glaeda-route App, read-only on runners, so the diff --git a/scripts/ci/manual_dispatch_guard.py b/scripts/ci/manual_dispatch_guard.py index 0f515f37670b..1b61f9504597 100755 --- a/scripts/ci/manual_dispatch_guard.py +++ b/scripts/ci/manual_dispatch_guard.py @@ -17,12 +17,15 @@ class Decision: reason: str -def has_covering_ci_run(runs: Sequence[Mapping[str, Any]], sha: str) -> bool: - """Whether a successful or active full-suite pull-request run covers this head.""" +def has_covering_ci_run( + runs: Sequence[Mapping[str, Any]], sha: str, fingerprint: str +) -> bool: + """Whether an active or successful PR run has equivalent coverage.""" return any( item.get("event") == "pull_request" and item.get("head_sha") == sha and item.get("full_suite") is True + and item.get("coverage_fingerprint") == fingerprint and (item.get("status") != "completed" or item.get("conclusion") == "success") for item in runs ) @@ -30,7 +33,8 @@ def has_covering_ci_run(runs: Sequence[Mapping[str, Any]], sha: str) -> bool: def decide(*, event: str, repository: str, ref_name: str, sha: str, pull_requests: Sequence[Mapping[str, Any]], - normal_ci_runs: Sequence[Mapping[str, Any]] = ()) -> Decision: + normal_ci_runs: Sequence[Mapping[str, Any]] = (), + coverage_fingerprint: str = "") -> Decision: """Return the cancellation decision without network or environment access.""" if event != "workflow_dispatch": return Decision(False, "not a manual dispatch") @@ -43,9 +47,11 @@ def decide(*, event: str, repository: str, ref_name: str, sha: str, if not matching: return Decision(False, "no open pull request for this branch") if any((item.get("head") or {}).get("sha") == sha for item in matching): - if has_covering_ci_run(normal_ci_runs, sha): + if coverage_fingerprint and has_covering_ci_run( + normal_ci_runs, sha, coverage_fingerprint + ): return Decision(True, "pull request run covers this head") - return Decision(False, "pull request head matches but its CI run is not present") + return Decision(False, "pull request head matches but equivalent CI coverage is not present") return Decision(True, "branch moved past the pull request head") @@ -82,21 +88,29 @@ def normal_ci_runs(self, sha: str) -> list[Mapping[str, Any]]: f"/repos/{self.repository}/actions/workflows/ci.yml/runs?{query}" ) runs = body.get("workflow_runs", []) if isinstance(body, Mapping) else [] + marker = "full-suite-coverage" for run in runs: try: jobs = self._request( f"/repos/{self.repository}/actions/runs/{run['id']}/jobs?per_page=100" ) + jobs_list = jobs.get("jobs", []) if isinstance(jobs, Mapping) else [] + marker_jobs = [ + job for job in jobs_list + if str(job.get("name", "")) == marker + ] run["full_suite"] = any( - job.get("name") == "full-suite-coverage" - and ( - job.get("status") != "completed" - or job.get("conclusion") == "success" - ) - for job in jobs.get("jobs", []) - ) if isinstance(jobs, Mapping) else False + job.get("status") != "completed" or job.get("conclusion") == "success" + for job in marker_jobs + ) + run["coverage_fingerprint"] = ( + str(run.get("display_title", "")) + if marker_jobs and str(run.get("display_title", "")).startswith("v1;") + else "" + ) except (KeyError, OSError, ValueError, TypeError): run["full_suite"] = False + run["coverage_fingerprint"] = "" return runs def cancel(self, run_id: str) -> None: @@ -122,13 +136,14 @@ def main(env: Mapping[str, str] | None = None, *, check_only: bool = False) -> i ref_name = env.get("SOURCE_REF_NAME", env.get("GITHUB_REF_NAME", "")) sha = env.get("SOURCE_SHA", env.get("GITHUB_SHA", "")) run_id = env.get("SOURCE_RUN_ID", env.get("GITHUB_RUN_ID", "")) + coverage_fingerprint = env.get("SOURCE_COVERAGE_FINGERPRINT", "") pull_requests = api.open_pull_requests(ref_name) matching_sha = any( (item.get("head") or {}).get("sha") == sha and item.get("state", "open") == "open" and (item.get("head") or {}).get("repo", {}).get("full_name") == repository and (item.get("head") or {}).get("ref") == ref_name - for item in pull_requests + for item in pull_requests ) normal_ci_runs = api.normal_ci_runs(sha) if matching_sha else [] decision = decide( @@ -138,12 +153,11 @@ def main(env: Mapping[str, str] | None = None, *, check_only: bool = False) -> i sha=sha, pull_requests=pull_requests, normal_ci_runs=normal_ci_runs, + coverage_fingerprint=coverage_fingerprint, ) print(f"manual dispatch: {decision.reason}", file=sys.stderr) if decision.cancel: if check_only: - # Fail changes before any consumer can start expensive work. - # The default-branch watcher cancels the run with its own token. return 1 api.cancel(run_id) except Exception as error: # noqa: BLE001 - fail open keeps CI available diff --git a/tests/test_ci_manual_dispatch_guard.py b/tests/test_ci_manual_dispatch_guard.py index 6149606c3353..7763becd8979 100644 --- a/tests/test_ci_manual_dispatch_guard.py +++ b/tests/test_ci_manual_dispatch_guard.py @@ -15,11 +15,12 @@ def pr(ref="feat/custom-sidebar-templates", sha="6611c69", state="open"): return {"state": state, "head": {"ref": ref, "sha": sha, "repo": {"full_name": "manaflow-ai/cmux"}}} +FP = "v1;sha=6611c69;ref=feat/custom-sidebar-templates;cache=r2;release=universal;coverage=full" + + class ManualDispatchGuard(unittest.TestCase): def test_changes_check_blocks_stale_run_without_cancelling(self): - env = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GH_TOKEN": "test", - "GITHUB_REPOSITORY": "manaflow-ai/cmux", "GITHUB_REF_NAME": "topic", - "GITHUB_SHA": "old", "GITHUB_RUN_ID": "42"} + env = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GH_TOKEN": "test", "GITHUB_REPOSITORY": "manaflow-ai/cmux", "GITHUB_REF_NAME": "topic", "GITHUB_SHA": "old", "GITHUB_RUN_ID": "42"} with patch.object(module, "GitHub") as constructor: api = constructor.return_value api.open_pull_requests.return_value = [pr("topic", "new")] @@ -30,99 +31,86 @@ def test_changes_check_blocks_stale_run_without_cancelling(self): api.cancel.assert_called_once_with("42") def test_watcher_uses_source_identity_and_full_suite_marker(self): - env = {"SOURCE_EVENT_NAME": "workflow_dispatch", "GH_TOKEN": "test", - "SOURCE_REPOSITORY": "manaflow-ai/cmux", "SOURCE_REF_NAME": "topic", - "SOURCE_SHA": "same", "SOURCE_RUN_ID": "42", - "GITHUB_EVENT_NAME": "pull_request", "GITHUB_REPOSITORY": "wrong"} + fp = FP.replace("6611c69", "same").replace("feat/custom-sidebar-templates", "topic") + env = {"SOURCE_EVENT_NAME": "workflow_dispatch", "GH_TOKEN": "test", "SOURCE_REPOSITORY": "manaflow-ai/cmux", "SOURCE_REF_NAME": "topic", "SOURCE_SHA": "same", "SOURCE_RUN_ID": "42", "SOURCE_COVERAGE_FINGERPRINT": fp, "GITHUB_EVENT_NAME": "pull_request", "GITHUB_REPOSITORY": "wrong"} with patch.object(module, "GitHub") as constructor: api = constructor.return_value api.open_pull_requests.return_value = [pr("topic", "same")] - api.normal_ci_runs.return_value = [{ - "event": "pull_request", "head_sha": "same", "status": "in_progress", - "full_suite": True, - }] + api.normal_ci_runs.return_value = [{"event": "pull_request", "head_sha": "same", "status": "in_progress", "full_suite": True, "coverage_fingerprint": fp}] self.assertEqual(module.main(env, check_only=True), 1) api.cancel.assert_not_called() def test_changes_check_fails_open_on_api_error(self): - env = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GH_TOKEN": "test", - "GITHUB_REPOSITORY": "manaflow-ai/cmux"} + env = {"GITHUB_EVENT_NAME": "workflow_dispatch", "GH_TOKEN": "test", "GITHUB_REPOSITORY": "manaflow-ai/cmux"} with patch.object(module, "GitHub") as constructor: constructor.return_value.open_pull_requests.side_effect = OSError("offline") self.assertEqual(module.main(env, check_only=True), 0) constructor.return_value.cancel.assert_not_called() def test_duplicate_dispatch_is_cancelled(self): - result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", - ref_name="feat/custom-sidebar-templates", sha="6611c69", - pull_requests=[pr()], normal_ci_runs=[ - {"id": 42, "event": "pull_request", "head_sha": "6611c69", "status": "in_progress", "full_suite": True} - ]) + result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", ref_name="feat/custom-sidebar-templates", sha="6611c69", pull_requests=[pr()], normal_ci_runs=[{"id": 42, "event": "pull_request", "head_sha": "6611c69", "status": "in_progress", "full_suite": True, "coverage_fingerprint": FP}], coverage_fingerprint=FP) self.assertEqual(result, module.Decision(True, "pull request run covers this head")) def test_matching_head_without_normal_ci_run_is_kept(self): - result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", - ref_name="feat/custom-sidebar-templates", sha="6611c69", - pull_requests=[pr()]) - self.assertEqual(result, module.Decision(False, "pull request head matches but its CI run is not present")) + result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", ref_name="feat/custom-sidebar-templates", sha="6611c69", pull_requests=[pr()]) + self.assertEqual(result, module.Decision(False, "pull request head matches but equivalent CI coverage is not present")) def test_cancelled_or_skipped_ci_run_does_not_cover_head(self): - runs = [ - {"event": "pull_request", "head_sha": "6611c69", "status": "completed", "conclusion": "cancelled", "full_suite": True}, - {"event": "pull_request", "head_sha": "6611c69", "status": "completed", "conclusion": "skipped", "full_suite": True}, - ] - self.assertFalse(module.has_covering_ci_run(runs, "6611c69")) + runs = [{"event": "pull_request", "head_sha": "6611c69", "status": "completed", "conclusion": "cancelled", "full_suite": True, "coverage_fingerprint": FP}, {"event": "pull_request", "head_sha": "6611c69", "status": "completed", "conclusion": "skipped", "full_suite": True, "coverage_fingerprint": FP}] + self.assertFalse(module.has_covering_ci_run(runs, "6611c69", FP)) def test_compile_only_marker_does_not_cover_dispatch(self): - self.assertFalse(module.has_covering_ci_run([ - {"event": "pull_request", "head_sha": "6611c69", "status": "in_progress", "full_suite": False}, - ], "6611c69")) + self.assertFalse(module.has_covering_ci_run([{"event": "pull_request", "head_sha": "6611c69", "status": "in_progress", "full_suite": False}], "6611c69", FP)) + + def test_compile_only_pr_stays_eligible_for_full_manual_dispatch(self): + result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", ref_name="feat/custom-sidebar-templates", sha="6611c69", pull_requests=[pr()], coverage_fingerprint=FP, normal_ci_runs=[{"event": "pull_request", "head_sha": "6611c69", "status": "in_progress", "full_suite": False}]) + self.assertFalse(result.cancel) + + def test_unknown_coverage_stays_eligible(self): + self.assertFalse(module.has_covering_ci_run([{"event": "pull_request", "head_sha": "6611c69", "status": "in_progress", "full_suite": True}], "6611c69", FP)) + + def test_fuller_manual_configuration_stays_eligible(self): + manual = FP.replace("cache=r2", "cache=warp") + self.assertFalse(module.has_covering_ci_run([{"event": "pull_request", "head_sha": "6611c69", "status": "in_progress", "full_suite": True, "coverage_fingerprint": FP}], "6611c69", manual)) def test_completed_full_suite_success_covers_head(self): - self.assertTrue(module.has_covering_ci_run([ - {"event": "pull_request", "head_sha": "6611c69", "status": "completed", "conclusion": "success", "full_suite": True}, - ], "6611c69")) + self.assertTrue(module.has_covering_ci_run([{"event": "pull_request", "head_sha": "6611c69", "status": "completed", "conclusion": "success", "full_suite": True, "coverage_fingerprint": FP}], "6611c69", FP)) def test_completed_failed_full_suite_does_not_cover_head(self): - self.assertFalse(module.has_covering_ci_run([ - {"event": "pull_request", "head_sha": "6611c69", "status": "completed", "conclusion": "failure", "full_suite": True}, - ], "6611c69")) + self.assertFalse(module.has_covering_ci_run([{"event": "pull_request", "head_sha": "6611c69", "status": "completed", "conclusion": "failure", "full_suite": True, "coverage_fingerprint": FP}], "6611c69", FP)) def test_failed_coverage_marker_is_not_published_as_coverage(self): api = module.GitHub("test", "manaflow-ai/cmux") - with patch.object(api, "_request", side_effect=[ - {"workflow_runs": [{"id": 7}]}, - {"jobs": [{"name": "full-suite-coverage", "status": "completed", "conclusion": "failure"}]}, - ]): + with patch.object(api, "_request", side_effect=[{"workflow_runs": [{"id": 7, "display_title": FP}]}, {"jobs": [{"name": "full-suite-coverage", "status": "completed", "conclusion": "failure"}]}]): self.assertEqual(api.normal_ci_runs("6611c69")[0]["full_suite"], False) + def test_marker_reads_fingerprint_from_run_title(self): + api = module.GitHub("test", "manaflow-ai/cmux") + with patch.object(api, "_request", side_effect=[{"workflow_runs": [{"id": 7, "display_title": FP}]}, {"jobs": [{"name": "full-suite-coverage", "status": "in_progress"}]}]): + run = api.normal_ci_runs("6611c69")[0] + self.assertTrue(run["full_suite"]) + self.assertEqual(run["coverage_fingerprint"], FP) + def test_other_sha_or_event_does_not_cover_head(self): - runs = [{"event": "workflow_dispatch", "head_sha": "6611c69", "status": "in_progress"}] - self.assertFalse(module.has_covering_ci_run(runs, "6611c69")) + self.assertFalse(module.has_covering_ci_run([{"event": "workflow_dispatch", "head_sha": "6611c69", "status": "in_progress"}], "6611c69", FP)) def test_stale_dispatch_is_cancelled(self): - result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", - ref_name="feat/custom-sidebar-templates", sha="7814d8f", - pull_requests=[pr()]) + result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", ref_name="feat/custom-sidebar-templates", sha="7814d8f", pull_requests=[pr()]) self.assertEqual(result, module.Decision(True, "branch moved past the pull request head")) def test_current_branch_without_pr_is_kept(self): - result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", - ref_name="topic", sha="123", pull_requests=[]) + result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", ref_name="topic", sha="123", pull_requests=[]) self.assertFalse(result.cancel) def test_non_dispatch_is_never_cancelled(self): - result = module.decide(event="pull_request", repository="manaflow-ai/cmux", - ref_name="feat", sha="123", pull_requests=[pr("feat", "123")]) + result = module.decide(event="pull_request", repository="manaflow-ai/cmux", ref_name="feat", sha="123", pull_requests=[pr("feat", "123")]) self.assertFalse(result.cancel) def test_closed_or_other_repository_pr_is_ignored(self): closed = pr(state="closed") other = pr() other["head"]["repo"]["full_name"] = "teamleaderleo/cmux" - result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", - ref_name="feat/custom-sidebar-templates", sha="7814d8f", - pull_requests=[closed, other]) + result = module.decide(event="workflow_dispatch", repository="manaflow-ai/cmux", ref_name="feat/custom-sidebar-templates", sha="7814d8f", pull_requests=[closed, other]) self.assertFalse(result.cancel) diff --git a/tests/test_ci_manual_dispatch_guard_workflow.py b/tests/test_ci_manual_dispatch_guard_workflow.py index 9aca66f85bb4..1eb8244c654f 100644 --- a/tests/test_ci_manual_dispatch_guard_workflow.py +++ b/tests/test_ci_manual_dispatch_guard_workflow.py @@ -23,7 +23,7 @@ def test_watcher_is_requested_ci_workflow_run() -> None: assert document["env"]["SOURCE_WORKFLOW_PATHS"] == ".github/workflows/ci.yml" assert document["permissions"] == {} watcher_env = document["jobs"]["guard"]["steps"][-1]["env"] - for key in ("SOURCE_EVENT_NAME", "SOURCE_REPOSITORY", "SOURCE_REF_NAME", "SOURCE_SHA", "SOURCE_RUN_ID"): + for key in ("SOURCE_EVENT_NAME", "SOURCE_REPOSITORY", "SOURCE_REF_NAME", "SOURCE_SHA", "SOURCE_RUN_ID", "SOURCE_COVERAGE_FINGERPRINT"): assert key in watcher_env assert "GITHUB_EVENT_NAME" not in watcher_env @@ -33,6 +33,8 @@ def test_full_suite_coverage_marker_is_only_for_full_suite() -> None: marker = document["jobs"]["full-suite-coverage"] assert marker["needs"] == "changes" assert "needs.changes.outputs.full_suite == 'true'" in marker["if"] + assert marker.get("name", "full-suite-coverage") == "full-suite-coverage" + assert "coverage_fingerprint" in document["jobs"]["changes"]["outputs"] def test_only_manual_dispatches_get_a_writer() -> None: