diff --git a/scripts/ci/find_admitted_build.py b/scripts/ci/find_admitted_build.py index 3309f0e9a487..2937fed31a15 100755 --- a/scripts/ci/find_admitted_build.py +++ b/scripts/ci/find_admitted_build.py @@ -23,6 +23,8 @@ ADMISSION_JOB = "macOS compile admission" ARTIFACT_PREFIX = "build-inputs-" RUNS_TO_CHECK = 6 +JOB_PAGES_TO_CHECK = 3 +JOBS_PER_PAGE = 100 Api = Callable[[str], dict] @@ -43,14 +45,20 @@ def admitted_run(api: Api, repository: str, branch: str, fingerprint: str, curre for run in runs: if run["id"] == current_run_id or run["head_repository"]["full_name"] != repository: continue - jobs = api(f"repos/{repository}/actions/runs/{run['id']}/jobs?filter=all&per_page=100").get("jobs", []) - admitted_attempts = { - job["run_attempt"] for job in jobs if job["name"] == ADMISSION_JOB and job["conclusion"] == "success" - } - for attempt in sorted(admitted_attempts): - artifact_query = urlencode({"name": artifact_name(fingerprint, attempt)}) - if api(f"repos/{repository}/actions/runs/{run['id']}/artifacts?{artifact_query}").get("total_count"): - return run["html_url"] + # filter=all includes reruns, whose jobs can fill more than one + # page. Bound this optional lookup; a miss just compiles again. + for page in range(1, JOB_PAGES_TO_CHECK + 1): + jobs_query = urlencode({"filter": "all", "per_page": JOBS_PER_PAGE, "page": page}) + jobs = api(f"repos/{repository}/actions/runs/{run['id']}/jobs?{jobs_query}").get("jobs", []) + admitted_attempts = { + job["run_attempt"] for job in jobs if job["name"] == ADMISSION_JOB and job["conclusion"] == "success" + } + for attempt in sorted(admitted_attempts): + artifact_query = urlencode({"name": artifact_name(fingerprint, attempt)}) + if api(f"repos/{repository}/actions/runs/{run['id']}/artifacts?{artifact_query}").get("total_count"): + return run["html_url"] + if len(jobs) < JOBS_PER_PAGE: + break except (subprocess.CalledProcessError, json.JSONDecodeError, KeyError, TypeError, AttributeError) as error: print(f"lookup failed, compiling: {error}", file=sys.stderr) return None diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 0296817eeecb..791c73f144d2 100644 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -1168,7 +1168,10 @@ def api(path: str) -> dict: (name,) = query["name"] return {"total_count": artifacts.get(run_id, []).count(name)} assert query["filter"] == ["all"], path - return {"jobs": jobs.get(run_id, [])} + page = int(query.get("page", ["1"])[0]) + per_page = int(query["per_page"][0]) + run_jobs = jobs.get(run_id, []) + return {"jobs": run_jobs[(page - 1) * per_page:page * per_page]} return api @@ -1233,6 +1236,84 @@ def find(fingerprint: str, artifacts: list[str], jobs: list[dict]) -> str | None assert find("old", [artifact_name("old", 1)], jobs) is None +def test_admission_lookup_finds_matching_attempt_beyond_the_first_jobs_page() -> None: + original_path = sys.path.copy() + try: + sys.path.insert(0, str(ROOT / "scripts/ci")) + from find_admitted_build import admitted_run, artifact_name + finally: + sys.path[:] = original_path + + # An earlier attempt's fan-out fills the first page. Only the later + # attempt compiled the desired inputs successfully. + earlier_jobs = [admission_job("failure")] * 100 + jobs = earlier_jobs + [admission_job("success", run_attempt=2)] + for fingerprint, expected in (("new", "https://example/8"), ("old", None)): + api = admission_api( + [admission_run(8)], + {8: [artifact_name("old", 1), artifact_name("new", 2)]}, + {8: jobs}, + ) + assert admitted_run(api, "manaflow-ai/cmux", "feature", fingerprint, current_run_id=9) == expected + + +def test_admission_lookup_falls_back_when_a_later_jobs_page_fails() -> None: + original_path = sys.path.copy() + try: + sys.path.insert(0, str(ROOT / "scripts/ci")) + from find_admitted_build import admitted_run, artifact_name + from urllib.parse import parse_qs, urlsplit + finally: + sys.path[:] = original_path + + base_api = admission_api( + [admission_run(8)], {8: [artifact_name("abc", 2)]}, + {8: [admission_job("failure")] * 100 + [admission_job("success", run_attempt=2)]}, + ) + pages = [] + + def api(path: str) -> dict: + url = urlsplit(path) + if url.path.endswith("/jobs"): + page = int(parse_qs(url.query).get("page", ["1"])[0]) + pages.append(page) + if page == 2: + raise subprocess.CalledProcessError(1, "gh") + return base_api(path) + + assert admitted_run(api, "manaflow-ai/cmux", "feature", "abc", current_run_id=9) is None + assert pages == [1, 2], "the lookup must reach the failed page before falling back" + + +def test_admission_lookup_bounds_job_pages_and_stops_after_a_match() -> None: + original_path = sys.path.copy() + try: + sys.path.insert(0, str(ROOT / "scripts/ci")) + from find_admitted_build import admitted_run, artifact_name + from urllib.parse import parse_qs, urlsplit + finally: + sys.path[:] = original_path + + # Cap lookup work even when a run has many attempts. Missing an old + # admission is safe: this candidate compiles normally instead. + for jobs, expected, expected_pages in ( + ([admission_job("failure")] * 1000, None, [1, 2, 3]), + ([admission_job("success")] * 100, "https://example/8", [1]), + ([], None, [1]), + ): + base_api = admission_api([admission_run(8)], {8: [artifact_name("abc", 1)]}, {8: jobs}) + pages = [] + + def api(path: str) -> dict: + url = urlsplit(path) + if url.path.endswith("/jobs"): + pages.append(int(parse_qs(url.query).get("page", ["1"])[0])) + return base_api(path) + + assert admitted_run(api, "manaflow-ai/cmux", "feature", "abc", current_run_id=9) == expected + assert pages == expected_pages + + def test_admission_lookup_sends_reserved_branch_characters_literally() -> None: sys.path.insert(0, str(ROOT / "scripts/ci")) from find_admitted_build import admitted_run, artifact_name