Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
189 changes: 86 additions & 103 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1385,102 +1385,6 @@ jobs:
secrets:
GLAEDA_ROUTE_APP_KEY: ${{ secrets.GLAEDA_ROUTE_APP_KEY }}

tests:
name: tests
needs:
- changes
- linux-preflight
- macos
- web
# Keep the aggregate verdict after a failed dependency, but let a
# cancelled run stop without spending another runner minute on a rollup.
if: ${{ !cancelled() }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
timeout-minutes: 5
steps:
- name: Check platform workflow routing
env:
TESTS_NEEDS: ${{ toJSON(needs) }}
run: |
python3 - <<'PY'
import json
import os
import sys

needs = json.loads(os.environ["TESTS_NEEDS"])
# A cancelled job reaches this gate as a `cancelled` result, and the
# routing verdict below would report it as though the pull request had
# a routing defect. It does not: the run was stopped before it tested
# anything. Say that instead, and still fail. This gate is rolled
# into the required ci-status check, so reporting success on a
# cancelled run would put a green on a required check that no suite
# stood behind, and a pull request could merge with nothing having
# tested it. Rerunning the run clears it.
cancelled = sorted(name for name, data in needs.items()
if data["result"] == "cancelled")
if cancelled:
print("cancelled: " + ", ".join(cancelled), file=sys.stderr)
print("this run was stopped before it reported a test verdict; "
"rerun it, or push again, to get one", file=sys.stderr)
sys.exit(1)
changes = needs["changes"]
if changes["result"] != "success":
print(f"changes: {changes['result']}", file=sys.stderr)
sys.exit(1)
outputs = changes.get("outputs", {})
macos_route = outputs.get("macos")
if macos_route not in {"true", "false"}:
print(f"invalid route macos={macos_route!r}", file=sys.stderr)
sys.exit(1)
preflight = needs["linux-preflight"]
if macos_route == "true":
if preflight["result"] != "success":
print(f"linux preflight did not pass: {preflight['result']}", file=sys.stderr)
sys.exit(1)
elif preflight["result"] not in {"success", "skipped"}:
print(
f"linux preflight had unexpected result: {preflight['result']}",
file=sys.stderr,
)
sys.exit(1)
macos_result = needs["macos"]["result"]
full_suite = outputs.get("full_suite")
compile_admitted = outputs.get("compile_admitted")
# The package-test lane is routed independently of compile admission
# and of the macOS area, so it can require the called workflow on its
# own. Keep this in sync with the `macos` job's own `if:`.
swift_packages = outputs.get("swift_packages") == "true"
# `unit-ci` asks for app-host unit tests without the full suite, so a
# routed macOS run is required even when compile admission reused a
# product and full_suite stayed false.
unit_suite = outputs.get("unit_suite") == "true"
macos_work_required = swift_packages or (
(
unit_suite
or outputs.get("cli") == "true"
or not (full_suite == "false" and compile_admitted == "true")
)
and (macos_route == "true" or outputs.get("cli") == "true")
)
if macos_work_required:
if macos_result != "success":
print(f"macOS workflow did not pass: {macos_result}", file=sys.stderr)
sys.exit(1)
elif macos_result not in {"success", "skipped"}:
print(f"macOS workflow had unexpected result: {macos_result}", file=sys.stderr)
sys.exit(1)
web_result = needs["web"]["result"]
if web_result not in {"success", "skipped"}:
print(f"web workflow did not pass: {web_result}", file=sys.stderr)
sys.exit(1)
print(f"changes.macos={macos_route}")
print(f"changes.full_suite={outputs.get('full_suite')}")
print(f"changes.swift_packages={outputs.get('swift_packages')}")
print(f"linux-preflight={preflight['result']}")
print(f"macos={macos_result}")
print(f"web={web_result}")
PY

linux-preflight:
name: linux-preflight
# Retain the Linux verdict in the required final statuses. It no longer
Expand Down Expand Up @@ -1741,9 +1645,6 @@ jobs:
- name: Download the pinned GhosttyKit
run: ./scripts/download-prebuilt-ghosttykit.sh

- name: Select Xcode
run: ./scripts/select-ci-xcode.sh

- name: Build and package the production-authenticated app
id: artifact
env:
Expand Down Expand Up @@ -1793,16 +1694,99 @@ jobs:
- linux-preflight
- macos-admission-gate
- macos
- tests
# A cancelled run already has its terminal outcome; the required status
# remains cancelled and does not need a post-cancel rollup.
if: ${{ !cancelled() }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
# One step that reads `needs` and decides. Its siblings are bounded at 5
# to 20 minutes; this one was inheriting the six-hour default, as the
# These verdict steps read `needs` and decide. Their siblings are bounded
# at 5 to 20 minutes; this job was inheriting the six-hour default, as the
# required check every pull request waits on.
timeout-minutes: 5
steps:
- name: Check platform workflow routing
env:
PLATFORM_NEEDS: ${{ toJSON(needs) }}
run: |
python3 - <<'PY'
import json
import os
import sys

needs = json.loads(os.environ["PLATFORM_NEEDS"])
# A cancelled job reaches this gate as a `cancelled` result, and the
# routing verdict below would report it as though the pull request had
# a routing defect. It does not: the run was stopped before it tested
# anything. Say that instead, and still fail. This gate is rolled
# into the required ci-status check, so reporting success on a
# cancelled run would put a green on a required check that no suite
# stood behind, and a pull request could merge with nothing having
# tested it. Rerunning the run clears it.
platform_names = ("changes", "linux-preflight", "macos", "web")
cancelled = sorted(name for name in platform_names
if needs[name]["result"] == "cancelled")
if cancelled:
print("cancelled: " + ", ".join(cancelled), file=sys.stderr)
print("this run was stopped before it reported a test verdict; "
"rerun it, or push again, to get one", file=sys.stderr)
sys.exit(1)
changes = needs["changes"]
if changes["result"] != "success":
print(f"changes: {changes['result']}", file=sys.stderr)
sys.exit(1)
outputs = changes.get("outputs", {})
macos_route = outputs.get("macos")
if macos_route not in {"true", "false"}:
print(f"invalid route macos={macos_route!r}", file=sys.stderr)
sys.exit(1)
preflight = needs["linux-preflight"]
if macos_route == "true":
if preflight["result"] != "success":
print(f"linux preflight did not pass: {preflight['result']}", file=sys.stderr)
sys.exit(1)
elif preflight["result"] not in {"success", "skipped"}:
print(
f"linux preflight had unexpected result: {preflight['result']}",
file=sys.stderr,
)
sys.exit(1)
macos_result = needs["macos"]["result"]
full_suite = outputs.get("full_suite")
compile_admitted = outputs.get("compile_admitted")
# The package-test lane is routed independently of compile admission
# and of the macOS area, so it can require the called workflow on its
# own. Keep this in sync with the `macos` job's own `if:`.
swift_packages = outputs.get("swift_packages") == "true"
# `unit-ci` asks for app-host unit tests without the full suite, so a
# routed macOS run is required even when compile admission reused a
# product and full_suite stayed false.
unit_suite = outputs.get("unit_suite") == "true"
macos_work_required = swift_packages or (
(
unit_suite
or outputs.get("cli") == "true"
or not (full_suite == "false" and compile_admitted == "true")
)
and (macos_route == "true" or outputs.get("cli") == "true")
)
if macos_work_required:
if macos_result != "success":
print(f"macOS workflow did not pass: {macos_result}", file=sys.stderr)
sys.exit(1)
elif macos_result not in {"success", "skipped"}:
print(f"macOS workflow had unexpected result: {macos_result}", file=sys.stderr)
sys.exit(1)
web_result = needs["web"]["result"]
if web_result not in {"success", "skipped"}:
print(f"web workflow did not pass: {web_result}", file=sys.stderr)
sys.exit(1)
print(f"changes.macos={macos_route}")
print(f"changes.full_suite={outputs.get('full_suite')}")
print(f"changes.swift_packages={outputs.get('swift_packages')}")
print(f"linux-preflight={preflight['result']}")
print(f"macos={macos_result}")
print(f"web={web_result}")
PY

- name: Check routed CI jobs
env:
CI_NEEDS: ${{ toJSON(needs) }}
Expand Down Expand Up @@ -1857,7 +1841,6 @@ jobs:
- linux-preflight
- macos-admission-gate
- macos
- tests
if: ${{ !cancelled() }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
timeout-minutes: 5
Expand Down
2 changes: 1 addition & 1 deletion scripts/ci/classify_failures.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
# Jobs that only read other jobs' results. Their names as the jobs API reports
# them; tests/test_ci_classify_failures.py derives each from its workflow.
GATE_JOBS = frozenset({
"ci-status", "tests", "CI timing", "linux-preflight", "macOS admission gate",
"ci-status", "CI timing", "linux-preflight", "macOS admission gate",
"macos / macOS status", "guards / Guard status", "web / Web status",
})
MAX_EVIDENCE_CHARS = 300
Expand Down
50 changes: 25 additions & 25 deletions tests/test_ci_change_areas.py
Original file line number Diff line number Diff line change
Expand Up @@ -3077,7 +3077,7 @@ def route_ci_workflow_edit(

def test_routing_policy_edits_skip_the_mac_standalone_lanes() -> None:
real = CI_WORKFLOW.read_text(encoding="utf-8")
for job in ("changes", "ci-status", "guards", "tests", "linux-preflight", "macos-admission-gate"):
for job in ("changes", "ci-status", "guards", "linux-preflight", "macos-admission-gate"):
values = route_ci_workflow_edit(edit_job(real, job))
for name in MAC_STANDALONE_OUTPUTS:
assert values[name] == "false", (job, name, values)
Expand Down Expand Up @@ -3305,20 +3305,20 @@ def test_ci_status_job_accepts_skipped_routed_jobs() -> None:
"web",
"linux-preflight",
"macos",
"tests",
]:
assert f" - {job_name}" in block
for job_name in MACOS_JOBS:
assert f" - {job_name}" not in block

assert "if: ${{ !cancelled() }}" in block
assert "PLATFORM_NEEDS: ${{ toJSON(needs) }}" in block
assert 'allowed = {"success", "skipped"}' in block


def test_required_tests_status_waits_for_platform_workflows() -> None:
block = workflow_job_block("tests")
def test_ci_status_waits_for_platform_workflows() -> None:
block = workflow_job_block("ci-status")

assert "name: tests" in block
assert "name: Check platform workflow routing" in block
for job_name in ("changes", "linux-preflight", "macos", "web"):
assert f" - {job_name}" in block
for job_name in MACOS_JOBS:
Expand Down Expand Up @@ -3821,19 +3821,19 @@ def test_macos_admission_gate_admits_whenever_it_cannot_prove_a_failure() -> Non
assert "already failed" not in result.stdout + result.stderr, label


def run_tests_gate(needs: dict) -> subprocess.CompletedProcess:
script = workflow_job_step_script("tests", "Check platform workflow routing")
def run_platform_gate(needs: dict) -> subprocess.CompletedProcess:
script = workflow_job_step_script("ci-status", "Check platform workflow routing")
body = script.split("python3 - <<'PY'\n", 1)[1].rsplit("\nPY", 1)[0]
return subprocess.run(
[sys.executable, "-c", textwrap.dedent(body)],
env={**os.environ, "TESTS_NEEDS": json.dumps(needs)},
env={**os.environ, "PLATFORM_NEEDS": json.dumps(needs)},
capture_output=True,
text=True,
check=False,
)


def tests_gate_needs(
def platform_gate_needs(
macos: str = "true",
macos_result: str = "success",
web_result: str = "skipped",
Expand All @@ -3855,26 +3855,26 @@ def tests_gate_needs(
}


def test_platform_workflow_results_gate_tests_status() -> None:
assert run_tests_gate(tests_gate_needs()).returncode == 0
assert run_tests_gate(tests_gate_needs(macos_result="failure")).returncode == 1
assert run_tests_gate(tests_gate_needs(macos_result="skipped")).returncode == 1
assert run_tests_gate(tests_gate_needs(macos="false", macos_result="skipped")).returncode == 0
assert run_tests_gate(
tests_gate_needs(
def test_platform_workflow_results_gate_ci_status() -> None:
assert run_platform_gate(platform_gate_needs()).returncode == 0
assert run_platform_gate(platform_gate_needs(macos_result="failure")).returncode == 1
assert run_platform_gate(platform_gate_needs(macos_result="skipped")).returncode == 1
assert run_platform_gate(platform_gate_needs(macos="false", macos_result="skipped")).returncode == 0
assert run_platform_gate(
platform_gate_needs(
macos_result="skipped",
full_suite="false",
compile_admitted="true",
)
).returncode == 0
assert run_tests_gate(tests_gate_needs(web_result="failure")).returncode == 1
assert run_platform_gate(platform_gate_needs(web_result="failure")).returncode == 1


def test_linux_failure_still_blocks_tests_after_macos_succeeds() -> None:
def test_linux_failure_still_blocks_ci_status_after_macos_succeeds() -> None:
for outcome in ("failure", "cancelled", "skipped"):
needs = tests_gate_needs(macos_result="success")
needs = platform_gate_needs(macos_result="success")
needs["linux-preflight"]["result"] = outcome
result = run_tests_gate(needs)
result = run_platform_gate(needs)
assert result.returncode != 0, outcome
if outcome == "cancelled":
assert "cancelled: linux-preflight" in result.stderr
Expand Down Expand Up @@ -4370,10 +4370,10 @@ def test_published_fingerprint_artifact_is_the_one_the_lookup_reads() -> None:


def test_full_suite_runs_still_require_the_suite() -> None:
assert run_tests_gate(tests_gate_needs("true", macos_result="success")).returncode == 0
assert run_tests_gate(tests_gate_needs("true", macos_result="skipped")).returncode == 1
assert run_platform_gate(platform_gate_needs("true", macos_result="success")).returncode == 0
assert run_platform_gate(platform_gate_needs("true", macos_result="skipped")).returncode == 1
# A missing route output must never relax the aggregate platform gate.
assert run_tests_gate(tests_gate_needs(None, macos_result="skipped")).returncode == 1
assert run_platform_gate(platform_gate_needs(None, macos_result="skipped")).returncode == 1


def test_only_pull_requests_under_the_compile_only_policy_skip_the_suite() -> None:
Expand Down Expand Up @@ -5562,9 +5562,9 @@ def test_the_unit_tier_is_routed_end_to_end() -> None:


def test_a_unit_ci_run_still_requires_the_macos_workflow_to_pass() -> None:
# The tests job restates the macos `if:` as a result contract; a routed
# The ci-status job restates the macos `if:` as a result contract; a routed
# unit-ci run that skipped macOS must not read as legitimately unrouted.
gate = workflow_job_block("tests")
gate = workflow_job_block("ci-status")
assert 'unit_suite = outputs.get("unit_suite") == "true"' in gate
assert "unit_suite" in gate.split("macos_work_required")[1].split(")")[0]

Expand Down
4 changes: 2 additions & 2 deletions tests/test_ci_cli_product_routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,8 +94,8 @@ def test_required_status_rejects_missing_targeted_cli_work(self):
result = run_step(job, "Check routed macOS jobs", env)
self.assertNotEqual(result.returncode, 0, (macos, name, outcome))

def test_outer_tests_gate_requires_targeted_workflow(self):
job = self.caller["tests"]
def test_outer_platform_gate_requires_targeted_workflow(self):
job = self.caller["ci-status"]
step = next(step for step in job["steps"] if "macos_work_required" in step.get("run", ""))
for macos in ("false", "true"):
for outcome in ("success", "skipped", "failure", "cancelled"):
Expand Down
12 changes: 6 additions & 6 deletions tests/test_ci_linux_guard_routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
module,
run_guard_status,
run_linux_preflight,
run_tests_gate,
tests_gate_needs,
run_platform_gate,
platform_gate_needs,
workflow_job_block,
workflow_job_step_script,
)
Expand Down Expand Up @@ -334,14 +334,14 @@ def test_linux_preflight_skips_when_macos_route_is_false(self):
block,
)

no_macos = tests_gate_needs(macos="false", macos_result="skipped")
no_macos = platform_gate_needs(macos="false", macos_result="skipped")
no_macos["linux-preflight"]["result"] = "skipped"
result = run_tests_gate(no_macos)
result = run_platform_gate(no_macos)
self.assertEqual(result.returncode, 0, result.stderr)

macos = tests_gate_needs()
macos = platform_gate_needs()
macos["linux-preflight"]["result"] = "skipped"
result = run_tests_gate(macos)
result = run_platform_gate(macos)
self.assertNotEqual(result.returncode, 0)
self.assertIn("linux preflight did not pass: skipped", result.stderr)

Expand Down
Loading