Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions tests/test_ci_self_hosted_guard.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1595,6 +1595,60 @@ check_no_self_hosted_fleet_runners() {
# Capability is accepted only as the complete NEXT trusted selector.
local next_trusted_selector="fromJSON(format('[\"{0}\", \"{1}\", \"glaeda-trusted-next-nightly\"]', vars.CI_NIGHTLY_NEXT_TRUSTED_POOL, vars.CI_NIGHTLY_NEXT_TRUSTED_RUNNER))"
local next_repository_guard="github.repository == 'manaflow-ai/cmux' && "
# The main metadata bootstrap has no preceding picker. Accept its complete
# main-only selector in this job alone, without exempting other fleet labels
# or allowing secrets onto the shared nightly pool. RC and forks keep their
# existing cloud routes; untrusted workflow triggers cannot use this exception.
local nightly_bootstrap_line
nightly_bootstrap_line="$(python3 - "$ROOT_DIR/.github/workflows/nightly.yml" <<'PYTHON'
import json
import re
import sys
from pathlib import Path

import yaml

path = Path(sys.argv[1])
if not path.exists():
sys.exit(0)
text = path.read_text()
# An alias can give decide another job's source position. Do not grant a
# line exemption through aliases or merges.
if any(isinstance(token, yaml.AliasToken) for token in yaml.scan(text)):
sys.exit(0)
document = yaml.load(text, Loader=yaml.BaseLoader) or {}
events = document.get("on", {})
if isinstance(events, str):
events = [events]
if not events or not set(events) <= {"push", "schedule", "workflow_dispatch"}:
sys.exit(0)
job = (document.get("jobs") or {}).get("decide") or {}
selector = ("${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || "
"github.ref == 'refs/heads/main' && "
"(startsWith(vars.CI_NIGHTLY_NEXT_BUILD_RUNNER, 'glaeda-nightly-') && "
"vars.CI_NIGHTLY_NEXT_BUILD_RUNNER || 'glaeda-nightly-std-xcode-26.6') || "
"'blacksmith-2vcpu-ubuntu-2404' }}")
if job.get("runs-on") != selector or "environment" in job:
sys.exit(0)
if re.search(r"\bsecrets\b|create-github-app-token|\bprivate-key\b",
json.dumps([document.get("env"), job]), re.IGNORECASE):
sys.exit(0)
# Source positions bind the exemption to this job's runs-on, not another
# job that happens to spell the same selector. Duplicate keys fail closed.
def mapping(node):
if not isinstance(node, yaml.MappingNode):
raise ValueError("expected a YAML mapping")
values = {key.value: value for key, value in node.value}
if len(values) != len(node.value):
raise ValueError("duplicate YAML keys")
return values

root = mapping(yaml.compose(text, Loader=yaml.BaseLoader))
node = mapping(mapping(root["jobs"])["decide"])["runs-on"]
if node.start_mark.line == node.end_mark.line:
print(f"{path}:{node.start_mark.line + 1}:")
PYTHON
)" || return 1
local hits="" line content content_without_allowed
# Inspect runner-selection lines only: runs-on:, matrix `os:`, and scalar list
# items (` - <label>`, which covers dispatch runner dropdowns and multi-line
Expand All @@ -1603,6 +1657,7 @@ check_no_self_hosted_fleet_runners() {
# before matching the value so the checkout path (which contains "cmux") can
# never match the bare `cmux` label.
while IFS= read -r line; do
[[ -n "$nightly_bootstrap_line" && "$line" == "$nightly_bootstrap_line"* ]] && continue
content="${line#*:*:}"
if [[ "$(basename "${line%%:*}")" == *nightly*.yml || "$(basename "${line%%:*}")" == *notarization*.yml ]]; then
content="$(printf '%s\n' "$content" | sed -E "s/$nightly_hosted//g")"
Expand Down
73 changes: 73 additions & 0 deletions tests/test_runner_label_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,79 @@ def check():
self.assertNotEqual(check().returncode, 0)


class MainNightlyBootstrapGuard(unittest.TestCase):
SELECTOR = ("${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || "
"github.ref == 'refs/heads/main' && "
"(startsWith(vars.CI_NIGHTLY_NEXT_BUILD_RUNNER, 'glaeda-nightly-') && "
"vars.CI_NIGHTLY_NEXT_BUILD_RUNNER || 'glaeda-nightly-std-xcode-26.6') || "
"'blacksmith-2vcpu-ubuntu-2404' }}")

def check(self, *, selector=SELECTOR, job="decide", workflow="nightly.yml",
trigger="on: [push, schedule, workflow_dispatch]", extra="", shared=""):
return self.run_guard(
f"{trigger}\n{shared}jobs:\n {job}:\n runs-on: {selector}\n"
f"{extra} steps:\n - run: echo metadata\n", workflow)

def run_guard(self, text, workflow="nightly.yml"):
import subprocess
import tempfile

script = ('set -euo pipefail\nROOT_DIR="$1"\nE2E_FILE="$1/.github/workflows/test-e2e.yml"\n'
'check_no_self_hosted_fleet_runners() {\n'
+ _guard_function(GUARD_SCRIPT.read_text())
+ '\n}\ncheck_no_self_hosted_fleet_runners')
with tempfile.TemporaryDirectory() as directory:
workflows = Path(directory) / ".github/workflows"
workflows.mkdir(parents=True)
(workflows / "test-e2e.yml").write_text("on: workflow_dispatch\n")
(workflows / workflow).write_text(text)
return subprocess.run(["bash", "-c", script, "guard", directory],
capture_output=True, text=True)

def test_approved_main_bootstrap_passes_the_real_guard(self):
result = self.check()
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)

def test_real_main_workflow_accepts_only_its_bootstrap_line(self):
text = (ROOT / ".github/workflows/nightly.yml").read_text()
text, count = re.subn(r"(\n decide:\n[\s\S]*?\n runs-on: )[^\n]+",
lambda match: match[1] + self.SELECTOR, text, count=1)
self.assertEqual(count, 1)
result = self.run_guard(text)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
text += "\n other:\n runs-on: " + self.SELECTOR + "\n"
result = self.run_guard(text)
self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)

def test_other_routes_and_jobs_stay_forbidden(self):
cases = {
"RC instead of main": {"selector": self.SELECTOR.replace("refs/heads/main", "refs/heads/rc/1.0")},
"no ref gate": {"selector": self.SELECTOR.replace("github.ref == 'refs/heads/main' && ", "")},
"no owner gate": {"selector": self.SELECTOR.replace("github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || ", "")},
"other job": {"job": "build"},
"other workflow": {"workflow": "another.yml"},
"untrusted trigger": {"trigger": "on: [push, pull_request_target]"},
"job secret": {"extra": " env:\n TOKEN: ${{ secrets.API_KEY }}\n"},
"shared secret": {"shared": "env:\n TOKEN: ${{ secrets.API_KEY }}\n"},
"bracket secret": {"extra": " env:\n TOKEN: ${{ secrets['API_KEY'] }}\n"},
"whole secret context": {"extra": " env:\n TOKEN: ${{ toJSON(SECRETS) }}\n"},
"environment": {"extra": " environment: release\n"},
"extra fleet fallback": {"selector": self.SELECTOR.replace(" }}", " || 'glaeda-side-std-xcode-26.6' }}")},
}
for name, kwargs in cases.items():
with self.subTest(case=name):
result = self.check(**kwargs)
self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("FAIL: workflow references a self-hosted", result.stdout)

def test_alias_cannot_exempt_another_jobs_runner(self):
text = ("on: push\njobs:\n other:\n runs-on: &route " + self.SELECTOR
+ "\n decide:\n runs-on: *route\n")
result = self.run_guard(text)
self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("FAIL: workflow references a self-hosted", result.stdout)


class OwnedPoolLabels(unittest.TestCase):
OWNED = ("glaeda-std-xcode-26.6", "glaeda-light-xcode-26.6", "glaeda-xl-xcode-26")

Expand Down
Loading