Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions scripts/glaeda-cmux-runner-hook
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,9 @@ EXCLUSIVE_UNITS = 1 << 16
EXCLUSIVE_CLASS = "exclusive"
EXCLUSIVE_CAPACITY_WAIT_S = 300.0 # NIGHTLY_CAPACITY_WAIT_S: the same owned-pool rescue window
JOB_CLAIM_ENV = "GLAEDA_JOB_CLAIM"
# The sha256 of this hook file, written beside JOB_CLAIM_ENV: a job that holds the whole mini can tell which hook
# admitted it (cmux's signing jobs accept only hooks that wait out an earlier job's leftover processes).
HOOK_SHA_ENV = "GLAEDA_HOOK_SHA256"
CLASS_COST: dict[str, tuple[int, tuple[str, ...]]] = {
"compile": (5, ("persistent-dd", "root")), "gui": (1, ("gui", "root")), "product": (1, ("gui", "root")),
"compile-gui": (4, ("root",)), "gui-step": (1, ("gui", "root")),
Expand Down Expand Up @@ -2023,9 +2026,10 @@ def export_developer_dir(xcode_app: str) -> str:


def export_job_claim(capacity_units: int, taken: dict[str, Any], note: str) -> str:
"""Tell an admitted job, through $GITHUB_ENV, that it holds the whole mini (JOB_CLAIM_ENV=exclusive): an exclusive
class job that holds every capacity unit, or any job holding the fleet host lock itself (a mini without weighted
capacity). Nothing for any other job, and nothing when there is no $GITHUB_ENV. Returns a note, or ''."""
"""Tell an admitted job, through $GITHUB_ENV, that it holds the whole mini (JOB_CLAIM_ENV=exclusive) and which
hook admitted it (HOOK_SHA_ENV, this file's sha256): an exclusive class job that holds every capacity unit, or any
job holding the fleet host lock itself (a mini without weighted capacity). Nothing for any other job, and nothing
when there is no $GITHUB_ENV. Returns a note, or ''."""
if capacity_units > 0:
whole = taken.get("class") == EXCLUSIVE_CLASS and taken.get("units") == capacity_units
else:
Expand All @@ -2034,8 +2038,11 @@ def export_job_claim(capacity_units: int, taken: dict[str, Any], note: str) -> s
if not whole or not target:
return ""
with contextlib.suppress(OSError):
lines = f"{JOB_CLAIM_ENV}={EXCLUSIVE_CLASS}\n"
with contextlib.suppress(OSError): # an unreadable hook writes no sha, and a job that needs one refuses itself
lines += f"{HOOK_SHA_ENV}={hashlib.sha256(Path(__file__).resolve().read_bytes()).hexdigest()}\n"
with open(target, "a", encoding="utf-8") as env_file:
env_file.write(f"{JOB_CLAIM_ENV}={EXCLUSIVE_CLASS}\n")
env_file.write(lines)
return f"{JOB_CLAIM_ENV}={EXCLUSIVE_CLASS} for the job (it holds the whole mini)"
return ""

Expand Down
1 change: 1 addition & 0 deletions scripts/glaeda-disk
Original file line number Diff line number Diff line change
Expand Up @@ -843,6 +843,7 @@ CMUX_NEXT_DRAIN_HOOK_SHA256 = frozenset({
"33deff2a5a3a025cc542678e7a856eae0e5ca7932e4549c0b230add25e86d482",
"fb0442d31e23f01326118b7d6be2effd0ba91bff9a0b59c62016c39bc23116da",
"2695209c13db5fe7514daf76d3e8e547b9569ad9a545225e52f7e07dd83d69e4",
"41e29cd9dca8cdee88ce069b61956674bb79c2c3da90af5520e3e397bfbedeef",
})
CMUX_NEXT_CI_PRESERVED_NAMES = frozenset({
"SourcePackages", "ModuleCache.noindex", "SDKStatCaches.noindex",
Expand Down
11 changes: 10 additions & 1 deletion scripts/test-glaeda-cmux-runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -2046,6 +2046,8 @@ def test_a_signing_job_takes_the_whole_mini(self) -> None:
admitted.stdout)
self.assertIn("GLAEDA_JOB_CLAIM=exclusive for the job", admitted.stdout)
self.assertIn("GLAEDA_JOB_CLAIM=exclusive\n", github_env.read_text())
self.assertIn(f"GLAEDA_HOOK_SHA256={hashlib.sha256(HOOK.read_bytes()).hexdigest()}\n",
github_env.read_text(), "the admitted job learns which hook admitted it")
for name, runner in (("swift-package-tests", "s1"), ("verify-signed-nightly", "s2")):
beside = self.job(name, runner, 14, None, "--gui-wait", "0")
self.assertEqual(beside.returncode, 1, beside.stdout + beside.stderr)
Expand Down Expand Up @@ -2164,9 +2166,16 @@ def test_job_claim_names_only_a_whole_mini(self) -> None:
self.assertFalse(github_env.exists())
self.assertIn("holds the whole mini", hook.export_job_claim(14, {"class": "exclusive", "units": 14}, ""))
self.assertIn("holds the whole mini", hook.export_job_claim(0, {}, f"{hook.HOST_LOCK_HELD} (held)"))
self.assertEqual(github_env.read_text(), "GLAEDA_JOB_CLAIM=exclusive\n" * 2)
claim = f"GLAEDA_JOB_CLAIM=exclusive\nGLAEDA_HOOK_SHA256={hashlib.sha256(HOOK.read_bytes()).hexdigest()}\n"
self.assertEqual(github_env.read_text(), claim * 2, "the claim names the hook that admitted the job")
with mock.patch.dict(os.environ, {}, clear=True):
self.assertEqual(hook.export_job_claim(14, {"class": "exclusive", "units": 14}, ""), "")
github_env.unlink()
with mock.patch.dict(os.environ, {"GITHUB_ENV": os.fspath(github_env)}), \
mock.patch.object(hook, "__file__", os.fspath(self.dir / "no-such-hook")):
self.assertIn("holds the whole mini", hook.export_job_claim(14, {"class": "exclusive", "units": 14}, ""))
self.assertEqual(github_env.read_text(), "GLAEDA_JOB_CLAIM=exclusive\n",
"a hook that cannot read itself names no sha, so a job that needs one refuses itself")

def test_a_nightly_sign_or_verify_job_needs_no_compile_tokens(self) -> None:
"""nightly.yml's sign and verify jobs were unknown jobs, so compiles: first in line, they still waited for the
Expand Down
Loading