diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5081f9e81..7f8da2fcb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,12 @@ jobs: Path("scripts/owned_linux_admission.py"), Path("scripts/owned-admission-control"), Path("scripts/test-owned-linux-admission.py"), + Path("scripts/cmux_fleet.py"), + Path("scripts/cmux_fleet_bootstrap.py"), + Path("scripts/cmux_execution_roles.py"), + Path("scripts/test-cmux-fleet.py"), + Path("scripts/test-cmux-execution-roles.py"), + Path("scripts/test-cmux-fleet-bootstrap.py"), Path("scripts/test-verify-focused.py"), Path("scripts/verify-changed-rustfmt"), Path("scripts/test-hot-run.py"), @@ -100,6 +106,9 @@ jobs: sh -n scripts/hot-run bash -n scripts/benchmark-developer-loop bash -n scripts/test-bootstrap.sh + bash -n scripts/cmux-fleet + bash -n scripts/cmux-fleet-bootstrap-macos + bash -n scripts/cmux-fleet-bootstrap-linux bash scripts/test-bootstrap.sh python3 scripts/test-benchmark-developer-loop.py python3 scripts/test-benchmark-hot-state-fanout.py @@ -109,6 +118,9 @@ jobs: python3 scripts/test-hot-run.py python3 scripts/test-verify-focused.py python3 scripts/test-owned-linux-admission.py + python3 scripts/test-cmux-fleet.py + python3 scripts/test-cmux-fleet-bootstrap.py + python3 scripts/test-cmux-execution-roles.py python3 scripts/test-verify.py python3 scripts/test-verify-changed-rustfmt.py python3 scripts/test-front-door.py diff --git a/docs/CMUX_EXECUTION_ROLES.md b/docs/CMUX_EXECUTION_ROLES.md new file mode 100644 index 000000000..e5412f0cd --- /dev/null +++ b/docs/CMUX_EXECUTION_ROLES.md @@ -0,0 +1,219 @@ +# CMUX execution roles and workload capabilities + +Tracking issues: #1056, #1057, #1058, #546, #743, #760, #840. + +This model sits between accepted CMUX fleet enrollment and placement. Enrollment establishes node identity and exact accepted generations. Role canaries establish what work a node may perform. Contention evidence establishes how much work a reviewed resource profile may admit. Placement preference stays advisory and comes from evidence. + +The implementation is the pure model in `scripts/cmux_execution_roles.py`. It performs no remote scheduling and grants no execution authority. + +## Closed role vocabulary + +Use the repository's existing role names and keep specialization in capabilities and operations: + +| Role | Platform | Typical capabilities | +| --- | --- | --- | +| `cmux_macos_native_build` | macOS arm64 | `native_xcode_build`, `native_release_build`, native Glaeda build | +| `cmux_macos_test` | macOS arm64 | `app_host_test` | +| `cmux_linux_ci` | Linux | `linux_ci`, `web_ci`, `background_verification` | +| `cmux_linux_agent` | Linux | `linux_agent`, `build_helper` | +| `artifact_cache` | cross-platform | `artifact_service` | +| `background_replay` | cross-platform | `background_replay` | +| `benchmark` | cross-platform | `benchmark` | +| `diagnostic` | cross-platform | `diagnostic` | + +The draft names map into this vocabulary as follows: + +- compile, release, and native developer builds use `cmux_macos_native_build` with different required capabilities; +- app-host work uses `cmux_macos_test`; +- Apple toolchain canary is acceptance/lifecycle evidence, not a routable role; +- web CI and background verification use `cmux_linux_ci`; +- build helpers use `cmux_linux_agent`. + +A new specialization becomes a capability or operation first. A new role requires a distinct acceptance boundary that cannot be represented safely by an existing role. + +#1056 currently has reviewed enrollment workloads for `cmux_macos_native_build` and `cmux_linux_ci`. The other names remain reserved until their exact acceptance workloads land. This model does not widen that gate. + +## Role eligibility + +A node is eligible for a role only when all of these agree: + +1. the node is in a routable lifecycle state; +2. the platform, reviewed OS version class, and architecture match the role; +3. Linux execution roles carry the reviewed kernel-6-or-newer capability from bootstrap/acceptance; +4. reviewed CPU, memory, and disk classes meet the role minimums; +5. required node capabilities are present; +6. a current accepted role canary exists; +7. the canary matches the current #1056 enrollment generation; +8. the canary matches the current Glaeda generation; +9. the canary matches the current #1056 per-role acceptance-workload generation; +10. the canary matches the current execution capability generation; +11. the canary accepted every capability needed by the role; +12. toolchain-bound roles bind a semantic profile to the exact accepted #1056 toolchain generation. + +Enrollment alone yields zero role eligibility. + +The node capability projection carries `enrollmentGeneration`, exact `glaedaGeneration`, and `roleWorkloadGenerations` from #1056 plus a separate `capabilityGeneration`. Any acceptance-relevant OS, hardware-class, Glaeda, toolchain, SDK, workload, or capability change makes the affected role canary stale. #1058 then drives the node through canary before routing resumes. + +`toolchainProfiles` maps an immutable semantic name such as `apple-xcode-26-sdk-26` to the exact accepted #1056 toolchain-generation digest. A toolchain update changes that digest or publishes a new profile; the previous canary cannot authorize the changed toolchain. + +## Machine capability classes + +The initial reviewed OS classes follow #1056: macOS 15/26 for macOS roles, Ubuntu 24.04 or Debian 12 plus kernel-major-6-or-newer capability for Linux execution roles. + +Remote placement consumes reviewed classes instead of arbitrary host controls: + +- CPU: `small`, `medium`, `large`; +- memory: `small`, `medium`, `large`, `xlarge`; +- disk: `small`, `medium`, `large`; +- pressure: bounded CPU, memory, swap, and thermal classes. + +The model exposes no caller-selected core counts, byte counts, cgroup settings, hostnames, or machine purchase metadata. + +Platform cost, machine age, and "newest host" are absent from eligibility and preference policy. + +## Workload requirement object + +A workload names useful work and semantic requirements. It never names a machine. + +Example: + +```json +{ + "schema": "glaeda-cmux-workload-requirement/v1", + "operation": "cmux_macos_compile_admission", + "role": "cmux_macos_native_build", + "platform": "macos", + "architecture": "arm64", + "toolchainProfile": "apple-xcode-26-sdk-26", + "minimumCpuClass": "medium", + "minimumMemoryClass": "medium", + "requiredCapabilities": ["native_xcode_build"], + "resourceProfile": "medium" +} +``` + +Operations currently model: + +- `cmux_macos_compile_admission`; +- `cmux_macos_release_admission`; +- `cmux_macos_app_host_test`; +- `cmux_linux_ci_admission`; +- `cmux_web_ci_admission`; +- `cmux_linux_agent_admission`; +- `cmux_build_helper_admission`; +- `background_verification`; +- `artifact_cache_service`; +- `background_replay`; +- `benchmark`; +- `diagnostic`. + +A scheduler may select only nodes whose current eligibility and measured resource profile satisfy the object. Local admission repeats the checks against fresh node state. + +## Resource profiles and measured concurrency + +The caller chooses only one reviewed profile: + +`small`, `medium`, `large`, or `exclusive`. + +Every accepted role/profile pair requires current capacity evidence. Profiles with no heavy lease claim still require measurement. + +A `glaeda-cmux-role-capacity/v1` receipt binds: + +- node, enrollment generation, Glaeda generation, capability generation, role, and resource profile; +- exact role-workload generation plus toolchain profile/generation context; +- one local slot class and measured maximum concurrency; +- `contentionEvidenceGeneration`, an immutable digest of the reviewed #760-style evidence; +- validated completions; +- p50 and p90 final-result latency; +- CPU pressure; +- memory pressure; +- swap class; +- thermal behavior where available; +- unfinished work. + +#760 contention windows remain the evidence producer. This model consumes reduced evidence and does not invent concurrency from core count or RAM size. #840 native Linux measurements can therefore establish values such as four medium jobs or one heavy job only after the semantic and pressure evidence supports them. + +Mixed workloads require evidence appropriate to their claims. A compile lane measured at one stays one even when a different role/profile has demonstrated higher concurrency on another lane. + +## Physical lease boundary + +Workload operations compile to local scarce-resource claims. Remote callers never name the claim directly. + +Current claims include: + +- `mac_native_build_lane`; +- `mac_app_host_test_slot`; +- `artifact_publisher_slot`; +- `linux_medium_slot`; +- `linux_heavy_slot`; +- `browser_ui_test_slot`. + +The `mac_native_build_lane`, `linux_heavy_slot`, and `artifact_publisher_slot` names reuse #1057's vocabulary. + +`local_admission` is `admission_only`. It reports `physical_execution_lease` as the lease boundary for work with scarce claims. The local #1057 lease owner supplies current held-slot observations and must acquire the lease before launch. This prevents two orchestrators from converting the same advisory placement into overlapping execution. + +A node that becomes critically pressured after remote selection refuses at local admission before a lease is granted. + +## Eligibility and preference stay separate + +Eligibility comes only from exact acceptance. + +Preference is observation-only evidence for #546. A node may be: + +- preferred for latency-sensitive native compile; +- eligible for app-host tests; +- measured for background replay. + +Changing preference never adds a role or repairs a failed canary. A preference record contains a role and evidence generation with `authority: observation_only`. + +#546 may rank only among already eligible candidates and reviewed resource profiles. Cost assumptions, platform stereotypes, and machine newness are outside this model. + +## Founder/operator status + +The operator projection contains the decision-ready subset: + +```text +node cmux-mac-001 +eligible: + cmux_macos_native_build + cmux_macos_test +temporarily unavailable: + artifact_cache: role_canary_pending +capacity: + mac_app_host_test_slot: 2 + mac_native_build_lane: 1 +preferred: + cmux_macos_native_build +state: active +``` + +Internal enrollment/capability/workload generation objects stay out of the human view. The machine-readable form retains bounded reason codes. + +## Acceptance coverage + +`scripts/test-cmux-execution-roles.py` covers: + +- exact role requirements; +- wrong Xcode/toolchain profile; +- unsupported OS version class and missing Linux kernel class; +- insufficient CPU and memory classes; +- failed role canary; +- stale role canary after re-enrollment; +- stale role canary after Glaeda or reviewed workload generation changes; +- exact toolchain-generation change behind the same semantic profile; +- draining node; +- role invalidation after capability/toolchain upgrade; +- multiple roles sharing a scarce physical lane; +- profile-specific concurrency so another role cannot inflate a lane; +- external scheduler request for an ineligible role; +- pressure changing between selection and local admission; +- unmeasured resource profile refusal, including background work; +- stale #760 capacity evidence after an acceptance-workload generation change; +- #760-style capacity evidence fields; +- one synthetic Mac with two roles; +- one synthetic Linux node with two roles; +- preference separated from eligibility; +- compact founder/operator output; +- workload objects with no machine selector or raw CPU/RAM/cgroup controls. + +Physical acceptance remains a later step. It should attach exact receipts proving at least one CMUX-owned Mac with two accepted roles and one CMUX-owned Linux node with two accepted roles, then demonstrate an incompatible workload refusal on each relevant path. diff --git a/docs/CMUX_FLEET_ENROLLMENT.md b/docs/CMUX_FLEET_ENROLLMENT.md new file mode 100644 index 000000000..1f2d16e89 --- /dev/null +++ b/docs/CMUX_FLEET_ENROLLMENT.md @@ -0,0 +1,308 @@ +# CMUX-owned fleet enrollment + +Tracking issue: #1056. + +This path turns a CMUX-controlled Mac or Linux host into a reviewed fleet node while the existing owners keep their responsibilities. GitHub Actions, CMUX's controller, direct SSH, MDM/configuration management, hosted runners, and future schedulers continue to manage the parts they already own. Glaeda owns the bounded enrollment record, role acceptance binding, lifecycle state, and a machine-status projection that reports role/candidate eligibility. + +## Supported v1 bootstrap classes + +The repository-owned bootstrap commands are read-only. They inspect an exact CMUX checkout and exact Glaeda executable, produce a bounded receipt, and leave package installation, MDM, SSH, networking, and machine account policy with their existing owners. + +- macOS: Apple Silicon or x86_64; macOS major 15 or 26; CMUX `.xcode-version` must be `26.0`; the selected Xcode and macOS SDK must be generation 26; Git present; at least 120 GiB free by default; AC system sleep disabled for unattended work. `cmux-mac-build-large` and `cmux-mac-test-large` require at least 8 logical CPUs and 16 GiB total memory. +- Linux: x86_64 or arm64; Ubuntu 24.04 or Debian 12; kernel major 6 or newer; Git, Python, systemd, bubblewrap, cgroup v2, pressure signals; at least 40 GiB free and 8 GiB available memory by default. `cmux-linux-ci-medium` and `cmux-linux-agent-medium` require at least 4 logical CPUs and 8 GiB total memory. `cmux_linux_ci` also checks `curl`, `tar`, `gzip`, and `ldd`. + +A different machine class starts as a bootstrap refusal until the reviewed policy changes. + +## Roles + +The role vocabulary is stable enough to name future work: + +| Role | v1 enrollment | Exact acceptance owner | +| --- | --- | --- | +| `cmux_macos_native_build` | supported | `manaflow-ai/cmux/scripts/fleet-accept-macos-native-build` | +| `cmux_linux_ci` | supported | `manaflow-ai/cmux/scripts/fleet-accept-linux-ci` | +| `cmux_macos_test` | reserved | acceptance workload must land first | +| `cmux_linux_agent` | reserved | acceptance workload must land first | +| `artifact_cache` | reserved | acceptance workload must land first | +| `background_replay` | reserved | acceptance workload must land first | +| `benchmark` | reserved | acceptance workload must land first | +| `diagnostic` | reserved | acceptance workload must land first | + +Execution capabilities, reviewed resource profiles, physical lease claims, and placement preference are defined in `docs/CMUX_EXECUTION_ROLES.md`. + +The v1 enrollment validator and bootstrap refuse reserved roles. A role becomes enrolable only when an exact reviewed workload exists; this keeps "advertised" equivalent to "can be proven." For supported roles, eligibility then requires node state `eligible` plus a current accepted receipt matching node ID, enrollment generation, Glaeda generation, enrolled toolchain generation, and the role's current reviewed workload generation. + +The versioned schema carries the full role vocabulary in `$defs.knownRole` while `allowedExecutionRoles` accepts only the v1 supported subset. + +## Enrollment identity and privacy + +`examples/cmux-fleet/enrollment-v1.schema.json` is the public v1 schema. The record contains only: + +- opaque `cmux-*` node ID; +- architecture; +- OS family and version class; +- hardware capability class; +- accepted toolchain generations; +- current reviewed workload generation for every allowed role; +- allowed roles; +- operator/fleet ownership scope; +- enrollment generation; +- installed Glaeda generation; +- lifecycle state and reviewed quarantine reason. + +The schema has no hostname, serial number, username, private address, SSH identity, MDM identifier, or raw hardware inventory fields. Unknown top-level fields are rejected. + +## Lifecycle + +States are: + +`discovered -> enrolling -> eligible -> draining/quarantined -> retired` + +Supported recovery paths also include `draining -> eligible` and `quarantined -> enrolling`. Every transition into `eligible` requires at least one current accepted role receipt. Leaving quarantine advances the enrollment generation, so every pre-quarantine acceptance receipt becomes stale immediately. + +`scripts/cmux-fleet status` reports role/candidate eligibility only. Its `routingCandidateEligible` field means the enrollment and current acceptance receipt satisfy this contract; `automaticDispatchAuthorized` stays `false`. #546 or another separately approved routing-promotion gate owns queue selection and dispatch, including queue/start prediction, hot locality, pressure, allowance scarcity, workload-class evidence, and operator policy. The status projection includes only bounded architecture, OS class, hardware class, accepted toolchain generations, role workload generations, Glaeda generation, fleet scope, lifecycle state, and per-role eligibility. A node in `draining`, `quarantined`, or `retired` produces zero eligible roles even when an older acceptance receipt exists. An `eligible` node still produces zero eligible roles when its acceptance evidence is absent, rejected, or stale. Live pressure/heat remains a fresh local admission veto: #970 may publish advisory bounded snapshots, while #546/local execution admission re-observes the machine before dispatch. + +Reviewed quarantine reasons are: + +- `toolchain_mismatch` +- `disk_pressure` +- `failed_acceptance` +- `dirty_canonical_checkout` +- `service_mismatch` +- `hardware_failure` +- `stale_glaeda_generation` +- `unexplained_process_settlement` + +## Onboard a Mac + +Use an exact reviewed Glaeda checkout and the CMUX checkout that will run acceptance. + +```bash +./scripts/bootstrap +cargo build --locked --release --bin glaeda + +GLAEDA_INSTALL_ROOT="${XDG_DATA_HOME:-$HOME/.local/share}/glaeda/cmux-fleet" +GLAEDA_BIN="$GLAEDA_INSTALL_ROOT/glaeda" +install -d -m 700 "$GLAEDA_INSTALL_ROOT" +if test -f "$GLAEDA_BIN" && ! test -e "$GLAEDA_INSTALL_ROOT/glaeda.rollback"; then + cp -p "$GLAEDA_BIN" "$GLAEDA_INSTALL_ROOT/glaeda.rollback" +fi +install -m 755 target/release/glaeda "$GLAEDA_INSTALL_ROOT/.glaeda.next" +mv "$GLAEDA_INSTALL_ROOT/.glaeda.next" "$GLAEDA_BIN" + +CMUX_ROOT=/absolute/path/to/cmux +CMUX_CACHE_ROOT=/absolute/path/to/cmux-native-cache +( + cd "$CMUX_ROOT" + ./scripts/setup.sh +) +export PATH="${CARGO_HOME:-$HOME/.cargo}/bin:/opt/homebrew/bin:/usr/local/bin:$PATH" + +FLEET_ROOT="${XDG_CONFIG_HOME:-$HOME/.config}/glaeda/cmux-fleet" +umask 077 +install -d -m 700 "$FLEET_ROOT" "$FLEET_ROOT/acceptance" +BOOTSTRAP="$(mktemp)" +ACCEPTANCE_EVIDENCE="$(mktemp)" +ENROLLMENT="$FLEET_ROOT/enrollment.json" +ACCEPTANCE="$FLEET_ROOT/acceptance/cmux_macos_native_build.json" +NODE_ID=cmux-mac-001 + +bash scripts/cmux-fleet-bootstrap-macos \ + --cmux-root "$CMUX_ROOT" \ + --glaeda "$GLAEDA_BIN" \ + --cache-root "$CMUX_CACHE_ROOT" \ + --hardware-class cmux-mac-build-large \ + --role cmux_macos_native_build \ + > "$BOOTSTRAP" + +ENROLLMENT_NEXT="$(mktemp "$FLEET_ROOT/.enrollment.XXXXXX")" +python3 scripts/cmux_fleet.py enroll "$BOOTSTRAP" \ + --node-id "$NODE_ID" \ + --scope cmux-founders \ + --generation 1 \ + > "$ENROLLMENT_NEXT" +chmod 600 "$ENROLLMENT_NEXT" +mv "$ENROLLMENT_NEXT" "$ENROLLMENT" + +CMUX_COMMIT="$(git -C "$CMUX_ROOT" rev-parse HEAD)" +bash "$CMUX_ROOT/scripts/fleet-accept-macos-native-build" \ + --commit "$CMUX_COMMIT" \ + --node-id "$NODE_ID" \ + --enrollment-generation 1 \ + --glaeda-generation "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["glaedaGeneration"])' "$ENROLLMENT")" \ + --toolchain-generation "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["supportedToolchainGenerations"][0])' "$ENROLLMENT")" \ + --output "$ACCEPTANCE_EVIDENCE" + +ACCEPTANCE_NEXT="$(mktemp "$FLEET_ROOT/acceptance/.cmux_macos_native_build.XXXXXX")" +python3 scripts/cmux_fleet.py finalize-acceptance \ + "$ENROLLMENT" "$ACCEPTANCE_EVIDENCE" > "$ACCEPTANCE_NEXT" +chmod 600 "$ACCEPTANCE_NEXT" +mv "$ACCEPTANCE_NEXT" "$ACCEPTANCE" + +ENROLLMENT_NEXT="$(mktemp "$FLEET_ROOT/.enrollment.XXXXXX")" +python3 scripts/cmux_fleet.py transition "$ENROLLMENT" --to eligible \ + --acceptance "$ACCEPTANCE" \ + > "$ENROLLMENT_NEXT" +chmod 600 "$ENROLLMENT_NEXT" +mv "$ENROLLMENT_NEXT" "$ENROLLMENT" + +bash scripts/cmux-fleet status "$ENROLLMENT" \ + --acceptance "$ACCEPTANCE" +``` + +The macOS preparation reuses CMUX's reviewed `scripts/setup.sh` for submodules, the pinned Rust toolchain, Zig compatibility, GhosttyKit, and the Xcode Metal component check. The bootstrap re-observes those prerequisites read-only, requires a clean canonical checkout, and verifies the operator-owned native cache root is writable with the same disk headroom threshold without publishing its path. The acceptance creates a fresh private DerivedData root, runs an exact clean Debug build with code signing disabled, verifies the produced `cmux DEV` executable is Mach-O, rechecks the canonical checkout, and refuses a surviving acceptance process group. + +## Onboard Linux + +```bash +./scripts/bootstrap +cargo build --locked --release --bin glaeda + +GLAEDA_INSTALL_ROOT="${XDG_DATA_HOME:-$HOME/.local/share}/glaeda/cmux-fleet" +GLAEDA_BIN="$GLAEDA_INSTALL_ROOT/glaeda" +install -d -m 700 "$GLAEDA_INSTALL_ROOT" +if test -f "$GLAEDA_BIN" && ! test -e "$GLAEDA_INSTALL_ROOT/glaeda.rollback"; then + cp -p "$GLAEDA_BIN" "$GLAEDA_INSTALL_ROOT/glaeda.rollback" +fi +install -m 755 target/release/glaeda "$GLAEDA_INSTALL_ROOT/.glaeda.next" +mv "$GLAEDA_INSTALL_ROOT/.glaeda.next" "$GLAEDA_BIN" + +CMUX_ROOT=/absolute/path/to/cmux +FLEET_ROOT="${XDG_CONFIG_HOME:-$HOME/.config}/glaeda/cmux-fleet" +umask 077 +install -d -m 700 "$FLEET_ROOT" "$FLEET_ROOT/acceptance" +BOOTSTRAP="$(mktemp)" +ACCEPTANCE_EVIDENCE="$(mktemp)" +ENROLLMENT="$FLEET_ROOT/enrollment.json" +ACCEPTANCE="$FLEET_ROOT/acceptance/cmux_linux_ci.json" +NODE_ID=cmux-linux-001 + +bash scripts/cmux-fleet-bootstrap-linux \ + --cmux-root "$CMUX_ROOT" \ + --glaeda "$GLAEDA_BIN" \ + --hardware-class cmux-linux-ci-medium \ + --role cmux_linux_ci \ + > "$BOOTSTRAP" + +ENROLLMENT_NEXT="$(mktemp "$FLEET_ROOT/.enrollment.XXXXXX")" +python3 scripts/cmux_fleet.py enroll "$BOOTSTRAP" \ + --node-id "$NODE_ID" \ + --scope cmux-founders \ + --generation 1 \ + > "$ENROLLMENT_NEXT" +chmod 600 "$ENROLLMENT_NEXT" +mv "$ENROLLMENT_NEXT" "$ENROLLMENT" + +CMUX_COMMIT="$(git -C "$CMUX_ROOT" rev-parse HEAD)" +bash "$CMUX_ROOT/scripts/fleet-accept-linux-ci" \ + --commit "$CMUX_COMMIT" \ + --node-id "$NODE_ID" \ + --enrollment-generation 1 \ + --glaeda-generation "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["glaedaGeneration"])' "$ENROLLMENT")" \ + --toolchain-generation "$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["supportedToolchainGenerations"][0])' "$ENROLLMENT")" \ + --output "$ACCEPTANCE_EVIDENCE" + +ACCEPTANCE_NEXT="$(mktemp "$FLEET_ROOT/acceptance/.cmux_linux_ci.XXXXXX")" +python3 scripts/cmux_fleet.py finalize-acceptance \ + "$ENROLLMENT" "$ACCEPTANCE_EVIDENCE" > "$ACCEPTANCE_NEXT" +chmod 600 "$ACCEPTANCE_NEXT" +mv "$ACCEPTANCE_NEXT" "$ACCEPTANCE" + +ENROLLMENT_NEXT="$(mktemp "$FLEET_ROOT/.enrollment.XXXXXX")" +python3 scripts/cmux_fleet.py transition "$ENROLLMENT" --to eligible \ + --acceptance "$ACCEPTANCE" \ + > "$ENROLLMENT_NEXT" +chmod 600 "$ENROLLMENT_NEXT" +mv "$ENROLLMENT_NEXT" "$ENROLLMENT" + +bash scripts/cmux-fleet status "$ENROLLMENT" \ + --acceptance "$ACCEPTANCE" +``` + +The Linux acceptance archives the exact commit into a private temporary tree, runs the CMUX self-hosted-runner guard and Linux routing unittest there, verifies the archive artifact, rechecks the canonical checkout, and refuses a surviving acceptance process group. + +After either onboarding path, remove only the transient evidence files: + +```bash +rm -f "$BOOTSTRAP" "$ACCEPTANCE_EVIDENCE" +``` + +The canonical enrollment and finalized role receipts stay under `$FLEET_ROOT` across reboot. They contain no credentials or project secrets. + +## Drain, quarantine, recover, and retire + +Drain before planned operator work: + +```bash +ENROLLMENT_NEXT="$(mktemp "$FLEET_ROOT/.enrollment.XXXXXX")" +python3 scripts/cmux_fleet.py transition "$ENROLLMENT" --to draining \ + > "$ENROLLMENT_NEXT" +chmod 600 "$ENROLLMENT_NEXT" +mv "$ENROLLMENT_NEXT" "$ENROLLMENT" +bash scripts/cmux-fleet status "$ENROLLMENT" --acceptance "$ACCEPTANCE" +``` + +Quarantine on a concrete reviewed reason: + +```bash +ENROLLMENT_NEXT="$(mktemp "$FLEET_ROOT/.enrollment.XXXXXX")" +python3 scripts/cmux_fleet.py transition "$ENROLLMENT" \ + --to quarantined --reason toolchain_mismatch \ + > "$ENROLLMENT_NEXT" +chmod 600 "$ENROLLMENT_NEXT" +mv "$ENROLLMENT_NEXT" "$ENROLLMENT" +``` + +After a toolchain, OS, hardware class, or Glaeda update, rerun bootstrap and create a fresh enrollment with generation N+1. Old acceptance receipts then become stale by construction. + +Rollback an onboarding before candidate promotion, or retire an active node, by preserving the record in the terminal `retired` state: + +```bash +ENROLLMENT_NEXT="$(mktemp "$FLEET_ROOT/.enrollment.XXXXXX")" +python3 scripts/cmux_fleet.py transition "$ENROLLMENT" --to retired \ + > "$ENROLLMENT_NEXT" +chmod 600 "$ENROLLMENT_NEXT" +mv "$ENROLLMENT_NEXT" "$ENROLLMENT" +bash scripts/cmux-fleet status "$ENROLLMENT" +``` + +This leaves an auditable local tombstone and zero candidate-eligible roles. + +## Operator-owned prerequisite changes + +The bootstrap commands deliberately avoid installing packages, changing SSH/Tailscale, enrolling MDM, editing GitHub runner registration, or changing service ownership. Use the machine-management system that already owns those changes, then rerun bootstrap. + +For a Mac used unattended without an existing MDM power profile, this reference change is reversible: + +```bash +OLD_AC_SLEEP="$(pmset -g custom | awk '/AC Power:/{on=1;next}/Power:/{on=0} on && $1=="sleep"{print $2; exit}')" +test -n "$OLD_AC_SLEEP" +sudo pmset -c sleep 0 +# rollback +sudo pmset -c sleep "$OLD_AC_SLEEP" +``` + +For Glaeda distribution, the bootstrap consumes an exact executable path and records its SHA-256 generation. The examples above install the reviewed repository build into a stable per-user CMUX-fleet location with an atomic rename. The first update attempt preserves the previously installed binary as `glaeda.rollback`; later retries leave that copy untouched until the candidate enrollment generation is accepted. A release package, MDM payload, or configuration manager may supply the same exact file instead. + +Rollback a just-installed repository build before re-enrollment with: + +```bash +test -f "$GLAEDA_INSTALL_ROOT/glaeda.rollback" +mv "$GLAEDA_INSTALL_ROOT/glaeda.rollback" "$GLAEDA_BIN" +``` + +After any install, update, or rollback, rerun bootstrap and advance the enrollment generation before candidate eligibility resumes. Once the new generation is accepted, remove the one-step rollback copy: + +```bash +rm -f "$GLAEDA_INSTALL_ROOT/glaeda.rollback" +``` + +## CI and synthetic evidence + +Hosted CI needs no physical CMUX machine. Glaeda CI runs the enrollment and bootstrap fixture suites. CMUX CI runs the acceptance-harness fixture suite. Physical Mac/Linux acceptance receipts are attached only after the corresponding CMUX-controlled hosts exist and have been explicitly approved for the workload. + +Related Glaeda work: #743, #546, #365, #492, #970, #1048, #1008, #1010. +Related CMUX work: manaflow-ai/cmux#13091, #13095, #13198, #13325. + + +Role eligibility also binds the current CMUX acceptance-workload generation. Bootstrap hashes the reviewed `scripts/fleet_acceptance.py` for every enrolled role, and status rejects an acceptance receipt when that generation no longer matches. diff --git a/examples/cmux-fleet/enrollment-v1.schema.json b/examples/cmux-fleet/enrollment-v1.schema.json new file mode 100644 index 000000000..74079b4fb --- /dev/null +++ b/examples/cmux-fleet/enrollment-v1.schema.json @@ -0,0 +1,289 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/teamleaderleo/glaeda/examples/cmux-fleet/enrollment-v1.schema.json", + "title": "Glaeda CMUX fleet enrollment v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "nodeId", + "architecture", + "os", + "hardwareCapabilityClass", + "supportedToolchainGenerations", + "roleWorkloadGenerations", + "allowedExecutionRoles", + "operatorFleetScope", + "enrollmentGeneration", + "glaedaGeneration", + "state", + "quarantineReason" + ], + "properties": { + "schema": { + "const": "glaeda-cmux-fleet-enrollment/v1" + }, + "nodeId": { + "type": "string", + "pattern": "^cmux-[a-z0-9][a-z0-9-]{2,59}$" + }, + "architecture": { + "enum": [ + "arm64", + "x86_64" + ] + }, + "os": { + "type": "object", + "additionalProperties": false, + "required": [ + "family", + "versionClass" + ], + "properties": { + "family": { + "enum": [ + "macos", + "linux" + ] + }, + "versionClass": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,79}$" + } + } + }, + "hardwareCapabilityClass": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,79}$" + }, + "supportedToolchainGenerations": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + }, + "allowedExecutionRoles": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "uniqueItems": true, + "items": { + "enum": [ + "cmux_linux_ci", + "cmux_macos_native_build" + ] + } + }, + "operatorFleetScope": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9._-]{0,79}$" + }, + "enrollmentGeneration": { + "type": "integer", + "minimum": 1, + "maximum": 2147483647 + }, + "glaedaGeneration": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "state": { + "enum": [ + "discovered", + "enrolling", + "eligible", + "draining", + "quarantined", + "retired" + ] + }, + "quarantineReason": { + "oneOf": [ + { + "type": "null" + }, + { + "enum": [ + "dirty_canonical_checkout", + "disk_pressure", + "failed_acceptance", + "hardware_failure", + "service_mismatch", + "stale_glaeda_generation", + "toolchain_mismatch", + "unexplained_process_settlement" + ] + } + ] + }, + "roleWorkloadGenerations": { + "type": "object", + "additionalProperties": false, + "properties": { + "cmux_macos_native_build": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + }, + "cmux_linux_ci": { + "type": "string", + "pattern": "^sha256:[0-9a-f]{64}$" + } + }, + "minProperties": 1, + "maxProperties": 2 + } + }, + "allOf": [ + { + "if": { + "properties": { + "state": { + "const": "quarantined" + } + } + }, + "then": { + "properties": { + "quarantineReason": { + "type": "string" + } + } + }, + "else": { + "properties": { + "quarantineReason": { + "type": "null" + } + } + } + }, + { + "if": { + "properties": { + "os": { + "properties": { + "family": { + "const": "macos" + } + } + } + } + }, + "then": { + "properties": { + "allowedExecutionRoles": { + "items": { + "not": { + "enum": [ + "cmux_linux_agent", + "cmux_linux_ci" + ] + } + } + } + } + } + }, + { + "if": { + "properties": { + "os": { + "properties": { + "family": { + "const": "linux" + } + } + } + } + }, + "then": { + "properties": { + "allowedExecutionRoles": { + "items": { + "not": { + "enum": [ + "cmux_macos_native_build", + "cmux_macos_test" + ] + } + } + } + } + } + }, + { + "if": { + "properties": { + "os": { + "properties": { + "family": { + "const": "macos" + } + } + } + } + }, + "then": { + "properties": { + "roleWorkloadGenerations": { + "required": [ + "cmux_macos_native_build" + ], + "not": { + "required": [ + "cmux_linux_ci" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "os": { + "properties": { + "family": { + "const": "linux" + } + } + } + } + }, + "then": { + "properties": { + "roleWorkloadGenerations": { + "required": [ + "cmux_linux_ci" + ], + "not": { + "required": [ + "cmux_macos_native_build" + ] + } + } + } + } + } + ], + "$defs": { + "knownRole": { + "description": "CMUX fleet role vocabulary. Roles become enrolable only after an exact reviewed acceptance workload lands.", + "enum": [ + "artifact_cache", + "background_replay", + "benchmark", + "cmux_linux_agent", + "cmux_linux_ci", + "cmux_macos_native_build", + "cmux_macos_test", + "diagnostic" + ] + } + } +} diff --git a/examples/cmux-fleet/linux-enrollment.example.json b/examples/cmux-fleet/linux-enrollment.example.json new file mode 100644 index 000000000..1b72e0ed2 --- /dev/null +++ b/examples/cmux-fleet/linux-enrollment.example.json @@ -0,0 +1,24 @@ +{ + "schema": "glaeda-cmux-fleet-enrollment/v1", + "nodeId": "cmux-example-linux-001", + "architecture": "x86_64", + "os": { + "family": "linux", + "versionClass": "ubuntu-24.04" + }, + "hardwareCapabilityClass": "cmux-linux-ci-medium", + "supportedToolchainGenerations": [ + "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + ], + "roleWorkloadGenerations": { + "cmux_linux_ci": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + "allowedExecutionRoles": [ + "cmux_linux_ci" + ], + "operatorFleetScope": "cmux-founders", + "enrollmentGeneration": 1, + "glaedaGeneration": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "state": "enrolling", + "quarantineReason": null +} diff --git a/examples/cmux-fleet/macos-enrollment.example.json b/examples/cmux-fleet/macos-enrollment.example.json new file mode 100644 index 000000000..c2dec91af --- /dev/null +++ b/examples/cmux-fleet/macos-enrollment.example.json @@ -0,0 +1,24 @@ +{ + "schema": "glaeda-cmux-fleet-enrollment/v1", + "nodeId": "cmux-example-mac-001", + "architecture": "arm64", + "os": { + "family": "macos", + "versionClass": "macos-26" + }, + "hardwareCapabilityClass": "cmux-mac-build-large", + "supportedToolchainGenerations": [ + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ], + "roleWorkloadGenerations": { + "cmux_macos_native_build": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "allowedExecutionRoles": [ + "cmux_macos_native_build" + ], + "operatorFleetScope": "cmux-founders", + "enrollmentGeneration": 1, + "glaedaGeneration": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "state": "enrolling", + "quarantineReason": null +} diff --git a/scripts/cmux-fleet b/scripts/cmux-fleet new file mode 100644 index 000000000..649b536b2 --- /dev/null +++ b/scripts/cmux-fleet @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +script_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +exec python3 "$script_dir/cmux_fleet.py" "$@" diff --git a/scripts/cmux-fleet-bootstrap-linux b/scripts/cmux-fleet-bootstrap-linux new file mode 100644 index 000000000..7c713f685 --- /dev/null +++ b/scripts/cmux-fleet-bootstrap-linux @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +script_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +exec python3 "$script_dir/cmux_fleet_bootstrap.py" --platform linux "$@" diff --git a/scripts/cmux-fleet-bootstrap-macos b/scripts/cmux-fleet-bootstrap-macos new file mode 100644 index 000000000..2ed9ac080 --- /dev/null +++ b/scripts/cmux-fleet-bootstrap-macos @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +script_dir="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +exec python3 "$script_dir/cmux_fleet_bootstrap.py" --platform macos "$@" diff --git a/scripts/cmux_execution_roles.py b/scripts/cmux_execution_roles.py new file mode 100644 index 000000000..74258af1f --- /dev/null +++ b/scripts/cmux_execution_roles.py @@ -0,0 +1,1072 @@ +#!/usr/bin/env python3 +"""Pure CMUX fleet execution-role, capability, and local-admission model.""" +from __future__ import annotations + +import json +import re +from typing import Any, Iterable + +NODE_SCHEMA = "glaeda-cmux-node-capabilities/v1" +ROLE_CANARY_SCHEMA = "glaeda-cmux-role-canary/v1" +CAPACITY_SCHEMA = "glaeda-cmux-role-capacity/v1" +WORKLOAD_SCHEMA = "glaeda-cmux-workload-requirement/v1" +PREFERENCE_SCHEMA = "glaeda-cmux-placement-preference/v1" +STATUS_SCHEMA = "glaeda-cmux-execution-role-status/v1" +DECISION_SCHEMA = "glaeda-cmux-local-admission/v1" + +TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9._:-]{0,95}\Z") +SHA256_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +NODE_RE = re.compile(r"cmux-[a-z0-9][a-z0-9-]{2,59}\Z") + +ROLES = ( + "artifact_cache", + "background_replay", + "benchmark", + "cmux_linux_agent", + "cmux_linux_ci", + "cmux_macos_native_build", + "cmux_macos_test", + "diagnostic", +) + +RESOURCE_PROFILES = ("small", "medium", "large", "exclusive") +CPU_CLASSES = ("small", "medium", "large") +MEMORY_CLASSES = ("small", "medium", "large", "xlarge") +DISK_CLASSES = ("small", "medium", "large") +PRESSURE_CLASSES = ("normal", "elevated", "critical", "unknown") +SWAP_CLASSES = ("none", "low", "elevated", "critical", "unknown") +THERMAL_CLASSES = ("stable", "elevated", "critical", "unknown") +ROUTABLE_STATES = {"eligible", "active"} +KNOWN_STATES = { + "new", + "discovered", + "bootstrapping", + "enrolling", + "canary", + "eligible", + "active", + "draining", + "maintenance", + "quarantined", + "retired", +} + +# Role identity stays small. More precise work belongs in capability and operation vocabulary. +ROLE_REQUIREMENTS = { + "cmux_macos_native_build": { + "platform": "macos", + "allowedOsVersionClasses": {"macos-15", "macos-26"}, + "architectures": {"arm64"}, + "minimumCpuClass": "medium", + "minimumMemoryClass": "medium", + "minimumDiskClass": "medium", + "requiredCapabilities": { + "apple_silicon", + "native_glaeda_build", + "native_xcode_build", + }, + "requiresToolchainProfile": True, + }, + "cmux_macos_test": { + "platform": "macos", + "allowedOsVersionClasses": {"macos-15", "macos-26"}, + "architectures": {"arm64"}, + "minimumCpuClass": "medium", + "minimumMemoryClass": "medium", + "minimumDiskClass": "medium", + "requiredCapabilities": {"apple_silicon", "app_host_test"}, + "requiresToolchainProfile": True, + }, + "cmux_linux_ci": { + "platform": "linux", + "allowedOsVersionClasses": {"ubuntu-24.04", "debian-12"}, + "architectures": {"x86_64", "arm64"}, + "minimumCpuClass": "medium", + "minimumMemoryClass": "medium", + "minimumDiskClass": "medium", + "requiredCapabilities": { + "cgroup_v2", + "linux_ci_toolchain", + "linux_kernel_6_plus", + "systemd_execution", + "task_isolation", + }, + "requiresToolchainProfile": True, + }, + "cmux_linux_agent": { + "platform": "linux", + "allowedOsVersionClasses": {"ubuntu-24.04", "debian-12"}, + "architectures": {"x86_64", "arm64"}, + "minimumCpuClass": "medium", + "minimumMemoryClass": "medium", + "minimumDiskClass": "medium", + "requiredCapabilities": { + "cgroup_v2", + "linux_kernel_6_plus", + "systemd_execution", + "task_isolation", + }, + "requiresToolchainProfile": True, + }, + "artifact_cache": { + "platform": None, + "architectures": {"x86_64", "arm64"}, + "minimumCpuClass": "small", + "minimumMemoryClass": "small", + "minimumDiskClass": "large", + "requiredCapabilities": {"artifact_service"}, + "requiresToolchainProfile": False, + }, + "background_replay": { + "platform": None, + "architectures": {"x86_64", "arm64"}, + "minimumCpuClass": "small", + "minimumMemoryClass": "small", + "minimumDiskClass": "small", + "requiredCapabilities": {"background_replay"}, + "requiresToolchainProfile": False, + }, + "benchmark": { + "platform": None, + "architectures": {"x86_64", "arm64"}, + "minimumCpuClass": "small", + "minimumMemoryClass": "small", + "minimumDiskClass": "small", + "requiredCapabilities": {"benchmark"}, + "requiresToolchainProfile": False, + }, + "diagnostic": { + "platform": None, + "architectures": {"x86_64", "arm64"}, + "minimumCpuClass": "small", + "minimumMemoryClass": "small", + "minimumDiskClass": "small", + "requiredCapabilities": {"diagnostic"}, + "requiresToolchainProfile": False, + }, +} + +# Useful work names semantic requirements. Scarce slot claims stay local. +OPERATION_CONTRACTS = { + "cmux_macos_compile_admission": { + "role": "cmux_macos_native_build", + "requiredCapabilities": {"native_xcode_build"}, + "slotClaims": ("mac_native_build_lane",), + }, + "cmux_macos_release_admission": { + "role": "cmux_macos_native_build", + "requiredCapabilities": {"native_release_build"}, + "slotClaims": ("artifact_publisher_slot", "mac_native_build_lane"), + }, + "cmux_macos_app_host_test": { + "role": "cmux_macos_test", + "requiredCapabilities": {"app_host_test"}, + "slotClaims": ("mac_app_host_test_slot",), + "largeSlotClaims": ("mac_app_host_test_slot", "mac_native_build_lane"), + }, + "cmux_linux_ci_admission": { + "role": "cmux_linux_ci", + "requiredCapabilities": {"linux_ci"}, + "slotClaims": ("linux_medium_slot",), + "largeSlotClaims": ("linux_heavy_slot",), + }, + "cmux_web_ci_admission": { + "role": "cmux_linux_ci", + "requiredCapabilities": {"web_ci"}, + "slotClaims": ("browser_ui_test_slot",), + }, + "cmux_linux_agent_admission": { + "role": "cmux_linux_agent", + "requiredCapabilities": {"linux_agent"}, + "slotClaims": ("linux_medium_slot",), + "largeSlotClaims": ("linux_heavy_slot",), + }, + "cmux_build_helper_admission": { + "role": "cmux_linux_agent", + "requiredCapabilities": {"build_helper"}, + "slotClaims": ("linux_medium_slot",), + "largeSlotClaims": ("linux_heavy_slot",), + }, + "background_verification": { + "role": "cmux_linux_ci", + "requiredCapabilities": {"background_verification"}, + "slotClaims": ("linux_medium_slot",), + }, + "artifact_cache_service": { + "role": "artifact_cache", + "requiredCapabilities": {"artifact_service"}, + "slotClaims": (), + }, + "background_replay": { + "role": "background_replay", + "requiredCapabilities": {"background_replay"}, + "slotClaims": (), + }, + "benchmark": { + "role": "benchmark", + "requiredCapabilities": {"benchmark"}, + "slotClaims": (), + }, + "diagnostic": { + "role": "diagnostic", + "requiredCapabilities": {"diagnostic"}, + "slotClaims": (), + }, +} + + +class RoleModelError(RuntimeError): + """Closed refusal for invalid role/capability evidence.""" + + +def _exact(value: object, keys: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != keys: + raise RoleModelError(f"{label} has unknown or missing fields") + return value + + +def _token(value: object, label: str, *, optional: bool = False) -> str | None: + if value is None and optional: + return None + if not isinstance(value, str) or TOKEN_RE.fullmatch(value) is None: + raise RoleModelError(f"{label} is invalid") + return value + + +def _sha(value: object, label: str) -> str: + if not isinstance(value, str) or SHA256_RE.fullmatch(value) is None: + raise RoleModelError(f"{label} is invalid") + return value + + +def _positive_int(value: object, label: str, *, maximum: int = 2**31 - 1) -> int: + if type(value) is not int or value < 1 or value > maximum: + raise RoleModelError(f"{label} is invalid") + return value + + +def _nonnegative_int(value: object, label: str, *, maximum: int = 2**31 - 1) -> int: + if type(value) is not int or value < 0 or value > maximum: + raise RoleModelError(f"{label} is invalid") + return value + + +def _sorted_tokens( + value: object, label: str, *, allowed: set[str] | None = None +) -> list[str]: + if ( + not isinstance(value, list) + or len(value) > 64 + or any(not isinstance(v, str) for v in value) + ): + raise RoleModelError(f"{label} is invalid") + if value != sorted(set(value)): + raise RoleModelError(f"{label} must be sorted and unique") + for item in value: + _token(item, label) + if allowed is not None and item not in allowed: + raise RoleModelError(f"{label} contains an unsupported value") + return value + + +def _sha_map( + value: object, + label: str, + *, + allowed_keys: set[str] | None = None, +) -> dict[str, str]: + if not isinstance(value, dict) or len(value) > 64: + raise RoleModelError(f"{label} is invalid") + for key, generation in value.items(): + _token(key, label) + _sha(generation, label) + if allowed_keys is not None and key not in allowed_keys: + raise RoleModelError(f"{label} contains an unsupported key") + return value + + +def _class_at_least(current: str, required: str, classes: tuple[str, ...]) -> bool: + return classes.index(current) >= classes.index(required) + + +def validate_node(value: object) -> dict[str, Any]: + doc = _exact( + value, + { + "schema", + "nodeId", + "state", + "platform", + "architecture", + "osVersionClass", + "enrollmentGeneration", + "glaedaGeneration", + "cpuClass", + "memoryClass", + "diskClass", + "capabilityGeneration", + "toolchainProfiles", + "roleWorkloadGenerations", + "capabilities", + "pressure", + }, + "node capability", + ) + if doc["schema"] != NODE_SCHEMA: + raise RoleModelError("node capability schema is unsupported") + if not isinstance(doc["nodeId"], str) or NODE_RE.fullmatch(doc["nodeId"]) is None: + raise RoleModelError("nodeId is invalid") + if doc["state"] not in KNOWN_STATES: + raise RoleModelError("node state is unsupported") + if doc["platform"] not in {"macos", "linux"}: + raise RoleModelError("node platform is unsupported") + if doc["architecture"] not in {"arm64", "x86_64"}: + raise RoleModelError("node architecture is unsupported") + _token(doc["osVersionClass"], "OS version class") + _positive_int(doc["enrollmentGeneration"], "enrollment generation") + _sha(doc["glaedaGeneration"], "Glaeda generation") + if doc["cpuClass"] not in CPU_CLASSES: + raise RoleModelError("CPU class is unsupported") + if doc["memoryClass"] not in MEMORY_CLASSES: + raise RoleModelError("memory class is unsupported") + if doc["diskClass"] not in DISK_CLASSES: + raise RoleModelError("disk class is unsupported") + _positive_int(doc["capabilityGeneration"], "capability generation") + _sha_map(doc["toolchainProfiles"], "toolchain profiles") + workload_generations = _sha_map( + doc["roleWorkloadGenerations"], + "role workload generations", + allowed_keys=set(ROLES), + ) + if not workload_generations: + raise RoleModelError("at least one enrolled role workload generation is required") + _sorted_tokens(doc["capabilities"], "node capabilities") + pressure = _exact( + doc["pressure"], {"cpu", "memory", "swap", "thermal"}, "pressure" + ) + if ( + pressure["cpu"] not in PRESSURE_CLASSES + or pressure["memory"] not in PRESSURE_CLASSES + ): + raise RoleModelError("CPU/memory pressure class is unsupported") + if pressure["swap"] not in SWAP_CLASSES: + raise RoleModelError("swap class is unsupported") + if pressure["thermal"] not in THERMAL_CLASSES: + raise RoleModelError("thermal class is unsupported") + return doc + + +def validate_role_canary(value: object) -> dict[str, Any]: + doc = _exact( + value, + { + "schema", + "nodeId", + "enrollmentGeneration", + "glaedaGeneration", + "capabilityGeneration", + "role", + "toolchainProfile", + "toolchainGeneration", + "acceptedCapabilities", + "acceptedResourceProfiles", + "workloadGeneration", + "result", + }, + "role canary", + ) + if doc["schema"] != ROLE_CANARY_SCHEMA: + raise RoleModelError("role canary schema is unsupported") + if not isinstance(doc["nodeId"], str) or NODE_RE.fullmatch(doc["nodeId"]) is None: + raise RoleModelError("role canary nodeId is invalid") + _positive_int(doc["enrollmentGeneration"], "role canary enrollment generation") + _sha(doc["glaedaGeneration"], "role canary Glaeda generation") + _positive_int( + doc["capabilityGeneration"], "role canary capability generation" + ) + if doc["role"] not in ROLES: + raise RoleModelError("role canary role is unsupported") + profile = _token( + doc["toolchainProfile"], "role canary toolchain profile", optional=True + ) + generation = doc["toolchainGeneration"] + if generation is not None: + _sha(generation, "role canary toolchain generation") + if (profile is None) != (generation is None): + raise RoleModelError("role canary toolchain profile/generation must be paired") + _sorted_tokens(doc["acceptedCapabilities"], "accepted capabilities") + _sorted_tokens( + doc["acceptedResourceProfiles"], + "accepted resource profiles", + allowed=set(RESOURCE_PROFILES), + ) + _sha(doc["workloadGeneration"], "role canary workload generation") + if doc["result"] not in {"accepted", "rejected"}: + raise RoleModelError("role canary result is invalid") + return doc + + +def validate_capacity(value: object) -> dict[str, Any]: + doc = _exact( + value, + { + "schema", + "nodeId", + "enrollmentGeneration", + "glaedaGeneration", + "capabilityGeneration", + "role", + "workloadGeneration", + "toolchainProfile", + "toolchainGeneration", + "resourceProfile", + "slotClass", + "maxConcurrent", + "contentionEvidenceGeneration", + "measurement", + "result", + }, + "capacity evidence", + ) + if doc["schema"] != CAPACITY_SCHEMA: + raise RoleModelError("capacity schema is unsupported") + if not isinstance(doc["nodeId"], str) or NODE_RE.fullmatch(doc["nodeId"]) is None: + raise RoleModelError("capacity nodeId is invalid") + _positive_int(doc["enrollmentGeneration"], "capacity enrollment generation") + _sha(doc["glaedaGeneration"], "capacity Glaeda generation") + _positive_int(doc["capabilityGeneration"], "capacity generation") + if doc["role"] not in ROLES: + raise RoleModelError("capacity role is unsupported") + _sha(doc["workloadGeneration"], "capacity workload generation") + profile = _token( + doc["toolchainProfile"], "capacity toolchain profile", optional=True + ) + generation = doc["toolchainGeneration"] + if generation is not None: + _sha(generation, "capacity toolchain generation") + if (profile is None) != (generation is None): + raise RoleModelError("capacity toolchain profile/generation must be paired") + if doc["resourceProfile"] not in RESOURCE_PROFILES: + raise RoleModelError("resource profile is unsupported") + _token(doc["slotClass"], "slot class") + _positive_int(doc["maxConcurrent"], "max concurrent", maximum=16) + _sha(doc["contentionEvidenceGeneration"], "contention evidence generation") + measurement = _exact( + doc["measurement"], + { + "validatedCompletions", + "p50Millis", + "p90Millis", + "cpuPressure", + "memoryPressure", + "swapClass", + "thermalBehavior", + "unfinishedWork", + }, + "capacity measurement", + ) + _nonnegative_int( + measurement["validatedCompletions"], + "validated completions", + maximum=65535, + ) + _positive_int(measurement["p50Millis"], "p50 millis") + _positive_int(measurement["p90Millis"], "p90 millis") + if measurement["p90Millis"] < measurement["p50Millis"]: + raise RoleModelError("p90 cannot be lower than p50") + if measurement["cpuPressure"] not in PRESSURE_CLASSES: + raise RoleModelError("capacity CPU pressure is unsupported") + if measurement["memoryPressure"] not in PRESSURE_CLASSES: + raise RoleModelError("capacity memory pressure is unsupported") + if measurement["swapClass"] not in SWAP_CLASSES: + raise RoleModelError("capacity swap class is unsupported") + if measurement["thermalBehavior"] not in THERMAL_CLASSES: + raise RoleModelError("capacity thermal class is unsupported") + _nonnegative_int( + measurement["unfinishedWork"], "unfinished work", maximum=65535 + ) + if doc["result"] not in {"accepted", "rejected"}: + raise RoleModelError("capacity result is invalid") + return doc + + +def validate_workload(value: object) -> dict[str, Any]: + doc = _exact( + value, + { + "schema", + "operation", + "role", + "platform", + "architecture", + "toolchainProfile", + "minimumCpuClass", + "minimumMemoryClass", + "requiredCapabilities", + "resourceProfile", + }, + "workload requirement", + ) + if doc["schema"] != WORKLOAD_SCHEMA: + raise RoleModelError("workload schema is unsupported") + operation = _token(doc["operation"], "operation") + if operation not in OPERATION_CONTRACTS: + raise RoleModelError("operation is unsupported") + contract = OPERATION_CONTRACTS[operation] + if doc["role"] != contract["role"]: + raise RoleModelError("operation and role disagree") + if doc["platform"] not in {"macos", "linux"}: + raise RoleModelError("workload platform is unsupported") + if doc["architecture"] not in {"arm64", "x86_64"}: + raise RoleModelError("workload architecture is unsupported") + _token( + doc["toolchainProfile"], "workload toolchain profile", optional=True + ) + if doc["minimumCpuClass"] not in CPU_CLASSES: + raise RoleModelError("workload CPU class is unsupported") + if doc["minimumMemoryClass"] not in MEMORY_CLASSES: + raise RoleModelError("workload memory class is unsupported") + required = _sorted_tokens( + doc["requiredCapabilities"], "required capabilities" + ) + if not set(contract["requiredCapabilities"]).issubset(required): + raise RoleModelError("workload omits operation-required capability") + if doc["resourceProfile"] not in RESOURCE_PROFILES: + raise RoleModelError("workload resource profile is unsupported") + return doc + + +def validate_preference(value: object) -> dict[str, Any]: + doc = _exact( + value, + { + "schema", + "nodeId", + "role", + "evidenceGeneration", + "preference", + "authority", + }, + "placement preference", + ) + if doc["schema"] != PREFERENCE_SCHEMA: + raise RoleModelError("preference schema is unsupported") + if not isinstance(doc["nodeId"], str) or NODE_RE.fullmatch(doc["nodeId"]) is None: + raise RoleModelError("preference nodeId is invalid") + if doc["role"] not in ROLES: + raise RoleModelError("preference role is unsupported") + _sha(doc["evidenceGeneration"], "preference evidence generation") + if doc["preference"] not in {"preferred", "neutral"}: + raise RoleModelError("preference value is unsupported") + if doc["authority"] != "observation_only": + raise RoleModelError( + "preference authority must remain observation_only" + ) + return doc + + +def role_eligibility( + node_value: object, canary_values: Iterable[object] +) -> dict[str, dict[str, Any]]: + node = validate_node(node_value) + canaries: dict[str, dict[str, Any]] = {} + for value in canary_values: + canary = validate_role_canary(value) + if canary["nodeId"] != node["nodeId"]: + raise RoleModelError("role canary belongs to a different node") + if canary["role"] in canaries: + raise RoleModelError("duplicate role canary") + canaries[canary["role"]] = canary + + results: dict[str, dict[str, Any]] = {} + capabilities = set(node["capabilities"]) + toolchains = node["toolchainProfiles"] + for role in ROLES: + requirement = ROLE_REQUIREMENTS[role] + if node["state"] not in ROUTABLE_STATES: + results[role] = { + "eligible": False, + "reason": f"node_{node['state']}", + } + continue + platform = requirement["platform"] + if platform is not None and platform != node["platform"]: + results[role] = { + "eligible": False, + "reason": "platform_mismatch", + } + continue + allowed_os = requirement.get("allowedOsVersionClasses") + if allowed_os is not None and node["osVersionClass"] not in allowed_os: + results[role] = { + "eligible": False, + "reason": "unsupported_os_version_class", + } + continue + if node["architecture"] not in requirement["architectures"]: + results[role] = { + "eligible": False, + "reason": "architecture_mismatch", + } + continue + expected_workload = node["roleWorkloadGenerations"].get(role) + if expected_workload is None: + results[role] = { + "eligible": False, + "reason": "role_not_enrolled", + } + continue + if not _class_at_least( + node["cpuClass"], + requirement["minimumCpuClass"], + CPU_CLASSES, + ): + results[role] = { + "eligible": False, + "reason": "insufficient_cpu_class", + } + continue + if not _class_at_least( + node["memoryClass"], + requirement["minimumMemoryClass"], + MEMORY_CLASSES, + ): + results[role] = { + "eligible": False, + "reason": "insufficient_memory_class", + } + continue + if not _class_at_least( + node["diskClass"], + requirement["minimumDiskClass"], + DISK_CLASSES, + ): + results[role] = { + "eligible": False, + "reason": "insufficient_disk_class", + } + continue + if not requirement["requiredCapabilities"].issubset(capabilities): + results[role] = { + "eligible": False, + "reason": "node_capability_missing", + } + continue + canary = canaries.get(role) + if canary is None: + results[role] = { + "eligible": False, + "reason": "role_canary_pending", + } + continue + if canary["result"] != "accepted": + results[role] = { + "eligible": False, + "reason": "role_canary_failed", + } + continue + if canary["enrollmentGeneration"] != node["enrollmentGeneration"]: + results[role] = { + "eligible": False, + "reason": "role_canary_enrollment_stale", + } + continue + if canary["glaedaGeneration"] != node["glaedaGeneration"]: + results[role] = { + "eligible": False, + "reason": "role_canary_glaeda_stale", + } + continue + if canary["workloadGeneration"] != expected_workload: + results[role] = { + "eligible": False, + "reason": "role_canary_workload_stale", + } + continue + if canary["capabilityGeneration"] != node["capabilityGeneration"]: + results[role] = { + "eligible": False, + "reason": "role_canary_pending", + } + continue + if not set(requirement["requiredCapabilities"]).issubset( + canary["acceptedCapabilities"] + ): + results[role] = { + "eligible": False, + "reason": "role_canary_capability_incomplete", + } + continue + if requirement["requiresToolchainProfile"]: + profile = canary["toolchainProfile"] + generation = canary["toolchainGeneration"] + if ( + profile is None + or generation is None + or toolchains.get(profile) != generation + ): + results[role] = { + "eligible": False, + "reason": "toolchain_canary_pending", + } + continue + elif ( + canary["toolchainProfile"] is not None + or canary["toolchainGeneration"] is not None + ): + results[role] = { + "eligible": False, + "reason": "role_canary_context_mismatch", + } + continue + results[role] = { + "eligible": True, + "reason": "accepted", + "glaedaGeneration": canary["glaedaGeneration"], + "workloadGeneration": canary["workloadGeneration"], + "toolchainProfile": canary["toolchainProfile"], + "toolchainGeneration": canary["toolchainGeneration"], + "acceptedCapabilities": canary["acceptedCapabilities"], + "acceptedResourceProfiles": canary[ + "acceptedResourceProfiles" + ], + } + return results + + +def _capacity_matches_context( + node: dict[str, Any], + receipt: dict[str, Any], + role_status: dict[str, Any], +) -> bool: + return ( + role_status.get("eligible") is True + and receipt["nodeId"] == node["nodeId"] + and receipt["enrollmentGeneration"] == node["enrollmentGeneration"] + and receipt["glaedaGeneration"] == node["glaedaGeneration"] + and receipt["capabilityGeneration"] == node["capabilityGeneration"] + and receipt["workloadGeneration"] == role_status["workloadGeneration"] + and receipt["toolchainProfile"] == role_status["toolchainProfile"] + and receipt["toolchainGeneration"] == role_status["toolchainGeneration"] + ) + + +def measured_slot_capacity( + node_value: object, + canary_values: Iterable[object], + capacity_values: Iterable[object], +) -> dict[str, int]: + canary_values = list(canary_values) + capacity_values = list(capacity_values) + node = validate_node(node_value) + eligibility = role_eligibility(node, canary_values) + slots: dict[str, int] = {} + for value in capacity_values: + receipt = validate_capacity(value) + if receipt["nodeId"] != node["nodeId"]: + raise RoleModelError( + "capacity evidence belongs to a different node" + ) + role = receipt["role"] + if not _capacity_matches_context(node, receipt, eligibility[role]): + continue + if receipt["result"] != "accepted": + continue + if ( + receipt["resourceProfile"] + not in eligibility[role]["acceptedResourceProfiles"] + ): + continue + slots[receipt["slotClass"]] = max( + slots.get(receipt["slotClass"], 0), receipt["maxConcurrent"] + ) + return dict(sorted(slots.items())) + + +def _slot_claims(workload: dict[str, Any]) -> tuple[str, ...]: + contract = OPERATION_CONTRACTS[workload["operation"]] + if ( + workload["resourceProfile"] in {"large", "exclusive"} + and "largeSlotClaims" in contract + ): + return contract["largeSlotClaims"] + return contract["slotClaims"] + + +def _profile_has_capacity( + node: dict[str, Any], + workload: dict[str, Any], + eligibility: dict[str, dict[str, Any]], + capacity_values: Iterable[object], +) -> bool: + role = workload["role"] + if ( + workload["resourceProfile"] + not in eligibility[role].get("acceptedResourceProfiles", []) + ): + return False + claims = set(_slot_claims(workload)) + matched = set() + measured_profile = False + for value in capacity_values: + receipt = validate_capacity(value) + if ( + receipt["role"] != role + or not _capacity_matches_context(node, receipt, eligibility[role]) + or receipt["resourceProfile"] + != workload["resourceProfile"] + or receipt["result"] != "accepted" + ): + continue + measured_profile = True + if receipt["slotClass"] in claims: + matched.add(receipt["slotClass"]) + if not measured_profile: + return False + return matched == claims + + +def select_eligible( + workload_value: object, + node_value: object, + canary_values: Iterable[object], + capacity_values: Iterable[object], +) -> tuple[bool, str]: + canary_values = list(canary_values) + capacity_values = list(capacity_values) + workload = validate_workload(workload_value) + node = validate_node(node_value) + eligibility = role_eligibility(node, canary_values) + role = workload["role"] + if not eligibility[role]["eligible"]: + return False, eligibility[role]["reason"] + if workload["platform"] != node["platform"]: + return False, "platform_mismatch" + if workload["architecture"] != node["architecture"]: + return False, "architecture_mismatch" + if not _class_at_least( + node["cpuClass"], + workload["minimumCpuClass"], + CPU_CLASSES, + ): + return False, "insufficient_cpu_class" + if not _class_at_least( + node["memoryClass"], + workload["minimumMemoryClass"], + MEMORY_CLASSES, + ): + return False, "insufficient_memory_class" + if not set(workload["requiredCapabilities"]).issubset( + node["capabilities"] + ): + return False, "node_capability_missing" + if not set(workload["requiredCapabilities"]).issubset( + eligibility[role].get("acceptedCapabilities", []) + ): + return False, "role_canary_capability_incomplete" + requested_toolchain = workload["toolchainProfile"] + if requested_toolchain is not None: + if requested_toolchain not in node["toolchainProfiles"]: + return False, "toolchain_profile_missing" + if eligibility[role].get("toolchainProfile") != requested_toolchain: + return False, "toolchain_profile_ineligible" + if not _profile_has_capacity( + node, workload, eligibility, capacity_values + ): + return False, "resource_profile_unmeasured" + return True, "eligible" + + +def _workload_slot_capacities( + node: dict[str, Any], + workload: dict[str, Any], + eligibility: dict[str, dict[str, Any]], + capacity_values: Iterable[object], +) -> dict[str, int]: + capacities: dict[str, int] = {} + for value in capacity_values: + receipt = validate_capacity(value) + if ( + receipt["role"] != workload["role"] + or not _capacity_matches_context( + node, receipt, eligibility[workload["role"]] + ) + or receipt["resourceProfile"] + != workload["resourceProfile"] + or receipt["result"] != "accepted" + ): + continue + capacities[receipt["slotClass"]] = max( + capacities.get(receipt["slotClass"], 0), + receipt["maxConcurrent"], + ) + return capacities + + +def local_admission( + workload_value: object, + node_value: object, + canary_values: Iterable[object], + capacity_values: Iterable[object], + held_slots: dict[str, int], +) -> dict[str, Any]: + canary_values = list(canary_values) + capacity_values = list(capacity_values) + workload = validate_workload(workload_value) + node = validate_node(node_value) + ok, reason = select_eligible( + workload, node, canary_values, capacity_values + ) + if not ok: + return { + "schema": DECISION_SCHEMA, + "authority": "admission_only", + "leaseBoundary": None, + "nodeId": node["nodeId"], + "operation": workload["operation"], + "accepted": False, + "reason": reason, + "resourceProfile": workload["resourceProfile"], + "physicalLeaseRequired": False, + "slotClaims": [], + } + + pressure = node["pressure"] + if ( + pressure["cpu"] == "critical" + or pressure["memory"] == "critical" + or pressure["swap"] == "critical" + or pressure["thermal"] == "critical" + ): + return { + "schema": DECISION_SCHEMA, + "authority": "admission_only", + "leaseBoundary": None, + "nodeId": node["nodeId"], + "operation": workload["operation"], + "accepted": False, + "reason": "node_pressured", + "resourceProfile": workload["resourceProfile"], + "physicalLeaseRequired": False, + "slotClaims": [], + } + + eligibility = role_eligibility(node, canary_values) + capacities = _workload_slot_capacities( + node, workload, eligibility, capacity_values + ) + claims = list(_slot_claims(workload)) + for slot in claims: + if capacities.get(slot, 0) <= held_slots.get(slot, 0): + return { + "schema": DECISION_SCHEMA, + "authority": "admission_only", + "leaseBoundary": "physical_execution_lease", + "nodeId": node["nodeId"], + "operation": workload["operation"], + "accepted": False, + "reason": "physical_slot_unavailable", + "resourceProfile": workload["resourceProfile"], + "physicalLeaseRequired": True, + "slotClaims": claims, + } + return { + "schema": DECISION_SCHEMA, + "authority": "admission_only", + "leaseBoundary": "physical_execution_lease" if claims else None, + "nodeId": node["nodeId"], + "operation": workload["operation"], + "accepted": True, + "reason": "admitted", + "resourceProfile": workload["resourceProfile"], + "physicalLeaseRequired": bool(claims), + "slotClaims": claims, + } + + +def operator_status( + node_value: object, + canary_values: Iterable[object], + capacity_values: Iterable[object], + preference_values: Iterable[object] = (), +) -> dict[str, Any]: + canary_values = list(canary_values) + capacity_values = list(capacity_values) + preference_values = list(preference_values) + node = validate_node(node_value) + eligibility = role_eligibility(node, canary_values) + preferred: list[str] = [] + for value in preference_values: + preference = validate_preference(value) + if preference["nodeId"] != node["nodeId"]: + raise RoleModelError( + "preference belongs to a different node" + ) + if preference["preference"] == "preferred": + preferred.append(preference["role"]) + preferred = sorted(set(preferred)) + eligible = [ + role for role in ROLES if eligibility[role]["eligible"] + ] + unavailable = [ + {"role": role, "reason": eligibility[role]["reason"]} + for role in ROLES + if not eligibility[role]["eligible"] + and eligibility[role]["reason"] + not in {"platform_mismatch", "architecture_mismatch"} + ] + return { + "schema": STATUS_SCHEMA, + "node": node["nodeId"], + "eligible": eligible, + "temporarilyUnavailable": unavailable, + "capacity": measured_slot_capacity( + node, canary_values, capacity_values + ), + "preferred": preferred, + "state": node["state"], + } + + +def render_status(status_value: object) -> str: + status = _exact( + status_value, + { + "schema", + "node", + "eligible", + "temporarilyUnavailable", + "capacity", + "preferred", + "state", + }, + "operator status", + ) + if status["schema"] != STATUS_SCHEMA: + raise RoleModelError("operator status schema is unsupported") + lines = [f"node {status['node']}", "eligible:"] + lines.extend(f" {role}" for role in status["eligible"]) + lines.append("temporarily unavailable:") + for item in status["temporarilyUnavailable"]: + lines.append(f" {item['role']}: {item['reason']}") + lines.append("capacity:") + lines.extend( + f" {slot}: {count}" + for slot, count in status["capacity"].items() + ) + lines.append("preferred:") + lines.extend(f" {role}" for role in status["preferred"]) + lines.append(f"state: {status['state']}") + return "\n".join(lines) + "\n" + + +def render_json(value: object) -> str: + return ( + json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ) \ No newline at end of file diff --git a/scripts/cmux_fleet.py b/scripts/cmux_fleet.py new file mode 100644 index 000000000..7818ccd8e --- /dev/null +++ b/scripts/cmux_fleet.py @@ -0,0 +1,518 @@ +#!/usr/bin/env python3 +"""CMUX-owned fleet enrollment, role acceptance, and bounded status projection.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from pathlib import Path +from typing import Any + +ENROLLMENT_SCHEMA = "glaeda-cmux-fleet-enrollment/v1" +ACCEPTANCE_EVIDENCE_SCHEMA = "glaeda-cmux-fleet-acceptance-evidence/v1" +ACCEPTANCE_SCHEMA = "glaeda-cmux-fleet-acceptance/v1" +STATUS_SCHEMA = "glaeda-cmux-fleet-node-status/v1" +BOOTSTRAP_SCHEMA = "glaeda-cmux-fleet-bootstrap/v1" +MAX_DOCUMENT_BYTES = 64 * 1024 +MAX_STATUS_BYTES = 16 * 1024 +SHA256_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +COMMIT_RE = re.compile(r"[0-9a-f]{40}\Z") +TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9._-]{0,79}\Z") +NODE_RE = re.compile(r"cmux-[a-z0-9][a-z0-9-]{2,59}\Z") +REPOSITORY_RE = re.compile(r"[A-Za-z0-9_.-]{1,64}/[A-Za-z0-9_.-]{1,100}\Z") + +ROLES = ( + "artifact_cache", + "background_replay", + "benchmark", + "cmux_linux_agent", + "cmux_linux_ci", + "cmux_macos_native_build", + "cmux_macos_test", + "diagnostic", +) +ENROLLABLE_ROLES = { + "cmux_linux_ci", + "cmux_macos_native_build", +} +STATES = ("discovered", "enrolling", "eligible", "draining", "quarantined", "retired") +QUARANTINE_REASONS = ( + "dirty_canonical_checkout", + "disk_pressure", + "failed_acceptance", + "hardware_failure", + "service_mismatch", + "stale_glaeda_generation", + "toolchain_mismatch", + "unexplained_process_settlement", +) +ROLE_OS = { + "cmux_macos_native_build": "macos", + "cmux_macos_test": "macos", + "cmux_linux_ci": "linux", + "cmux_linux_agent": "linux", +} +TRANSITIONS = { + "discovered": {"enrolling", "retired"}, + "enrolling": {"eligible", "quarantined", "retired"}, + "eligible": {"draining", "quarantined", "retired"}, + "draining": {"eligible", "quarantined", "retired"}, + "quarantined": {"enrolling", "retired"}, + "retired": set(), +} +ENROLLMENT_KEYS = { + "schema", "nodeId", "architecture", "os", "hardwareCapabilityClass", + "supportedToolchainGenerations", "roleWorkloadGenerations", + "allowedExecutionRoles", "operatorFleetScope", + "enrollmentGeneration", "glaedaGeneration", "state", "quarantineReason", +} +ACCEPTANCE_EVIDENCE_KEYS = { + "schema", "nodeId", "enrollmentGeneration", "role", "source", + "toolchainGeneration", "glaedaGeneration", "workloadGeneration", "checks", +} +CHECK_KEYS = {"workload", "semanticVerifier", "artifact", "processSettlement"} + + +class FleetError(RuntimeError): + """A closed fleet enrollment/status refusal.""" + + +def canonical(value: object) -> bytes: + return (json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n").encode() + + +def digest(value: object) -> str: + return "sha256:" + hashlib.sha256(canonical(value)).hexdigest() + + +def exact_keys(value: object, keys: set[str], label: str) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != keys: + raise FleetError(f"{label} has unknown or missing fields") + return value + + +def token(value: object, label: str) -> str: + if not isinstance(value, str) or TOKEN_RE.fullmatch(value) is None: + raise FleetError(f"{label} is invalid") + return value + + +def sha256(value: object, label: str, *, optional: bool = False) -> str | None: + if value is None and optional: + return None + if not isinstance(value, str) or SHA256_RE.fullmatch(value) is None: + raise FleetError(f"{label} is invalid") + return value + + +def positive_int(value: object, label: str) -> int: + if type(value) is not int or not 1 <= value <= 2**31 - 1: + raise FleetError(f"{label} is invalid") + return value + + +def sorted_unique_strings(value: object, label: str, *, allowed: set[str] | None = None) -> list[str]: + if not isinstance(value, list) or len(value) > 32 or any(not isinstance(v, str) for v in value): + raise FleetError(f"{label} is invalid") + if value != sorted(set(value)): + raise FleetError(f"{label} must be sorted and unique") + if allowed is not None and any(v not in allowed for v in value): + raise FleetError(f"{label} contains an unsupported value") + return value + + +def validate_enrollment(value: object) -> dict[str, Any]: + doc = exact_keys(value, ENROLLMENT_KEYS, "enrollment") + if doc["schema"] != ENROLLMENT_SCHEMA: + raise FleetError("enrollment schema is unsupported") + if not isinstance(doc["nodeId"], str) or NODE_RE.fullmatch(doc["nodeId"]) is None: + raise FleetError("nodeId must be an opaque cmux-* identifier") + if doc["architecture"] not in {"arm64", "x86_64"}: + raise FleetError("architecture is unsupported") + os_doc = exact_keys(doc["os"], {"family", "versionClass"}, "os") + if os_doc["family"] not in {"macos", "linux"}: + raise FleetError("OS family is unsupported") + token(os_doc["versionClass"], "OS version class") + token(doc["hardwareCapabilityClass"], "hardware capability class") + token(doc["operatorFleetScope"], "operator/fleet scope") + generations = sorted_unique_strings(doc["supportedToolchainGenerations"], "supported toolchain generations") + if not generations or any(SHA256_RE.fullmatch(v) is None for v in generations): + raise FleetError("supported toolchain generations are invalid") + roles = sorted_unique_strings(doc["allowedExecutionRoles"], "allowed execution roles", allowed=set(ROLES)) + if not roles: + raise FleetError("at least one execution role is required") + workload_generations = doc["roleWorkloadGenerations"] + if ( + not isinstance(workload_generations, dict) + or set(workload_generations) != set(roles) + or any( + not isinstance(value, str) or SHA256_RE.fullmatch(value) is None + for value in workload_generations.values() + ) + ): + raise FleetError("role workload generations must exactly cover allowed roles") + unreviewed = [role for role in roles if role not in ENROLLABLE_ROLES] + if unreviewed: + raise FleetError( + "execution role lacks a reviewed v1 acceptance workload: " + + ",".join(unreviewed) + ) + for role in roles: + required_os = ROLE_OS.get(role) + if required_os is not None and required_os != os_doc["family"]: + raise FleetError(f"role {role} is incompatible with OS family {os_doc['family']}") + positive_int(doc["enrollmentGeneration"], "enrollment generation") + sha256(doc["glaedaGeneration"], "Glaeda generation") + if doc["state"] not in STATES: + raise FleetError("node state is unsupported") + reason = doc["quarantineReason"] + if doc["state"] == "quarantined": + if reason not in QUARANTINE_REASONS: + raise FleetError("quarantine reason is required and must be reviewed") + elif reason is not None: + raise FleetError("quarantine reason is only valid for quarantined nodes") + return doc + + +def enrollment_from_bootstrap( + bootstrap_value: object, + *, + node_id: str, + operator_fleet_scope: str, + enrollment_generation: int, +) -> dict[str, Any]: + if ( + not isinstance(bootstrap_value, dict) + or bootstrap_value.get("schema") != BOOTSTRAP_SCHEMA + ): + raise FleetError("bootstrap receipt schema is unsupported") + if bootstrap_value.get("authority") != "observation_only": + raise FleetError("bootstrap receipt authority is invalid") + if bootstrap_value.get("eligibleForEnrollment") is not True: + raise FleetError("bootstrap receipt has blocking checks") + enrollment = { + "schema": ENROLLMENT_SCHEMA, + "nodeId": node_id, + "architecture": bootstrap_value.get("architecture"), + "os": { + "family": bootstrap_value.get("platform"), + "versionClass": bootstrap_value.get("osVersionClass"), + }, + "hardwareCapabilityClass": bootstrap_value.get("hardwareCapabilityClass"), + "supportedToolchainGenerations": [ + bootstrap_value.get("toolchainGeneration") + ], + "roleWorkloadGenerations": bootstrap_value.get("roleWorkloadGenerations"), + "allowedExecutionRoles": bootstrap_value.get("roles"), + "operatorFleetScope": operator_fleet_scope, + "enrollmentGeneration": enrollment_generation, + "glaedaGeneration": bootstrap_value.get("glaedaGeneration"), + "state": "enrolling", + "quarantineReason": None, + } + return validate_enrollment(enrollment) + + +def validate_acceptance_evidence(value: object) -> dict[str, Any]: + doc = exact_keys(value, ACCEPTANCE_EVIDENCE_KEYS, "acceptance evidence") + if doc["schema"] != ACCEPTANCE_EVIDENCE_SCHEMA: + raise FleetError("acceptance evidence schema is unsupported") + if not isinstance(doc["nodeId"], str) or NODE_RE.fullmatch(doc["nodeId"]) is None: + raise FleetError("acceptance nodeId is invalid") + positive_int(doc["enrollmentGeneration"], "acceptance enrollment generation") + if doc["role"] not in ENROLLABLE_ROLES: + raise FleetError("acceptance role lacks a reviewed v1 workload") + source = exact_keys(doc["source"], {"repository", "commit"}, "source") + if not isinstance(source["repository"], str) or REPOSITORY_RE.fullmatch(source["repository"]) is None: + raise FleetError("source repository is invalid") + if not isinstance(source["commit"], str) or COMMIT_RE.fullmatch(source["commit"]) is None: + raise FleetError("source commit is invalid") + sha256(doc["toolchainGeneration"], "acceptance toolchain generation") + sha256(doc["glaedaGeneration"], "acceptance Glaeda generation") + sha256(doc["workloadGeneration"], "acceptance workload generation") + checks = exact_keys(doc["checks"], CHECK_KEYS, "acceptance checks") + if any(v not in {"pass", "fail"} for v in checks.values()): + raise FleetError("acceptance checks must be pass or fail") + return doc + + +ACCEPTANCE_RECEIPT_KEYS = { + "schema", "nodeId", "enrollmentGeneration", "role", "source", + "toolchainGeneration", "glaedaGeneration", "workloadGeneration", "checks", + "result", "evidenceSha256", +} + + +def validate_acceptance_receipt(value: object) -> dict[str, Any]: + doc = exact_keys(value, ACCEPTANCE_RECEIPT_KEYS, "acceptance receipt") + if doc["schema"] != ACCEPTANCE_SCHEMA: + raise FleetError("acceptance receipt schema is unsupported") + if not isinstance(doc["nodeId"], str) or NODE_RE.fullmatch(doc["nodeId"]) is None: + raise FleetError("acceptance receipt nodeId is invalid") + positive_int( + doc["enrollmentGeneration"], + "acceptance receipt enrollment generation", + ) + if doc["role"] not in ENROLLABLE_ROLES: + raise FleetError("acceptance receipt role lacks a reviewed v1 workload") + source = exact_keys( + doc["source"], + {"repository", "commit"}, + "acceptance receipt source", + ) + if ( + not isinstance(source["repository"], str) + or REPOSITORY_RE.fullmatch(source["repository"]) is None + ): + raise FleetError("acceptance receipt source repository is invalid") + if ( + not isinstance(source["commit"], str) + or COMMIT_RE.fullmatch(source["commit"]) is None + ): + raise FleetError("acceptance receipt source commit is invalid") + sha256( + doc["toolchainGeneration"], + "acceptance receipt toolchain generation", + ) + sha256( + doc["glaedaGeneration"], + "acceptance receipt Glaeda generation", + ) + sha256( + doc["workloadGeneration"], + "acceptance receipt workload generation", + ) + sha256(doc["evidenceSha256"], "acceptance evidence digest") + checks = exact_keys( + doc["checks"], + CHECK_KEYS, + "acceptance receipt checks", + ) + if any(value not in {"pass", "fail"} for value in checks.values()): + raise FleetError("acceptance receipt checks must be pass or fail") + if doc["result"] not in {"accepted", "rejected"}: + raise FleetError("acceptance receipt result is invalid") + all_pass = all(value == "pass" for value in checks.values()) + if (doc["result"] == "accepted") != all_pass: + raise FleetError("acceptance receipt result disagrees with checks") + return doc + + +def acceptance_matches_enrollment(enrollment: dict[str, Any], receipt: dict[str, Any], role: str) -> tuple[bool, str]: + if receipt.get("schema") != ACCEPTANCE_SCHEMA or receipt.get("result") != "accepted": + return False, "acceptance_missing_or_rejected" + if receipt.get("nodeId") != enrollment["nodeId"] or receipt.get("role") != role: + return False, "acceptance_identity_mismatch" + if receipt.get("enrollmentGeneration") != enrollment["enrollmentGeneration"]: + return False, "acceptance_enrollment_stale" + if receipt.get("glaedaGeneration") != enrollment["glaedaGeneration"]: + return False, "acceptance_glaeda_stale" + if receipt.get("toolchainGeneration") not in enrollment["supportedToolchainGenerations"]: + return False, "acceptance_toolchain_stale" + if receipt.get("workloadGeneration") != enrollment["roleWorkloadGenerations"].get(role): + return False, "acceptance_workload_stale" + return True, "accepted" + + +def finalize_acceptance(enrollment_value: object, evidence_value: object) -> dict[str, Any]: + enrollment = validate_enrollment(enrollment_value) + evidence = validate_acceptance_evidence(evidence_value) + if evidence["nodeId"] != enrollment["nodeId"]: + raise FleetError("acceptance node identity differs from enrollment") + if evidence["enrollmentGeneration"] != enrollment["enrollmentGeneration"]: + raise FleetError("acceptance enrollment generation differs from current enrollment") + if evidence["role"] not in enrollment["allowedExecutionRoles"]: + raise FleetError("acceptance role is outside the enrollment allowlist") + if evidence["toolchainGeneration"] not in enrollment["supportedToolchainGenerations"]: + raise FleetError("acceptance toolchain generation is outside the enrollment allowlist") + if evidence["glaedaGeneration"] != enrollment["glaedaGeneration"]: + raise FleetError("acceptance Glaeda generation differs from current enrollment") + if evidence["workloadGeneration"] != enrollment["roleWorkloadGenerations"].get(evidence["role"]): + raise FleetError("acceptance workload generation differs from current enrollment") + result = "accepted" if all(v == "pass" for v in evidence["checks"].values()) else "rejected" + receipt = { + "schema": ACCEPTANCE_SCHEMA, + "nodeId": evidence["nodeId"], + "enrollmentGeneration": evidence["enrollmentGeneration"], + "role": evidence["role"], + "source": evidence["source"], + "toolchainGeneration": evidence["toolchainGeneration"], + "glaedaGeneration": evidence["glaedaGeneration"], + "workloadGeneration": evidence["workloadGeneration"], + "checks": evidence["checks"], + "result": result, + "evidenceSha256": digest(evidence), + } + if len(canonical(receipt)) > MAX_STATUS_BYTES: + raise FleetError("acceptance receipt exceeds size ceiling") + return receipt + + +def node_status(enrollment_value: object, acceptance_values: list[object]) -> dict[str, Any]: + enrollment = validate_enrollment(enrollment_value) + receipts: dict[str, dict[str, Any]] = {} + for value in acceptance_values: + receipt = validate_acceptance_receipt(value) + role = receipt["role"] + if role in receipts: + raise FleetError("duplicate acceptance receipt for role") + receipts[role] = receipt + role_status = [] + for role in enrollment["allowedExecutionRoles"]: + if enrollment["state"] != "eligible": + eligible, reason = False, f"node_{enrollment['state']}" + else: + receipt = receipts.get(role) + if receipt is None: + eligible, reason = False, "acceptance_missing_or_rejected" + else: + eligible, reason = acceptance_matches_enrollment(enrollment, receipt, role) + role_status.append({"role": role, "eligible": eligible, "reason": reason}) + status = { + "schema": STATUS_SCHEMA, + "nodeId": enrollment["nodeId"], + "enrollmentGeneration": enrollment["enrollmentGeneration"], + "state": enrollment["state"], + "routingCandidateEligible": any(r["eligible"] for r in role_status), + "automaticDispatchAuthorized": False, + "capability": { + "architecture": enrollment["architecture"], + "osFamily": enrollment["os"]["family"], + "osVersionClass": enrollment["os"]["versionClass"], + "hardwareCapabilityClass": enrollment["hardwareCapabilityClass"], + "supportedToolchainGenerations": enrollment[ + "supportedToolchainGenerations" + ], + "roleWorkloadGenerations": enrollment["roleWorkloadGenerations"], + "glaedaGeneration": enrollment["glaedaGeneration"], + "operatorFleetScope": enrollment["operatorFleetScope"], + }, + "roles": role_status, + "privacy": { + "containsSerialNumber": False, + "containsPrivateNetworking": False, + "containsUsername": False, + }, + } + if len(canonical(status)) > MAX_STATUS_BYTES: + raise FleetError("status document exceeds size ceiling") + return status + + +def transition( + enrollment_value: object, + target: str, + reason: str | None, + acceptance_values: list[object] | None = None, +) -> dict[str, Any]: + enrollment = dict(validate_enrollment(enrollment_value)) + current = enrollment["state"] + if target not in STATES or target not in TRANSITIONS[current]: + raise FleetError(f"transition {current}->{target} is unsupported") + if target == "quarantined": + if reason not in QUARANTINE_REASONS: + raise FleetError("a reviewed quarantine reason is required") + enrollment["quarantineReason"] = reason + else: + if reason is not None: + raise FleetError("transition reason is only accepted for quarantine") + enrollment["quarantineReason"] = None + if current == "quarantined" and target == "enrolling": + if enrollment["enrollmentGeneration"] == 2**31 - 1: + raise FleetError("enrollment generation is exhausted") + enrollment["enrollmentGeneration"] += 1 + enrollment["state"] = target + enrollment = validate_enrollment(enrollment) + if target == "eligible": + status = node_status(enrollment, acceptance_values or []) + if not status["routingCandidateEligible"]: + raise FleetError( + "eligible transition requires a current accepted role receipt" + ) + return enrollment + + +def load(path: Path) -> object: + raw = path.read_bytes() + if len(raw) > MAX_DOCUMENT_BYTES: + raise FleetError("fleet document exceeds size ceiling") + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise FleetError("fleet document is invalid JSON") from error + return value + + +def emit(value: object) -> None: + sys.stdout.buffer.write(canonical(value)) + + +def parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + sub = p.add_subparsers(dest="command", required=True) + e = sub.add_parser("enroll") + e.add_argument("bootstrap", type=Path) + e.add_argument("--node-id", required=True) + e.add_argument("--scope", required=True) + e.add_argument("--generation", required=True, type=int) + v = sub.add_parser("validate") + v.add_argument("enrollment", type=Path) + f = sub.add_parser("fingerprint") + f.add_argument("enrollment", type=Path) + s = sub.add_parser("status") + s.add_argument("enrollment", type=Path) + s.add_argument("--acceptance", action="append", type=Path, default=[]) + t = sub.add_parser("transition") + t.add_argument("enrollment", type=Path) + t.add_argument("--to", required=True, choices=STATES) + t.add_argument("--reason", choices=QUARANTINE_REASONS) + t.add_argument("--acceptance", action="append", type=Path, default=[]) + a = sub.add_parser("finalize-acceptance") + a.add_argument("enrollment", type=Path) + a.add_argument("evidence", type=Path) + return p + + +def main() -> int: + try: + args = parser().parse_args() + if args.command == "enroll": + emit( + enrollment_from_bootstrap( + load(args.bootstrap), + node_id=args.node_id, + operator_fleet_scope=args.scope, + enrollment_generation=args.generation, + ) + ) + return 0 + enrollment = load(args.enrollment) + if args.command == "validate": + emit(validate_enrollment(enrollment)) + elif args.command == "fingerprint": + emit({"schema": ENROLLMENT_SCHEMA, "enrollmentSha256": digest(validate_enrollment(enrollment))}) + elif args.command == "status": + emit(node_status(enrollment, [load(path) for path in args.acceptance])) + elif args.command == "transition": + emit( + transition( + enrollment, + args.to, + args.reason, + [load(path) for path in args.acceptance], + ) + ) + elif args.command == "finalize-acceptance": + emit(finalize_acceptance(enrollment, load(args.evidence))) + else: + raise FleetError("unsupported command") + return 0 + except (OSError, FleetError) as error: + print(json.dumps({"error": str(error)}, sort_keys=True, separators=(",", ":")), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/cmux_fleet_bootstrap.py b/scripts/cmux_fleet_bootstrap.py new file mode 100644 index 000000000..fcab62c88 --- /dev/null +++ b/scripts/cmux_fleet_bootstrap.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +"""Read-only CMUX fleet bootstrap observation for macOS and Linux nodes.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import re +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +SCHEMA = "glaeda-cmux-fleet-bootstrap/v1" +MAX_OUTPUT_BYTES = 16 * 1024 +MAX_COMMAND_OUTPUT_BYTES = 16 * 1024 +ENROLLABLE_ROLES = { + "cmux_macos_native_build", + "cmux_linux_ci", +} +ROLE_OS = { + "cmux_macos_native_build": "macos", + "cmux_macos_test": "macos", + "cmux_linux_ci": "linux", + "cmux_linux_agent": "linux", + "artifact_cache": None, + "background_replay": None, + "benchmark": None, + "diagnostic": None, +} + + +class BootstrapError(RuntimeError): + pass + + +def canonical(value: object) -> bytes: + return (json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n").encode() + + +def digest_bytes(value: bytes) -> str: + return "sha256:" + hashlib.sha256(value).hexdigest() + + +def digest_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + h.update(chunk) + return "sha256:" + h.hexdigest() + + +def run( + argv: list[str], + timeout: int = 10, + cwd: Path | None = None, +) -> str: + environment = { + "LC_ALL": "C", + "PATH": os.environ.get("PATH", "/usr/bin:/bin:/usr/sbin:/sbin"), + } + for name in ("HOME", "CARGO_HOME", "RUSTUP_HOME", "DEVELOPER_DIR"): + value = os.environ.get(name) + if value: + environment[name] = value + result = subprocess.run( + argv, + check=False, + cwd=cwd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=environment, + timeout=timeout, + text=True, + ) + if len(result.stdout.encode("utf-8", errors="replace")) > MAX_COMMAND_OUTPUT_BYTES: + raise BootstrapError( + f"required command output is too large: {Path(argv[0]).name}" + ) + if result.returncode != 0: + raise BootstrapError(f"required command failed: {Path(argv[0]).name}") + return result.stdout.strip() + + +def executable(name: str) -> str: + value = shutil.which(name) + if value is None: + raise BootstrapError(f"required command is missing: {name}") + return os.path.abspath(value) + + +def normalize_arch(value: str) -> str: + if value in {"arm64", "aarch64"}: + return "arm64" + if value == "x86_64": + return value + raise BootstrapError(f"unsupported architecture: {value}") + + +def disk_free_gib(path: Path) -> int: + return shutil.disk_usage(path).free // (1024**3) + + +def role_workload_generations(cmux_root: Path, roles: list[str]) -> dict[str, str]: + workload = cmux_root / "scripts/fleet_acceptance.py" + if not workload.is_file(): + raise BootstrapError("CMUX fleet acceptance workload is unavailable") + generation = digest_file(workload) + return {role: generation for role in sorted(set(roles))} + + +def mac_sleep_disabled_on_ac(raw: str) -> bool: + in_ac = False + for line in raw.splitlines(): + stripped = line.strip() + if stripped == "AC Power:": + in_ac = True + continue + if stripped.endswith("Power:"): + in_ac = False + continue + if in_ac: + match = re.fullmatch(r"sleep\s+(\d+)", stripped) + if match: + return int(match.group(1)) == 0 + return False + + +def cmux_checkout_clean(root: Path) -> bool: + return ( + run( + [ + executable("git"), + "status", + "--porcelain=v1", + "--untracked-files=all", + ], + cwd=root, + ) + == "" + ) + + +def cmux_submodules_ready(root: Path) -> bool: + output = run( + [executable("git"), "submodule", "status", "--recursive"], + cwd=root, + ) + lines = [line for line in output.splitlines() if line] + return bool(lines) and all(line[0] == " " for line in lines) + + +def cmux_required_zig_version(root: Path) -> str: + manifest = root / "ghostty/build.zig.zon" + match = re.search( + r'^\s*\.minimum_zig_version\s*=\s*"([0-9]+\.[0-9]+\.[0-9]+)"', + manifest.read_text(encoding="utf-8"), + re.MULTILINE, + ) + if match is None: + raise BootstrapError("Ghostty minimum Zig version is unavailable") + return match.group(1) + + +def zig_version_compatible(actual: str, required: str) -> bool: + def parse(value: str) -> tuple[int, int, int] | None: + core = re.split(r"[-+]", value, maxsplit=1)[0] + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)", core) + if match is None: + return None + return tuple(int(part) for part in match.groups()) + + actual_parts = parse(actual) + required_parts = parse(required) + return bool( + actual_parts + and required_parts + and actual_parts[:2] == required_parts[:2] + and actual_parts[2] >= required_parts[2] + ) + + +def cmux_diff_rust_toolchain(root: Path) -> str: + content = (root / "Native/DiffSidecar/rust-toolchain.toml").read_text( + encoding="utf-8" + ) + match = re.search( + r'^\s*channel\s*=\s*"([^"]+)"', + content, + re.MULTILINE, + ) + if match is None: + raise BootstrapError("CMUX DiffSidecar Rust toolchain is unavailable") + return match.group(1) + + +def collect_macos( + cmux_root: Path, + glaeda: Path, + min_free_gib: int, + hardware_class: str, + cache_root: Path | None, +) -> dict[str, Any]: + if platform.system() != "Darwin": + raise BootstrapError("macOS bootstrap requires Darwin") + version = platform.mac_ver()[0] + match = re.fullmatch(r"(\d+)\.\d+(?:\.\d+)?", version) + if match is None: + raise BootstrapError("macOS version is unavailable") + major = int(match.group(1)) + pin = (cmux_root / ".xcode-version").read_text(encoding="utf-8").strip() + xcode = run([executable("xcodebuild"), "-version"]) + xcrun = executable("xcrun") + sdk = run([xcrun, "--sdk", "macosx", "--show-sdk-version"]) + metal = run([xcrun, "metal", "--version"]) + git = run([executable("git"), "--version"]) + zig = run([executable("zig"), "version"]) + zig_required = cmux_required_zig_version(cmux_root) + rustup = executable("rustup") + rustup_version = run([rustup, "--version"]) + cargo = run([executable("cargo"), "--version"]) + rustc = run([executable("rustc"), "--version"]) + diff_rust = cmux_diff_rust_toolchain(cmux_root) + diff_cargo = run([rustup, "run", diff_rust, "cargo", "--version"]) + diff_rustc = run([rustup, "run", diff_rust, "rustc", "--version"]) + pmset = run([executable("pmset"), "-g", "custom"]) + xcode_match = re.search(r"^Xcode\s+(\d+(?:\.\d+)*)$", xcode, re.MULTILINE) + sdk_match = re.fullmatch(r"(\d+)(?:\.\d+)*", sdk) + toolchain = { + "cmuxXcodePin": pin, + "xcodeVersion": xcode_match.group(1) if xcode_match else "unknown", + "macosSdkVersion": sdk, + "gitVersion": git, + "metalVersion": metal, + "zigVersion": zig, + "zigRequired": zig_required, + "rustupVersion": rustup_version, + "cargoVersion": cargo, + "rustcVersion": rustc, + "diffRustToolchain": diff_rust, + "diffCargoVersion": diff_cargo, + "diffRustcVersion": diff_rustc, + } + free_gib = disk_free_gib(cmux_root) + cache_required = cache_root is not None + cache_ready = ( + cache_root is not None + and cache_root.is_dir() + and os.access(cache_root, os.W_OK | os.X_OK) + ) + cache_free_gib = disk_free_gib(cache_root) if cache_ready and cache_root else 0 + cpus = os.cpu_count() or 0 + memory_gib = mac_total_memory_gib() + return { + "platform": "macos", + "architecture": normalize_arch(platform.machine()), + "osVersionClass": f"macos-{major}", + "glaedaGeneration": digest_file(glaeda), + "toolchainGeneration": digest_bytes(canonical(toolchain)), + "checks": { + "supportedOs": major in {15, 26}, + "hardwareCapability": hardware_class_ready( + "macos", + hardware_class, + cpus, + memory_gib, + ), + "cmuxCheckout": (cmux_root / ".git").exists() + and (cmux_root / ".xcode-version").is_file(), + "canonicalCheckoutClean": cmux_checkout_clean(cmux_root), + "submodulesReady": cmux_submodules_ready(cmux_root), + "cmuxSetupArtifacts": ( + (cmux_root / "ghostty/include/ghostty.h").is_file() + and ( + cmux_root / "ghostty/macos/GhosttyKit.xcframework" + ).is_dir() + ), + "xcodePin": bool( + xcode_match + and sdk_match + and pin == "26.0" + and xcode_match.group(1).startswith("26") + and int(sdk_match.group(1)) == 26 + ), + "git": git.startswith("git version "), + "zig": zig_version_compatible(zig, zig_required), + "rust": ( + rustup_version.startswith("rustup ") + and cargo.startswith("cargo ") + and rustc.startswith("rustc ") + and diff_cargo.startswith("cargo ") + and diff_rustc.startswith("rustc ") + ), + "metalToolchain": bool(metal), + "glaedaExecutable": glaeda.is_file() and os.access(glaeda, os.X_OK), + "diskAdmission": free_gib >= min_free_gib, + "nativeCacheRoot": (not cache_required) or cache_ready, + "nativeCacheDiskAdmission": ( + (not cache_required) + or (cache_ready and cache_free_gib >= min_free_gib) + ), + "unattendedPower": mac_sleep_disabled_on_ac(pmset), + }, + "observed": { + "freeDiskGiBClass": ( + f"ge-{min_free_gib}" if free_gib >= min_free_gib else f"lt-{min_free_gib}" + ), + "xcodePin": pin, + "nativeCacheFreeDiskGiBClass": ( + "unconfigured" + if not cache_required + else ( + f"ge-{min_free_gib}" + if cache_ready and cache_free_gib >= min_free_gib + else f"lt-{min_free_gib}" + ) + ), + "logicalCpuClass": "ge-8" if cpus >= 8 else "lt-8", + "totalMemoryGiBClass": ( + "ge-16" if memory_gib >= 16 else "lt-16" + ), + }, + } + + +def read_os_release() -> tuple[str, str]: + values: dict[str, str] = {} + for line in Path("/etc/os-release").read_text(encoding="utf-8").splitlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + values[key] = value.strip().strip('"') + return values.get("ID", ""), values.get("VERSION_ID", "") + + +def linux_memory_gib(field: str) -> int: + for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): + if line.startswith(field + ":"): + return int(line.split()[1]) // (1024**2) + raise BootstrapError(f"{field} is unavailable") + + +def mac_total_memory_gib() -> int: + raw = run([executable("sysctl"), "-n", "hw.memsize"]) + return int(raw) // (1024**3) + + +def hardware_class_ready( + platform_name: str, + hardware_class: str, + cpus: int, + memory_gib: int, +) -> bool: + minimums = { + ("macos", "cmux-mac-build-large"): (8, 16), + ("macos", "cmux-mac-test-large"): (8, 16), + ("linux", "cmux-linux-ci-medium"): (4, 8), + ("linux", "cmux-linux-agent-medium"): (4, 8), + } + required = minimums.get((platform_name, hardware_class)) + return ( + required is not None + and cpus >= required[0] + and memory_gib >= required[1] + ) + + +def collect_linux( + cmux_root: Path, + glaeda: Path, + min_free_gib: int, + roles: list[str], + hardware_class: str, +) -> dict[str, Any]: + if platform.system() != "Linux": + raise BootstrapError("Linux bootstrap requires Linux") + distro, version = read_os_release() + git = run([executable("git"), "--version"]) + python = run([executable("python3"), "--version"]) + systemd = run([executable("systemctl"), "--version"]).splitlines()[0] + bwrap = run([executable("bwrap"), "--version"]) + kernel_major = int(platform.release().split(".", 1)[0]) + actions_ok = True + if "cmux_linux_ci" in roles: + for name in ("curl", "tar", "gzip", "ldd"): + actions_ok = actions_ok and shutil.which(name) is not None + toolchain = { + "distribution": f"{distro}-{version}", + "kernelMajor": kernel_major, + "gitVersion": git, + "pythonVersion": python, + } + supported_distro = (distro, version) in { + ("ubuntu", "24.04"), + ("debian", "12"), + } + cgroup2 = Path("/sys/fs/cgroup/cgroup.controllers").is_file() + pressure = all( + Path(f"/proc/pressure/{name}").is_file() + for name in ("cpu", "memory", "io") + ) + available_gib = linux_memory_gib("MemAvailable") + total_gib = linux_memory_gib("MemTotal") + cpus = os.cpu_count() or 0 + free_gib = disk_free_gib(cmux_root) + return { + "platform": "linux", + "architecture": normalize_arch(platform.machine()), + "osVersionClass": f"{distro}-{version}", + "glaedaGeneration": digest_file(glaeda), + "toolchainGeneration": digest_bytes(canonical(toolchain)), + "checks": { + "supportedOs": supported_distro and kernel_major >= 6, + "hardwareCapability": hardware_class_ready( + "linux", + hardware_class, + cpus, + total_gib, + ), + "cmuxCheckout": (cmux_root / ".git").exists(), + "canonicalCheckoutClean": cmux_checkout_clean(cmux_root), + "git": git.startswith("git version "), + "glaedaExecutable": glaeda.is_file() and os.access(glaeda, os.X_OK), + "systemd": systemd.startswith("systemd "), + "bubblewrap": bwrap.startswith("bubblewrap "), + "cgroupV2": cgroup2, + "pressureSignals": pressure, + "diskAdmission": free_gib >= min_free_gib, + "memoryAdmission": available_gib >= 8, + "actionsPrerequisites": actions_ok, + }, + "observed": { + "freeDiskGiBClass": ( + f"ge-{min_free_gib}" if free_gib >= min_free_gib else f"lt-{min_free_gib}" + ), + "availableMemoryGiBClass": ( + "ge-8" if available_gib >= 8 else "lt-8" + ), + "logicalCpuClass": "ge-4" if cpus >= 4 else "lt-4", + "totalMemoryGiBClass": ( + "ge-8" if total_gib >= 8 else "lt-8" + ), + }, + } + + +def evaluate( + observation: dict[str, Any], + roles: list[str], + hardware_class: str, +) -> dict[str, Any]: + platform_name = observation.get("platform") + if platform_name not in {"macos", "linux"}: + raise BootstrapError("bootstrap platform is invalid") + roles = sorted(set(roles)) + if not roles or any(role not in ROLE_OS for role in roles): + raise BootstrapError("bootstrap roles are invalid") + unreviewed = [role for role in roles if role not in ENROLLABLE_ROLES] + if unreviewed: + raise BootstrapError( + "bootstrap role lacks a reviewed v1 acceptance workload: " + + ",".join(unreviewed) + ) + for role in roles: + required = ROLE_OS[role] + if required is not None and required != platform_name: + raise BootstrapError(f"role {role} is incompatible with {platform_name}") + if re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,79}", hardware_class) is None: + raise BootstrapError("hardware capability class is invalid") + checks = observation.get("checks") + if ( + not isinstance(checks, dict) + or not checks + or any(type(value) is not bool for value in checks.values()) + ): + raise BootstrapError("bootstrap checks are invalid") + workload_generations = observation.get("roleWorkloadGenerations") + if ( + not isinstance(workload_generations, dict) + or set(workload_generations) != set(roles) + or any( + not isinstance(value, str) + or re.fullmatch(r"sha256:[0-9a-f]{64}", value) is None + for value in workload_generations.values() + ) + ): + raise BootstrapError("role workload generations are invalid") + failures = sorted(key for key, passed in checks.items() if not passed) + result = { + "schema": SCHEMA, + "platform": platform_name, + "architecture": observation["architecture"], + "osVersionClass": observation["osVersionClass"], + "hardwareCapabilityClass": hardware_class, + "roles": roles, + "glaedaGeneration": observation["glaedaGeneration"], + "toolchainGeneration": observation["toolchainGeneration"], + "roleWorkloadGenerations": workload_generations, + "checks": checks, + "observed": observation.get("observed", {}), + "eligibleForEnrollment": not failures, + "blockingChecks": failures, + "authority": "observation_only", + } + if len(canonical(result)) > MAX_OUTPUT_BYTES: + raise BootstrapError("bootstrap receipt exceeds size ceiling") + return result + + +def parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--platform", required=True, choices=("macos", "linux")) + p.add_argument("--cmux-root", type=Path, required=True) + p.add_argument("--glaeda", type=Path, required=True) + p.add_argument("--hardware-class", required=True) + p.add_argument("--role", action="append", required=True) + p.add_argument( + "--cache-root", + type=Path, + help="Existing operator-owned native build/cache root; path is never emitted", + ) + p.add_argument("--min-free-gib", type=int) + return p + + +def main() -> int: + try: + args = parser().parse_args() + cmux_root = args.cmux_root.resolve(strict=True) + glaeda = args.glaeda.resolve(strict=True) + cache_root = ( + args.cache_root.resolve(strict=True) + if args.cache_root is not None + else None + ) + if ( + args.platform == "macos" + and any( + role == "cmux_macos_native_build" + for role in args.role + ) + and cache_root is None + ): + raise BootstrapError( + "macOS native-build role requires --cache-root" + ) + minimum = args.min_free_gib or (120 if args.platform == "macos" else 40) + if minimum <= 0: + raise BootstrapError("minimum free disk must be positive") + observation = ( + collect_macos( + cmux_root, + glaeda, + minimum, + args.hardware_class, + cache_root, + ) + if args.platform == "macos" + else collect_linux( + cmux_root, + glaeda, + minimum, + args.role, + args.hardware_class, + ) + ) + observation["roleWorkloadGenerations"] = role_workload_generations( + cmux_root, + args.role, + ) + sys.stdout.buffer.write( + canonical(evaluate(observation, args.role, args.hardware_class)) + ) + return 0 + except ( + OSError, + ValueError, + BootstrapError, + subprocess.TimeoutExpired, + ) as error: + print( + json.dumps({"error": str(error)}, sort_keys=True, separators=(",", ":")), + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-cmux-execution-roles.py b/scripts/test-cmux-execution-roles.py new file mode 100644 index 000000000..636f6d1c7 --- /dev/null +++ b/scripts/test-cmux-execution-roles.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import copy +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = Path(__file__).with_name('cmux_execution_roles.py') +SPEC = importlib.util.spec_from_file_location('cmux_execution_roles', MODULE_PATH) +assert SPEC and SPEC.loader +m = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(m) + +GEN = 'sha256:' + 'a' * 64 +PREF = 'sha256:' + 'b' * 64 +TOOLCHAIN = 'sha256:' + 'c' * 64 +GLAEDA = 'sha256:' + 'd' * 64 +ALT_WORKLOAD = 'sha256:' + 'e' * 64 +ALT_TOOLCHAIN = 'sha256:' + 'f' * 64 +ALT_GLAEDA = 'sha256:' + '1' * 64 + + +def mac_node(state='eligible', generation=7, xcode='apple-xcode-26-sdk-26', memory='large', pressure=None): + return { + 'schema': m.NODE_SCHEMA, + 'nodeId': 'cmux-mac-001', + 'state': state, + 'platform': 'macos', + 'architecture': 'arm64', + 'osVersionClass': 'macos-26', + 'enrollmentGeneration': 3, + 'glaedaGeneration': GLAEDA, + 'cpuClass': 'large', + 'memoryClass': memory, + 'diskClass': 'large', + 'capabilityGeneration': generation, + 'toolchainProfiles': {xcode: TOOLCHAIN}, + 'roleWorkloadGenerations': { + 'benchmark': GEN, + 'cmux_macos_native_build': GEN, + 'cmux_macos_test': GEN, + 'diagnostic': GEN, + }, + 'capabilities': sorted({ + 'apple_silicon', + 'app_host_test', + 'benchmark', + 'diagnostic', + 'native_glaeda_build', + 'native_release_build', + 'native_xcode_build', + }), + 'pressure': pressure or { + 'cpu': 'normal', + 'memory': 'normal', + 'swap': 'none', + 'thermal': 'stable', + }, + } + + +def linux_node(state='eligible', generation=4, memory='large'): + return { + 'schema': m.NODE_SCHEMA, + 'nodeId': 'cmux-linux-001', + 'state': state, + 'platform': 'linux', + 'architecture': 'x86_64', + 'osVersionClass': 'ubuntu-24.04', + 'enrollmentGeneration': 2, + 'glaedaGeneration': GLAEDA, + 'cpuClass': 'large', + 'memoryClass': memory, + 'diskClass': 'large', + 'capabilityGeneration': generation, + 'toolchainProfiles': {'linux-rust-ci-2026-09': TOOLCHAIN}, + 'roleWorkloadGenerations': { + 'artifact_cache': GEN, + 'background_replay': GEN, + 'benchmark': GEN, + 'cmux_linux_agent': GEN, + 'cmux_linux_ci': GEN, + 'diagnostic': GEN, + }, + 'capabilities': sorted({ + 'artifact_service', + 'background_replay', + 'background_verification', + 'benchmark', + 'build_helper', + 'cgroup_v2', + 'diagnostic', + 'linux_agent', + 'linux_ci', + 'linux_ci_toolchain', + 'linux_kernel_6_plus', + 'systemd_execution', + 'task_isolation', + 'web_ci', + }), + 'pressure': { + 'cpu': 'normal', + 'memory': 'normal', + 'swap': 'none', + 'thermal': 'stable', + }, + } + + +def canary(node, role, *, result='accepted', toolchain=None, capabilities=None, profiles=None): + if capabilities is None: + capabilities = set(m.ROLE_REQUIREMENTS[role]['requiredCapabilities']) + if role == 'cmux_macos_native_build': + capabilities |= {'native_release_build'} + if role == 'cmux_macos_test': + capabilities |= {'app_host_test'} + if role == 'cmux_linux_ci': + capabilities |= {'linux_ci', 'web_ci', 'background_verification'} + if role == 'cmux_linux_agent': + capabilities |= {'linux_agent', 'build_helper'} + requires_toolchain = m.ROLE_REQUIREMENTS[role]['requiresToolchainProfile'] + default_toolchain = next(iter(node['toolchainProfiles'])) if requires_toolchain else None + profile = toolchain if toolchain is not None else default_toolchain + toolchain_generation = ( + node['toolchainProfiles'].get(profile) if profile is not None else None + ) + return { + 'schema': m.ROLE_CANARY_SCHEMA, + 'nodeId': node['nodeId'], + 'enrollmentGeneration': node['enrollmentGeneration'], + 'glaedaGeneration': node['glaedaGeneration'], + 'capabilityGeneration': node['capabilityGeneration'], + 'role': role, + 'toolchainProfile': profile, + 'toolchainGeneration': toolchain_generation, + 'acceptedCapabilities': sorted(capabilities), + 'acceptedResourceProfiles': profiles or ['large', 'medium', 'small'], + 'workloadGeneration': node['roleWorkloadGenerations'][role], + 'result': result, + } + + +def capacity(node, role, profile, slot, concurrent, *, result='accepted', pressure='normal', unfinished=0): + requires_toolchain = m.ROLE_REQUIREMENTS[role]['requiresToolchainProfile'] + toolchain_profile = next(iter(node['toolchainProfiles'])) if requires_toolchain else None + toolchain_generation = ( + node['toolchainProfiles'].get(toolchain_profile) + if toolchain_profile is not None + else None + ) + return { + 'schema': m.CAPACITY_SCHEMA, + 'nodeId': node['nodeId'], + 'enrollmentGeneration': node['enrollmentGeneration'], + 'glaedaGeneration': node['glaedaGeneration'], + 'capabilityGeneration': node['capabilityGeneration'], + 'role': role, + 'workloadGeneration': node['roleWorkloadGenerations'][role], + 'toolchainProfile': toolchain_profile, + 'toolchainGeneration': toolchain_generation, + 'resourceProfile': profile, + 'slotClass': slot, + 'maxConcurrent': concurrent, + 'contentionEvidenceGeneration': GEN, + 'measurement': { + 'validatedCompletions': 8, + 'p50Millis': 1000, + 'p90Millis': 1400, + 'cpuPressure': pressure, + 'memoryPressure': pressure, + 'swapClass': 'none', + 'thermalBehavior': 'stable', + 'unfinishedWork': unfinished, + }, + 'result': result, + } + + +def workload(operation, role, platform, architecture, capabilities, *, toolchain=None, memory='medium', profile='medium'): + return { + 'schema': m.WORKLOAD_SCHEMA, + 'operation': operation, + 'role': role, + 'platform': platform, + 'architecture': architecture, + 'toolchainProfile': toolchain, + 'minimumCpuClass': 'medium', + 'minimumMemoryClass': memory, + 'requiredCapabilities': sorted(capabilities), + 'resourceProfile': profile, + } + + +class RoleModelTests(unittest.TestCase): + def test_closed_role_vocabulary_keeps_workload_specialization_as_capabilities(self): + self.assertEqual( + set(m.ROLES), + { + 'artifact_cache', + 'background_replay', + 'benchmark', + 'cmux_linux_agent', + 'cmux_linux_ci', + 'cmux_macos_native_build', + 'cmux_macos_test', + 'diagnostic', + }, + ) + self.assertEqual(m.OPERATION_CONTRACTS['cmux_macos_release_admission']['role'], 'cmux_macos_native_build') + self.assertIn('native_release_build', m.OPERATION_CONTRACTS['cmux_macos_release_admission']['requiredCapabilities']) + self.assertEqual(m.OPERATION_CONTRACTS['cmux_web_ci_admission']['role'], 'cmux_linux_ci') + + def test_node_meets_exact_mac_role_and_workload_requirements(self): + node = mac_node() + canaries = [canary(node, 'cmux_macos_native_build')] + caps = [ + capacity(node, 'cmux_macos_native_build', 'medium', 'mac_native_build_lane', 1), + ] + req = workload( + 'cmux_macos_compile_admission', + 'cmux_macos_native_build', + 'macos', + 'arm64', + {'native_xcode_build'}, + toolchain='apple-xcode-26-sdk-26', + ) + self.assertEqual(m.select_eligible(req, node, canaries, caps), (True, 'eligible')) + decision = m.local_admission(req, node, canaries, caps, {}) + self.assertTrue(decision['accepted']) + self.assertEqual(decision['authority'], 'admission_only') + self.assertEqual(decision['slotClaims'], ['mac_native_build_lane']) + + def test_wrong_xcode_profile_refuses_cleanly(self): + node = mac_node() + canaries = [canary(node, 'cmux_macos_native_build')] + caps = [capacity(node, 'cmux_macos_native_build', 'medium', 'mac_native_build_lane', 1)] + req = workload( + 'cmux_macos_compile_admission', + 'cmux_macos_native_build', + 'macos', + 'arm64', + {'native_xcode_build'}, + toolchain='apple-xcode-27-sdk-27', + ) + self.assertEqual(m.select_eligible(req, node, canaries, caps), (False, 'toolchain_profile_missing')) + + def test_unsupported_os_version_class_refuses(self): + node = mac_node() + node['osVersionClass'] = 'macos-27' + canaries = [canary(node, 'cmux_macos_native_build')] + eligibility = m.role_eligibility(node, canaries) + self.assertEqual( + eligibility['cmux_macos_native_build']['reason'], + 'unsupported_os_version_class', + ) + + def test_linux_kernel_class_is_required(self): + node = linux_node() + node['capabilities'].remove('linux_kernel_6_plus') + canaries = [canary(node, 'cmux_linux_ci')] + eligibility = m.role_eligibility(node, canaries) + self.assertEqual( + eligibility['cmux_linux_ci']['reason'], + 'node_capability_missing', + ) + + def test_insufficient_memory_class_refuses(self): + node = mac_node(memory='small') + canaries = [canary(node, 'cmux_macos_native_build')] + eligibility = m.role_eligibility(node, canaries) + self.assertEqual(eligibility['cmux_macos_native_build']['reason'], 'insufficient_memory_class') + + def test_insufficient_cpu_class_refuses_linux_role(self): + node = linux_node() + node['cpuClass'] = 'small' + canaries = [canary(node, 'cmux_linux_ci')] + eligibility = m.role_eligibility(node, canaries) + self.assertEqual(eligibility['cmux_linux_ci']['reason'], 'insufficient_cpu_class') + + def test_reenrollment_invalidates_old_role_canary(self): + node = mac_node() + receipt = canary(node, 'cmux_macos_native_build') + node['enrollmentGeneration'] += 1 + eligibility = m.role_eligibility(node, [receipt]) + self.assertEqual( + eligibility['cmux_macos_native_build']['reason'], + 'role_canary_enrollment_stale', + ) + + def test_role_workload_generation_change_invalidates_canary(self): + node = mac_node() + receipt = canary(node, 'cmux_macos_native_build') + node['roleWorkloadGenerations']['cmux_macos_native_build'] = ALT_WORKLOAD + eligibility = m.role_eligibility(node, [receipt]) + self.assertEqual( + eligibility['cmux_macos_native_build']['reason'], + 'role_canary_workload_stale', + ) + + def test_glaeda_generation_change_invalidates_canary(self): + node = mac_node() + receipt = canary(node, 'cmux_macos_native_build') + node['glaedaGeneration'] = ALT_GLAEDA + eligibility = m.role_eligibility(node, [receipt]) + self.assertEqual( + eligibility['cmux_macos_native_build']['reason'], + 'role_canary_glaeda_stale', + ) + + def test_exact_toolchain_generation_change_invalidates_canary(self): + node = mac_node() + receipt = canary(node, 'cmux_macos_native_build') + node['toolchainProfiles']['apple-xcode-26-sdk-26'] = ALT_TOOLCHAIN + eligibility = m.role_eligibility(node, [receipt]) + self.assertEqual( + eligibility['cmux_macos_native_build']['reason'], + 'toolchain_canary_pending', + ) + + def test_failed_role_canary_refuses(self): + node = mac_node() + eligibility = m.role_eligibility(node, [canary(node, 'cmux_macos_native_build', result='rejected')]) + self.assertEqual(eligibility['cmux_macos_native_build']['reason'], 'role_canary_failed') + + def test_draining_node_refuses_every_role(self): + node = mac_node(state='draining') + eligibility = m.role_eligibility(node, [canary(node, 'cmux_macos_native_build')]) + self.assertEqual(eligibility['cmux_macos_native_build']['reason'], 'node_draining') + + def test_role_is_removed_after_upgrade_until_recanary(self): + old = mac_node(generation=7) + receipt = canary(old, 'cmux_macos_native_build') + upgraded = mac_node(generation=8, xcode='apple-xcode-27-sdk-27') + eligibility = m.role_eligibility(upgraded, [receipt]) + self.assertEqual(eligibility['cmux_macos_native_build']['reason'], 'role_canary_pending') + + def test_multiple_roles_share_one_scarce_slot(self): + node = mac_node() + canaries = [ + canary(node, 'cmux_macos_native_build'), + canary(node, 'cmux_macos_test'), + ] + caps = [ + capacity(node, 'cmux_macos_native_build', 'large', 'mac_native_build_lane', 1), + capacity(node, 'cmux_macos_test', 'large', 'mac_app_host_test_slot', 2), + capacity(node, 'cmux_macos_test', 'large', 'mac_native_build_lane', 2), + ] + compile_req = workload( + 'cmux_macos_compile_admission', 'cmux_macos_native_build', 'macos', 'arm64', + {'native_xcode_build'}, toolchain='apple-xcode-26-sdk-26', profile='large' + ) + test_req = workload( + 'cmux_macos_app_host_test', 'cmux_macos_test', 'macos', 'arm64', + {'app_host_test'}, toolchain='apple-xcode-26-sdk-26', profile='large' + ) + admitted = m.local_admission(compile_req, node, canaries, caps, {}) + self.assertTrue(admitted['accepted']) + self.assertEqual(admitted['authority'], 'admission_only') + self.assertEqual(admitted['leaseBoundary'], 'physical_execution_lease') + compile_refused = m.local_admission( + compile_req, node, canaries, caps, {'mac_native_build_lane': 1} + ) + self.assertFalse(compile_refused['accepted']) + refused = m.local_admission(test_req, node, canaries, caps, {'mac_native_build_lane': 2}) + self.assertFalse(refused['accepted']) + self.assertEqual(refused['reason'], 'physical_slot_unavailable') + self.assertIn('mac_native_build_lane', refused['slotClaims']) + + def test_external_scheduler_cannot_request_ineligible_role(self): + node = linux_node() + canaries = [canary(node, 'cmux_linux_ci')] + caps = [capacity(node, 'cmux_linux_ci', 'medium', 'linux_medium_slot', 4)] + req = workload( + 'cmux_linux_agent_admission', 'cmux_linux_agent', 'linux', 'x86_64', + {'linux_agent'}, toolchain='linux-rust-ci-2026-09' + ) + decision = m.local_admission(req, node, canaries, caps, {}) + self.assertFalse(decision['accepted']) + self.assertEqual(decision['reason'], 'role_canary_pending') + def test_node_pressure_between_selection_and_local_admission_vetoes(self): + selected = mac_node() + canaries = [canary(selected, 'cmux_macos_native_build')] + caps = [capacity(selected, 'cmux_macos_native_build', 'medium', 'mac_native_build_lane', 1)] + req = workload( + 'cmux_macos_compile_admission', 'cmux_macos_native_build', 'macos', 'arm64', + {'native_xcode_build'}, toolchain='apple-xcode-26-sdk-26' + ) + self.assertEqual(m.select_eligible(req, selected, canaries, caps), (True, 'eligible')) + pressured = copy.deepcopy(selected) + pressured['pressure']['memory'] = 'critical' + decision = m.local_admission(req, pressured, canaries, caps, {}) + self.assertFalse(decision['accepted']) + self.assertEqual(decision['reason'], 'node_pressured') + + def test_resource_profile_requires_reviewed_measurement(self): + node = linux_node() + canaries = [canary(node, 'cmux_linux_ci')] + req = workload( + 'cmux_linux_ci_admission', 'cmux_linux_ci', 'linux', 'x86_64', + {'linux_ci'}, toolchain='linux-rust-ci-2026-09', profile='large' + ) + caps = [capacity(node, 'cmux_linux_ci', 'medium', 'linux_medium_slot', 4)] + self.assertEqual(m.select_eligible(req, node, canaries, caps), (False, 'resource_profile_unmeasured')) + + def test_stale_capacity_evidence_refuses_after_workload_generation_change(self): + node = linux_node() + canaries = [canary(node, 'cmux_linux_ci')] + old_capacity = capacity( + node, 'cmux_linux_ci', 'medium', 'linux_medium_slot', 4 + ) + node['roleWorkloadGenerations']['cmux_linux_ci'] = ALT_WORKLOAD + canaries = [canary(node, 'cmux_linux_ci')] + req = workload( + 'cmux_linux_ci_admission', 'cmux_linux_ci', 'linux', 'x86_64', + {'linux_ci'}, toolchain='linux-rust-ci-2026-09' + ) + self.assertEqual( + m.select_eligible(req, node, canaries, [old_capacity]), + (False, 'resource_profile_unmeasured'), + ) + + def test_background_profile_still_requires_measurement(self): + node = linux_node() + canaries = [canary(node, 'background_replay', profiles=['small'])] + req = workload( + 'background_replay', 'background_replay', 'linux', 'x86_64', + {'background_replay'}, profile='small', memory='small' + ) + req['minimumCpuClass'] = 'small' + self.assertEqual( + m.select_eligible(req, node, canaries, []), + (False, 'resource_profile_unmeasured'), + ) + evidence = [capacity(node, 'background_replay', 'small', 'background_capacity', 4)] + self.assertEqual(m.select_eligible(req, node, canaries, evidence), (True, 'eligible')) + + def test_capacity_receipt_records_contention_window_fields_without_remote_raw_cpu_ram(self): + node = linux_node() + receipt = capacity(node, 'cmux_linux_ci', 'medium', 'linux_medium_slot', 4) + validated = m.validate_capacity(receipt) + measurement = validated['measurement'] + self.assertEqual(validated['contentionEvidenceGeneration'], GEN) + self.assertEqual( + set(measurement), + { + 'validatedCompletions', 'p50Millis', 'p90Millis', 'cpuPressure', + 'memoryPressure', 'swapClass', 'thermalBehavior', 'unfinishedWork' + }, + ) + self.assertNotIn('cpu', validated) + self.assertNotIn('ramBytes', validated) + self.assertNotIn('cgroup', validated) + + def test_one_mac_and_one_linux_can_each_hold_two_roles(self): + mac = mac_node() + mac_roles = m.role_eligibility(mac, [ + canary(mac, 'cmux_macos_native_build'), + canary(mac, 'cmux_macos_test'), + ]) + self.assertTrue(mac_roles['cmux_macos_native_build']['eligible']) + self.assertTrue(mac_roles['cmux_macos_test']['eligible']) + + linux = linux_node() + linux_roles = m.role_eligibility(linux, [ + canary(linux, 'cmux_linux_ci'), + canary(linux, 'cmux_linux_agent'), + ]) + self.assertTrue(linux_roles['cmux_linux_ci']['eligible']) + self.assertTrue(linux_roles['cmux_linux_agent']['eligible']) + + def test_preferred_is_advisory_and_separate_from_eligibility(self): + node = mac_node() + canaries = [canary(node, 'cmux_macos_test')] + preference = { + 'schema': m.PREFERENCE_SCHEMA, + 'nodeId': node['nodeId'], + 'role': 'cmux_macos_native_build', + 'evidenceGeneration': PREF, + 'preference': 'preferred', + 'authority': 'observation_only', + } + status = m.operator_status(node, canaries, [], [preference]) + self.assertIn('cmux_macos_native_build', status['preferred']) + self.assertNotIn('cmux_macos_native_build', status['eligible']) + + def test_compact_operator_status_hides_internal_objects(self): + node = mac_node(state='active') + canaries = [ + canary(node, 'cmux_macos_native_build'), + canary(node, 'cmux_macos_test'), + ] + caps = [ + capacity(node, 'cmux_macos_native_build', 'medium', 'mac_native_build_lane', 1), + capacity(node, 'cmux_macos_test', 'medium', 'mac_app_host_test_slot', 2), + ] + status = m.operator_status(node, canaries, caps) + human = m.render_status(status) + self.assertIn('node cmux-mac-001', human) + self.assertIn('cmux_macos_native_build', human) + self.assertIn('mac_native_build_lane: 1', human) + self.assertIn('state: active', human) + self.assertNotIn('capabilityGeneration', human) + self.assertNotIn('workloadGeneration', human) + + def test_workload_has_no_machine_selector_or_raw_resource_controls(self): + req = workload( + 'cmux_macos_compile_admission', 'cmux_macos_native_build', 'macos', 'arm64', + {'native_xcode_build'}, toolchain='apple-xcode-26-sdk-26' + ) + m.validate_workload(req) + self.assertNotIn('nodeId', req) + self.assertNotIn('cpu', req) + self.assertNotIn('ramBytes', req) + self.assertNotIn('cgroup', req) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/scripts/test-cmux-fleet-bootstrap.py b/scripts/test-cmux-fleet-bootstrap.py new file mode 100644 index 000000000..58afdc8b7 --- /dev/null +++ b/scripts/test-cmux-fleet-bootstrap.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +import importlib.util +from pathlib import Path +import unittest + +MODULE_PATH = Path(__file__).with_name("cmux_fleet_bootstrap.py") +SPEC = importlib.util.spec_from_file_location("cmux_fleet_bootstrap", MODULE_PATH) +assert SPEC and SPEC.loader +b = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(b) + +A = "sha256:" + "a" * 64 +B = "sha256:" + "b" * 64 +W = "sha256:" + "e" * 64 + + +def observation(platform="macos", failed=()): + checks = { + "supportedOs": True, + "hardwareCapability": True, + "git": True, + "glaedaExecutable": True, + "diskAdmission": True, + } + if platform == "macos": + checks.update( + {"cmuxCheckout": True, "xcodePin": True, "unattendedPower": True} + ) + else: + checks.update( + { + "cmuxCheckout": True, + "systemd": True, + "bubblewrap": True, + "cgroupV2": True, + "pressureSignals": True, + "memoryAdmission": True, + "actionsPrerequisites": True, + } + ) + for key in failed: + checks[key] = False + return { + "platform": platform, + "architecture": "arm64" if platform == "macos" else "x86_64", + "osVersionClass": "macos-26" if platform == "macos" else "ubuntu-24.04", + "glaedaGeneration": A, + "toolchainGeneration": B, + "roleWorkloadGenerations": { + "cmux_macos_native_build" if platform == "macos" else "cmux_linux_ci": W + }, + "checks": checks, + "observed": {}, + } + + +class Tests(unittest.TestCase): + def test_mac_ready(self): + result = b.evaluate( + observation(), + ["cmux_macos_native_build"], + "cmux-mac-build-large", + ) + self.assertTrue(result["eligibleForEnrollment"]) + self.assertEqual(result["authority"], "observation_only") + self.assertEqual( + result["roleWorkloadGenerations"], + {"cmux_macos_native_build": W}, + ) + + def test_linux_ready(self): + result = b.evaluate( + observation("linux"), + ["cmux_linux_ci"], + "cmux-linux-ci-medium", + ) + self.assertTrue(result["eligibleForEnrollment"]) + + def test_failures_block_enrollment(self): + result = b.evaluate( + observation(failed=("xcodePin", "diskAdmission")), + ["cmux_macos_native_build"], + "cmux-mac-build-large", + ) + self.assertFalse(result["eligibleForEnrollment"]) + self.assertEqual(result["blockingChecks"], ["diskAdmission", "xcodePin"]) + + def test_wrong_os_role_refused(self): + with self.assertRaisesRegex(b.BootstrapError, "incompatible"): + b.evaluate( + observation("linux"), + ["cmux_macos_native_build"], + "cmux-linux-ci-medium", + ) + + def test_reviewed_hardware_classes_have_minimums(self): + self.assertTrue( + b.hardware_class_ready( + "macos", + "cmux-mac-build-large", + 8, + 16, + ) + ) + self.assertFalse( + b.hardware_class_ready( + "macos", + "cmux-mac-build-large", + 4, + 16, + ) + ) + self.assertTrue( + b.hardware_class_ready( + "linux", + "cmux-linux-ci-medium", + 4, + 8, + ) + ) + self.assertFalse( + b.hardware_class_ready( + "linux", + "unknown", + 64, + 256, + ) + ) + + def test_role_without_workload_refused(self): + with self.assertRaisesRegex(b.BootstrapError, "reviewed v1 acceptance workload"): + b.evaluate( + observation(), + ["artifact_cache"], + "cmux-mac-build-large", + ) + + def test_zig_version_compatibility_matches_cmux_policy(self): + self.assertTrue(b.zig_version_compatible("0.16.0", "0.16.0")) + self.assertTrue(b.zig_version_compatible("0.16.4", "0.16.0")) + self.assertFalse(b.zig_version_compatible("0.15.9", "0.16.0")) + self.assertFalse(b.zig_version_compatible("0.17.0", "0.16.0")) + self.assertFalse(b.zig_version_compatible("nightly", "0.16.0")) + + def test_workload_generation_must_cover_exact_roles(self): + observed = observation() + observed["roleWorkloadGenerations"] = {} + with self.assertRaisesRegex(b.BootstrapError, "workload generations"): + b.evaluate( + observed, + ["cmux_macos_native_build"], + "cmux-mac-build-large", + ) + + def test_power_posture_parser(self): + raw = ( + "Battery Power:\n" + " sleep 1\n" + "AC Power:\n" + " sleep 0\n" + " displaysleep 10\n" + ) + self.assertTrue(b.mac_sleep_disabled_on_ac(raw)) + self.assertFalse( + b.mac_sleep_disabled_on_ac( + raw.replace("sleep 0", "sleep 1") + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test-cmux-fleet.py b/scripts/test-cmux-fleet.py new file mode 100644 index 000000000..d185d9441 --- /dev/null +++ b/scripts/test-cmux-fleet.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import copy +import json +import importlib.util +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] + +MODULE_PATH = Path(__file__).with_name("cmux_fleet.py") +SPEC = importlib.util.spec_from_file_location("cmux_fleet", MODULE_PATH) +assert SPEC and SPEC.loader +f = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(f) + +A = "sha256:" + "a" * 64 +B = "sha256:" + "b" * 64 +C = "sha256:" + "c" * 64 +D = "sha256:" + "d" * 64 +COMMIT = "1" * 40 + + +def enrollment(os_family="macos", state="eligible", roles=None): + if roles is None: + roles = ( + ["cmux_macos_native_build"] + if os_family == "macos" + else ["cmux_linux_ci"] + ) + return { + "schema": f.ENROLLMENT_SCHEMA, + "nodeId": "cmux-fixture-001", + "architecture": "arm64" if os_family == "macos" else "x86_64", + "os": { + "family": os_family, + "versionClass": "macos-26" if os_family == "macos" else "ubuntu-24.04", + }, + "hardwareCapabilityClass": ( + "cmux-mac-build-large" if os_family == "macos" else "cmux-linux-ci-medium" + ), + "supportedToolchainGenerations": [A, B], + "roleWorkloadGenerations": {role: B for role in sorted(roles)}, + "allowedExecutionRoles": sorted(roles), + "operatorFleetScope": "cmux-founders", + "enrollmentGeneration": 3, + "glaedaGeneration": C, + "state": state, + "quarantineReason": None, + } + + +def evidence(role="cmux_macos_native_build", toolchain=A, checks=None): + return { + "schema": f.ACCEPTANCE_EVIDENCE_SCHEMA, + "nodeId": "cmux-fixture-001", + "enrollmentGeneration": 3, + "role": role, + "source": {"repository": "manaflow-ai/cmux", "commit": COMMIT}, + "toolchainGeneration": toolchain, + "glaedaGeneration": C, + "workloadGeneration": B, + "checks": checks or { + "workload": "pass", + "semanticVerifier": "pass", + "artifact": "pass", + "processSettlement": "pass", + }, + } + + +class FleetTests(unittest.TestCase): + def test_current_accepted_role_is_eligible(self): + e = enrollment() + r = f.finalize_acceptance(e, evidence()) + status = f.node_status(e, [r]) + by_role = {v["role"]: v for v in status["roles"]} + self.assertTrue(by_role["cmux_macos_native_build"]["eligible"]) + self.assertEqual(set(by_role), {"cmux_macos_native_build"}) + self.assertTrue(status["routingCandidateEligible"]) + self.assertFalse(status["automaticDispatchAuthorized"]) + + def test_enrollment_presence_does_not_grant_role(self): + status = f.node_status(enrollment(), []) + self.assertFalse(status["routingCandidateEligible"]) + self.assertTrue(all(not r["eligible"] for r in status["roles"])) + + def test_non_routable_states_exclude_every_role(self): + for state in ("draining", "quarantined", "retired"): + with self.subTest(state=state): + e = enrollment(state=state) + if state == "quarantined": + e["quarantineReason"] = "disk_pressure" + receipt = f.finalize_acceptance( + {**e, "state": "eligible", "quarantineReason": None}, + evidence(), + ) + status = f.node_status(e, [receipt]) + self.assertFalse(status["routingCandidateEligible"]) + self.assertTrue( + all(r["reason"] == f"node_{state}" for r in status["roles"]) + ) + + def test_null_glaeda_generation_is_rejected(self): + e = enrollment() + e["glaedaGeneration"] = None + with self.assertRaisesRegex(f.FleetError, "Glaeda generation is invalid"): + f.validate_enrollment(e) + + bad_evidence = evidence() + bad_evidence["glaedaGeneration"] = None + with self.assertRaisesRegex(f.FleetError, "acceptance Glaeda generation is invalid"): + f.validate_acceptance_evidence(bad_evidence) + + receipt = f.finalize_acceptance(enrollment(), evidence()) + receipt["glaedaGeneration"] = None + with self.assertRaisesRegex( + f.FleetError, "acceptance receipt Glaeda generation is invalid" + ): + f.validate_acceptance_receipt(receipt) + + def test_stale_acceptance_generation_is_ineligible(self): + e = enrollment() + r = f.finalize_acceptance(e, evidence()) + r["enrollmentGeneration"] = 2 + status = f.node_status(e, [r]) + native = next( + v for v in status["roles"] if v["role"] == "cmux_macos_native_build" + ) + self.assertEqual(native["reason"], "acceptance_enrollment_stale") + + def test_stale_acceptance_workload_generation_is_ineligible(self): + e = enrollment() + r = f.finalize_acceptance(e, evidence()) + e["roleWorkloadGenerations"]["cmux_macos_native_build"] = D + status = f.node_status(e, [r]) + native = next( + v for v in status["roles"] if v["role"] == "cmux_macos_native_build" + ) + self.assertFalse(native["eligible"]) + self.assertEqual(native["reason"], "acceptance_workload_stale") + + def test_private_identity_fields_are_rejected(self): + for field in ("hostname", "serialNumber", "privateIp", "username"): + with self.subTest(field=field): + e = enrollment() + e[field] = "secret-ish" + with self.assertRaisesRegex(f.FleetError, "unknown or missing"): + f.validate_enrollment(e) + + def test_role_without_reviewed_workload_is_rejected(self): + with self.assertRaisesRegex(f.FleetError, "reviewed v1 acceptance workload"): + f.validate_enrollment( + enrollment(roles=["artifact_cache"]) + ) + + def test_os_role_mismatch_is_rejected(self): + with self.assertRaisesRegex(f.FleetError, "incompatible"): + f.validate_enrollment( + enrollment("linux", roles=["cmux_macos_native_build"]) + ) + + def test_eligible_transition_requires_current_acceptance(self): + e = enrollment(state="enrolling") + with self.assertRaisesRegex(f.FleetError, "accepted role receipt"): + f.transition(e, "eligible", None) + receipt = f.finalize_acceptance(e, evidence()) + eligible = f.transition(e, "eligible", None, [receipt]) + self.assertEqual(eligible["state"], "eligible") + + def test_quarantine_and_reenroll_path(self): + e = enrollment() + q = f.transition(e, "quarantined", "service_mismatch") + self.assertEqual(q["quarantineReason"], "service_mismatch") + enrolling = f.transition(q, "enrolling", None) + self.assertIsNone(enrolling["quarantineReason"]) + self.assertEqual(enrolling["state"], "enrolling") + self.assertEqual( + enrolling["enrollmentGeneration"], + e["enrollmentGeneration"] + 1, + ) + + def test_acceptance_binds_current_identity(self): + e = enrollment() + bad = evidence(toolchain="sha256:" + "d" * 64) + with self.assertRaisesRegex(f.FleetError, "toolchain generation"): + f.finalize_acceptance(e, bad) + bad = evidence() + bad["glaedaGeneration"] = "sha256:" + "d" * 64 + with self.assertRaisesRegex(f.FleetError, "Glaeda generation"): + f.finalize_acceptance(e, bad) + bad = evidence() + bad["enrollmentGeneration"] = 4 + with self.assertRaisesRegex(f.FleetError, "enrollment generation"): + f.finalize_acceptance(e, bad) + bad = evidence() + bad["workloadGeneration"] = D + with self.assertRaisesRegex(f.FleetError, "workload generation"): + f.finalize_acceptance(e, bad) + + def test_failed_settlement_rejects_role(self): + checks = { + "workload": "pass", + "semanticVerifier": "pass", + "artifact": "pass", + "processSettlement": "fail", + } + e = enrollment() + r = f.finalize_acceptance(e, evidence(checks=checks)) + self.assertEqual(r["result"], "rejected") + status = f.node_status(e, [r]) + native = next( + v for v in status["roles"] if v["role"] == "cmux_macos_native_build" + ) + self.assertFalse(native["eligible"]) + + def test_linux_fixture_can_be_eligible(self): + e = enrollment("linux") + r = f.finalize_acceptance(e, evidence(role="cmux_linux_ci")) + status = f.node_status(e, [r]) + linux = next(v for v in status["roles"] if v["role"] == "cmux_linux_ci") + self.assertTrue(linux["eligible"]) + + def test_status_rejects_forged_acceptance_receipt(self): + e = enrollment() + forged = f.finalize_acceptance(e, evidence()) + forged["checks"]["processSettlement"] = "fail" + with self.assertRaisesRegex(f.FleetError, "disagrees"): + f.node_status(e, [forged]) + + def test_status_rejects_unknown_acceptance_fields(self): + e = enrollment() + receipt = f.finalize_acceptance(e, evidence()) + receipt["hostname"] = "hidden-host" + with self.assertRaisesRegex(f.FleetError, "unknown or missing"): + f.node_status(e, [receipt]) + + def test_enrollment_is_derived_from_accepted_bootstrap(self): + bootstrap = { + "schema": f.BOOTSTRAP_SCHEMA, + "platform": "macos", + "architecture": "arm64", + "osVersionClass": "macos-26", + "hardwareCapabilityClass": "cmux-mac-build-large", + "roles": ["cmux_macos_native_build"], + "glaedaGeneration": C, + "toolchainGeneration": A, + "roleWorkloadGenerations": {"cmux_macos_native_build": B}, + "checks": {"ready": True}, + "observed": {}, + "eligibleForEnrollment": True, + "blockingChecks": [], + "authority": "observation_only", + } + result = f.enrollment_from_bootstrap( + bootstrap, + node_id="cmux-fixture-002", + operator_fleet_scope="cmux-founders", + enrollment_generation=1, + ) + self.assertEqual(result["state"], "enrolling") + self.assertEqual(result["supportedToolchainGenerations"], [A]) + self.assertEqual( + result["roleWorkloadGenerations"], + {"cmux_macos_native_build": B}, + ) + + def test_blocked_bootstrap_cannot_enroll(self): + bootstrap = { + "schema": f.BOOTSTRAP_SCHEMA, + "authority": "observation_only", + "eligibleForEnrollment": False, + } + with self.assertRaisesRegex(f.FleetError, "blocking checks"): + f.enrollment_from_bootstrap( + bootstrap, + node_id="cmux-fixture-002", + operator_fleet_scope="cmux-founders", + enrollment_generation=1, + ) + + def test_versioned_schema_and_examples_parse(self): + schema = json.loads( + (ROOT / "examples/cmux-fleet/enrollment-v1.schema.json").read_text( + encoding="utf-8" + ) + ) + self.assertFalse(schema["additionalProperties"]) + for name in ( + "macos-enrollment.example.json", + "linux-enrollment.example.json", + ): + document = json.loads( + (ROOT / "examples/cmux-fleet" / name).read_text( + encoding="utf-8" + ) + ) + self.assertEqual(f.validate_enrollment(document), document) + + def test_fingerprint_is_canonical(self): + e = enrollment() + self.assertEqual(f.digest(copy.deepcopy(e)), f.digest(e)) + self.assertLess(len(f.canonical(f.node_status(e, []))), f.MAX_STATUS_BYTES) + + +if __name__ == "__main__": + unittest.main()