diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index d2a704e6871..d5b2eed9007 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -11,7 +11,7 @@ on: options: [srv1, srv2, srv3, srv4] type: choice mode: - description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown + description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown required: true options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot, mtcollins1_fan_observe] type: choice @@ -625,6 +625,8 @@ jobs: ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_microvm_host_ready.dag --function runner_microvm_host_converge_wet cat "$ROOT/target/runner-microvm-host-standing.txt" + env: + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} if: github.event.inputs.mode == 'microvm_host_converge' timeout-minutes: 30 - name: Upload runner micro-VM host standing receipt @@ -643,6 +645,8 @@ jobs: ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_microvm_boot_probe.dag --function runner_microvm_boot_probe_wet cat "$ROOT/target/runner-microvm-boot-probe.txt" + env: + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} if: github.event.inputs.mode == 'microvm_boot_probe' timeout-minutes: 30 - name: Upload runner micro-VM boot probe console receipt diff --git a/dag/extdeps/access/posix.dag b/dag/extdeps/access/posix.dag index d97a82e8004..1cca0a3dc9c 100644 --- a/dag/extdeps/access/posix.dag +++ b/dag/extdeps/access/posix.dag @@ -3,6 +3,7 @@ module extdeps.access.posix import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import std.types { Bool, Int, NonEmptyStr, String } +import std.algebra { trim } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { scheme: Https @@ -129,3 +130,71 @@ fn file_mode_octal(mode: FileMode) -> String { to_string(permission_bits_octal_digit(bits: mode.other)), ], "") } + +// THE SAME RELATION READ BACKWARD, which is why it lives here and not at the call site that wanted +// it. file_mode_octal above renders a FileMode as chmod's numeric spelling; this reads that +// spelling back. One grammar, two directions (DESIGN section 4) -- a parser authored beside a +// consumer would be a second, informal authority for the digit layout, and the two would drift the +// first time a special bit mattered. +// +// THREE DIGITS AND FOUR ARE THE SAME NUMBER. stat's %a prints the special digit only when it is +// non-zero, so "660" and "0660" are one mode spelled two ways; reading the text as an integer and +// slicing digits positionally from the LEAST significant end handles both without a length branch +// and without the leading-zero question ever arising. +// +// EVERY REFUSAL ARM IS A REFUSAL AND NOT A ZERO. An unparseable field, a negative number, a digit +// above 7 or a value wider than four digits means the observation did not happen; folding any of +// them into a FileMode would hand a consumer a permission set nothing observed, and the consumer +// here is deciding whether a device is delegated to a group. +fn permission_bits_of_octal_digit(digit: Int) -> PermissionBits? { + if digit < 0 || digit > 7 { + none + } else { + Present { + value: PermissionBits { + read: (digit / 4) % 2 == 1, + write: (digit / 2) % 2 == 1, + execute: digit % 2 == 1, + }, + } + } +} + +fn file_mode_of_octal_text(text: String) -> FileMode? { + match parse_int(s: trim(s: text)) { + Absent => none + Present { value: n } => + if n < 0 || n > 7777 { + none + } else { + match permission_bits_of_octal_digit(digit: (n / 100) % 10) { + Absent => none + Present { value: owner } => + match permission_bits_of_octal_digit(digit: (n / 10) % 10) { + Absent => none + Present { value: group } => + match permission_bits_of_octal_digit(digit: n % 10) { + Absent => none + Present { value: other } => { + let special = (n / 1000) % 10 + if special < 0 || special > 7 { + none + } else { + Present { + value: FileMode { + owner: owner, + group: group, + other: other, + setuid: (special / 4) % 2 == 1, + setgid: (special / 2) % 2 == 1, + sticky: special % 2 == 1, + }, + } + } + } + } + } + } + } + } +} diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index b3601da4bde..70e0311cb72 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -143,6 +143,7 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.posix.test_utility", decl_name: "test_exists_command"), decl_ref(module_path: "extdeps.posix.test_utility", decl_name: "test_writable_command"), decl_ref(module_path: "extdeps.posix.test_utility", decl_name: "test_executable_command"), + decl_ref(module_path: "extdeps.tools.id", decl_name: "id_group_names_command"), decl_ref(module_path: "extdeps.virtualization.firecracker", decl_name: "firecracker_version_command"), decl_ref(module_path: "extdeps.virtualization.firecracker", decl_name: "firecracker_run_config_command"), decl_ref(module_path: "extdeps.virtualization.firecracker", decl_name: "firecracker_jailer_run_command"), diff --git a/dag/extdeps/tools/coreutils_stat.dag b/dag/extdeps/tools/coreutils_stat.dag index b1e1526efcf..31b822a35de 100644 --- a/dag/extdeps/tools/coreutils_stat.dag +++ b/dag/extdeps/tools/coreutils_stat.dag @@ -2,7 +2,7 @@ module extdeps.tools.coreutils_stat import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } -import extdeps.access.posix { FileOwnership } +import extdeps.access.posix { FileOwnership, FileMode, file_mode_of_octal_text } import std.types { Bool, Int, List, NonEmptyStr, Unit } import std.string_type { String } import std.algebra { trim } @@ -29,6 +29,17 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { // four fields can contain one: uid and gid are decimal, and a POSIX login name is restricted to the // portable filename character set, which excludes the colon. That is a property of the fields, not // an assumption about the data, which is why the separator is not a configurable parameter. +// +// THE MODE IS ITS OWN PROBE, NOT A FIFTH FIELD ON THE OWNERSHIP ONE. The colon separator above is +// safe because none of those four fields can contain one, which is a property of the fields; the +// mode is a different question with a different failure -- a path can be readable for ownership and +// the answer still be that its permission bits are what nobody expected -- and joining them would +// make one unreadable half destroy the other. It is also the shape this module's sibling +// extdeps.tools.stat already commits to: one token per probe. +// +// %a IS THE ACCESS BITS IN OCTAL WITH NO INTERPRETATION. extdeps.access.posix owns what those digits +// MEAN and reads them back through file_mode_of_octal_text; PathMode's whole job is to fetch the +// spelling. service coreutils.Stat { operation PathOwnership { input { path: NonEmptyStr } @@ -43,6 +54,20 @@ service coreutils.Stat { nonzero => String "stat could not read the path" } } + + operation PathMode { + input { path: NonEmptyStr } + output { + mode_text: String from "stdout" + success: Bool from "exit_success" + } + readonly + transport shell { argv: ["stat", "-c", "%a", "--", "{path}"] } + exit { + 0 => Unit + nonzero => String "stat could not read the path" + } + } } // AN UNREADABLE PATH IS A REFUSAL, NOT AN OWNERSHIP OF NOBODY, for the same reason an unreadable @@ -133,3 +158,24 @@ fn coreutils_stat_ownership_record( } } } + +// THE MODE OBSERVATION, WITH THE SAME REFUSAL DISCIPLINE AS THE OWNERSHIP ONE ABOVE. A path whose +// permission bits could not be read must not arrive at a consumer as a mode with everything false, +// because "nobody may" and "we did not look" are the two answers a delegation check must keep +// apart -- the first says the host is misconfigured, the second says the probe is. +type PathModeObservation + = PathModeObserved { path: NonEmptyStr, mode: FileMode } + | PathModeUnobservable { path: NonEmptyStr, reason: String } + +fn coreutils_stat_path_mode(path: NonEmptyStr) -> PathModeObservation { + let observed = coreutils.Stat.PathMode(path: path) + if observed.success == false { + PathModeUnobservable { path: path, reason: "stat could not read the path" } + } else { + match file_mode_of_octal_text(text: observed.mode_text) { + Absent => + PathModeUnobservable { path: path, reason: join(["stat printed a mode this cannot read as octal permission bits: '", trim(s: observed.mode_text), "'"], "") } + Present { value: mode } => PathModeObserved { path: path, mode: mode } + } + } +} diff --git a/dag/extdeps/tools/id.dag b/dag/extdeps/tools/id.dag index 6809637ec00..34889778e5c 100644 --- a/dag/extdeps/tools/id.dag +++ b/dag/extdeps/tools/id.dag @@ -4,6 +4,7 @@ import std.types { NonEmptyStr, List, Bool, Unit } import std.string_type { String } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } +import extdeps.exec.command { ArgvCommand, argv_command } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -18,12 +19,29 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { // the grant authorizes -- and gunbc.host_effect_realize needs both spellings for the same tool. data id_binary_path: NonEmptyStr = "/usr/bin/id" +// THE PATH-RESOLVED SPELLING, DECLARED ONCE. The argv rows below and the command builder all name +// the same program, and spelling it three times would be three places one word could drift. +data id_program: NonEmptyStr = "id" + fn id_uid_argv() -> List { - ["id", "-u"] + [id_program as String, "-u"] } fn id_gid_argv() -> List { - ["id", "-g"] + [id_program as String, "-g"] +} + +// -nG NAMES THE USER EXPLICITLY, AND THAT IS THE WHOLE REASON THIS BUILDER TAKES ONE. id(1) with an +// operand answers from the account database; id(1) with no operand answers from the CALLING +// PROCESS's own credentials, which were established at its exec and never change afterwards. A +// caller asking whether a supplementary group grant has landed must ask the database -- the running +// process cannot have picked the group up, so the no-operand form would answer the older question +// and read as a refusal of a grant that is in fact installed. +// +// -n asks for names rather than numeric gids, because the group a caller holds is named by the +// authority it came from (a udev rule naming "kvm"), not by a gid that varies per host. +fn id_group_names_command(user: NonEmptyStr) -> ArgvCommand { + argv_command(program: id_program, arguments: ["-nG", "--", user as String]) } service os.Id { diff --git a/dag/gunbc/fleet/fleet_converge_plan.dag b/dag/gunbc/fleet/fleet_converge_plan.dag index b4c1b4ced69..c260cdfc537 100644 --- a/dag/gunbc/fleet/fleet_converge_plan.dag +++ b/dag/gunbc/fleet/fleet_converge_plan.dag @@ -79,6 +79,7 @@ import gunbc.fabric_cell_effect { import gunbc.runner_host_grants { fabric_cell_effect_operations, fabric_allocation_store_effect_operations, + runner_host_wide_refused_operations, } import gunbc.fabric_allocation_store_substrate { fabric_allocation_store_substrate_spec, @@ -89,7 +90,7 @@ import gunbc.fabric_allocation_store_substrate { AllocationStoreDirectoryPresentUnexpected, allocation_store_substrate_prestate_wire, } -import gunbc.executor_privileged_operation { executor_privileged_operation_shell_command } +import gunbc.executor_privileged_operation { executor_privileged_operation_shell_command, executor_privileged_operation_converge_command } import gunbc.fabric_cell_observation_admission { FabricCellObservationDecision, FabricCellObservationOutcome, @@ -186,7 +187,10 @@ import product.placement_supply { HostIdentity } import gunbc.runner_host_deploy { RunnerHostDeployFound, RunnerHostDeployUnmodeled, + RunnerHostSpecFound, + RunnerHostSpecAbsent, runner_host_deploy_for, + runner_host_spec_for, admit_runner_host, } import gunbc.runner_service_activation { @@ -1552,6 +1556,20 @@ fn activation_apply_lines( ) } +// THE HEADER AN OPERATOR READS BEFORE APPROVING A HOST MUTATION, and it said something the fold +// below does not do. It read "ADD only", immediately above a family that can also emit +// `systemctl mask --now` for an authorized surplus unit and remove a retired tree. The inner +// headers in slot_apply_lines have always named both of those honestly; the OUTER one -- the first +// line a reader meets, and the one that frames everything under it -- claimed a narrower blast +// radius than the script carries. A reader who trusts it stops reading at the right moment to miss +// the only two lines that stop anything. +// +// It names all three dispositions now, and each is bounded by what the fold actually admits: no +// desired runner is restarted or reinstalled, inhibition is `mask --now` on units the retirement +// admission authorized as surplus, and a tree is removed only for a unit whose RetirementComplete +// was independently re-observed. +data runner_slot_apply_family_header: String = "# runner-slot membership (gunbc.runner_slot_provision): ADDITIONS, plus retirement inhibition (systemctl mask --now) for units the retirement admission authorized as surplus, plus tree removal ONLY for a unit with an independently reobserved RetirementComplete. No desired runner is stopped, restarted or reinstalled; see slot_apply_lines for the per-disposition lines" + fn slot_apply_lines( host: NonEmptyStr, observed_slots: List, @@ -2490,7 +2508,7 @@ fn fleet_converge_apply_shell( fn fleet_converge_apply_shell_axis_note(request: FleetConvergeRequest) -> String { match request { FullHostConverge { host: _, observed_timers: _, observed_caps: _, observed_slots: _, observed_fabric_cells: _, observed_activation_readiness: _, observed_unit_standings: _, observed_retirement: _ } => - "this script runs timer teardown, runner-slot ADD, and fabric-cell ensure effects" + "this script runs timer teardown, the host-wide executor grants, runner-slot reconciliation (additions, authorized retirement inhibition via systemctl mask --now, and completed-retirement tree removal), and fabric-cell ensure effects" LaunchEnvironmentConverge { host: _, observed_timers: _ } => "scope:launch-environment — this script runs legacy fleet-converge timer teardown and nothing else; runner-slot, runner-service activation, the cap axis and fabric cells are not in this scope and have no lines here" FabricExecutionCellsConverge { host: _, observed_fabric_cells: _ } => @@ -2520,7 +2538,9 @@ fn fleet_converge_apply_shell_axis_lines(request: FleetConvergeRequest) -> List< [concat("sudo -n ", concat(ci_runner_sudo_binary_path(b: SudoSystemctl) as String, " daemon-reload"))], ), concat( - ["# runner-slot membership (gunbc.runner_slot_provision) — ADD only; see slot_apply_lines"], + host_wide_grant_apply_lines(host: host), + concat( + [runner_slot_apply_family_header], concat( slot_apply_lines(host: host, observed_slots: observed_slots, retirement_evidence: retirement_evidence), concat( @@ -2532,6 +2552,7 @@ fn fleet_converge_apply_shell_axis_lines(request: FleetConvergeRequest) -> List< ), ), ), + ), ) LaunchEnvironmentConverge { host: _, observed_timers: observed_timers } => concat( @@ -2588,6 +2609,50 @@ fn fabric_cell_apply_lines( } } +// THE HOST-WIDE EXECUTOR GRANTS, APPLIED WHERE EVERY OTHER HOST MUTATION IS APPLIED. These are the +// operations the executor owes the machine itself rather than any runner population: its converge +// state directory, the runner tree's roots, membership of the group udev hands /dev/kvm to, and the +// daemon reload. gunbc.runner.runner_host_grants has emitted them since it was written and +// runner_host_sudoers_content rendered PERMISSION for each one, but nothing executed them -- so the +// KVM grant arrived on srv2 by an operator typing the rendered argv by hand on 2026-09-05, which is +// the rostered class gunbc.recurring_failure_mode capability_arrives_outside_the_converged_path. +// +// THEY BELONG HERE AND NOT BESIDE THE OBSERVER THAT NEEDS THEM, which is that row's explicit +// ruling: an applier reached outside this spine inherits none of fleet_converge_plan_held_lease, so +// two applies against one baseline could both proceed, and an applier bolted beside +// gunbc.runner_microvm_host_ready -- the module that DISCLAIMED the grant -- inherits the +// disclaimer's blind spots. Emitted into apply.sh, each line is one plan-time-resolved argv that +// the operator reads before approving, the whole script runs under the generation flock, and the +// readback that decides whether the capability actually landed stays with the module that claims it +// (runner_microvm_host_ready's kvm_group standing). +// +// THE SPEC, NOT THE DEPLOY, AND THE DIFFERENCE IS A HOST. Every other family here resolves through +// runner_host_deploy_for and contributes nothing for a host whose width refuses. These rows depend +// on no runner population -- that is exactly why runner_host_wide_refused_operations takes the spec +// -- so resolving them through the deploy would silently deny srv2, a rostered host with a refused +// width, the group membership its own installed sudoers already authorizes. +// +// EVERY ROW IS AN IDEMPOTENT ENSURE, so a script stopped part-way by `set -euo pipefail` reconciles +// forward on the next apply: `usermod -aG` appending a group the login already holds exits 0 +// (measured on srv1, twice -- gunbc.executor_privileged_operation's precondition roster), and the +// directory rows are ensures by construction. +data host_wide_grant_apply_header: String = "# host-wide executor grants (gunbc.runner_host_grants runner_host_wide_refused_operations) — resolved at plan time, authorized by this host's rendered sudoers" + +fn host_wide_grant_apply_lines(host: NonEmptyStr) -> List { + match runner_host_spec_for(host: host as HostIdentity) { + RunnerHostSpecAbsent { host: h } => + [concat("# host-wide executor grants: no runner host spec for ", concat(h as String, " — no grants are declared for it, so none are applied"))] + RunnerHostSpecFound { spec: spec } => + concat( + [host_wide_grant_apply_header], + map( + runner_host_wide_refused_operations(spec: spec), + op => executor_privileged_operation_converge_command(op: op), + ), + ) + } +} + // AN OBSERVATION REFUSAL REACHES THE SCRIPT AS A REFUSAL LINE rather than as no lines at all, for // the same reason it reaches the baseline as a row: a host that could not be read must not look // like a host with no fabric work. diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index a508ea52593..0a2766ec3de 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -1043,17 +1043,33 @@ fn fleet_converge_guest_image_receipt_upload_step() -> Step { } } -// THE HOST CONVERGE INSTALLS THE VMM AND REFUSES THE REST. It fetches the cited Firecracker -// release into an executor-owned root under a verified digest; it does NOT grant kvm group -// membership, which is a runner-host grant belonging to the sudoers roster rather than to this -// reconciler, and the receipt says so by name rather than failing vaguely. +// THE HOST CONVERGE INSTALLS THE VMM AND READS THE GRANT BACK. It fetches the cited Firecracker +// release into an executor-owned root under a verified digest. It does NOT apply the kvm group +// membership -- that is emitted into the full-host apply script by gunbc.fleet.fleet_converge_plan +// host_wide_grant_apply_lines, under the generation lease with every other host mutation -- and this +// mode READS BACK whether the grant landed, from the account database and the device's own +// delegation, refusing by name when it has not. The earlier wording here said this mode "refuses the +// kvm grant by name", which was true of the disclaiming version and is now false in the direction +// that matters: an operator reading it would not know the grant has an executor at all. +// +// THE SELECTED HOST IS PASSED IN, AND WITHOUT IT THIS MODE REFUSES. The entry seals the operator's +// choice against the machine's own hostname before any fetch, install or receipt write +// (gunbc.runner_microvm_host_ready microvm_host_binding_wet), so a run that is not told which host +// was selected cannot tell a correct dispatch from a mis-routed one and declines to mutate anything. +// That makes this env binding load-bearing rather than decorative: omit it and the mode exits +// ExpectedHostAbsent on every dispatch, which is exactly what it did between the binding landing and +// this line being added. fn fleet_converge_microvm_host_converge_step() -> Step { RunStep { name: Present { value: "Runner micro-VM host: install the cited Firecracker release and re-observe" }, id: Present { value: "microvm_host_converge" }, run: gunbc_ci_runner_microvm_host_converge_invoke(), shell: none, - env: none, + env: Present { + value: [ + kv(key: fleet_converge_expected_host_env_name, value: yaml_string(s: "${{ github.event.inputs.host }}")), + ] + }, working_directory: none, if_condition: Present { value: fleet_converge_microvm_host_converge_step_if }, continue_on_error: none, @@ -1190,13 +1206,20 @@ fn fleet_converge_runner_host_file_receipt_upload_step() -> Step { } } +// THE PROBE TAKES THE SELECTED HOST FOR THE SAME REASON THE CONVERGE DOES: it reads the host +// standing before booting anything, and that standing is now bound to the host the dispatch chose. +// A boot verdict attributed to the wrong machine is worse than no verdict. fn fleet_converge_microvm_boot_probe_step() -> Step { RunStep { name: Present { value: "Runner micro-VM: boot the guest image and read the serial console" }, id: Present { value: "microvm_boot_probe" }, run: gunbc_ci_runner_microvm_boot_probe_invoke(), shell: none, - env: none, + env: Present { + value: [ + kv(key: fleet_converge_expected_host_env_name, value: yaml_string(s: "${{ github.event.inputs.host }}")), + ] + }, working_directory: none, if_condition: Present { value: fleet_converge_microvm_boot_probe_step_if }, continue_on_error: none, @@ -2060,7 +2083,7 @@ data fleet_converge_workflow: Workflow = Workflow { }, DispatchInput { name: "mode", - description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown" }, + description: Present { value: "plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown" }, required: true, default: none, type: InputChoice { options: fleet_converge_mode_options }, diff --git a/dag/gunbc/recurring_failure_mode/capability_arrives_outside_the_converged_path.dag b/dag/gunbc/recurring_failure_mode/capability_arrives_outside_the_converged_path.dag index 4f674bbb643..50abf3c93f4 100644 --- a/dag/gunbc/recurring_failure_mode/capability_arrives_outside_the_converged_path.dag +++ b/dag/gunbc/recurring_failure_mode/capability_arrives_outside_the_converged_path.dag @@ -35,6 +35,10 @@ data capability_arrives_outside_the_converged_path: RecurringFailureMode = Recur "THE PREFERRED REPAIR IS MORE EXPENSIVE THAN THE RULING ABOVE READS, MEASURED WHEN A LANE TRIED TO TAKE THE CHEAP ROUTE INSTEAD. Closing this in fleet converge is not a call-site move: `gunbc.fleet.fleet_converge_plan` imports only `fabric_cell_effect_operations` and `fabric_allocation_store_effect_operations` from `gunbc.runner.runner_host_grants`, and NOTHING in the corpus consumes `runner_host_privileged_operations` except `runner_host_sudoers_content`, the renderer that writes permission. So the preferred arm requires the converge plan to CARRY and APPLY host privileged operations, with the lease and receipt story attached -- `fleet_converge_plan_held_lease` is generation-scoped so two applies against one baseline cannot both proceed, and an apply reached outside that spine inherits none of it. That is a piece of the converge spine, not a wiring. THIS IS WHY THE CHEAP ROUTE KEEPS BEING ATTEMPTED AND MUST KEEP BEING REFUSED. A lane wired the existing applier from `gunbc.runner_microvm_host_ready` -- the disclaiming module -- as its own CI run step, which is exactly the arm this ruling forbids, and did not notice it had done so. The refusal is not stylistic: an applier that lives beside the disclaimer inherits the disclaimer position and therefore its blind spots, which is how the capability arrived outside the converged path to begin with. Recording the cost here rather than softening the ruling, because the ruling being expensive is the reason nobody has discharged it -- not evidence that it should not be discharged.", + "THE PREFERRED ARM LANDED FOR THE HOST-WIDE GRANTS, AND THE ROW SAYS EXACTLY HOW FAR THAT GOES. `gunbc.fleet.fleet_converge_plan host_wide_grant_apply_lines` now emits `runner_host_wide_refused_operations` into the FULL-HOST apply script beside the fabric-cell effects, so the /dev/kvm membership, the converge state directory and the runner tree roots are applied under `fleet_converge_plan_held_lease` with the rest of the host mutation, read by an operator in plan.txt before approval, and rendered through `executor_privileged_operation_converge_command` -- the same value the sudoers line is rendered from. The first draft of this repair took the cheap route this row forbids, wiring the applier from `gunbc.runner_microvm_host_ready` into its own converge mode; the ruling was re-read and the call site moved before it was pushed, which is the only reason it is a footnote rather than a fourth specimen. WHAT IT DOES NOT DO, stated so the row is not read as discharged: the grants resolve through `runner_host_spec_for`, so a host with no roster row still gets a stated refusal line and no grant, and NOTHING here attributes a capability to the run that placed it. The next-rung trigger above is untouched -- a standing that refuses a capability it cannot attribute -- and this change does not reach it: it makes the modeled path the one that DELIVERS, not the one that can tell whether it did.", + + "AND THE READBACK STAYS WITH THE OBSERVER, WHICH IS WHERE THE DISCLAIMER WAS RIGHT. `gunbc.runner_microvm_host_ready` now carries the executor's group enrolment as its own typed standing (`KvmGroupEnrolment`, read from the account database with `id -nG `), because the thing that was actually missing from that module was not an applier but the ability to tell a landed grant from a missing one. The reading is deliberately NOT `test -w /dev/kvm`: supplementary groups enter a process's credentials at its exec, so the converge that applies the grant, and any process already running, still read present-not-writable -- a readback on this process would refuse a host that had just converged correctly. Readiness therefore splits in two, and the split is the claim: `firecracker_host_ready` admits enrolment (the converge's question, effective at the next incarnation) and `firecracker_host_ready_in_this_process` requires writability (the boot probe's question, because it opens the device itself). The receipt states that the grant is effective for incarnations started AFTER it, so nobody reads present-not-writable beside enrolled as a reason to restart a live slot.", + "REPAIR, IN PREFERENCE ORDER: bake the capability into a host image so the question cannot be asked, or close it in fleet converge. NOT an applier bolted beside the disclaiming module, and never the sudoers argv by hand -- that converts a modeled hole into a second undeclared deviation, which is how specimen A was produced. The reviewer's question for any host-capability claim: which run put it there, and can you name it.", ], diff --git a/dag/gunbc/runner/runner_microvm_boot_probe.dag b/dag/gunbc/runner/runner_microvm_boot_probe.dag index 2789499a679..87d125acf61 100644 --- a/dag/gunbc/runner/runner_microvm_boot_probe.dag +++ b/dag/gunbc/runner/runner_microvm_boot_probe.dag @@ -27,9 +27,11 @@ import gunbc.runner_guest_image { runner_guest_base_artifact_path, runner_guest_kernel_artifact, } import gunbc.runner_microvm_host_ready { - FirecrackerHostStanding, firecracker_host_ready, firecracker_installed_binary, + FirecrackerHostStanding, firecracker_host_ready_in_this_process, firecracker_installed_binary, firecracker_install_root, firecracker_host_standing_text, observe_firecracker_host_standing_wet, + MicrovmHostBinding, MicrovmHostBound, MicrovmHostBindingRefused, + microvm_host_binding_wet, microvm_host_binding_refusal_text, } import gunbc.command_runner { ProcessObservation, run_shell_command_observe, run_shell_commands, LocalExec, @@ -260,11 +262,20 @@ fn runner_microvm_boot_probe_wet() -> ProcessExit Absent => exit_failure(reason: "runner_microvm_boot_probe: HOME is unset, so neither the image root nor the Firecracker install root can be named") Present { value: home } => { let fc_root = firecracker_install_root(executor_home: home) - let standing = observe_firecracker_host_standing_wet(root: fc_root) - if !firecracker_host_ready(standing: standing) { + // THE LOGIN COMES FROM THE HOST'S ROSTER ROW, because the standing now carries the executor's + // group enrolment and that is a fact about a named account. A probe on a host with no roster + // row has no executor to ask about, which is a BootNotAttempted cause like any other. + match microvm_host_binding_wet() { + MicrovmHostBindingRefused { cause: c } => + runner_microvm_boot_probe_receipt_wet( + verdict: BootNotAttempted { cause: join(["the host this probe is running on is not bound to the host the dispatch selected, so no executor is named to ask about and nothing was booted: ", microvm_host_binding_refusal_text(c: c)], "") }, + ) + MicrovmHostBound { spec: spec, host: bound_host } => { + let standing = observe_firecracker_host_standing_wet(root: fc_root, login: spec.job_user) + if !firecracker_host_ready_in_this_process(standing: standing) { runner_microvm_boot_probe_receipt_wet( verdict: BootNotAttempted { - cause: join(["the host cannot run a micro-VM at all, so no image verdict is available --\n", firecracker_host_standing_text(s: standing)], ""), + cause: join(["the host cannot run a micro-VM at all, so no image verdict is available --\n", firecracker_host_standing_text(s: standing, host: bound_host, expected_login: spec.job_user)], ""), }, ) } else { @@ -319,6 +330,8 @@ fn runner_microvm_boot_probe_wet() -> ProcessExit } } } + } + } } } } diff --git a/dag/gunbc/runner/runner_microvm_host_ready.dag b/dag/gunbc/runner/runner_microvm_host_ready.dag index 5cbcb1d31ca..ec187a06e6c 100644 --- a/dag/gunbc/runner/runner_microvm_host_ready.dag +++ b/dag/gunbc/runner/runner_microvm_host_ready.dag @@ -25,21 +25,34 @@ import extdeps.virtualization.firecracker { FirecrackerRelease, firecracker_release_aarch64, firecracker_requires_kvm_device, firecracker_release_tarball_filename, firecracker_release_download_url, firecracker_release_binary_path_in_tarball, firecracker_release_aarch64_tarball_sha256, - firecracker_version_command, firecracker_version_line, + firecracker_version_command, firecracker_version_line, kvm_device_group, } import gunbc.command_runner { ProcessOutcome, ProcessOutputPresent, ProcessOutputAbsent, ProcessRefused, run_shell_command_capture, run_shell_commands, process_outcome_admitted, LocalExec, } +import extdeps.tools.id { id_group_names_command } +import extdeps.tools.coreutils_stat { + PathOwnershipObservation, PathOwnershipObserved, PathOwnershipUnobservable, coreutils_stat_path_ownership, + PathModeObservation, PathModeObserved, PathModeUnobservable, coreutils_stat_path_mode, +} +import gunbc.host_effect { LocalShell } +import gunbc.hostname_read { HostnameShortCaptured, HostnameShortCaptureRefused, hostname_short_read } +import gunbc.actions_run_binding { + actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent, fleet_converge_expected_host_env_name, +} +import product.placement_supply { HostIdentity } +import gunbc.runner_host_deploy { + RunnerHostSpec, RunnerHostSpecFound, RunnerHostSpecAbsent, runner_host_spec_for, +} // THE HOST HALF OF THE RUNNER MICRO-VM, OBSERVED BEFORE ANYTHING IS BUILT ON IT. gunbc.runner_microvm // models what a runner guest is and admits a VM config; nothing had ever asked a host whether it // could run one. Two facts decide that: the KVM device exists and the executor may write to it, and // the cited Firecracker release is installed and answers `--version` with the modeled version. This -// module observes both as a typed standing, plans the one install it owns (the release tarball, -// digest-verified, into an executor-owned root, so no host grant is spent on it), and refuses the -// rest: the KVM group membership is a host grant that belongs to the runner host's sudoers roster, -// not to this reconciler. +// module observes both as a typed standing and installs the one release it owns (the release tarball, +// digest-verified, into an executor-owned root, so no host grant is spent on it). The KVM group +// membership is applied by the fleet converge spine, and READ BACK here. // // THE FIRST MEASUREMENT HERE WAS INVALIDATED BY THE MEASURER, WHICH IS WHY IT IS DATED AND // ATTRIBUTED RATHER THAN STATED AS THE HOST'S STANDING. Observed on srv2 2026-09-05: /dev/kvm @@ -52,19 +65,67 @@ import gunbc.command_runner { // is a property of one shell session, NOT of convergence, and it must not be read as evidence that // a host reaches this standing on its own. // -// THE GAP THAT PERMITTED IT is that EnsureSupplementaryGroupMembership is AUTHORIZED and never -// APPLIED: gunbc.runner.runner_host_grants runner_host_privileged_operations emits the row, and its -// only consumer is runner_host_sudoers_content, which renders permission to run the argv. This -// reconciler disclaims it (above). So the operation is granted by one module, disclaimed by -// another, and executed by neither -- a hole an operator falls into once per host, by hand. The -// refusal below is CORRECT and stays; what is missing is an applier for the host-grant ops, which -// is an ownership decision about host actuation and not this module's to take unilaterally. +// THE GAP THAT PERMITTED IT WAS THAT EnsureSupplementaryGroupMembership WAS AUTHORIZED AND NEVER +// APPLIED, AND IT IS THE CONVERGE SPINE THAT NOW APPLIES IT. gunbc.runner.runner_host_grants +// runner_host_wide_refused_operations emits the row and runner_host_sudoers_content renders +// PERMISSION to run its argv; until now nothing ran it, so the operation was granted by one module, +// disclaimed by this one, and executed by neither -- the rostered class +// gunbc.recurring_failure_mode capability_arrives_outside_the_converged_path, whose observable form +// is one hand-typed argv per host. The executor is gunbc.fleet.fleet_converge_plan, which emits the +// host-wide operations into the full-host apply script beside the fabric-cell effects, under the +// same generation lease as every other host mutation. This module stays the OBSERVER, which is what +// its disclaimer was right about; what was wrong was that its observation could not tell a landed +// grant from a missing one. +// +// THE CONVERGE MAY NOT OBSERVE ITS OWN GRANT LANDING, AND SAYING OTHERWISE WOULD BE THE FABRICATED +// ARM. A supplementary group enters a process's credentials at exec and never afterwards, so the +// process that ran `usermod -aG` still holds the old set and `test -w /dev/kvm` under it still +// answers false. Reading that as "the grant failed" would refuse a converged host forever; reading +// it as success would assert a capability nobody observed. So the readback asks the ACCOUNT +// DATABASE instead -- `id -nG ` names the login explicitly, which is the fact that decides +// what the NEXT incarnation gets -- and the standing carries process writability and database +// enrolment as two separate observations, with the receipt stating which one carried the host. type KvmDeviceStanding = KvmDeviceWritable | KvmDevicePresentNotWritable | KvmDeviceAbsent +// THE DEVICE AND THE GRANT ARE TWO FACTS, AND COLLAPSING THEM IS WHAT MADE THE OLD STANDING UNABLE +// TO DESCRIBE A CONVERGED HOST. KvmDeviceStanding above answers "can THIS process open the device", +// which is a property of credentials established before the converge started. This answers "does +// the account database enrol the executor login in the device's group", which is what every later +// incarnation will inherit. A host mid-converge reads present-not-writable AND enrolled, and that +// pair is the truth rather than a contradiction. +// +// UNREADABLE IS ITS OWN ARM BECAUSE "we could not ask" AND "the login is not a member" HAVE +// OPPOSITE REMEDIES: the first is a broken probe on the converging host, the second is a grant that +// did not land. Folding them would make a failed `id` read as a missing grant and send an operator +// to re-run an apply that already worked. +type KvmGroupEnrolment + = KvmGroupEnrolled { login: NonEmptyStr, group: NonEmptyStr } + | KvmGroupNotEnrolled { login: NonEmptyStr, group: NonEmptyStr } + | KvmGroupEnrolmentUnreadable { login: NonEmptyStr, cause: String } + +// ENROLMENT PLUS PRESENCE DOES NOT PROVE THE NEXT INCARNATION CAN OPEN THE DEVICE, and this type is +// what closes that gap. `present-not-writable` beside `enrolled` has TWO explanations: the grant has +// landed and this process simply predates it, or /dev/kvm is not delegated to the group the grant +// names at all. udev's Ubuntu default hands the device to group kvm at mode 0660 +// (50-udev-default.rules, cited at extdeps.virtualization.firecracker kvm_device_group), but that is +// UPSTREAM DESIRED STATE, not a readback of this host -- a local rule, a different distribution or a +// hand-chmod makes it false, and reading a capability off an upstream default is asserting as +// deduced what was only inferred (DESIGN section 4d). +// +// So the delegation is OBSERVED: the device's own group name and its group-write bit, joined into +// one verdict. Contradicted carries what was actually seen, because the remedy differs by which +// half failed -- a device owned by a different group is a udev question, a device owned by kvm with +// no group-write is a mode question -- and Unreadable stays separate from both for the reason every +// other arm here does. +type KvmDeviceDelegation + = KvmDeviceDelegated { group: NonEmptyStr } + | KvmDeviceDelegationContradicted { expected_group: NonEmptyStr, observed_group: NonEmptyStr, group_writable: Bool } + | KvmDeviceDelegationUnreadable { cause: String } + type FirecrackerBinaryStanding = FirecrackerInstalled { binary: String, version_line: String } | FirecrackerVersionMismatch { binary: String, observed: String } @@ -98,6 +159,8 @@ type FootprintInstrumentStanding { type FirecrackerHostStanding { kvm: KvmDeviceStanding + kvm_group: KvmGroupEnrolment + kvm_delegation: KvmDeviceDelegation binary: FirecrackerBinaryStanding footprint_instruments: FootprintInstrumentStanding } @@ -151,15 +214,71 @@ fn binary_standing_of(binary: String, executable: Bool, version_output: String, } } -fn firecracker_host_ready(standing: FirecrackerHostStanding) -> Bool { +// THE CONVERGE'S QUESTION IS ABOUT THE NEXT INCARNATION, SO IT IS ANSWERED ENTIRELY FROM DURABLE +// HOST STATE AND NEVER FROM THIS PROCESS'S CREDENTIALS. Three facts, all required: the device +// exists, the host delegates it to the group the grant names with group-write, and the account +// database enrols the executor login in that group. Any incarnation started after this line +// inherits all three; none of them can be read off whether the CURRENT process happens to hold an +// open path to the device. +// +// THIS DELIBERATELY DOES NOT SHORT-CIRCUIT ON KvmDeviceWritable, AND THAT WAS THE DEFECT. An +// earlier revision admitted a writable device outright, which answers the next-incarnation question +// from the current incarnation: a process that inherited the group before the grant was revoked can +// still open the device, so the host would read ready while the login was no longer enrolled at +// all -- and the next incarnation, the one the whole split exists to speak for, would fail. A +// writable device is now evidence for `firecracker_host_ready_in_this_process` and for nothing +// else. +// THE POSITIVE ARMS CARRY A LOGIN AND A GROUP, AND DISCARDING THEM IS HOW TWO TRUE OBSERVATIONS +// COMPOSE INTO A FALSE CONCLUSION. An earlier revision matched `KvmGroupEnrolled { login: _, group: _ }` +// and `KvmDeviceDelegated { group: _ }` for their SHAPE alone, so "the device is delegated to group A" +// beside "some login is enrolled in group B" read as ready -- and the login it believed was whichever +// one the observation happened to be asked about, not the one this host's roster row names. Three +// equalities are checked rather than assumed: the enrolled login IS the bound spec's job_user, and the +// enrolled group and the delegated group are BOTH the group udev hands the device to. +fn kvm_reachable(standing: FirecrackerHostStanding, expected_login: NonEmptyStr) -> Bool { + (match standing.kvm { + KvmDeviceAbsent => false + KvmDeviceWritable => true + KvmDevicePresentNotWritable => true + }) + && (match standing.kvm_delegation { + KvmDeviceDelegated { group: dg } => (dg as String) == (kvm_device_group as String) + KvmDeviceDelegationContradicted { expected_group: _, observed_group: _, group_writable: _ } => false + KvmDeviceDelegationUnreadable { cause: _ } => false + }) + && (match standing.kvm_group { + KvmGroupEnrolled { login: l, group: eg } => + (l as String) == (expected_login as String) && (eg as String) == (kvm_device_group as String) + KvmGroupNotEnrolled { login: _, group: _ } => false + KvmGroupEnrolmentUnreadable { login: _, cause: _ } => false + }) +} + +fn firecracker_binary_installed(standing: FirecrackerHostStanding) -> Bool { + match standing.binary { FirecrackerInstalled { binary: _, version_line: _ } => true _ => false } +} + +// THE CONVERGE'S QUESTION: has this host been brought to the state where its executor can run a +// guest? That is answered by the grant, whose effect lands at the next incarnation. +fn firecracker_host_ready(standing: FirecrackerHostStanding, expected_login: NonEmptyStr) -> Bool { + kvm_reachable(standing: standing, expected_login: expected_login) + && firecracker_binary_installed(standing: standing) +} + +// A DIFFERENT QUESTION WITH A DIFFERENT ANSWER, AND KEEPING THEM APART IS THE POINT. A caller that +// is about to OPEN /dev/kvm in its own process -- the boot probe -- is not asking whether the host +// converged; it is asking whether ITS credentials reach the device right now. An enrolment this +// process did not inherit answers the first question and not the second, so a probe gated on +// readiness would attempt a boot it cannot perform and report a VMM error as an IMAGE verdict. +fn firecracker_host_ready_in_this_process(standing: FirecrackerHostStanding) -> Bool { (match standing.kvm { KvmDeviceWritable => true _ => false }) - && (match standing.binary { FirecrackerInstalled { binary: _, version_line: _ } => true _ => false }) + && firecracker_binary_installed(standing: standing) } fn kvm_standing_text(k: KvmDeviceStanding) -> String { match k { KvmDeviceWritable => "kvm=writable" - KvmDevicePresentNotWritable => join(["kvm=present-not-writable (", firecracker_requires_kvm_device as String, " exists but the executor may not open it; the kvm group membership is a runner-host grant, not this reconciler's)"], "") + KvmDevicePresentNotWritable => join(["kvm=present-not-writable (", firecracker_requires_kvm_device as String, " exists but THIS process may not open it; a process's supplementary groups are fixed at its exec, so a grant applied during this run is not visible here)"], "") KvmDeviceAbsent => join(["kvm=absent (", firecracker_requires_kvm_device as String, " does not exist; the KVM module is not loaded or the host does not virtualize)"], "") } } @@ -172,10 +291,44 @@ fn binary_standing_text(b: FirecrackerBinaryStanding) -> String { } } -fn firecracker_host_standing_text(s: FirecrackerHostStanding) -> String { - join([kvm_standing_text(k: s.kvm), "\n", binary_standing_text(b: s.binary), "\n", +// THE RECEIPT SAYS WHEN THE GRANT TAKES EFFECT, because an operator reading "enrolled" beside +// "present-not-writable" will otherwise reach for a restart to force the pickup. Nothing needs to be +// restarted: the runner slots are just-in-time incarnations, so the next one to start after this +// line was written holds the group, which happens on its own when the current job ends. +fn kvm_group_standing_text(g: KvmGroupEnrolment) -> String { + match g { + KvmGroupEnrolled { login: l, group: gr } => + join(["kvm-group=enrolled (", l as String, " is a member of ", gr as String, + " in the account database; effective for executor incarnations started AFTER this line, not for any process already running)"], "") + KvmGroupNotEnrolled { login: l, group: gr } => + join(["kvm-group=not-enrolled (", l as String, " is not a member of ", gr as String, ")"], "") + KvmGroupEnrolmentUnreadable { login: l, cause: c } => + join(["kvm-group=unreadable (membership of ", l as String, " could not be read: ", c, ")"], "") + } +} + +fn kvm_delegation_text(d: KvmDeviceDelegation) -> String { + match d { + KvmDeviceDelegated { group: g } => + join(["kvm-delegation=delegated (", firecracker_requires_kvm_device as String, " is owned by group ", g as String, " and is group-writable, so membership of that group is what opens it)"], "") + KvmDeviceDelegationContradicted { expected_group: e, observed_group: o, group_writable: w } => + join(["kvm-delegation=contradicted (the grant enrols in group ", e as String, " but ", firecracker_requires_kvm_device as String, + " is owned by group ", o as String, " and is ", if w { "group-writable" } else { "NOT group-writable" }, + "; membership of ", e as String, " would not open this device)"], "") + KvmDeviceDelegationUnreadable { cause: c } => + join(["kvm-delegation=unreadable (", c, ")"], "") + } +} + +// THE RECEIPT NAMES THE BOUND HOST AND THE BOUND LOGIN, because a standing that says ready without +// saying ready-for-whom-on-which-machine is the certification defect this module was just repaired +// for. Both come from the sealed binding, never from the observation being described. +fn firecracker_host_standing_text(s: FirecrackerHostStanding, host: NonEmptyStr, expected_login: NonEmptyStr) -> String { + join(["bound-host=", host as String, " executor=", expected_login as String, "\n", + kvm_standing_text(k: s.kvm), "\n", kvm_delegation_text(d: s.kvm_delegation), "\n", kvm_group_standing_text(g: s.kvm_group), "\n", + binary_standing_text(b: s.binary), "\n", footprint_instrument_text(f: s.footprint_instruments), "\n", - if firecracker_host_ready(standing: s) { "standing=ready" } else { "standing=not-ready" }], "") + if firecracker_host_ready(standing: s, expected_login: expected_login) { "standing=ready" } else { "standing=not-ready" }], "") } // The install is the one effect this module owns: fetch the cited tarball, verify the published @@ -201,6 +354,135 @@ fn firecracker_install_commands(root: String, rel: FirecrackerRelease, scratch: ] } +// THE SUBJECT IS THE HOST THE PROCESS IS RUNNING ON, AND IT IS RESOLVED RATHER THAN PASSED. The +// mode is host-pinned by the workflow, so "which host" is not a parameter of the operation but its +// subject (gunbc.fleet_converge_workflow says the same about the two micro-VM modes). The executor +// login whose grant is applied and read back comes from the roster row for that host, never from a +// literal here: a second spelling of "ghrunner" in this module would be a grant applied to one +// account and authorized for another. +// RESOLVING THE HOSTNAME STOPS THE WRONG SPEC; IT DOES NOT STOP MUTATING THE WRONG HOST. An earlier +// revision read the kernel hostname and looked up its spec, which prevents srv1's spec being applied +// to srv3 -- and then converges srv3 and writes a receipt certifying it, because the run's own +// concurrency group is keyed to the host the OPERATOR REQUESTED. A mutable scheduling label is +// routing evidence, not machine identity (the subject-binding rule of gunbc#11751), so the operator's +// choice and the machine's own answer are TWO facts and this type is their join. +// +// THE BINDING IS SEALED BEFORE ANY MUTATION, which is the whole point rather than a nicety: a +// mismatch must refuse having performed zero install operations, so nothing downstream of this join +// may fetch, install, create a receipt directory or reach a readiness verdict. The converge below +// matches this FIRST, outside every other arm. +type MicrovmHostBinding + = MicrovmHostBound { spec: RunnerHostSpec, host: NonEmptyStr } + | MicrovmHostBindingRefused { cause: MicrovmHostBindingRefusal } + +// FOUR CAUSES, NOT ONE STRING, because they send an operator to four different places: a dispatch +// that named no host, a machine that could not name itself, a machine that is not the one the +// dispatch selected, and a machine no roster describes. +type MicrovmHostBindingRefusal + = ExpectedHostAbsent { detail: String } + | HostObservationFailed { reason: String } + | HostBindingRefused { expected: NonEmptyStr, observed: NonEmptyStr } + | HostNotRostered { host: NonEmptyStr } + +// THE JOIN IS PURE SO THE MISMATCH IS DISCRIMINABLE WITHOUT A HOST. Its inputs are the two supplied +// values a witness can construct; the wet wrapper below is the only thing that has to run on a +// machine to obtain them (DESIGN section 3: a witness discriminates at one interface). +fn microvm_host_binding_of(expected: NonEmptyStr, observed: NonEmptyStr) -> MicrovmHostBinding { + if (trim(s: expected as String)) != (trim(s: observed as String)) { + MicrovmHostBindingRefused { cause: HostBindingRefused { expected: expected, observed: observed } } + } else { + match runner_host_spec_for(host: (trim(s: observed as String)) as HostIdentity) { + RunnerHostSpecFound { spec: sp } => MicrovmHostBound { spec: sp, host: observed } + RunnerHostSpecAbsent { host: h } => MicrovmHostBindingRefused { cause: HostNotRostered { host: h } } + } + } +} + +fn microvm_host_binding_refusal_text(c: MicrovmHostBindingRefusal) -> String { + match c { + ExpectedHostAbsent { detail: d } => + join(["ExpectedHostAbsent -- ", fleet_converge_expected_host_env_name, " ", d, + "; the subject is the host the dispatch selected and this run was given none, so it will not converge whatever machine it happens to be running on"], "") + HostObservationFailed { reason: r } => + join(["HostObservationFailed -- the host could not name itself (", r, "), so its identity cannot be joined to the dispatch's selection"], "") + HostBindingRefused { expected: e, observed: o } => + join(["HostBindingRefused -- the dispatch selected ", e as String, " and this machine answers ", o as String, + "; NOTHING has been fetched, installed or written. A run that continued here would mutate ", o as String, + " and certify it under ", e as String, "'s name"], "") + HostNotRostered { host: h } => + join(["HostNotRostered -- ", h as String, " carries no runner host spec, so no executor login and no grant roster are declared for it"], "") + } +} + +fn microvm_host_binding_wet() -> MicrovmHostBinding { + match actions_variable_read(name: fleet_converge_expected_host_env_name) { + ActionsVariableAbsent { variable: _, detail: d } => + MicrovmHostBindingRefused { cause: ExpectedHostAbsent { detail: d } } + ActionsVariablePresent { value: expected } => + match hostname_short_read(transport: LocalShell) { + HostnameShortCaptureRefused { reason: r } => + MicrovmHostBindingRefused { cause: HostObservationFailed { reason: r } } + HostnameShortCaptured { value: v } => + if trim(s: expected) == "" { + MicrovmHostBindingRefused { cause: ExpectedHostAbsent { detail: "is set to an empty value" } } + } else if trim(s: v) == "" { + MicrovmHostBindingRefused { cause: HostObservationFailed { reason: "hostname -s returned empty output" } } + } else { + microvm_host_binding_of(expected: trim(s: expected) as NonEmptyStr, observed: trim(s: v) as NonEmptyStr) + } + } + } +} + +// A WORD MATCH OVER `id -nG`, WHICH PRINTS NAMES SEPARATED BY SINGLE SPACES. A substring test would +// admit "kvm-admin" as evidence of membership in "kvm", which is exactly the kind of near-miss a +// grant readback must not accept. +fn group_names_contain(output: String, group: NonEmptyStr) -> Bool { + fold(split(trim(s: output), " "), init: false, f: (acc, w) => acc || trim(s: w) == (group as String)) +} + +// THE TWO HALVES ARE OBSERVED SEPARATELY AND JOINED HERE, because either can fail on its own and +// the join is where "what did we see" turns into "does membership of this group open this device". +// Both probes are typed metadata observations over extdeps.tools.coreutils_stat; neither reads a +// mode as a string, because extdeps.access.posix owns what the digits mean. +fn observe_kvm_device_delegation_wet() -> KvmDeviceDelegation { + let path = (firecracker_requires_kvm_device as String) as NonEmptyStr + match coreutils_stat_path_ownership(path: path) { + PathOwnershipUnobservable { path: _, reason: r } => + KvmDeviceDelegationUnreadable { cause: join(["the device's owning group could not be read: ", r], "") } + PathOwnershipObserved { path: _, ownership: _, owner_name: _, group_name: g } => + match coreutils_stat_path_mode(path: path) { + PathModeUnobservable { path: _, reason: r } => + KvmDeviceDelegationUnreadable { cause: join(["the device's permission bits could not be read: ", r], "") } + PathModeObserved { path: _, mode: m } => + if (g as String) == (kvm_device_group as String) && m.group.write { + KvmDeviceDelegated { group: kvm_device_group } + } else { + KvmDeviceDelegationContradicted { + expected_group: kvm_device_group, + observed_group: g, + group_writable: m.group.write, + } + } + } + } +} + +fn observe_kvm_group_enrolment_wet(login: NonEmptyStr) -> KvmGroupEnrolment { + match run_shell_command_capture(command: id_group_names_command(user: login), transport: LocalExec) { + ProcessOutputPresent { text: t } => + if group_names_contain(output: t, group: kvm_device_group) { + KvmGroupEnrolled { login: login, group: kvm_device_group } + } else { + KvmGroupNotEnrolled { login: login, group: kvm_device_group } + } + ProcessOutputAbsent => + KvmGroupEnrolmentUnreadable { login: login, cause: "id -nG exited successfully and printed nothing, so no group list was read" } + ProcessRefused { exit_code: c, stderr: e } => + KvmGroupEnrolmentUnreadable { login: login, cause: join(["id -nG exited ", to_string(c), ": ", trim(s: e)], "") } + } +} + fn executor_home_wet() -> String? { match shell.Env.Get(name: "HOME" as NonEmptyStr).value { Absent => none @@ -214,7 +496,7 @@ fn executor_home_wet() -> String? { // let "we could not ask" and "the host cannot measure" render identically to whoever reads the // receipt and decides what can measure the reserve. -fn observe_firecracker_host_standing_wet(root: String) -> FirecrackerHostStanding +fn observe_firecracker_host_standing_wet(root: String, login: NonEmptyStr) -> FirecrackerHostStanding uses net: Network { let rel = firecracker_release_aarch64 @@ -240,6 +522,8 @@ fn observe_firecracker_host_standing_wet(root: String) -> FirecrackerHostStandin } FirecrackerHostStanding { kvm: kvm_standing_of(exists: exists, writable: writable), + kvm_group: observe_kvm_group_enrolment_wet(login: login), + kvm_delegation: observe_kvm_device_delegation_wet(), binary: binary_standing_of(binary: binary, executable: executable, version_output: version_output, rel: rel), footprint_instruments: FootprintInstrumentStanding { instruments: [ @@ -265,68 +549,97 @@ fn observe_firecracker_host_standing_wet(root: String) -> FirecrackerHostStandin } } +fn write_host_standing_receipt(text: String) -> String? { + let _ = run_shell_commands(commands: [mkdir_parents_command(path: runner_microvm_host_standing_receipt_dir)], transport: LocalExec) + let w = Filesystem.Write(path: runner_microvm_host_standing_receipt_path, content: concat(text, "\n")) + if w.success { none } else { Present { value: w.error } } +} + +// The receipt is written BEFORE the verdict is decided, on every path, so a not-ready host leaves +// the observation that says why rather than only an exit code. +fn host_standing_verdict(entry: String, standing: FirecrackerHostStanding, host: NonEmptyStr, expected_login: NonEmptyStr) -> ProcessExit { + let text = firecracker_host_standing_text(s: standing, host: host, expected_login: expected_login) + match write_host_standing_receipt(text: text) { + Present { value: e } => exit_failure(reason: join([entry, ": receipt write refused: ", e], "")) + Absent => + if firecracker_host_ready(standing: standing, expected_login: expected_login) { + ExitSuccess + } else { + exit_failure(reason: join([entry, ": host is not ready --\n", text], "")) + } + } +} + fn runner_microvm_host_observe_wet() -> ProcessExit uses net: Network { - match executor_home_wet() { - Absent => exit_failure(reason: "runner_microvm_host_observe: HOME is unset, so the executor-owned install root cannot be named") - Present { value: home } => { - let standing = observe_firecracker_host_standing_wet(root: firecracker_install_root(executor_home: home)) - let text = firecracker_host_standing_text(s: standing) - let _ = run_shell_commands(commands: [mkdir_parents_command(path: runner_microvm_host_standing_receipt_dir)], transport: LocalExec) - let w = Filesystem.Write(path: runner_microvm_host_standing_receipt_path, content: concat(text, "\n")) - if !w.success { - exit_failure(reason: join(["runner_microvm_host_observe: receipt write refused: ", w.error], "")) - } else if firecracker_host_ready(standing: standing) { - ExitSuccess - } else { - exit_failure(reason: join(["runner_microvm_host_observe: host is not ready --\n", text], "")) + match microvm_host_binding_wet() { + MicrovmHostBindingRefused { cause: c } => + exit_failure(reason: join(["runner_microvm_host_observe: ", microvm_host_binding_refusal_text(c: c)], "")) + MicrovmHostBound { spec: spec, host: host } => + match executor_home_wet() { + Absent => exit_failure(reason: "runner_microvm_host_observe: HOME is unset, so the executor-owned install root cannot be named") + Present { value: home } => + host_standing_verdict( + entry: "runner_microvm_host_observe", + standing: observe_firecracker_host_standing_wet(root: firecracker_install_root(executor_home: home), login: spec.job_user), + host: host, + expected_login: spec.job_user, + ) } - } } } -// Converge = observe, install only the binary if absent, observe again. The KVM half is never -// actuated here: a device this executor cannot open is reported, and the report names the grant. +// Converge = resolve the subject, observe, install the binary if absent, observe again, report. +// THE KVM GRANT IS APPLIED BY THE FLEET CONVERGE SPINE AND NOT HERE, AND THAT PLACEMENT IS A +// RULING RATHER THAN A PREFERENCE. gunbc.recurring_failure_mode +// capability_arrives_outside_the_converged_path forbids an applier bolted beside this module -- the +// one that disclaimed the grant -- because an apply reached outside gunbc.fleet.fleet_converge_plan +// inherits none of fleet_converge_plan_held_lease, so two applies against one baseline could both +// proceed, and an applier that lives beside the disclaimer inherits the disclaimer's blind spots. +// The grant is emitted into the full-host apply script by gunbc.fleet.fleet_converge_plan +// host_wide_grant_apply_lines. +// WHAT THIS MODULE OWNS IS THE READBACK, at the point the capability is CLAIMED: the standing above +// carries the executor's group enrolment as its own typed fact, so a host whose grant never landed +// refuses here with a cause that names the login and the group. fn runner_microvm_host_converge_wet() -> ProcessExit uses net: Network { - match executor_home_wet() { - Absent => exit_failure(reason: "runner_microvm_host_converge: HOME is unset, so the executor-owned install root cannot be named") - Present { value: home } => { - let root = firecracker_install_root(executor_home: home) - let before = observe_firecracker_host_standing_wet(root: root) - let install = match before.binary { - FirecrackerInstalled { binary: _, version_line: _ } => ExitSuccess - _ => { - let scratch = join([root, "/scratch"], "") - match run_shell_commands(commands: firecracker_fetch_commands(root: root, rel: firecracker_release_aarch64, scratch: scratch), transport: LocalExec) { - ExitSuccess => - match sha256sum_verify_via_shell(path: firecracker_tarball_path(scratch: scratch, rel: firecracker_release_aarch64) as NonEmptyStr, expected_digest: firecracker_release_aarch64_tarball_sha256) { - Sha256Matches => run_shell_commands(commands: firecracker_install_commands(root: root, rel: firecracker_release_aarch64, scratch: scratch), transport: LocalExec) - Sha256Mismatch => exit_failure(reason: join(["runner_microvm_host_converge: FirecrackerTarballDigestMismatch -- the fetched ", firecracker_release_tarball_filename(rel: firecracker_release_aarch64), " does not hash to the published digest; installing NOTHING"], "")) - Sha256VerifyInfraFailure { reason: r } => exit_failure(reason: join(["runner_microvm_host_converge: digest verification could not run (", r as String, "); installing NOTHING"], "")) + match microvm_host_binding_wet() { + MicrovmHostBindingRefused { cause: c } => + exit_failure(reason: join(["runner_microvm_host_converge: ", microvm_host_binding_refusal_text(c: c)], "")) + MicrovmHostBound { spec: spec, host: host } => + match executor_home_wet() { + Absent => exit_failure(reason: "runner_microvm_host_converge: HOME is unset, so the executor-owned install root cannot be named") + Present { value: home } => { + let root = firecracker_install_root(executor_home: home) + let before = observe_firecracker_host_standing_wet(root: root, login: spec.job_user) + let install = match before.binary { + FirecrackerInstalled { binary: _, version_line: _ } => ExitSuccess + _ => { + let scratch = join([root, "/scratch"], "") + match run_shell_commands(commands: firecracker_fetch_commands(root: root, rel: firecracker_release_aarch64, scratch: scratch), transport: LocalExec) { + ExitSuccess => + match sha256sum_verify_via_shell(path: firecracker_tarball_path(scratch: scratch, rel: firecracker_release_aarch64) as NonEmptyStr, expected_digest: firecracker_release_aarch64_tarball_sha256) { + Sha256Matches => run_shell_commands(commands: firecracker_install_commands(root: root, rel: firecracker_release_aarch64, scratch: scratch), transport: LocalExec) + Sha256Mismatch => exit_failure(reason: join(["runner_microvm_host_converge: FirecrackerTarballDigestMismatch -- the fetched ", firecracker_release_tarball_filename(rel: firecracker_release_aarch64), " does not hash to the published digest; installing NOTHING"], "")) + Sha256VerifyInfraFailure { reason: r } => exit_failure(reason: join(["runner_microvm_host_converge: digest verification could not run (", r as String, "); installing NOTHING"], "")) + } + other => other } - other => other + } } - } - } - match install { - ExitSuccess => { - let after = observe_firecracker_host_standing_wet(root: root) - let text = firecracker_host_standing_text(s: after) - let _ = run_shell_commands(commands: [mkdir_parents_command(path: runner_microvm_host_standing_receipt_dir)], transport: LocalExec) - let w = Filesystem.Write(path: runner_microvm_host_standing_receipt_path, content: concat(text, "\n")) - if !w.success { - exit_failure(reason: join(["runner_microvm_host_converge: receipt write refused: ", w.error], "")) - } else if firecracker_host_ready(standing: after) { - ExitSuccess - } else { - exit_failure(reason: join(["runner_microvm_host_converge: converged what it owns; host is still not ready --\n", text], "")) + match install { + ExitSuccess => + host_standing_verdict( + entry: "runner_microvm_host_converge", + standing: observe_firecracker_host_standing_wet(root: root, login: spec.job_user), + host: host, + expected_login: spec.job_user, + ) + other => other } } - other => other } - } } } diff --git a/dag/test/claim/fleet/fleet_converge_plan_witness_test.dag b/dag/test/claim/fleet/fleet_converge_plan_witness_test.dag index 54bab312ad4..1f192db6d13 100644 --- a/dag/test/claim/fleet/fleet_converge_plan_witness_test.dag +++ b/dag/test/claim/fleet/fleet_converge_plan_witness_test.dag @@ -74,6 +74,11 @@ import product.placement_supply { HostIdentity } import gunbc.build_cache_instance { RunnerSlotIdentity } import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv3, operator_host_srv4 } import gunbc.ownership { Owned } +import gunbc.fleet_converge_workflow { + fleet_converge_microvm_host_converge_step, fleet_converge_microvm_boot_probe_step, +} +import gunbc.actions_run_binding { fleet_converge_expected_host_env_name } +import extdeps.github.actions { Step, RunStep } import gunbc.fleet_converge_plan { fleet_converge_cap_lines, fleet_converge_cap_refusal_count, @@ -642,6 +647,104 @@ test fn apply_shell_carries_dissolve_on_marker_not_host_gate() -> Bool { } } +// THE FIRST LINE AN APPROVER READS IS ITS OWN SUBJECT, so this control extracts THAT LINE and +// asserts over it alone. A whole-script substring check cannot do this job: the truthful detailed +// header sits lower in the same script and would satisfy any document-wide search for the retirement +// words while the admission-gate summary at the top still said "runner-slot ADD". That is exactly how +// the ADD-only defect survived its first repair -- fixed at one rendering site, left standing at the +// earlier one, with the existing witness looking only at the family header. +fn apply_shell_admission_gate_line(apply_shell: String) -> String { + fold( + split(apply_shell, "\n"), + init: "", + f: (acc, l) => if starts_with(s: l, prefix: "# Admission gates enforced") { l } else { acc }, + ) +} + +test fn the_top_admission_gate_line_names_retirement_not_add_only() -> Bool { + let artifact = fleet_converge_plan_artifact( + host: witness_host_srv1, + prior_generation: 0, + observed_timers: [], + observed_caps: [], + observed_slots: [], + observed_fabric_cells: fabric_cell_observation_unobserved(host: operator_host_srv3), + observed_activation_readiness: runner_activation_readiness_unobserved_no_transaction, + observed_unit_standings: runner_unit_standings_unobserved_no_transaction, + observed_retirement: SlotRetirementEvidenceAbsent { cause: "this fixture binds no retirement observation" }, + ) + let gate_line = apply_shell_admission_gate_line(apply_shell: artifact.apply_shell) + gate_line != "" + && string_contains(s: gate_line, pattern: "runner-slot reconciliation") + && string_contains(s: gate_line, pattern: "retirement inhibition via systemctl mask --now") + && string_contains(s: gate_line, pattern: "completed-retirement tree removal") + && !string_contains(s: gate_line, pattern: "runner-slot ADD") + && !string_contains(s: gate_line, pattern: "ADD only") +} + +// THE GRANT THAT NOBODY EXECUTED IS NOW IN THE SCRIPT THAT EXECUTES, and this claim is what makes +// that a fact rather than an intention. gunbc.recurring_failure_mode +// capability_arrives_outside_the_converged_path was opened because /dev/kvm membership was granted +// in sudoers and applied by an operator's hand -- so the discriminating assertion is the EXACT +// elevated argv, not the presence of the word "kvm": a line that names the group but a different +// binary, login or flag would satisfy a looser check and would not grant anything. +// +// THE NEGATIVE HALF IS A HOST WITH NO ROSTER ROW, which must contribute a stated refusal line and +// no executable grant. Without it this claim would pass on a fold that emitted the same grants for +// every host, which is a privilege placed on a machine no roster names. +test fn the_full_host_apply_script_carries_the_hosts_own_executor_grants() -> Bool { + let rostered = fleet_converge_plan_artifact( + host: witness_host_srv1, + prior_generation: 0, + observed_timers: [], + observed_caps: [], + observed_slots: [], + observed_fabric_cells: fabric_cell_observation_unobserved(host: operator_host_srv3), + observed_activation_readiness: runner_activation_readiness_unobserved_no_transaction, + observed_unit_standings: runner_unit_standings_unobserved_no_transaction, + observed_retirement: SlotRetirementEvidenceAbsent { cause: "this fixture binds no retirement observation" }, + ) + let unrostered = fleet_converge_plan_artifact( + host: "srv-not-rostered" as NonEmptyStr, + prior_generation: 0, + observed_timers: [], + observed_caps: [], + observed_slots: [], + observed_fabric_cells: fabric_cell_observation_unobserved(host: operator_host_srv3), + observed_activation_readiness: runner_activation_readiness_unobserved_no_transaction, + observed_unit_standings: runner_unit_standings_unobserved_no_transaction, + observed_retirement: SlotRetirementEvidenceAbsent { cause: "this fixture binds no retirement observation" }, + ) + string_contains(s: rostered.apply_shell, pattern: "'/usr/bin/sudo' '-n' '/usr/sbin/usermod' '-aG' 'kvm' 'ghrunner'") + && string_contains(s: rostered.apply_shell, pattern: "host-wide executor grants (gunbc.runner_host_grants runner_host_wide_refused_operations)") + && !string_contains(s: unrostered.apply_shell, pattern: "usermod") + && string_contains(s: unrostered.apply_shell, pattern: "no runner host spec for srv-not-rostered") +} + +// THE HEADER MUST DESCRIBE THE FOLD UNDER IT, because that line is what an operator reads before +// approving a host mutation. It said "ADD only" directly above a family that can also emit +// `systemctl mask --now` for an authorized surplus unit and remove a retired tree -- a narrower +// blast radius than the script carries, which is the one claim that matters to a reader deciding +// whether it is safe to run while jobs are live. The negative half is the whole point: the retired +// wording must not come back. +test fn the_slot_family_header_names_every_disposition_the_fold_can_emit() -> Bool { + let artifact = fleet_converge_plan_artifact( + host: witness_host_srv1, + prior_generation: 0, + observed_timers: [], + observed_caps: [], + observed_slots: [], + observed_fabric_cells: fabric_cell_observation_unobserved(host: operator_host_srv3), + observed_activation_readiness: runner_activation_readiness_unobserved_no_transaction, + observed_unit_standings: runner_unit_standings_unobserved_no_transaction, + observed_retirement: SlotRetirementEvidenceAbsent { cause: "this fixture binds no retirement observation" }, + ) + !string_contains(s: artifact.apply_shell, pattern: "ADD only") + && string_contains(s: artifact.apply_shell, pattern: "retirement inhibition (systemctl mask --now)") + && string_contains(s: artifact.apply_shell, pattern: "independently reobserved RetirementComplete") + && string_contains(s: artifact.apply_shell, pattern: "No desired runner is stopped, restarted or reinstalled") +} + // The stale diagnostic names the observed generation as ONE word. Split across two echo // arguments it renders `observed= 4`, which reads as an empty observation followed by a // number — precisely the confusion this PR removes from the guard itself (review 54519). @@ -2502,3 +2605,30 @@ fn wt_srv3_deploy() -> RunnerHostDeploy { Absent => wt_missing_deploy() } } + + +// THE BINDING IS ONLY A WALL IF THE STEP TELLS IT WHICH HOST WAS SELECTED, and for one revision it +// did not. gunbc.runner_microvm_host_ready seals the operator's choice against the machine's own +// hostname before any fetch, install or receipt write, reading that choice from +// FLEET_CONVERGE_EXPECTED_HOST -- and both micro-VM steps carried `env: none`, so every dispatch of +// either mode would have exited ExpectedHostAbsent having done nothing. The readback the repair +// exists to deliver had a route that terminated in a refusal, which is DESIGN section 3c's +// consumption question answered in the negative: a named consumer that cannot start. +// +// This asserts over the STEP VALUES rather than the emitted YAML text, because the text is a +// projection and the defect was in what the projection was given. Both modes are checked: they fail +// independently, so a control over one would have left the other dark. +fn microvm_step_env_names(step: Step) -> List { + match step { + RunStep { name: _, id: _, run: _, shell: _, env: e, working_directory: _, if_condition: _, continue_on_error: _, timeout_minutes: _ } => + match e { Present { value: kvs } => map(kvs, p => p.key) Absent => [] } + _ => [] + } +} + +test fn both_microvm_steps_pass_the_selected_host_to_the_binding() -> Bool { + let converge_env = microvm_step_env_names(step: fleet_converge_microvm_host_converge_step()) + let probe_env = microvm_step_env_names(step: fleet_converge_microvm_boot_probe_step()) + any(converge_env, n => n == fleet_converge_expected_host_env_name) + && any(probe_env, n => n == fleet_converge_expected_host_env_name) +} diff --git a/dag/test/claim/managed_directory_witness_test.dag b/dag/test/claim/managed_directory_witness_test.dag index 9121af4feb0..6238bfb921c 100644 --- a/dag/test/claim/managed_directory_witness_test.dag +++ b/dag/test/claim/managed_directory_witness_test.dag @@ -6,6 +6,8 @@ import extdeps.access.posix { PermissionBits, FileMode, file_mode_octal, + file_mode_of_octal_text, + permission_bits_of_octal_digit, } import std.effect_grant { Read, Write, Execute } import gunbc.ownership { Owned } @@ -128,3 +130,61 @@ test fn witness_octal_renders_special_bits_digit() -> Bool { } file_mode_octal(mode: mode) == "2750" } + +// THE INVERSE IS THE SAME GRAMMAR READ BACKWARD (DESIGN section 4), so the discriminating check is +// that the round trip preserves the mode -- not that one hand-picked string parses. stat's %a omits +// the special digit when it is zero, so "660" and "0660" must reach the same FileMode; that pair is +// the reason the reader slices digits from the least significant end rather than branching on +// length. +test fn the_octal_parse_is_the_inverse_of_the_render_for_three_and_four_digit_forms() -> Bool { + let setgid_mode = FileMode { + owner: PermissionBits { read: true, write: true, execute: true }, + group: PermissionBits { read: true, write: false, execute: true }, + other: PermissionBits { read: false, write: false, execute: false }, + setuid: false, + setgid: true, + sticky: false, + } + let kvm_mode = FileMode { + owner: PermissionBits { read: true, write: true, execute: false }, + group: PermissionBits { read: true, write: true, execute: false }, + other: PermissionBits { read: false, write: false, execute: false }, + setuid: false, + setgid: false, + sticky: false, + } + (match file_mode_of_octal_text(text: file_mode_octal(mode: setgid_mode)) { + Present { value: m } => file_mode_octal(mode: m) == "2750" && m.setgid && !m.group.write + Absent => false + }) + && (match file_mode_of_octal_text(text: "660") { + Present { value: m } => file_mode_octal(mode: m) == file_mode_octal(mode: kvm_mode) && m.group.write + Absent => false + }) + && (match file_mode_of_octal_text(text: "0660") { + Present { value: m } => file_mode_octal(mode: m) == file_mode_octal(mode: kvm_mode) + Absent => false + }) + && (match file_mode_of_octal_text(text: "640") { + Present { value: m } => !m.group.write && m.group.read + Absent => false + }) +} + +// AN UNPARSEABLE MODE IS A REFUSAL AND NEVER A MODE WITH EVERYTHING FALSE, because the consumer is +// deciding whether a device is delegated to a group: a fabricated all-denied mode would render as +// "this host does not delegate", sending an operator to fix a udev rule that is fine, when what +// actually happened is that the probe failed. An octal digit above 7 is the sharp case -- "680" is +// digits all the way down and still not a mode. +test fn an_unreadable_octal_mode_refuses_rather_than_reading_as_all_denied() -> Bool { + (match file_mode_of_octal_text(text: "") { Absent => true Present { value: _ } => false }) + && (match file_mode_of_octal_text(text: "rw-rw----") { Absent => true Present { value: _ } => false }) + && (match file_mode_of_octal_text(text: "680") { Absent => true Present { value: _ } => false }) + && (match file_mode_of_octal_text(text: "-1") { Absent => true Present { value: _ } => false }) + && (match file_mode_of_octal_text(text: "77777") { Absent => true Present { value: _ } => false }) + && (match permission_bits_of_octal_digit(digit: 8) { Absent => true Present { value: _ } => false }) + && (match permission_bits_of_octal_digit(digit: 6) { + Present { value: b } => b.read && b.write && !b.execute + Absent => false + }) +} diff --git a/dag/test/claim/runner/runner_microvm_host_ready_witness_test.dag b/dag/test/claim/runner/runner_microvm_host_ready_witness_test.dag index d808459a30c..00458720681 100644 --- a/dag/test/claim/runner/runner_microvm_host_ready_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_host_ready_witness_test.dag @@ -14,7 +14,13 @@ import gunbc.runner_microvm_host_ready { KvmDeviceStanding, KvmDeviceWritable, KvmDevicePresentNotWritable, KvmDeviceAbsent, FirecrackerBinaryStanding, FirecrackerInstalled, FirecrackerVersionMismatch, FirecrackerBinaryAbsent, FirecrackerHostStanding, - kvm_standing_of, binary_standing_of, firecracker_host_ready, firecracker_host_standing_text, + KvmGroupEnrolment, KvmGroupEnrolled, KvmGroupNotEnrolled, KvmGroupEnrolmentUnreadable, + KvmDeviceDelegation, KvmDeviceDelegated, KvmDeviceDelegationContradicted, KvmDeviceDelegationUnreadable, + MicrovmHostBinding, MicrovmHostBound, MicrovmHostBindingRefused, + MicrovmHostBindingRefusal, HostBindingRefused, HostNotRostered, + microvm_host_binding_of, microvm_host_binding_refusal_text, + kvm_standing_of, binary_standing_of, firecracker_host_ready, firecracker_host_ready_in_this_process, + group_names_contain, firecracker_host_standing_text, firecracker_fetch_commands, firecracker_install_commands, firecracker_installed_binary, firecracker_install_root, } import extdeps.systemd.systemctl { SystemdReportsMemoryPeak, SystemdTooOldForMemoryPeak, SystemdVersionUnreadable, systemd_memory_peak_support } @@ -34,25 +40,220 @@ test fn the_release_artifact_is_identified_by_version_arch_and_digest() -> Bool && firecracker_version_line(rel: firecracker_release_aarch64) == "Firecracker v1.16.1" } +// THE FIXTURE IS SUPPLIED AT ONE INTERFACE (DESIGN section 3): every claim below discriminates over +// firecracker_host_ready / _in_this_process / the standing renderer, so the standing is constructed +// rather than produced by running the host probes those claims are not about. The inhabitance claim +// that the real producer emits this shape is the converge itself, which runs on a host and writes +// runner-microvm-host-standing.txt; no witness can execute it and none pretends to. +data witness_instruments: FootprintInstrumentStanding = FootprintInstrumentStanding { + instruments: [ + InstrumentStanding { instrument: GnuTimeMaxRss, observation: InstrumentAbsent }, + InstrumentStanding { instrument: CgroupMemoryPeak, observation: InstrumentAbsent }, + InstrumentStanding { instrument: SystemdMemoryPeak, observation: InstrumentUnreadable { cause: "systemctl --version produced no readable line" } }, + ], +} + +data witness_installed: FirecrackerBinaryStanding = FirecrackerInstalled { binary: "/x/firecracker", version_line: "Firecracker v1.16.1" } + +data witness_login: NonEmptyStr = "ghrunner" + +data witness_host: NonEmptyStr = "srv1" + +data witness_enrolled: KvmGroupEnrolment = KvmGroupEnrolled { login: "ghrunner" as NonEmptyStr, group: "kvm" as NonEmptyStr } + +data witness_not_enrolled: KvmGroupEnrolment = KvmGroupNotEnrolled { login: "ghrunner" as NonEmptyStr, group: "kvm" as NonEmptyStr } + +data witness_delegated: KvmDeviceDelegation = KvmDeviceDelegated { group: "kvm" as NonEmptyStr } + +// THE DEFAULT FIXTURE DELEGATES, so every claim that is NOT about delegation reads the same as it +// did before the fact existed; the claims that ARE about it construct their own arm. +fn witness_standing(kvm: KvmDeviceStanding, group: KvmGroupEnrolment, binary: FirecrackerBinaryStanding) -> FirecrackerHostStanding { + FirecrackerHostStanding { kvm: kvm, kvm_group: group, kvm_delegation: witness_delegated, binary: binary, footprint_instruments: witness_instruments } +} + +fn witness_standing_with_delegation(kvm: KvmDeviceStanding, group: KvmGroupEnrolment, delegation: KvmDeviceDelegation) -> FirecrackerHostStanding { + FirecrackerHostStanding { kvm: kvm, kvm_group: group, kvm_delegation: delegation, binary: witness_installed, footprint_instruments: witness_instruments } +} + // The standing folds two facts; readiness needs both, and each refusal names what it lacks. -test fn readiness_needs_a_writable_kvm_device_and_the_modeled_version() -> Bool { - let instruments = FootprintInstrumentStanding { - instruments: [ - InstrumentStanding { instrument: GnuTimeMaxRss, observation: InstrumentAbsent }, - InstrumentStanding { instrument: CgroupMemoryPeak, observation: InstrumentAbsent }, - InstrumentStanding { instrument: SystemdMemoryPeak, observation: InstrumentUnreadable { cause: "systemctl --version produced no readable line" } }, - ], - } - let ready = FirecrackerHostStanding { kvm: KvmDeviceWritable, binary: FirecrackerInstalled { binary: "/x/firecracker", version_line: "Firecracker v1.16.1" }, footprint_instruments: instruments } - let no_kvm = FirecrackerHostStanding { kvm: KvmDeviceAbsent, binary: ready.binary, footprint_instruments: instruments } - let unwritable = FirecrackerHostStanding { kvm: KvmDevicePresentNotWritable, binary: ready.binary, footprint_instruments: instruments } - let absent = FirecrackerHostStanding { kvm: KvmDeviceWritable, binary: FirecrackerBinaryAbsent { binary: "/x/firecracker" }, footprint_instruments: instruments } - firecracker_host_ready(standing: ready) - && !firecracker_host_ready(standing: no_kvm) - && !firecracker_host_ready(standing: unwritable) - && !firecracker_host_ready(standing: absent) - && string_contains(s: firecracker_host_standing_text(s: unwritable), pattern: "kvm group membership is a runner-host grant") - && string_contains(s: firecracker_host_standing_text(s: ready), pattern: "standing=ready") +test fn readiness_needs_a_reachable_kvm_device_and_the_modeled_version() -> Bool { + let ready = witness_standing(kvm: KvmDeviceWritable, group: witness_enrolled, binary: witness_installed) + let no_kvm = witness_standing(kvm: KvmDeviceAbsent, group: witness_enrolled, binary: witness_installed) + let unwritable = witness_standing(kvm: KvmDevicePresentNotWritable, group: witness_not_enrolled, binary: witness_installed) + let absent = witness_standing(kvm: KvmDeviceWritable, group: witness_enrolled, binary: FirecrackerBinaryAbsent { binary: "/x/firecracker" }) + firecracker_host_ready(standing: ready, expected_login: witness_login) + && !firecracker_host_ready(standing: no_kvm, expected_login: witness_login) + && !firecracker_host_ready(standing: unwritable, expected_login: witness_login) + && !firecracker_host_ready(standing: absent, expected_login: witness_login) + && string_contains(s: firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: unwritable), pattern: "kvm-group=not-enrolled") + && string_contains(s: firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: ready), pattern: "standing=ready") +} + +// THE CLAIM THE WHOLE GRANT WIRING RESTS ON. A supplementary group enters a process at its exec, so +// the converge that applies the grant observes present-not-writable from its own credentials. If +// readiness required THIS process to open the device, a converged host would refuse forever and the +// mode would report failure for work it had just completed correctly. Enrolment carries it; and the +// carry is not unconditional -- an ABSENT device is not made reachable by any membership. +test fn enrolment_carries_a_host_whose_converging_process_cannot_yet_open_the_device() -> Bool { + let mid_converge = witness_standing(kvm: KvmDevicePresentNotWritable, group: witness_enrolled, binary: witness_installed) + let ungranted = witness_standing(kvm: KvmDevicePresentNotWritable, group: witness_not_enrolled, binary: witness_installed) + let no_device = witness_standing(kvm: KvmDeviceAbsent, group: witness_enrolled, binary: witness_installed) + firecracker_host_ready(standing: mid_converge, expected_login: witness_login) + && !firecracker_host_ready(standing: ungranted, expected_login: witness_login) + && !firecracker_host_ready(standing: no_device, expected_login: witness_login) + && string_contains(s: firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: mid_converge), pattern: "effective for executor incarnations started AFTER this line") +} + +// UNREADABLE IS NOT NOT-ENROLLED. A probe that could not answer must not carry a host to ready, and +// must not render as a missing grant either -- the two send an operator to opposite remedies. +test fn an_unreadable_membership_neither_carries_nor_reads_as_a_missing_grant() -> Bool { + let unreadable = witness_standing( + kvm: KvmDevicePresentNotWritable, + group: KvmGroupEnrolmentUnreadable { login: "ghrunner" as NonEmptyStr, cause: "id -nG exited 1: no such user" }, + binary: witness_installed, + ) + !firecracker_host_ready(standing: unreadable, expected_login: witness_login) + && string_contains(s: firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: unreadable), pattern: "kvm-group=unreadable") + && !string_contains(s: firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: unreadable), pattern: "kvm-group=not-enrolled") +} + +// A WRITABLE DEVICE MAY NOT ANSWER THE NEXT INCARNATION'S QUESTION, AND THIS IS THE CLAIM THAT SAYS +// SO. An earlier revision short-circuited converge readiness on KvmDeviceWritable, which answers a +// question about the NEXT incarnation from the CURRENT one: a process that inherited the group +// before the grant was revoked still holds an open path to the device, so the host read ready while +// the login was no longer enrolled at all -- and the next incarnation, the one the split exists to +// speak for, would fail. Both writable-but-ungranted shapes are covered, because an unreadable +// probe and a revoked grant reach the same wrong answer by different routes. +test fn a_writable_device_does_not_carry_a_login_the_database_no_longer_enrols() -> Bool { + let writable_not_enrolled = witness_standing(kvm: KvmDeviceWritable, group: witness_not_enrolled, binary: witness_installed) + let writable_unreadable = witness_standing( + kvm: KvmDeviceWritable, + group: KvmGroupEnrolmentUnreadable { login: "ghrunner" as NonEmptyStr, cause: "id -nG exited 1: name service unavailable" }, + binary: witness_installed, + ) + !firecracker_host_ready(standing: writable_not_enrolled, expected_login: witness_login) + && !firecracker_host_ready(standing: writable_unreadable, expected_login: witness_login) + && firecracker_host_ready_in_this_process(standing: writable_not_enrolled) + && firecracker_host_ready_in_this_process(standing: writable_unreadable) +} + +// ENROLMENT PLUS PRESENCE HAS A SECOND EXPLANATION, and until the delegation was observed the +// standing could not tell them apart: either the grant landed and this process predates it, or +// /dev/kvm is not handed to the group the grant names at all. Ubuntu's udev default is UPSTREAM +// DESIRED STATE, not a readback of this host, so reading the capability off it would be asserting +// as deduced what was only inferred (DESIGN section 4d). Both contradiction shapes refuse -- a +// device owned by another group, and a device owned by kvm with no group-write -- and so does an +// unreadable probe. +test fn enrolment_does_not_carry_a_device_this_host_does_not_delegate_to_that_group() -> Bool { + let delegated = witness_standing_with_delegation(kvm: KvmDevicePresentNotWritable, group: witness_enrolled, delegation: witness_delegated) + let wrong_group = witness_standing_with_delegation( + kvm: KvmDevicePresentNotWritable, + group: witness_enrolled, + delegation: KvmDeviceDelegationContradicted { expected_group: "kvm" as NonEmptyStr, observed_group: "root" as NonEmptyStr, group_writable: true }, + ) + let no_group_write = witness_standing_with_delegation( + kvm: KvmDevicePresentNotWritable, + group: witness_enrolled, + delegation: KvmDeviceDelegationContradicted { expected_group: "kvm" as NonEmptyStr, observed_group: "kvm" as NonEmptyStr, group_writable: false }, + ) + let unreadable = witness_standing_with_delegation( + kvm: KvmDevicePresentNotWritable, + group: witness_enrolled, + delegation: KvmDeviceDelegationUnreadable { cause: "stat could not read the path" }, + ) + firecracker_host_ready(standing: delegated, expected_login: witness_login) + && !firecracker_host_ready(standing: wrong_group, expected_login: witness_login) + && !firecracker_host_ready(standing: no_group_write, expected_login: witness_login) + && !firecracker_host_ready(standing: unreadable, expected_login: witness_login) + && string_contains(s: firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: wrong_group), pattern: "owned by group root") + && string_contains(s: firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: no_group_write), pattern: "NOT group-writable") + && string_contains(s: firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: unreadable), pattern: "kvm-delegation=unreadable") +} + +// THE REQUIRED RED FOR THE SUBJECT BINDING. Resolving the kernel hostname stops srv1's SPEC being +// applied to srv3; it does not stop srv3 being converged and certified under srv1's name, because the +// run's concurrency group is keyed to the host the operator REQUESTED. So the dispatch's selection +// and the machine's own answer are joined, and a disagreement refuses carrying BOTH names -- an +// operator reading "HostBindingRefused" has to be told which machine answered, or they cannot tell a +// mis-dispatch from a renamed host. +// +// The join is pure, so this claim supplies the two values rather than running a host (DESIGN section +// 3). What it cannot establish by itself is the ORDERING obligation -- that a refusal has performed +// zero install operations -- which holds by construction instead: microvm_host_binding_wet is matched +// first in both wet entries, outside the arm that reaches HOME, the fetch, the install and the +// receipt write. +test fn a_host_that_is_not_the_dispatched_one_refuses_and_names_both() -> Bool { + let mismatch = microvm_host_binding_of(expected: "srv1" as NonEmptyStr, observed: "srv3" as NonEmptyStr) + let agreed = microvm_host_binding_of(expected: "srv1" as NonEmptyStr, observed: "srv1" as NonEmptyStr) + let unrostered = microvm_host_binding_of(expected: "srv-nope" as NonEmptyStr, observed: "srv-nope" as NonEmptyStr) + (match mismatch { + MicrovmHostBindingRefused { cause: c } => + (match c { + HostBindingRefused { expected: e, observed: o } => (e as String) == "srv1" && (o as String) == "srv3" + _ => false + }) + && string_contains(s: microvm_host_binding_refusal_text(c: c), pattern: "NOTHING has been fetched, installed or written") + MicrovmHostBound { spec: _, host: _ } => false + }) + && (match agreed { + MicrovmHostBound { spec: sp, host: h } => (h as String) == "srv1" && (sp.job_user as String) == "ghrunner" + MicrovmHostBindingRefused { cause: _ } => false + }) + && (match unrostered { + MicrovmHostBindingRefused { cause: c } => match c { HostNotRostered { host: _ } => true _ => false } + MicrovmHostBound { spec: _, host: _ } => false + }) +} + +// THE POSITIVE ARMS CARRY A LOGIN AND A GROUP AND READINESS MUST READ THEM. Matching those arms for +// their SHAPE alone lets two true observations compose into a false conclusion: a device delegated to +// one group beside a DIFFERENT login enrolled in a DIFFERENT group reads as ready. Each of the three +// equalities is broken on its own here, because a check that only fires when all three are wrong is +// satisfied by any one of them being enforced. +test fn readiness_checks_the_carried_login_and_group_rather_than_their_shape() -> Bool { + let good = witness_standing(kvm: KvmDevicePresentNotWritable, group: witness_enrolled, binary: witness_installed) + let other_login = witness_standing( + kvm: KvmDevicePresentNotWritable, + group: KvmGroupEnrolled { login: "someone-else" as NonEmptyStr, group: "kvm" as NonEmptyStr }, + binary: witness_installed, + ) + let other_enrolled_group = witness_standing( + kvm: KvmDevicePresentNotWritable, + group: KvmGroupEnrolled { login: "ghrunner" as NonEmptyStr, group: "libvirt" as NonEmptyStr }, + binary: witness_installed, + ) + let other_delegated_group = witness_standing_with_delegation( + kvm: KvmDevicePresentNotWritable, + group: witness_enrolled, + delegation: KvmDeviceDelegated { group: "libvirt" as NonEmptyStr }, + ) + firecracker_host_ready(standing: good, expected_login: witness_login) + && !firecracker_host_ready(standing: other_login, expected_login: witness_login) + && !firecracker_host_ready(standing: other_enrolled_group, expected_login: witness_login) + && !firecracker_host_ready(standing: other_delegated_group, expected_login: witness_login) +} + +// THE BOOT PROBE ASKS A DIFFERENT QUESTION AND MUST GET A DIFFERENT ANSWER. It opens /dev/kvm in +// its own process, so an enrolment it did not inherit does not help it; gating it on converge +// readiness would make it attempt a boot it cannot perform and report a VMM failure as an IMAGE +// verdict. +test fn the_in_process_predicate_refuses_exactly_where_the_converge_predicate_carries() -> Bool { + let mid_converge = witness_standing(kvm: KvmDevicePresentNotWritable, group: witness_enrolled, binary: witness_installed) + let picked_up = witness_standing(kvm: KvmDeviceWritable, group: witness_enrolled, binary: witness_installed) + firecracker_host_ready(standing: mid_converge, expected_login: witness_login) + && !firecracker_host_ready_in_this_process(standing: mid_converge) + && firecracker_host_ready_in_this_process(standing: picked_up) + && !firecracker_host_ready_in_this_process(standing: witness_standing(kvm: KvmDeviceWritable, group: witness_enrolled, binary: FirecrackerBinaryAbsent { binary: "/x/firecracker" })) +} + +// THE READBACK MATCHES WHOLE NAMES. `id -nG` prints names separated by single spaces, and a +// substring test would accept "kvm-admin" -- an unrelated group -- as evidence that the executor may +// open /dev/kvm. +test fn the_group_readback_matches_a_whole_name_and_not_a_prefix() -> Bool { + group_names_contain(output: "ghrunner docker kvm\n", group: "kvm" as NonEmptyStr) + && group_names_contain(output: "kvm", group: "kvm" as NonEmptyStr) + && !group_names_contain(output: "ghrunner kvm-admin libvirt\n", group: "kvm" as NonEmptyStr) + && !group_names_contain(output: "", group: "kvm" as NonEmptyStr) } // RED control: a binary that answers another version is a mismatch, never installed. @@ -132,9 +333,9 @@ test fn host_readiness_is_independent_of_the_footprint_instruments() -> Bool { ], } let installed = FirecrackerInstalled { binary: "/x/firecracker", version_line: "Firecracker v1.16.1" } - firecracker_host_ready(standing: FirecrackerHostStanding { kvm: KvmDeviceWritable, binary: installed, footprint_instruments: bare }) - && firecracker_host_ready(standing: FirecrackerHostStanding { kvm: KvmDeviceWritable, binary: installed, footprint_instruments: equipped }) - && !firecracker_host_ready(standing: FirecrackerHostStanding { kvm: KvmDeviceAbsent, binary: installed, footprint_instruments: equipped }) + firecracker_host_ready(standing: FirecrackerHostStanding { kvm: KvmDeviceWritable, kvm_group: witness_enrolled, kvm_delegation: witness_delegated, binary: installed, footprint_instruments: bare }, expected_login: witness_login) + && firecracker_host_ready(standing: FirecrackerHostStanding { kvm: KvmDeviceWritable, kvm_group: witness_enrolled, kvm_delegation: witness_delegated, binary: installed, footprint_instruments: equipped }, expected_login: witness_login) + && !firecracker_host_ready(standing: FirecrackerHostStanding { kvm: KvmDeviceAbsent, kvm_group: witness_enrolled, kvm_delegation: witness_delegated, binary: installed, footprint_instruments: equipped }, expected_login: witness_login) } // THE VERSION GATE, WHICH NOTHING EXERCISED UNTIL NOW AND WHICH WAS WRONG THE WHOLE TIME. Before @@ -186,6 +387,8 @@ test fn the_memory_peak_threshold_admits_254_and_refuses_253() -> Bool { test fn the_standing_text_reports_the_instruments() -> Bool { let equipped = FirecrackerHostStanding { kvm: KvmDeviceWritable, + kvm_group: witness_enrolled, + kvm_delegation: witness_delegated, binary: FirecrackerInstalled { binary: "/x/firecracker", version_line: "Firecracker v1.16.1" }, footprint_instruments: FootprintInstrumentStanding { instruments: [ @@ -195,7 +398,7 @@ test fn the_standing_text_reports_the_instruments() -> Bool { ], }, } - let text = firecracker_host_standing_text(s: equipped) + let text = firecracker_host_standing_text(host: witness_host, expected_login: witness_login, s: equipped) string_contains(s: text, pattern: "gnu-time-max-rss=present") && string_contains(s: text, pattern: "cgroup-memory-peak=absent") && string_contains(s: text, pattern: "systemd-memory-peak=unreadable(")