Apply the host-wide executor grants from the converge spine, and read the kvm grant back

.github/workflows/fleet-converge.yml

@@ -11,7 +11,7 @@ on:
         options: [srv1, srv2, srv3, srv4]
         type: choice
       mode:
-        description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown
+        description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown
         required: true
         options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot, mtcollins1_fan_observe]
         type: choice
@@ -625,6 +625,8 @@ jobs:
           ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
           "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_microvm_host_ready.dag --function runner_microvm_host_converge_wet
           cat "$ROOT/target/runner-microvm-host-standing.txt"
+        env:
+          FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }}
         if: github.event.inputs.mode == 'microvm_host_converge'
         timeout-minutes: 30
       - name: Upload runner micro-VM host standing receipt
@@ -643,6 +645,8 @@ jobs:
           ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
           "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_microvm_boot_probe.dag --function runner_microvm_boot_probe_wet
           cat "$ROOT/target/runner-microvm-boot-probe.txt"
+        env:
+          FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }}
         if: github.event.inputs.mode == 'microvm_boot_probe'
         timeout-minutes: 30
       - name: Upload runner micro-VM boot probe console receipt

dag/extdeps/access/posix.dag

@@ -3,6 +3,7 @@ module extdeps.access.posix
 import extdeps.external_authority { ExternalAuthority }
 import extdeps.uri { Uri, Https }
 import std.types { Bool, Int, NonEmptyStr, String }
+import std.algebra { trim }
 data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
   uri: Uri {
     scheme: Https
@@ -129,3 +130,71 @@ fn file_mode_octal(mode: FileMode) -> String {
     to_string(permission_bits_octal_digit(bits: mode.other)),
   ], "")
 }
+
+// THE SAME RELATION READ BACKWARD, which is why it lives here and not at the call site that wanted
+// it. file_mode_octal above renders a FileMode as chmod's numeric spelling; this reads that
+// spelling back. One grammar, two directions (DESIGN section 4) -- a parser authored beside a
+// consumer would be a second, informal authority for the digit layout, and the two would drift the
+// first time a special bit mattered.
+//
+// THREE DIGITS AND FOUR ARE THE SAME NUMBER. stat's %a prints the special digit only when it is
+// non-zero, so "660" and "0660" are one mode spelled two ways; reading the text as an integer and
+// slicing digits positionally from the LEAST significant end handles both without a length branch
+// and without the leading-zero question ever arising.
+//
+// EVERY REFUSAL ARM IS A REFUSAL AND NOT A ZERO. An unparseable field, a negative number, a digit
+// above 7 or a value wider than four digits means the observation did not happen; folding any of
+// them into a FileMode would hand a consumer a permission set nothing observed, and the consumer
+// here is deciding whether a device is delegated to a group.
+fn permission_bits_of_octal_digit(digit: Int) -> PermissionBits? {
+  if digit < 0 || digit > 7 {
+    none
+  } else {
+    Present {
+      value: PermissionBits {
+        read: (digit / 4) % 2 == 1,
+        write: (digit / 2) % 2 == 1,
+        execute: digit % 2 == 1,
+      },
+    }
+  }
+}
+
+fn file_mode_of_octal_text(text: String) -> FileMode? {
+  match parse_int(s: trim(s: text)) {
+    Absent => none
+    Present { value: n } =>
+      if n < 0 || n > 7777 {
+        none
+      } else {
+        match permission_bits_of_octal_digit(digit: (n / 100) % 10) {
+          Absent => none
+          Present { value: owner } =>
+            match permission_bits_of_octal_digit(digit: (n / 10) % 10) {
+              Absent => none
+              Present { value: group } =>
+                match permission_bits_of_octal_digit(digit: n % 10) {
+                  Absent => none
+                  Present { value: other } => {
+                    let special = (n / 1000) % 10
+                    if special < 0 || special > 7 {
+                      none
+                    } else {
+                      Present {
+                        value: FileMode {
+                          owner: owner,
+                          group: group,
+                          other: other,
+                          setuid: (special / 4) % 2 == 1,
+                          setgid: (special / 2) % 2 == 1,
+                          sticky: special % 2 == 1,
+                        },
+                      }
+                    }
+                  }
+                }
+            }
+        }
+      }
+  }
+}

dag/extdeps/exec/command.dag

@@ -143,6 +143,7 @@ fn argv_command(program: NonEmptyStr, arguments: List<String>) -> ArgvCommand
     decl_ref(module_path: "extdeps.posix.test_utility", decl_name: "test_exists_command"),
     decl_ref(module_path: "extdeps.posix.test_utility", decl_name: "test_writable_command"),
     decl_ref(module_path: "extdeps.posix.test_utility", decl_name: "test_executable_command"),
+    decl_ref(module_path: "extdeps.tools.id", decl_name: "id_group_names_command"),
     decl_ref(module_path: "extdeps.virtualization.firecracker", decl_name: "firecracker_version_command"),
     decl_ref(module_path: "extdeps.virtualization.firecracker", decl_name: "firecracker_run_config_command"),
     decl_ref(module_path: "extdeps.virtualization.firecracker", decl_name: "firecracker_jailer_run_command"),

dag/extdeps/tools/coreutils_stat.dag

@@ -2,7 +2,7 @@ module extdeps.tools.coreutils_stat
 
 import extdeps.external_authority { ExternalAuthority }
 import extdeps.uri { Uri, Https }
-import extdeps.access.posix { FileOwnership }
+import extdeps.access.posix { FileOwnership, FileMode, file_mode_of_octal_text }
 import std.types { Bool, Int, List, NonEmptyStr, Unit }
 import std.string_type { String }
 import std.algebra { trim }
@@ -29,6 +29,17 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
 // four fields can contain one: uid and gid are decimal, and a POSIX login name is restricted to the
 // portable filename character set, which excludes the colon. That is a property of the fields, not
 // an assumption about the data, which is why the separator is not a configurable parameter.
+//
+// THE MODE IS ITS OWN PROBE, NOT A FIFTH FIELD ON THE OWNERSHIP ONE. The colon separator above is
+// safe because none of those four fields can contain one, which is a property of the fields; the
+// mode is a different question with a different failure -- a path can be readable for ownership and
+// the answer still be that its permission bits are what nobody expected -- and joining them would
+// make one unreadable half destroy the other. It is also the shape this module's sibling
+// extdeps.tools.stat already commits to: one token per probe.
+//
+// %a IS THE ACCESS BITS IN OCTAL WITH NO INTERPRETATION. extdeps.access.posix owns what those digits
+// MEAN and reads them back through file_mode_of_octal_text; PathMode's whole job is to fetch the
+// spelling.
 service coreutils.Stat {
   operation PathOwnership {
     input { path: NonEmptyStr }
@@ -43,6 +54,20 @@ service coreutils.Stat {
       nonzero => String "stat could not read the path"
     }
   }
+
+  operation PathMode {
+    input { path: NonEmptyStr }
+    output {
+      mode_text: String from "stdout"
+      success: Bool from "exit_success"
+    }
+    readonly
+    transport shell { argv: ["stat", "-c", "%a", "--", "{path}"] }
+    exit {
+      0 => Unit
+      nonzero => String "stat could not read the path"
+    }
+  }
 }
 
 // AN UNREADABLE PATH IS A REFUSAL, NOT AN OWNERSHIP OF NOBODY, for the same reason an unreadable
@@ -133,3 +158,24 @@ fn coreutils_stat_ownership_record(
     }
   }
 }
+
+// THE MODE OBSERVATION, WITH THE SAME REFUSAL DISCIPLINE AS THE OWNERSHIP ONE ABOVE. A path whose
+// permission bits could not be read must not arrive at a consumer as a mode with everything false,
+// because "nobody may" and "we did not look" are the two answers a delegation check must keep
+// apart -- the first says the host is misconfigured, the second says the probe is.
+type PathModeObservation
+  = PathModeObserved { path: NonEmptyStr, mode: FileMode }
+  | PathModeUnobservable { path: NonEmptyStr, reason: String }
+
+fn coreutils_stat_path_mode(path: NonEmptyStr) -> PathModeObservation {
+  let observed = coreutils.Stat.PathMode(path: path)
+  if observed.success == false {
+    PathModeUnobservable { path: path, reason: "stat could not read the path" }
+  } else {
+    match file_mode_of_octal_text(text: observed.mode_text) {
+      Absent =>
+        PathModeUnobservable { path: path, reason: join(["stat printed a mode this cannot read as octal permission bits: '", trim(s: observed.mode_text), "'"], "") }
+      Present { value: mode } => PathModeObserved { path: path, mode: mode }
+    }
+  }
+}

dag/extdeps/tools/id.dag

@@ -4,6 +4,7 @@ import std.types { NonEmptyStr, List, Bool, Unit }
 import std.string_type { String }
 import extdeps.external_authority { ExternalAuthority }
 import extdeps.uri { Uri, Https }
+import extdeps.exec.command { ArgvCommand, argv_command }
 
 data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
   uri: Uri {
@@ -18,12 +19,29 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
 // the grant authorizes -- and gunbc.host_effect_realize needs both spellings for the same tool.
 data id_binary_path: NonEmptyStr = "/usr/bin/id"
 
+// THE PATH-RESOLVED SPELLING, DECLARED ONCE. The argv rows below and the command builder all name
+// the same program, and spelling it three times would be three places one word could drift.
+data id_program: NonEmptyStr = "id"
+
 fn id_uid_argv() -> List<String> {
-  ["id", "-u"]
+  [id_program as String, "-u"]
 }
 
 fn id_gid_argv() -> List<String> {
-  ["id", "-g"]
+  [id_program as String, "-g"]
+}
+
+// -nG NAMES THE USER EXPLICITLY, AND THAT IS THE WHOLE REASON THIS BUILDER TAKES ONE. id(1) with an
+// operand answers from the account database; id(1) with no operand answers from the CALLING
+// PROCESS's own credentials, which were established at its exec and never change afterwards. A
+// caller asking whether a supplementary group grant has landed must ask the database -- the running
+// process cannot have picked the group up, so the no-operand form would answer the older question
+// and read as a refusal of a grant that is in fact installed.
+//
+// -n asks for names rather than numeric gids, because the group a caller holds is named by the
+// authority it came from (a udev rule naming "kvm"), not by a gid that varies per host.
+fn id_group_names_command(user: NonEmptyStr) -> ArgvCommand {
+  argv_command(program: id_program, arguments: ["-nG", "--", user as String])
 }
 
 service os.Id {