microVM guest runner bootstrap

dag/extdeps/github/actions_runner.dag

@@ -174,6 +174,34 @@ data actions_runner_jit_settings_ephemeral_member: String = "Ephemeral"
 
 data actions_runner_base_dir: String = "/opt/actions-runner"
 
+// THE LISTENER TAKES ANY COMMAND-LINE ARGUMENT FROM ITS ENVIRONMENT, AND DELETES IT ONCE READ.
+// src/Runner.Listener/CommandSettings.cs (v2.337.0) scans the process environment for variables
+// prefixed `ACTIONS_RUNNER_INPUT_`, records each suffix as the argument of that name, and calls
+// SetEnvironmentVariable(fullKey, null) on it -- so a job the listener spawns does not inherit it.
+// GetJitConfig() reads the argument named `jitconfig` through that same lookup. These are upstream
+// spellings, cited rather than chosen. Both the prefix test and the argument dictionary compare with
+// StringComparison.OrdinalIgnoreCase, so the variable is the prefix followed by the argument name
+// as cited -- no second, uppercased spelling of the same argument is needed or declared.
+data actions_runner_input_env_prefix: String = "ACTIONS_RUNNER_INPUT_"
+data actions_runner_jitconfig_argument: String = "jitconfig"
+
+fn actions_runner_jitconfig_env_name() -> String {
+  concat(actions_runner_input_env_prefix, actions_runner_jitconfig_argument)
+}
+
+// `run.sh` is a LOOP AROUND THE LISTENER, NOT THE LISTENER. src/Misc/layoutroot/run.sh (v2.337.0)
+// re-executes run-helper.sh whenever it returns 2 (Constants.Runner.ReturnCode.RetryableError) and
+// exits otherwise. The listener itself is `bin/Runner.Listener run`, which is what run-helper.sh
+// execs; a consumer that invokes the listener directly has no relaunch loop at all, which is why no
+// return-code row is declared here: nothing in this corpus branches on one.
+data actions_runner_listener_relative_path: String = "/bin/Runner.Listener"
+data actions_runner_listener_run_verb: String = "run"
+
+// THE LISTENER'S OWN READINESS LINE. src/Runner.Listener/Runner.cs writes
+// `{DateTime.UtcNow:u}: Listening for Jobs` to its terminal once its session is established. The
+// timestamp prefix varies, so the stable part is this suffix.
+data actions_runner_listening_line_suffix: String = "Listening for Jobs"
+
 // `config.sh` writes `.runner` beside the binaries when a runner registers with GitHub, and
 // `config.sh remove` deletes it; the file's presence is the runner's own record that this tree is
 // registered. It is the registration signal a fleet observer reads, cited here because it is the

dag/extdeps/systemd/unit_file.dag

@@ -125,13 +125,76 @@ type SystemdUnitDirective
   | Wants { target: NonEmptyStr }
   | Requires { target: NonEmptyStr }
   | StartLimitIntervalSec { duration: NonEmptyStr }
+  | SuccessAction { action: SystemdUnitStateAction }
+  | FailureAction { action: SystemdUnitStateAction }
+
+// systemd.unit(5) (v255), SuccessAction=/FailureAction=: what the MANAGER does when the unit stops
+// cleanly or enters the failed state. The value set is upstream's and CLOSED, so it is a coproduct
+// here, exactly as ExitType= and KillMode= are: a misspelled action has no constructor, rather than
+// rendering a unit systemd refuses at load inside a guest nobody is watching. The -force strengths
+// skip the orderly stop of other units but still kill every process and unmount; -immediate skips
+// even that.
+type SystemdUnitStateAction
+  = UnitActionNone
+  | UnitActionReboot
+  | UnitActionRebootForce
+  | UnitActionRebootImmediate
+  | UnitActionPoweroff
+  | UnitActionPoweroffForce
+  | UnitActionPoweroffImmediate
+  | UnitActionExit
+  | UnitActionExitForce
+  | UnitActionSoftReboot
+  | UnitActionSoftRebootForce
+  | UnitActionKexec
+  | UnitActionKexecForce
+  | UnitActionKexecImmediate
+  | UnitActionHalt
+  | UnitActionHaltForce
+  | UnitActionHaltImmediate
+
+fn systemd_unit_state_action_wire(action: SystemdUnitStateAction) -> String {
+  match action {
+    UnitActionNone => "none"
+    UnitActionReboot => "reboot"
+    UnitActionRebootForce => "reboot-force"
+    UnitActionRebootImmediate => "reboot-immediate"
+    UnitActionPoweroff => "poweroff"
+    UnitActionPoweroffForce => "poweroff-force"
+    UnitActionPoweroffImmediate => "poweroff-immediate"
+    UnitActionExit => "exit"
+    UnitActionExitForce => "exit-force"
+    UnitActionSoftReboot => "soft-reboot"
+    UnitActionSoftRebootForce => "soft-reboot-force"
+    UnitActionKexec => "kexec"
+    UnitActionKexecForce => "kexec-force"
+    UnitActionKexecImmediate => "kexec-immediate"
+    UnitActionHalt => "halt"
+    UnitActionHaltForce => "halt-force"
+    UnitActionHaltImmediate => "halt-immediate"
+  }
+}
+
+// systemd.service(5), "Command lines": `${NAME}` in an Exec*= line is replaced by that environment
+// variable's value as one argument. The environment includes what systemd itself sets for the
+// spawned process -- for ExecStop=/ExecStopPost=, $EXIT_CODE and $EXIT_STATUS (systemd.exec(5),
+// "Environment Variables in Spawned Processes"). The dollar and brace are joined rather than written
+// adjacently because `${` inside a .dag string literal is .dag's own interpolation.
+fn systemd_environment_reference(name: NonEmptyStr) -> String {
+  join(["$", "{", name as String, "}"], "")
+}
+
+data systemd_exit_code_variable: NonEmptyStr = "EXIT_CODE" as NonEmptyStr
+data systemd_exit_status_variable: NonEmptyStr = "EXIT_STATUS" as NonEmptyStr
 
 type SystemdServiceDirective
   = ServiceType { mode: NonEmptyStr }
   | ExecStartPre { command: NonEmptyStr }
   | ExecStart { command: NonEmptyStr }
+  | ExecStartPost { command: NonEmptyStr }
   | ExecStop { command: NonEmptyStr }
   | ExecStopPost { command: NonEmptyStr }
+  | ExecStopPostIgnoringFailure { command: NonEmptyStr }
   | ServiceStandardOutput { stream: SystemdOutputStream }
   | ServiceStandardError { stream: SystemdOutputStream }
   | Restart { policy: NonEmptyStr }
@@ -532,6 +595,8 @@ fn systemd_unit_directive_line(directive: SystemdUnitDirective) -> String {
     Wants { target: v } => join(["Wants=", v as String], "")
     Requires { target: v } => join(["Requires=", v as String], "")
     StartLimitIntervalSec { duration: v } => join(["StartLimitIntervalSec=", v as String], "")
+    SuccessAction { action: a } => join(["SuccessAction=", systemd_unit_state_action_wire(action: a)], "")
+    FailureAction { action: a } => join(["FailureAction=", systemd_unit_state_action_wire(action: a)], "")
   }
 }
 
@@ -540,8 +605,10 @@ fn systemd_service_directive_line(directive: SystemdServiceDirective) -> String
     ServiceType { mode: v } => join(["Type=", v as String], "")
     ExecStartPre { command: v } => join(["ExecStartPre=", v as String], "")
     ExecStart { command: v } => join(["ExecStart=", v as String], "")
+    ExecStartPost { command: v } => join(["ExecStartPost=", v as String], "")
     ExecStop { command: v } => join(["ExecStop=", v as String], "")
     ExecStopPost { command: v } => join(["ExecStopPost=", v as String], "")
+    ExecStopPostIgnoringFailure { command: v } => join(["ExecStopPost=-", v as String], "")
     ServiceStandardOutput { stream: v } =>
       join(["StandardOutput=", systemd_output_stream_wire(stream: v)], "")
     ServiceStandardError { stream: v } =>

dag/extdeps/virtualization/firecracker.dag

@@ -3,6 +3,7 @@ module extdeps.virtualization.firecracker
 import extdeps.exec.command { ArgvCommand, argv_command }
 import std.types { String, NonEmptyStr, Int, Bool, List, FilePath }
 import std.disposition { Disposition, Terminal }
+import std.measure { ByteSize, byte_size }
 import extdeps.external_authority { ExternalAuthority }
 import extdeps.uri { Uri, Https }
 import extdeps.languages.json.emit {
@@ -113,6 +114,13 @@ type FirecrackerBootSource {
   boot_args: NonEmptyStr
 }
 
+// A DRIVE IS EXPOSED TO THE GUEST IN WHOLE 512-BYTE SECTORS, AND A PARTIAL TAIL IS DROPPED.
+// firecracker v1.16.1 src/vmm/src/devices/virtio/block/virtio/mod.rs: SECTOR_SIZE = 1 << SECTOR_SHIFT
+// (9). device.rs, on opening the backing file: "If the image is not a multiple of the sector size,
+// the tail bits are not exposed" -- it logs a warning and the guest sees the size rounded down. A
+// backing file whose content matters to its last byte must therefore be a whole number of these.
+data firecracker_block_sector_size: ByteSize = byte_size(count: 512)
+
 // A drive is identified by a caller-chosen id and is EITHER the root device or not. Upstream lets
 // both flags be set independently; modelling is_root_device as a Bool beside is_read_only keeps
 // the upstream shape rather than inventing a coproduct the API does not have.
@@ -159,6 +167,42 @@ fn firecracker_config_has_single_root(config: FirecrackerVmConfig) -> Bool {
   firecracker_root_drive_count(config: config) == 1
 }
 
+// WHICH GUEST DEVICE A DRIVE BECOMES IS ITS POSITION IN `drives`. Firecracker attaches the
+// virtio-blk devices in the order of the configuration's drives array, and the Linux virtio_blk
+// driver names them vda, vdb, ... in the order it probes them (virtblk_name_format). On this fleet's
+// aarch64 hosts that order was observed, not only read: the Mt. Collins run
+// (gunbc.runner.runner_first_job_receipt execution_chain) booted root=/dev/vda and read its second
+// drive's credential off /dev/vdb. Only the first 26 drives are named here; a drive beyond them, or
+// one that is not attached, has no name rather than a guessed one.
+type DriveIndexScan {
+  next: Int
+  found: Int?
+}
+
+data firecracker_guest_block_letters: String = "abcdefghijklmnopqrstuvwxyz"
+
+fn firecracker_guest_block_device_of_index(index: Int) -> String? {
+  if index >= string_length(s: firecracker_guest_block_letters) {
+    none
+  } else {
+    Present { value: concat("/dev/vd", substring(s: firecracker_guest_block_letters, start: index, end: index + 1)) }
+  }
+}
+
+fn firecracker_drive_guest_device(drives: List<FirecrackerDrive>, drive_id: NonEmptyStr) -> String? {
+  let scan = fold(drives, init: DriveIndexScan { next: 0, found: none }, f: fn(acc, d) {
+    let found = match acc.found {
+      Present { value: i } => Present { value: i }
+      Absent => if d.drive_id == drive_id { Present { value: acc.next } } else { none }
+    }
+    DriveIndexScan { next: acc.next + 1, found: found }
+  })
+  match scan.found {
+    Absent => none
+    Present { value: i } => firecracker_guest_block_device_of_index(index: i)
+  }
+}
+
 // --- the configuration FILE, which is upstream's own wire shape and not a rendering of ours ---
 
 // FIRECRACKER READS ONE JSON DOCUMENT WHOSE KEYS ARE HYPHENATED WHERE THE API PATHS ARE. The

dag/gunbc/host/host_hygiene_liveness_observe.dag

@@ -5,6 +5,7 @@ import std.disposition { Disposition, Terminal }
 import std.realization_reconcile { ShowEffectiveRead, ReadObserved, ReadAbsent, Reconciliation }
 import extdeps.clock { clock_unix_millis_read, ClockUnixMillisObserved, ClockUnixMillisRefused }
 import extdeps.systemd.journalctl
+import extdeps.github.actions_runner { actions_runner_listening_line_suffix }
 import gunbc.systemctl_list_units { systemctl_list_units_all_states }
 import gunbc.host_converge { gunbc_runner_all_slots_unit_glob }
 import gunbc.host_hygiene_liveness {
@@ -23,14 +24,12 @@ data gunbc_host_hygiene_liveness_observe_scope_disposition: Disposition = Termin
   reason: "The OBSERVE half of the liveness vertical (brief: split observe from mutate). Runs LocalShell only: unit enumeration reuses gunbc.systemctl_list_units's all-states seam (the same one-op, different-binding move the reaper observe module uses — a listener slot is worth checking whether active, inactive, or failed, since a failed slot is exactly the outage this exists to catch), and the journal read goes through the new extdeps.systemd.Journalctl.UnitLog typed op rather than a piped `journalctl | grep | tail` shell body. Marker-matching (last 'Listening for Jobs' line) and freshness (age against extdeps.clock's typed Clock.UnixMillis read) are pure .dag folds over one typed stdout read, replacing what the deleted host_hygiene_liveness_script.dag did as one hand-written pipeline in a single shell body."
 }
 
-data host_hygiene_listening_marker: String = "Listening for Jobs"
-
 fn host_hygiene_journal_lines(stdout: String) -> List<String> {
   filter(map(split(s: trim(stdout), delimiter: "\n"), l => trim(l)), l => l != "")
 }
 
 fn host_hygiene_journal_matching_lines(stdout: String) -> List<String> {
-  filter(host_hygiene_journal_lines(stdout: stdout), l => string_contains(s: l, pattern: host_hygiene_listening_marker))
+  filter(host_hygiene_journal_lines(stdout: stdout), l => string_contains(s: l, pattern: actions_runner_listening_line_suffix))
 }
 
 // journalctl -o short-unix prints "SECONDS.MICROSECONDS host proc[pid]: msg" — the first

dag/gunbc/runner/runner_attempt_launch.dag

@@ -80,7 +80,7 @@ data jit_mint_http_realization_frontier: DissolutionCondition = unbound_dissolut
 // firecracker_jailer_chroot_root. SUFFICIENT FOR deleting this row. NOT satisfied by
 // creating an empty jit.bin beside the workspace.
 data jail_jit_device_staging_frontier: DissolutionCondition = unbound_dissolution(
-  description: "NAMED CONSUMER: gunbc.runner_attempt_launch plan_authorized_steps. TRIGGER: gunbc#10923 mint supplies encoded_jit_config and this planner writes that blob at firecracker_jailer_chroot_root/.../jit.bin so the in-jail path /jit.bin names a file the VMM can open. SUFFICIENT FOR deleting this row. NOT satisfied by an empty placeholder beside work.img." as NonEmptyStr,
+  description: "NAMED CONSUMER: gunbc.runner_attempt_launch plan_authorized_steps. TRIGGER: gunbc#10923 mint supplies encoded_jit_config and this planner writes gunbc.runner_microvm jit_drive_content(encoded_jit_config) -- the sector-padded environment-file bytes the guest agent reads from the device jit_drive_guest_device resolves, never the bare blob -- at firecracker_jailer_chroot_root/.../jit.bin so the in-jail path /jit.bin names a file the VMM can open. SUFFICIENT FOR deleting this row. NOT satisfied by an empty placeholder beside work.img." as NonEmptyStr,
 )
 
 // SHADOW MEANS SHADOW. No fleet-converge entry, no dual dispatch, no "use Y when available".