diff --git a/dag/extdeps/github/actions_runner.dag b/dag/extdeps/github/actions_runner.dag index dcaca47741e..17194c42280 100644 --- a/dag/extdeps/github/actions_runner.dag +++ b/dag/extdeps/github/actions_runner.dag @@ -174,6 +174,34 @@ data actions_runner_jit_settings_ephemeral_member: String = "Ephemeral" data actions_runner_base_dir: String = "/opt/actions-runner" +// THE LISTENER TAKES ANY COMMAND-LINE ARGUMENT FROM ITS ENVIRONMENT, AND DELETES IT ONCE READ. +// src/Runner.Listener/CommandSettings.cs (v2.337.0) scans the process environment for variables +// prefixed `ACTIONS_RUNNER_INPUT_`, records each suffix as the argument of that name, and calls +// SetEnvironmentVariable(fullKey, null) on it -- so a job the listener spawns does not inherit it. +// GetJitConfig() reads the argument named `jitconfig` through that same lookup. These are upstream +// spellings, cited rather than chosen. Both the prefix test and the argument dictionary compare with +// StringComparison.OrdinalIgnoreCase, so the variable is the prefix followed by the argument name +// as cited -- no second, uppercased spelling of the same argument is needed or declared. +data actions_runner_input_env_prefix: String = "ACTIONS_RUNNER_INPUT_" +data actions_runner_jitconfig_argument: String = "jitconfig" + +fn actions_runner_jitconfig_env_name() -> String { + concat(actions_runner_input_env_prefix, actions_runner_jitconfig_argument) +} + +// `run.sh` is a LOOP AROUND THE LISTENER, NOT THE LISTENER. src/Misc/layoutroot/run.sh (v2.337.0) +// re-executes run-helper.sh whenever it returns 2 (Constants.Runner.ReturnCode.RetryableError) and +// exits otherwise. The listener itself is `bin/Runner.Listener run`, which is what run-helper.sh +// execs; a consumer that invokes the listener directly has no relaunch loop at all, which is why no +// return-code row is declared here: nothing in this corpus branches on one. +data actions_runner_listener_relative_path: String = "/bin/Runner.Listener" +data actions_runner_listener_run_verb: String = "run" + +// THE LISTENER'S OWN READINESS LINE. src/Runner.Listener/Runner.cs writes +// `{DateTime.UtcNow:u}: Listening for Jobs` to its terminal once its session is established. The +// timestamp prefix varies, so the stable part is this suffix. +data actions_runner_listening_line_suffix: String = "Listening for Jobs" + // `config.sh` writes `.runner` beside the binaries when a runner registers with GitHub, and // `config.sh remove` deletes it; the file's presence is the runner's own record that this tree is // registered. It is the registration signal a fleet observer reads, cited here because it is the diff --git a/dag/extdeps/systemd/unit_file.dag b/dag/extdeps/systemd/unit_file.dag index 54e15ca194b..c20d5568b0f 100644 --- a/dag/extdeps/systemd/unit_file.dag +++ b/dag/extdeps/systemd/unit_file.dag @@ -125,13 +125,76 @@ type SystemdUnitDirective | Wants { target: NonEmptyStr } | Requires { target: NonEmptyStr } | StartLimitIntervalSec { duration: NonEmptyStr } + | SuccessAction { action: SystemdUnitStateAction } + | FailureAction { action: SystemdUnitStateAction } + +// systemd.unit(5) (v255), SuccessAction=/FailureAction=: what the MANAGER does when the unit stops +// cleanly or enters the failed state. The value set is upstream's and CLOSED, so it is a coproduct +// here, exactly as ExitType= and KillMode= are: a misspelled action has no constructor, rather than +// rendering a unit systemd refuses at load inside a guest nobody is watching. The -force strengths +// skip the orderly stop of other units but still kill every process and unmount; -immediate skips +// even that. +type SystemdUnitStateAction + = UnitActionNone + | UnitActionReboot + | UnitActionRebootForce + | UnitActionRebootImmediate + | UnitActionPoweroff + | UnitActionPoweroffForce + | UnitActionPoweroffImmediate + | UnitActionExit + | UnitActionExitForce + | UnitActionSoftReboot + | UnitActionSoftRebootForce + | UnitActionKexec + | UnitActionKexecForce + | UnitActionKexecImmediate + | UnitActionHalt + | UnitActionHaltForce + | UnitActionHaltImmediate + +fn systemd_unit_state_action_wire(action: SystemdUnitStateAction) -> String { + match action { + UnitActionNone => "none" + UnitActionReboot => "reboot" + UnitActionRebootForce => "reboot-force" + UnitActionRebootImmediate => "reboot-immediate" + UnitActionPoweroff => "poweroff" + UnitActionPoweroffForce => "poweroff-force" + UnitActionPoweroffImmediate => "poweroff-immediate" + UnitActionExit => "exit" + UnitActionExitForce => "exit-force" + UnitActionSoftReboot => "soft-reboot" + UnitActionSoftRebootForce => "soft-reboot-force" + UnitActionKexec => "kexec" + UnitActionKexecForce => "kexec-force" + UnitActionKexecImmediate => "kexec-immediate" + UnitActionHalt => "halt" + UnitActionHaltForce => "halt-force" + UnitActionHaltImmediate => "halt-immediate" + } +} + +// systemd.service(5), "Command lines": `${NAME}` in an Exec*= line is replaced by that environment +// variable's value as one argument. The environment includes what systemd itself sets for the +// spawned process -- for ExecStop=/ExecStopPost=, $EXIT_CODE and $EXIT_STATUS (systemd.exec(5), +// "Environment Variables in Spawned Processes"). The dollar and brace are joined rather than written +// adjacently because `${` inside a .dag string literal is .dag's own interpolation. +fn systemd_environment_reference(name: NonEmptyStr) -> String { + join(["$", "{", name as String, "}"], "") +} + +data systemd_exit_code_variable: NonEmptyStr = "EXIT_CODE" as NonEmptyStr +data systemd_exit_status_variable: NonEmptyStr = "EXIT_STATUS" as NonEmptyStr type SystemdServiceDirective = ServiceType { mode: NonEmptyStr } | ExecStartPre { command: NonEmptyStr } | ExecStart { command: NonEmptyStr } + | ExecStartPost { command: NonEmptyStr } | ExecStop { command: NonEmptyStr } | ExecStopPost { command: NonEmptyStr } + | ExecStopPostIgnoringFailure { command: NonEmptyStr } | ServiceStandardOutput { stream: SystemdOutputStream } | ServiceStandardError { stream: SystemdOutputStream } | Restart { policy: NonEmptyStr } @@ -532,6 +595,8 @@ fn systemd_unit_directive_line(directive: SystemdUnitDirective) -> String { Wants { target: v } => join(["Wants=", v as String], "") Requires { target: v } => join(["Requires=", v as String], "") StartLimitIntervalSec { duration: v } => join(["StartLimitIntervalSec=", v as String], "") + SuccessAction { action: a } => join(["SuccessAction=", systemd_unit_state_action_wire(action: a)], "") + FailureAction { action: a } => join(["FailureAction=", systemd_unit_state_action_wire(action: a)], "") } } @@ -540,8 +605,10 @@ fn systemd_service_directive_line(directive: SystemdServiceDirective) -> String ServiceType { mode: v } => join(["Type=", v as String], "") ExecStartPre { command: v } => join(["ExecStartPre=", v as String], "") ExecStart { command: v } => join(["ExecStart=", v as String], "") + ExecStartPost { command: v } => join(["ExecStartPost=", v as String], "") ExecStop { command: v } => join(["ExecStop=", v as String], "") ExecStopPost { command: v } => join(["ExecStopPost=", v as String], "") + ExecStopPostIgnoringFailure { command: v } => join(["ExecStopPost=-", v as String], "") ServiceStandardOutput { stream: v } => join(["StandardOutput=", systemd_output_stream_wire(stream: v)], "") ServiceStandardError { stream: v } => diff --git a/dag/extdeps/virtualization/firecracker.dag b/dag/extdeps/virtualization/firecracker.dag index f4454b58f4b..7852381c568 100644 --- a/dag/extdeps/virtualization/firecracker.dag +++ b/dag/extdeps/virtualization/firecracker.dag @@ -3,6 +3,7 @@ module extdeps.virtualization.firecracker import extdeps.exec.command { ArgvCommand, argv_command } import std.types { String, NonEmptyStr, Int, Bool, List, FilePath } import std.disposition { Disposition, Terminal } +import std.measure { ByteSize, byte_size } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import extdeps.languages.json.emit { @@ -113,6 +114,13 @@ type FirecrackerBootSource { boot_args: NonEmptyStr } +// A DRIVE IS EXPOSED TO THE GUEST IN WHOLE 512-BYTE SECTORS, AND A PARTIAL TAIL IS DROPPED. +// firecracker v1.16.1 src/vmm/src/devices/virtio/block/virtio/mod.rs: SECTOR_SIZE = 1 << SECTOR_SHIFT +// (9). device.rs, on opening the backing file: "If the image is not a multiple of the sector size, +// the tail bits are not exposed" -- it logs a warning and the guest sees the size rounded down. A +// backing file whose content matters to its last byte must therefore be a whole number of these. +data firecracker_block_sector_size: ByteSize = byte_size(count: 512) + // A drive is identified by a caller-chosen id and is EITHER the root device or not. Upstream lets // both flags be set independently; modelling is_root_device as a Bool beside is_read_only keeps // the upstream shape rather than inventing a coproduct the API does not have. @@ -159,6 +167,42 @@ fn firecracker_config_has_single_root(config: FirecrackerVmConfig) -> Bool { firecracker_root_drive_count(config: config) == 1 } +// WHICH GUEST DEVICE A DRIVE BECOMES IS ITS POSITION IN `drives`. Firecracker attaches the +// virtio-blk devices in the order of the configuration's drives array, and the Linux virtio_blk +// driver names them vda, vdb, ... in the order it probes them (virtblk_name_format). On this fleet's +// aarch64 hosts that order was observed, not only read: the Mt. Collins run +// (gunbc.runner.runner_first_job_receipt execution_chain) booted root=/dev/vda and read its second +// drive's credential off /dev/vdb. Only the first 26 drives are named here; a drive beyond them, or +// one that is not attached, has no name rather than a guessed one. +type DriveIndexScan { + next: Int + found: Int? +} + +data firecracker_guest_block_letters: String = "abcdefghijklmnopqrstuvwxyz" + +fn firecracker_guest_block_device_of_index(index: Int) -> String? { + if index >= string_length(s: firecracker_guest_block_letters) { + none + } else { + Present { value: concat("/dev/vd", substring(s: firecracker_guest_block_letters, start: index, end: index + 1)) } + } +} + +fn firecracker_drive_guest_device(drives: List, drive_id: NonEmptyStr) -> String? { + let scan = fold(drives, init: DriveIndexScan { next: 0, found: none }, f: fn(acc, d) { + let found = match acc.found { + Present { value: i } => Present { value: i } + Absent => if d.drive_id == drive_id { Present { value: acc.next } } else { none } + } + DriveIndexScan { next: acc.next + 1, found: found } + }) + match scan.found { + Absent => none + Present { value: i } => firecracker_guest_block_device_of_index(index: i) + } +} + // --- the configuration FILE, which is upstream's own wire shape and not a rendering of ours --- // FIRECRACKER READS ONE JSON DOCUMENT WHOSE KEYS ARE HYPHENATED WHERE THE API PATHS ARE. The diff --git a/dag/gunbc/host/host_hygiene_liveness_observe.dag b/dag/gunbc/host/host_hygiene_liveness_observe.dag index a8298cff5b0..bfe6d55ed7b 100644 --- a/dag/gunbc/host/host_hygiene_liveness_observe.dag +++ b/dag/gunbc/host/host_hygiene_liveness_observe.dag @@ -5,6 +5,7 @@ import std.disposition { Disposition, Terminal } import std.realization_reconcile { ShowEffectiveRead, ReadObserved, ReadAbsent, Reconciliation } import extdeps.clock { clock_unix_millis_read, ClockUnixMillisObserved, ClockUnixMillisRefused } import extdeps.systemd.journalctl +import extdeps.github.actions_runner { actions_runner_listening_line_suffix } import gunbc.systemctl_list_units { systemctl_list_units_all_states } import gunbc.host_converge { gunbc_runner_all_slots_unit_glob } import gunbc.host_hygiene_liveness { @@ -23,14 +24,12 @@ data gunbc_host_hygiene_liveness_observe_scope_disposition: Disposition = Termin reason: "The OBSERVE half of the liveness vertical (brief: split observe from mutate). Runs LocalShell only: unit enumeration reuses gunbc.systemctl_list_units's all-states seam (the same one-op, different-binding move the reaper observe module uses — a listener slot is worth checking whether active, inactive, or failed, since a failed slot is exactly the outage this exists to catch), and the journal read goes through the new extdeps.systemd.Journalctl.UnitLog typed op rather than a piped `journalctl | grep | tail` shell body. Marker-matching (last 'Listening for Jobs' line) and freshness (age against extdeps.clock's typed Clock.UnixMillis read) are pure .dag folds over one typed stdout read, replacing what the deleted host_hygiene_liveness_script.dag did as one hand-written pipeline in a single shell body." } -data host_hygiene_listening_marker: String = "Listening for Jobs" - fn host_hygiene_journal_lines(stdout: String) -> List { filter(map(split(s: trim(stdout), delimiter: "\n"), l => trim(l)), l => l != "") } fn host_hygiene_journal_matching_lines(stdout: String) -> List { - filter(host_hygiene_journal_lines(stdout: stdout), l => string_contains(s: l, pattern: host_hygiene_listening_marker)) + filter(host_hygiene_journal_lines(stdout: stdout), l => string_contains(s: l, pattern: actions_runner_listening_line_suffix)) } // journalctl -o short-unix prints "SECONDS.MICROSECONDS host proc[pid]: msg" — the first diff --git a/dag/gunbc/runner/runner_attempt_launch.dag b/dag/gunbc/runner/runner_attempt_launch.dag index 6d41be2e0f5..cd6a0e11b89 100644 --- a/dag/gunbc/runner/runner_attempt_launch.dag +++ b/dag/gunbc/runner/runner_attempt_launch.dag @@ -80,7 +80,7 @@ data jit_mint_http_realization_frontier: DissolutionCondition = unbound_dissolut // firecracker_jailer_chroot_root. SUFFICIENT FOR deleting this row. NOT satisfied by // creating an empty jit.bin beside the workspace. data jail_jit_device_staging_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: gunbc.runner_attempt_launch plan_authorized_steps. TRIGGER: gunbc#10923 mint supplies encoded_jit_config and this planner writes that blob at firecracker_jailer_chroot_root/.../jit.bin so the in-jail path /jit.bin names a file the VMM can open. SUFFICIENT FOR deleting this row. NOT satisfied by an empty placeholder beside work.img." as NonEmptyStr, + description: "NAMED CONSUMER: gunbc.runner_attempt_launch plan_authorized_steps. TRIGGER: gunbc#10923 mint supplies encoded_jit_config and this planner writes gunbc.runner_microvm jit_drive_content(encoded_jit_config) -- the sector-padded environment-file bytes the guest agent reads from the device jit_drive_guest_device resolves, never the bare blob -- at firecracker_jailer_chroot_root/.../jit.bin so the in-jail path /jit.bin names a file the VMM can open. SUFFICIENT FOR deleting this row. NOT satisfied by an empty placeholder beside work.img." as NonEmptyStr, ) // SHADOW MEANS SHADOW. No fleet-converge entry, no dual dispatch, no "use Y when available". diff --git a/dag/gunbc/runner/runner_guest_image.dag b/dag/gunbc/runner/runner_guest_image.dag index fd18383d3d2..46de55dce90 100644 --- a/dag/gunbc/runner/runner_guest_image.dag +++ b/dag/gunbc/runner/runner_guest_image.dag @@ -6,6 +6,7 @@ import std.measure { Mebibyte, mebibyte } import std.process { ProcessExit, ExitSuccess, exit_failure } import std.resources { Network } import std.disposition { Disposition, SingleAuthority } +import std.dissolution { DissolutionCondition, unbound_dissolution } import extdeps.shell import extdeps.exec.command { ArgvCommand } import extdeps.crypto.hash { Digest, sha256_digest } @@ -23,6 +24,8 @@ import extdeps.github.actions_runner { ActionsRunnerRelease, ActionsRunnerBinaryArtifact, actions_runner_base_dir, actions_runner_binary_artifact_for_arch, actions_runner_download_url, actions_runner_tarball_filename, + actions_runner_listener_relative_path, actions_runner_listener_run_verb, + actions_runner_listening_line_suffix, } import extdeps.virtualization.firecracker { firecracker_ci_smoke_kernel_url, firecracker_ci_smoke_rootfs_url, @@ -30,12 +33,17 @@ import extdeps.virtualization.firecracker { import extdeps.systemd.unit_file { SystemdUnitFile, ServiceUnitFile, SystemdInstallSection, Installable, SystemdDropIn, SystemdDropInSection, DropInUnitSection, - Description, After, Wants, - ServiceType, ExecStartPre, ExecStart, ExecStopPost, WorkingDirectory, + Description, After, Wants, SuccessAction, FailureAction, SystemdUnitStateAction, UnitActionPoweroffForce, + systemd_environment_reference, systemd_exit_code_variable, systemd_exit_status_variable, + ServiceType, ExecStartPre, ExecStart, ExecStartPost, ExecStopPostIgnoringFailure, WorkingDirectory, + Restart, Environment, EnvironmentFileIfPresent, ServiceStandardOutput, ServiceStandardError, OutputJournalAndConsole, WantedBy, serialize_systemd_unit_file, serialize_systemd_drop_in, } -import gunbc.runner_microvm { RunnerGuestImage, gunbc_runner_guest_image } +import gunbc.runner_microvm { + RunnerGuestImage, gunbc_runner_guest_image, + JitDriveDeviceResolution, JitDriveDevice, JitDriveNotAttached, jit_drive_guest_device, +} import gunbc.command_runner { run_shell_commands, LocalExec, } @@ -215,15 +223,10 @@ fn runner_guest_image_dir(root: String) -> String { } // THE GUEST AGENT IS A UNIT FILE, NOT A SCRIPT, and that is the difference between a modeled guest -// and a hand-authored one. Everything the agent has to do -- run one command, print to a console -// somebody can read, and power the machine off afterwards -- is a systemd directive, so it is rows -// through extdeps.systemd.unit_file rather than shell this repository would then own forever. -// -// THE POWER-OFF IS ExecStopPost= RATHER THAN A SECOND ExecStart= LINE, and the placement is the -// property. systemd runs ExecStopPost on BOTH arms -- the main command succeeding and the main -// command failing -- so a guest whose job died still stops existing. A poweroff appended after the -// main command would run only on success, and the one VM that most needs to disappear, the one that -// failed holding a credential, is exactly the one that would stay up. +// and a hand-authored one. Everything the agent has to do -- read one credential, run one process, +// report four events on a surface the host reads, and power the machine off afterwards -- is a +// systemd directive, so it is rows through extdeps.systemd.unit_file rather than shell this +// repository would then own forever. // // StandardOutput IS journal+console FOR A REASON THAT IS STRUCTURAL RATHER THAN VERBOSE: a guest // that powers off takes its journal with it, so anything written only to the journal was never @@ -242,30 +245,210 @@ data runner_guest_agent_target: NonEmptyStr = "multi-user.target" as NonEmptyStr // `10-gunbc-runner-agent.service`, which systemd skips because it does not end in `.conf`. Two // different mechanisms, the same outcome: an activation file that is written, looks right, and is // never read. -// -// THE WITNESS ALREADY DEMANDED THIS AND WAS NEVER RUN. the_agent_is_pulled_in_by_a_drop_in_systemd_ -// will_actually_read asserts `.conf` in the path; it would have gone red on the first repair. It was -// compiled and not executed, which is precisely the specification-without-execution DESIGN section 5 -// names -- a typecheck is not a consumer. data runner_guest_agent_drop_in_suffix: String = ".conf" -data runner_guest_agent_poweroff_command: NonEmptyStr = "/usr/sbin/poweroff -f" as NonEmptyStr +// --- the host channel --- + +// THE CHANNEL IS THE SERIAL CONSOLE, NOT vsock, AND THE CHOICE IS WHAT THE HOST ALREADY READS. +// extdeps.virtualization.firecracker delivers ttyS0 on the VMM's stdout, and +// gunbc.runner_microvm_boot_probe already judges a boot from exactly that stream. vsock would need +// a second device in the VM configuration, a listener on the host side and a writer in the guest -- +// three new mechanisms to carry four lines -- and it would still not replace the console, which is +// where the kernel and systemd report a failure that happens before any agent runs. +// +// WHAT THE CHANNEL CANNOT CARRY IS STATED BECAUSE IT DECIDES WHO OWNS THE IDENTITY. A console line is +// written by the guest, so it is a claim by the workload about itself. The guest therefore asserts +// only what it can know -- which event, and which image it is. Joining a line to the attempt tuple +// (host, slot, generation, attempt, InvocationID, VMM identity, registration) is the host-side +// lifecycle controller's job, by the fact that the line arrived on THAT VMM's stdout; that +// controller is not built yet, and today's only reader is the boot probe, which has no attempt. A guest that printed a slot or attempt id would be +// forging a key the host must hold independently. And per the program rule, any of these events may +// TRIGGER the host's teardown; none of them may ESTABLISH that the cell is clean. +// +// THE GUEST EMITS THREE OF THE FOUR, AND THE TYPE SAYS WHICH. RunnerListening is known only to the +// listener, which prints its own readiness line; a guest-emitted marker for it would be a second line +// claiming the same fact. So the emitter takes a GuestEmittedEvent, which has no RunnerListening +// constructor, and only the reader's GuestObservationKind can name it. +type GuestEmittedEvent + = GuestBooted + | RunnerStarted + | RunnerExited + +type GuestObservationKind + = GuestEmitted { event: GuestEmittedEvent } + | RunnerListening + +// THE MARKER IS A FIXED TOKEN RATHER THAN A PHRASE, because the console interleaves kernel and systemd +// output into any line and a reader matching prose has already been fooled once on this lane (the +// Mt. Collins boot-marker grep, gunbc.runner.runner_first_job_receipt defects_found). +data guest_observation_marker: String = "gunbc-guest-observation" + +fn guest_emitted_event_wire(event: GuestEmittedEvent) -> String { + match event { + GuestBooted => "guest-booted" + RunnerStarted => "runner-started" + RunnerExited => "runner-exited" + } +} + +// A LABEL FOR RECEIPTS AND WITNESSES, NOT A WIRE: RunnerListening's label is never printed by the +// guest, it names what the reader recognised in upstream's line. +fn guest_observation_kind_label(kind: GuestObservationKind) -> String { + match kind { + GuestEmitted { event: e } => guest_emitted_event_wire(event: e) + RunnerListening => "runner-listening" + } +} + +fn guest_observation_line(event: GuestEmittedEvent, image: RunnerGuestImage) -> String { + join([guest_observation_marker, " ", guest_emitted_event_wire(event: event), " image=", runner_guest_image_name(image: image)], "") +} + +// PID 1 GIVES A SERVICE NO PATH WORTH TRUSTING, SO EVERY PROGRAM IS NAMED ABSOLUTELY. The Mt. Collins +// run accepted and executed a job that then died on "sh: command not found", because the process +// that launched it inherited no PATH (runner_first_job_receipt defects_found). systemd does hand +// services a default PATH, but the job's steps inherit the LISTENER's environment, so the PATH the job +// sees is declared here rather than left to whichever init started the guest. +data runner_guest_echo_program: String = "/usr/bin/echo" +data runner_guest_job_path: NonEmptyStr = "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" as NonEmptyStr +data runner_guest_job_home: NonEmptyStr = "HOME=/root" as NonEmptyStr + +fn guest_observation_emit_command(event: GuestEmittedEvent, image: RunnerGuestImage) -> NonEmptyStr { + join([runner_guest_echo_program, " ", guest_observation_line(event: event, image: image)], "") as NonEmptyStr +} + +// RunnerExited CARRIES THE EXIT THE MANAGER RECORDED, via systemd's own ExecStopPost environment +// ($EXIT_CODE, $EXIT_STATUS: systemd.exec(5), "Environment Variables in Spawned Processes"). It is +// placed as ExecStopPostIgnoringFailure -- systemd's `-` prefix, modeled in extdeps as its own +// variant like EnvironmentFileIfPresent -- so a failed echo can never be what fails the unit. +fn guest_runner_exited_emit_command(image: RunnerGuestImage) -> NonEmptyStr { + join([ + runner_guest_echo_program, " ", guest_observation_line(event: RunnerExited, image: image), + " exit_code=", systemd_environment_reference(name: systemd_exit_code_variable), + " exit_status=", systemd_environment_reference(name: systemd_exit_status_variable), + ], "") as NonEmptyStr +} + +// RunnerListening IS THE ONE EVENT THE GUEST AGENT DOES NOT PRINT, because only the listener knows +// it. Its own readiness line (extdeps.github.actions_runner actions_runner_listening_line_suffix) +// reaches the same console through the same StandardOutput, so the host reads upstream's line rather +// than a second line of ours claiming the same thing. +type GuestObservationRead + = GuestObservationSeen { kind: GuestObservationKind, line: String } + | NotAGuestObservation + +fn guest_emitted_event_in_line(line: String, event: GuestEmittedEvent) -> Bool { + string_contains(s: line, pattern: join([guest_observation_marker, " ", guest_emitted_event_wire(event: event), " "], "")) +} + +fn guest_observation_of_console_line(line: String) -> GuestObservationRead { + if guest_emitted_event_in_line(line: line, event: GuestBooted) { + GuestObservationSeen { kind: GuestEmitted { event: GuestBooted }, line: line } + } else if guest_emitted_event_in_line(line: line, event: RunnerStarted) { + GuestObservationSeen { kind: GuestEmitted { event: RunnerStarted }, line: line } + } else if guest_emitted_event_in_line(line: line, event: RunnerExited) { + GuestObservationSeen { kind: GuestEmitted { event: RunnerExited }, line: line } + } else if string_contains(s: line, pattern: actions_runner_listening_line_suffix) { + GuestObservationSeen { kind: RunnerListening, line: line } + } else { + NotAGuestObservation + } +} + +// THE WHOLE CONSOLE, READ LINE BY LINE, AS THE HOST'S ONE READER. gunbc.runner_microvm_boot_probe +// consumes it today to require GuestBooted; the lifecycle controller consumes the same fold for the +// rest. +fn guest_console_observations(console: String) -> List { + fold(split(s: console, delimiter: "\n"), init: [] as List, f: fn(acc, line) { + match guest_observation_of_console_line(line: line) { + GuestObservationSeen { kind: k, line: _ } => append(acc, items: [k]) + NotAGuestObservation => acc + } + }) +} -fn runner_guest_agent_unit() -> SystemdUnitFile { +fn guest_console_carries(console: String, kind: GuestObservationKind) -> Bool { + any(xs: guest_console_observations(console: console), predicate: fn(k) { + guest_observation_kind_label(kind: k) == guest_observation_kind_label(kind: kind) + }) +} + +// DECLARED FRONTIER: THE AGENT HAS NOT YET RUN A JOB. Everything above is exercised by witnesses over +// the rendered unit and the console reader, and the boot probe exercises the no-credential arm on a +// real VM. The credential arm -- a minted JIT config on the drive, the listener registering, a job +// dispatched under a test label (never the floor label), RunnerExited and the VM gone -- needs the +// host to mint and stage that credential, and no modeled host path does yet. A hand-staged +// credential would be the out-of-band actuation DESIGN section 6 names, so the run waits for the +// real path. +data guest_agent_real_job_acceptance_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: gunbc.runner_guest_image runner_guest_agent_unit, credential arm. TRIGGER: gunbc.runner_attempt_launch jit_mint_http_realization_frontier AND jail_jit_device_staging_frontier both dissolve, so a modeled host path mints a JIT config and writes gunbc.runner_microvm jit_drive_content to the attempt's credential drive. SUFFICIENT FOR deleting this row only together with one real job run on srv1 under a test label whose console shows GuestBooted, RunnerStarted, the listener's Listening line, RunnerExited, and a VMM that exited. NOT satisfied by the boot probe, by a hand-staged credential, or by a floor-label job." as NonEmptyStr, +) + +// --- the agent --- + +// THE PRINCIPAL WORKLOAD IS THE LISTENER ITSELF, NOT run.sh, AND THAT IS HOW THE RELAUNCH LOOP IS +// ACCOUNTED FOR. run.sh re-executes the listener whenever it returns 2 (RetryableError); inside a +// one-job guest that loop would keep a VM alive past its job, holding a spent registration. +// Invoking `Runner.Listener run` directly means there is no loop to reach: the listener's exit IS the +// service's exit, whatever the code. `Restart=no` is written rather than defaulted so a later author +// adding a restart policy has to delete a declaration to do it. +// +// THE CREDENTIAL IS READ BY systemd FROM THE READ-ONLY DEVICE (gunbc.runner_microvm +// jit_drive_guest_device, passed in as credential_device) as an environment file whose single assignment is the listener's own +// argument variable. No `--jitconfig ` appears on any command line. +fn runner_guest_version_readback_command() -> NonEmptyStr { + join([actions_runner_base_dir, actions_runner_listener_relative_path, " --version"], "") as NonEmptyStr +} + +fn runner_guest_listener_command() -> NonEmptyStr { + join([actions_runner_base_dir, actions_runner_listener_relative_path, " ", actions_runner_listener_run_verb], "") as NonEmptyStr +} + +// THE POWER-OFF IS THE MANAGER'S, NOT A COMMAND'S. SuccessAction= and FailureAction= are both +// `poweroff-force`, so every way the unit can end powers the VM off -- the listener exiting 0, it +// exiting nonzero, it being killed, AND the start failing before any command ran. The earlier +// `ExecStopPost=/usr/sbin/poweroff -f` covered only the arms on which systemd could still spawn a +// command. +// +// THE CREDENTIAL FILE IS `EnvironmentFile=-` (IF PRESENT), AND THAT DOES NOT MAKE A MISSING ONE +// SILENT. systemd loads an environment file for EVERY command it spawns, so a hard `EnvironmentFile=` +// on an absent device would fail GuestBooted, the version readback and RunnerExited alike -- the host +// would see a VM that printed nothing. With the prefix, the listener starts without a credential, +// upstream refuses it ("Runner is not configured.", TerminatedError = 1), and the console carries +// GuestBooted, the version, no RunnerListening and RunnerExited exit_status=1: a located refusal +// instead of an empty console. +// +// THE VERSION READBACK STAYS, AND IT IS THE runner_first_job_receipt LESSON. The Mt. Collins canary +// ran 2.337.0 while the model said 2.336.0 and nothing here caught it. `Runner.Listener --version` +// prints the release actually baked into the image on the console the host reads, where +// gunbc.runner_microvm_boot_probe boot_probe_verdict compares it with the modeled release. +// `poweroff-force` skips the orderly stop of other units but still kills every process and unmounts +// before powering off -- a one-job VM has nothing else whose orderly stop matters. +data runner_guest_agent_end_action: SystemdUnitStateAction = UnitActionPoweroffForce + +fn runner_guest_agent_unit(image: RunnerGuestImage, credential_device: NonEmptyStr) -> SystemdUnitFile { ServiceUnitFile { unit: [ Description { text: "gunbc ephemeral Actions runner guest agent" as NonEmptyStr }, Wants { target: "network-online.target" as NonEmptyStr }, After { target: "network-online.target" as NonEmptyStr }, + SuccessAction { action: runner_guest_agent_end_action }, + FailureAction { action: runner_guest_agent_end_action }, ], service: [ - ServiceType { mode: "oneshot" as NonEmptyStr }, + ServiceType { mode: "exec" as NonEmptyStr }, + Restart { policy: "no" as NonEmptyStr }, WorkingDirectory { path: actions_runner_base_dir as NonEmptyStr }, + Environment { assignment: runner_guest_job_path }, + Environment { assignment: runner_guest_job_home }, + EnvironmentFileIfPresent { path: credential_device }, ServiceStandardOutput { stream: OutputJournalAndConsole }, ServiceStandardError { stream: OutputJournalAndConsole }, - ExecStartPre { command: runner_guest_agent_version_command() }, - ExecStart { command: runner_guest_agent_version_command() }, - ExecStopPost { command: runner_guest_agent_poweroff_command }, + ExecStartPre { command: guest_observation_emit_command(event: GuestBooted, image: image) }, + ExecStartPre { command: runner_guest_version_readback_command() }, + ExecStart { command: runner_guest_listener_command() }, + ExecStartPost { command: guest_observation_emit_command(event: RunnerStarted, image: image) }, + ExecStopPostIgnoringFailure { command: guest_runner_exited_emit_command(image: image) }, ], install: Installable { directives: [WantedBy { target: runner_guest_agent_target }], @@ -273,21 +456,6 @@ fn runner_guest_agent_unit() -> SystemdUnitFile { } } -// THE AGENT'S MAIN COMMAND IS THE RUNNER'S OWN VERSION READBACK, AND IT IS THE WHOLE AGENT TODAY. -// -// This is the slice-1 acceptance and nothing beyond it: a VM that boots this image prints the -// version of the runner that was actually baked into it, on a console outside the VM, and powers -// off. It is a READBACK rather than a greeting -- it can only print if the tarball extracted, if -// the interpreter it needs is present, and if the guest reached multi-user -- so it discriminates -// on the facts the build is supposed to have established. -// -// THE JIT REGISTRATION IS DELIBERATELY ABSENT and its absence is a refusal rather than a gap left -// open: no credential is modeled to reach this guest yet, and an ExecStart that tried to register -// without one would fail in the guest, where nothing is watching, instead of failing here. -fn runner_guest_agent_version_command() -> NonEmptyStr { - join([actions_runner_base_dir, "/bin/Runner.Listener --version"], "") as NonEmptyStr -} - fn runner_guest_agent_unit_path(root: String, image: RunnerGuestImage) -> String { join([runner_guest_tree_dir(root: root, image: image), "/etc/systemd/system/", runner_guest_agent_unit_name], "") } @@ -547,7 +715,15 @@ fn measure_artifact_wet(root: String, artifact: GuestBaseArtifact) -> GuestArtif } fn write_guest_agent_unit_wet(root: String, image: RunnerGuestImage) -> ProcessExit { - let text = serialize_systemd_unit_file(file: runner_guest_agent_unit()) + match jit_drive_guest_device() { + JitDriveNotAttached { cause: c } => + exit_failure(reason: join(["runner_guest_image: no guest device names the credential drive, so no agent was rendered: ", c], "")) + JitDriveDevice { device: d } => write_guest_agent_unit_for_device_wet(root: root, image: image, credential_device: d) + } +} + +fn write_guest_agent_unit_for_device_wet(root: String, image: RunnerGuestImage, credential_device: NonEmptyStr) -> ProcessExit { + let text = serialize_systemd_unit_file(file: runner_guest_agent_unit(image: image, credential_device: credential_device)) let a = Filesystem.Write(path: runner_guest_agent_unit_path(root: root, image: image), content: text) if !a.success { exit_failure(reason: join(["runner_guest_image: guest agent unit write refused: ", a.error], "")) diff --git a/dag/gunbc/runner/runner_microvm.dag b/dag/gunbc/runner/runner_microvm.dag index bc1bcf348bd..242ac49ee90 100644 --- a/dag/gunbc/runner/runner_microvm.dag +++ b/dag/gunbc/runner/runner_microvm.dag @@ -9,6 +9,7 @@ import std.measure { gibibyte_to_byte_size, HardwareThreadCount, byte_size_count, + byte_size, mebibyte, mebibyte_count, mebibyte_to_byte_size, @@ -38,7 +39,9 @@ import gunbc.runner_unit { lifecycle_intent_stops_finished_incarnation, gunbc_runner_main_exits_when_incarnation_ends, } -import extdeps.github.actions_runner { ActionsRunnerRelease, actions_runner_release_2_337_0 } +import extdeps.github.actions_runner { + ActionsRunnerRelease, actions_runner_release_2_337_0, actions_runner_jitconfig_env_name, +} import extdeps.virtualization.firecracker { FirecrackerVmConfig, FirecrackerBootSource, @@ -48,7 +51,10 @@ import extdeps.virtualization.firecracker { FirecrackerRelease, firecracker_release_aarch64, firecracker_config_has_single_root, + firecracker_block_sector_size, + firecracker_drive_guest_device, } +import std.bytes { utf8_encode_bytes, bytes_octets } data runner_microvm_disposition: Disposition = SingleAuthority @@ -112,6 +118,86 @@ data gunbc_runner_jit_transport: JitCredentialTransport = JitConfigOnReadOnlyDri drive_id: "jitcfg", } +// WHAT THE CREDENTIAL DRIVE CONTAINS, AS THE ONE CONTRACT BOTH ENDS DERIVE FROM. The host writes +// these bytes and the guest reads them; a format stated on only one side is a fork the other side +// discovers at boot, inside a VM nobody is watching. +// +// THE CONTENT IS ONE systemd ENVIRONMENT-FILE ASSIGNMENT, because that makes the guest's reader a +// directive rather than a program. The guest agent names the device in `EnvironmentFile=`, systemd +// parses it, and the listener takes the value from its environment under upstream's own argument +// convention (extdeps.github.actions_runner actions_runner_jitconfig_env_name) and unsets it once +// read. No shell, no `cat`, and the secret never appears on a command line where `ps` would show it. +// +// THE PADDING IS NEWLINES BECAUSE A BLOCK DEVICE HAS NO LENGTH OF ITS OWN. Firecracker exposes a +// backing file only in whole sectors (extdeps.virtualization.firecracker +// firecracker_block_sector_size) and drops a partial tail, so an unpadded 4,076-byte config would +// reach the guest cut to 3,584 bytes. Padding to the sector boundary with blank lines keeps every +// byte, and blank lines are what systemd's environment-file parser skips -- a NUL fill would not be. +// The length is measured in UTF-8 BYTES, the unit the sector is denominated in, not in characters. + +// THE GUEST DEVICE IS DERIVED FROM THE DRIVE LIST, NOT WRITTEN BESIDE IT. runner_microvm_drives is +// the one drive list runner_microvm_vm_config attaches, and extdeps.virtualization.firecracker +// firecracker_drive_guest_device maps the credential drive's position in it to its guest name. The +// guest image renders its agent from this resolution, so prepending or reordering a drive moves the +// name the guest reads with it -- the rootfs can no longer be read as the credential by an edit that +// left a literal behind. The order does not depend on the backing paths, so the resolution names the +// attempt layout's own file names; a credential drive that is not attached resolves to a refusal, +// never to a guessed device. +type JitDriveDeviceResolution + = JitDriveDevice { device: NonEmptyStr } + | JitDriveNotAttached { cause: String } + +fn jit_drive_guest_device() -> JitDriveDeviceResolution { + let drives = runner_microvm_drives( + rootfs_path: "rootfs.ext4", + jit_config_path: "jit.img", + transport: gunbc_runner_jit_transport, + ) + match firecracker_drive_guest_device(drives: drives, drive_id: jit_transport_drive_id(t: gunbc_runner_jit_transport)) { + Present { value: d } => JitDriveDevice { device: d as NonEmptyStr } + Absent => JitDriveNotAttached { + cause: "the credential drive is not among the first 26 drives runner_microvm_drives attaches, so no guest device names it", + } + } +} + +fn jit_drive_assignment(encoded_jit_config: NonEmptyStr) -> String { + join([actions_runner_jitconfig_env_name(), "=", encoded_jit_config as String, "\n"], "") +} + +fn jit_drive_utf8_size(text: String) -> ByteSize { + byte_size(count: count(bytes_octets(b: utf8_encode_bytes(s: text)))) +} + +// ONE SECTOR OF NEWLINES, AS A SOURCE TO CUT PADDING FROM. Built by doubling to 512 and checked +// against the sector size by the_credential_drive_is_whole_sectors_of_one_assignment: a newline is +// one UTF-8 byte, so this string's byte size is its length. +fn jit_drive_newline_sector() -> String { + let n2 = "\n\n" + let n8 = concat(concat(n2, n2), concat(n2, n2)) + let n32 = concat(concat(n8, n8), concat(n8, n8)) + let n128 = concat(concat(n32, n32), concat(n32, n32)) + concat(concat(n128, n128), concat(n128, n128)) +} + +// A DRIVE THAT HONESTLY CARRIES NO CREDENTIAL: one sector of blank lines, which the guest's +// environment-file read accepts and assigns nothing from. A VM booted with it reaches the listener, +// which refuses as unconfigured -- the boot-only probe's arm, where no registration may be spent. +fn jit_drive_without_credential() -> String { + jit_drive_newline_sector() +} + +fn jit_drive_content(encoded_jit_config: NonEmptyStr) -> String { + let body = jit_drive_assignment(encoded_jit_config: encoded_jit_config) + let sector = byte_size_count(b: firecracker_block_sector_size) + let tail = byte_size_count(b: jit_drive_utf8_size(text: body)) % sector + if tail == 0 { + body + } else { + concat(body, substring(s: jit_drive_newline_sector(), start: 0, end: sector - tail)) + } +} + // WHAT ONE RUNNER VM IS SIZED AT, IN GROUNDED MAGNITUDES RATHER THAN BARE INTEGERS. // // An earlier revision carried these as `Int`, mirroring Firecracker's own field names inward. That @@ -578,20 +664,7 @@ fn runner_microvm_vm_config( kernel_image_path: kernel_path, boot_args: "keep_bootcon console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw", }, - drives: [ - FirecrackerDrive { - drive_id: "rootfs", - path_on_host: rootfs_path, - is_root_device: true, - is_read_only: false, - }, - FirecrackerDrive { - drive_id: jit_transport_drive_id(t: transport), - path_on_host: jit_config_path, - is_root_device: false, - is_read_only: true, - }, - ], + drives: runner_microvm_drives(rootfs_path: rootfs_path, jit_config_path: jit_config_path, transport: transport), network_interfaces: runner_microvm_network_interfaces(net: net), machine_config: FirecrackerMachineConfig { vcpu_count: hardware_thread_count_value(t: size.vcpu_count), @@ -600,6 +673,27 @@ fn runner_microvm_vm_config( } } +fn runner_microvm_drives( + rootfs_path: FilePath, + jit_config_path: FilePath, + transport: JitCredentialTransport, +) -> List { + [ + FirecrackerDrive { + drive_id: "rootfs", + path_on_host: rootfs_path, + is_root_device: true, + is_read_only: false, + }, + FirecrackerDrive { + drive_id: jit_transport_drive_id(t: transport), + path_on_host: jit_config_path, + is_root_device: false, + is_read_only: true, + }, + ] +} + fn jit_transport_drive_id(t: JitCredentialTransport) -> NonEmptyStr { match t { JitConfigOnReadOnlyDrive { drive_id: d } => d diff --git a/dag/gunbc/runner/runner_microvm_boot_probe.dag b/dag/gunbc/runner/runner_microvm_boot_probe.dag index 7401d545eb5..2789499a679 100644 --- a/dag/gunbc/runner/runner_microvm_boot_probe.dag +++ b/dag/gunbc/runner/runner_microvm_boot_probe.dag @@ -19,9 +19,10 @@ import gunbc.runner_microvm { RunnerGuestImage, gunbc_runner_guest_image, RunnerMicroVmNetworking, GuestNetworkAbsent, RunnerMicroVmConfigResolution, MicroVmConfigResolved, MicroVmConfigUnresolved, - gunbc_runner_microvm_vm_config, + gunbc_runner_microvm_vm_config, jit_drive_without_credential, } import gunbc.runner_guest_image { + guest_console_carries, GuestEmitted, GuestBooted, runner_guest_image_root, runner_guest_image_path, runner_guest_base_artifact_path, runner_guest_kernel_artifact, } @@ -98,14 +99,16 @@ fn runner_microvm_boot_probe_stage_commands(root: String, image: RunnerGuestImag ] } -// THE JIT DRIVE IS THE ROOT DRIVE'S OWN PATH TODAY, AND THAT IS A GAP RATHER THAN A DESIGN. -// gunbc.runner_microvm's configuration builder attaches a second, read-only drive carrying the -// registration credential, because that carrier is the whole reason a microVM closes the shared- -// credential class. No credential is minted yet -- that is the JIT slice -- so there is nothing to -// put on it. A drive pointing at a file that does not exist makes Firecracker refuse at device -// setup, which would render as an image defect; pointing it at the rootfs copy attaches a real file -// the guest ignores. The moment a credential exists this argument is the credential's path and this -// paragraph deletes. +// THE JIT DRIVE CARRIES NO CREDENTIAL, BY THE CONTRACT RATHER THAN BY ACCIDENT. The guest agent reads +// that device as a systemd environment file (gunbc.runner_microvm jit_drive_content), so the +// earlier stand-in -- the rootfs copy's own path -- would now hand systemd an 8 GiB ext4 image to +// parse. The probe instead writes gunbc.runner_microvm jit_drive_without_credential: whole sectors +// of blank lines. The guest boots, prints its runner version and GuestBooted, the listener refuses as +// unconfigured, and the manager powers the VM off -- a boot probe spends no registration. +fn runner_microvm_boot_probe_jit_drive(root: String) -> String { + join([runner_microvm_boot_probe_dir(root: root), "/probe-jit.img"], "") +} + // NO IMAGE IDENTITY IS TAKEN HERE, AND THAT IS THE POINT: the probe boots a FIXED path, and the // identity is spent upstream by runner_microvm_boot_probe_stage_commands, which copies // runner_guest_image_path(root, image) onto it. Taking an identity this function could not honour @@ -114,7 +117,7 @@ fn runner_microvm_boot_probe_stage_commands(root: String, image: RunnerGuestImag fn runner_microvm_boot_probe_config(root: String) -> RunnerMicroVmConfigResolution { gunbc_runner_microvm_vm_config( rootfs_path: runner_microvm_boot_probe_rootfs(root: root), - jit_config_path: runner_microvm_boot_probe_rootfs(root: root), + jit_config_path: runner_microvm_boot_probe_jit_drive(root: root), kernel_path: runner_guest_base_artifact_path(root: root, artifact: runner_guest_kernel_artifact), net: runner_microvm_boot_probe_networking(), ) @@ -122,7 +125,7 @@ fn runner_microvm_boot_probe_config(root: String) -> RunnerMicroVmConfigResoluti // THE DEADLINE IS THE HOST'S, AND IT MUST NOT DEPEND ON THE GUEST AGREEING TO IT. // -// The guest agent powers the machine off on ExecStopPost=, which runs on both arms of the job -- and +// The guest agent powers the machine off by SuccessAction=/FailureAction=, which cover every way the agent unit ends -- and // that only ever covers the case where the agent STARTED. It does not cover an agent systemd never // pulled in (the exact defect this module's sibling just carried), a kernel that panics before // userspace, an init that hangs, or a guest that never reaches its target at all. In every one of @@ -160,6 +163,7 @@ type BootProbeVerdict = BootPrintedRunnerVersion { version: String } | BootProducedNoConsole { cause: String } | BootConsoleLacksRunnerVersion { expected: String, console: String } + | BootConsoleLacksGuestBooted { console: String } | BootNotAttempted { cause: String } | BootExceededHostDeadline { deadline: String, console: String, detail: String } | BootVmmFailed { exit_code: Int, console: String, detail: String } @@ -184,12 +188,33 @@ fn boot_probe_verdict(runner: ActionsRunnerRelease, observed: ProcessObservation } } +// A LIVE BOOT OF THIS IMAGE MUST ALSO CARRY THE AGENT'S GuestBooted MARKER, read through +// gunbc.runner_guest_image guest_console_carries -- the host's one console reader -- so a version +// string that reached the console some other way is not read as the agent having run. It is a layer +// over boot_probe_verdict rather than inside it because that verdict also judges consoles captured +// before the marker existed (gunbc.runner_observed_version_check's committed Mt. Collins capture), +// where requiring it would fail a true version readback for a reason unrelated to the version. +fn guest_agent_boot_verdict(runner: ActionsRunnerRelease, observed: ProcessObservation) -> BootProbeVerdict { + let base = boot_probe_verdict(runner: runner, observed: observed) + match base { + BootPrintedRunnerVersion { version: _ } => + if guest_console_carries(console: observed.stdout, kind: GuestEmitted { event: GuestBooted }) { + base + } else { + BootConsoleLacksGuestBooted { console: observed.stdout } + } + _ => base + } +} + fn boot_probe_verdict_text(v: BootProbeVerdict) -> String { match v { BootPrintedRunnerVersion { version: x } => join(["boot=printed-runner-version ", x], "") BootProducedNoConsole { cause: c } => join(["boot=no-console (", c, ")"], "") BootConsoleLacksRunnerVersion { expected: e, console: c } => join(["boot=console-without-runner-version expected=", e, "\n--- console ---\n", c, "\n--- end console ---"], "") + BootConsoleLacksGuestBooted { console: c } => + join(["boot=console-without-guest-booted (the guest agent never reported GuestBooted)\n--- console ---\n", c, "\n--- end console ---"], "") BootNotAttempted { cause: c } => join(["boot=not-attempted (", c, ")"], "") BootExceededHostDeadline { deadline: d, console: c, detail: e } => join(["boot=host-deadline-expired after ", d, @@ -206,6 +231,7 @@ fn boot_probe_admitted(v: BootProbeVerdict) -> Bool { BootPrintedRunnerVersion { version: _ } => true BootProducedNoConsole { cause: _ } => false BootConsoleLacksRunnerVersion { expected: _, console: _ } => false + BootConsoleLacksGuestBooted { console: _ } => false BootNotAttempted { cause: _ } => false BootExceededHostDeadline { deadline: _, console: _, detail: _ } => false BootVmmFailed { exit_code: _, console: _, detail: _ } => false @@ -254,11 +280,21 @@ fn runner_microvm_boot_probe_wet() -> ProcessExit MicroVmConfigResolved { config: cfg } => match run_shell_commands(commands: runner_microvm_boot_probe_stage_commands(root: root, image: image), transport: LocalExec) { ExitSuccess => { + let j = Filesystem.Write( + path: runner_microvm_boot_probe_jit_drive(root: root), + content: jit_drive_without_credential(), + ) let w = Filesystem.Write( path: runner_microvm_boot_probe_config_path(root: root), content: firecracker_vm_config_document(config: cfg), ) - if !w.success { + if !j.success { + runner_microvm_boot_probe_receipt_wet( + verdict: BootNotAttempted { + cause: join(["the credential-free JIT drive could not be written: ", j.error], ""), + }, + ) + } else if !w.success { runner_microvm_boot_probe_receipt_wet( verdict: BootNotAttempted { cause: join(["the VM configuration document could not be written: ", w.error], ""), @@ -266,7 +302,7 @@ fn runner_microvm_boot_probe_wet() -> ProcessExit ) } else { runner_microvm_boot_probe_receipt_wet( - verdict: boot_probe_verdict( + verdict: guest_agent_boot_verdict( runner: image.runner, observed: run_shell_command_observe( command: runner_microvm_boot_probe_bounded_command( diff --git a/dag/test/claim/runner/runner_guest_image_witness_test.dag b/dag/test/claim/runner/runner_guest_image_witness_test.dag index 49bcdb158ec..ae710bcfeb9 100644 --- a/dag/test/claim/runner/runner_guest_image_witness_test.dag +++ b/dag/test/claim/runner/runner_guest_image_witness_test.dag @@ -6,7 +6,15 @@ import extdeps.github.actions_runner { ActionsRunnerRelease } import extdeps.exec.command { ArgvCommand } import extdeps.tools.e2fsprogs { Ext4, mke2fs_populate_command } import extdeps.systemd.unit_file { serialize_systemd_unit_file, serialize_systemd_drop_in } -import gunbc.runner_microvm { RunnerGuestImage, gunbc_runner_guest_image } +import gunbc.runner_microvm { + RunnerGuestImage, gunbc_runner_guest_image, jit_drive_content, jit_drive_newline_sector, + jit_drive_guest_device, JitDriveDevice, JitDriveNotAttached, runner_microvm_vm_config, attach_workspace_drive, jit_transport_drive_id, + gunbc_runner_jit_transport, gunbc_runner_microvm_declared_size, GuestNetworkAbsent, +} +import extdeps.virtualization.firecracker { + firecracker_block_sector_size, firecracker_drive_guest_device, FirecrackerDrive, +} +import std.measure { byte_size_count } import gunbc.runner_guest_image { GuestBaseArtifact, GuestArtifactIntegrity, ArtifactDigestPinned, ArtifactDigestUnpinned, ArtifactDigestNotPinnable, runner_guest_built_image_artifact, @@ -19,12 +27,23 @@ import gunbc.runner_guest_image { runner_guest_agent_unit, runner_guest_agent_unit_path, runner_guest_agent_drop_in_path, runner_guest_agent_target_drop_in, runner_guest_image_size, runner_guest_image_filesystem, + GuestObservationKind, GuestEmitted, RunnerListening, + GuestBooted, RunnerStarted, RunnerExited, + guest_observation_line, guest_observation_kind_label, + guest_console_observations, guest_console_carries, } import extdeps.tools.sha256sum { Sha256FileDigest, Sha256FileDigestUnavailable } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly +fn the_credential_device() -> NonEmptyStr { + match jit_drive_guest_device() { + JitDriveDevice { device: d } => d + JitDriveNotAttached { cause: _ } => "/dev/credential-drive-not-attached" as NonEmptyStr + } +} + data a_digest: NonEmptyStr = "1111111111111111111111111111111111111111111111111111111111111111" as NonEmptyStr data b_digest: NonEmptyStr = "2222222222222222222222222222222222222222222222222222222222222222" as NonEmptyStr @@ -134,16 +153,76 @@ test fn two_images_differing_only_in_the_runner_release_do_not_share_a_path() -> && runner_guest_tree_dir(root: "/r", image: a) != runner_guest_tree_dir(root: "/r", image: b) } -// THE POWER-OFF IS UNCONDITIONAL, AND THIS IS THE ASSERTION THAT SAYS SO IN BYTES. `ExecStopPost=` -// runs on both the success and the failure arm; an `ExecStart=` carrying the poweroff would leave a -// failed job's VM standing. A refactor that "simplifies" the two into one ExecStart reds here. +// THE POWER-OFF IS UNCONDITIONAL, AND THIS IS THE ASSERTION THAT SAYS SO IN BYTES. SuccessAction= and +// FailureAction= are the manager's own reaction to the unit ending, so they cover the arm where no +// command can be spawned at all. The red is either one missing, or the power-off regressing to a +// command line (which cannot run on the start-failed arm). test fn the_guest_agent_powers_the_machine_off_on_every_arm() -> Bool { - let text = serialize_systemd_unit_file(file: runner_guest_agent_unit()) - string_contains(s: text, pattern: "ExecStopPost=/usr/sbin/poweroff -f") + let text = serialize_systemd_unit_file(file: runner_guest_agent_unit(image: gunbc_runner_guest_image, credential_device: the_credential_device())) + string_contains(s: text, pattern: "SuccessAction=poweroff-force") + && string_contains(s: text, pattern: "FailureAction=poweroff-force") + && !string_contains(s: text, pattern: "poweroff -f") && string_contains(s: text, pattern: "StandardOutput=journal+console") - && string_contains(s: text, pattern: "Type=oneshot") && string_contains(s: text, pattern: "WantedBy=multi-user.target") - && string_contains(s: text, pattern: "ExecStopPost=") +} + +// THE LISTENER IS THE PRINCIPAL WORKLOAD AND run.sh IS NOT, so a return code of 2 has no loop to +// re-enter: the listener's exit is the unit's exit. The red is any revision that launches run.sh (or +// run-helper.sh), or adds a restart policy, either of which keeps a spent one-job VM alive. +test fn the_listener_runs_once_with_no_relaunch_loop() -> Bool { + let text = serialize_systemd_unit_file(file: runner_guest_agent_unit(image: gunbc_runner_guest_image, credential_device: the_credential_device())) + string_contains(s: text, pattern: "ExecStart=/opt/actions-runner/bin/Runner.Listener run") + && !string_contains(s: text, pattern: "run.sh") + && !string_contains(s: text, pattern: "run-helper") + && string_contains(s: text, pattern: "Restart=no") + && string_contains(s: text, pattern: "Type=exec") +} + +// THE CREDENTIAL IS READ FROM THE DEVICE BY systemd, NEVER PASSED ON A COMMAND LINE, and every program +// the agent runs is named absolutely (the Mt. Collins PATH lesson). The job PATH is declared. +test fn the_credential_arrives_by_environment_file_and_every_program_is_absolute() -> Bool { + let text = serialize_systemd_unit_file(file: runner_guest_agent_unit(image: gunbc_runner_guest_image, credential_device: the_credential_device())) + string_contains(s: text, pattern: "EnvironmentFile=-/dev/vdb") + && !string_contains(s: text, pattern: "--jitconfig") + && string_contains(s: text, pattern: "Environment=PATH=/usr/local/sbin:") + && string_contains(s: text, pattern: "ExecStartPre=/usr/bin/echo gunbc-guest-observation guest-booted") + && string_contains(s: text, pattern: "ExecStartPre=/opt/actions-runner/bin/Runner.Listener --version") + && string_contains(s: text, pattern: "ExecStartPost=/usr/bin/echo gunbc-guest-observation runner-started") + && string_contains(s: text, pattern: "ExecStopPost=-/usr/bin/echo gunbc-guest-observation runner-exited") + && string_contains(s: text, pattern: join([" exit_code=", "$", "{", "EXIT_CODE", "}", " exit_status=", "$", "{", "EXIT_STATUS", "}"], "")) +} + +// THE DRIVE BYTES SURVIVE A BLOCK DEVICE. Firecracker drops a partial tail sector, so the contract +// pads to a whole number of extdeps.virtualization.firecracker firecracker_block_sector_size; the red is an unpadded content (a real 4,076-byte config would lose its +// last 492 bytes). The padding must be newlines, which systemd's parser skips, and the assignment +// must be upstream's own argument variable. +test fn the_credential_drive_is_whole_sectors_of_one_assignment() -> Bool { + let c = jit_drive_content(encoded_jit_config: "eyJhIjoiYiJ9" as NonEmptyStr) + let sector = byte_size_count(b: firecracker_block_sector_size) + string_length(s: c) % sector == 0 + && string_length(s: c) == sector + && string_length(s: jit_drive_newline_sector()) == sector + && starts_with(s: c, prefix: "ACTIONS_RUNNER_INPUT_jitconfig=eyJhIjoiYiJ9\n\n") + && !string_contains(s: c, pattern: " ") +} + +// THE HOST READS THE FOUR EVENTS OFF THE CONSOLE, and the reader is driven with the exact lines the +// unit emits -- so an edit to the emitter that the reader does not follow reds here rather than on a +// VM. RunnerListening is read from upstream's own line, and a line merely mentioning the marker +// without a known kind is not an observation. +test fn every_emitted_observation_is_read_back_as_its_own_kind() -> Bool { + let img = gunbc_runner_guest_image + let console = join([ + "[ 3.1] ", guest_observation_line(event: GuestBooted, image: img), "\n", + guest_observation_line(event: RunnerStarted, image: img), "\n", + "2026-09-19 05:20:04Z: Listening for Jobs\n", + guest_observation_line(event: RunnerExited, image: img), " exit_code=exited exit_status=0\n", + "gunbc-guest-observation something-else image=x\n", + ], "") + let labels = list_map(xs: guest_console_observations(console: console), f: fn(k) { guest_observation_kind_label(kind: k) }) + labels == ["guest-booted", "runner-started", "runner-listening", "runner-exited"] + && guest_console_carries(console: console, kind: RunnerListening) + && !guest_console_carries(console: "Listening for nothing", kind: GuestEmitted { event: GuestBooted }) } // THE REGRESSION CONTROL FOR THE ACTIVATION DEFECT, and it is written against the mechanism systemd @@ -192,3 +271,31 @@ test fn the_filesystem_size_operand_is_denominated_rather_than_a_block_count() - && any(xs: c.arguments, predicate: fn(w) { w == "ext4" }) && any(xs: c.arguments, predicate: fn(w) { w == "-d" }) } + +// THE DEVICE THE GUEST READS IS DERIVED FROM THE DRIVE LIST, AND THE DERIVATION MATCHES WHAT A REAL +// GUEST SAW. The oracle is /dev/vdb as observed on hardware (gunbc.runner.runner_first_job_receipt: +// the Mt. Collins guest read its credential off /dev/vdb with root=/dev/vda), not a copy of a +// production row. Applied to the real runner_microvm_vm_config before and after the workspace drive, +// and to the rendered unit: prepending a drive moves the derived name, reds the hardware pin here, +// and the rendered EnvironmentFile= follows the derivation rather than a literal. Controls: root is +// /dev/vda, an unattached id has no name. +test fn the_credential_device_is_where_the_vm_config_puts_the_credential_drive() -> Bool { + let cfg = runner_microvm_vm_config( + rootfs_path: "/r/rootfs.ext4", + jit_config_path: "/r/jit.img", + kernel_path: "/r/vmlinux", + net: GuestNetworkAbsent { cause: "witness" }, + size: gunbc_runner_microvm_declared_size, + transport: gunbc_runner_jit_transport, + ) + let with_work = attach_workspace_drive(config: cfg, path: "/r/work.img") + let jit = jit_transport_drive_id(t: gunbc_runner_jit_transport) + let observed_on_hardware = "/dev/vdb" + let unit = serialize_systemd_unit_file(file: runner_guest_agent_unit(image: gunbc_runner_guest_image, credential_device: the_credential_device())) + (match jit_drive_guest_device() { JitDriveDevice { device: d } => d as String == observed_on_hardware JitDriveNotAttached { cause: _ } => false }) + && string_contains(s: unit, pattern: concat("EnvironmentFile=-", observed_on_hardware)) + && (match firecracker_drive_guest_device(drives: cfg.drives, drive_id: jit) { Present { value: d } => d == observed_on_hardware Absent => false }) + && (match firecracker_drive_guest_device(drives: with_work.drives, drive_id: jit) { Present { value: d } => d == observed_on_hardware Absent => false }) + && (match firecracker_drive_guest_device(drives: cfg.drives, drive_id: "rootfs" as NonEmptyStr) { Present { value: d } => d == "/dev/vda" Absent => false }) + && (match firecracker_drive_guest_device(drives: cfg.drives, drive_id: "absent" as NonEmptyStr) { Present { value: _ } => false Absent => true }) +} diff --git a/dag/test/claim/runner/runner_microvm_boot_probe_witness_test.dag b/dag/test/claim/runner/runner_microvm_boot_probe_witness_test.dag index 02c42c618ba..e13c73595a2 100644 --- a/dag/test/claim/runner/runner_microvm_boot_probe_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_boot_probe_witness_test.dag @@ -13,11 +13,13 @@ import gunbc.runner_microvm { } import gunbc.runner_microvm_boot_probe { BootProbeVerdict, BootPrintedRunnerVersion, BootProducedNoConsole, - BootConsoleLacksRunnerVersion, BootNotAttempted, BootExceededHostDeadline, BootVmmFailed, - boot_probe_verdict, boot_probe_admitted, + BootConsoleLacksRunnerVersion, BootConsoleLacksGuestBooted, + BootNotAttempted, BootExceededHostDeadline, BootVmmFailed, + boot_probe_verdict, boot_probe_admitted, guest_agent_boot_verdict, runner_microvm_boot_probe_rootfs, runner_microvm_boot_probe_networking, runner_microvm_boot_probe_bounded_command, } +import gunbc.runner_guest_image { guest_observation_line, GuestBooted } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -162,3 +164,20 @@ test fn the_bounded_command_escalates_rather_than_asking_politely() -> Bool { && any(xs: c.arguments, predicate: fn(w) { w == "/fc/firecracker" }) && any(xs: c.arguments, predicate: fn(w) { w == "--no-api" }) } + +// A LIVE BOOT IS JUDGED THROUGH THE HOST'S CONSOLE READER, AND THIS IS THE PAIR. The same version +// readback with and without the agent's GuestBooted marker: only the marked console is a boot of +// this image's agent. The marker is produced by the emitter's own line builder, so an emitter edit +// the reader does not follow reds here. +test fn a_live_boot_requires_the_agents_guest_booted_marker() -> Bool { + let marker = guest_observation_line(event: GuestBooted, image: gunbc_runner_guest_image) + let marked = guest_agent_boot_verdict( + runner: actions_runner_release_2_336_0, + observed: process_observation(exit_code: 0, stdout: join(["[ 3.1] ", marker, "\n2.", "336.0\n"], ""), stderr: ""), + ) + let unmarked = guest_agent_boot_verdict(runner: actions_runner_release_2_336_0, observed: console_carrying_version()) + boot_probe_admitted(v: marked) + && !boot_probe_admitted(v: unmarked) + && (match unmarked { BootConsoleLacksGuestBooted { console: _ } => true _ => false }) + && boot_probe_admitted(v: boot_probe_verdict(runner: actions_runner_release_2_336_0, observed: console_carrying_version())) +}