approval loop D: converge materialization, loopback bind, mtcollins1_boot mode

.github/workflows/fleet-converge.yml

@@ -13,7 +13,7 @@ on:
       mode:
         description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown
         required: true
-        options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe]
+        options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot]
         type: choice
       target:
         description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact"
@@ -494,6 +494,73 @@ jobs:
           retention-days: 30
         if: always() && github.event.inputs.mode == 'r2_object_write_mint'
         timeout-minutes: 10
+      - name: Materialize approval MAC keys on srv1 (SM pinned -> sha256 prefix -> /etc/gunbc-roadmap 0640)
+        id: approval_keyring_converge
+        run: |
+          # 🟡 dissolve-on: gunbc_ci_approval_keyring_converge_invoke - orch-emitted foreign-executor (GitHub Actions run:) credential runner: WIF access token by env, two PINNED secret versions (capability and submission MAC keys) fetched over curl, 0600 files under RUNNER_TEMP with a trap armed BEFORE the credentials touch disk. The pipeline steps are modeled (gunbc_ci_approval_keyring_converge_prelude) but the runner transport itself remains hand-shell; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell-to-intent Phase 2) realizes the credential runner through orchestration emit or typed host_effect_apply without a medium-as-string concat scaffold
+          set -euo pipefail
+          umask 077
+          HDR_FILE="$RUNNER_TEMP/approval-keyring-auth-header"
+          RESP_FILE="$RUNNER_TEMP/approval-keyring-sm.json"
+          CAP_FILE="$RUNNER_TEMP/approval-capability-mac-key"
+          SUB_FILE="$RUNNER_TEMP/approval-submission-mac-key"
+          trap 'rm -f "$HDR_FILE" "$RESP_FILE" "$CAP_FILE" "$SUB_FILE"' EXIT
+          printf 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE"
+          DEST_FILE="$CAP_FILE"; curl -sSf -o "$RESP_FILE" -H @"$HDR_FILE" "https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/approval-capability-mac-key/versions/1:access"; python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$DEST_FILE"
+          chmod 600 "$CAP_FILE"
+          DEST_FILE="$SUB_FILE"; curl -sSf -o "$RESP_FILE" -H @"$HDR_FILE" "https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/approval-submission-mac-key/versions/1:access"; python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$DEST_FILE"
+          chmod 600 "$SUB_FILE"
+          export GUNBC_APPROVAL_CAPABILITY_MAC_KEY_FILE="$CAP_FILE"
+          export GUNBC_APPROVAL_SUBMISSION_MAC_KEY_FILE="$SUB_FILE"
+          ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
+          "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/auth/approval_keyring_converge.dag --function approval_keyring_converge_wet
+        env:
+          WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }}
+          FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }}
+        if: github.event.inputs.mode == 'approval_keyring_converge'
+        timeout-minutes: 5
+      - name: Mt. Collins unit 1 approval-gated diskless boot (file request, poll, grant, attach, CD handoff)
+        id: mtcollins1_boot
+        run: |
+          # 🟡 dissolve-on: gunbc_ci_mtcollins1_boot_invoke - orch-emitted foreign-executor (GitHub Actions run:) credential runner: WIF access token by env, PINNED BMC and submission-MAC secret versions fetched over curl, 0600 files under RUNNER_TEMP with a trap armed BEFORE the credentials touch disk. SOL hold is gunbc.machine_intake.sol_hold ActivateHeld (its own scaffold). The pipeline steps are modeled (gunbc_ci_mtcollins1_boot_credential_prelude) but the runner transport itself remains hand-shell; DISSOLVES WHEN bash-emit (#5828 / ROADMAP 6-shell-slice0 / shell-to-intent Phase 2) realizes the credential runner through orchestration emit or typed host_effect_apply without a medium-as-string concat scaffold
+          set -euo pipefail
+          umask 077
+          HDR_FILE="$RUNNER_TEMP/mtcollins1-boot-auth-header"
+          RESP_FILE="$RUNNER_TEMP/mtcollins1-boot-sm.json"
+          DEST_FILE="$RUNNER_TEMP/mtcollins1-bmc-credential"
+          SUB_FILE="$RUNNER_TEMP/approval-submission-mac-key"
+          SOL_PID_FILE="$RUNNER_TEMP/mtcollins1-sol.pid"
+          SOL_OUT="$GITHUB_WORKSPACE/artifacts/mtcollins1-boot-sol.capture"
+          trap 'if [ -f "$SOL_PID_FILE" ]; then kill "$(cat "$SOL_PID_FILE")" 2>/dev/null || true; fi; rm -f "$HDR_FILE" "$RESP_FILE" "$DEST_FILE" "$SUB_FILE" "$SOL_PID_FILE"' EXIT
+          printf 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE"
+          curl -sSf -o "$RESP_FILE" -H @"$HDR_FILE" "https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/bmc-mtcollins1-gunbc/versions/2:access"; python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$DEST_FILE"
+          chmod 600 "$DEST_FILE"
+          export GUNBC_HOST_RESET_BMC_CREDENTIAL_FILE="$DEST_FILE"
+          HOLD_BMC="$DEST_FILE"
+          DEST_FILE="$SUB_FILE"; curl -sSf -o "$RESP_FILE" -H @"$HDR_FILE" "https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/approval-submission-mac-key/versions/1:access"; python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$DEST_FILE"; DEST_FILE="$HOLD_BMC"
+          chmod 600 "$SUB_FILE"
+          export GUNBC_APPROVAL_SUBMISSION_MAC_KEY_FILE="$SUB_FILE"
+          mkdir -p "$GITHUB_WORKSPACE/artifacts"
+          : > "$SOL_OUT"
+          export GUNBC_MTCOLLINS1_SOL_CAPTURE="$SOL_OUT"
+          export GUNBC_MTCOLLINS1_SOL_PID_FILE="$SOL_PID_FILE"
+          ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd)
+          "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/machine_intake/mtcollins1_boot_run.dag --function mtcollins1_boot_wet
+        env:
+          WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }}
+          FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }}
+        if: github.event.inputs.mode == 'mtcollins1_boot'
+        timeout-minutes: 38
+      - name: Upload Mt. Collins unit 1 boot SOL capture and receipts
+        id: mtcollins1_boot_receipt_upload
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f
+        with:
+          name: mtcollins1-boot-receipts
+          path: artifacts/mtcollins1-boot-*
+          if-no-files-found: warn
+          retention-days: 30
+        if: always() && github.event.inputs.mode == 'mtcollins1_boot'
+        timeout-minutes: 10
       - name: "Runner micro-VM guest image: observe base artifacts and their digests"
         id: guest_image_observe
         run: |

dag/extdeps/bmc/ipmi.dag

@@ -102,6 +102,26 @@ service diagnostic.ipmi.Tool {
     }
   }
 
+  operation ChassisBootDevWithOptions {
+    input { bmc_host: NonEmptyStr, username: NonEmptyStr, password_file: NonEmptyStr, device: NonEmptyStr, options: NonEmptyStr }
+    output { stdout: String from "stdout", success: Bool from "exit_success" }
+    transport shell { argv: ["ipmitool", "-H", "{bmc_host}", "-I", "lanplus", "-U", "{username}", "-f", "{password_file}", "chassis", "bootdev", "{device}", "options={options}"] }
+    exit {
+      0 => Unit
+      nonzero => String "ipmitool chassis bootdev with options failed"
+    }
+  }
+
+  operation SolDeactivate {
+    input { bmc_host: NonEmptyStr, username: NonEmptyStr, password_file: NonEmptyStr }
+    output { stdout: String from "stdout", success: Bool from "exit_success" }
+    transport shell { argv: ["ipmitool", "-H", "{bmc_host}", "-I", "lanplus", "-U", "{username}", "-f", "{password_file}", "sol", "deactivate"] }
+    exit {
+      0 => Unit
+      nonzero => String "ipmitool sol deactivate failed"
+    }
+  }
+
   operation ChassisBootParamGet {
     input { bmc_host: NonEmptyStr, username: NonEmptyStr, password_file: NonEmptyStr, parameter: NonEmptyStr }
     output { stdout: String from "stdout", success: Bool from "exit_success" }

dag/extdeps/bmc/ipmi_boot_selection.dag

@@ -0,0 +1,143 @@
+module extdeps.bmc.ipmi_boot_selection
+
+import std.types { NonEmptyStr, String }
+import extdeps.external_authority { ExternalAuthority }
+import extdeps.uri { Uri, Https }
+import extdeps.toolchain.types { Architecture, Aarch64 }
+
+// THE CLI SURFACE IS ipmitool's BOOT-FLAG ENCODER, not the IPMI wire itself. The IPMI 2.0
+// chassis boot parameter has separate valid, persistence and boot-type bits; ipmitool exposes
+// them through the final `options=...` argv word. The exact implementation authority is the
+// upstream source whose option table names `valid`, `persistent` and `efiboot` and whose
+// set-bootdev path sets the valid bit before writing parameter 5.
+data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority {
+  uri: Uri {
+    scheme: Https
+    locator: "github.com/ipmitool/ipmitool/blob/master/lib/ipmi_chassis.c"
+  }
+}
+
+type IpmiBootDevice
+  = IpmiBootPxe
+  | IpmiBootCdrom
+
+type IpmiFirmwareBootMode
+  = IpmiBootLegacy
+  | IpmiBootEfi
+
+type IpmiBootPersistence
+  = IpmiBootOnce
+  | IpmiBootPersistent
+
+// A VALID BOOT SELECTION, NOT THREE INDEPENDENT SCALARS. This record is sole_constructor so a
+// caller cannot place Aarch64 beside Legacy or manufacture an opaque options word. The only
+// constructor currently exported is aarch64_uefi_once_boot_selection: its signature has no mode
+// or persistence parameter, hence `(Aarch64, Legacy, Cdrom)` and persistent intake overrides
+// have no source spelling at all. A future x86 legacy requirement adds a different constructor
+// here; it does not widen this one with a Boolean or raw-string escape hatch.
+//
+// The wire and read-back spellings travel with the semantic axes because they are one CLI
+// serialization. A caller selects `IpmiBootCdrom`; it never authors `cdrom`,
+// an `options=` word, or one of the four strings used to adjudicate parameter 5.
+// The chassis-bootdev device word is `ipmi_boot_device_label`; the options token is
+// `ipmi_boot_selection_options_token`; parameter-5 readback words are
+// `ipmi_boot_expected_*_rendering`. None of those strings is stored beside the axes.
+type IpmiBootSelection sole_constructor {
+  architecture: Architecture
+  device: IpmiBootDevice
+  firmware_mode: IpmiFirmwareBootMode
+  persistence: IpmiBootPersistence
+}
+
+fn aarch64_uefi_once_boot_selection(device: IpmiBootDevice) -> IpmiBootSelection {
+  match device {
+    IpmiBootPxe => IpmiBootSelection {
+      architecture: Aarch64,
+      device: IpmiBootPxe,
+      firmware_mode: IpmiBootEfi,
+      persistence: IpmiBootOnce,
+    }
+    IpmiBootCdrom => IpmiBootSelection {
+      architecture: Aarch64,
+      device: IpmiBootCdrom,
+      firmware_mode: IpmiBootEfi,
+      persistence: IpmiBootOnce,
+    }
+  }
+}
+
+fn ipmi_boot_device_label(device: IpmiBootDevice) -> NonEmptyStr {
+  match device {
+    IpmiBootPxe => "pxe"
+    IpmiBootCdrom => "cdrom"
+  }
+}
+
+fn ipmi_boot_mode_label(mode: IpmiFirmwareBootMode) -> NonEmptyStr {
+  match mode {
+    IpmiBootLegacy => "legacy"
+    IpmiBootEfi => "efi"
+  }
+}
+
+fn ipmi_boot_persistence_label(persistence: IpmiBootPersistence) -> NonEmptyStr {
+  match persistence {
+    IpmiBootOnce => "once"
+    IpmiBootPersistent => "persistent"
+  }
+}
+
+fn ipmi_boot_selection_label(selection: IpmiBootSelection) -> NonEmptyStr {
+  join([
+    ipmi_boot_device_label(device: selection.device) as String,
+    "/",
+    ipmi_boot_mode_label(mode: selection.firmware_mode) as String,
+    "/",
+    ipmi_boot_persistence_label(persistence: selection.persistence) as String,
+  ], "") as NonEmptyStr
+}
+
+fn ipmi_boot_selection_options_token(selection: IpmiBootSelection) -> NonEmptyStr {
+  match selection.firmware_mode {
+    IpmiBootEfi =>
+      match selection.persistence {
+        IpmiBootOnce => "efiboot"
+        IpmiBootPersistent => "efiboot,persistent"
+      }
+    IpmiBootLegacy =>
+      match selection.persistence {
+        IpmiBootOnce => "valid"
+        IpmiBootPersistent => "persistent"
+      }
+  }
+}
+
+data ipmi_boot_flag_valid_rendering: NonEmptyStr = "Boot Flag Valid"
+
+fn ipmi_boot_expected_flag_rendering(selection: IpmiBootSelection) -> NonEmptyStr {
+  match selection.device {
+    IpmiBootPxe => ipmi_boot_flag_valid_rendering
+    IpmiBootCdrom => ipmi_boot_flag_valid_rendering
+  }
+}
+
+fn ipmi_boot_expected_device_rendering(selection: IpmiBootSelection) -> NonEmptyStr {
+  match selection.device {
+    IpmiBootPxe => "Force PXE"
+    IpmiBootCdrom => "Force Boot from CD/DVD"
+  }
+}
+
+fn ipmi_boot_expected_mode_rendering(selection: IpmiBootSelection) -> NonEmptyStr {
+  match selection.firmware_mode {
+    IpmiBootEfi => "BIOS EFI boot"
+    IpmiBootLegacy => "BIOS PC Compatible (legacy) boot"
+  }
+}
+
+fn ipmi_boot_expected_persistence_rendering(selection: IpmiBootSelection) -> NonEmptyStr {
+  match selection.persistence {
+    IpmiBootOnce => "Options apply to only next boot"
+    IpmiBootPersistent => "Options apply to all future boots"
+  }
+}

dag/extdeps/bmc/megarac.dag

@@ -405,14 +405,14 @@ data megarac_ipmi_sel_layout: MegaRacEventLogLayout = MegaRacIpmiSelPayloadPrese
 // per probe and closing none. Every operation sequence that opens a session owes a close on every
 // exit, including refusals — an operation that failed still allocated the session that carried it.
 //
-// THE CREDENTIAL IS FORM-ENCODED IN THE SESSION CALL AND NOWHERE ELSE. It reaches argv there, which
-// is a real weakness and is stated rather than hidden: curl offers no netrc equivalent for a
-// form-encoded POST body, and the alternative spellings (--data @file) put the credential in a file
-// whose lifetime this operation does not control. Every operation AFTER the session carries no
-// credential at all — the cookie is the bearer — so the exposure is one call, not the whole flow.
-// The next-rung repair is a transport that reads a form body from a file the caller materializes
-// and removes, which is the same missing mechanism the removed ipmi UserSetPassword operation
-// needs.
+// THE CREDENTIAL IS READ FROM A CALLER FILE ON OpenSession AND NOWHERE ELSE. curl --data-urlencode
+// password@FILE keeps the secret out of argv; every later call carries only the cookie. THE FILE IS
+// THE PASSWORD BYTES EXACTLY: curl(1) documents name@file as URL-encoding the file's content
+// INCLUDING any newline, so a file ending in \n posts password=...%0A and the controller refuses the
+// login as a wrong password with no hint (review 68456). The caller's prelude writes the Secret
+// Manager payload bytes verbatim (base64-decoded, no trailer), so the contract lives on the stored
+// payload: it is minted without a trailing newline, and boot 22 (run 35447067113) is the executed
+// positive control that the stored bytes log in.
 //
 // StartMedia's json_body MUST CARRY image_redirection AND ITS ABSENCE IS OPAQUE. The contract note
 // on the operation row above records that omitting it makes the firmware refuse with codes
@@ -423,17 +423,30 @@ data megarac_ipmi_sel_layout: MegaRacEventLogLayout = MegaRacIpmiSelPayloadPrese
 service megarac.Media {
 
   operation OpenSession {
-    input { bmc_host: NonEmptyStr, username: NonEmptyStr, password: NonEmptyStr, cookie_jar: NonEmptyStr }
+    input { bmc_host: NonEmptyStr, username: NonEmptyStr, password_file: NonEmptyStr, cookie_jar: NonEmptyStr }
     output { body: String from "stdout", success: Bool from "exit_success" }
     transport shell {
-      argv: ["curl", "--fail-with-body", "-sS", "-k", "-c", "{cookie_jar}", "-X", "POST", "-d", "username={username}", "-d", "password={password}", "https://{bmc_host}/api/session"]
+      argv: ["curl", "--fail-with-body", "-sS", "-k", "-c", "{cookie_jar}", "-X", "POST", "--data-urlencode", "username={username}", "--data-urlencode", "password@{password_file}", "https://{bmc_host}/api/session"]
     }
     exit {
       0 => Unit
       nonzero => String "megarac session POST failed"
     }
   }
 
+  operation ProbeSessionOnMediaRoute {
+    input { bmc_host: NonEmptyStr, cookie_jar: NonEmptyStr, csrf_token: NonEmptyStr }
+    output { body: String from "stdout", success: Bool from "exit_success" }
+    readonly
+    transport shell {
+      argv: ["curl", "--fail-with-body", "-sS", "-k", "-b", "{cookie_jar}", "-H", "X-CSRFTOKEN: {csrf_token}", "-w", "\\n%\{http_code}", "https://{bmc_host}/api/settings/media/remote/configurations"]
+    }
+    exit {
+      0 => Unit
+      nonzero => String "megarac session probe on the media route failed"
+    }
+  }
+
   operation GetMediaGeneral {
     input { bmc_host: NonEmptyStr, cookie_jar: NonEmptyStr, csrf_token: NonEmptyStr }
     output { body: String from "stdout", success: Bool from "exit_success" }