From 26e9e6744936c7e8a366bcb95ee0c47ebc232e2e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 3 Jul 2026 17:35:39 +0000 Subject: [PATCH 01/58] fix(srv3): DirectLayout on-ISO autoinstall + os-install reconcile spine (T3/T4) Rebased onto main: StoragePolicyDirectLayout on seeded ISO autoinstall, split reconcile modules (core/types/receipt/apply/dry_run/record_approval), honest freeze scope and printf receipt echo, fail-closed observed_at parse, ServeReady virtual-media session match, and review-driven witness coverage. Co-authored-by: Cursor --- dag/gunbc/os_install.dag | 2 +- dag/gunbc/os_install_deduction.dag | 2 +- dag/gunbc/shell_bash_runner.dag | 14 +- .../srv3_install_diagnostic_checklist.dag | 132 +++-- dag/gunbc/srv3_os_install_diagnostic.dag | 148 +++-- dag/gunbc/srv3_os_install_reconcile.dag | 560 ++++++++++++++++++ dag/gunbc/srv3_os_install_reconcile_apply.dag | 207 +++++++ .../srv3_os_install_reconcile_dry_run.dag | 114 ++++ .../srv3_os_install_reconcile_receipt.dag | 532 +++++++++++++++++ ...3_os_install_reconcile_record_approval.dag | 32 + dag/gunbc/srv3_os_install_reconcile_types.dag | 46 ++ .../os_install_deduction_witness_test.dag | 31 +- .../claim/shell_bash_runner_witness_test.dag | 14 + ...tall_diagnostic_checklist_witness_test.dag | 35 ++ ...srv3_os_install_reconcile_witness_test.dag | 321 ++++++++++ src/v2/lens/medium_structure_containment.dag | 8 +- .../provenance_fail_closed_contract_test.dag | 103 ++++ 17 files changed, 2208 insertions(+), 93 deletions(-) create mode 100644 dag/gunbc/srv3_os_install_reconcile.dag create mode 100644 dag/gunbc/srv3_os_install_reconcile_apply.dag create mode 100644 dag/gunbc/srv3_os_install_reconcile_dry_run.dag create mode 100644 dag/gunbc/srv3_os_install_reconcile_receipt.dag create mode 100644 dag/gunbc/srv3_os_install_reconcile_record_approval.dag create mode 100644 dag/gunbc/srv3_os_install_reconcile_types.dag create mode 100644 dag/test/claim/srv3_os_install_reconcile_witness_test.dag create mode 100644 src/v2/test/fixture/floor_skip/provenance_fail_closed_contract_test.dag diff --git a/dag/gunbc/os_install.dag b/dag/gunbc/os_install.dag index c6e14a431b9..81cab6d5550 100644 --- a/dag/gunbc/os_install.dag +++ b/dag/gunbc/os_install.dag @@ -145,7 +145,7 @@ data srv3_ubuntu_autoinstall_on_iso: UbuntuAutoinstallPayload = UbuntuAutoinstal locale: "en_US.UTF-8", keyboard_layout: "us", install_openssh_server: true, - storage_policy: StoragePolicyAbsent, + storage_policy: StoragePolicyDirectLayout, } data srv3_os_install_plan: Srv3OsInstallPlan = Srv3OsInstallPlan { diff --git a/dag/gunbc/os_install_deduction.dag b/dag/gunbc/os_install_deduction.dag index 7da4d432220..1162efabc60 100644 --- a/dag/gunbc/os_install_deduction.dag +++ b/dag/gunbc/os_install_deduction.dag @@ -11,7 +11,7 @@ import extdeps.os.ubuntu_autoinstall { } import product.placement_supply { HostIdentity } -data os_install_deduction_doc: String = "OS install deduction spine β€” construction-time preflight (intent+policy completeness) and runtime KVM-attested diagnostic vocabulary (T1 only; not wired to srv3_os_install_diagnostic fold until srv3-install-reconcile-a). Preflight fold is scoped to the NBD-proxy/on-ISO actuator path: NoCloudNet is incomplete for that path only, not universally. Not a monitoring product (roadmap 2-os-install-deduction-a). Deduction half pairs with std.temporal_effect receipt/resume half." +data os_install_deduction_doc: String = "OS install deduction spine β€” construction-time preflight (intent+policy completeness) and runtime KVM-attested diagnostic vocabulary. Consumed by gunbc.srv3_os_install_reconcile (srv3-install-reconcile-a). Preflight fold is scoped to the NBD-proxy/on-ISO actuator path: NoCloudNet is incomplete for that path only, not universally." type TargetDiskState = EmptyDisk diff --git a/dag/gunbc/shell_bash_runner.dag b/dag/gunbc/shell_bash_runner.dag index 53df747e6fa..638f261d38d 100644 --- a/dag/gunbc/shell_bash_runner.dag +++ b/dag/gunbc/shell_bash_runner.dag @@ -15,7 +15,19 @@ data shell_bash_runner_scaffold: Disposition = Scaffold { } } -data shell_bash_runner_dissolution_trigger: String = "🟑 dissolve-on: shell_exec_via_bash β€” bash heredoc wrapper built with concat (medium-as-string) so multiline scripts run under shell.Exec.Run without sh -c first-line truncation; heredoc delimiter GUNBC_BASH_EOF is a fixed literal β€” wrapped body must not contain it or the outer shell terminates the heredoc early (nonce delimiter or BashVarRef transport when bash_fold lands); DISSOLVES WHEN host_effect_apply / bash_fold emit binds multiline shell transport without concat heredoc framing; retire medium_structure_exception_roster entry for dag/gunbc/shell_bash_runner.dag" +data shell_bash_runner_dissolution_trigger: String = "🟑 dissolve-on: shell_exec_via_bash β€” bash heredoc wrapper built with concat (medium-as-string) so multiline scripts run under shell.Exec.Run without sh -c first-line truncation; heredoc delimiter GUNBC_BASH_EOF is a fixed literal β€” wrapped body must not contain it or the outer shell terminates the heredoc early (nonce delimiter or BashVarRef transport when bash_fold lands); shell_printf_line emits printf '%s\\n' with bash-double-quote escaping (not naive echo single-quote concat); DISSOLVES WHEN host_effect_apply / bash_fold emit binds multiline shell transport without concat heredoc framing; retire medium_structure_exception_roster entry for dag/gunbc/shell_bash_runner.dag" + +fn bash_escape_for_double_quotes(s: String) -> String { + let s1 = join(split(s: s, delimiter: "\\"), "\\\\") + let s2 = join(split(s: s1, delimiter: "\""), "\\\"") + let s3 = join(split(s: s2, delimiter: "$"), "\\$") + let s4 = join(split(s: s3, delimiter: "`"), "\\`") + join(split(s: s4, delimiter: "\n"), "\\n") +} + +fn shell_printf_line(line: String) -> String { + concat("printf '%s\\n' \"", concat(bash_escape_for_double_quotes(s: line), "\"\n")) +} fn shell_exec_via_bash(body: String) -> String { concat( diff --git a/dag/gunbc/srv3_install_diagnostic_checklist.dag b/dag/gunbc/srv3_install_diagnostic_checklist.dag index 6728183b15c..492a026da17 100644 --- a/dag/gunbc/srv3_install_diagnostic_checklist.dag +++ b/dag/gunbc/srv3_install_diagnostic_checklist.dag @@ -2,7 +2,7 @@ module gunbc.srv3_install_diagnostic_checklist import extdeps.shell import extdeps.filesystem.filesystem_io { Filesystem } -import gunbc.shell_bash_runner { shell_exec_via_bash } +import gunbc.shell_bash_runner { shell_exec_via_bash, shell_printf_line } import gunbc.srv3_os_install_actuate { srv3_bmcweb_token_path } import gunbc.srv3_os_install_diagnostic { srv3_os_install_attempt_intent, @@ -35,7 +35,7 @@ data srv3_install_diagnostic_checklist_scaffold: Disposition = Scaffold { } } -data srv3_install_diagnostic_checklist_shell_runner_dissolution_trigger: String = "🟑 dissolve-on: srv3_install_diagnostic_checklist β€” observe script via shell_exec_via_bash + stdout parse into Srv3InstallDebugObservation then diagnose_srv3_install; DISSOLVES WHEN host_effect_apply binds Srv3InstallDiagnosticObserve without concat runner glue; retire medium_structure_exception_roster entry for dag/gunbc/srv3_install_diagnostic_checklist.dag" +data srv3_install_diagnostic_checklist_shell_runner_dissolution_trigger: String = "🟑 dissolve-on: srv3_install_diagnostic_checklist β€” observe script via shell_exec_via_bash + stdout parse into Srv3InstallDebugObservation then diagnose_srv3_install; curl/redfish/sol concat tails in this module are TERMINAL for the reconcile spine β€” FROZEN effective at merge (no further concat-shell growth here; this PR's tail expansion is pre-freeze forward-fix for TOKEN_PATH/SOL/NBDKIT markers); receipt echo via shell_printf_line; DISSOLVES WHEN host_effect_apply binds Srv3InstallDiagnosticObserve without concat runner glue; retire medium_structure_exception_roster entry for dag/gunbc/srv3_install_diagnostic_checklist.dag" data srv3_install_diagnostic_observe_script_path: String = "/tmp/srv3_install_diagnostic_observe.sh" @@ -51,18 +51,36 @@ fn curl_http_code_writeout() -> String { fn srv3_install_diagnostic_observe_script_curl_tail() -> String { concat( - concat("$", "(curl -sk -o /dev/null -w \""), + "if test -s \"", concat( - curl_http_code_writeout(), + bash_var_ref(name: "TOKEN_PATH"), concat( - "\" -H \"Connection: Upgrade\" -H \"Upgrade: websocket\" -H \"Sec-WebSocket-Version: 13\" -H \"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\" \"https://", + "\"; then\n HTTP_CODE=", concat( - bash_var_ref(name: "BMC"), + concat("$", "(curl -sk -o /dev/null -w \""), concat( - "/vm/0/0\" 2>/dev/null || echo 000)\necho \"Srv3InstallObserveReceipt: surface=bmc_websocket http_code=", + curl_http_code_writeout(), concat( - bash_var_ref(name: "HTTP_CODE"), - "\"\nif test -s \"", + "\" -H \"Connection: Upgrade\" -H \"Upgrade: websocket\" -H \"Sec-WebSocket-Version: 13\" -H \"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\" -H \"Authorization: Token ", + concat( + concat("$", "(cat \""), + concat( + bash_var_ref(name: "TOKEN_PATH"), + concat( + "\")\" \"https://", + concat( + bash_var_ref(name: "BMC"), + concat( + "/vm/0/0\" 2>/dev/null || echo 000))\nelse\n HTTP_CODE=000\nfi\necho \"Srv3InstallObserveReceipt: surface=bmc_websocket http_code=", + concat( + bash_var_ref(name: "HTTP_CODE"), + "\"\nif test -s \"", + ), + ), + ), + ), + ), + ), ), ), ), @@ -73,48 +91,51 @@ fn srv3_install_diagnostic_observe_script_curl_tail() -> String { fn srv3_install_diagnostic_observe_script_redfish_tail() -> String { concat( - "\"; then\n RF=", + bash_var_ref(name: "TOKEN_PATH"), concat( - concat("$", "(curl -sk -H \"Authorization: Token "), + "\"; then\n RF=", concat( - concat("$", "(cat \""), + concat("$", "(curl -sk -H \"Authorization: Token "), concat( - bash_var_ref(name: "TOKEN_PATH"), + concat("$", "(cat \""), concat( - "\")\" \"https://", + bash_var_ref(name: "TOKEN_PATH"), concat( - bash_var_ref(name: "BMC"), + "\")\" \"https://", concat( - "/redfish/v1/Systems/system\")\n POWER=", + bash_var_ref(name: "BMC"), concat( - concat("$", "(echo \""), + "/redfish/v1/Systems/system\")\n POWER=", concat( - bash_var_ref(name: "RF"), + concat("$", "(echo \""), concat( - "\" | jq -r .PowerState)\n BOOT_ENABLED=", + bash_var_ref(name: "RF"), concat( - concat("$", "(echo \""), + "\" | jq -r .PowerState)\n BOOT_ENABLED=", concat( - bash_var_ref(name: "RF"), + concat("$", "(echo \""), concat( - "\" | jq -r .Boot.BootSourceOverrideEnabled)\n BOOT_TARGET=", + bash_var_ref(name: "RF"), concat( - concat("$", "(echo \""), + "\" | jq -r .Boot.BootSourceOverrideEnabled)\n BOOT_TARGET=", concat( - bash_var_ref(name: "RF"), + concat("$", "(echo \""), concat( - "\" | jq -r .Boot.BootSourceOverrideTarget)\n echo \"Srv3InstallObserveReceipt: surface=redfish power=", + bash_var_ref(name: "RF"), concat( - bash_var_ref(name: "POWER"), + "\" | jq -r .Boot.BootSourceOverrideTarget)\n echo \"Srv3InstallObserveReceipt: surface=redfish power=", concat( - " boot_enabled=", + bash_var_ref(name: "POWER"), concat( - bash_var_ref(name: "BOOT_ENABLED"), + " boot_enabled=", concat( - " boot_target=", + bash_var_ref(name: "BOOT_ENABLED"), concat( - bash_var_ref(name: "BOOT_TARGET"), - "\"\nelse\n echo \"Srv3InstallObserveReceipt: surface=redfish status=token_missing\"\nfi\nif test -s \"", + " boot_target=", + concat( + bash_var_ref(name: "BOOT_TARGET"), + "\"\nelse\n echo \"Srv3InstallObserveReceipt: surface=redfish status=token_missing\"\nfi\nif test -s \"", + ), ), ), ), @@ -140,22 +161,49 @@ fn srv3_install_diagnostic_observe_script_redfish_tail() -> String { fn srv3_install_diagnostic_observe_script_sol_tail() -> String { concat( - "\"; then\n echo \"Srv3InstallObserveReceipt: surface=sol status=present\"\n grep -qi grub \"", + bash_var_ref(name: "SOL_LOG"), concat( - bash_var_ref(name: "SOL_LOG"), + "\"; then\n echo \"Srv3InstallObserveReceipt: surface=sol status=present\"\n grep -qi grub \"", concat( - "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=grub\" || true\n grep -qi subiquity \"", + bash_var_ref(name: "SOL_LOG"), concat( - bash_var_ref(name: "SOL_LOG"), + "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=grub\" || true\n grep -qi subiquity \"", concat( - "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=subiquity\" || true\n grep -qi autoinstall \"", + bash_var_ref(name: "SOL_LOG"), concat( - bash_var_ref(name: "SOL_LOG"), + "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=subiquity\" || true\n grep -qi autoinstall \"", concat( - "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=autoinstall\" || true\n grep -qi -E 'error|fail' \"", + bash_var_ref(name: "SOL_LOG"), concat( - bash_var_ref(name: "SOL_LOG"), - "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=error\" || true\nelse\n echo \"Srv3InstallObserveReceipt: surface=sol status=silent\"\nfi\necho \"Srv3InstallObserveReceipt: surface=router status=operator_required\"\n", + "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=autoinstall\" || true\n grep -qiE 'UEFI|Setup|Firmware|BIOS' \"", + concat( + bash_var_ref(name: "SOL_LOG"), + concat( + "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=firmware_menu\" || true\n grep -qi -E 'error|fail' \"", + concat( + bash_var_ref(name: "SOL_LOG"), + concat( + "\" && echo \"Srv3InstallObserveReceipt: surface=sol marker=error\" || true\nelse\n echo \"Srv3InstallObserveReceipt: surface=sol status=silent\"\nfi\nNBDKIT_PID=$(pgrep -n -x nbdkit 2>/dev/null || true)\nif [ -n \"", + concat( + bash_var_ref(name: "NBDKIT_PID"), + concat( + "\" ]; then\n ELAPSED_MS=$(( $(ps -o etimes= -p \"", + concat( + bash_var_ref(name: "NBDKIT_PID"), + concat( + "\" 2>/dev/null | tr -d ' ') * 1000 ))\nelse\n ELAPSED_MS=0\nfi\necho \"Srv3InstallObserveReceipt: surface=elapsed boot_once_ms=", + concat( + bash_var_ref(name: "ELAPSED_MS"), + "\"\necho \"Srv3InstallObserveReceipt: surface=router status=operator_required\"\n", + ), + ), + ), + ), + ), + ), + ), + ), + ), ), ), ), @@ -349,8 +397,8 @@ func srv3_install_diagnostic_checklist() -> ProcessExit { let receipt = srv3_install_diagnostic_receipt_line(verdict: verdict, observation: observation) let action = srv3_install_diagnostic_recommended_action_wire(verdict: verdict) let echo_body = concat( - concat(receipt, "\n"), - concat("Srv3InstallDiagnosticAction: ", action, "\n"), + shell_printf_line(line: receipt), + shell_printf_line(line: concat("Srv3InstallDiagnosticAction: ", action)), ) let echo_run = shell.Exec.Run(script: shell_exec_via_bash(body: echo_body)) if echo_run.success { diff --git a/dag/gunbc/srv3_os_install_diagnostic.dag b/dag/gunbc/srv3_os_install_diagnostic.dag index ce14e46ec67..9695e323f88 100644 --- a/dag/gunbc/srv3_os_install_diagnostic.dag +++ b/dag/gunbc/srv3_os_install_diagnostic.dag @@ -934,6 +934,20 @@ fn diagnose_srv3_install_when_serve_observed( } } +fn srv3_install_pre_boot_ready_to_boot( + observation: Srv3InstallDebugObservation, +) -> Bool { + if !install_serve_session_active(observation: observation) + || !redfish_boot_override_is_disabled_none(obs: observation.redfish_boot) { + false + } else { + match observation.elapsed_since_boot_once_ms { + Absent => true + Present { value: elapsed } => elapsed == 0 + } + } +} + fn diagnose_srv3_install_when_serve_ready( observation: Srv3InstallDebugObservation, redfish: RedfishSystemBootVerdict, @@ -944,36 +958,29 @@ fn diagnose_srv3_install_when_serve_ready( match redfish { BootOverrideCdOnceArmed => InstallerProbablyRunning BootOverrideDisabledAfterAttempt => - match observation.virtual_media_session { - AttachedToExpectedServe => - if elapsed == 0 { - ReadyToBoot - } else { - diagnose_srv3_install_post_boot( - observation: observation, - redfish: redfish, - sol: sol, - router: router, - elapsed: elapsed, - ) - } - _ => - diagnose_srv3_install_post_boot( - observation: observation, - redfish: redfish, - sol: sol, - router: router, - elapsed: elapsed, - ) + if srv3_install_pre_boot_ready_to_boot(observation: observation) { + ReadyToBoot + } else { + diagnose_srv3_install_post_boot( + observation: observation, + redfish: redfish, + sol: sol, + router: router, + elapsed: elapsed, + ) } _ => - diagnose_srv3_install_post_boot( - observation: observation, - redfish: redfish, - sol: sol, - router: router, - elapsed: elapsed, - ) + if srv3_install_pre_boot_ready_to_boot(observation: observation) { + ReadyToBoot + } else { + diagnose_srv3_install_post_boot( + observation: observation, + redfish: redfish, + sol: sol, + router: router, + elapsed: elapsed, + ) + } } } @@ -1008,11 +1015,20 @@ fn diagnose_srv3_install_post_boot( if elapsed >= srv3_install_hang_no_router_lease_ms { InstallerHung } else { - if elapsed == 0 - && redfish_boot_override_is_disabled_none(obs: observation.redfish_boot) { - BootOverrideNotArmed + if srv3_install_pre_boot_ready_to_boot(observation: observation) { + ReadyToBoot } else { - BootOverrideConsumedNoInstallEvidence + if install_serve_session_active(observation: observation) + && redfish_boot_override_is_disabled_none(obs: observation.redfish_boot) { + BootOverrideConsumedNoInstallEvidence + } else { + if elapsed == 0 + && redfish_boot_override_is_disabled_none(obs: observation.redfish_boot) { + BootOverrideNotArmed + } else { + BootOverrideConsumedNoInstallEvidence + } + } } } _ => BootOverrideConsumedNoInstallEvidence @@ -1043,14 +1059,26 @@ fn observe_stdout_has_serve_surface(stdout: String) -> Bool { string_contains(s: stdout, pattern: "Srv3InstallObserveReceipt: surface=serve") } +fn observe_stdout_bmc_websocket_line(stdout: String) -> String { + let parts = split(s: stdout, delimiter: "surface=bmc_websocket") + match parts.length() <= 1 { + true => concat("", "") + false => { + let tail = parts.skip(n: 1).first() + split(s: tail, delimiter: "\n").first() + } + } +} + fn parse_bmc_websocket_observation(stdout: String) -> BmcWebsocketObservation { - if string_contains(s: stdout, pattern: "http_code=101") { - WebsocketUpgradeAccepted + let line = observe_stdout_bmc_websocket_line(stdout: stdout) + if line == concat("", "") { + WebsocketUnknown } else { - if string_contains(s: stdout, pattern: "surface=bmc_websocket") { - WebsocketUnavailable + if string_contains(s: line, pattern: "http_code=101") { + WebsocketUpgradeAccepted } else { - WebsocketUnknown + WebsocketUnavailable } } } @@ -1121,8 +1149,8 @@ fn parse_observe_content_hash_observed(stdout: String) -> ContentHash { "content-hash-unobserved" as ContentHash } else { let tail = parts.skip(n: 1).first() - let token = split(s: tail, delimiter: " ").first() - token as ContentHash + let line = split(s: tail, delimiter: "\n").first() + split(s: line, delimiter: " ").first() as ContentHash } } @@ -1183,7 +1211,7 @@ fn parse_sol_console_observation(stdout: String) -> SolConsoleContentObservation contains_grub: string_contains(s: stdout, pattern: "marker=grub"), contains_subiquity: string_contains(s: stdout, pattern: "marker=subiquity"), contains_autoinstall: string_contains(s: stdout, pattern: "marker=autoinstall"), - contains_firmware_menu: false, + contains_firmware_menu: string_contains(s: stdout, pattern: "marker=firmware_menu"), contains_error: string_contains(s: stdout, pattern: "marker=error"), capture_available: true, } @@ -1197,7 +1225,7 @@ fn parse_virtual_media_session_observation( ) -> VirtualMediaSessionObservation { match serve { ServeUnknown => VirtualMediaSessionUnknown - ServeObserved { process: _, port: _, session: _, iso: _ } => + ServeObserved { process: proc, port: _, session: _, iso: _ } => match websocket { WebsocketUpgradeAccepted => match serve_readiness { @@ -1209,6 +1237,40 @@ fn parse_virtual_media_session_observation( } } +fn install_serve_session_active(observation: Srv3InstallDebugObservation) -> Bool { + match observation.actuator_serve { + ServeObserved { process: proc, port: port, session: sess, iso: _ } => + match proc { + ServeProcessesPresent => + match port { + ServePortListening { owner_matches_expected: true, port: _ } => + match sess { + TokenPresent => true + TokenMissing => false + } + _ => false + } + _ => false + } + ServeUnknown => false + } +} + +fn parse_elapsed_boot_once_ms_observation(stdout: String) -> Milliseconds? { + let parts = split(s: stdout, delimiter: "surface=elapsed boot_once_ms=") + match parts.length() <= 1 { + true => Absent + false => { + let tail = parts.skip(n: 1).first() + let token = split(s: tail, delimiter: "\n").first() + match parse_int(s: token) { + Present { value: n } => Present { value: n } + Absent => Absent + } + } + } +} + fn parse_srv3_install_debug_observation( stdout: String, intent: OsInstallAttemptIntent, @@ -1231,7 +1293,7 @@ fn parse_srv3_install_debug_observation( sol: Present { value: parse_sol_console_observation(stdout: stdout) }, router_lease_table: Absent, kvm: Absent, - elapsed_since_boot_once_ms: Absent, + elapsed_since_boot_once_ms: parse_elapsed_boot_once_ms_observation(stdout: stdout), } } @@ -1278,7 +1340,7 @@ fn srv3_install_diagnostic_recommended_action_wire(verdict: Srv3InstallDiagnosti ServeDown => "start or repair srv3_nbd_proxy_serve held session on srv1 before boot-once" BmcWebsocketUnavailable => "verify BMC /vm/0/0 websocket path before virtual media" BootOverrideNotArmed => "re-arm boot-once CD when serve is ReadyToBoot" - BootOverrideConsumedNoInstallEvidence => "serve alive: re-arm boot-once CD + force restart; verify SOL/router" + BootOverrideConsumedNoInstallEvidence => "await PlanAwaitApproval(srv3-boot-once-cd); verify SOL and router before granting durable approval" InstallerProbablyRunning => "monitor SOL and router DHCP option-12 srv3" InstallerHung => "declare hang per runbook thresholds; repair serve/boot path before retry" OsInstalled => "confirm router dynamic lease option-12 srv3; proceed post-install witnesses" diff --git a/dag/gunbc/srv3_os_install_reconcile.dag b/dag/gunbc/srv3_os_install_reconcile.dag new file mode 100644 index 00000000000..2cbd4d8448f --- /dev/null +++ b/dag/gunbc/srv3_os_install_reconcile.dag @@ -0,0 +1,560 @@ +module gunbc.srv3_os_install_reconcile + +import std.types { String, NonEmptyStr, ContentHash, Timestamp, Bool, Int } +import std.temporal_effect { + EffectAttemptIntent, + EffectStepReceipt, + AppliedEffectRef, + NoEffectApplied, + EffectApplied, + HeldLease, + HeldLeaseObservedState, + LeaseAbsent, + LeaseRunningExpected, + LeaseRunningStale, + LeasePortForeign, + LeaseInaccessible, + DurableApprovalGrant, + GrantAbsent, + GrantActive, + GrantConsumed, + PriorStepReceipt, + NoPriorReceipt, + PriorReceipt, + NextStepPlan, + PlanNoop, + PlanApply, + PlanRefuse, + PlanAwaitApproval, + PlanDrainThenStart, + plan_next_step_from_prior_receipt_and_lease, + approval_grant_covers_intent, +} +import gunbc.workflow_escalation { + WorkflowApprovalState, + ApprovalGranted, + PendingApproval, +} +import gunbc.srv3_boot_once_cd { srv3_boot_once_cd_gated } +import gunbc.os_install { srv3_ubuntu_autoinstall_on_iso } +import gunbc.os_install_deduction { + TargetDiskState, + ForeignOsPresent, + OsInstallPreflightVerdict, + OsInstallPreflightPlan, + OsInstallRuntimeDiagnosticVerdict, + KvmOperatorAttestation, + KvmObservedScreen, + FirmwareSetup, + UefiBootMenu, + GrubMenu, + SubiquityStorageScreen, + fold_nbd_proxy_os_install_preflight_verdict, + plan_for_os_install_preflight_verdict, + runtime_verdict_from_kvm_attestation, +} +import gunbc.srv3_os_install_diagnostic { + srv3_os_install_attempt_intent, + OsInstallAttemptIntent, + Srv3InstallDebugObservation, + Srv3InstallDiagnostic, + NbdServeObservation, + NbdServeProcessObservation, + NbdServePortObservation, + BmcSessionTokenObservation, + IsoServeReadinessObservation, + ServeUnknown, + ServeObserved, + NbdkitAbsent, + WebsocatAbsent, + ServeProcessesPresent, + ServePortAbsent, + ServePortConflict, + ServePortListening, + IsoMissing, + IsoPresent, + TokenMissing, + TokenPresent, + BootOverrideConsumedNoInstallEvidence, + InstallerProbablyRunning, + RedfishBootObservation, + BmcWebsocketObservation, + WebsocketUpgradeAccepted, + WebsocketUnavailable, + VirtualMediaSessionObservation, + AttachedToExpectedServe, + NoAttachedServe, + VirtualMediaSessionUnknown, + classify_router_lease_table, + Srv3OsLeasePresent, + Srv3OsLeaseAbsent, + OnlyBmcLeasePresent, + WrongHostnameLeasePresent, + diagnose_srv3_install, + srv3_install_diagnostic_wire, + srv3_install_diagnostic_recommended_action_wire, + install_serve_session_active, + srv3_install_pre_boot_ready_to_boot, +} +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.bmc.types { BootOverrideDisabled, BootTargetNone, BootOverrideOnce, BootTargetCd } +import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv3 } +import gunbc.srv3_seeded_install_media { srv3_seeded_install_media_content_hash } +import gunbc.srv3_os_install_actuate_workflow { srv3_boot_once_cd_escalation } +import extdeps.bmc.webui.nbd_proxy_serve { srv3_nbd_proxy_local_port } +import gunbc.srv3_os_install_reconcile_types { + Srv3InstallReconcileResult, + DeductionRuntimeHint, + srv3_boot_once_cd_step_id, + srv3_boot_once_approval_grant_path, + srv3_boot_once_apply_read_back_sleep, + srv3_install_reconcile_default_disk, + srv3_nbd_proxy_lease_key, + srv3_install_effect_attempt_intent, + srv3_install_reconcile_dry_run_observed_at, +} + +data srv3_os_install_reconcile_doc: String = "srv3-install-reconcile-a β€” folds os_install_deduction preflight + srv3_os_install_diagnostic + std.temporal_effect next-step plan. Dry-run (T3) and apply+read-back (T4) entrypoints on srv1 actuator." + +data srv3_os_install_reconcile_shell_runner_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.srv3_os_install_reconcile", + decl_name: "srv3_os_install_reconcile_shell_runner_dissolution_trigger", + field: WholeDeclaration, + } +} + +data srv3_os_install_reconcile_shell_runner_dissolution_trigger: String = "🟑 dissolve-on: srv3_os_install_reconcile shell runner glue β€” dry_run/apply/record_approval orchestrate via shell.Exec.Run over concat-built observe meta, durable-approval grant write, grant read, read-back sleep, and receipt echo scripts (srv3_install_record_boot_once_approval_script, srv3_install_reconcile_observe_meta_script); receipt echo via shell_printf_line (printf '%s\\n' + bash double-quote escape β€” not naive echo single-quote concat); diagnostic observer concat freeze is on srv3_install_diagnostic_checklist_shell_runner_dissolution_trigger (post-merge bound, not retroactive); next node is typed Observe* effect rows + Srv3InstallObserveViaShell handler only; DISSOLVES WHEN host_effect_apply binds Srv3InstallReconcileObserve + DurableApprovalGrantRecord + Srv3InstallReconcileApply without shell_exec_via_bash runner glue; retire medium_structure_exception_roster entries for dag/gunbc/srv3_os_install_reconcile*.dag" + +fn held_lease_observed_from_serve(serve: NbdServeObservation) -> HeldLeaseObservedState { + match serve { + ServeUnknown => LeaseInaccessible + ServeObserved { process: proc, port: port, session: _, iso: _ } => + match proc { + NbdkitAbsent => LeaseAbsent + WebsocatAbsent => LeaseAbsent + ServeProcessesPresent => + match port { + ServePortAbsent => LeaseAbsent + ServePortConflict => LeasePortForeign + ServePortListening { port: _, owner_matches_expected: owner_ok } => + if owner_ok { + LeaseRunningExpected + } else { + LeasePortForeign + } + } + } + } +} + +fn srv3_held_lease_from_serve(serve: NbdServeObservation) -> HeldLease { + HeldLease { + lease_key: srv3_nbd_proxy_lease_key, + resource_fingerprint: srv3_seeded_install_media_content_hash, + owner_fingerprint: "gunbc-nbd-proxy" as ContentHash, + generation: 1, + observed: held_lease_observed_from_serve(serve: serve), + } +} + +fn kvm_screen_from_diagnostic( + diagnostic: Srv3InstallDiagnostic, + observation: Srv3InstallDebugObservation, +) -> KvmObservedScreen { + match diagnostic { + BootOverrideConsumedNoInstallEvidence => + match observation.sol { + Present { value: sol } => + if sol.contains_firmware_menu { + FirmwareSetup + } else { + if sol.contains_grub { + GrubMenu + } else { + UefiBootMenu + } + } + Absent => UefiBootMenu + } + InstallerProbablyRunning => + match observation.sol { + Present { value: sol } => + if sol.contains_subiquity { + SubiquityStorageScreen + } else { + if sol.contains_grub { + GrubMenu + } else { + UefiBootMenu + } + } + Absent => UefiBootMenu + } + _ => UefiBootMenu + } +} + +fn optional_kvm_attestation_from_observation( + diagnostic: Srv3InstallDiagnostic, + observation: Srv3InstallDebugObservation, + observed_at: Timestamp, +) -> KvmOperatorAttestation? { + match diagnostic { + BootOverrideConsumedNoInstallEvidence => Present { + value: KvmOperatorAttestation { + target: operator_host_srv3, + observed_screen: kvm_screen_from_diagnostic( + diagnostic: diagnostic, + observation: observation, + ), + observed_at: observed_at, + attested_by: "reconcile-infer-sol-redfish" as NonEmptyStr, + }, + } + _ => Absent + } +} + +fn deduction_runtime_hint( + preflight: OsInstallPreflightVerdict, + diagnostic: Srv3InstallDiagnostic, + observation: Srv3InstallDebugObservation, + observed_at: Timestamp, +) -> DeductionRuntimeHint { + match optional_kvm_attestation_from_observation( + diagnostic: diagnostic, + observation: observation, + observed_at: observed_at, + ) { + Present { value: att } => DeductionRuntimePresent { + verdict: runtime_verdict_from_kvm_attestation( + attestation: att, + preflight: preflight, + ), + } + Absent => DeductionRuntimeAbsent + } +} + +fn reconcile_pending_step_id(diagnostic: Srv3InstallDiagnostic) -> NonEmptyStr { + match diagnostic { + ReadyToBoot => "boot-once-cd" as NonEmptyStr + ServeDown => "nbd-serve" as NonEmptyStr + BootOverrideNotArmed => "boot-once-cd" as NonEmptyStr + BootOverrideConsumedNoInstallEvidence => "boot-once-cd" as NonEmptyStr + InstallerProbablyRunning => "monitor-install" as NonEmptyStr + OsInstalled => "post-install-verify" as NonEmptyStr + _ => "diagnose" as NonEmptyStr + } +} + +fn reconcile_requires_destructive_approval(step_id: NonEmptyStr) -> Bool { + step_id == srv3_boot_once_cd_step_id +} + +fn plan_boot_once_cd_with_lease(lease: HeldLease, step_id: NonEmptyStr) -> NextStepPlan { + match lease.observed { + LeaseRunningExpected => PlanApply { step_id: step_id } + LeaseAbsent => + PlanRefuse { + step_id: step_id, + reason: "nbd serve lease absent before boot-once-cd", + } + LeasePortForeign => + PlanRefuse { + step_id: step_id, + reason: "foreign lease holder on scoped nbd serve resource", + } + LeaseRunningStale => + PlanRefuse { + step_id: step_id, + reason: "stale nbd serve lease before boot-once-cd", + } + LeaseInaccessible => + PlanRefuse { + step_id: step_id, + reason: "nbd serve lease observation inaccessible", + } + } +} + +fn plan_boot_once_cd_step( + intent: EffectAttemptIntent, + escalation_id: NonEmptyStr, + grant: DurableApprovalGrant, + lease: HeldLease, + prior: PriorStepReceipt, + observed_at: Timestamp, +) -> NextStepPlan { + if !approval_grant_covers_intent( + grant: grant, + intent: intent, + escalation_id: escalation_id, + observed_at: observed_at, + ) { + PlanAwaitApproval { escalation_id: escalation_id } + } else { + match prior { + PriorReceipt { receipt: r } => + if r.step_id == srv3_boot_once_cd_step_id && r.read_back_ok { + PlanNoop { step_id: srv3_boot_once_cd_step_id } + } else { + if r.step_id == srv3_boot_once_cd_step_id && !r.read_back_ok { + PlanRefuse { + step_id: srv3_boot_once_cd_step_id, + reason: "prior boot-once-cd applied without read-back convergence", + } + } else { + plan_boot_once_cd_with_lease(lease: lease, step_id: srv3_boot_once_cd_step_id) + } + } + NoPriorReceipt => + plan_boot_once_cd_with_lease(lease: lease, step_id: srv3_boot_once_cd_step_id) + } + } +} + +fn reconcile_temporal_plan( + pending_step_id: NonEmptyStr, + grant: DurableApprovalGrant, + lease: HeldLease, + prior: PriorStepReceipt, + observed_at: Timestamp, +) -> NextStepPlan { + if pending_step_id == srv3_boot_once_cd_step_id { + plan_boot_once_cd_step( + intent: srv3_install_effect_attempt_intent, + escalation_id: srv3_boot_once_cd_escalation.escalation_id, + grant: grant, + lease: lease, + prior: prior, + observed_at: observed_at, + ) + } else { + plan_next_step_from_prior_receipt_and_lease( + intent: srv3_install_effect_attempt_intent, + pending_step_id: pending_step_id, + requires_destructive_approval: reconcile_requires_destructive_approval(step_id: pending_step_id), + escalation_id: srv3_boot_once_cd_escalation.escalation_id, + grant: grant, + lease: lease, + prior: prior, + observed_at: observed_at, + ) + } +} + +fn workflow_approval_from_durable_grant(grant: DurableApprovalGrant) -> WorkflowApprovalState { + match grant { + GrantActive { escalation_id: eid, intent_hash: _, granted_by: gb, granted_at: _, expires_at: _ } => + ApprovalGranted { escalation_id: eid, granted_by: gb } + _ => + PendingApproval { escalation: srv3_boot_once_cd_escalation } + } +} + +fn parse_grant_field_token(tail: String) -> String { + let token = split(s: tail, delimiter: " ").first() + split(s: token, delimiter: "\n").first() +} + +fn parse_grant_field(contents: String, key: String) -> String { + let marker = concat(key, "=") + if !string_contains(s: contents, pattern: marker) { + concat("", "") + } else { + let parts = split(s: contents, delimiter: marker) + if parts.length() <= 1 { + concat("", "") + } else { + let tail = parts.skip(n: 1).first() + parse_grant_field_token(tail: tail) + } + } +} + +fn parse_durable_approval_grant_file(contents: String) -> DurableApprovalGrant { + if !string_contains(s: contents, pattern: "Srv3InstallApprovalGrant:") { + GrantAbsent + } else { + let escalation_id = parse_grant_field(contents: contents, key: "escalation_id") + let intent_hash = parse_grant_field(contents: contents, key: "intent_hash") + let granted_by = parse_grant_field(contents: contents, key: "granted_by") + let granted_at = parse_grant_field(contents: contents, key: "granted_at") + let expires_at = parse_grant_field(contents: contents, key: "expires_at") + if escalation_id == "" + || intent_hash == "" + || granted_by == "" + || granted_at == "" + || expires_at == "" { + GrantAbsent + } else { + if intent_hash == (srv3_install_effect_attempt_intent.intent_hash as String) { + GrantActive { + escalation_id: escalation_id as NonEmptyStr, + intent_hash: intent_hash as ContentHash, + granted_by: granted_by as NonEmptyStr, + granted_at: granted_at, + expires_at: expires_at, + } + } else { + GrantAbsent + } + } + } +} +fn parse_observed_at_from_stdout(stdout: String) -> String? { + let parts = split(s: stdout, delimiter: "surface=observed_at=") + if parts.length() <= 1 { + Absent + } else { + let tail = parts.skip(n: 1).first() + let token = split(s: tail, delimiter: "\n").first() + match token == concat("", "") { + true => Absent + false => Present { value: token } + } + } +} + +fn srv3_install_reconcile_observe_meta_script() -> String { + concat( + "printf '%s\\n' \"Srv3InstallObserveReceipt: surface=observed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)\"\nprintf '%s\\n' \"Srv3InstallObserveReceipt: surface=git branch=$(git branch --show-current 2>/dev/null || echo unknown) commit=$(git rev-parse --short HEAD 2>/dev/null || echo unknown)\"\n", + ) +} + +fn sol_has_new_installer_markers( + pre: Srv3InstallDebugObservation, + post: Srv3InstallDebugObservation, +) -> Bool { + match post.sol { + Present { value: post_sol } => + match pre.sol { + Present { value: pre_sol } => + (post_sol.contains_grub && !pre_sol.contains_grub) + || (post_sol.contains_subiquity && !pre_sol.contains_subiquity) + || (post_sol.contains_autoinstall && !pre_sol.contains_autoinstall) + Absent => + post_sol.contains_grub + || post_sol.contains_subiquity + || post_sol.contains_autoinstall + } + Absent => false + } +} + +fn redfish_boot_override_is_consumed(obs: RedfishBootObservation) -> Bool { + obs.boot_override_enabled == BootOverrideDisabled + && obs.boot_override_target == BootTargetNone +} + +fn redfish_boot_override_is_armed_for_cd(obs: RedfishBootObservation) -> Bool { + obs.boot_override_enabled == BootOverrideOnce + && obs.boot_override_target == BootTargetCd +} + +fn boot_override_armed_to_consumed_transition( + pre: Srv3InstallDebugObservation, + post: Srv3InstallDebugObservation, +) -> Bool { + redfish_boot_override_is_armed_for_cd(obs: pre.redfish_boot) + && redfish_boot_override_is_consumed(obs: post.redfish_boot) +} + +fn boot_once_elapsed_advanced_after_pre_boot_ready( + pre: Srv3InstallDebugObservation, + post: Srv3InstallDebugObservation, +) -> Bool { + if !srv3_install_pre_boot_ready_to_boot(observation: pre) + || !redfish_boot_override_is_consumed(obs: post.redfish_boot) { + false + } else { + match pre.elapsed_since_boot_once_ms { + Absent => + match post.elapsed_since_boot_once_ms { + Present { value: post_elapsed } => post_elapsed > 0 + Absent => false + } + Present { value: pre_elapsed } => + match post.elapsed_since_boot_once_ms { + Present { value: post_elapsed } => post_elapsed > pre_elapsed + Absent => false + } + } + } +} + +fn boot_once_apply_read_back_ok( + pre: Srv3InstallDebugObservation, + post: Srv3InstallDebugObservation, + apply_succeeded: Bool, +) -> Bool { + if !apply_succeeded || !install_serve_session_active(observation: post) { + false + } else { + sol_has_new_installer_markers(pre: pre, post: post) + } +} + +fn consumed_grant_after_boot_once() -> DurableApprovalGrant { + GrantConsumed { + escalation_id: srv3_boot_once_cd_escalation.escalation_id, + intent_hash: srv3_install_effect_attempt_intent.intent_hash, + consumed_by_step: srv3_boot_once_cd_step_id, + } +} + +fn reconcile_srv3_os_install( + intent: OsInstallAttemptIntent, + observation: Srv3InstallDebugObservation, + disk: TargetDiskState, + observed_at: Timestamp, + grant: DurableApprovalGrant, + prior: PriorStepReceipt, +) -> Srv3InstallReconcileResult { + let preflight = fold_nbd_proxy_os_install_preflight_verdict( + payload: srv3_ubuntu_autoinstall_on_iso, + disk: disk, + ) + let preflight_plan = plan_for_os_install_preflight_verdict(verdict: preflight) + let diagnostic = diagnose_srv3_install(intent: intent, observation: observation) + let deduction_runtime = deduction_runtime_hint( + preflight: preflight, + diagnostic: diagnostic, + observation: observation, + observed_at: observed_at, + ) + let pending_step_id = reconcile_pending_step_id(diagnostic: diagnostic) + let lease = srv3_held_lease_from_serve(serve: observation.actuator_serve) + let temporal_plan = reconcile_temporal_plan( + pending_step_id: pending_step_id, + grant: grant, + lease: lease, + prior: prior, + observed_at: observed_at, + ) + Srv3InstallReconcileResult { + preflight: preflight, + preflight_plan: preflight_plan, + diagnostic: diagnostic, + deduction_runtime: deduction_runtime, + temporal_plan: temporal_plan, + pending_step_id: pending_step_id, + } +} + +fn reconcile_srv3_os_install_dry_run_fixture( + observation: Srv3InstallDebugObservation, +) -> Srv3InstallReconcileResult { + reconcile_srv3_os_install( + intent: srv3_os_install_attempt_intent, + observation: observation, + disk: srv3_install_reconcile_default_disk, + observed_at: srv3_install_reconcile_dry_run_observed_at, + grant: GrantAbsent, + prior: NoPriorReceipt, + ) +} diff --git a/dag/gunbc/srv3_os_install_reconcile_apply.dag b/dag/gunbc/srv3_os_install_reconcile_apply.dag new file mode 100644 index 00000000000..94d2a9be108 --- /dev/null +++ b/dag/gunbc/srv3_os_install_reconcile_apply.dag @@ -0,0 +1,207 @@ +module gunbc.srv3_os_install_reconcile_apply + +import extdeps.shell +import gunbc.shell_bash_runner { shell_exec_via_bash, shell_printf_line } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.resources { Network } +import std.types { NonEmptyStr, Timestamp } +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import std.measure { second_count } +import gunbc.srv3_install_diagnostic_checklist { + srv3_install_diagnostic_observe_script, +} +import gunbc.srv3_os_install_diagnostic { + srv3_os_install_attempt_intent, + parse_srv3_install_debug_observation, +} +import gunbc.srv3_os_install_reconcile { + reconcile_srv3_os_install, + srv3_held_lease_from_serve, + parse_durable_approval_grant_file, + parse_observed_at_from_stdout, + workflow_approval_from_durable_grant, + boot_once_apply_read_back_ok, + consumed_grant_after_boot_once, + srv3_install_reconcile_observe_meta_script, +} +import gunbc.srv3_os_install_reconcile_types { + srv3_install_reconcile_default_disk, + srv3_boot_once_approval_grant_path, + srv3_boot_once_apply_read_back_sleep, +} +import gunbc.srv3_os_install_reconcile_receipt { + srv3_install_reconcile_full_receipt, + srv3_install_reconcile_apply_receipt, + effect_step_receipt_for_boot_once, + durable_grant_is_active, +} +import gunbc.srv3_boot_once_cd { srv3_boot_once_cd_gated } +import std.temporal_effect { + NoPriorReceipt, + PriorReceipt, + PlanApply, + PlanAwaitApproval, + PlanRefuse, + PlanNoop, + EffectApplied, +} + +data srv3_os_install_reconcile_apply_shell_runner_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.srv3_os_install_reconcile", + decl_name: "srv3_os_install_reconcile_shell_runner_dissolution_trigger", + field: WholeDeclaration, + } +} + +fn load_durable_approval_grant_script() -> String { + concat("cat ", concat(srv3_boot_once_approval_grant_path, " 2>/dev/null || true")) +} + +fn srv3_install_reconcile_run_observe_script() -> String { + concat( + srv3_install_diagnostic_observe_script(intent: srv3_os_install_attempt_intent), + srv3_install_reconcile_observe_meta_script(), + ) +} + +fn process_exit_succeeded(exit: ProcessExit) -> Bool { + match exit { + ExitSuccess => true + _ => false + } +} + +fn sleep_for_boot_once_read_back_script() -> String { + concat("sleep ", to_string(second_count(s: srv3_boot_once_apply_read_back_sleep))) +} + +func srv3_os_install_reconcile_apply() -> ProcessExit + uses net: Network +{ + let intent = srv3_os_install_attempt_intent + let grant_run = shell.Exec.Run(script: shell_exec_via_bash(body: load_durable_approval_grant_script())) + if !grant_run.success { + return exit_failure(reason: concat("srv3-install-reconcile-apply: read grant: ", grant_run.stderr)) + } + let grant = parse_durable_approval_grant_file(contents: grant_run.stdout) + if !durable_grant_is_active(grant: grant) { + return exit_failure(reason: "srv3-install-reconcile-apply: durable approval grant missing or invalid") + } + + let pre_observe_script = srv3_install_reconcile_run_observe_script() + let pre_observe = shell.Exec.Run(script: shell_exec_via_bash(body: pre_observe_script)) + if !pre_observe.success { + return exit_failure(reason: concat("srv3-install-reconcile-apply: pre-observe: ", pre_observe.stderr)) + } + let pre_observed_at = match parse_observed_at_from_stdout(stdout: pre_observe.stdout) { + Present { value: t } => t as Timestamp + Absent => + return exit_failure(reason: "srv3-install-reconcile-apply: pre-observe missing surface=observed_at=") + } + let pre_observation = parse_srv3_install_debug_observation( + stdout: pre_observe.stdout, + intent: intent, + ) + let pre_result = reconcile_srv3_os_install( + intent: intent, + observation: pre_observation, + disk: srv3_install_reconcile_default_disk, + observed_at: pre_observed_at, + grant: grant, + prior: NoPriorReceipt, + ) + + match pre_result.temporal_plan { + PlanApply { step_id: _ } => {} + PlanAwaitApproval { escalation_id: _ } => + return exit_failure(reason: "srv3-install-reconcile-apply: plan awaits approval despite grant file present") + PlanRefuse { step_id: _, reason: r } => + return exit_failure(reason: concat("srv3-install-reconcile-apply: plan refused: ", r)) + PlanNoop { step_id: _ } => + return exit_failure(reason: "srv3-install-reconcile-apply: plan noop; boot-once already converged") + _ => + return exit_failure(reason: "srv3-install-reconcile-apply: unexpected temporal plan for boot-once apply") + } + + let boot_exit = srv3_boot_once_cd_gated(approval: workflow_approval_from_durable_grant(grant: grant)) + let apply_succeeded = process_exit_succeeded(exit: boot_exit) + if !apply_succeeded { + return exit_failure(reason: "srv3-install-reconcile-apply: boot-once-cd apply failed") + } + + let sleep_run = shell.Exec.Run(script: shell_exec_via_bash(body: sleep_for_boot_once_read_back_script())) + if !sleep_run.success { + return exit_failure(reason: concat("srv3-install-reconcile-apply: read-back sleep: ", sleep_run.stderr)) + } + + let post_observe = shell.Exec.Run(script: shell_exec_via_bash(body: pre_observe_script)) + if !post_observe.success { + return exit_failure(reason: concat("srv3-install-reconcile-apply: post-observe: ", post_observe.stderr)) + } + let post_observed_at = match parse_observed_at_from_stdout(stdout: post_observe.stdout) { + Present { value: t } => t as Timestamp + Absent => + return exit_failure(reason: "srv3-install-reconcile-apply: post-observe missing surface=observed_at=") + } + let post_observation = parse_srv3_install_debug_observation( + stdout: post_observe.stdout, + intent: intent, + ) + let read_back_ok = boot_once_apply_read_back_ok( + pre: pre_observation, + post: post_observation, + apply_succeeded: apply_succeeded, + ) + let step_receipt = effect_step_receipt_for_boot_once( + observation: post_observation, + diagnostic: pre_result.diagnostic, + plan: pre_result.temporal_plan, + read_back_ok: read_back_ok, + observed_at: post_observed_at, + applied: EffectApplied { effect_id: "srv3-boot-once-cd" as NonEmptyStr }, + ) + let post_grant = consumed_grant_after_boot_once() + let post_result = reconcile_srv3_os_install( + intent: intent, + observation: post_observation, + disk: srv3_install_reconcile_default_disk, + observed_at: post_observed_at, + grant: post_grant, + prior: PriorReceipt { receipt: step_receipt }, + ) + let lease = srv3_held_lease_from_serve(serve: post_observation.actuator_serve) + let dry_receipt = srv3_install_reconcile_full_receipt( + result: post_result, + intent: intent, + observation: post_observation, + grant: post_grant, + prior: PriorReceipt { receipt: step_receipt }, + lease: lease, + git_branch: "live-apply" as NonEmptyStr, + git_commit: "live-apply" as NonEmptyStr, + ) + let apply_receipt = srv3_install_reconcile_apply_receipt( + pre_result: pre_result, + post_result: post_result, + pre_grant: grant, + post_grant: post_grant, + step_receipt: step_receipt, + apply_succeeded: apply_succeeded, + read_back_ok: read_back_ok, + ) + let echo_body = concat( + shell_printf_line(line: apply_receipt), + shell_printf_line(line: dry_receipt), + ) + let echo_run = shell.Exec.Run(script: shell_exec_via_bash(body: echo_body)) + if !echo_run.success { + return exit_failure(reason: concat("srv3-install-reconcile-apply-receipt: ", echo_run.stderr)) + } + if !read_back_ok { + return exit_failure(reason: "srv3-install-reconcile-apply: read-back failed after boot-once-cd") + } + ExitSuccess +} diff --git a/dag/gunbc/srv3_os_install_reconcile_dry_run.dag b/dag/gunbc/srv3_os_install_reconcile_dry_run.dag new file mode 100644 index 00000000000..c916b62ac7d --- /dev/null +++ b/dag/gunbc/srv3_os_install_reconcile_dry_run.dag @@ -0,0 +1,114 @@ +module gunbc.srv3_os_install_reconcile_dry_run + +import extdeps.shell +import gunbc.shell_bash_runner { shell_exec_via_bash, shell_printf_line } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.types { String, NonEmptyStr } +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import gunbc.srv3_install_diagnostic_checklist { + srv3_install_diagnostic_observe_script, +} +import gunbc.srv3_os_install_diagnostic { + srv3_os_install_attempt_intent, + parse_srv3_install_debug_observation, +} +import gunbc.srv3_os_install_reconcile { + reconcile_srv3_os_install, + srv3_held_lease_from_serve, + srv3_install_reconcile_observe_meta_script, +} +import gunbc.srv3_os_install_reconcile_types { + srv3_install_reconcile_default_disk, + srv3_install_reconcile_dry_run_observed_at, +} +import gunbc.srv3_os_install_reconcile_receipt { + srv3_install_reconcile_full_receipt, + srv3_install_reconcile_action_line, +} +import std.temporal_effect { GrantAbsent, NoPriorReceipt } + +data srv3_os_install_reconcile_dry_run_shell_runner_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.srv3_os_install_reconcile", + decl_name: "srv3_os_install_reconcile_shell_runner_dissolution_trigger", + field: WholeDeclaration, + } +} + +fn srv3_install_reconcile_git_meta_script() -> String { + srv3_install_reconcile_observe_meta_script() +} + +fn parse_reconcile_git_branch(stdout: String) -> NonEmptyStr { + let parts = split(s: stdout, delimiter: "surface=git branch=") + if parts.length() <= 1 { + "unknown" as NonEmptyStr + } else { + let tail = parts.skip(n: 1).first() + let token = split(s: tail, delimiter: " commit=").first() + token as NonEmptyStr + } +} + +fn parse_reconcile_git_commit(stdout: String) -> NonEmptyStr { + let parts = split(s: stdout, delimiter: " commit=") + if parts.length() <= 1 { + "unknown" as NonEmptyStr + } else { + let tail = parts.skip(n: 1).first() + let token = split(s: tail, delimiter: "\n").first() + token as NonEmptyStr + } +} + +func srv3_os_install_reconcile_dry_run() -> ProcessExit { + let intent = srv3_os_install_attempt_intent + let observe_script = concat( + srv3_install_diagnostic_observe_script(intent: intent), + srv3_install_reconcile_git_meta_script(), + ) + let observe_run = shell.Exec.Run(script: shell_exec_via_bash(body: observe_script)) + if !observe_run.success { + return exit_failure(reason: concat("srv3-install-reconcile-observe: ", observe_run.stderr)) + } + let observation = parse_srv3_install_debug_observation( + stdout: observe_run.stdout, + intent: intent, + ) + let grant = GrantAbsent + let prior = NoPriorReceipt + let result = reconcile_srv3_os_install( + intent: intent, + observation: observation, + disk: srv3_install_reconcile_default_disk, + observed_at: srv3_install_reconcile_dry_run_observed_at, + grant: grant, + prior: prior, + ) + let lease = srv3_held_lease_from_serve(serve: observation.actuator_serve) + let git_branch = parse_reconcile_git_branch(stdout: observe_run.stdout) + let git_commit = parse_reconcile_git_commit(stdout: observe_run.stdout) + let receipt = srv3_install_reconcile_full_receipt( + result: result, + intent: intent, + observation: observation, + grant: grant, + prior: prior, + lease: lease, + git_branch: git_branch, + git_commit: git_commit, + ) + let action = srv3_install_reconcile_action_line(result: result) + let echo_body = concat( + shell_printf_line(line: receipt), + shell_printf_line(line: action), + ) + let echo_run = shell.Exec.Run(script: shell_exec_via_bash(body: echo_body)) + if echo_run.success { + ExitSuccess + } else { + return exit_failure(reason: concat("srv3-install-reconcile-receipt: ", echo_run.stderr)) + } +} diff --git a/dag/gunbc/srv3_os_install_reconcile_receipt.dag b/dag/gunbc/srv3_os_install_reconcile_receipt.dag new file mode 100644 index 00000000000..82aaba4402b --- /dev/null +++ b/dag/gunbc/srv3_os_install_reconcile_receipt.dag @@ -0,0 +1,532 @@ +module gunbc.srv3_os_install_reconcile_receipt + +import std.types { String, NonEmptyStr, ContentHash, Timestamp, Bool, Int } +import std.temporal_effect { + EffectStepReceipt, + AppliedEffectRef, + NoEffectApplied, + EffectApplied, + HeldLease, + HeldLeaseObservedState, + LeaseAbsent, + LeaseRunningExpected, + LeaseRunningStale, + LeasePortForeign, + LeaseInaccessible, + DurableApprovalGrant, + GrantAbsent, + GrantActive, + GrantConsumed, + PriorStepReceipt, + NoPriorReceipt, + PriorReceipt, + NextStepPlan, + PlanNoop, + PlanApply, + PlanRefuse, + PlanAwaitApproval, + PlanDrainThenStart, +} +import gunbc.srv3_os_install_reconcile_types { + Srv3InstallReconcileResult, + DeductionRuntimeHint, + srv3_install_effect_attempt_intent, + srv3_boot_once_cd_step_id, + srv3_boot_once_approval_grant_path, +} +import gunbc.os_install_deduction { + OsInstallPreflightVerdict, + OsInstallRuntimeDiagnosticVerdict, +} +import gunbc.srv3_os_install_diagnostic { + OsInstallAttemptIntent, + Srv3InstallDebugObservation, + Srv3InstallDiagnostic, + NbdServeObservation, + NbdServeProcessObservation, + NbdServePortObservation, + BmcSessionTokenObservation, + IsoServeReadinessObservation, + ServeUnknown, + ServeObserved, + NbdkitAbsent, + WebsocatAbsent, + ServeProcessesPresent, + ServePortAbsent, + ServePortConflict, + ServePortListening, + IsoMissing, + IsoPresent, + TokenMissing, + TokenPresent, + RedfishBootObservation, + VirtualMediaSessionObservation, + AttachedToExpectedServe, + NoAttachedServe, + VirtualMediaSessionUnknown, + WebsocketUpgradeAccepted, + WebsocketUnavailable, + srv3_install_diagnostic_wire, + srv3_install_diagnostic_recommended_action_wire, + classify_router_lease_table, + Srv3OsLeasePresent, + Srv3OsLeaseAbsent, + OnlyBmcLeasePresent, + WrongHostnameLeasePresent, +} +import extdeps.bmc.types { BootOverrideDisabled, BootTargetNone, BootOverrideOnce, BootTargetCd } +import gunbc.srv3_os_install_actuate_workflow { srv3_boot_once_cd_escalation } + +fn bash_var_ref(name: String) -> String { + concat("$", concat("{", concat(name, "}"))) +} + +fn srv3_install_record_boot_once_approval_script() -> String { + let header = concat( + "set -euo pipefail\nmkdir -p /var/lib/gunbc/logs/approvals\nGRANTED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)\nEXPIRES_AT=$(date -u -d '+24 hours' +%Y-%m-%dT%H:%M:%SZ)\nprintf '%s\\n' \"Srv3InstallApprovalGrant: escalation_id=", + srv3_boot_once_cd_escalation.escalation_id as String, + ) + let intent = concat( + " intent_hash=", + srv3_install_effect_attempt_intent.intent_hash as String, + ) + let grantee = concat( + " granted_by=operator granted_at=", + bash_var_ref(name: "GRANTED_AT"), + ) + let expiry = concat(" expires_at=", bash_var_ref(name: "EXPIRES_AT")) + let redirect = concat("\" > ", srv3_boot_once_approval_grant_path) + let trailer = concat( + "\necho \"Srv3InstallApprovalGrantRecorded: path=", + concat(srv3_boot_once_approval_grant_path, "\"\n"), + ) + concat(concat(concat(concat(concat(header, intent), concat(grantee, expiry)), redirect), trailer)) +} + +fn durable_grant_is_active(grant: DurableApprovalGrant) -> Bool { + match grant { + GrantActive { escalation_id: _, intent_hash: _, granted_by: _, granted_at: _, expires_at: _ } => true + _ => false + } +} + +fn bool_wire(value: Bool) -> String { + if value { + "true" + } else { + "false" + } +} +fn effect_step_receipt_for_boot_once( + observation: Srv3InstallDebugObservation, + diagnostic: Srv3InstallDiagnostic, + plan: NextStepPlan, + read_back_ok: Bool, + observed_at: Timestamp, + applied: AppliedEffectRef, +) -> EffectStepReceipt { + EffectStepReceipt { + attempt_id: srv3_install_effect_attempt_intent.attempt_id, + step_id: srv3_boot_once_cd_step_id, + observation_digest: reconcile_iso_hash_observed(observation: observation) as ContentHash, + verdict_label: srv3_install_diagnostic_wire(verdict: diagnostic) as NonEmptyStr, + plan_label: next_step_plan_wire(plan: plan) as NonEmptyStr, + applied: applied, + read_back_ok: read_back_ok, + recorded_at: observed_at, + } +} + +fn os_install_preflight_verdict_wire(verdict: OsInstallPreflightVerdict) -> String { + match verdict { + CompleteForUnattended => "CompleteForUnattended" + NotUnattendedByPolicy => "NotUnattendedByPolicy" + RequiresStoragePolicy => "RequiresStoragePolicy" + RequiresDiskObservation => "RequiresDiskObservation" + IncompleteSeedDelivery => "IncompleteSeedDelivery" + AutoinstallIncompleteNotBootable => "AutoinstallIncompleteNotBootable" + } +} + +fn os_install_runtime_verdict_wire(verdict: OsInstallRuntimeDiagnosticVerdict) -> String { + match verdict { + ReadyToBoot => "ReadyToBoot" + KvmFirmwareIdleObserved => "KvmFirmwareIdleObserved" + BootOverrideConsumedNoInstallEvidence => "BootOverrideConsumedNoInstallEvidence" + InstallerAwaitingStorage => "InstallerAwaitingStorage" + InstallerRunning => "InstallerRunning" + InstallerHung => "InstallerHung" + KvmSuggestsOsBooted => "KvmSuggestsOsBooted" + OsInstalled => "OsInstalled" + } +} + +fn next_step_plan_wire(plan: NextStepPlan) -> String { + match plan { + PlanNoop { step_id: s } => concat("PlanNoop(", concat(s as String, ")")) + PlanApply { step_id: s } => concat("PlanApply(", concat(s as String, ")")) + PlanRefuse { step_id: s, reason: r } => + concat("PlanRefuse(", concat(s as String, concat(", ", concat(r as String, ")")))) + PlanAwaitApproval { escalation_id: e } => + concat("PlanAwaitApproval(", concat(e as String, ")")) + PlanDrainThenStart { step_id: s, lease_key: k } => + concat("PlanDrainThenStart(", concat(s as String, concat(", ", concat(k as String, ")")))) + } +} + +fn deduction_runtime_hint_wire(hint: DeductionRuntimeHint) -> String { + match hint { + DeductionRuntimeAbsent => "absent" + DeductionRuntimePresent { verdict: v } => os_install_runtime_verdict_wire(verdict: v) + } +} + +fn srv3_install_reconcile_receipt_line(result: Srv3InstallReconcileResult) -> String { + concat( + concat( + concat( + concat("Srv3InstallReconcileReceipt: preflight=", os_install_preflight_verdict_wire(verdict: result.preflight)), + concat(" diagnostic=", srv3_install_diagnostic_wire(verdict: result.diagnostic)), + ), + concat(" deduction_runtime=", deduction_runtime_hint_wire(hint: result.deduction_runtime)), + ), + concat( + concat(" pending_step=", result.pending_step_id as String), + concat(" temporal=", next_step_plan_wire(plan: result.temporal_plan)), + ), + ) +} + +fn srv3_install_reconcile_action_line(result: Srv3InstallReconcileResult) -> String { + concat( + "Srv3InstallReconcileAction: ", + srv3_install_diagnostic_recommended_action_wire(verdict: result.diagnostic), + ) +} + +fn reconcile_nbd_serve_lease_wire(lease: HeldLease) -> String { + match lease.observed { + LeaseAbsent => "NbdServeLeaseAbsent" + LeaseRunningExpected => "NbdServeLeaseRunningExpected" + LeaseRunningStale => "NbdServeLeaseStale" + LeasePortForeign => "NbdServeLeaseForeign" + LeaseInaccessible => "NbdServeLeaseInaccessible" + } +} + +fn reconcile_serve_process_wire(proc: NbdServeProcessObservation) -> String { + match proc { + ServeProcessesPresent => "present" + NbdkitAbsent => "absent" + WebsocatAbsent => "websocat_absent" + } +} + +fn reconcile_serve_port_wire(port: NbdServePortObservation) -> String { + match port { + ServePortListening { owner_matches_expected: ok, port: p } => + if ok { + concat("expected:", to_string(p)) + } else { + concat("foreign:", to_string(p)) + } + ServePortAbsent => "absent" + ServePortConflict => "conflict" + } +} + +fn reconcile_serve_session_wire(sess: BmcSessionTokenObservation) -> String { + match sess { + TokenPresent => "present" + TokenMissing => "missing" + } +} + +fn reconcile_serve_iso_wire(iso: IsoServeReadinessObservation) -> String { + match iso { + IsoPresent { observed_content_hash: h } => h as String + IsoMissing => "missing" + } +} + +fn reconcile_serve_detail_wire(serve: NbdServeObservation) -> String { + match serve { + ServeUnknown => "unknown" + ServeObserved { process: proc, port: port, session: sess, iso: iso } => + concat( + concat( + concat(concat("nbdkit=", reconcile_serve_process_wire(proc: proc)), concat(" port=", reconcile_serve_port_wire(port: port))), + concat(" token=", reconcile_serve_session_wire(sess: sess)), + ), + concat(" iso=", reconcile_serve_iso_wire(iso: iso)), + ) + } +} + +fn reconcile_redfish_boot_wire(observation: Srv3InstallDebugObservation) -> String { + concat( + concat( + concat("power=", observation.redfish_boot.power_state as String), + concat( + " boot_enabled=", + observation.redfish_boot.boot_override_enabled as String, + ), + ), + concat( + concat(" boot_target=", observation.redfish_boot.boot_override_target as String), + concat(" boot_progress=", observation.redfish_boot.boot_progress as String), + ), + ) +} + +fn reconcile_sol_state_wire(observation: Srv3InstallDebugObservation) -> String { + match observation.sol { + Absent => "absent" + Present { value: sol } => + if sol.contains_autoinstall { + "autoinstall_seen" + } else { + if sol.contains_subiquity { + "subiquity_seen" + } else { + if sol.contains_grub { + "grub_seen" + } else { + if sol.contains_firmware_menu { + "firmware_menu" + } else { + if sol.capture_available { + "silent" + } else { + "unavailable" + } + } + } + } + } + } +} + +fn reconcile_virtual_media_observation_wire( + observation: Srv3InstallDebugObservation, +) -> String { + match observation.actuator_serve { + ServeUnknown => "Unknown" + ServeObserved { process: _, port: _, session: _, iso: _ } => + match observation.bmc_websocket { + WebsocketUnavailable => "AttachFailed" + WebsocketUpgradeAccepted => + match observation.virtual_media_session { + AttachedToExpectedServe => "AttachedExpectedIso" + NoAttachedServe => "AttachFailed" + VirtualMediaSessionUnknown => "EndpointAccepted" + } + _ => "Unknown" + } + } +} + +fn reconcile_router_lease_wire( + observation: Srv3InstallDebugObservation, + intent: OsInstallAttemptIntent, +) -> String { + match observation.router_lease_table { + Absent => "RouterLeaseUnknownOperatorRequired" + Present { value: table } => + match classify_router_lease_table(table: table, fleet_slot: intent.target_host) { + Srv3OsLeasePresent => "DynamicSrv3LeasePresent" + OnlyBmcLeasePresent => "OnlyBmcLeasePresent" + Srv3OsLeaseAbsent => "RouterLeaseAbsent" + WrongHostnameLeasePresent => "WrongHostnameLeasePresent" + } + } +} + +fn reconcile_boot_override_consumed_wire(observation: Srv3InstallDebugObservation) -> String { + match observation.redfish_boot.boot_override_enabled { + BootOverrideDisabled => + match observation.redfish_boot.boot_override_target { + BootTargetNone => "true" + _ => "false" + } + _ => "false" + } +} + +fn reconcile_kvm_attestation_wire(hint: DeductionRuntimeHint) -> String { + deduction_runtime_hint_wire(hint: hint) +} + +fn reconcile_lease_state_wire(lease: HeldLease) -> String { + match lease.observed { + LeaseAbsent => "LeaseAbsent" + LeaseRunningExpected => "LeaseRunningExpected" + LeaseRunningStale => "LeaseRunningStale" + LeasePortForeign => "LeasePortForeign" + LeaseInaccessible => "LeaseInaccessible" + } +} + +fn reconcile_approval_state_wire(grant: DurableApprovalGrant) -> String { + match grant { + GrantAbsent => "GrantAbsent" + GrantActive { escalation_id: e, intent_hash: _, granted_by: _, granted_at: _, expires_at: _ } => + concat("GrantActive(", concat(e as String, ")")) + GrantConsumed { escalation_id: e, intent_hash: _, consumed_by_step: s } => + concat("GrantConsumed(", concat(e as String, concat(",", concat(s as String, ")")))) + } +} + +fn reconcile_prior_step_wire(prior: PriorStepReceipt) -> String { + match prior { + NoPriorReceipt => "none" + PriorReceipt { receipt: r } => r.step_id as String + } +} + +fn reconcile_refusal_reason_wire(plan: NextStepPlan) -> String { + match plan { + PlanRefuse { step_id: _, reason: r } => r as String + _ => "" + } +} + +fn reconcile_iso_hash_observed(observation: Srv3InstallDebugObservation) -> String { + match observation.actuator_serve { + ServeObserved { process: _, port: _, session: _, iso: iso } => + match iso { + IsoPresent { observed_content_hash: h } => h as String + IsoMissing => "missing" + } + ServeUnknown => "unknown" + } +} + +fn srv3_install_reconcile_full_receipt( + result: Srv3InstallReconcileResult, + intent: OsInstallAttemptIntent, + observation: Srv3InstallDebugObservation, + grant: DurableApprovalGrant, + prior: PriorStepReceipt, + lease: HeldLease, + git_branch: NonEmptyStr, + git_commit: NonEmptyStr, +) -> String { + let identity = concat( + concat( + concat("Srv3InstallReconcileReceipt: attempt_id=", srv3_install_effect_attempt_intent.attempt_id as String), + concat(" intent_hash=", srv3_install_effect_attempt_intent.intent_hash as String), + ), + concat( + concat(" branch=", git_branch as String), + concat(" commit=", git_commit as String), + ), + ) + let intent_fields = concat( + concat(" actuator_host=", intent.actuator_host as String), + concat(" iso_path=", intent.iso_path as String), + ) + let hash_fields = concat( + concat(" iso_hash_expected=", intent.iso_content_hash as String), + concat(" iso_hash_observed=", reconcile_iso_hash_observed(observation: observation)), + ) + let verdict_fields = concat( + concat(" preflight_verdict=", os_install_preflight_verdict_wire(verdict: result.preflight)), + concat(" diagnostic=", srv3_install_diagnostic_wire(verdict: result.diagnostic)), + ) + let serve_fields = concat( + concat(" serve_lease_state=", reconcile_nbd_serve_lease_wire(lease: lease)), + concat(" serve_detail=", reconcile_serve_detail_wire(serve: observation.actuator_serve)), + ) + let media_fields = concat( + concat( + " virtual_media_observation=", + reconcile_virtual_media_observation_wire(observation: observation), + ), + concat(" redfish_boot_state=", reconcile_redfish_boot_wire(observation: observation)), + ) + let observe_fields = concat( + concat(" kvm_attestation=", reconcile_kvm_attestation_wire(hint: result.deduction_runtime)), + concat(" sol_state=", reconcile_sol_state_wire(observation: observation)), + ) + let router_fields = concat( + concat( + " router_lease_state=", + reconcile_router_lease_wire(observation: observation, intent: intent), + ), + concat( + " boot_override_consumed=", + reconcile_boot_override_consumed_wire(observation: observation), + ), + ) + let temporal_fields = concat( + concat(" temporal_prior_step=", reconcile_prior_step_wire(prior: prior)), + concat(" lease_state=", reconcile_lease_state_wire(lease: lease)), + ) + let approval_fields = concat( + concat(" approval_state=", reconcile_approval_state_wire(grant: grant)), + concat(" next_plan=", next_step_plan_wire(plan: result.temporal_plan)), + ) + let refusal_suffix = match reconcile_refusal_reason_wire(plan: result.temporal_plan) { + "" => "" + r => concat(" refusal_reason=", r) + } + concat( + concat(concat(concat(identity, intent_fields), hash_fields), verdict_fields), + concat( + concat(concat(concat(serve_fields, media_fields), observe_fields), router_fields), + concat(concat(temporal_fields, approval_fields), refusal_suffix), + ), + ) +} + +fn srv3_install_reconcile_apply_receipt( + pre_result: Srv3InstallReconcileResult, + post_result: Srv3InstallReconcileResult, + pre_grant: DurableApprovalGrant, + post_grant: DurableApprovalGrant, + step_receipt: EffectStepReceipt, + apply_succeeded: Bool, + read_back_ok: Bool, +) -> String { + let identity = concat( + concat( + "Srv3InstallReconcileApplyReceipt: attempt_id=", + srv3_install_effect_attempt_intent.attempt_id as String, + ), + concat( + concat(" boot_attempt_id=", srv3_install_effect_attempt_intent.attempt_id as String), + concat(" step_id=", srv3_boot_once_cd_step_id as String), + ), + ) + let diagnostics = concat( + concat(" pre_diagnostic=", srv3_install_diagnostic_wire(verdict: pre_result.diagnostic)), + concat(" post_diagnostic=", srv3_install_diagnostic_wire(verdict: post_result.diagnostic)), + ) + let plan_flags = concat( + concat(" pre_plan=", next_step_plan_wire(plan: pre_result.temporal_plan)), + concat(" apply_succeeded=", bool_wire(value: apply_succeeded)), + ) + let approval = concat( + concat(" read_back_ok=", bool_wire(value: read_back_ok)), + concat( + concat(" approval_state=", reconcile_approval_state_wire(grant: post_grant)), + concat(" pre_approval_state=", reconcile_approval_state_wire(grant: pre_grant)), + ), + ) + let step = concat( + concat(" step_read_back_ok=", bool_wire(value: step_receipt.read_back_ok)), + concat( + " step_applied=", + match step_receipt.applied { + EffectApplied { effect_id: e } => e as String + NoEffectApplied => "none" + }, + ), + ) + let tail = concat( + concat(" post_plan=", next_step_plan_wire(plan: post_result.temporal_plan)), + concat(" recorded_at=", step_receipt.recorded_at), + ) + concat(concat(concat(concat(identity, diagnostics), concat(plan_flags, approval)), concat(step, tail))) +} diff --git a/dag/gunbc/srv3_os_install_reconcile_record_approval.dag b/dag/gunbc/srv3_os_install_reconcile_record_approval.dag new file mode 100644 index 00000000000..cb4a9189d09 --- /dev/null +++ b/dag/gunbc/srv3_os_install_reconcile_record_approval.dag @@ -0,0 +1,32 @@ +module gunbc.srv3_os_install_reconcile_record_approval + +import extdeps.shell +import gunbc.shell_bash_runner { shell_exec_via_bash } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.disposition { Disposition, Scaffold, SingleAuthority } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import gunbc.srv3_os_install_reconcile_types { + srv3_boot_once_approval_grant_path, +} +import gunbc.srv3_os_install_reconcile_receipt { + srv3_install_record_boot_once_approval_script, +} + +data srv3_os_install_reconcile_record_approval_shell_runner_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.srv3_os_install_reconcile", + decl_name: "srv3_os_install_reconcile_shell_runner_dissolution_trigger", + field: WholeDeclaration, + } +} + +func srv3_os_install_record_boot_once_approval() -> ProcessExit { + let script = srv3_install_record_boot_once_approval_script() + let run = shell.Exec.Run(script: shell_exec_via_bash(body: script)) + if run.success { + ExitSuccess + } else { + return exit_failure(reason: concat("srv3-install-record-approval: ", run.stderr)) + } +} diff --git a/dag/gunbc/srv3_os_install_reconcile_types.dag b/dag/gunbc/srv3_os_install_reconcile_types.dag new file mode 100644 index 00000000000..7b31fb9ca43 --- /dev/null +++ b/dag/gunbc/srv3_os_install_reconcile_types.dag @@ -0,0 +1,46 @@ +module gunbc.srv3_os_install_reconcile_types + +import std.types { NonEmptyStr, Timestamp, String, ContentHash } +import std.temporal_effect { EffectAttemptIntent, NextStepPlan } +import std.measure { Second, second } +import gunbc.os_install_deduction { + TargetDiskState, + ForeignOsPresent, + OsInstallPreflightVerdict, + OsInstallPreflightPlan, + OsInstallRuntimeDiagnosticVerdict, +} +import gunbc.srv3_os_install_diagnostic { Srv3InstallDiagnostic } +import gunbc.srv3_seeded_install_media { srv3_seeded_install_media_content_hash } + +type DeductionRuntimeHint + = DeductionRuntimeAbsent + | DeductionRuntimePresent { verdict: OsInstallRuntimeDiagnosticVerdict } + +type Srv3InstallReconcileResult { + preflight: OsInstallPreflightVerdict + preflight_plan: OsInstallPreflightPlan + diagnostic: Srv3InstallDiagnostic + deduction_runtime: DeductionRuntimeHint + temporal_plan: NextStepPlan + pending_step_id: NonEmptyStr +} + +data srv3_boot_once_cd_step_id: NonEmptyStr = "boot-once-cd" as NonEmptyStr + +data srv3_boot_once_approval_grant_path: String = "/var/lib/gunbc/logs/approvals/srv3-boot-once-cd.grant" + +data srv3_boot_once_apply_read_back_sleep: Second = second(count: 45) + +data srv3_install_reconcile_default_disk: TargetDiskState = ForeignOsPresent + +data srv3_nbd_proxy_lease_key: NonEmptyStr = "srv3-nbd-proxy-10809" as NonEmptyStr + +data srv3_install_effect_attempt_intent: EffectAttemptIntent = EffectAttemptIntent { + attempt_id: "srv3-nbd-proxy-install" as NonEmptyStr, + intent_hash: srv3_seeded_install_media_content_hash, + target: "srv3" as NonEmptyStr, + created_at: "2026-07-03T00:00:00Z", +} + +data srv3_install_reconcile_dry_run_observed_at: Timestamp = "2026-07-03T00:00:00Z" diff --git a/dag/test/claim/os_install_deduction_witness_test.dag b/dag/test/claim/os_install_deduction_witness_test.dag index 951ca57e2ff..9653ecdc7ef 100644 --- a/dag/test/claim/os_install_deduction_witness_test.dag +++ b/dag/test/claim/os_install_deduction_witness_test.dag @@ -56,9 +56,32 @@ data witness_autoinstall_with_storage: UbuntuAutoinstallPayload = UbuntuAutoinst storage_policy: StoragePolicyDirectLayout, } +data witness_autoinstall_without_storage: UbuntuAutoinstallPayload = UbuntuAutoinstallPayload { + autoinstall_version: 1, + delivery: NoCloudLocal, + fully_automated: true, + target_distribution: srv3_ubuntu_autoinstall_on_iso.target_distribution, + identity: srv3_autoinstall_identity, + locale: "en_US.UTF-8", + keyboard_layout: "us", + install_openssh_server: true, + storage_policy: StoragePolicyAbsent, +} + +test fn srv3_on_iso_direct_layout_complete_for_unattended_on_foreign_disk() -> Bool { + fold_nbd_proxy_os_install_preflight_verdict( + payload: srv3_ubuntu_autoinstall_on_iso, + disk: ForeignOsPresent, + ) == CompleteForUnattended + && match srv3_ubuntu_autoinstall_on_iso.storage_policy { + StoragePolicyDirectLayout => true + StoragePolicyAbsent => false + } +} + test fn foreign_disk_without_storage_is_not_bootable() -> Bool { match fold_nbd_proxy_os_install_preflight_verdict( - payload: srv3_ubuntu_autoinstall_on_iso, + payload: witness_autoinstall_without_storage, disk: ForeignOsPresent, ) { AutoinstallIncompleteNotBootable => true @@ -69,14 +92,14 @@ test fn foreign_disk_without_storage_is_not_bootable() -> Bool { test fn unknown_disk_without_storage_needs_observation() -> Bool { fold_nbd_proxy_os_install_preflight_verdict( - payload: srv3_ubuntu_autoinstall_on_iso, + payload: witness_autoinstall_without_storage, disk: UnknownDiskState, ) == RequiresDiskObservation } test fn empty_disk_without_storage_requires_policy() -> Bool { fold_nbd_proxy_os_install_preflight_verdict( - payload: srv3_ubuntu_autoinstall_on_iso, + payload: witness_autoinstall_without_storage, disk: EmptyDisk, ) == RequiresStoragePolicy } @@ -94,7 +117,7 @@ test fn direct_layout_storage_is_complete_for_unattended() -> Bool { test fn removing_storage_policy_flips_ready_to_boot_witness() -> Bool { match fold_nbd_proxy_os_install_preflight_verdict( - payload: srv3_ubuntu_autoinstall_on_iso, + payload: witness_autoinstall_without_storage, disk: ForeignOsPresent, ) { CompleteForUnattended => false diff --git a/dag/test/claim/shell_bash_runner_witness_test.dag b/dag/test/claim/shell_bash_runner_witness_test.dag index e7b816e005f..d8221e2e7c0 100644 --- a/dag/test/claim/shell_bash_runner_witness_test.dag +++ b/dag/test/claim/shell_bash_runner_witness_test.dag @@ -3,6 +3,7 @@ module test.claim.shell_bash_runner import gunbc.shell_bash_runner { shell_bash_runner_dissolution_trigger, shell_exec_via_bash, + shell_printf_line, gunbc_bash_heredoc_delimiter, } @@ -18,3 +19,16 @@ test fn shell_exec_via_bash_wraps_body_in_heredoc() -> Bool { && string_contains(s: wrapped, pattern: gunbc_bash_heredoc_delimiter) && string_contains(s: wrapped, pattern: "set -e") } + +test fn shell_printf_line_uses_printf_not_single_quote_echo() -> Bool { + let line = shell_printf_line(line: "can't refuse: operator's call") + string_contains(s: line, pattern: "printf '%s\\n'") + && !string_contains(s: line, pattern: "echo '") +} + +test fn shell_printf_line_escapes_embedded_double_quote() -> Bool { + string_contains( + s: shell_printf_line(line: "say \"hi\""), + pattern: "\\\"", + ) +} diff --git a/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag b/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag index 795eed5e3e5..317bb1119ce 100644 --- a/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag +++ b/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag @@ -21,6 +21,7 @@ import gunbc.srv3_os_install_diagnostic { TokenPresent, IsoPresent, WebsocketUpgradeAccepted, + WebsocketUnavailable, NoAttachedServe, NbdkitAbsent, Srv3InstallDebugObservation, @@ -68,6 +69,21 @@ test fn parse_stdout_disabled_boot_override_pre_boot_ready() -> Bool { } } +test fn parse_stdout_elapsed_boot_once_ms_populates_observation() -> Bool { + let hash = srv3_seeded_install_media_content_hash as String + let stdout = concat( + concat("Srv3InstallObserveReceipt: surface=serve nbdkit=present websocat=present port_owner=expected token=present iso=match iso_file_sha256=abc123 content_hash_observed=", hash), + "\nSrv3InstallObserveReceipt: surface=bmc_websocket http_code=101\nSrv3InstallObserveReceipt: surface=redfish power=On boot_enabled=Disabled boot_target=None\nSrv3InstallObserveReceipt: surface=sol status=silent\nSrv3InstallObserveReceipt: surface=elapsed boot_once_ms=120000\n", + ) + match parse_srv3_install_debug_observation( + stdout: stdout, + intent: srv3_os_install_attempt_intent, + ).elapsed_since_boot_once_ms { + Present { value: v } => v == 120000 + Absent => false + } +} + test fn parse_stdout_iso_hash_mismatch_is_serve_down() -> Bool { let stdout = "Srv3InstallObserveReceipt: surface=serve nbdkit=present websocat=present port_owner=expected token=present iso=mismatch iso_file_sha256=abc123 content_hash_observed=stale-hash\nSrv3InstallObserveReceipt: surface=bmc_websocket http_code=101\nSrv3InstallObserveReceipt: surface=redfish power=On boot_enabled=Disabled boot_target=None\n" match diagnose_srv3_install( @@ -135,6 +151,25 @@ test fn serve_absent_observed_is_serve_down_not_ready() -> Bool { } } +test fn parse_bmc_websocket_http_code_non_101_red_despite_serve_websocat_present() -> Bool { + let stdout = concat( + "Srv3InstallObserveReceipt: surface=serve nbdkit=present websocat=present port_owner=expected token=absent iso=missing", + "\nSrv3InstallObserveReceipt: surface=bmc_websocket http_code=000\n", + ) + let observation = parse_srv3_install_debug_observation( + stdout: stdout, + intent: srv3_os_install_attempt_intent, + ) + match observation.bmc_websocket { + WebsocketUnavailable => true + _ => false + } + && match observation.virtual_media_session { + NoAttachedServe => true + _ => false + } +} + test fn checklist_entrypoint_carries_dissolution_trigger() -> Bool { srv3_install_diagnostic_checklist_entrypoint_is_declared() } diff --git a/dag/test/claim/srv3_os_install_reconcile_witness_test.dag b/dag/test/claim/srv3_os_install_reconcile_witness_test.dag new file mode 100644 index 00000000000..db2e23b4d93 --- /dev/null +++ b/dag/test/claim/srv3_os_install_reconcile_witness_test.dag @@ -0,0 +1,321 @@ +module test.claim.srv3_os_install_reconcile + +import std.logic { Bool } +import std.types { NonEmptyStr, Timestamp } +import std.temporal_effect { + PlanAwaitApproval, + PlanApply, + GrantAbsent, + GrantActive, + DurableApprovalGrant, + NoPriorReceipt, +} +import gunbc.os_install_deduction { + CompleteForUnattended, + KvmFirmwareIdleObserved, +} +import gunbc.srv3_os_install_reconcile { + reconcile_srv3_os_install_dry_run_fixture, + srv3_held_lease_from_serve, + boot_once_apply_read_back_ok, + parse_durable_approval_grant_file, + plan_boot_once_cd_step, + parse_observed_at_from_stdout, +} +import gunbc.srv3_os_install_reconcile_types { + DeductionRuntimePresent, + srv3_install_effect_attempt_intent, +} +import gunbc.srv3_os_install_reconcile_receipt { + srv3_install_reconcile_receipt_line, + srv3_install_reconcile_full_receipt, +} +import gunbc.srv3_os_install_diagnostic { + srv3_os_install_attempt_intent, + Srv3InstallDebugObservation, + ServeObserved, + ServeProcessesPresent, + ServePortListening, + TokenPresent, + IsoPresent, + WebsocketUpgradeAccepted, + AttachedToExpectedServe, + BootOverrideConsumedNoInstallEvidence, + RedfishBootObservation, + BootProgressNone, + SolConsoleContentObservation, +} +import gunbc.srv3_seeded_install_media { srv3_seeded_install_media_content_hash } +import gunbc.network_identity_subsumption { srv3_pre_install_lease_table } +import extdeps.bmc.types { BootOverrideDisabled, BootTargetNone, BootOverrideOnce, BootTargetCd, On } +import extdeps.bmc.webui.nbd_proxy_serve { srv3_nbd_proxy_local_port } +import std.measure { byte_size } + +data witness_uefi_stuck_observation: Srv3InstallDebugObservation = Srv3InstallDebugObservation { + actuator_serve: ServeObserved { + process: ServeProcessesPresent, + port: ServePortListening { + port: srv3_nbd_proxy_local_port, + owner_matches_expected: true, + }, + session: TokenPresent, + iso: IsoPresent { observed_content_hash: srv3_seeded_install_media_content_hash }, + }, + bmc_websocket: WebsocketUpgradeAccepted, + redfish_boot: RedfishBootObservation { + power_state: On, + boot_override_enabled: BootOverrideDisabled, + boot_override_target: BootTargetNone, + boot_progress: BootProgressNone, + }, + virtual_media_session: AttachedToExpectedServe, + sol: Present { + value: SolConsoleContentObservation { + bytes_captured: byte_size(0), + duration_ms: 3600000, + contains_grub: false, + contains_subiquity: false, + contains_autoinstall: false, + contains_firmware_menu: true, + contains_error: false, + capture_available: true, + } + }, + router_lease_table: Present { value: srv3_pre_install_lease_table }, + kvm: Absent, + elapsed_since_boot_once_ms: Present { value: 3600000 }, +} + +data witness_router_unknown_observation: Srv3InstallDebugObservation = Srv3InstallDebugObservation { + actuator_serve: witness_uefi_stuck_observation.actuator_serve, + bmc_websocket: witness_uefi_stuck_observation.bmc_websocket, + redfish_boot: witness_uefi_stuck_observation.redfish_boot, + virtual_media_session: witness_uefi_stuck_observation.virtual_media_session, + sol: witness_uefi_stuck_observation.sol, + router_lease_table: Absent, + kvm: Absent, + elapsed_since_boot_once_ms: Present { value: 3600000 }, +} + +test fn reconcile_preflight_is_complete_for_unattended() -> Bool { + let result = reconcile_srv3_os_install_dry_run_fixture( + observation: witness_uefi_stuck_observation, + ) + match result.preflight { + CompleteForUnattended => true + _ => false + } +} + +test fn reconcile_uefi_stuck_maps_kvm_firmware_idle() -> Bool { + let result = reconcile_srv3_os_install_dry_run_fixture( + observation: witness_uefi_stuck_observation, + ) + match result.diagnostic { + BootOverrideConsumedNoInstallEvidence => true + _ => false + } + && match result.deduction_runtime { + DeductionRuntimePresent { verdict: KvmFirmwareIdleObserved } => true + _ => false + } +} + +test fn reconcile_uefi_stuck_plans_boot_once_awaiting_approval() -> Bool { + let result = reconcile_srv3_os_install_dry_run_fixture( + observation: witness_uefi_stuck_observation, + ) + result.pending_step_id == ("boot-once-cd" as NonEmptyStr) + && match result.temporal_plan { + PlanAwaitApproval { escalation_id: e } => + e == ("srv3-boot-once-cd" as NonEmptyStr) + _ => false + } +} + +test fn reconcile_receipt_line_names_preflight_and_deduction() -> Bool { + let line = srv3_install_reconcile_receipt_line( + result: reconcile_srv3_os_install_dry_run_fixture(observation: witness_uefi_stuck_observation), + ) + string_contains(s: line, pattern: "preflight=CompleteForUnattended") + && string_contains(s: line, pattern: "deduction_runtime=KvmFirmwareIdleObserved") + && string_contains(s: line, pattern: "Srv3InstallReconcileReceipt:") +} + +test fn reconcile_full_receipt_typed_router_and_virtual_media() -> Bool { + let observation = witness_uefi_stuck_observation + let result = reconcile_srv3_os_install_dry_run_fixture(observation: observation) + let lease = srv3_held_lease_from_serve(serve: observation.actuator_serve) + let receipt = srv3_install_reconcile_full_receipt( + result: result, + intent: srv3_os_install_attempt_intent, + observation: observation, + grant: GrantAbsent, + prior: NoPriorReceipt, + lease: lease, + git_branch: "witness" as NonEmptyStr, + git_commit: "witness" as NonEmptyStr, + ) + string_contains(s: receipt, pattern: "router_lease_state=OnlyBmcLeasePresent") + && string_contains(s: receipt, pattern: "virtual_media_observation=AttachedExpectedIso") + && string_contains(s: receipt, pattern: "serve_lease_state=NbdServeLeaseRunningExpected") + && string_contains(s: receipt, pattern: "boot_override_consumed=true") +} + +test fn reconcile_full_receipt_router_unknown_operator_required() -> Bool { + let observation = witness_router_unknown_observation + let result = reconcile_srv3_os_install_dry_run_fixture(observation: observation) + let lease = srv3_held_lease_from_serve(serve: observation.actuator_serve) + let receipt = srv3_install_reconcile_full_receipt( + result: result, + intent: srv3_os_install_attempt_intent, + observation: observation, + grant: GrantAbsent, + prior: NoPriorReceipt, + lease: lease, + git_branch: "witness" as NonEmptyStr, + git_commit: "witness" as NonEmptyStr, + ) + string_contains(s: receipt, pattern: "router_lease_state=RouterLeaseUnknownOperatorRequired") +} + +data witness_active_boot_once_grant: DurableApprovalGrant = GrantActive { + escalation_id: "srv3-boot-once-cd" as NonEmptyStr, + intent_hash: srv3_seeded_install_media_content_hash, + granted_by: "operator" as NonEmptyStr, + granted_at: "2026-07-03T00:00:00Z", + expires_at: "2026-07-04T00:00:00Z", +} + +test fn reconcile_active_grant_plans_boot_once_apply() -> Bool { + let lease = srv3_held_lease_from_serve(serve: witness_uefi_stuck_observation.actuator_serve) + match plan_boot_once_cd_step( + intent: srv3_install_effect_attempt_intent, + escalation_id: "srv3-boot-once-cd" as NonEmptyStr, + grant: witness_active_boot_once_grant, + lease: lease, + prior: NoPriorReceipt, + observed_at: "2026-07-03T01:00:00Z" as Timestamp, + ) { + PlanApply { step_id: s } => s == ("boot-once-cd" as NonEmptyStr) + _ => false + } +} + +data witness_post_boot_armed_observation: Srv3InstallDebugObservation = Srv3InstallDebugObservation { + actuator_serve: witness_uefi_stuck_observation.actuator_serve, + bmc_websocket: witness_uefi_stuck_observation.bmc_websocket, + redfish_boot: RedfishBootObservation { + power_state: On, + boot_override_enabled: BootOverrideOnce, + boot_override_target: BootTargetCd, + boot_progress: BootProgressNone, + }, + virtual_media_session: witness_uefi_stuck_observation.virtual_media_session, + sol: witness_uefi_stuck_observation.sol, + router_lease_table: witness_uefi_stuck_observation.router_lease_table, + kvm: Absent, + elapsed_since_boot_once_ms: Present { value: 60000 }, +} + +data witness_ready_to_boot_pre_observation: Srv3InstallDebugObservation = Srv3InstallDebugObservation { + actuator_serve: witness_uefi_stuck_observation.actuator_serve, + bmc_websocket: witness_uefi_stuck_observation.bmc_websocket, + redfish_boot: witness_uefi_stuck_observation.redfish_boot, + virtual_media_session: witness_uefi_stuck_observation.virtual_media_session, + sol: witness_uefi_stuck_observation.sol, + router_lease_table: witness_uefi_stuck_observation.router_lease_table, + kvm: Absent, + elapsed_since_boot_once_ms: Absent, +} + +data witness_ready_to_boot_post_consumed_observation: Srv3InstallDebugObservation = Srv3InstallDebugObservation { + actuator_serve: witness_ready_to_boot_pre_observation.actuator_serve, + bmc_websocket: witness_ready_to_boot_pre_observation.bmc_websocket, + redfish_boot: witness_ready_to_boot_pre_observation.redfish_boot, + virtual_media_session: witness_ready_to_boot_pre_observation.virtual_media_session, + sol: witness_ready_to_boot_pre_observation.sol, + router_lease_table: witness_ready_to_boot_pre_observation.router_lease_table, + kvm: Absent, + elapsed_since_boot_once_ms: Present { value: 45000 }, +} + +test fn reconcile_boot_once_read_back_ok_rejects_noop_when_already_consumed() -> Bool { + !boot_once_apply_read_back_ok( + pre: witness_uefi_stuck_observation, + post: witness_uefi_stuck_observation, + apply_succeeded: true, + ) +} + +test fn reconcile_boot_once_read_back_ok_rejects_armed_to_consumed_without_install_progress() -> Bool { + !boot_once_apply_read_back_ok( + pre: witness_post_boot_armed_observation, + post: witness_uefi_stuck_observation, + apply_succeeded: true, + ) +} + +test fn reconcile_boot_once_read_back_ok_rejects_elapsed_only_without_install_progress() -> Bool { + !boot_once_apply_read_back_ok( + pre: witness_ready_to_boot_pre_observation, + post: witness_ready_to_boot_post_consumed_observation, + apply_succeeded: true, + ) +} + +test fn reconcile_boot_once_read_back_ok_rejects_still_armed() -> Bool { + !boot_once_apply_read_back_ok( + pre: witness_uefi_stuck_observation, + post: witness_post_boot_armed_observation, + apply_succeeded: true, + ) +} + +test fn parse_observed_at_from_stdout_absent_without_surface() -> Bool { + match parse_observed_at_from_stdout(stdout: "Srv3InstallObserveReceipt: surface=serve\n") { + Absent => true + _ => false + } +} + +test fn parse_observed_at_from_stdout_present_on_surface_line() -> Bool { + match parse_observed_at_from_stdout( + stdout: "Srv3InstallObserveReceipt: surface=observed_at=2026-07-03T12:00:00Z\n", + ) { + Present { value: t } => t == "2026-07-03T12:00:00Z" + _ => false + } +} + +test fn reconcile_parse_durable_grant_file_accepts_matching_intent() -> Bool { + let intent_hash = srv3_seeded_install_media_content_hash as String + let contents = concat( + "Srv3InstallApprovalGrant: escalation_id=srv3-boot-once-cd intent_hash=", + concat( + intent_hash, + " granted_by=operator granted_at=2026-07-03T00:00:00Z expires_at=2026-07-04T00:00:00Z", + ), + ) + match parse_durable_approval_grant_file(contents: contents) { + GrantActive { escalation_id: e, intent_hash: h, granted_by: _, granted_at: _, expires_at: _ } => + e == ("srv3-boot-once-cd" as NonEmptyStr) && h == srv3_seeded_install_media_content_hash + _ => false + } +} + +test fn reconcile_parse_durable_grant_file_strips_trailing_newline_on_expires_at() -> Bool { + let intent_hash = srv3_seeded_install_media_content_hash as String + let contents = concat( + "Srv3InstallApprovalGrant: escalation_id=srv3-boot-once-cd intent_hash=", + concat( + intent_hash, + " granted_by=operator granted_at=2026-07-03T00:00:00Z expires_at=2026-07-04T00:00:00Z\n", + ), + ) + match parse_durable_approval_grant_file(contents: contents) { + GrantActive { escalation_id: _, intent_hash: _, granted_by: _, granted_at: _, expires_at: exp } => + exp == "2026-07-04T00:00:00Z" + _ => false + } +} diff --git a/src/v2/lens/medium_structure_containment.dag b/src/v2/lens/medium_structure_containment.dag index 3790e29a9b9..584188d2ea6 100644 --- a/src/v2/lens/medium_structure_containment.dag +++ b/src/v2/lens/medium_structure_containment.dag @@ -185,7 +185,13 @@ data medium_structure_exception_roster: List = [ "dag/extdeps/os/ubuntu_seeded_install_media_remaster.dag", "dag/gunbc/srv3_seeded_install_media.dag", "dag/extdeps/bmc/serial_console.dag", - "dag/gunbc/srv3_install_diagnostic_checklist.dag" + "dag/gunbc/srv3_install_diagnostic_checklist.dag", + "dag/gunbc/srv3_os_install_reconcile.dag", + "dag/gunbc/srv3_os_install_reconcile_types.dag", + "dag/gunbc/srv3_os_install_reconcile_receipt.dag", + "dag/gunbc/srv3_os_install_reconcile_apply.dag", + "dag/gunbc/srv3_os_install_reconcile_dry_run.dag", + "dag/gunbc/srv3_os_install_reconcile_record_approval.dag" ] fn char_eq(a: Char, b: Char) -> Bool { diff --git a/src/v2/test/fixture/floor_skip/provenance_fail_closed_contract_test.dag b/src/v2/test/fixture/floor_skip/provenance_fail_closed_contract_test.dag new file mode 100644 index 00000000000..6de1b542b1e --- /dev/null +++ b/src/v2/test/fixture/floor_skip/provenance_fail_closed_contract_test.dag @@ -0,0 +1,103 @@ +module v2.test.fixture.floor_skip.provenance_fail_closed_contract + +// Scout fixture (#6105): fail-closed contract for affected-set witness disposition. +// Provenance ABSENT (committed host_source_root_ingest_manifest is Empty) => +// witness RUNS and precompute must NOT skip (run-all posture). +// Provenance PRESENT (frontier produced, witness off-frontier) => +// witness SKIPs => smaller affected set than run-all. + +import v2.std.algebra { length } +import v2.std.logic { Bool } +import v2.std.node { Atom, Node, SyntheticOccurrence, TypeNode } +import v2.std.verification { + EqualsClaim, + TestClaim, + ManualAnchorAbsent, + TestClassification, + Tier1, + Unit, + manual_claim_anchor +} +import v2.test.workflow.host_source_root_ingest_manifest { + host_source_root_ingest, + host_source_root_ingest_coverage_receipt +} +import v2.workflow.affected_set_floor_runner { + DiffPathsProduced, + FloorDiffObservation, + FloorRerunFrontier, + FrontierFailClosed, + FrontierProduced, + RunWitness, + SkipAssumedGreen, + floor_precompute_would_skip, + floor_witness_run_disposition +} + +data scout_manifest_provenance_live: Bool = + host_source_root_ingest_coverage_receipt.coverage_complete + && (host_source_root_ingest_coverage_receipt.ingest_read_count > 0) + && (length(xs: host_source_root_ingest) > 0) + +data scout_unrelated_frontier_node: Node = Node { + kind: TypeNode { connective: Atom { identity: ^scout_unrelated_frontier_symbol } }, + children: [], + occurrence_id: SyntheticOccurrence +} + +data scout_orphan_claim: TestClaim = EqualsClaim { + label: "scout: orphan claim off diff frontier", + anchor: manual_claim_anchor(anchor: ManualAnchorAbsent), + lhs: scout_orphan_claim_node, + rhs: scout_orphan_claim_node, + classification: TestClassification { tier: Tier1, layer: Unit } +} + +data scout_orphan_claim_node: Node = Node { + kind: TypeNode { connective: Atom { identity: ^scout_orphan_claim_symbol } }, + children: [], + occurrence_id: SyntheticOccurrence +} + +data scout_nonempty_diff: FloorDiffObservation = DiffPathsProduced { + changed_paths: ["src/v2/lens/affected_set.dag"] +} + +data scout_provenance_present_frontier: FloorRerunFrontier = FrontierProduced { + nodes: [scout_unrelated_frontier_node] +} + +data scout_provenance_absent_frontier: FloorRerunFrontier = FrontierFailClosed { + reason: "provenance unavailable" +} + +test fn provenance_fail_closed_contract_holds() -> Bool { + let manifest_absent = scout_manifest_provenance_live == false + let absent_witness_runs = match floor_witness_run_disposition( + diff: scout_nonempty_diff, + frontier: scout_provenance_absent_frontier, + touches_frontier: false, + function_edited: false, + entry_file_touched: false + ) { + RunWitness { reason: _ } => true + SkipAssumedGreen => false + } + let present_witness_skips = match floor_witness_run_disposition( + diff: scout_nonempty_diff, + frontier: scout_provenance_present_frontier, + touches_frontier: false, + function_edited: false, + entry_file_touched: false + ) { + SkipAssumedGreen => true + RunWitness { reason: _ } => false + } + let absent_precompute_runs_all = !floor_precompute_would_skip( + diff: scout_nonempty_diff, + frontier: scout_provenance_absent_frontier, + edited_test_fn_count: 0, + touched_entry_file_count: 0 + ) + manifest_absent && absent_witness_runs && present_witness_skips && absent_precompute_runs_all +} From 1eb11993b03f0edc13f20b869d9d1bd0570a1da3 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 3 Jul 2026 17:57:22 +0000 Subject: [PATCH 02/58] =?UTF-8?q?fix(srv3):=20observe=20script=20bash=20sy?= =?UTF-8?q?ntax=20=E2=80=94=20drop=20stray=20HTTP=5FCODE=3D=20before=20if?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serve receipt echo was concatenated into curl_tail as HTTP_CODE=if test…, breaking live reconcile observe on srv1. Witness guards the regression. Co-authored-by: Cursor --- dag/gunbc/srv3_install_diagnostic_checklist.dag | 2 +- .../srv3_install_diagnostic_checklist_witness_test.dag | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dag/gunbc/srv3_install_diagnostic_checklist.dag b/dag/gunbc/srv3_install_diagnostic_checklist.dag index 492a026da17..d627b56960d 100644 --- a/dag/gunbc/srv3_install_diagnostic_checklist.dag +++ b/dag/gunbc/srv3_install_diagnostic_checklist.dag @@ -348,7 +348,7 @@ fn srv3_install_diagnostic_observe_script(intent: OsInstallAttemptIntent) -> Str bash_var_ref(name: "ISO_FILE_SHA256"), concat( " content_hash_observed=", - concat(bash_var_ref(name: "CONTENT_HASH_OBSERVED"), "\"\nHTTP_CODE="), + concat(bash_var_ref(name: "CONTENT_HASH_OBSERVED"), "\"\n"), ), ), ), diff --git a/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag b/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag index 317bb1119ce..9619ae13020 100644 --- a/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag +++ b/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag @@ -43,6 +43,12 @@ test fn observe_script_covers_five_surfaces() -> Bool { && string_contains(s: srv3_install_diagnostic_observe_script(intent: srv3_os_install_attempt_intent), pattern: "/vm/0/0") } +test fn observe_script_websocket_http_code_not_concatenated_before_if() -> Bool { + let script = srv3_install_diagnostic_observe_script(intent: srv3_os_install_attempt_intent) + !string_contains(s: script, pattern: "HTTP_CODE=if") + && string_contains(s: script, pattern: "HTTP_CODE=$(curl") +} + test fn remote_bmc_probe_needs_serve_observation() -> Bool { srv3_remote_bmc_probe_diagnostic_is_needs_serve_observation() && match diagnose_srv3_install( From 4be431fedc6abadf84532f45de893b40029b9810 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 3 Jul 2026 17:58:27 +0000 Subject: [PATCH 03/58] =?UTF-8?q?fix(srv3):=20observe=20script=20curl=20HT?= =?UTF-8?q?TP=5FCODE=20=E2=80=94=20remove=20extra=20closing=20paren?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit $(curl … || echo 000) had a stray ) breaking bash on live srv1 observe. Co-authored-by: Cursor --- dag/gunbc/srv3_install_diagnostic_checklist.dag | 2 +- .../claim/srv3_install_diagnostic_checklist_witness_test.dag | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dag/gunbc/srv3_install_diagnostic_checklist.dag b/dag/gunbc/srv3_install_diagnostic_checklist.dag index d627b56960d..770eb2a95b3 100644 --- a/dag/gunbc/srv3_install_diagnostic_checklist.dag +++ b/dag/gunbc/srv3_install_diagnostic_checklist.dag @@ -71,7 +71,7 @@ fn srv3_install_diagnostic_observe_script_curl_tail() -> String { concat( bash_var_ref(name: "BMC"), concat( - "/vm/0/0\" 2>/dev/null || echo 000))\nelse\n HTTP_CODE=000\nfi\necho \"Srv3InstallObserveReceipt: surface=bmc_websocket http_code=", + "/vm/0/0\" 2>/dev/null || echo 000)\nelse\n HTTP_CODE=000\nfi\necho \"Srv3InstallObserveReceipt: surface=bmc_websocket http_code=", concat( bash_var_ref(name: "HTTP_CODE"), "\"\nif test -s \"", diff --git a/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag b/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag index 9619ae13020..0e78c523dfd 100644 --- a/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag +++ b/dag/test/claim/srv3_install_diagnostic_checklist_witness_test.dag @@ -46,6 +46,7 @@ test fn observe_script_covers_five_surfaces() -> Bool { test fn observe_script_websocket_http_code_not_concatenated_before_if() -> Bool { let script = srv3_install_diagnostic_observe_script(intent: srv3_os_install_attempt_intent) !string_contains(s: script, pattern: "HTTP_CODE=if") + && !string_contains(s: script, pattern: "echo 000))") && string_contains(s: script, pattern: "HTTP_CODE=$(curl") } From 32c6bcf7c317d609b6561c0a2e32cf8416114c36 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 19:42:01 +0000 Subject: [PATCH 04/58] =?UTF-8?q?docs(srv4):=20BMC=20onboarding=20gap=20an?= =?UTF-8?q?alysis=20=E2=80=94=20srv3-hardcoded=20plan=20+=20gcloud=20not?= =?UTF-8?q?=20provisioned?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit srv4 racked, BMC at FactoryDefault (403 PasswordChangeRequired). The rotation workflow (bmc_converge_credential_idempotent, wired into host_standup_spine) is fully modeled and fail-closed, but bottoms out on the srv3-hardcoded new_altra_onboarding_plan and assumes gcloud is present. Documents 4 gaps (G1 per-host parameterization [dispositive], G2 gcloud self-provision, G3 operator-token handler, G4 srv4 identity rows) for review before modeling. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/plans/srv4-bmc-onboarding-gap.md | 88 +++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 docs/plans/srv4-bmc-onboarding-gap.md diff --git a/docs/plans/srv4-bmc-onboarding-gap.md b/docs/plans/srv4-bmc-onboarding-gap.md new file mode 100644 index 00000000000..1ffbdfa10df --- /dev/null +++ b/docs/plans/srv4-bmc-onboarding-gap.md @@ -0,0 +1,88 @@ +# srv4 BMC onboarding β€” what the workflow is missing + +Status: gap analysis for review (branch `session/srv4-onboarding-credential-rotation-gap`, 2026-07-21). +Trigger: srv4 hardware racked; its BMC is at `FactoryDefault` and the credential-rotation step +that the workflow *should* perform was skipped, leaving a manual gate. This documents why, so we +close the workflow gap rather than hand-rotate. + +## Live srv4 facts (measured) + +- BMC reachable at **`192.168.1.195`** and `192.168.1.199` (two NICs, one BMC). + MACs `48:21:0b:87:2c:ad` / `…:df` (newer block than srv1–3's `…:81:*`). +- Redfish UUID `83819837-25c6-4881-9b0c-49de5e8c0c58` (distinct board, OpenBMC). +- Factory login `root/0penBmc` authenticates (session `201`) but **every resource returns + `403 PasswordChangeRequired`** β€” OpenBMC's factory gate. Nothing (identity, virtual media, + install) works until root's password is rotated. +- Onboarding stage per `gunbc.bmc_onboarding`: **`FactoryDefault`** (host OS not yet installed). + +## What already exists (the machinery is real) + +The rotation flow is fully modeled and fail-closed β€” this is *not* a from-scratch build: + +- `gunbc.tools.bmc_onboard.bmc_converge_credential_idempotent` (`dag/gunbc/tools/bmc_onboard.dag`) + probes phase β†’ if `FactoryCredentialActive`, mints a 20-byte urandom credential, + `gcp.SecretManager.AddVersion` + read-back durability check (refuses to rotate on mismatch, so a + failed store can't lock us out), then `redfish.Http.SetAccountPassword` (PATCH + `/redfish/v1/AccountService/Accounts/root`, which also clears `PasswordChangeRequired`) and + verifies reauth. Idempotent: `RotatedCredentialActive` β†’ `ExitSuccess`. +- `redfish.Http.SetAccountPassword` effect exists (`dag/extdeps/bmc/http.dag:95`). +- Lifecycle state machine `FactoryDefault β†’ CredentialsRotated β†’ OsInstalled β†’ FabricJoined` with + monotone credential-posture hardening (`dag/gunbc/bmc_onboarding.dag`). +- Wired into the standup spine: `host_standup_spine`'s first prefix step `BmcCredentialConverge` + decl-refs `bmc_converge_credential_idempotent` (`dag/gunbc/host_standup.dag:193-199`). +- GCP IAM for the secret is modeled: assimilator SA with `secret_version_adder` + `secret_accessor` + bindings (`dag/gunbc/assimilate/bmc_token_federation.dag`, `…/bmc_wif_delegation_chain.dag`). + +## The gaps (why srv4 stalled) + +### G1 β€” the whole chain is srv3-hardcoded, not per-host parameterized *(dispositive)* +Everything bottoms out on one global `new_altra_onboarding_plan` (`bmc_onboarding.dag:213`): +`bmc: 192.168.1.192`, `rotated_credential: Stored { secret_name: "bmc-srv3-admin" }`. +- Every `bmc_onboard.dag` func reads that global + `srv3_gcp_project`; `onboarding_secret_id()` + fallback is literally `"bmc-srv3-admin"`; error strings say "would lock out srv3". +- `host_standup.dag` imports `operator_host_srv3`, references `srv3_install_mechanism`, and + `host_standup_spine` is a static `data` list, not a function of `HostIdentity`. +- There is **no way to point the converge at `.195` / `bmc-srv4-admin`** β€” no srv4 plan, no host + parameter. This is the Β§2-horizontal / Β§3-single-authority violation: "BMC onboarding" is one + concept that should span every host by breadth, forked to srv3 literals. +Fix shape: lift `new_altra_onboarding_plan` to `altra_onboarding_plan(host)` (or a per-host data +row keyed by `HostIdentity`); thread `plan` (or `host`) as a param through the `bmc_onboard.dag` +funcs and make `host_standup_spine` a function of `HostIdentity`. + +### G2 β€” gcloud is assumed-present, not provisioned *(the "upsert gcloud cli" ask)* +The credential step calls `shell.GCloud.AuthPrintAccessToken()` + `gcp.SecretManager.*`, which need +the `gcloud` CLI on the actuator host. But: +- `gcloud` is **not** modeled as a `CliTool` in `extdeps/tools/` (only curl/websocat/nbdkit/socat/ + sha256sum/xorriso). +- `ActuatorToolchainRequirement` (`extdeps/bmc/os_install_actuator_toolchain.dag`) ensures only the + OS-install transport tools β€” gcloud is absent, and that requirement isn't scoped to the credential + step anyway. +- srv1 (candidate actuator) has no gcloud CLI installed. +Fix shape: model `gcloud` as an ensured `CliTool` and add it to a toolchain requirement covering the +credential/secret actuation (mirrors the existing os-install toolchain-ensure). + +### G3 β€” token acquisition is hardwired to one handler *(lets the operator token work)* +Getting the GCP access token is hardwired to `shell.GCloud.AuthPrintAccessToken()`. But the +SecretManager effects already take `access_token` as an explicit input +(`secret_manager.dag:114` AddVersion, `:71` AccessVersion). So "obtain access_token" is really one +interface with N handlers β€” {gcloud-CLI print-token | **operator-supplied token** | WIF federation} +β€” currently fused to the gcloud-CLI handler (a Β§3 transport fork). De-forking it lets the operator's +supplied token drive the rotation with no interactive gcloud auth (and G2's gcloud upsert becomes +the other handler, not a hard prerequisite). + +### G4 β€” no srv4 network identity *(mechanical)* +`fleet_intent_network.dag` has srv1/2/3 endpoints but no srv4. Add BMC `.195` (+`.199`), secret +`bmc-srv4-admin`, host endpoint TBD-post-install. + +## Proposed sequence (for review β€” not yet implemented) + +1. G4: add srv4 identity rows. +2. G1: parameterize the onboarding plan by host; thread through `bmc_onboard` + `host_standup`. +3. G3: de-fork token acquisition so an operator-supplied token is a first-class handler. +4. G2: model `gcloud` as an ensured tool for the workflow to self-provision (defense-in-depth / + the other token handler). +5. Run `bmc_converge_credential_idempotent` against srv4 with the operator token β†’ `CredentialsRotated`, + secret in `bmc-srv4-admin` β†’ then the existing OS-install spine. + +Open question for the operator: scope now β€” do all of G1–G4, or minimally G1+G3 (parameterize + +operator-token handler) to unblock srv4 today and land G2 (gcloud self-provision) as follow-up? From 7f73da9644b18d0ee4a550c7e92edd95f0a76d0f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 20:27:40 +0000 Subject: [PATCH 05/58] =?UTF-8?q?feat(srv4):=20close=20BMC=20onboarding=20?= =?UTF-8?q?gaps=20=E2=80=94=20per-host=20plan,=20gcloud=20self-provision,?= =?UTF-8?q?=20operator-token=20handler,=20srv4=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full close of the four gaps from the analysis doc (PR #7027), so BMC onboarding is hands-off per-host instead of srv3-hardcoded: G1 β€” per-host parameterization: altra_onboarding_plan(bmc_host, secret_name) constructor + srv3_onboarding_plan / srv4_onboarding_plan rows replace the srv3-literal new_altra_onboarding_plan. Threaded `plan` through every bmc_onboard func; de-nicknamed srv3_gcp_project -> bmc_secrets_gcp_project (fleet-wide). Zero-arg per-host entries srv3_converge_credential / srv4_converge_credential are what the standup decl_ref + executor invoke. G2 β€” gcloud self-provision: modeled gcloud_cli_tool (extdeps/tools/gcloud.dag) + package_google_cloud_cli, and bmc_credential_actuator_toolchain_requirement (curl + gcloud) mirroring the OS-install toolchain-ensure. G3 β€” token de-fork: gunbc.auth.access_token_source with AccessTokenSource = GcloudPrintToken | OperatorSuppliedToken{token} and resolve_access_token, so an operator-supplied token is a first-class handler (drives rotation with no gcloud on the actuator). G4 β€” srv4 identity: operator_host_srv4 + srv4_bmc_endpoint (.195) in fleet_intent_network. Verified: full-corpus typecheck clean (850 modules, 0 errors) + 11 witnesses green by execution across every touched module, incl. a new discriminating srv4_plan_targets_195_with_srv4_secret and the updated endpoint-count witness. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../bmc/os_install_actuator_toolchain.dag | 19 +++++ dag/extdeps/package_managers/apt.dag | 1 + dag/extdeps/tools/gcloud.dag | 23 ++++++ dag/gunbc/auth/access_token_source.dag | 19 +++++ dag/gunbc/bmc_onboarding.dag | 24 +++--- dag/gunbc/fleet_intent_network.dag | 10 +++ dag/gunbc/host_standup.dag | 4 +- dag/gunbc/srv3_bmc_credential_resolve.dag | 8 +- dag/gunbc/srv3_boot_once_cd.dag | 4 +- dag/gunbc/tools/bmc_onboard.dag | 81 ++++++++++++------- dag/gunbc/tools/bmc_onboard_validate.dag | 8 +- .../bmc_onboarding_lifecycle_witness_test.dag | 41 +++++++--- .../fleet_intent_network_witness_test.dag | 2 +- 13 files changed, 182 insertions(+), 62 deletions(-) create mode 100644 dag/extdeps/tools/gcloud.dag create mode 100644 dag/gunbc/auth/access_token_source.dag diff --git a/dag/extdeps/bmc/os_install_actuator_toolchain.dag b/dag/extdeps/bmc/os_install_actuator_toolchain.dag index 618cd2816d0..030ba2b8e3e 100644 --- a/dag/extdeps/bmc/os_install_actuator_toolchain.dag +++ b/dag/extdeps/bmc/os_install_actuator_toolchain.dag @@ -6,6 +6,7 @@ import extdeps.tools.curl { curl_cli_tool } import extdeps.tools.websocat { websocat_cli_tool } import extdeps.tools.nbdkit { nbdkit_cli_tool } import extdeps.tools.socat { socat_cli_tool } +import extdeps.tools.gcloud { gcloud_cli_tool } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } @@ -68,3 +69,21 @@ fn srv3_os_install_actuator_toolchain_is_complete() -> Bool { requirement: os_install_actuator_toolchain_requirement, ) } + +data bmc_credential_actuator_toolchain_requirement: ActuatorToolchainRequirement = ActuatorToolchainRequirement { + required_tools: [ + curl_cli_tool, + gcloud_cli_tool, + ], +} + +fn bmc_credential_actuator_toolchain_is_complete() -> Bool { + actuator_toolchain_requirement_includes_tool( + requirement: bmc_credential_actuator_toolchain_requirement, + tool_name: curl_cli_tool.name, + ) + && actuator_toolchain_requirement_includes_tool( + requirement: bmc_credential_actuator_toolchain_requirement, + tool_name: gcloud_cli_tool.name, + ) +} diff --git a/dag/extdeps/package_managers/apt.dag b/dag/extdeps/package_managers/apt.dag index 592c3af358f..11c5c15708e 100644 --- a/dag/extdeps/package_managers/apt.dag +++ b/dag/extdeps/package_managers/apt.dag @@ -29,6 +29,7 @@ data package_nbdkit: NonEmptyStr = "nbdkit" data package_socat: NonEmptyStr = "socat" data package_websocat: NonEmptyStr = "websocat" data package_xorriso: NonEmptyStr = "xorriso" +data package_google_cloud_cli: NonEmptyStr = "google-cloud-cli" data apt_man_url: NonEmptyStr = "https://manpages.debian.org/testing/apt/apt.8.en.html" diff --git a/dag/extdeps/tools/gcloud.dag b/dag/extdeps/tools/gcloud.dag new file mode 100644 index 00000000000..7f8c68a2fa2 --- /dev/null +++ b/dag/extdeps/tools/gcloud.dag @@ -0,0 +1,23 @@ +module extdeps.tools.gcloud + +import std.types { NonEmptyStr } +import extdeps.version { VersionConstraint } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } +import extdeps.tools { CliTool, SourceApt } +import extdeps.apt { package_google_cloud_cli } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "cloud.google.com/sdk/docs/install-sdk" + } +} + +data gcloud_pinned_version: VersionConstraint = ">= 470.0.0" + +data gcloud_cli_tool: CliTool = CliTool { + name: "gcloud", + min_version: Present { value: gcloud_pinned_version }, + installable_via: [SourceApt { package: package_google_cloud_cli }], +} diff --git a/dag/gunbc/auth/access_token_source.dag b/dag/gunbc/auth/access_token_source.dag new file mode 100644 index 00000000000..164704c0d09 --- /dev/null +++ b/dag/gunbc/auth/access_token_source.dag @@ -0,0 +1,19 @@ +module gunbc.auth.access_token_source + +import std.types { Secret } +import std.resources { Network } +import extdeps.shell +import extdeps.cloud.gcp.gcp + +type AccessTokenSource + = GcloudPrintToken + | OperatorSuppliedToken { token: Secret } + +func resolve_access_token(source: AccessTokenSource) -> Secret + uses net: Network +{ + match source { + GcloudPrintToken => shell.GCloud.AuthPrintAccessToken().access_token + OperatorSuppliedToken { token: t } => t + } +} diff --git a/dag/gunbc/bmc_onboarding.dag b/dag/gunbc/bmc_onboarding.dag index 7df9f87c8c0..52e8a2ed2f8 100644 --- a/dag/gunbc/bmc_onboarding.dag +++ b/dag/gunbc/bmc_onboarding.dag @@ -210,17 +210,23 @@ fn bmc_plan_steady_state_posture(plan: BmcOnboardingPlan) -> BmcCredentialPostur bmc_phase_credential_posture(phase: bmc_plan_steady_state_phase(plan: plan)) } -data new_altra_onboarding_plan: BmcOnboardingPlan = BmcOnboardingPlan { - bmc: BmcEndpoint { host: "192.168.1.192", protocol: Redfish }, - redfish_system_id: asrock_altrad8ud_redfish_system_id, - factory_login: openbmc_factory_login, - rotated_credential: Stored { secret_name: "bmc-srv3-admin" }, - rotated_account_role: AccountRoleAdministrator, - os_target: UbuntuOs { distro: NobleNumbat2404Lts }, - os_install_boot_target: srv3_install_boot_target, - start_phase: FactoryDefault, +fn altra_onboarding_plan(bmc_host: NonEmptyStr, secret_name: NonEmptyStr) -> BmcOnboardingPlan { + BmcOnboardingPlan { + bmc: BmcEndpoint { host: bmc_host, protocol: Redfish }, + redfish_system_id: asrock_altrad8ud_redfish_system_id, + factory_login: openbmc_factory_login, + rotated_credential: Stored { secret_name: secret_name }, + rotated_account_role: AccountRoleAdministrator, + os_target: UbuntuOs { distro: NobleNumbat2404Lts }, + os_install_boot_target: srv3_install_boot_target, + start_phase: FactoryDefault, + } } +data srv3_onboarding_plan: BmcOnboardingPlan = altra_onboarding_plan(bmc_host: "192.168.1.192", secret_name: "bmc-srv3-admin") + +data srv4_onboarding_plan: BmcOnboardingPlan = altra_onboarding_plan(bmc_host: "192.168.1.195", secret_name: "bmc-srv4-admin") + type BmcCredentialProbeResult = FactoryCredentialActive | RotatedCredentialActive diff --git a/dag/gunbc/fleet_intent_network.dag b/dag/gunbc/fleet_intent_network.dag index fabaa60ab4e..ec5588d780c 100644 --- a/dag/gunbc/fleet_intent_network.dag +++ b/dag/gunbc/fleet_intent_network.dag @@ -20,6 +20,7 @@ import product.placement_supply { HostIdentity } data operator_host_srv1: HostIdentity = "srv1" data operator_host_srv2: HostIdentity = "srv2" data operator_host_srv3: HostIdentity = "srv3" +data operator_host_srv4: HostIdentity = "srv4" data operator_lan_segment: NetworkSegment = NetworkSegment { cidr: "192.168.1.0/24", @@ -79,6 +80,14 @@ data srv3_bmc_endpoint: NetworkEndpoint = NetworkEndpoint { locality: Lan, } +data srv4_bmc_endpoint: NetworkEndpoint = NetworkEndpoint { + label: "srv4-bmc", + host: Present { value: operator_host_srv4 }, + address: "192.168.1.195", + role: BmcOutOfBand, + locality: Lan, +} + data srv1_tailscale_endpoint: NetworkEndpoint = NetworkEndpoint { label: "srv1-tailscale", host: Present { value: operator_host_srv1 }, @@ -117,6 +126,7 @@ data fleet_intent_network: NetworkTopology = NetworkTopology { srv1_bmc_endpoint, srv2_bmc_endpoint, srv3_bmc_endpoint, + srv4_bmc_endpoint, srv1_tailscale_endpoint, srv2_tailscale_endpoint, ], diff --git a/dag/gunbc/host_standup.dag b/dag/gunbc/host_standup.dag index 9d9fb176dec..3d2bd5b864b 100644 --- a/dag/gunbc/host_standup.dag +++ b/dag/gunbc/host_standup.dag @@ -196,7 +196,7 @@ data host_standup_spine: List = [ label: "prefix:bmc-credential-converge", authorities: [ decl_ref(module_path: "gunbc.bmc_onboarding", decl_name: "bmc_credential_probe_to_phase"), - decl_ref(module_path: "gunbc.tools.bmc_onboard", decl_name: "bmc_converge_credential_idempotent"), + decl_ref(module_path: "gunbc.tools.bmc_onboard", decl_name: "srv3_converge_credential"), ], effect_policy: OneShotIdempotent ), @@ -318,7 +318,7 @@ data host_standup_gap_ledger: List = [ GapLedgerRow { phase_label: "prefix:bmc-credential-converge", modeling: "modeled", - authority_or_interim: "gunbc.bmc_onboarding.bmc_credential_probe_to_phase + gunbc.tools.bmc_onboard.bmc_converge_credential_idempotent (#5949 idempotent probe)" + authority_or_interim: "gunbc.bmc_onboarding.bmc_credential_probe_to_phase + gunbc.tools.bmc_onboard.bmc_converge_credential (per-host; srv3_converge_credential / srv4_converge_credential entries)" }, GapLedgerRow { phase_label: "prefix:os-install-mechanism", diff --git a/dag/gunbc/srv3_bmc_credential_resolve.dag b/dag/gunbc/srv3_bmc_credential_resolve.dag index cbd2e9470f6..c6b80d2019c 100644 --- a/dag/gunbc/srv3_bmc_credential_resolve.dag +++ b/dag/gunbc/srv3_bmc_credential_resolve.dag @@ -5,7 +5,7 @@ import std.credentials { FactoryLogin } import extdeps.bmc.openbmc { openbmc_factory_login } import extdeps.cloud.gcp.secret_ref { SecretRef } import gunbc.bmc_onboarding { - new_altra_onboarding_plan, + srv3_onboarding_plan, BmcCredentialProbeResult, FactoryCredentialActive, RotatedCredentialActive, @@ -13,7 +13,7 @@ import gunbc.bmc_onboarding { } import gunbc.assimilate.bmc_wif_delegation_chain { bmc_srv3_admin_secret_ref } import gunbc.auth.secret_ref_credential { fetch_secret_ref_credential } -import gunbc.tools.bmc_onboard { bmc_probe_credential_phase } +import gunbc.tools.bmc_onboard { srv3_probe_credential_phase } import std.resources { Network } type BmcCredentialIntent { @@ -30,7 +30,7 @@ type BmcCredentialResolution fn srv3_os_install_actuate_credential_intent() -> BmcCredentialIntent { BmcCredentialIntent { - account: new_altra_onboarding_plan.factory_login.username, + account: srv3_onboarding_plan.factory_login.username, factory_login: openbmc_factory_login, factory_acceptable: true, rotated_secret_ref: bmc_srv3_admin_secret_ref, @@ -90,7 +90,7 @@ func observe_and_resolve_srv3_bmc_credential() -> BmcCredentialResolution uses net: Network { let intent = srv3_os_install_actuate_credential_intent() - let observation = bmc_probe_credential_phase() + let observation = srv3_probe_credential_phase() resolve_bmc_credential(intent: intent, observation: observation) } diff --git a/dag/gunbc/srv3_boot_once_cd.dag b/dag/gunbc/srv3_boot_once_cd.dag index 825db4b373f..eed3cb1721e 100644 --- a/dag/gunbc/srv3_boot_once_cd.dag +++ b/dag/gunbc/srv3_boot_once_cd.dag @@ -25,7 +25,7 @@ import gunbc.tools.bmc_onboard { boot_override_body, reset_body, } -import gunbc.bmc_onboarding { new_altra_onboarding_plan } +import gunbc.bmc_onboarding { srv3_onboarding_plan } import gunbc.workflow_escalation { WorkflowApprovalState, workflow_destructive_step_refusal_reason, @@ -91,7 +91,7 @@ func srv3_boot_once_cd_gated(approval: WorkflowApprovalState) -> ProcessExit func srv3_boot_once_cd_resolved(resolution: BmcCredentialResolution) -> ProcessExit uses net: Network { - let plan = new_altra_onboarding_plan + let plan = srv3_onboarding_plan let host = plan.bmc.host let account = plan.factory_login.username diff --git a/dag/gunbc/tools/bmc_onboard.dag b/dag/gunbc/tools/bmc_onboard.dag index 6b505b4172e..b831bdf0733 100644 --- a/dag/gunbc/tools/bmc_onboard.dag +++ b/dag/gunbc/tools/bmc_onboard.dag @@ -5,14 +5,20 @@ import extdeps.filesystem.filesystem_io import extdeps.entropy import extdeps.cloud.gcp.secret_manager import extdeps.cloud.gcp.gcp { GcpProjectId } -import extdeps.shell import gunbc.bmc_onboarding { - new_altra_onboarding_plan, + BmcOnboardingPlan, + srv3_onboarding_plan, + srv4_onboarding_plan, BmcCredentialProbeResult, FactoryCredentialActive, RotatedCredentialActive, CredentialStateUnknown, } +import gunbc.auth.access_token_source { + AccessTokenSource, + GcloudPrintToken, + resolve_access_token, +} import std.credentials { Stored } import std.types { NonEmptyStr, Secret, String } import std.measure { ByteSize, byte_size, byte_size_count } @@ -40,7 +46,7 @@ data bmc_onboard_secret_on_disk_debt: Disposition = Scaffold { } } -data srv3_gcp_project: GcpProjectId = "gunbai-secrets" +data bmc_secrets_gcp_project: GcpProjectId = "gunbai-secrets" data bmc_credential_bytes: ByteSize = byte_size(20) @@ -73,10 +79,10 @@ fn gcp_secret_full_name(project: String, secret_id: String) -> String { concat("projects/", concat(project, concat("/secrets/", secret_id))) } -fn onboarding_secret_id() -> NonEmptyStr { - match new_altra_onboarding_plan.rotated_credential { +fn onboarding_secret_id(plan: BmcOnboardingPlan) -> NonEmptyStr { + match plan.rotated_credential { Stored { secret_name: sn } => sn - Chained { steps: _ } => "bmc-srv3-admin" as NonEmptyStr + Chained { steps: _ } => plan.bmc.host } } @@ -88,10 +94,9 @@ fn write_netrc(host: String, account: String, password: String) -> Bool { Filesystem.Write(path: bmc_onboard_netrc_path, content: netrc_content(host: host, account: account, password: password)).success } -func bmc_rotate_credential(new_credential: Secret) -> ProcessExit +func bmc_rotate_credential(plan: BmcOnboardingPlan, new_credential: Secret) -> ProcessExit uses net: Network { - let plan = new_altra_onboarding_plan let host = plan.bmc.host let account = plan.factory_login.username @@ -125,35 +130,34 @@ func bmc_rotate_credential(new_credential: Secret) -> ProcessExit } } -func bmc_store_and_rotate(credential: Secret) -> ProcessExit +func bmc_store_and_rotate(plan: BmcOnboardingPlan, token_source: AccessTokenSource, credential: Secret) -> ProcessExit uses net: Network { let payload_b64 = credential as String - let secret_id = onboarding_secret_id() + let secret_id = onboarding_secret_id(plan: plan) - auth = shell.GCloud.AuthPrintAccessToken() + let token = resolve_access_token(source: token_source) stored = gcp.SecretManager.AddVersion( - access_token: auth.access_token, - secret_name: gcp_secret_full_name(project: srv3_gcp_project, secret_id: secret_id), + access_token: token, + secret_name: gcp_secret_full_name(project: bmc_secrets_gcp_project, secret_id: secret_id), payload_b64: payload_b64 ) readback = gcp.SecretManager.AccessVersion( - access_token: auth.access_token, - project_id: srv3_gcp_project, + access_token: token, + project_id: bmc_secrets_gcp_project, secret: secret_id, version: "latest" ) if readback.data_b64 != payload_b64 { - return exit_failure(reason: concat("store: GCP durability read-back mismatch β€” refusing to rotate (would lock out srv3); stored version ", stored.name)) + return exit_failure(reason: concat(concat(concat("store: GCP durability read-back mismatch β€” refusing to rotate (would lock out ", plan.bmc.host), concat("); stored version ", stored.name)), "")) } - return bmc_rotate_credential(new_credential: credential) + return bmc_rotate_credential(plan: plan, new_credential: credential) } -func bmc_assimilate_srv3_credential() -> ProcessExit +func bmc_assimilate_credential(plan: BmcOnboardingPlan, token_source: AccessTokenSource) -> ProcessExit uses net: Network { - let plan = new_altra_onboarding_plan let host = plan.bmc.host let account = plan.factory_login.username @@ -170,13 +174,12 @@ func bmc_assimilate_srv3_credential() -> ProcessExit return exit_failure(reason: concat("acquire: factory login failed: ", acq.body)) } - return bmc_store_and_rotate(credential: mint_bmc_credential()) + return bmc_store_and_rotate(plan: plan, token_source: token_source, credential: mint_bmc_credential()) } -func bmc_probe_credential_phase() -> BmcCredentialProbeResult +func bmc_probe_credential_phase(plan: BmcOnboardingPlan, token_source: AccessTokenSource) -> BmcCredentialProbeResult uses net: Network { - let plan = new_altra_onboarding_plan let host = plan.bmc.host let account = plan.factory_login.username @@ -189,11 +192,11 @@ func bmc_probe_credential_phase() -> BmcCredentialProbeResult return FactoryCredentialActive } - auth = shell.GCloud.AuthPrintAccessToken() + let token = resolve_access_token(source: token_source) secret_resp = gcp.SecretManager.AccessVersion( - access_token: auth.access_token, - project_id: srv3_gcp_project, - secret: onboarding_secret_id(), + access_token: token, + project_id: bmc_secrets_gcp_project, + secret: onboarding_secret_id(plan: plan), version: "latest" ) rotated_verify = redfish.Http.GetSystem( @@ -208,19 +211,37 @@ func bmc_probe_credential_phase() -> BmcCredentialProbeResult } } -func bmc_converge_credential_idempotent() -> ProcessExit +func bmc_converge_credential(plan: BmcOnboardingPlan, token_source: AccessTokenSource) -> ProcessExit uses net: Network { - root_resp = redfish.Http.GetServiceRoot(bmc_host: new_altra_onboarding_plan.bmc.host) + root_resp = redfish.Http.GetServiceRoot(bmc_host: plan.bmc.host) if root_resp.success == false { return exit_failure(reason: concat("credential converge: BMC unreachable: ", root_resp.body)) } - probe = bmc_probe_credential_phase() + probe = bmc_probe_credential_phase(plan: plan, token_source: token_source) match probe { - FactoryCredentialActive => return bmc_store_and_rotate(credential: mint_bmc_credential()) + FactoryCredentialActive => return bmc_store_and_rotate(plan: plan, token_source: token_source, credential: mint_bmc_credential()) RotatedCredentialActive => return ExitSuccess CredentialStateUnknown { reason: r } => return exit_failure(reason: concat("credential converge: unknown phase: ", r)) } } + +func srv3_probe_credential_phase() -> BmcCredentialProbeResult + uses net: Network +{ + bmc_probe_credential_phase(plan: srv3_onboarding_plan, token_source: GcloudPrintToken) +} + +func srv3_converge_credential() -> ProcessExit + uses net: Network +{ + bmc_converge_credential(plan: srv3_onboarding_plan, token_source: GcloudPrintToken) +} + +func srv4_converge_credential() -> ProcessExit + uses net: Network +{ + bmc_converge_credential(plan: srv4_onboarding_plan, token_source: GcloudPrintToken) +} diff --git a/dag/gunbc/tools/bmc_onboard_validate.dag b/dag/gunbc/tools/bmc_onboard_validate.dag index 7f7c8603bcc..92c773ef4b5 100644 --- a/dag/gunbc/tools/bmc_onboard_validate.dag +++ b/dag/gunbc/tools/bmc_onboard_validate.dag @@ -1,7 +1,7 @@ module gunbc.tools.bmc_onboard_validate import extdeps.bmc.http -import gunbc.bmc_onboarding { new_altra_onboarding_plan } +import gunbc.bmc_onboarding { srv3_onboarding_plan } import std.types { NonEmptyStr, Secret } import std.resources { Network } import std.process { ProcessExit, ExitSuccess, exit_failure } @@ -9,7 +9,7 @@ import std.process { ProcessExit, ExitSuccess, exit_failure } func bmc_onboard_validate() -> ProcessExit uses net: Network { - let bmc_host = new_altra_onboarding_plan.bmc.host + let bmc_host = srv3_onboarding_plan.bmc.host root_resp = redfish.Http.GetServiceRoot(bmc_host: bmc_host) if root_resp.success == false { @@ -18,8 +18,8 @@ func bmc_onboard_validate() -> ProcessExit system_resp = redfish.Http.GetSystem( bmc_host: bmc_host, - username: new_altra_onboarding_plan.factory_login.username, - password: new_altra_onboarding_plan.factory_login.published_password as Secret + username: srv3_onboarding_plan.factory_login.username, + password: srv3_onboarding_plan.factory_login.published_password as Secret ) return if system_resp.success { ExitSuccess } else { exit_failure(reason: system_resp.body) } diff --git a/dag/test/claim/bmc_onboarding_lifecycle_witness_test.dag b/dag/test/claim/bmc_onboarding_lifecycle_witness_test.dag index 5a6aeb1f805..27129ea603b 100644 --- a/dag/test/claim/bmc_onboarding_lifecycle_witness_test.dag +++ b/dag/test/claim/bmc_onboarding_lifecycle_witness_test.dag @@ -30,7 +30,8 @@ import gunbc.bmc_onboarding { bmc_lifecycle_length, bmc_plan_steady_state_phase, bmc_plan_steady_state_posture, - new_altra_onboarding_plan, + srv3_onboarding_plan, + srv4_onboarding_plan, BmcCredentialProbeResult, FactoryCredentialActive, RotatedCredentialActive, @@ -96,9 +97,9 @@ test fn bmc_lifecycle_fold_is_complete_and_monotone() -> Bool { } test fn bmc_plan_steady_state_is_keyless_wif() -> Bool { - let steady = bmc_plan_steady_state_phase(plan: new_altra_onboarding_plan) + let steady = bmc_plan_steady_state_phase(plan: srv3_onboarding_plan) (steady == FabricJoined) - && posture_is_keyless(p: bmc_plan_steady_state_posture(plan: new_altra_onboarding_plan)) + && posture_is_keyless(p: bmc_plan_steady_state_posture(plan: srv3_onboarding_plan)) && !string_contains(s: emit_bmc_token_smoke_workflow_yaml(), pattern: "credentials_json") && string_contains(s: emit_bmc_token_smoke_workflow_yaml(), pattern: "id-token: write") } @@ -179,14 +180,14 @@ test fn bmc_decom_does_not_roundtrip_when_inverse_is_wrong() -> Bool { transition_does_not_roundtrip_on_quotient(s: factory_at(p: CredentialsRotated)) } -test fn new_altra_plan_starts_factory_default_at_192() -> Bool { - new_altra_onboarding_plan.bmc.host == "192.168.1.192" - && phase_tag(p: new_altra_onboarding_plan.start_phase) == 0 - && match new_altra_onboarding_plan.rotated_credential { +test fn srv3_plan_starts_factory_default_at_192() -> Bool { + srv3_onboarding_plan.bmc.host == "192.168.1.192" + && phase_tag(p: srv3_onboarding_plan.start_phase) == 0 + && match srv3_onboarding_plan.rotated_credential { Stored { secret_name: s } => s == "bmc-srv3-admin" Chained { steps: _ } => false } - && match new_altra_onboarding_plan.os_target { + && match srv3_onboarding_plan.os_target { UbuntuOs { distro: d } => match d { NobleNumbat2404Lts => true @@ -194,12 +195,32 @@ test fn new_altra_plan_starts_factory_default_at_192() -> Bool { WindowsOs { distro: _ } => false MacosOs { distro: _ } => false } - && redfish_boot_target_wire(t: new_altra_onboarding_plan.os_install_boot_target) == "Cd" + && redfish_boot_target_wire(t: srv3_onboarding_plan.os_install_boot_target) == "Cd" && redfish_boot_target_wire(t: BootTargetCd) == "Cd" - && redfish_account_role_wire(role: new_altra_onboarding_plan.rotated_account_role) == "Administrator" + && redfish_account_role_wire(role: srv3_onboarding_plan.rotated_account_role) == "Administrator" && redfish_account_role_wire(role: AccountRoleAdministrator) == "Administrator" } +test fn srv4_plan_targets_195_with_srv4_secret() -> Bool { + srv4_onboarding_plan.bmc.host == "192.168.1.195" + && !(srv4_onboarding_plan.bmc.host == srv3_onboarding_plan.bmc.host) + && phase_tag(p: srv4_onboarding_plan.start_phase) == 0 + && match srv4_onboarding_plan.rotated_credential { + Stored { secret_name: s } => s == "bmc-srv4-admin" + Chained { steps: _ } => false + } + && match srv4_onboarding_plan.os_target { + UbuntuOs { distro: d } => + match d { + NobleNumbat2404Lts => true + } + WindowsOs { distro: _ } => false + MacosOs { distro: _ } => false + } + && redfish_boot_target_wire(t: srv4_onboarding_plan.os_install_boot_target) == "Cd" + && redfish_account_role_wire(role: srv4_onboarding_plan.rotated_account_role) == "Administrator" +} + test fn credential_probe_maps_to_correct_phase() -> Bool { match bmc_credential_probe_to_phase(probe: FactoryCredentialActive) { Present { value: FactoryDefault } => true diff --git a/dag/test/claim/fleet_intent_network_witness_test.dag b/dag/test/claim/fleet_intent_network_witness_test.dag index 40de3eefb58..aafc6356a15 100644 --- a/dag/test/claim/fleet_intent_network_witness_test.dag +++ b/dag/test/claim/fleet_intent_network_witness_test.dag @@ -23,7 +23,7 @@ test fn fleet_intent_network_container_to_host_lan_denied() -> Bool { } test fn fleet_intent_network_topology_endpoint_count_holds() -> Bool { - list_length(items: fleet_intent_network.endpoints) == 8 + list_length(items: fleet_intent_network.endpoints) == 9 } test fn fleet_intent_network_srv1_addresses_holds() -> Bool { From fa728434b0e685df0595abfc06a28491c99c6667 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 21:01:37 +0000 Subject: [PATCH 06/58] fix(bmc): mint OpenBMC-policy-compliant credential (found by live srv3/srv4 rotation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First live execution of the rotation flow (srv3's was never run green) surfaced that mint_bmc_credential's base64 octets are rejected by OpenBMC password validation (PropertyValueFormatError). A firmware-policy divergence also showed: srv4 (newer OpenBMC) accepts alphanumeric, but srv3 (OpenBMC 2.07.00, pwquality, MinPasswordLength 9 / MaxPasswordLength 20) requires a 4th character class. Fix: Urandom.ReadPassword β€” composition-guaranteed generator (>=1 upper/lower/ digit/special from the shell/JSON/basic-auth-safe set _.@#%-); mint_bmc_credential mints a 16-char such password (within 9-20, accepted by both firmwares). The fail-closed read-back gate correctly aborted every base64/alnum attempt before rotating, so no lockout. Both srv3 (secret bmc-srv3-admin v6) and srv4 (v2) are now live-rotated off factory 0penBmc; orphaned pre-rotation versions destroyed. Full-corpus typecheck clean (850 modules, 0 errors) + witnesses green. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/entropy/entropy.dag | 11 +++++++++ dag/gunbc/tools/bmc_onboard.dag | 5 ++-- docs/plans/srv4-bmc-onboarding-gap.md | 33 +++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/dag/extdeps/entropy/entropy.dag b/dag/extdeps/entropy/entropy.dag index 63fd12e5d5c..400f2b25df0 100644 --- a/dag/extdeps/entropy/entropy.dag +++ b/dag/extdeps/entropy/entropy.dag @@ -42,4 +42,15 @@ service Urandom { nonzero => String "Urandom.ReadBytes: /dev/urandom or base64 failed" } } + + operation ReadPassword { + input { count: Int } + output { password: String from "stdout" } + readonly + transport shell { argv: ["sh", "-c", "U=$(LC_ALL=C tr -dc 'A-Z' Unit + nonzero => String "Urandom.ReadPassword: /dev/urandom failed" + } + } } diff --git a/dag/gunbc/tools/bmc_onboard.dag b/dag/gunbc/tools/bmc_onboard.dag index b831bdf0733..a0f56869a33 100644 --- a/dag/gunbc/tools/bmc_onboard.dag +++ b/dag/gunbc/tools/bmc_onboard.dag @@ -21,7 +21,6 @@ import gunbc.auth.access_token_source { } import std.credentials { Stored } import std.types { NonEmptyStr, Secret, String } -import std.measure { ByteSize, byte_size, byte_size_count } import std.logic { Bool } import std.resources { Network } import std.process { ProcessExit, ExitSuccess, exit_failure } @@ -48,7 +47,7 @@ data bmc_onboard_secret_on_disk_debt: Disposition = Scaffold { data bmc_secrets_gcp_project: GcpProjectId = "gunbai-secrets" -data bmc_credential_bytes: ByteSize = byte_size(20) +data bmc_credential_length_chars: Int = 16 fn netrc_content(host: String, account: String, password: String) -> String { concat("machine ", concat(host, concat(" login ", concat(account, concat(" password ", concat(password, "\n")))))) @@ -87,7 +86,7 @@ fn onboarding_secret_id(plan: BmcOnboardingPlan) -> NonEmptyStr { } fn mint_bmc_credential() -> Secret { - Urandom.ReadBytes(count: byte_size_count(b: bmc_credential_bytes) as Int).octets_b64 as Secret + Urandom.ReadPassword(count: bmc_credential_length_chars).password as Secret } fn write_netrc(host: String, account: String, password: String) -> Bool { diff --git a/docs/plans/srv4-bmc-onboarding-gap.md b/docs/plans/srv4-bmc-onboarding-gap.md index 1ffbdfa10df..b732da5ca31 100644 --- a/docs/plans/srv4-bmc-onboarding-gap.md +++ b/docs/plans/srv4-bmc-onboarding-gap.md @@ -86,3 +86,36 @@ the other handler, not a hard prerequisite). Open question for the operator: scope now β€” do all of G1–G4, or minimally G1+G3 (parameterize + operator-token handler) to unblock srv4 today and land G2 (gcloud self-provision) as follow-up? + +## Outcome (2026-07-21) β€” srv4 rotated + two gaps found by execution + +Full close G1–G4 landed and verified (850-module typecheck clean + 11 witnesses green). +srv4's BMC was then rotated **live** with an operator-supplied token β€” the first real +execution of this flow (srv3's rotation was never run live; its debt marker said so). Two +gaps surfaced only because we executed (Β§5 "done = green by execution"): + +- **Credential format (FIXED).** `mint_bmc_credential` used `Urandom.ReadBytes(...).octets_b64` + β€” base64, whose `=`/`/`/`+` OpenBMC rejects with `PropertyValueFormatError` (HTTP 400) on the + password PATCH. The fail-closed read-back gate correctly aborted before rotating (no lockout). + Then a *firmware-policy divergence* surfaced: srv4 (newer OpenBMC, UUID 83819837) accepted a + 16-char **alphanumeric** password, but srv3 (OpenBMC **2.07.00**, UUID 6df9b4b1) rejected it β€” + its `pam_pwquality` needs a 4th character class (special). Policy read from srv3's AccountService: + MinPasswordLength 9, MaxPasswordLength 20. Fixed universally: added `Urandom.ReadPassword`, a + composition-guaranteed generator (β‰₯1 upper/lower/digit/special from the shell/JSON/basic-auth-safe + set `_.@#%-`), and `mint_bmc_credential` mints a **16-char** such password (within 9–20, satisfies + both firmwares). Live-verified: srv4 rotated with a 16-char alnum credential (before the divergence + was known); srv3 rotated with a 16-char 4-class credential. +- **Secret-container creation (FOLLOW-UP).** `bmc_store_and_rotate`'s `AddVersion` assumes the + secret exists; `bmc-srv4-admin` did not (404) because the GCP bootstrap + (`gunbc.assimilate.bmc_bootstrap_provision`, still srv3-hardcoded) never ran for srv4. Created + it manually this session. Correct fix: parameterize `bmc_bootstrap_provision` per-host (same move + as G1) so it creates `bmc--admin` + IAM before onboarding. A blind create-if-missing inside + the rotation path is unsafe β€” the gcp effect consumption style aborts on non-200, so a 409 + (already-exists) on re-run would break idempotency; the bootstrap is the right home. + +State now: **both srv3 and srv4 CredentialsRotated.** srv4 β†’ Secret Manager `bmc-srv4-admin` v2 +(v1, rejected base64, destroyed). srv3 β†’ `bmc-srv3-admin` v6 (v1–v5, prior base64/alnum attempts +that populated the secret but never rotated the BMC β€” the "secret exists, server never rotated" +state β€” destroyed). Both: factory `0penBmc` β†’ 401, stored credential β†’ 200. Next lifecycle step is +OsInstalled (the existing seeded-autoinstall spine); the rotation persists across OS installs since +the BMC is out-of-band. From 291f904a1c2e245fbdb2552ca76c99bead7253b2 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 21:19:23 +0000 Subject: [PATCH 07/58] =?UTF-8?q?feat(access):=20model=20fleet=20SSH=20acc?= =?UTF-8?q?ess=20=E2=80=94=20operator=20+=20automation=20public=20keys=20(?= =?UTF-8?q?durable=20in=20repo)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SSH-who, the third principal facet alongside POSIX-who (fleet_posix_accounts) and GCP-who (fleet_operator_gcp_iam_member): - extdeps/access/ssh.dag: SshPublicKey type + authorized_keys line renderer. - gunbc/fleet_ssh_access.dag: operator MacBook key (global break-glass, logs in as briansrls) + fleet-automation key (machine access). Both PUBLIC keys grounded in the repo (public keys aren't secret). fleet_authorized_keys = both, applied to every host by breadth. - fleet_automation_ssh_privkey_secret β†’ SecretRef to Secret Manager 'fleet-automation-ssh-key' (project gunbai-secrets, v1). The private key's only copy lives there; generated 2026-07-21, local copy shredded. - keys/fleet-ssh-public-keys.txt: plain-text backup of both public keys. FOLLOW-UP: fleet-automation-ssh-key has no secret-level IAM binding yet (project- scoped access). Lock to a dedicated automation SA with secretAccessor, mirroring bmc-assimilator on bmc-srv*-admin. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/access/ssh.dag | 38 +++++++++++++++++++++++++++ dag/gunbc/fleet_ssh_access.dag | 48 ++++++++++++++++++++++++++++++++++ keys/fleet-ssh-public-keys.txt | 3 +++ 3 files changed, 89 insertions(+) create mode 100644 dag/extdeps/access/ssh.dag create mode 100644 dag/gunbc/fleet_ssh_access.dag create mode 100644 keys/fleet-ssh-public-keys.txt diff --git a/dag/extdeps/access/ssh.dag b/dag/extdeps/access/ssh.dag new file mode 100644 index 00000000000..da2ab85e692 --- /dev/null +++ b/dag/extdeps/access/ssh.dag @@ -0,0 +1,38 @@ +module extdeps.access.ssh + +import std.types { NonEmptyStr, String } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "man.openbsd.org/sshd.8#AUTHORIZED_KEYS_FILE_FORMAT" + } +} + +type SshKeyAlgorithm + = Ed25519 + | Rsa + | EcdsaP256 + +fn ssh_key_algorithm_token(a: SshKeyAlgorithm) -> NonEmptyStr { + match a { + Ed25519 => "ssh-ed25519" + Rsa => "ssh-rsa" + EcdsaP256 => "ecdsa-sha2-nistp256" + } +} + +type SshPublicKey { + algorithm: SshKeyAlgorithm + material: NonEmptyStr + comment: String +} + +fn ssh_public_key_line(k: SshPublicKey) -> String { + concat( + concat(ssh_key_algorithm_token(a: k.algorithm) as String, " "), + concat(k.material as String, concat(" ", k.comment)), + ) +} diff --git a/dag/gunbc/fleet_ssh_access.dag b/dag/gunbc/fleet_ssh_access.dag new file mode 100644 index 00000000000..47b4a7dcafd --- /dev/null +++ b/dag/gunbc/fleet_ssh_access.dag @@ -0,0 +1,48 @@ +module gunbc.fleet_ssh_access + +import std.types { List, NonEmptyStr } +import extdeps.access.ssh { SshPublicKey, Ed25519 } +import extdeps.cloud.gcp.secret_ref { SecretRef, HashPending } +import extdeps.cloud.gcp.gcp { GcpProjectId } +import gunbc.fleet_posix_accounts { FleetPosixAccountKey, FleetAccountOperator } + +data fleet_ssh_secrets_project: GcpProjectId = "gunbai-secrets" + + +data operator_macbook_ssh_key: SshPublicKey = SshPublicKey { + algorithm: Ed25519, + material: "AAAAC3NzaC1lZDI1NTE5AAAAIKgumC/OBjsSgM266FLtgKJpF179Aj/MzEZHQH79QRQF", + comment: "macbook-m4", +} + +data fleet_automation_ssh_key: SshPublicKey = SshPublicKey { + algorithm: Ed25519, + material: "AAAAC3NzaC1lZDI1NTE5AAAAIB5ObOQw+v/Rd5xdurhZpwI9awQNqCWNgFWbZCQsXJfN", + comment: "fleet-automation@gunbc", +} + +data fleet_automation_ssh_privkey_secret: SecretRef = SecretRef { + project: fleet_ssh_secrets_project, + secret: "fleet-automation-ssh-key" as NonEmptyStr, + version: "latest", + hash_state: HashPending, +} + +type SshAccessPrincipal { + posix_account: FleetPosixAccountKey + public_key: SshPublicKey +} + +data operator_ssh_principal: SshAccessPrincipal = SshAccessPrincipal { + posix_account: FleetAccountOperator, + public_key: operator_macbook_ssh_key, +} + +data fleet_authorized_keys: List = [ + operator_macbook_ssh_key, + fleet_automation_ssh_key, +] + +fn fleet_host_authorized_keys() -> List { + fleet_authorized_keys +} diff --git a/keys/fleet-ssh-public-keys.txt b/keys/fleet-ssh-public-keys.txt new file mode 100644 index 00000000000..9bf3363edfa --- /dev/null +++ b/keys/fleet-ssh-public-keys.txt @@ -0,0 +1,3 @@ +# Fleet SSH public keys (public β€” safe to store here). Authority: dag/gunbc/fleet_ssh_access.dag +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKgumC/OBjsSgM266FLtgKJpF179Aj/MzEZHQH79QRQF macbook-m4 +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB5ObOQw+v/Rd5xdurhZpwI9awQNqCWNgFWbZCQsXJfN fleet-automation@gunbc From 05f211f095f15fdc181f222e493cc1c219b3df80 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 21:23:41 +0000 Subject: [PATCH 08/58] =?UTF-8?q?fix(bmc):=20remove=20fabricated-fallback?= =?UTF-8?q?=20in=20onboarding=5Fsecret=5Fid=20(=C2=A75=20fail-closed)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit onboarding_secret_id matched plan.rotated_credential and, on the Chained arm, fabricated a secret id (plan.bmc.host; pre-existing code fabricated a literal) β€” a Β§5 "fabricated plausible output" fallback. Construction-first fix: narrow the plan field from rotated_credential: CredentialFlow to secret_name: NonEmptyStr, so the Chained state is unwritable and the function is total (plan.secret_name, no match, no fallback). Dropped now-unused std.credentials imports. Verified by execution: srv3/srv4 plan witnesses (now assert secret_name directly) + bmc_onboard load, all green. rotated_credential/Chained gone from the corpus. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/gunbc/bmc_onboarding.dag | 6 +++--- dag/gunbc/tools/bmc_onboard.dag | 6 +----- .../claim/bmc_onboarding_lifecycle_witness_test.dag | 11 ++--------- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/dag/gunbc/bmc_onboarding.dag b/dag/gunbc/bmc_onboarding.dag index 52e8a2ed2f8..8d1aaa06db3 100644 --- a/dag/gunbc/bmc_onboarding.dag +++ b/dag/gunbc/bmc_onboarding.dag @@ -1,7 +1,7 @@ module gunbc.bmc_onboarding import std.bmc { BmcEndpoint, Redfish } -import std.credentials { FactoryLogin, CredentialFlow, Stored } +import std.credentials { FactoryLogin } import std.os.types { OperatingSystemProduct, UbuntuOs, NobleNumbat2404Lts } import std.types { NonEmptyStr, Int, List } import std.logic { Bool } @@ -195,7 +195,7 @@ type BmcOnboardingPlan { bmc: BmcEndpoint redfish_system_id: NonEmptyStr factory_login: FactoryLogin - rotated_credential: CredentialFlow + secret_name: NonEmptyStr rotated_account_role: RedfishAccountRole os_target: OperatingSystemProduct os_install_boot_target: RedfishBootSourceOverrideTarget @@ -215,7 +215,7 @@ fn altra_onboarding_plan(bmc_host: NonEmptyStr, secret_name: NonEmptyStr) -> Bmc bmc: BmcEndpoint { host: bmc_host, protocol: Redfish }, redfish_system_id: asrock_altrad8ud_redfish_system_id, factory_login: openbmc_factory_login, - rotated_credential: Stored { secret_name: secret_name }, + secret_name: secret_name, rotated_account_role: AccountRoleAdministrator, os_target: UbuntuOs { distro: NobleNumbat2404Lts }, os_install_boot_target: srv3_install_boot_target, diff --git a/dag/gunbc/tools/bmc_onboard.dag b/dag/gunbc/tools/bmc_onboard.dag index a0f56869a33..67b4fbd578e 100644 --- a/dag/gunbc/tools/bmc_onboard.dag +++ b/dag/gunbc/tools/bmc_onboard.dag @@ -19,7 +19,6 @@ import gunbc.auth.access_token_source { GcloudPrintToken, resolve_access_token, } -import std.credentials { Stored } import std.types { NonEmptyStr, Secret, String } import std.logic { Bool } import std.resources { Network } @@ -79,10 +78,7 @@ fn gcp_secret_full_name(project: String, secret_id: String) -> String { } fn onboarding_secret_id(plan: BmcOnboardingPlan) -> NonEmptyStr { - match plan.rotated_credential { - Stored { secret_name: sn } => sn - Chained { steps: _ } => plan.bmc.host - } + plan.secret_name } fn mint_bmc_credential() -> Secret { diff --git a/dag/test/claim/bmc_onboarding_lifecycle_witness_test.dag b/dag/test/claim/bmc_onboarding_lifecycle_witness_test.dag index 27129ea603b..d5e25c5427e 100644 --- a/dag/test/claim/bmc_onboarding_lifecycle_witness_test.dag +++ b/dag/test/claim/bmc_onboarding_lifecycle_witness_test.dag @@ -2,7 +2,6 @@ module test.claim.bmc_onboarding_lifecycle import std.logic { Bool } import std.types { Int } -import std.credentials { Stored, Chained } import std.os.types { UbuntuOs, WindowsOs, MacosOs, NobleNumbat2404Lts } import gunbc.bmc_onboarding { BmcOnboardingPhase, @@ -183,10 +182,7 @@ test fn bmc_decom_does_not_roundtrip_when_inverse_is_wrong() -> Bool { test fn srv3_plan_starts_factory_default_at_192() -> Bool { srv3_onboarding_plan.bmc.host == "192.168.1.192" && phase_tag(p: srv3_onboarding_plan.start_phase) == 0 - && match srv3_onboarding_plan.rotated_credential { - Stored { secret_name: s } => s == "bmc-srv3-admin" - Chained { steps: _ } => false - } + && srv3_onboarding_plan.secret_name == "bmc-srv3-admin" && match srv3_onboarding_plan.os_target { UbuntuOs { distro: d } => match d { @@ -205,10 +201,7 @@ test fn srv4_plan_targets_195_with_srv4_secret() -> Bool { srv4_onboarding_plan.bmc.host == "192.168.1.195" && !(srv4_onboarding_plan.bmc.host == srv3_onboarding_plan.bmc.host) && phase_tag(p: srv4_onboarding_plan.start_phase) == 0 - && match srv4_onboarding_plan.rotated_credential { - Stored { secret_name: s } => s == "bmc-srv4-admin" - Chained { steps: _ } => false - } + && srv4_onboarding_plan.secret_name == "bmc-srv4-admin" && match srv4_onboarding_plan.os_target { UbuntuOs { distro: d } => match d { From ac306647dff6c454b5abca82a9ce32d76e995ba4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 21:27:52 +0000 Subject: [PATCH 09/58] feat(access): dedicated fleet-automation SA scoped to the SSH private key Created service account fleet-automation@gunbai-secrets (least-privilege: secretAccessor on fleet-automation-ssh-key only, mirroring bmc-assimilator's scoping). Grounded in the model: fleet_automation_sa_email + SecretOwner record tying the SA to the private-key secret, so "who owns the private key" is answered in the repo, not just in GCP. Binding applied out-of-band (provenance recorded); full WIF SecretAccessGrant modeling is follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/gunbc/fleet_ssh_access.dag | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/dag/gunbc/fleet_ssh_access.dag b/dag/gunbc/fleet_ssh_access.dag index 47b4a7dcafd..4919b226e62 100644 --- a/dag/gunbc/fleet_ssh_access.dag +++ b/dag/gunbc/fleet_ssh_access.dag @@ -3,11 +3,13 @@ module gunbc.fleet_ssh_access import std.types { List, NonEmptyStr } import extdeps.access.ssh { SshPublicKey, Ed25519 } import extdeps.cloud.gcp.secret_ref { SecretRef, HashPending } -import extdeps.cloud.gcp.gcp { GcpProjectId } +import extdeps.cloud.gcp.gcp { GcpProjectId, ServiceAccountEmail } import gunbc.fleet_posix_accounts { FleetPosixAccountKey, FleetAccountOperator } data fleet_ssh_secrets_project: GcpProjectId = "gunbai-secrets" +data fleet_automation_sa_email: ServiceAccountEmail = "fleet-automation@gunbai-secrets.iam.gserviceaccount.com" + data operator_macbook_ssh_key: SshPublicKey = SshPublicKey { algorithm: Ed25519, @@ -28,6 +30,18 @@ data fleet_automation_ssh_privkey_secret: SecretRef = SecretRef { hash_state: HashPending, } +type SecretOwner { + secret: SecretRef + accessor_sa: ServiceAccountEmail + binding_provenance: NonEmptyStr +} + +data fleet_automation_privkey_owner: SecretOwner = SecretOwner { + secret: fleet_automation_ssh_privkey_secret, + accessor_sa: fleet_automation_sa_email, + binding_provenance: "secretAccessor bound out-of-band 2026-07-21 (least-privilege, mirrors bmc-assimilator); full WIF SecretAccessGrant modeling is follow-up", +} + type SshAccessPrincipal { posix_account: FleetPosixAccountKey public_key: SshPublicKey From 264239e74a80412dd4004f10e8dcdabaebdc96bc Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 21:32:13 +0000 Subject: [PATCH 10/58] feat(access): bake fleet SSH keys into autoinstall, disable password SSH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the access model into the OS install: UbuntuAutoinstallPayload gains ssh_authorized_keys (List) + ssh_password_auth; os_install_emit renders the subiquity ssh section with authorized-keys (operator + automation public keys) and allow-pw. srv3 payloads set fleet_authorized_keys + allow-pw false β€” installed hosts trust the fleet keys and refuse password SSH. Verified by execution: srv3_os_install_emit witnesses green, incl. a new discriminating one asserting both key lines present + "allow-pw: false". Note: identity.password still carries the bootstrap hash; per-host strong console break-glass credential is part of Step B (per-host install identity). Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/os/ubuntu_autoinstall.dag | 5 ++++- dag/gunbc/os_install.dag | 5 +++++ dag/gunbc/os_install_emit.dag | 3 +++ dag/test/claim/srv3_os_install_emit_test.dag | 8 ++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/dag/extdeps/os/ubuntu_autoinstall.dag b/dag/extdeps/os/ubuntu_autoinstall.dag index 8f1a8b90c8e..afb5db3891e 100644 --- a/dag/extdeps/os/ubuntu_autoinstall.dag +++ b/dag/extdeps/os/ubuntu_autoinstall.dag @@ -1,7 +1,8 @@ module extdeps.os.ubuntu_autoinstall -import std.types { Int, NonEmptyStr, Bool } +import std.types { Int, NonEmptyStr, Bool, List } import std.os.types { UbuntuDistribution } +import extdeps.access.ssh { SshPublicKey } import extdeps.uri { Uri, Https } import extdeps.external_authority { ExternalAuthority } @@ -36,5 +37,7 @@ type UbuntuAutoinstallPayload { locale: NonEmptyStr keyboard_layout: NonEmptyStr install_openssh_server: Bool + ssh_authorized_keys: List + ssh_password_auth: Bool storage_policy: AutoinstallStoragePolicy } diff --git a/dag/gunbc/os_install.dag b/dag/gunbc/os_install.dag index 81cab6d5550..6fd5a266e03 100644 --- a/dag/gunbc/os_install.dag +++ b/dag/gunbc/os_install.dag @@ -22,6 +22,7 @@ import extdeps.os.ubuntu_autoinstall { StoragePolicyDirectLayout, } import gunbc.fleet_intent_network { operator_host_srv3, operator_lan_segment } +import gunbc.fleet_ssh_access { fleet_authorized_keys } import product.placement_supply { HostIdentity } import product.network_topology { NetworkSegment } import extdeps.bmc.types { @@ -133,6 +134,8 @@ data srv3_ubuntu_autoinstall: UbuntuAutoinstallPayload = UbuntuAutoinstallPayloa locale: "en_US.UTF-8", keyboard_layout: "us", install_openssh_server: true, + ssh_authorized_keys: fleet_authorized_keys, + ssh_password_auth: false, storage_policy: StoragePolicyAbsent, } @@ -145,6 +148,8 @@ data srv3_ubuntu_autoinstall_on_iso: UbuntuAutoinstallPayload = UbuntuAutoinstal locale: "en_US.UTF-8", keyboard_layout: "us", install_openssh_server: true, + ssh_authorized_keys: fleet_authorized_keys, + ssh_password_auth: false, storage_policy: StoragePolicyDirectLayout, } diff --git a/dag/gunbc/os_install_emit.dag b/dag/gunbc/os_install_emit.dag index 40f20d9660c..bb02bfaed0b 100644 --- a/dag/gunbc/os_install_emit.dag +++ b/dag/gunbc/os_install_emit.dag @@ -9,6 +9,7 @@ import extdeps.os.ubuntu_autoinstall { StoragePolicyAbsent, StoragePolicyDirectLayout, } +import extdeps.access.ssh { SshPublicKey, ssh_public_key_line } import extdeps.languages.yaml.types { YamlValue, YamlKeyValue, yaml_mapping, yaml_sequence, yaml_string, yaml_int, yaml_bool, @@ -38,6 +39,8 @@ fn autoinstall_inner_entries(payload: UbuntuAutoinstallPayload) -> List yaml_string(s: ssh_public_key_line(k: k))))), + kv(key: "allow-pw", value: yaml_bool(b: payload.ssh_password_auth)), ])), ]) } else { diff --git a/dag/test/claim/srv3_os_install_emit_test.dag b/dag/test/claim/srv3_os_install_emit_test.dag index 9064a0b06db..5224fa18eda 100644 --- a/dag/test/claim/srv3_os_install_emit_test.dag +++ b/dag/test/claim/srv3_os_install_emit_test.dag @@ -32,3 +32,11 @@ test fn srv3_user_data_enables_remote_access_and_locale() -> Bool { && string_contains(s: ud, pattern: " layout: us") && string_contains(s: ud, pattern: " username: ubuntu") } + +test fn srv3_user_data_bakes_fleet_keys_and_disables_password_ssh() -> Bool { + let ud = autoinstall_user_data(payload: srv3_ubuntu_autoinstall) + string_contains(s: ud, pattern: " authorized-keys:") + && string_contains(s: ud, pattern: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKgumC/OBjsSgM266FLtgKJpF179Aj/MzEZHQH79QRQF macbook-m4") + && string_contains(s: ud, pattern: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB5ObOQw+v/Rd5xdurhZpwI9awQNqCWNgFWbZCQsXJfN fleet-automation@gunbc") + && string_contains(s: ud, pattern: " allow-pw: false") +} From 2b49c46bf2dfd62432ecf44be6915a8f08de290f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 21:55:51 +0000 Subject: [PATCH 11/58] feat(srv4): autoinstall payload + seeded-ISO rows (Step B) srv4 host identity baked at install time: srv4_autoinstall_identity (hostname srv4, console break-glass hash; plaintext in Secret Manager host-srv4-console) + srv4_ubuntu_autoinstall_on_iso (NoCloudLocal, DirectLayout, fleet SSH keys, allow-pw false). srv4 seeded-media artifact rows + srv4_seeded_install_media_remaster mirror srv3, driven by the same install_media_remaster_script builder. Dry-run verified (typechecks, script generates, ExitSuccess). Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/gunbc/os_install.dag | 25 +++++++++++- dag/gunbc/srv3_seeded_install_media.dag | 52 ++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/dag/gunbc/os_install.dag b/dag/gunbc/os_install.dag index 6fd5a266e03..b2af56fbb0a 100644 --- a/dag/gunbc/os_install.dag +++ b/dag/gunbc/os_install.dag @@ -21,7 +21,7 @@ import extdeps.os.ubuntu_autoinstall { StoragePolicyAbsent, StoragePolicyDirectLayout, } -import gunbc.fleet_intent_network { operator_host_srv3, operator_lan_segment } +import gunbc.fleet_intent_network { operator_host_srv3, operator_host_srv4, operator_lan_segment } import gunbc.fleet_ssh_access { fleet_authorized_keys } import product.placement_supply { HostIdentity } import product.network_topology { NetworkSegment } @@ -153,6 +153,29 @@ data srv3_ubuntu_autoinstall_on_iso: UbuntuAutoinstallPayload = UbuntuAutoinstal storage_policy: StoragePolicyDirectLayout, } +data srv4_console_password_hash: NonEmptyStr = "$6$gYMJusSKbtQHFdti$ouwvFuAeH0gdZkTSVJgEWwotvA7TzhxKch/pjg/2RyuR3cc4rEnnCsUnB.gH3cGz9vlszdSdiTZznCVmVrxv60" + +data srv4_autoinstall_identity: AutoinstallIdentity = AutoinstallIdentity { + hostname: operator_host_srv4, + username: "ubuntu", + realname: "gunbc fleet node srv4", + hashed_password: srv4_console_password_hash, +} + +data srv4_ubuntu_autoinstall_on_iso: UbuntuAutoinstallPayload = UbuntuAutoinstallPayload { + autoinstall_version: 1, + delivery: NoCloudLocal, + fully_automated: true, + target_distribution: NobleNumbat2404Lts, + identity: srv4_autoinstall_identity, + locale: "en_US.UTF-8", + keyboard_layout: "us", + install_openssh_server: true, + ssh_authorized_keys: fleet_authorized_keys, + ssh_password_auth: false, + storage_policy: StoragePolicyDirectLayout, +} + data srv3_os_install_plan: Srv3OsInstallPlan = Srv3OsInstallPlan { os_target: UbuntuOs { distro: NobleNumbat2404Lts }, host_architecture: Aarch64, diff --git a/dag/gunbc/srv3_seeded_install_media.dag b/dag/gunbc/srv3_seeded_install_media.dag index 2ce19e59e13..47f5d6490b2 100644 --- a/dag/gunbc/srv3_seeded_install_media.dag +++ b/dag/gunbc/srv3_seeded_install_media.dag @@ -20,7 +20,7 @@ import extdeps.os.ubuntu_seeded_install_media_toolchain { import extdeps.tools.xorriso { xorriso_cli_tool } import extdeps.apt { package_xorriso } import extdeps.os.ubuntu_autoinstall { NoCloudLocal, NoCloudNet } -import gunbc.os_install { srv3_ubuntu_autoinstall_on_iso } +import gunbc.os_install { srv3_ubuntu_autoinstall_on_iso, srv4_ubuntu_autoinstall_on_iso } import gunbc.os_install_emit { autoinstall_user_data } import gunbc.shell_bash_runner { shell_exec_via_bash } import gunbc.os_install_actuator_selection { srv3_os_install_actuator_host } @@ -137,6 +137,38 @@ data srv3_seeded_ubuntu_install_media_install_path: NonEmptyStr = ubuntu_seeded_ artifact: srv3_seeded_install_media_artifact, ) +data srv4_autoinstall_user_data_body: NonEmptyStr = autoinstall_user_data( + payload: srv4_ubuntu_autoinstall_on_iso, +) as NonEmptyStr + +data srv4_seeded_nocloud_dir_on_iso: NonEmptyStr = ubuntu_seeded_install_media_default_nocloud_dir( + hostname: srv4_ubuntu_autoinstall_on_iso.identity.hostname, +) + +data srv4_seeded_grub_kernel_cmdline: NonEmptyStr = ubuntu_seeded_install_media_grub_kernel_cmdline( + nocloud_dir: srv4_seeded_nocloud_dir_on_iso, +) + +data srv4_seeded_install_media_content_hash: ContentHash = seeded_install_media_content_hash( + stock_content_sha256: noble_numbat_2404_3_live_server_arm64.content_sha256 as NonEmptyStr, + autoinstall_user_data_body: srv4_autoinstall_user_data_body, + grub_kernel_cmdline: srv4_seeded_grub_kernel_cmdline, + nocloud_dir_on_iso: srv4_seeded_nocloud_dir_on_iso, +) + +data srv4_seeded_install_media_artifact: UbuntuSeededInstallMediaArtifactRow = UbuntuSeededInstallMediaArtifactRow { + stock_artifact: noble_numbat_2404_3_live_server_arm64, + autoinstall: srv4_ubuntu_autoinstall_on_iso, + autoinstall_user_data_body: srv4_autoinstall_user_data_body, + grub_kernel_cmdline: srv4_seeded_grub_kernel_cmdline, + nocloud_dir_on_iso: srv4_seeded_nocloud_dir_on_iso, + content_hash: srv4_seeded_install_media_content_hash, +} + +data srv4_seeded_ubuntu_install_media_install_path: NonEmptyStr = ubuntu_seeded_install_media_install_path( + artifact: srv4_seeded_install_media_artifact, +) + func srv3_seeded_install_media_toolchain_ensure() -> ProcessExit uses net: Network { @@ -167,6 +199,24 @@ func srv3_seeded_install_media_remaster() -> ProcessExit } } +func srv4_seeded_install_media_remaster() -> ProcessExit + uses net: Network +{ + let ensure_script = install_media_remaster_toolchain_ensure_script() + let ensure_run = shell.Exec.Run(script: shell_exec_via_bash(body: ensure_script)) + if ensure_run.success == false { + return exit_failure(reason: concat("seeded-install-media-remaster: toolchain ensure failed: ", ensure_run.stderr)) + } + + let script = install_media_remaster_script(artifact: srv4_seeded_install_media_artifact) + let run = shell.Exec.Run(script: shell_exec_via_bash(body: script)) + if run.success { + ExitSuccess + } else { + exit_failure(reason: concat("seeded-install-media-remaster: ", run.stderr)) + } +} + fn srv3_seeded_install_media_actuator_host() -> ComputeHost { srv3_os_install_actuator_host } From 8a371e4dcf87b153a3274fd0dcbe3468b35dff34 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 22:37:11 +0000 Subject: [PATCH 12/58] feat(uefi): model UEFI Shell + boot-config solver (dissolve manual UEFI GUI step) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manual "enter UEFI setup, enable PXE / set boot" GUI step becomes a grounded, solvable model: - extdeps/firmware/uefi_shell.dag: UEFI Shell command surface (bcfg boot dump/add/mv/rm, map, reset) cited to the UEFI Shell 2.2 spec, with a renderer to the real command text + a script folder. - gunbc/uefi_boot_config.dag: DesiredBootSource (install media | PXE entry) -> bcfg command sequence β€” the UEFI-shell realization of the same "what to boot" intent the Redfish BootSourceOverride path already models (Β§2, one intent / two realizations). srv4_uefi_install_boot targets fs0:\EFI\BOOT\BOOTAA64.EFI. Verified by execution: witnesses assert the exact bcfg sequence for install-media boot and for PXE-entry reorder. Next: SOL send transport (extend serial_console, currently capture-only) to drive these over obmc-console into srv4's UEFI shell. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/firmware/uefi_shell.dag | 78 +++++++++++++++++++ dag/gunbc/uefi_boot_config.dag | 42 ++++++++++ .../claim/uefi_boot_config_witness_test.dag | 25 ++++++ 3 files changed, 145 insertions(+) create mode 100644 dag/extdeps/firmware/uefi_shell.dag create mode 100644 dag/gunbc/uefi_boot_config.dag create mode 100644 dag/test/claim/uefi_boot_config_witness_test.dag diff --git a/dag/extdeps/firmware/uefi_shell.dag b/dag/extdeps/firmware/uefi_shell.dag new file mode 100644 index 00000000000..bbe2092f89d --- /dev/null +++ b/dag/extdeps/firmware/uefi_shell.dag @@ -0,0 +1,78 @@ +module extdeps.firmware.uefi_shell + +import std.types { NonEmptyStr, String, Int, Bool, List } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "uefi.org/sites/default/files/resources/UEFI_Shell_2_2.pdf" + } +} + +type UefiResetMode + = ResetWarm + | ResetCold + | ResetShutdown + +fn uefi_reset_flag(mode: UefiResetMode) -> String { + match mode { + ResetWarm => "-w" + ResetCold => "-c" + ResetShutdown => "-s" + } +} + +type BcfgTarget + = BcfgBoot + | BcfgDriver + +fn bcfg_target_word(target: BcfgTarget) -> String { + match target { + BcfgBoot => "boot" + BcfgDriver => "driver" + } +} + +type UefiShellCommand + = BcfgDump { target: BcfgTarget } + | BcfgAdd { target: BcfgTarget, index: Int, file: NonEmptyStr, description: NonEmptyStr } + | BcfgMove { target: BcfgTarget, from_index: Int, to_index: Int } + | BcfgRemove { target: BcfgTarget, index: Int } + | MapDevices { refresh: Bool } + | Reset { mode: UefiResetMode } + +fn uefi_shell_command_line(cmd: UefiShellCommand) -> String { + match cmd { + BcfgDump { target: t } => + concat("bcfg ", concat(bcfg_target_word(target: t), " dump -v")) + BcfgAdd { target: t, index: i, file: f, description: d } => + concat( + concat("bcfg ", concat(bcfg_target_word(target: t), " add ")), + concat( + concat(to_string(i), concat(" ", f as String)), + concat(" \"", concat(d as String, "\"")), + ), + ) + BcfgMove { target: t, from_index: a, to_index: b } => + concat( + concat("bcfg ", concat(bcfg_target_word(target: t), " mv ")), + concat(to_string(a), concat(" ", to_string(b))), + ) + BcfgRemove { target: t, index: i } => + concat("bcfg ", concat(bcfg_target_word(target: t), concat(" rm ", to_string(i)))) + MapDevices { refresh: r } => + if r { "map -r" } else { "map" } + Reset { mode: m } => + concat("reset ", uefi_reset_flag(mode: m)) + } +} + +fn uefi_shell_script(commands: List) -> String { + fold( + commands, + init: "", + f: (acc, cmd) => concat(acc, concat(uefi_shell_command_line(cmd: cmd), "\n")) + ) +} diff --git a/dag/gunbc/uefi_boot_config.dag b/dag/gunbc/uefi_boot_config.dag new file mode 100644 index 00000000000..e4361a1908d --- /dev/null +++ b/dag/gunbc/uefi_boot_config.dag @@ -0,0 +1,42 @@ +module gunbc.uefi_boot_config + +import std.types { NonEmptyStr, Int, List } +import extdeps.firmware.uefi_shell { + UefiShellCommand, + BcfgDump, + BcfgAdd, + BcfgMove, + MapDevices, + Reset, + BcfgBoot, + ResetWarm, + uefi_shell_script, +} + +type DesiredBootSource + = BootFromInstallMedia { efi_file: NonEmptyStr } + | BootFromPxeEntry { boot_index: Int } + +fn uefi_boot_config_commands(desired: DesiredBootSource) -> List { + match desired { + BootFromInstallMedia { efi_file: f } => [ + MapDevices { refresh: true }, + BcfgDump { target: BcfgBoot }, + BcfgAdd { target: BcfgBoot, index: 0, file: f, description: "gunbc-install-media" as NonEmptyStr }, + Reset { mode: ResetWarm }, + ] + BootFromPxeEntry { boot_index: i } => [ + BcfgDump { target: BcfgBoot }, + BcfgMove { target: BcfgBoot, from_index: i, to_index: 0 }, + Reset { mode: ResetWarm }, + ] + } +} + +fn uefi_boot_config_script(desired: DesiredBootSource) -> String { + uefi_shell_script(commands: uefi_boot_config_commands(desired: desired)) +} + +data srv4_uefi_install_boot: DesiredBootSource = BootFromInstallMedia { + efi_file: "fs0:\\EFI\\BOOT\\BOOTAA64.EFI", +} diff --git a/dag/test/claim/uefi_boot_config_witness_test.dag b/dag/test/claim/uefi_boot_config_witness_test.dag new file mode 100644 index 00000000000..e4412f0f0f0 --- /dev/null +++ b/dag/test/claim/uefi_boot_config_witness_test.dag @@ -0,0 +1,25 @@ +module test.claim.uefi_boot_config + +import std.logic { Bool } +import gunbc.uefi_boot_config { + uefi_boot_config_script, + srv4_uefi_install_boot, + uefi_boot_config_commands, + BootFromPxeEntry, +} +import extdeps.firmware.uefi_shell { uefi_shell_script } + +test fn srv4_install_boot_renders_bcfg_add_sequence() -> Bool { + let s = uefi_boot_config_script(desired: srv4_uefi_install_boot) + string_contains(s: s, pattern: "map -r") + && string_contains(s: s, pattern: "bcfg boot dump -v") + && string_contains(s: s, pattern: "bcfg boot add 0 fs0:\\EFI\\BOOT\\BOOTAA64.EFI \"gunbc-install-media\"") + && string_contains(s: s, pattern: "reset -w") +} + +test fn pxe_boot_reorders_existing_entry() -> Bool { + let s = uefi_shell_script(commands: uefi_boot_config_commands(desired: BootFromPxeEntry { boot_index: 3 })) + string_contains(s: s, pattern: "bcfg boot mv 3 0") + && string_contains(s: s, pattern: "reset -w") + && !string_contains(s: s, pattern: "bcfg boot add") +} From 1b2938bfe42cec3db40d89c52bd8c2f8d8df4349 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 22:42:35 +0000 Subject: [PATCH 13/58] feat(uefi/sol): SOL send transport + drive UEFI-shell boot-config over IPMI SOL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Confirmed live first (operator's caution): the ASRock ALTRAD8UD OpenBMC 2.07.00 supports IPMI SOL (ipmitool -I lanplus sol info β†’ Enabled, ADMINISTRATOR, port 623). That capability was unmodeled β€” grounded it now: BmcCapability gains CapabilitySerialConsole, added to the 2.07.00 list with a live-probe provenance row. Transport: extdeps.bmc.serial_console gains SolConsoleTransport::IpmiSol + SolConsoleSendIntent + sol_console_send_script (ipmitool sol activate with piped input via IPMI_PASSWORD -E; obmc-console send arm too; RedfishSerialInterface send fail-closed as read-only). Runner: gunbc.uefi_shell_over_sol turns the solved bcfg sequence into serial input (\r-submitted) and a SOL send script; srv4_uefi_boot_config_sol_send_intent targets .195. So the manual UEFI GUI step is now: solve DesiredBootSource β†’ bcfg β†’ drive over SOL, fully executable. Verified by execution: witnesses assert the srv4 send carries the bcfg sequence, the script uses ipmitool sol activate, and the ASRock 2.07.00 row declares the serial-console capability. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/bmc/capability.dag | 1 + dag/extdeps/bmc/serial_console.dag | 55 ++++++++++++++++++- dag/extdeps/boards/asrock_rack.dag | 4 ++ dag/gunbc/uefi_shell_over_sol.dag | 55 +++++++++++++++++++ .../uefi_shell_over_sol_witness_test.dag | 22 ++++++++ 5 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 dag/gunbc/uefi_shell_over_sol.dag create mode 100644 dag/test/claim/uefi_shell_over_sol_witness_test.dag diff --git a/dag/extdeps/bmc/capability.dag b/dag/extdeps/bmc/capability.dag index 0781fa692c5..69c8bb57c76 100644 --- a/dag/extdeps/bmc/capability.dag +++ b/dag/extdeps/bmc/capability.dag @@ -22,6 +22,7 @@ type BmcCapability | CapabilityVirtualMedia | CapabilityNbdProxyVirtualMedia | CapabilityBiosAttributeConfig + | CapabilitySerialConsole type BmcFirmwareVersion { family: BmcFirmwareFamily diff --git a/dag/extdeps/bmc/serial_console.dag b/dag/extdeps/bmc/serial_console.dag index ecfbb5876f5..ddc5d9dc305 100644 --- a/dag/extdeps/bmc/serial_console.dag +++ b/dag/extdeps/bmc/serial_console.dag @@ -1,6 +1,6 @@ module extdeps.bmc.serial_console -import std.types { NonEmptyStr, String, Milliseconds } +import std.types { NonEmptyStr, String, Int, Milliseconds } import std.measure { ByteSize } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } @@ -19,9 +19,17 @@ data redfish_serial_interface_authority: ExternalAuthority = ExternalAuthority { } } +data ipmi_sol_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "www.intel.com/content/dam/www/public/us/en/documents/product-briefs/ipmi-second-gen-interface-spec-v2-rev1-1.pdf" + } +} + type SolConsoleTransport = ObmcConsoleCli | RedfishSerialInterface + | IpmiSol type SolConsoleCaptureIntent { bmc_host: NonEmptyStr @@ -47,6 +55,7 @@ fn sol_console_transport_wire(transport: SolConsoleTransport) -> NonEmptyStr { match transport { ObmcConsoleCli => "obmc-console" RedfishSerialInterface => "redfish-serial-interface" + IpmiSol => "ipmi-sol" } } @@ -54,11 +63,55 @@ fn sol_console_transport_authority_for(transport: SolConsoleTransport) -> Extern match transport { ObmcConsoleCli => extdeps_external_authority_anchor RedfishSerialInterface => redfish_serial_interface_authority + IpmiSol => ipmi_sol_authority } } data sol_console_capture_bash_emit_dissolution_trigger: String = "🟑 dissolve-on: sol_console_capture β€” obmc-console/Redfish SerialInterface log stream composed as shell transport (medium-as-string); DISSOLVES WHEN host_effect_apply binds SolConsoleCapture without concat-built runner glue; retire medium_structure_exception_roster entry for dag/extdeps/bmc/serial_console.dag" +type SolConsoleSendIntent { + bmc_host: NonEmptyStr + transport: SolConsoleTransport + username: NonEmptyStr + input: String + settle_seconds: Int + read_timeout_seconds: Int +} + +data sol_console_send_bash_emit_dissolution_trigger: String = "🟑 dissolve-on: sol_console_send β€” ipmitool/obmc-console interactive send composed as shell transport (medium-as-string, IPMI_PASSWORD via -E env); DISSOLVES WHEN host_effect_apply binds SolConsoleSend without concat-built runner glue; retire medium_structure_exception_roster entry for dag/extdeps/bmc/serial_console.dag" + +fn sol_console_send_script(intent: SolConsoleSendIntent) -> String { + let h = intent.bmc_host as String + let u = intent.username as String + let settle = to_string(intent.settle_seconds) + let read = to_string(intent.read_timeout_seconds) + let ipmi = concat(concat("ipmitool -I lanplus -H \"", h), concat("\" -U \"", concat(u, "\" -E"))) + let feed = concat( + concat(concat("{ sleep ", settle), concat("; printf '%b' '", intent.input)), + concat(concat("'; sleep ", read), "; printf '~.\\n'; }"), + ) + match intent.transport { + IpmiSol => + concat( + concat("#!/usr/bin/env bash\nset -uo pipefail\n", concat(ipmi, " sol deactivate >/dev/null 2>&1 || true\n")), + concat( + concat(feed, concat(" | ", concat(ipmi, " sol activate\n"))), + concat("echo \"SolConsoleSendReceipt: transport=ipmi-sol host=", concat(h, " sent\"\n")), + ), + ) + ObmcConsoleCli => + concat( + concat("#!/usr/bin/env bash\nset -uo pipefail\n", feed), + concat( + concat(" | obmc-console-client ", h), + concat("\necho \"SolConsoleSendReceipt: transport=obmc-console host=", concat(h, " sent\"\n")), + ), + ) + RedfishSerialInterface => + "#!/usr/bin/env bash\necho \"SolConsoleSendRefusal: RedfishSerialInterface is a read-only serial log surface, not an interactive send path\" >&2\nexit 1\n" + } +} + fn sol_console_capture_script(intent: SolConsoleCaptureIntent) -> String { let transport_wire = sol_console_transport_wire(transport: intent.transport) concat( diff --git a/dag/extdeps/boards/asrock_rack.dag b/dag/extdeps/boards/asrock_rack.dag index fbbd81a3a53..de9af267f94 100644 --- a/dag/extdeps/boards/asrock_rack.dag +++ b/dag/extdeps/boards/asrock_rack.dag @@ -16,6 +16,7 @@ import extdeps.bmc.capability { CapabilityAccountManagement, CapabilityFirmwareUpdate, CapabilityNbdProxyVirtualMedia, + CapabilitySerialConsole, } import extdeps.bmc.webui.nbd_proxy { BmcwebNbdProxyEndpointVariant, @@ -75,8 +76,11 @@ data altrad8ud_openbmc_2_07_00_capabilities_list: List = [ CapabilityAccountManagement, CapabilityFirmwareUpdate, CapabilityNbdProxyVirtualMedia, + CapabilitySerialConsole, ] +data altrad8ud_openbmc_2_07_00_serial_console_probe: NonEmptyStr = "IPMI SOL Enabled=true, Privilege=ADMINISTRATOR, Payload Port=623 β€” confirmed live 2026-07-21 via ipmitool -I lanplus sol info against srv4 (192.168.1.195, OpenBMC 2.07.00)" + data asrock_altrad8ud_openbmc_2_07_00_capabilities: BmcFirmwareCapabilityRow = BmcFirmwareCapabilityRow { firmware: BmcFirmwareVersion { family: OpenBmc, diff --git a/dag/gunbc/uefi_shell_over_sol.dag b/dag/gunbc/uefi_shell_over_sol.dag new file mode 100644 index 00000000000..e5abbcd19fd --- /dev/null +++ b/dag/gunbc/uefi_shell_over_sol.dag @@ -0,0 +1,55 @@ +module gunbc.uefi_shell_over_sol + +import std.types { NonEmptyStr, String, Int, List } +import extdeps.bmc.serial_console { + SolConsoleSendIntent, + IpmiSol, + sol_console_send_script, +} +import extdeps.firmware.uefi_shell { UefiShellCommand, uefi_shell_command_line } +import gunbc.uefi_boot_config { + DesiredBootSource, + uefi_boot_config_commands, + srv4_uefi_install_boot, +} + +fn uefi_shell_serial_input(commands: List) -> String { + fold( + commands, + init: "", + f: (acc, cmd) => concat(acc, concat(uefi_shell_command_line(cmd: cmd), "\\r")) + ) +} + +fn uefi_boot_config_sol_send_intent( + bmc_host: NonEmptyStr, + username: NonEmptyStr, + desired: DesiredBootSource, +) -> SolConsoleSendIntent { + SolConsoleSendIntent { + bmc_host: bmc_host, + transport: IpmiSol, + username: username, + input: uefi_shell_serial_input(commands: uefi_boot_config_commands(desired: desired)), + settle_seconds: 3, + read_timeout_seconds: 5, + } +} + +fn uefi_boot_config_sol_send_script( + bmc_host: NonEmptyStr, + username: NonEmptyStr, + desired: DesiredBootSource, +) -> String { + sol_console_send_script(intent: uefi_boot_config_sol_send_intent( + bmc_host: bmc_host, + username: username, + desired: desired, + )) +} + +data srv4_uefi_boot_config_sol_send_intent: SolConsoleSendIntent = uefi_boot_config_sol_send_intent( + bmc_host: "192.168.1.195", + username: "root", + desired: srv4_uefi_install_boot, +) diff --git a/dag/test/claim/uefi_shell_over_sol_witness_test.dag b/dag/test/claim/uefi_shell_over_sol_witness_test.dag new file mode 100644 index 00000000000..5421b5e3344 --- /dev/null +++ b/dag/test/claim/uefi_shell_over_sol_witness_test.dag @@ -0,0 +1,22 @@ +module test.claim.uefi_shell_over_sol + +import std.logic { Bool } +import gunbc.uefi_shell_over_sol { + srv4_uefi_boot_config_sol_send_intent, + uefi_boot_config_sol_send_script, +} +import gunbc.uefi_boot_config { srv4_uefi_install_boot } + +test fn srv4_sol_send_carries_bcfg_over_serial() -> Bool { + let intent = srv4_uefi_boot_config_sol_send_intent + string_contains(s: intent.input, pattern: "bcfg boot add 0 fs0:\\EFI\\BOOT\\BOOTAA64.EFI") + && string_contains(s: intent.input, pattern: "\\r") + && string_contains(s: intent.input, pattern: "reset -w") +} + +test fn srv4_sol_send_script_uses_ipmitool_sol() -> Bool { + let s = uefi_boot_config_sol_send_script(bmc_host: "192.168.1.195", username: "root", desired: srv4_uefi_install_boot) + string_contains(s: s, pattern: "ipmitool -I lanplus -H \"192.168.1.195\" -U \"root\" -E sol activate") + && string_contains(s: s, pattern: "sol deactivate") + && string_contains(s: s, pattern: "printf '%b'") +} From 83fec629acf847ad3c1e3984f3cfe4f05c92a9e6 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 22:46:01 +0000 Subject: [PATCH 14/58] fix(sol): IpmiSol exhaustiveness in srv3_sol_console_capture witness Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/test/claim/srv3_sol_console_capture_witness_test.dag | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dag/test/claim/srv3_sol_console_capture_witness_test.dag b/dag/test/claim/srv3_sol_console_capture_witness_test.dag index a940049c028..31e2b6471b0 100644 --- a/dag/test/claim/srv3_sol_console_capture_witness_test.dag +++ b/dag/test/claim/srv3_sol_console_capture_witness_test.dag @@ -8,6 +8,7 @@ import gunbc.srv3_sol_console_capture { import extdeps.bmc.serial_console { ObmcConsoleCli, RedfishSerialInterface, + IpmiSol, extdeps_external_authority_anchor, redfish_serial_interface_authority, sol_console_transport_authority_for, @@ -17,6 +18,7 @@ test fn srv3_sol_console_capture_intent_uses_obmc_cli() -> Bool { match srv3_sol_console_capture_intent.transport { ObmcConsoleCli => true RedfishSerialInterface => false + IpmiSol => false } && srv3_sol_console_capture_log_path_is_declared() && srv3_sol_console_capture_carries_dissolution_trigger() From 3622ceade710fe47c59d35dd286acf936938e3bd Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 22:51:31 +0000 Subject: [PATCH 15/58] feat(uefi): model the UEFI Shell command surface + observable environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Back up and ground the real shell interaction (not ad-hoc poking): - Command surface expanded: connect -r (ConnectRecursive), devices (ListDevices), ifconfig (list / set dhcp / set static) alongside bcfg/map/reset β€” the commands actually used driving srv4 over SOL, cited to the UEFI Shell 2.2 spec. - Observable environment types: UefiNetworkInterface (+ UefiMediaState), UefiBootOption, UefiDeviceMapping, UefiShellEnvironment β€” so the interaction is observe->decide->act. - gunbc.srv4_uefi_observed: srv4's ACTUAL environment captured live over SOL 2026-07-21 (map -r, bcfg boot dump -v, ifconfig -l): NVMe with Windows Boot Manager + EFI Shell, eth0/eth2 media present (eth0 link-local 169.254.0.18), eth1/eth3 disconnected. Finding that motivated this: UEFI network stack is ALREADY up in srv4's shell (eth0 has media) β€” PXE-enable via GUI is unnecessary; ifconfig -s eth0 dhcp reaches the network directly. Windows-on-disk confirmed (operator OK'd wipe). Verified by execution: new commands render, srv4 observed env asserts eth0-has-link + Windows-present. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/firmware/uefi_shell.dag | 58 +++++++++++++++++++ dag/gunbc/srv4_uefi_observed.dag | 43 ++++++++++++++ .../uefi_shell_environment_witness_test.dag | 29 ++++++++++ 3 files changed, 130 insertions(+) create mode 100644 dag/gunbc/srv4_uefi_observed.dag create mode 100644 dag/test/claim/uefi_shell_environment_witness_test.dag diff --git a/dag/extdeps/firmware/uefi_shell.dag b/dag/extdeps/firmware/uefi_shell.dag index bbe2092f89d..91f716da2f9 100644 --- a/dag/extdeps/firmware/uefi_shell.dag +++ b/dag/extdeps/firmware/uefi_shell.dag @@ -35,12 +35,20 @@ fn bcfg_target_word(target: BcfgTarget) -> String { } } +type UefiIfconfigAction + = IfconfigList + | IfconfigSetDhcp { interface: NonEmptyStr } + | IfconfigSetStatic { interface: NonEmptyStr, address: NonEmptyStr, netmask: NonEmptyStr, gateway: NonEmptyStr } + type UefiShellCommand = BcfgDump { target: BcfgTarget } | BcfgAdd { target: BcfgTarget, index: Int, file: NonEmptyStr, description: NonEmptyStr } | BcfgMove { target: BcfgTarget, from_index: Int, to_index: Int } | BcfgRemove { target: BcfgTarget, index: Int } | MapDevices { refresh: Bool } + | ConnectRecursive + | ListDevices + | Ifconfig { action: UefiIfconfigAction } | Reset { mode: UefiResetMode } fn uefi_shell_command_line(cmd: UefiShellCommand) -> String { @@ -64,11 +72,61 @@ fn uefi_shell_command_line(cmd: UefiShellCommand) -> String { concat("bcfg ", concat(bcfg_target_word(target: t), concat(" rm ", to_string(i)))) MapDevices { refresh: r } => if r { "map -r" } else { "map" } + ConnectRecursive => "connect -r" + ListDevices => "devices" + Ifconfig { action: a } => + match a { + IfconfigList => "ifconfig -l" + IfconfigSetDhcp { interface: i } => + concat("ifconfig -s ", concat(i as String, " dhcp")) + IfconfigSetStatic { interface: i, address: addr, netmask: nm, gateway: gw } => + concat( + concat("ifconfig -s ", concat(i as String, " static ")), + concat(concat(addr as String, " "), concat(concat(nm as String, " "), gw as String)), + ) + } Reset { mode: m } => concat("reset ", uefi_reset_flag(mode: m)) } } +type UefiMediaState + = MediaPresent + | MediaDisconnected + +type UefiNetworkInterface { + name: NonEmptyStr + media: UefiMediaState + ipv4: NonEmptyStr +} + +fn uefi_interface_has_link(iface: UefiNetworkInterface) -> Bool { + match iface.media { + MediaPresent => true + MediaDisconnected => false + } +} + +type UefiBootOption { + index: Int + variable: NonEmptyStr + description: String + device_path: String + optional: Bool +} + +type UefiDeviceMapping { + fs_alias: String + block_aliases: String + device_path: NonEmptyStr +} + +type UefiShellEnvironment { + mappings: List + boot_options: List + interfaces: List +} + fn uefi_shell_script(commands: List) -> String { fold( commands, diff --git a/dag/gunbc/srv4_uefi_observed.dag b/dag/gunbc/srv4_uefi_observed.dag new file mode 100644 index 00000000000..7422575ab66 --- /dev/null +++ b/dag/gunbc/srv4_uefi_observed.dag @@ -0,0 +1,43 @@ +module gunbc.srv4_uefi_observed + +import std.types { List } +import extdeps.firmware.uefi_shell { + UefiShellEnvironment, + UefiNetworkInterface, + UefiBootOption, + UefiDeviceMapping, + MediaPresent, + MediaDisconnected, +} + +data srv4_uefi_observed_environment: UefiShellEnvironment = UefiShellEnvironment { + mappings: [ + UefiDeviceMapping { + fs_alias: "FS0", + block_aliases: "HD0b;BLK1", + device_path: "PcieRoot(0x40000)/Pci(0x1,0x0)/Pci(0x0,0x0)/NVMe(0x1,04-4A-B4-81-58-38-25-00)/HD(1,GPT,3C65AC38-871A-4633-BE76-2F1FB65522D7,0x800,0x32000)" as NonEmptyStr, + }, + ], + boot_options: [ + UefiBootOption { + index: 0, + variable: "Boot0002", + description: "Windows Boot Manager", + device_path: "HD(1,GPT,3C65AC38-871A-4633-BE76-2F1FB65522D7,0x800,0x32000)/\\EFI\\MICROSOFT\\BOOT\\BOOTMGFW.EFI", + optional: true, + }, + UefiBootOption { + index: 1, + variable: "Boot0001", + description: "UEFI: Built-in EFI Shell", + device_path: "VenMedia(5023B95C-DB26-429B-A648-BD47664C8012)", + optional: true, + }, + ], + interfaces: [ + UefiNetworkInterface { name: "eth0", media: MediaPresent, ipv4: "169.254.0.18" }, + UefiNetworkInterface { name: "eth1", media: MediaDisconnected, ipv4: "0.0.0.0" }, + UefiNetworkInterface { name: "eth2", media: MediaPresent, ipv4: "0.0.0.0" }, + UefiNetworkInterface { name: "eth3", media: MediaDisconnected, ipv4: "0.0.0.0" }, + ], +} diff --git a/dag/test/claim/uefi_shell_environment_witness_test.dag b/dag/test/claim/uefi_shell_environment_witness_test.dag new file mode 100644 index 00000000000..d67dacb7cf5 --- /dev/null +++ b/dag/test/claim/uefi_shell_environment_witness_test.dag @@ -0,0 +1,29 @@ +module test.claim.uefi_shell_environment + +import std.logic { Bool } +import std.types { List } +import extdeps.firmware.uefi_shell { + UefiShellCommand, + UefiNetworkInterface, + uefi_interface_has_link, + uefi_shell_command_line, + ConnectRecursive, + Ifconfig, + IfconfigSetDhcp, + UefiShellEnvironment, +} +import gunbc.srv4_uefi_observed { srv4_uefi_observed_environment } + +test fn new_shell_commands_render() -> Bool { + uefi_shell_command_line(cmd: ConnectRecursive) == "connect -r" + && uefi_shell_command_line(cmd: Ifconfig { action: IfconfigSetDhcp { interface: "eth0" } }) == "ifconfig -s eth0 dhcp" +} + +test fn srv4_observed_env_has_link_and_windows() -> Bool { + let env = srv4_uefi_observed_environment + let eth0_up = fold(env.interfaces, init: false, f: (acc, i) => + acc || (i.name == "eth0" && uefi_interface_has_link(iface: i))) + let has_windows = fold(env.boot_options, init: false, f: (acc, b) => + acc || b.description == "Windows Boot Manager") + eth0_up && has_windows +} From 148f084efd0616fbcd36cbb676197632d9815750 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 22:56:40 +0000 Subject: [PATCH 16/58] =?UTF-8?q?feat(uefi):=20boot-install=20decision/dia?= =?UTF-8?q?gnostic=20model=20(observe=E2=86=92diagnose=E2=86=92decide?= =?UTF-8?q?=E2=86=92act)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generalize the "we're at a UEFI shell, now what?" case into a goal-directed decision procedure over the observed environment: - DesiredOutcome = InstalledFleetNode (the goal: wipe + unattended Ubuntu). - diagnose_boot_install(env) -> BootInstallSituation: pure read of the observed UefiShellEnvironment β†’ InstallMediaReady | NetworkReady | NetworkUpNeedsDhcp | NoBootSourceAvailable. Fail-closed: no media + no link says so, never pretends. - decide_boot_install(situation, goal) -> BootInstallAction: goal-directed; refusal is a first-class outcome (RefuseNoSource), not a silent no-op. - boot_install_commands(action) -> List?: Absent for a refusal β€” a caller cannot extract a "do nothing" sequence and mistake it for progress. Grounded on srv4's real observed env: diagnoses NetworkUpNeedsDhcp{eth0} (media up, link-local) β†’ DhcpThenNetworkBoot β†’ ifconfig -s eth0 dhcp. Discriminating fail-closed control witness: no-media/no-link env β†’ RefuseNoSource β†’ Absent commands. Verified by execution. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/gunbc/uefi_boot_install_decision.dag | 139 ++++++++++++++++++ ...efi_boot_install_decision_witness_test.dag | 53 +++++++ 2 files changed, 192 insertions(+) create mode 100644 dag/gunbc/uefi_boot_install_decision.dag create mode 100644 dag/test/claim/uefi_boot_install_decision_witness_test.dag diff --git a/dag/gunbc/uefi_boot_install_decision.dag b/dag/gunbc/uefi_boot_install_decision.dag new file mode 100644 index 00000000000..f06f132b011 --- /dev/null +++ b/dag/gunbc/uefi_boot_install_decision.dag @@ -0,0 +1,139 @@ +module gunbc.uefi_boot_install_decision + +import std.types { NonEmptyStr, String, Bool, List } +import extdeps.firmware.uefi_shell { + UefiShellEnvironment, + UefiNetworkInterface, + UefiDeviceMapping, + uefi_interface_has_link, + UefiShellCommand, + MapDevices, + BcfgDump, + BcfgAdd, + BcfgBoot, + Ifconfig, + IfconfigSetDhcp, + Reset, + ResetWarm, +} + +type DesiredOutcome + = | InstalledFleetNode + +type BootInstallSituation + = InstallMediaReady { boot_file: NonEmptyStr } + | NetworkReady { interface: NonEmptyStr } + | NetworkUpNeedsDhcp { interface: NonEmptyStr } + | NoBootSourceAvailable { reason: NonEmptyStr } + +fn ipv4_unconfigured(ipv4: NonEmptyStr) -> Bool { + ipv4 == "0.0.0.0" + || string_contains(s: ipv4 as String, pattern: "169.254.") +} + +fn mapping_is_removable_install_media(m: UefiDeviceMapping) -> Bool { + string_contains(s: m.device_path as String, pattern: "CDROM") + || string_contains(s: m.device_path as String, pattern: "USB") + || string_contains(s: m.device_path as String, pattern: "CDDVD") +} + +type MediaScan { + found: Bool + boot_file: NonEmptyStr +} + +fn scan_install_media(env: UefiShellEnvironment) -> MediaScan { + fold( + env.mappings, + init: MediaScan { found: false, boot_file: "none" }, + f: (acc, m) => + if acc.found { acc } + else if mapping_is_removable_install_media(m: m) { + MediaScan { found: true, boot_file: concat(m.fs_alias, ":\\EFI\\BOOT\\BOOTAA64.EFI") as NonEmptyStr } + } else { acc } + ) +} + +type IfaceScan { + found: Bool + name: NonEmptyStr + routable: Bool +} + +fn scan_up_interface(env: UefiShellEnvironment) -> IfaceScan { + fold( + env.interfaces, + init: IfaceScan { found: false, name: "none", routable: false }, + f: (acc, i) => + if acc.found { acc } + else if uefi_interface_has_link(iface: i) { + IfaceScan { found: true, name: i.name, routable: !ipv4_unconfigured(ipv4: i.ipv4) } + } else { acc } + ) +} + +fn diagnose_boot_install(env: UefiShellEnvironment) -> BootInstallSituation { + let media = scan_install_media(env: env) + if media.found { + InstallMediaReady { boot_file: media.boot_file } + } else { + let iface = scan_up_interface(env: env) + if iface.found { + if iface.routable { + NetworkReady { interface: iface.name } + } else { + NetworkUpNeedsDhcp { interface: iface.name } + } + } else { + NoBootSourceAvailable { + reason: "no removable install media mapped and no network interface with link β€” cannot reach an installer", + } + } + } +} + +type BootInstallAction + = BootFromMappedMedia { boot_file: NonEmptyStr } + | DhcpThenNetworkBoot { interface: NonEmptyStr } + | RefuseNoSource { reason: NonEmptyStr } + +fn decide_boot_install(situation: BootInstallSituation, goal: DesiredOutcome) -> BootInstallAction { + match goal { + InstalledFleetNode => + match situation { + InstallMediaReady { boot_file: f } => BootFromMappedMedia { boot_file: f } + NetworkReady { interface: i } => DhcpThenNetworkBoot { interface: i } + NetworkUpNeedsDhcp { interface: i } => DhcpThenNetworkBoot { interface: i } + NoBootSourceAvailable { reason: r } => RefuseNoSource { reason: r } + } + } +} + +fn boot_install_commands(action: BootInstallAction) -> List? { + match action { + BootFromMappedMedia { boot_file: f } => + Present { + value: [ + MapDevices { refresh: true }, + BcfgAdd { target: BcfgBoot, index: 0, file: f, description: "gunbc-install-media" as NonEmptyStr }, + Reset { mode: ResetWarm }, + ] + } + DhcpThenNetworkBoot { interface: i } => + Present { + value: [ + Ifconfig { action: IfconfigSetDhcp { interface: i } }, + BcfgDump { target: BcfgBoot }, + ] + } + RefuseNoSource { reason: _ } => Absent + } +} + +fn boot_install_is_refused(action: BootInstallAction) -> Bool { + match action { + RefuseNoSource { reason: _ } => true + BootFromMappedMedia { boot_file: _ } => false + DhcpThenNetworkBoot { interface: _ } => false + } +} diff --git a/dag/test/claim/uefi_boot_install_decision_witness_test.dag b/dag/test/claim/uefi_boot_install_decision_witness_test.dag new file mode 100644 index 00000000000..bd3a527f846 --- /dev/null +++ b/dag/test/claim/uefi_boot_install_decision_witness_test.dag @@ -0,0 +1,53 @@ +module test.claim.uefi_boot_install_decision + +import std.logic { Bool } +import std.types { List } +import extdeps.firmware.uefi_shell { + UefiShellEnvironment, + UefiNetworkInterface, + MediaDisconnected, + uefi_shell_script, +} +import gunbc.uefi_boot_install_decision { + diagnose_boot_install, + decide_boot_install, + boot_install_commands, + boot_install_is_refused, + InstalledFleetNode, + NetworkUpNeedsDhcp, +} +import gunbc.srv4_uefi_observed { srv4_uefi_observed_environment } + +test fn srv4_diagnoses_network_needs_dhcp_and_acts() -> Bool { + let sit = diagnose_boot_install(env: srv4_uefi_observed_environment) + let is_needs_dhcp = match sit { + NetworkUpNeedsDhcp { interface: i } => i == "eth0" + InstallMediaReady { boot_file: _ } => false + NetworkReady { interface: _ } => false + NoBootSourceAvailable { reason: _ } => false + } + let action = decide_boot_install(situation: sit, goal: InstalledFleetNode) + let cmds_have_dhcp = match boot_install_commands(action: action) { + Present { value: cs } => string_contains(s: uefi_shell_script(commands: cs), pattern: "ifconfig -s eth0 dhcp") + Absent => false + } + is_needs_dhcp && cmds_have_dhcp && !boot_install_is_refused(action: action) +} + +data no_source_env: UefiShellEnvironment = UefiShellEnvironment { + mappings: [], + boot_options: [], + interfaces: [ + UefiNetworkInterface { name: "eth0", media: MediaDisconnected, ipv4: "0.0.0.0" }, + ], +} + +test fn no_source_fails_closed() -> Bool { + let sit = diagnose_boot_install(env: no_source_env) + let action = decide_boot_install(situation: sit, goal: InstalledFleetNode) + let no_commands = match boot_install_commands(action: action) { + Present { value: _ } => false + Absent => true + } + boot_install_is_refused(action: action) && no_commands +} From 3960005cbcbd543a7feee47a0f2cc15f200ee51c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 23:13:13 +0000 Subject: [PATCH 17/58] feat(bmc): model self-contained BMC netboot serve (no runtime central server) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answering "why srv1?" β€” it isn't needed at runtime. Model the BMC as the netboot host: gunbc.bmc_netboot_serve.BmcNetbootServePlan + srv4 instance. - bmc_netboot_serve_command: busybox httpd -f -p 8080 -h /tmp/netboot (the BMC hosts the ~88MB bootstrap: kernel/initrd/grub + a staged static busybox). - bmc_netboot_grub_cfg: boots /vmlinuz + /initrd with url= at the Ubuntu MIRROR (host streams the ~1.5GB bulk directly, never on the BMC) and ds=nocloud-net;s= at the BMC's own seed dir. - bmc_netboot_nocloud_user_data: the served seed = autoinstall_user_data(srv4 payload) β€” carries fleet SSH keys + allow-pw:false, same emit as the on-ISO path. So provisioning is BMC + internet: no central serve host at runtime; srv1's only role is one-time (cacheable) extraction of the 88MB bootstrap from the ISO. Scaffold-marked (medium-as-string ssh/httpd glue) like nbd_proxy_serve. Verified by execution: grub.cfg targets mirror-bulk + BMC-seed, serve cmd is busybox httpd, served seed carries the fleet keys. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/gunbc/bmc_netboot_serve.dag | 74 +++++++++++++++++++ .../claim/bmc_netboot_serve_witness_test.dag | 28 +++++++ 2 files changed, 102 insertions(+) create mode 100644 dag/gunbc/bmc_netboot_serve.dag create mode 100644 dag/test/claim/bmc_netboot_serve_witness_test.dag diff --git a/dag/gunbc/bmc_netboot_serve.dag b/dag/gunbc/bmc_netboot_serve.dag new file mode 100644 index 00000000000..af3b12a7754 --- /dev/null +++ b/dag/gunbc/bmc_netboot_serve.dag @@ -0,0 +1,74 @@ +module gunbc.bmc_netboot_serve + +import std.types { NonEmptyStr, String, Int } +import extdeps.uri { Uri, Https, uri_wire } +import extdeps.os.ubuntu_autoinstall { UbuntuAutoinstallPayload } +import gunbc.os_install { srv4_ubuntu_autoinstall_on_iso } +import gunbc.os_install_emit { autoinstall_user_data } +import std.disposition { Disposition, Scaffold, SingleAuthority, RealizationDispatch } +import std.decl_ref { DeclarationRef, WholeDeclaration } + +data bmc_netboot_serve_scope: Disposition = Scaffold { + dissolves_to: RealizationDispatch, + bind: DeclarationRef { + module_path: "extdeps.bmc.serial_console", + decl_name: "SolConsoleSendIntent", + field: WholeDeclaration, + } +} + +data bmc_netboot_serve_bash_emit_dissolution_trigger: String = "🟑 dissolve-on: bmc_netboot_serve β€” busybox-httpd-on-BMC serve command + grub.cfg composed as medium-as-string (concat), staged over ssh; the BMC hosts the ~88MB bootstrap (kernel/initrd/grub + our nocloud seed) so a host UEFI-HTTP-boots self-contained and streams the bulk from the Ubuntu mirror (never on the BMC). DISSOLVES WHEN host_effect_apply binds a BmcNetbootServe transport handler without concat runner glue; retire medium_structure_exception_roster entry for dag/gunbc/bmc_netboot_serve.dag" + +type BmcNetbootServePlan { + bmc_host: NonEmptyStr + serve_port: Int + netboot_dir: NonEmptyStr + httpd_binary: NonEmptyStr + nocloud_subdir: NonEmptyStr + bulk_iso_url: Uri + autoinstall: UbuntuAutoinstallPayload +} + +data srv4_bmc_netboot_serve_plan: BmcNetbootServePlan = BmcNetbootServePlan { + bmc_host: "192.168.1.195", + serve_port: 8080, + netboot_dir: "/tmp/netboot", + httpd_binary: "/tmp/netboot/busybox", + nocloud_subdir: "nocloud", + bulk_iso_url: Uri { + scheme: Https, + locator: "cdimage.ubuntu.com/ubuntu/releases/24.04/release/ubuntu-24.04.3-live-server-arm64.iso", + }, + autoinstall: srv4_ubuntu_autoinstall_on_iso, +} + +fn bmc_netboot_base_url(plan: BmcNetbootServePlan) -> String { + concat(concat("http://", plan.bmc_host as String), concat(":", int_to_str(n: plan.serve_port))) +} + +fn int_to_str(n: Int) -> String { + to_string(n) +} + +fn bmc_netboot_grub_cfg(plan: BmcNetbootServePlan) -> String { + let base = bmc_netboot_base_url(plan: plan) + let seed = concat(concat(base, "/"), concat(plan.nocloud_subdir as String, "/")) + concat( + concat("set timeout=1\nmenuentry \"gunbc netboot install\" {\n linux /vmlinuz ip=dhcp url=", uri_wire(uri: plan.bulk_iso_url) as String), + concat( + concat(" autoinstall \"ds=nocloud-net;s=", seed), + concat("\" ---\n initrd /initrd\n}\n", ""), + ), + ) +} + +fn bmc_netboot_nocloud_user_data(plan: BmcNetbootServePlan) -> String { + autoinstall_user_data(payload: plan.autoinstall) +} + +fn bmc_netboot_serve_command(plan: BmcNetbootServePlan) -> String { + concat( + concat(plan.httpd_binary as String, " httpd -f -p "), + concat(int_to_str(n: plan.serve_port), concat(" -h ", plan.netboot_dir as String)), + ) +} diff --git a/dag/test/claim/bmc_netboot_serve_witness_test.dag b/dag/test/claim/bmc_netboot_serve_witness_test.dag new file mode 100644 index 00000000000..747ffe5d1a8 --- /dev/null +++ b/dag/test/claim/bmc_netboot_serve_witness_test.dag @@ -0,0 +1,28 @@ +module test.claim.bmc_netboot_serve + +import std.logic { Bool } +import gunbc.bmc_netboot_serve { + srv4_bmc_netboot_serve_plan, + bmc_netboot_grub_cfg, + bmc_netboot_serve_command, + bmc_netboot_nocloud_user_data, +} + +test fn srv4_grub_cfg_boots_mirror_bulk_with_bmc_seed() -> Bool { + let cfg = bmc_netboot_grub_cfg(plan: srv4_bmc_netboot_serve_plan) + string_contains(s: cfg, pattern: "linux /vmlinuz") + && string_contains(s: cfg, pattern: "url=https://cdimage.ubuntu.com/ubuntu/releases/24.04/release/ubuntu-24.04.3-live-server-arm64.iso") + && string_contains(s: cfg, pattern: "ds=nocloud-net;s=http://192.168.1.195:8080/nocloud/") + && string_contains(s: cfg, pattern: "initrd /initrd") +} + +test fn srv4_serve_command_is_busybox_httpd() -> Bool { + bmc_netboot_serve_command(plan: srv4_bmc_netboot_serve_plan) == "/tmp/netboot/busybox httpd -f -p 8080 -h /tmp/netboot" +} + +test fn srv4_bmc_served_seed_carries_fleet_keys() -> Bool { + let ud = bmc_netboot_nocloud_user_data(plan: srv4_bmc_netboot_serve_plan) + string_contains(s: ud, pattern: "hostname: srv4") + && string_contains(s: ud, pattern: "allow-pw: false") + && string_contains(s: ud, pattern: "fleet-automation@gunbc") +} From 789596313ceb1efc796a19019249858d944b48a5 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 23:20:25 +0000 Subject: [PATCH 18/58] refactor(bmc/netboot): model components structurally in extdeps (no concat blobs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per operator direction β€” model each piece appropriately in extdeps first, legibly, instead of hand-built concat strings: - extdeps/firmware/kernel_cmdline.dag: KernelCmdlineArg = KernelFlag | KernelKeyValue; kernel_cmdline_render via join/map (cited to kernel-parameters.rst). - extdeps/bootloader/grub.dag: GrubConfig / GrubMenuEntry (structured), grub_config_render via join β€” replaces the string-blob grub cmdline pattern (cited to the GRUB manual). - extdeps/tools/busybox.dag: busybox CliTool + service busybox.Httpd.Serve with structured argv transport (the idiomatic form, like curl.Http). - extdeps/firmware/uefi_http_boot.dag: UefiHttpBootEntry + provisioning variants (cited to UEFI 2.10 HTTP Boot). gunbc.bmc_netboot_serve recomposed to build a GrubConfig + UefiHttpBootEntry from the plan (no bespoke concat); grub.cfg, http-boot target, and BMC-served nocloud seed all derive from structured values. Bulk from the Ubuntu mirror, seed from the BMC. Verified by execution: grub.cfg renders the full structured cmdline, http-boot entry targets the BMC url, served seed carries the fleet keys. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/bootloader/grub.dag | 54 +++++++++++ dag/extdeps/firmware/kernel_cmdline.dag | 27 ++++++ dag/extdeps/firmware/uefi_http_boot.dag | 26 ++++++ dag/extdeps/package_managers/apt.dag | 1 + dag/extdeps/tools/busybox.dag | 42 +++++++++ dag/gunbc/bmc_netboot_serve.dag | 92 +++++++++++-------- .../claim/bmc_netboot_serve_witness_test.dag | 18 ++-- 7 files changed, 213 insertions(+), 47 deletions(-) create mode 100644 dag/extdeps/bootloader/grub.dag create mode 100644 dag/extdeps/firmware/kernel_cmdline.dag create mode 100644 dag/extdeps/firmware/uefi_http_boot.dag create mode 100644 dag/extdeps/tools/busybox.dag diff --git a/dag/extdeps/bootloader/grub.dag b/dag/extdeps/bootloader/grub.dag new file mode 100644 index 00000000000..13e812cbcbf --- /dev/null +++ b/dag/extdeps/bootloader/grub.dag @@ -0,0 +1,54 @@ +module extdeps.bootloader.grub + +import std.types { NonEmptyStr, String, Int, List } +import extdeps.firmware.kernel_cmdline { KernelCmdlineArg, kernel_cmdline_render } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "www.gnu.org/software/grub/manual/grub/grub.html#Simple-configuration" + } +} + +type GrubMenuEntry { + title: NonEmptyStr + linux_image: NonEmptyStr + linux_cmdline: List + initrd_image: NonEmptyStr +} + +type GrubConfig { + timeout_seconds: Int + default_entry: Int + entries: List +} + +fn grub_linux_line(entry: GrubMenuEntry) -> String { + join([" linux", entry.linux_image as String, kernel_cmdline_render(args: entry.linux_cmdline)], " ") +} + +fn grub_menuentry_render(entry: GrubMenuEntry) -> String { + join( + [ + concat(concat("menuentry \"", entry.title as String), "\" {"), + grub_linux_line(entry: entry), + join([" initrd", entry.initrd_image as String], " "), + "}", + ], + "\n" + ) +} + +fn grub_config_render(cfg: GrubConfig) -> String { + let header = join( + [ + join(["set timeout=", to_string(cfg.timeout_seconds)], ""), + join(["set default=", to_string(cfg.default_entry)], ""), + ], + "\n" + ) + let body = join(map(cfg.entries, e => grub_menuentry_render(entry: e)), "\n") + join([header, body], "\n") +} diff --git a/dag/extdeps/firmware/kernel_cmdline.dag b/dag/extdeps/firmware/kernel_cmdline.dag new file mode 100644 index 00000000000..8530b23cb19 --- /dev/null +++ b/dag/extdeps/firmware/kernel_cmdline.dag @@ -0,0 +1,27 @@ +module extdeps.firmware.kernel_cmdline + +import std.types { NonEmptyStr, String, List } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html" + } +} + +type KernelCmdlineArg + = KernelFlag { name: NonEmptyStr } + | KernelKeyValue { key: NonEmptyStr, value: String } + +fn kernel_cmdline_arg_wire(arg: KernelCmdlineArg) -> String { + match arg { + KernelFlag { name: n } => n as String + KernelKeyValue { key: k, value: v } => concat(k as String, concat("=", v)) + } +} + +fn kernel_cmdline_render(args: List) -> String { + join(map(args, a => kernel_cmdline_arg_wire(arg: a)), " ") +} diff --git a/dag/extdeps/firmware/uefi_http_boot.dag b/dag/extdeps/firmware/uefi_http_boot.dag new file mode 100644 index 00000000000..5357223a411 --- /dev/null +++ b/dag/extdeps/firmware/uefi_http_boot.dag @@ -0,0 +1,26 @@ +module extdeps.firmware.uefi_http_boot + +import std.types { NonEmptyStr, String } +import extdeps.uri { Uri, Https, uri_wire } +import extdeps.external_authority { ExternalAuthority } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "uefi.org/specs/UEFI/2.10/24_Network_Protocols_HTTP_Boot.html" + } +} + +type UefiHttpBootEntry { + description: NonEmptyStr + boot_file_uri: Uri + interface_name: NonEmptyStr +} + +fn uefi_http_boot_uri_wire(entry: UefiHttpBootEntry) -> String { + uri_wire(uri: entry.boot_file_uri) +} + +type UefiHttpBootProvisioning + = FirmwareHttpBootDhcp + | BcfgUriDevicePath { device_path: NonEmptyStr } diff --git a/dag/extdeps/package_managers/apt.dag b/dag/extdeps/package_managers/apt.dag index 11c5c15708e..8c57d965491 100644 --- a/dag/extdeps/package_managers/apt.dag +++ b/dag/extdeps/package_managers/apt.dag @@ -30,6 +30,7 @@ data package_socat: NonEmptyStr = "socat" data package_websocat: NonEmptyStr = "websocat" data package_xorriso: NonEmptyStr = "xorriso" data package_google_cloud_cli: NonEmptyStr = "google-cloud-cli" +data package_busybox: NonEmptyStr = "busybox-static" data apt_man_url: NonEmptyStr = "https://manpages.debian.org/testing/apt/apt.8.en.html" diff --git a/dag/extdeps/tools/busybox.dag b/dag/extdeps/tools/busybox.dag new file mode 100644 index 00000000000..65de9cf08bb --- /dev/null +++ b/dag/extdeps/tools/busybox.dag @@ -0,0 +1,42 @@ +module extdeps.tools.busybox + +import std.types { NonEmptyStr, String, Int, Bool } +import extdeps.version { VersionConstraint } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } +import extdeps.tools { CliTool, SourceApt } +import extdeps.apt { package_busybox } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "busybox.net/downloads/BusyBox.html" + } +} + +data busybox_pinned_version: VersionConstraint = ">= 1.36" + +data busybox_cli_tool: CliTool = CliTool { + name: "busybox", + min_version: Present { value: busybox_pinned_version }, + installable_via: [SourceApt { package: package_busybox }], +} + +service busybox.Httpd { + operation Serve { + input { document_root: NonEmptyStr, port: Int } + output { success: Bool from "exit_success" } + transport shell { + argv: [ + "busybox", + "httpd", + "-p", "{port}", + "-h", "{document_root}", + ] + } + exit { + 0 => Unit + nonzero => String "busybox httpd serve failed" + } + } +} diff --git a/dag/gunbc/bmc_netboot_serve.dag b/dag/gunbc/bmc_netboot_serve.dag index af3b12a7754..c073c8cc208 100644 --- a/dag/gunbc/bmc_netboot_serve.dag +++ b/dag/gunbc/bmc_netboot_serve.dag @@ -1,30 +1,20 @@ module gunbc.bmc_netboot_serve import std.types { NonEmptyStr, String, Int } -import extdeps.uri { Uri, Https, uri_wire } +import extdeps.uri { Uri, Http, Https, uri_wire } import extdeps.os.ubuntu_autoinstall { UbuntuAutoinstallPayload } +import extdeps.bootloader.grub { GrubConfig, GrubMenuEntry, grub_config_render } +import extdeps.firmware.kernel_cmdline { KernelCmdlineArg, KernelFlag, KernelKeyValue } +import extdeps.firmware.uefi_http_boot { UefiHttpBootEntry } +import extdeps.tools.busybox { busybox_cli_tool } import gunbc.os_install { srv4_ubuntu_autoinstall_on_iso } import gunbc.os_install_emit { autoinstall_user_data } -import std.disposition { Disposition, Scaffold, SingleAuthority, RealizationDispatch } -import std.decl_ref { DeclarationRef, WholeDeclaration } - -data bmc_netboot_serve_scope: Disposition = Scaffold { - dissolves_to: RealizationDispatch, - bind: DeclarationRef { - module_path: "extdeps.bmc.serial_console", - decl_name: "SolConsoleSendIntent", - field: WholeDeclaration, - } -} - -data bmc_netboot_serve_bash_emit_dissolution_trigger: String = "🟑 dissolve-on: bmc_netboot_serve β€” busybox-httpd-on-BMC serve command + grub.cfg composed as medium-as-string (concat), staged over ssh; the BMC hosts the ~88MB bootstrap (kernel/initrd/grub + our nocloud seed) so a host UEFI-HTTP-boots self-contained and streams the bulk from the Ubuntu mirror (never on the BMC). DISSOLVES WHEN host_effect_apply binds a BmcNetbootServe transport handler without concat runner glue; retire medium_structure_exception_roster entry for dag/gunbc/bmc_netboot_serve.dag" type BmcNetbootServePlan { bmc_host: NonEmptyStr serve_port: Int netboot_dir: NonEmptyStr - httpd_binary: NonEmptyStr - nocloud_subdir: NonEmptyStr + boot_interface: NonEmptyStr bulk_iso_url: Uri autoinstall: UbuntuAutoinstallPayload } @@ -33,8 +23,7 @@ data srv4_bmc_netboot_serve_plan: BmcNetbootServePlan = BmcNetbootServePlan { bmc_host: "192.168.1.195", serve_port: 8080, netboot_dir: "/tmp/netboot", - httpd_binary: "/tmp/netboot/busybox", - nocloud_subdir: "nocloud", + boot_interface: "eth0", bulk_iso_url: Uri { scheme: Https, locator: "cdimage.ubuntu.com/ubuntu/releases/24.04/release/ubuntu-24.04.3-live-server-arm64.iso", @@ -42,33 +31,58 @@ data srv4_bmc_netboot_serve_plan: BmcNetbootServePlan = BmcNetbootServePlan { autoinstall: srv4_ubuntu_autoinstall_on_iso, } -fn bmc_netboot_base_url(plan: BmcNetbootServePlan) -> String { - concat(concat("http://", plan.bmc_host as String), concat(":", int_to_str(n: plan.serve_port))) +fn bmc_netboot_host_port(plan: BmcNetbootServePlan) -> NonEmptyStr { + join([plan.bmc_host as String, ":", to_string(plan.serve_port)], "") as NonEmptyStr } -fn int_to_str(n: Int) -> String { - to_string(n) +fn bmc_netboot_uri(plan: BmcNetbootServePlan, path: NonEmptyStr) -> Uri { + Uri { + scheme: Http, + locator: join([bmc_netboot_host_port(plan: plan) as String, path as String], "") as NonEmptyStr, + } } -fn bmc_netboot_grub_cfg(plan: BmcNetbootServePlan) -> String { - let base = bmc_netboot_base_url(plan: plan) - let seed = concat(concat(base, "/"), concat(plan.nocloud_subdir as String, "/")) - concat( - concat("set timeout=1\nmenuentry \"gunbc netboot install\" {\n linux /vmlinuz ip=dhcp url=", uri_wire(uri: plan.bulk_iso_url) as String), - concat( - concat(" autoinstall \"ds=nocloud-net;s=", seed), - concat("\" ---\n initrd /initrd\n}\n", ""), - ), - ) +fn bmc_netboot_seed_datasource(plan: BmcNetbootServePlan) -> String { + join(["nocloud-net;s=", uri_wire(uri: bmc_netboot_uri(plan: plan, path: "/nocloud/"))], "") } -fn bmc_netboot_nocloud_user_data(plan: BmcNetbootServePlan) -> String { - autoinstall_user_data(payload: plan.autoinstall) +fn bmc_netboot_kernel_cmdline(plan: BmcNetbootServePlan) -> List { + [ + KernelKeyValue { key: "ip", value: "dhcp" }, + KernelKeyValue { key: "url", value: uri_wire(uri: plan.bulk_iso_url) }, + KernelFlag { name: "autoinstall" }, + KernelKeyValue { key: "ds", value: bmc_netboot_seed_datasource(plan: plan) }, + KernelFlag { name: "---" }, + ] } -fn bmc_netboot_serve_command(plan: BmcNetbootServePlan) -> String { - concat( - concat(plan.httpd_binary as String, " httpd -f -p "), - concat(int_to_str(n: plan.serve_port), concat(" -h ", plan.netboot_dir as String)), - ) +fn bmc_netboot_grub_config(plan: BmcNetbootServePlan) -> GrubConfig { + GrubConfig { + timeout_seconds: 1, + default_entry: 0, + entries: [ + GrubMenuEntry { + title: "gunbc netboot install", + linux_image: "/vmlinuz", + linux_cmdline: bmc_netboot_kernel_cmdline(plan: plan), + initrd_image: "/initrd", + }, + ], + } +} + +fn bmc_netboot_grub_cfg_text(plan: BmcNetbootServePlan) -> String { + grub_config_render(cfg: bmc_netboot_grub_config(plan: plan)) +} + +fn bmc_netboot_http_boot_entry(plan: BmcNetbootServePlan) -> UefiHttpBootEntry { + UefiHttpBootEntry { + description: "gunbc-bmc-netboot", + boot_file_uri: bmc_netboot_uri(plan: plan, path: "/grubaa64.efi"), + interface_name: plan.boot_interface, + } +} + +fn bmc_netboot_nocloud_user_data(plan: BmcNetbootServePlan) -> String { + autoinstall_user_data(payload: plan.autoinstall) } diff --git a/dag/test/claim/bmc_netboot_serve_witness_test.dag b/dag/test/claim/bmc_netboot_serve_witness_test.dag index 747ffe5d1a8..79e2c2340f7 100644 --- a/dag/test/claim/bmc_netboot_serve_witness_test.dag +++ b/dag/test/claim/bmc_netboot_serve_witness_test.dag @@ -1,23 +1,25 @@ module test.claim.bmc_netboot_serve import std.logic { Bool } +import extdeps.firmware.uefi_http_boot { uefi_http_boot_uri_wire } import gunbc.bmc_netboot_serve { srv4_bmc_netboot_serve_plan, - bmc_netboot_grub_cfg, - bmc_netboot_serve_command, + bmc_netboot_grub_cfg_text, + bmc_netboot_http_boot_entry, bmc_netboot_nocloud_user_data, } test fn srv4_grub_cfg_boots_mirror_bulk_with_bmc_seed() -> Bool { - let cfg = bmc_netboot_grub_cfg(plan: srv4_bmc_netboot_serve_plan) - string_contains(s: cfg, pattern: "linux /vmlinuz") - && string_contains(s: cfg, pattern: "url=https://cdimage.ubuntu.com/ubuntu/releases/24.04/release/ubuntu-24.04.3-live-server-arm64.iso") - && string_contains(s: cfg, pattern: "ds=nocloud-net;s=http://192.168.1.195:8080/nocloud/") + let cfg = bmc_netboot_grub_cfg_text(plan: srv4_bmc_netboot_serve_plan) + string_contains(s: cfg, pattern: "linux /vmlinuz ip=dhcp url=https://cdimage.ubuntu.com/ubuntu/releases/24.04/release/ubuntu-24.04.3-live-server-arm64.iso autoinstall ds=nocloud-net;s=http://192.168.1.195:8080/nocloud/ ---") && string_contains(s: cfg, pattern: "initrd /initrd") + && string_contains(s: cfg, pattern: "set timeout=1") } -test fn srv4_serve_command_is_busybox_httpd() -> Bool { - bmc_netboot_serve_command(plan: srv4_bmc_netboot_serve_plan) == "/tmp/netboot/busybox httpd -f -p 8080 -h /tmp/netboot" +test fn srv4_http_boot_entry_targets_bmc() -> Bool { + let entry = bmc_netboot_http_boot_entry(plan: srv4_bmc_netboot_serve_plan) + string_contains(s: uefi_http_boot_uri_wire(entry: entry), pattern: "http://192.168.1.195:8080/grubaa64.efi") + && entry.interface_name == "eth0" } test fn srv4_bmc_served_seed_carries_fleet_keys() -> Bool { From fd20fe1a20314f5e76abd0cf3f9b9d12849c9af4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 23:27:02 +0000 Subject: [PATCH 19/58] feat(bmc/netboot): model BMC ssh transport + structured staging manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - extdeps/bmc/ssh.dag: service bmc.Ssh (ExecScript + PutFile) over sshpass -e (password via SSHPASS env, never argv) β€” structured argv, mirrors curl.Http. - gunbc.bmc_netboot_serve: staging modeled as a structured manifest (BmcStagedFile / StagedContentSource = InlineText | LocalArtifact | FetchFromUri): busybox+kernel+initrd+grub as LocalArtifact, the rendered grub.cfg + nocloud user-data/meta-data as InlineText. Separates WHAT must be on the BMC from HOW it gets there. Verified: manifest stages the rendered grub.cfg (ds=nocloud-net) and the seed (fleet keys) inline, 7 files. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/bmc/ssh.dag | 52 ++++++++++++++++++ dag/gunbc/bmc_netboot_serve.dag | 55 ++++++++++++++++++- .../claim/bmc_netboot_serve_witness_test.dag | 24 ++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 dag/extdeps/bmc/ssh.dag diff --git a/dag/extdeps/bmc/ssh.dag b/dag/extdeps/bmc/ssh.dag new file mode 100644 index 00000000000..32a4f307ca9 --- /dev/null +++ b/dag/extdeps/bmc/ssh.dag @@ -0,0 +1,52 @@ +module extdeps.bmc.ssh + +import std.types { NonEmptyStr, String, Bool } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/openbmc/docs/blob/master/console.md" + } +} + +service bmc.Ssh { + operation ExecScript { + input { target: NonEmptyStr, script: String } + output { stdout: String from "stdout", success: Bool from "exit_success" } + transport shell { + argv: [ + "sshpass", "-e", + "ssh", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "{target}", + "sh", "-s", + ] + } + exit { + 0 => Unit + nonzero => String "bmc ssh exec failed" + } + } + + operation PutFile { + input { local_path: NonEmptyStr, remote_target: NonEmptyStr } + output { success: Bool from "exit_success" } + transport shell { + argv: [ + "sshpass", "-e", + "scp", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "{local_path}", + "{remote_target}", + ] + } + exit { + 0 => Unit + nonzero => String "bmc scp put failed" + } + } +} diff --git a/dag/gunbc/bmc_netboot_serve.dag b/dag/gunbc/bmc_netboot_serve.dag index c073c8cc208..081adc2d3d2 100644 --- a/dag/gunbc/bmc_netboot_serve.dag +++ b/dag/gunbc/bmc_netboot_serve.dag @@ -1,6 +1,6 @@ module gunbc.bmc_netboot_serve -import std.types { NonEmptyStr, String, Int } +import std.types { NonEmptyStr, String, Int, List } import extdeps.uri { Uri, Http, Https, uri_wire } import extdeps.os.ubuntu_autoinstall { UbuntuAutoinstallPayload } import extdeps.bootloader.grub { GrubConfig, GrubMenuEntry, grub_config_render } @@ -86,3 +86,56 @@ fn bmc_netboot_http_boot_entry(plan: BmcNetbootServePlan) -> UefiHttpBootEntry { fn bmc_netboot_nocloud_user_data(plan: BmcNetbootServePlan) -> String { autoinstall_user_data(payload: plan.autoinstall) } + +data bmc_netboot_bootstrap_artifact_dir: NonEmptyStr = "/var/lib/gunbc/artifacts/netboot" + +type StagedContentSource + = InlineText { text: String } + | LocalArtifact { path: NonEmptyStr } + | FetchFromUri { source: Uri } + +type BmcStagedFile { + remote_path: NonEmptyStr + content: StagedContentSource +} + +fn bmc_netboot_artifact(name: NonEmptyStr) -> NonEmptyStr { + join([bmc_netboot_bootstrap_artifact_dir as String, "/", name as String], "") as NonEmptyStr +} + +fn bmc_netboot_staged_path(plan: BmcNetbootServePlan, name: NonEmptyStr) -> NonEmptyStr { + join([plan.netboot_dir as String, "/", name as String], "") as NonEmptyStr +} + +fn bmc_netboot_staging_manifest(plan: BmcNetbootServePlan) -> List { + [ + BmcStagedFile { + remote_path: bmc_netboot_staged_path(plan: plan, name: "busybox"), + content: LocalArtifact { path: bmc_netboot_artifact(name: "busybox") }, + }, + BmcStagedFile { + remote_path: bmc_netboot_staged_path(plan: plan, name: "vmlinuz"), + content: LocalArtifact { path: bmc_netboot_artifact(name: "vmlinuz") }, + }, + BmcStagedFile { + remote_path: bmc_netboot_staged_path(plan: plan, name: "initrd"), + content: LocalArtifact { path: bmc_netboot_artifact(name: "initrd") }, + }, + BmcStagedFile { + remote_path: bmc_netboot_staged_path(plan: plan, name: "grubaa64.efi"), + content: LocalArtifact { path: bmc_netboot_artifact(name: "grubaa64.efi") }, + }, + BmcStagedFile { + remote_path: bmc_netboot_staged_path(plan: plan, name: "grub.cfg"), + content: InlineText { text: bmc_netboot_grub_cfg_text(plan: plan) }, + }, + BmcStagedFile { + remote_path: bmc_netboot_staged_path(plan: plan, name: "nocloud/user-data"), + content: InlineText { text: bmc_netboot_nocloud_user_data(plan: plan) }, + }, + BmcStagedFile { + remote_path: bmc_netboot_staged_path(plan: plan, name: "nocloud/meta-data"), + content: InlineText { text: join(["instance-id: ", plan.autoinstall.identity.hostname as String], "") }, + }, + ] +} diff --git a/dag/test/claim/bmc_netboot_serve_witness_test.dag b/dag/test/claim/bmc_netboot_serve_witness_test.dag index 79e2c2340f7..c77eec2bfc3 100644 --- a/dag/test/claim/bmc_netboot_serve_witness_test.dag +++ b/dag/test/claim/bmc_netboot_serve_witness_test.dag @@ -1,12 +1,17 @@ module test.claim.bmc_netboot_serve import std.logic { Bool } +import std.types { list_length } import extdeps.firmware.uefi_http_boot { uefi_http_boot_uri_wire } import gunbc.bmc_netboot_serve { srv4_bmc_netboot_serve_plan, bmc_netboot_grub_cfg_text, bmc_netboot_http_boot_entry, bmc_netboot_nocloud_user_data, + bmc_netboot_staging_manifest, + InlineText, + LocalArtifact, + FetchFromUri, } test fn srv4_grub_cfg_boots_mirror_bulk_with_bmc_seed() -> Bool { @@ -28,3 +33,22 @@ test fn srv4_bmc_served_seed_carries_fleet_keys() -> Bool { && string_contains(s: ud, pattern: "allow-pw: false") && string_contains(s: ud, pattern: "fleet-automation@gunbc") } + +test fn srv4_staging_manifest_stages_rendered_grub_and_seed() -> Bool { + let m = bmc_netboot_staging_manifest(plan: srv4_bmc_netboot_serve_plan) + let grub_ok = fold(m, init: false, f: (acc, file) => + acc || (string_contains(s: file.remote_path as String, pattern: "/grub.cfg") + && match file.content { + InlineText { text: t } => string_contains(s: t, pattern: "ds=nocloud-net") + LocalArtifact { path: _ } => false + FetchFromUri { source: _ } => false + })) + let seed_ok = fold(m, init: false, f: (acc, file) => + acc || (string_contains(s: file.remote_path as String, pattern: "nocloud/user-data") + && match file.content { + InlineText { text: t } => string_contains(s: t, pattern: "fleet-automation@gunbc") + LocalArtifact { path: _ } => false + FetchFromUri { source: _ } => false + })) + grub_ok && seed_ok && list_length(items: m) == 7 +} From e27fb26eccf5740fe8f04dbc2b45fa2b566f642c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 23:35:54 +0000 Subject: [PATCH 20/58] =?UTF-8?q?feat(exec):=20transport=20seam=20?= =?UTF-8?q?=E2=80=94=20one=20command,=20N=20transports=20(=C2=A73,=20no=20?= =?UTF-8?q?forked=20blobs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extdeps/exec/command.dag: ShellCommand { argv } + CommandTransport = LocalShell | SshExec { ssh_target }; command_over_transport wraps a command's argv with the transport prefix; shell_command_render joins to a string. One operation, chosen transport β€” not a per-site command blob. - busybox: service busybox.Httpd removed in favor of busybox_httpd_command -> ShellCommand (single authority for the command shape; runs local OR over BMC-ssh via the seam, no dual representation). - bmc.Ssh: ExecScript removed (superseded by SshExec transport); PutFile kept for file transfer. - bmc_netboot_serve: the serve command is busybox_httpd_command over bmc_transport (SshExec root@bmc); local and BMC renderings both derive from it. Discriminating witness serve_command_one_shape_two_transports: the same command renders "busybox httpd -p 8080 -h /tmp/netboot" locally and the sshpass-wrapped form over the BMC transport. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/bmc/ssh.dag | 21 +--------- dag/extdeps/exec/command.dag | 42 +++++++++++++++++++ dag/extdeps/tools/busybox.dag | 22 +++------- dag/gunbc/bmc_netboot_serve.dag | 29 ++++++++++++- .../claim/bmc_netboot_serve_witness_test.dag | 10 +++++ 5 files changed, 86 insertions(+), 38 deletions(-) create mode 100644 dag/extdeps/exec/command.dag diff --git a/dag/extdeps/bmc/ssh.dag b/dag/extdeps/bmc/ssh.dag index 32a4f307ca9..dd7d512828f 100644 --- a/dag/extdeps/bmc/ssh.dag +++ b/dag/extdeps/bmc/ssh.dag @@ -1,6 +1,6 @@ module extdeps.bmc.ssh -import std.types { NonEmptyStr, String, Bool } +import std.types { NonEmptyStr, Bool } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } @@ -12,25 +12,6 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { } service bmc.Ssh { - operation ExecScript { - input { target: NonEmptyStr, script: String } - output { stdout: String from "stdout", success: Bool from "exit_success" } - transport shell { - argv: [ - "sshpass", "-e", - "ssh", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "{target}", - "sh", "-s", - ] - } - exit { - 0 => Unit - nonzero => String "bmc ssh exec failed" - } - } - operation PutFile { input { local_path: NonEmptyStr, remote_target: NonEmptyStr } output { success: Bool from "exit_success" } diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag new file mode 100644 index 00000000000..4420b807de6 --- /dev/null +++ b/dag/extdeps/exec/command.dag @@ -0,0 +1,42 @@ +module extdeps.exec.command + +import std.types { NonEmptyStr, String, List } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "man7.org/linux/man-pages/man1/ssh.1.html" + } +} + +type ShellCommand { + argv: List +} + +type CommandTransport + = LocalShell + | SshExec { ssh_target: NonEmptyStr } + +fn ssh_exec_prefix(target: NonEmptyStr) -> List { + [ + "sshpass", "-e", + "ssh", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + target as String, + ] +} + +fn command_over_transport(command: ShellCommand, transport: CommandTransport) -> ShellCommand { + match transport { + LocalShell => command + SshExec { ssh_target: t } => + ShellCommand { argv: append(ssh_exec_prefix(target: t), items: command.argv) } + } +} + +fn shell_command_render(command: ShellCommand) -> String { + join(command.argv, " ") +} diff --git a/dag/extdeps/tools/busybox.dag b/dag/extdeps/tools/busybox.dag index 65de9cf08bb..2cda933f64b 100644 --- a/dag/extdeps/tools/busybox.dag +++ b/dag/extdeps/tools/busybox.dag @@ -1,11 +1,12 @@ module extdeps.tools.busybox -import std.types { NonEmptyStr, String, Int, Bool } +import std.types { NonEmptyStr, String, Int } import extdeps.version { VersionConstraint } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import extdeps.tools { CliTool, SourceApt } import extdeps.apt { package_busybox } +import extdeps.exec.command { ShellCommand } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -22,21 +23,8 @@ data busybox_cli_tool: CliTool = CliTool { installable_via: [SourceApt { package: package_busybox }], } -service busybox.Httpd { - operation Serve { - input { document_root: NonEmptyStr, port: Int } - output { success: Bool from "exit_success" } - transport shell { - argv: [ - "busybox", - "httpd", - "-p", "{port}", - "-h", "{document_root}", - ] - } - exit { - 0 => Unit - nonzero => String "busybox httpd serve failed" - } +fn busybox_httpd_command(document_root: NonEmptyStr, port: Int) -> ShellCommand { + ShellCommand { + argv: ["busybox", "httpd", "-p", to_string(port), "-h", document_root as String], } } diff --git a/dag/gunbc/bmc_netboot_serve.dag b/dag/gunbc/bmc_netboot_serve.dag index 081adc2d3d2..ed8863bc0a4 100644 --- a/dag/gunbc/bmc_netboot_serve.dag +++ b/dag/gunbc/bmc_netboot_serve.dag @@ -6,7 +6,15 @@ import extdeps.os.ubuntu_autoinstall { UbuntuAutoinstallPayload } import extdeps.bootloader.grub { GrubConfig, GrubMenuEntry, grub_config_render } import extdeps.firmware.kernel_cmdline { KernelCmdlineArg, KernelFlag, KernelKeyValue } import extdeps.firmware.uefi_http_boot { UefiHttpBootEntry } -import extdeps.tools.busybox { busybox_cli_tool } +import extdeps.tools.busybox { busybox_cli_tool, busybox_httpd_command } +import extdeps.exec.command { + ShellCommand, + CommandTransport, + LocalShell, + SshExec, + command_over_transport, + shell_command_render, +} import gunbc.os_install { srv4_ubuntu_autoinstall_on_iso } import gunbc.os_install_emit { autoinstall_user_data } @@ -87,6 +95,25 @@ fn bmc_netboot_nocloud_user_data(plan: BmcNetbootServePlan) -> String { autoinstall_user_data(payload: plan.autoinstall) } +fn bmc_transport(plan: BmcNetbootServePlan) -> CommandTransport { + SshExec { ssh_target: join(["root@", plan.bmc_host as String], "") as NonEmptyStr } +} + +fn bmc_netboot_serve_command_local(plan: BmcNetbootServePlan) -> ShellCommand { + busybox_httpd_command(document_root: plan.netboot_dir, port: plan.serve_port) +} + +fn bmc_netboot_serve_command(plan: BmcNetbootServePlan) -> ShellCommand { + command_over_transport( + command: bmc_netboot_serve_command_local(plan: plan), + transport: bmc_transport(plan: plan), + ) +} + +fn bmc_netboot_serve_command_text(plan: BmcNetbootServePlan) -> String { + shell_command_render(command: bmc_netboot_serve_command(plan: plan)) +} + data bmc_netboot_bootstrap_artifact_dir: NonEmptyStr = "/var/lib/gunbc/artifacts/netboot" type StagedContentSource diff --git a/dag/test/claim/bmc_netboot_serve_witness_test.dag b/dag/test/claim/bmc_netboot_serve_witness_test.dag index c77eec2bfc3..89929081eb6 100644 --- a/dag/test/claim/bmc_netboot_serve_witness_test.dag +++ b/dag/test/claim/bmc_netboot_serve_witness_test.dag @@ -3,12 +3,15 @@ module test.claim.bmc_netboot_serve import std.logic { Bool } import std.types { list_length } import extdeps.firmware.uefi_http_boot { uefi_http_boot_uri_wire } +import extdeps.exec.command { shell_command_render } import gunbc.bmc_netboot_serve { srv4_bmc_netboot_serve_plan, bmc_netboot_grub_cfg_text, bmc_netboot_http_boot_entry, bmc_netboot_nocloud_user_data, bmc_netboot_staging_manifest, + bmc_netboot_serve_command_local, + bmc_netboot_serve_command_text, InlineText, LocalArtifact, FetchFromUri, @@ -34,6 +37,13 @@ test fn srv4_bmc_served_seed_carries_fleet_keys() -> Bool { && string_contains(s: ud, pattern: "fleet-automation@gunbc") } +test fn serve_command_one_shape_two_transports() -> Bool { + let local = shell_command_render(command: bmc_netboot_serve_command_local(plan: srv4_bmc_netboot_serve_plan)) + let over_bmc = bmc_netboot_serve_command_text(plan: srv4_bmc_netboot_serve_plan) + local == "busybox httpd -p 8080 -h /tmp/netboot" + && over_bmc == "sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@192.168.1.195 busybox httpd -p 8080 -h /tmp/netboot" +} + test fn srv4_staging_manifest_stages_rendered_grub_and_seed() -> Bool { let m = bmc_netboot_staging_manifest(plan: srv4_bmc_netboot_serve_plan) let grub_ok = fold(m, init: false, f: (acc, file) => From f14d25c24f256c0581493e4fbe9f817932a69d21 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 23:50:52 +0000 Subject: [PATCH 21/58] feat(bmc/netboot): orchestration provision func (realize via the seam) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bmc_netboot_provision: sequences the staged manifest + serve β€” mkdir + serve go through the extdeps.exec.command seam (bmc_netboot_run_bmc), artifacts via bmc.Ssh.PutFile, rendered grub.cfg/seed via Filesystem.Write then PutFile. process_exit_first_failure collects the first failure (no fabricated success). srv4_bmc_netboot_provision is the zero-arg entry. bmc.Ssh.PutFile gains a mock_response for hermetic dry-run. Marked realization scaffold. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/bmc/ssh.dag | 3 ++ dag/gunbc/bmc_netboot_serve.dag | 73 +++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/dag/extdeps/bmc/ssh.dag b/dag/extdeps/bmc/ssh.dag index dd7d512828f..d8f92457ac6 100644 --- a/dag/extdeps/bmc/ssh.dag +++ b/dag/extdeps/bmc/ssh.dag @@ -29,5 +29,8 @@ service bmc.Ssh { 0 => Unit nonzero => String "bmc scp put failed" } + mock_response { + 0 => { success: true } "hermetic bmc.Ssh.PutFile success" + } } } diff --git a/dag/gunbc/bmc_netboot_serve.dag b/dag/gunbc/bmc_netboot_serve.dag index ed8863bc0a4..b2dc4f1aee3 100644 --- a/dag/gunbc/bmc_netboot_serve.dag +++ b/dag/gunbc/bmc_netboot_serve.dag @@ -15,6 +15,12 @@ import extdeps.exec.command { command_over_transport, shell_command_render, } +import extdeps.shell +import extdeps.bmc.ssh +import extdeps.filesystem.filesystem_io { Filesystem } +import gunbc.shell_bash_runner { shell_exec_via_bash } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.resources { Network } import gunbc.os_install { srv4_ubuntu_autoinstall_on_iso } import gunbc.os_install_emit { autoinstall_user_data } @@ -114,6 +120,73 @@ fn bmc_netboot_serve_command_text(plan: BmcNetbootServePlan) -> String { shell_command_render(command: bmc_netboot_serve_command(plan: plan)) } +fn bmc_ssh_target(plan: BmcNetbootServePlan) -> NonEmptyStr { + join(["root@", plan.bmc_host as String], "") as NonEmptyStr +} + +fn bmc_remote_target(plan: BmcNetbootServePlan, name: NonEmptyStr) -> NonEmptyStr { + join([bmc_ssh_target(plan: plan) as String, ":", bmc_netboot_staged_path(plan: plan, name: name) as String], "") as NonEmptyStr +} + +data bmc_netboot_provision_dissolution_trigger: String = "🟑 dissolve-on: bmc_netboot_provision β€” realization driver sequences the staging manifest + serve over shell.Exec.Run / bmc.Ssh.PutFile / Filesystem.Write; the BMC commands go through the extdeps.exec.command seam (no bespoke blob) but the per-file sequencing is hand-unrolled for the known plan. DISSOLVES WHEN a host_effect_apply binds the staging manifest fold generically; retire medium_structure_exception_roster entry for dag/gunbc/bmc_netboot_serve.dag" + +func bmc_netboot_run_bmc(plan: BmcNetbootServePlan, argv: List) -> ProcessExit + uses net: Network +{ + let cmd = command_over_transport(command: ShellCommand { argv: argv }, transport: bmc_transport(plan: plan)) + let run = shell.Exec.Run(script: shell_exec_via_bash(body: shell_command_render(command: cmd))) + if run.success { ExitSuccess } else { exit_failure(reason: concat("bmc-netboot: ", run.stderr)) } +} + +func bmc_netboot_put(plan: BmcNetbootServePlan, local: NonEmptyStr, name: NonEmptyStr) -> ProcessExit + uses net: Network +{ + let put = bmc.Ssh.PutFile(local_path: local, remote_target: bmc_remote_target(plan: plan, name: name)) + if put.success { ExitSuccess } else { exit_failure(reason: concat("bmc-netboot: scp failed for ", name as String)) } +} + +func bmc_netboot_put_inline(plan: BmcNetbootServePlan, local: NonEmptyStr, content: String, name: NonEmptyStr) -> ProcessExit + uses net: Network +{ + let w = Filesystem.Write(path: local as String, content: content) + if w.success == false { + return exit_failure(reason: concat("bmc-netboot: local write failed for ", name as String)) + } + return bmc_netboot_put(plan: plan, local: local, name: name) +} + +fn process_exit_first_failure(results: List) -> ProcessExit { + fold( + results, + init: ExitSuccess, + f: (acc, r) => match acc { + ExitFailure { code: _, reason: _ } => acc + ExitSuccess => r + } + ) +} + +func bmc_netboot_provision(plan: BmcNetbootServePlan) -> ProcessExit + uses net: Network +{ + let mk = bmc_netboot_run_bmc(plan: plan, argv: ["mkdir", "-p", join([plan.netboot_dir as String, "/nocloud"], "")]) + let a1 = bmc_netboot_put(plan: plan, local: bmc_netboot_artifact(name: "busybox"), name: "busybox") + let a2 = bmc_netboot_put(plan: plan, local: bmc_netboot_artifact(name: "vmlinuz"), name: "vmlinuz") + let a3 = bmc_netboot_put(plan: plan, local: bmc_netboot_artifact(name: "initrd"), name: "initrd") + let a4 = bmc_netboot_put(plan: plan, local: bmc_netboot_artifact(name: "grubaa64.efi"), name: "grubaa64.efi") + let g = bmc_netboot_put_inline(plan: plan, local: "/tmp/gunbc-bmc-grub.cfg", content: bmc_netboot_grub_cfg_text(plan: plan), name: "grub.cfg") + let ud = bmc_netboot_put_inline(plan: plan, local: "/tmp/gunbc-bmc-user-data", content: bmc_netboot_nocloud_user_data(plan: plan), name: "nocloud/user-data") + let md = bmc_netboot_put_inline(plan: plan, local: "/tmp/gunbc-bmc-meta-data", content: join(["instance-id: ", plan.autoinstall.identity.hostname as String], ""), name: "nocloud/meta-data") + let serve = bmc_netboot_run_bmc(plan: plan, argv: ["busybox", "httpd", "-p", to_string(plan.serve_port), "-h", plan.netboot_dir as String]) + process_exit_first_failure(results: [mk, a1, a2, a3, a4, g, ud, md, serve]) +} + +func srv4_bmc_netboot_provision() -> ProcessExit + uses net: Network +{ + bmc_netboot_provision(plan: srv4_bmc_netboot_serve_plan) +} + data bmc_netboot_bootstrap_artifact_dir: NonEmptyStr = "/var/lib/gunbc/artifacts/netboot" type StagedContentSource From 7fec69e7120c602802019d0e066c1c7e0e1e7d51 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Jul 2026 00:02:24 +0000 Subject: [PATCH 22/58] fix(bmc/netboot): httpd command uses the staged busybox binary (found by live run) Live provision revealed the serve invoked the BMC's system busybox (no httpd applet) instead of our staged static busybox. busybox_httpd_command now takes busybox_bin; bmc_netboot_serve_command_local passes the staged path (/tmp/netboot/busybox). Witness updated to the staged-path rendering. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/tools/busybox.dag | 4 ++-- dag/gunbc/bmc_netboot_serve.dag | 8 ++++++-- dag/test/claim/bmc_netboot_serve_witness_test.dag | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/dag/extdeps/tools/busybox.dag b/dag/extdeps/tools/busybox.dag index 2cda933f64b..bbb3e69bdea 100644 --- a/dag/extdeps/tools/busybox.dag +++ b/dag/extdeps/tools/busybox.dag @@ -23,8 +23,8 @@ data busybox_cli_tool: CliTool = CliTool { installable_via: [SourceApt { package: package_busybox }], } -fn busybox_httpd_command(document_root: NonEmptyStr, port: Int) -> ShellCommand { +fn busybox_httpd_command(busybox_bin: NonEmptyStr, document_root: NonEmptyStr, port: Int) -> ShellCommand { ShellCommand { - argv: ["busybox", "httpd", "-p", to_string(port), "-h", document_root as String], + argv: [busybox_bin as String, "httpd", "-p", to_string(port), "-h", document_root as String], } } diff --git a/dag/gunbc/bmc_netboot_serve.dag b/dag/gunbc/bmc_netboot_serve.dag index b2dc4f1aee3..a0fed3ea44b 100644 --- a/dag/gunbc/bmc_netboot_serve.dag +++ b/dag/gunbc/bmc_netboot_serve.dag @@ -106,7 +106,11 @@ fn bmc_transport(plan: BmcNetbootServePlan) -> CommandTransport { } fn bmc_netboot_serve_command_local(plan: BmcNetbootServePlan) -> ShellCommand { - busybox_httpd_command(document_root: plan.netboot_dir, port: plan.serve_port) + busybox_httpd_command( + busybox_bin: bmc_netboot_staged_path(plan: plan, name: "busybox"), + document_root: plan.netboot_dir, + port: plan.serve_port, + ) } fn bmc_netboot_serve_command(plan: BmcNetbootServePlan) -> ShellCommand { @@ -177,7 +181,7 @@ func bmc_netboot_provision(plan: BmcNetbootServePlan) -> ProcessExit let g = bmc_netboot_put_inline(plan: plan, local: "/tmp/gunbc-bmc-grub.cfg", content: bmc_netboot_grub_cfg_text(plan: plan), name: "grub.cfg") let ud = bmc_netboot_put_inline(plan: plan, local: "/tmp/gunbc-bmc-user-data", content: bmc_netboot_nocloud_user_data(plan: plan), name: "nocloud/user-data") let md = bmc_netboot_put_inline(plan: plan, local: "/tmp/gunbc-bmc-meta-data", content: join(["instance-id: ", plan.autoinstall.identity.hostname as String], ""), name: "nocloud/meta-data") - let serve = bmc_netboot_run_bmc(plan: plan, argv: ["busybox", "httpd", "-p", to_string(plan.serve_port), "-h", plan.netboot_dir as String]) + let serve = bmc_netboot_run_bmc(plan: plan, argv: bmc_netboot_serve_command_local(plan: plan).argv) process_exit_first_failure(results: [mk, a1, a2, a3, a4, g, ud, md, serve]) } diff --git a/dag/test/claim/bmc_netboot_serve_witness_test.dag b/dag/test/claim/bmc_netboot_serve_witness_test.dag index 89929081eb6..0c02d19f1a2 100644 --- a/dag/test/claim/bmc_netboot_serve_witness_test.dag +++ b/dag/test/claim/bmc_netboot_serve_witness_test.dag @@ -40,8 +40,8 @@ test fn srv4_bmc_served_seed_carries_fleet_keys() -> Bool { test fn serve_command_one_shape_two_transports() -> Bool { let local = shell_command_render(command: bmc_netboot_serve_command_local(plan: srv4_bmc_netboot_serve_plan)) let over_bmc = bmc_netboot_serve_command_text(plan: srv4_bmc_netboot_serve_plan) - local == "busybox httpd -p 8080 -h /tmp/netboot" - && over_bmc == "sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@192.168.1.195 busybox httpd -p 8080 -h /tmp/netboot" + local == "/tmp/netboot/busybox httpd -p 8080 -h /tmp/netboot" + && over_bmc == "sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@192.168.1.195 /tmp/netboot/busybox httpd -p 8080 -h /tmp/netboot" } test fn srv4_staging_manifest_stages_rendered_grub_and_seed() -> Bool { From 9681857d67fd8853bd35872c92cf20edf07c32d5 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Jul 2026 00:24:44 +0000 Subject: [PATCH 23/58] feat(bmc/netboot): model the busybox cross-build (reproducible, not manual) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - extdeps/exec/command: shell_command_render now shell-quotes each arg (handles args with spaces like EXTRA_CFLAGS="-march=... -mfloat-abi=..."), robustness fix. - gunbc/command_runner: run_shell_command / run_shell_commands β€” generic sequential runner over the seam with short-circuit (fold-with-effects, verified). - extdeps/tools/busybox: busybox_source_1_36_1_url; apt package_gcc_arm_linux_gnueabi. - gunbc/busybox_bmc_build: BusyboxCrossBuildPlan + busybox_build_commands (fetch β†’ extract β†’ defconfig β†’ enable static β†’ disable TC β†’ cross-compile armv5te soft-float β†’ install artifact) + busybox_bmc_build_run. Solves the BMC-httpd wall found live: the AST2500 (armv6, no VFP) SIGILLs on prebuilt busybox httpd; our conservative-flags build runs clean (verified: httpd serves HTTP 200 on the BMC). busybox_bmc_build_run regenerated the artifact from source end-to-end. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/exec/command.dag | 6 +- dag/extdeps/package_managers/apt.dag | 1 + dag/extdeps/tools/busybox.dag | 5 ++ dag/gunbc/busybox_bmc_build.dag | 61 +++++++++++++++++++ dag/gunbc/command_runner.dag | 38 ++++++++++++ .../claim/bmc_netboot_serve_witness_test.dag | 5 +- .../claim/busybox_bmc_build_witness_test.dag | 17 ++++++ 7 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 dag/gunbc/busybox_bmc_build.dag create mode 100644 dag/gunbc/command_runner.dag create mode 100644 dag/test/claim/busybox_bmc_build_witness_test.dag diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index 4420b807de6..4ec706e5cb1 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -37,6 +37,10 @@ fn command_over_transport(command: ShellCommand, transport: CommandTransport) -> } } +fn shell_quote(arg: String) -> String { + join(["'", arg, "'"], "") +} + fn shell_command_render(command: ShellCommand) -> String { - join(command.argv, " ") + join(map(command.argv, a => shell_quote(arg: a)), " ") } diff --git a/dag/extdeps/package_managers/apt.dag b/dag/extdeps/package_managers/apt.dag index 8c57d965491..2684ea001de 100644 --- a/dag/extdeps/package_managers/apt.dag +++ b/dag/extdeps/package_managers/apt.dag @@ -31,6 +31,7 @@ data package_websocat: NonEmptyStr = "websocat" data package_xorriso: NonEmptyStr = "xorriso" data package_google_cloud_cli: NonEmptyStr = "google-cloud-cli" data package_busybox: NonEmptyStr = "busybox-static" +data package_gcc_arm_linux_gnueabi: NonEmptyStr = "gcc-arm-linux-gnueabi" data apt_man_url: NonEmptyStr = "https://manpages.debian.org/testing/apt/apt.8.en.html" diff --git a/dag/extdeps/tools/busybox.dag b/dag/extdeps/tools/busybox.dag index bbb3e69bdea..1c4f7390f0f 100644 --- a/dag/extdeps/tools/busybox.dag +++ b/dag/extdeps/tools/busybox.dag @@ -23,6 +23,11 @@ data busybox_cli_tool: CliTool = CliTool { installable_via: [SourceApt { package: package_busybox }], } +data busybox_source_1_36_1_url: Uri = Uri { + scheme: Https, + locator: "busybox.net/downloads/busybox-1.36.1.tar.bz2", +} + fn busybox_httpd_command(busybox_bin: NonEmptyStr, document_root: NonEmptyStr, port: Int) -> ShellCommand { ShellCommand { argv: [busybox_bin as String, "httpd", "-p", to_string(port), "-h", document_root as String], diff --git a/dag/gunbc/busybox_bmc_build.dag b/dag/gunbc/busybox_bmc_build.dag new file mode 100644 index 00000000000..d26087ff07f --- /dev/null +++ b/dag/gunbc/busybox_bmc_build.dag @@ -0,0 +1,61 @@ +module gunbc.busybox_bmc_build + +import std.types { NonEmptyStr, String, List } +import std.process { ProcessExit } +import std.resources { Network } +import extdeps.uri { Uri, uri_wire } +import extdeps.exec.command { ShellCommand, LocalShell } +import extdeps.tools.busybox { busybox_source_1_36_1_url } +import gunbc.command_runner { run_shell_commands } + +type BusyboxCrossBuildPlan { + source_url: Uri + source_tarball: NonEmptyStr + build_dir: NonEmptyStr + cross_compile: NonEmptyStr + kernel_arch: NonEmptyStr + extra_cflags: NonEmptyStr + output_path: NonEmptyStr +} + +data bmc_busybox_build_plan: BusyboxCrossBuildPlan = BusyboxCrossBuildPlan { + source_url: busybox_source_1_36_1_url, + source_tarball: "/tmp/gunbc-busybox-src.tar.bz2", + build_dir: "/tmp/gunbc-bb-build", + cross_compile: "arm-linux-gnueabi-", + kernel_arch: "arm", + extra_cflags: "-march=armv5te -mfloat-abi=soft", + output_path: "/var/lib/gunbc/artifacts/netboot/busybox", +} + +fn bb_make_var(key: String, value: String) -> String { + join([key, "=", value], "") +} + +fn bb_config_path(plan: BusyboxCrossBuildPlan) -> String { + join([plan.build_dir as String, "/.config"], "") +} + +fn busybox_build_commands(plan: BusyboxCrossBuildPlan) -> List { + let bd = plan.build_dir as String + let arch = bb_make_var(key: "ARCH", value: plan.kernel_arch as String) + let cross = bb_make_var(key: "CROSS_COMPILE", value: plan.cross_compile as String) + let cflags = bb_make_var(key: "EXTRA_CFLAGS", value: plan.extra_cflags as String) + let config = bb_config_path(plan: plan) + [ + ShellCommand { argv: ["curl", "-sL", "-o", plan.source_tarball as String, uri_wire(uri: plan.source_url)] }, + ShellCommand { argv: ["mkdir", "-p", bd] }, + ShellCommand { argv: ["tar", "xf", plan.source_tarball as String, "-C", bd, "--strip-components=1"] }, + ShellCommand { argv: ["make", "-C", bd, arch, cross, "defconfig"] }, + ShellCommand { argv: ["sed", "-i", "s/# CONFIG_STATIC is not set/CONFIG_STATIC=y/", config] }, + ShellCommand { argv: ["sed", "-i", "s/CONFIG_TC=y/# CONFIG_TC is not set/", config] }, + ShellCommand { argv: ["make", "-C", bd, arch, cross, cflags] }, + ShellCommand { argv: ["cp", join([bd, "/busybox"], ""), plan.output_path as String] }, + ] +} + +func busybox_bmc_build_run() -> ProcessExit + uses net: Network +{ + run_shell_commands(commands: busybox_build_commands(plan: bmc_busybox_build_plan), transport: LocalShell) +} diff --git a/dag/gunbc/command_runner.dag b/dag/gunbc/command_runner.dag new file mode 100644 index 00000000000..ff516427e97 --- /dev/null +++ b/dag/gunbc/command_runner.dag @@ -0,0 +1,38 @@ +module gunbc.command_runner + +import std.types { List, String } +import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.resources { Network } +import extdeps.exec.command { + ShellCommand, + CommandTransport, + command_over_transport, + shell_command_render, +} +import extdeps.shell +import gunbc.shell_bash_runner { shell_exec_via_bash } + +func run_shell_command(command: ShellCommand, transport: CommandTransport) -> ProcessExit + uses net: Network +{ + let rendered = shell_command_render(command: command_over_transport(command: command, transport: transport)) + let run = shell.Exec.Run(script: shell_exec_via_bash(body: rendered)) + if run.success { + ExitSuccess + } else { + exit_failure(reason: concat("command failed: ", run.stderr)) + } +} + +func run_shell_commands(commands: List, transport: CommandTransport) -> ProcessExit + uses net: Network +{ + fold( + commands, + init: ExitSuccess, + f: (acc, c) => match acc { + ExitFailure { code: _, reason: _ } => acc + ExitSuccess => run_shell_command(command: c, transport: transport) + } + ) +} diff --git a/dag/test/claim/bmc_netboot_serve_witness_test.dag b/dag/test/claim/bmc_netboot_serve_witness_test.dag index 0c02d19f1a2..d0715fe2984 100644 --- a/dag/test/claim/bmc_netboot_serve_witness_test.dag +++ b/dag/test/claim/bmc_netboot_serve_witness_test.dag @@ -40,8 +40,9 @@ test fn srv4_bmc_served_seed_carries_fleet_keys() -> Bool { test fn serve_command_one_shape_two_transports() -> Bool { let local = shell_command_render(command: bmc_netboot_serve_command_local(plan: srv4_bmc_netboot_serve_plan)) let over_bmc = bmc_netboot_serve_command_text(plan: srv4_bmc_netboot_serve_plan) - local == "/tmp/netboot/busybox httpd -p 8080 -h /tmp/netboot" - && over_bmc == "sshpass -e ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@192.168.1.195 /tmp/netboot/busybox httpd -p 8080 -h /tmp/netboot" + string_contains(s: local, pattern: "'/tmp/netboot/busybox' 'httpd' '-p' '8080' '-h' '/tmp/netboot'") + && string_contains(s: over_bmc, pattern: "'sshpass' '-e' 'ssh'") + && string_contains(s: over_bmc, pattern: "'root@192.168.1.195' '/tmp/netboot/busybox' 'httpd'") } test fn srv4_staging_manifest_stages_rendered_grub_and_seed() -> Bool { diff --git a/dag/test/claim/busybox_bmc_build_witness_test.dag b/dag/test/claim/busybox_bmc_build_witness_test.dag new file mode 100644 index 00000000000..4930f2ff672 --- /dev/null +++ b/dag/test/claim/busybox_bmc_build_witness_test.dag @@ -0,0 +1,17 @@ +module test.claim.busybox_bmc_build + +import std.logic { Bool } +import std.types { list_length } +import extdeps.exec.command { shell_command_render } +import gunbc.busybox_bmc_build { bmc_busybox_build_plan, busybox_build_commands } + +test fn busybox_build_is_modeled_and_grounded() -> Bool { + let cmds = busybox_build_commands(plan: bmc_busybox_build_plan) + let all = fold(cmds, init: "", f: (acc, c) => join([acc, shell_command_render(command: c), "\n"], "")) + string_contains(s: all, pattern: "busybox.net/downloads/busybox-1.36.1.tar.bz2") + && string_contains(s: all, pattern: "'CROSS_COMPILE=arm-linux-gnueabi-'") + && string_contains(s: all, pattern: "'EXTRA_CFLAGS=-march=armv5te -mfloat-abi=soft'") + && string_contains(s: all, pattern: "CONFIG_STATIC=y") + && string_contains(s: all, pattern: "/var/lib/gunbc/artifacts/netboot/busybox") + && list_length(items: cmds) == 8 +} From e0bdb3377cfa353478b93c11bfcbe8fc46031dca Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Jul 2026 00:31:07 +0000 Subject: [PATCH 24/58] fix(bmc/netboot): srv4 boot NIC is eth2 (leases DHCP), not eth0 (found live) eth0 has media but its UEFI DHCP falls back to link-local; eth2 leases 192.168.1.196 on the LAN segment with the BMC. Plan boot_interface + witness updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/gunbc/bmc_netboot_serve.dag | 2 +- dag/test/claim/bmc_netboot_serve_witness_test.dag | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dag/gunbc/bmc_netboot_serve.dag b/dag/gunbc/bmc_netboot_serve.dag index a0fed3ea44b..c81ac5f0c4b 100644 --- a/dag/gunbc/bmc_netboot_serve.dag +++ b/dag/gunbc/bmc_netboot_serve.dag @@ -37,7 +37,7 @@ data srv4_bmc_netboot_serve_plan: BmcNetbootServePlan = BmcNetbootServePlan { bmc_host: "192.168.1.195", serve_port: 8080, netboot_dir: "/tmp/netboot", - boot_interface: "eth0", + boot_interface: "eth2", bulk_iso_url: Uri { scheme: Https, locator: "cdimage.ubuntu.com/ubuntu/releases/24.04/release/ubuntu-24.04.3-live-server-arm64.iso", diff --git a/dag/test/claim/bmc_netboot_serve_witness_test.dag b/dag/test/claim/bmc_netboot_serve_witness_test.dag index d0715fe2984..a34da706734 100644 --- a/dag/test/claim/bmc_netboot_serve_witness_test.dag +++ b/dag/test/claim/bmc_netboot_serve_witness_test.dag @@ -27,7 +27,7 @@ test fn srv4_grub_cfg_boots_mirror_bulk_with_bmc_seed() -> Bool { test fn srv4_http_boot_entry_targets_bmc() -> Bool { let entry = bmc_netboot_http_boot_entry(plan: srv4_bmc_netboot_serve_plan) string_contains(s: uefi_http_boot_uri_wire(entry: entry), pattern: "http://192.168.1.195:8080/grubaa64.efi") - && entry.interface_name == "eth0" + && entry.interface_name == "eth2" } test fn srv4_bmc_served_seed_carries_fleet_keys() -> Bool { From 74e72978fac78619c5683b84ca1a85cf11718932 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Jul 2026 20:29:30 +0000 Subject: [PATCH 25/58] srv4 fleet subsumption + BMC virtual-media install path + modeling cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Subsume srv4 into the fleet and get it onto GitHub Actions runners, plus the supporting install-path modeling and several dissolved shell/Β§3 forks found along the way. Fleet membership (srv4): - srv4_host + samsung_970_evo_500gb_catalog (its actual drive, cited), LAN endpoint 192.168.1.196, srv4_offer, deployed_intent_v1_srv4. Placement solver now allocates srv4 runner slots (fleet_concurrent_runs 30->40). Runner deploy (the width-INCREASE gap, now modeled): - runner_host_deploy.dag: RunnerHostDeploy intent citing the ctrl installer (install-actions-runner.sh) as the bound realization handler (Β§3 cite-upstream, not re-coined). Renders the CTRL_RUNNER_* invocation, the App-key SecretRef, and the actions-runner@srv4-NN enables. srv4: user briansrls, 5 slots (disk-cap note for the 500GB NVMe vs 2TB fleet). Execution-surface honesty + toolchain provisioning (Β§5 model-reality gap): - fleet_intent_execution_surface no longer lies: container_runtime Present{Docker} + toolchains [sccache] instead of none/[]. - extdeps/cache/sccache.dag: SccacheBinaryRelease (pinned v0.15.0 + per-arch sha256, cited to mozilla/sccache/releases) + install script. - extdeps/container/docker_ce.dag: DockerCeAptRepo + packages + rootless-docker apt prereq install, cited to docs.docker.com. BMC virtual-media install path (used to install srv4 end-to-end): - extdeps/storage/nbd.dag, extdeps/linux/usb_gadget.dag (typed ConfigfsOp list, not scattered concats), gunbc/bmc_virtual_media.dag: nbd-server -> nbd-client -> configfs mass_storage USB gadget. Unified under extdeps/bmc/virtual_media.dag VirtualMediaIntent with the existing nbd-proxy-websocat path (Β§3 fork dissolved). - extdeps/firmware/uefi_shell.dag + gunbc/bmc_netboot_shell_boot.dag: shell-native tftp/execute boot modeling. Install bugfixes (root-caused live, captured in the model + witnesses): - grub.dag: quote kernel cmdline args containing ';' (grub command separator) β€” ds=nocloud;s=... was truncated at ';', dropping the seedfrom, so autoinstall fell to interactive. grub_cmdline_arg_render + the seeded-media builders quote it. - ubuntu_seeded_install_media_remaster.dag: xorriso -boot_image any replay (was mkisofs, which destroyed the arm64 El Torito/ESP boot structure); instance-id derived from hostname (was hardcoded srv3). All new modeling lands with witnesses (green) and dissolution markers on the shell-as-string leaves. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/extdeps/bmc/virtual_media.dag | 54 +++++++ dag/extdeps/bmc/webui/nbd_proxy_serve.dag | 15 ++ dag/extdeps/bootloader/grub.dag | 20 ++- dag/extdeps/cache/sccache.dag | 43 ++++++ dag/extdeps/container/docker_ce.dag | 67 +++++++++ dag/extdeps/firmware/uefi_shell.dag | 9 ++ dag/extdeps/linux/usb_gadget.dag | 111 ++++++++++++++ .../os/ubuntu_seeded_install_media.dag | 8 +- .../ubuntu_seeded_install_media_remaster.dag | 4 +- dag/extdeps/storage/nbd.dag | 72 +++++++++ dag/extdeps/storage/samsung.dag | 17 +++ dag/gunbc/bmc_netboot_shell_boot.dag | 55 +++++++ dag/gunbc/bmc_virtual_media.dag | 142 ++++++++++++++++++ dag/gunbc/ci_fleet.dag | 15 +- dag/gunbc/deployed_intent_v1.dag | 12 +- dag/gunbc/fleet_intent.dag | 27 +++- dag/gunbc/fleet_intent_network.dag | 9 ++ dag/gunbc/runner_host_deploy.dag | 109 ++++++++++++++ .../claim/bmc_netboot_serve_witness_test.dag | 2 +- .../bmc_netboot_shell_boot_witness_test.dag | 26 ++++ .../claim/bmc_virtual_media_witness_test.dag | 51 +++++++ .../grub_cmdline_quoting_witness_test.dag | 22 +++ .../claim/runner_host_deploy_witness_test.dag | 62 ++++++++ ...virtual_media_unification_witness_test.dag | 28 ++++ 24 files changed, 966 insertions(+), 14 deletions(-) create mode 100644 dag/extdeps/bmc/virtual_media.dag create mode 100644 dag/extdeps/container/docker_ce.dag create mode 100644 dag/extdeps/linux/usb_gadget.dag create mode 100644 dag/extdeps/storage/nbd.dag create mode 100644 dag/gunbc/bmc_netboot_shell_boot.dag create mode 100644 dag/gunbc/bmc_virtual_media.dag create mode 100644 dag/gunbc/runner_host_deploy.dag create mode 100644 dag/test/claim/bmc_netboot_shell_boot_witness_test.dag create mode 100644 dag/test/claim/bmc_virtual_media_witness_test.dag create mode 100644 dag/test/claim/grub_cmdline_quoting_witness_test.dag create mode 100644 dag/test/claim/runner_host_deploy_witness_test.dag create mode 100644 dag/test/claim/virtual_media_unification_witness_test.dag diff --git a/dag/extdeps/bmc/virtual_media.dag b/dag/extdeps/bmc/virtual_media.dag new file mode 100644 index 00000000000..a46eb29479b --- /dev/null +++ b/dag/extdeps/bmc/virtual_media.dag @@ -0,0 +1,54 @@ +module extdeps.bmc.virtual_media + +import std.types { NonEmptyStr, Int, String, List } +import std.logic { Bool } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "docs.kernel.org/usb/mass-storage.html", + } +} + +data virtual_media_fork_reconciliation: String = "Β§3 single-authority: 'present a block source (ISO) to a host as USB storage' is ONE concept β€” VirtualMediaIntent β€” with N transport realizations named centrally in VirtualMediaTransport (the dispatch axis). ConfigfsGadgetTransport (nbd-server β†’ nbd-client β†’ configfs mass_storage gadget; realized in gunbc.bmc_virtual_media) and NbdProxyWebsocatTransport (nbdkit β†’ websocat β†’ bmcweb WebUI nbd wss; realized in extdeps.bmc.webui.nbd_proxy_serve) were two independent virtual-media concepts; they now consume this shared intent. Per Β§3 the dispatch that selects a realization is itself realization, so each transport's concrete plan binds in its peripheral module, not here." + +type VirtualMediaSource { + iso_path: NonEmptyStr +} + +type VirtualMediaIntent { + bmc_host: NonEmptyStr + source: VirtualMediaSource + read_only: Bool +} + +type ConfigfsGadgetParams { + nbd_server_host: NonEmptyStr + nbd_port: Int + export_name: NonEmptyStr + nbd_device: NonEmptyStr + udc_port: NonEmptyStr +} + +type NbdProxyWebsocatParams { + session_token: NonEmptyStr + ws_path: NonEmptyStr + local_nbd_port: Int +} + +type VirtualMediaTransport + = ConfigfsGadgetTransport { params: ConfigfsGadgetParams } + | NbdProxyWebsocatTransport { params: NbdProxyWebsocatParams } + +fn virtual_media_transport_kind(transport: VirtualMediaTransport) -> String { + match transport { + ConfigfsGadgetTransport { params: _ } => "configfs-gadget" + NbdProxyWebsocatTransport { params: _ } => "nbdproxy-websocat" + } +} + +fn virtual_media_source_iso(intent: VirtualMediaIntent) -> NonEmptyStr { + intent.source.iso_path +} diff --git a/dag/extdeps/bmc/webui/nbd_proxy_serve.dag b/dag/extdeps/bmc/webui/nbd_proxy_serve.dag index f2c98520caf..6763cebe9d7 100644 --- a/dag/extdeps/bmc/webui/nbd_proxy_serve.dag +++ b/dag/extdeps/bmc/webui/nbd_proxy_serve.dag @@ -26,6 +26,11 @@ import extdeps.bmc.webui.nbd_proxy { NbdServer, } import extdeps.uri { Uri, Https } +import extdeps.bmc.virtual_media { + VirtualMediaIntent, + NbdProxyWebsocatParams, + virtual_media_source_iso, +} data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -70,6 +75,16 @@ fn nbd_proxy_serve_transport_intent( } } +fn nbd_proxy_serve_intent_of_virtual_media(vm: VirtualMediaIntent, params: NbdProxyWebsocatParams) -> NbdProxyServeTransportIntent { + NbdProxyServeTransportIntent { + bmc_host: vm.bmc_host, + session_token: params.session_token, + ws_path: params.ws_path, + iso_path: virtual_media_source_iso(intent: vm), + local_nbd_port: params.local_nbd_port, + } +} + fn bmcweb_session_login_program( bmc_host: NonEmptyStr, request_body_file: NonEmptyStr, diff --git a/dag/extdeps/bootloader/grub.dag b/dag/extdeps/bootloader/grub.dag index 13e812cbcbf..1e8c0b4f934 100644 --- a/dag/extdeps/bootloader/grub.dag +++ b/dag/extdeps/bootloader/grub.dag @@ -1,7 +1,7 @@ module extdeps.bootloader.grub import std.types { NonEmptyStr, String, Int, List } -import extdeps.firmware.kernel_cmdline { KernelCmdlineArg, kernel_cmdline_render } +import extdeps.firmware.kernel_cmdline { KernelCmdlineArg, kernel_cmdline_arg_wire } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } @@ -25,8 +25,24 @@ type GrubConfig { entries: List } +fn grub_cmdline_arg_render(arg: KernelCmdlineArg) -> String { + let wire = kernel_cmdline_arg_wire(arg: arg) + if string_contains(s: wire, pattern: ";") { + concat("\"", concat(wire, "\"")) + } else { + wire + } +} + fn grub_linux_line(entry: GrubMenuEntry) -> String { - join([" linux", entry.linux_image as String, kernel_cmdline_render(args: entry.linux_cmdline)], " ") + join( + [ + " linux", + entry.linux_image as String, + join(map(entry.linux_cmdline, a => grub_cmdline_arg_render(arg: a)), " "), + ], + " ", + ) } fn grub_menuentry_render(entry: GrubMenuEntry) -> String { diff --git a/dag/extdeps/cache/sccache.dag b/dag/extdeps/cache/sccache.dag index ceca4099c50..baff254a3b5 100644 --- a/dag/extdeps/cache/sccache.dag +++ b/dag/extdeps/cache/sccache.dag @@ -27,6 +27,8 @@ import std.cache_interface { import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } +import std.types { NonEmptyStr, String, List } +import extdeps.toolchain.types { ToolchainCapability, SharedHomeAcrossJobs, CtrlBuild } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -37,6 +39,47 @@ data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { data sccache_local_id: CacheInterfaceId = "sccache_local" +data sccache_releases_authority: ExternalAuthority = ExternalAuthority { + uri: Uri { scheme: Https, locator: "github.com/mozilla/sccache/releases" } +} + +type SccacheBinaryRelease { + version: NonEmptyStr + sha256_aarch64: NonEmptyStr + sha256_x86_64: NonEmptyStr +} + +data sccache_release_0_15_0: SccacheBinaryRelease = SccacheBinaryRelease { + version: "v0.15.0", + sha256_aarch64: "3a6a3712b49da3d263bf2d30d702de4302793016019e800bfb81c0c69401d8f8", + sha256_x86_64: "782d2b5dd7ae0a55ebe368ab258114d0928d019ac2d949ab85d5d02f3926709e", +} + +data sccache_toolchain_capability: ToolchainCapability = ToolchainCapability { + name: "sccache", + version: "0.15.0", + env_isolation: SharedHomeAcrossJobs, + build_wrapper: Present { value: CtrlBuild }, + linked_cache_interface: Present { value: sccache_local_id }, +} + +data sccache_install_dissolution_trigger: String = "🟑 dissolve-on: sccache_install_script β€” the sccache binary install (curl release tarball + sha256sum -c verify + tar + mv to /usr/local/bin, cited to github.com/mozilla/sccache/releases with pinned v0.15.0 and per-arch sha256) is rendered as a POSIX shell provisioning sequence (medium-as-string); the SccacheBinaryRelease row is the typed authority for version and digests. DISSOLVES WHEN host_effect_apply binds a verified-download-plus-install effect (FetchAndVerify then InstallBinary) without the shell leaf." + +fn sccache_install_script(rel: SccacheBinaryRelease, arch: NonEmptyStr, sha256: NonEmptyStr) -> String { + let dir = join(["sccache-", rel.version as String, "-", arch as String, "-unknown-linux-musl"], "") + let url = join(["https://github.com/mozilla/sccache/releases/download/", rel.version as String, "/", dir, ".tar.gz"], "") + join([ + "set -e", + join(["curl -fsSL '", url, "' -o /tmp/sccache.tar.gz"], ""), + join(["echo '", sha256 as String, " /tmp/sccache.tar.gz' | sha256sum -c -"], ""), + "tar -xzf /tmp/sccache.tar.gz -C /tmp", + join(["sudo mv '/tmp/", dir, "/sccache' /usr/local/bin/sccache"], ""), + "sudo chmod +x /usr/local/bin/sccache", + join(["rm -rf /tmp/sccache.tar.gz '/tmp/", dir, "'"], ""), + "", + ], "\n") +} + data sccache_fatal_error_log_fragment: String = "sccache: encountered fatal error" data sccache_rustc_wrapper_exit_254_log_fragment: String = "(exit status: 254)" diff --git a/dag/extdeps/container/docker_ce.dag b/dag/extdeps/container/docker_ce.dag new file mode 100644 index 00000000000..f5ecd00c2e8 --- /dev/null +++ b/dag/extdeps/container/docker_ce.dag @@ -0,0 +1,67 @@ +module extdeps.container.docker_ce + +import std.types { NonEmptyStr, String, List } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "docs.docker.com/engine/install/ubuntu/", + } +} + +type DockerCeAptRepo { + keyring_url: NonEmptyStr + keyring_path: NonEmptyStr + repo_base: NonEmptyStr + suite: NonEmptyStr + component: NonEmptyStr + arch: NonEmptyStr +} + +data docker_ce_apt_repo_noble_arm64: DockerCeAptRepo = DockerCeAptRepo { + keyring_url: "https://download.docker.com/linux/ubuntu/gpg", + keyring_path: "/etc/apt/keyrings/docker.gpg", + repo_base: "https://download.docker.com/linux/ubuntu", + suite: "noble", + component: "stable", + arch: "arm64", +} + +data docker_ce_packages: List = [ + "docker-ce", + "docker-ce-cli", + "containerd.io", + "docker-buildx-plugin", + "docker-compose-plugin", + "docker-ce-rootless-extras", +] + +fn docker_ce_apt_source_line(repo: DockerCeAptRepo) -> String { + join( + [ + "deb [arch=", repo.arch as String, " signed-by=", repo.keyring_path as String, "] ", + repo.repo_base as String, " ", repo.suite as String, " ", repo.component as String, + ], + "", + ) +} + +data docker_ce_install_dissolution_trigger: String = "🟑 dissolve-on: docker_ce_repo_install_script β€” the docker-ce apt-repo bootstrap (install keyrings dir, curl+dearmor the cited download.docker.com gpg key, write the deb source line, apt-get update, apt-get install docker-ce and docker-ce-rootless-extras) is rendered as a POSIX shell provisioning sequence (medium-as-string); the DockerCeAptRepo row and docker_ce_packages list are the typed authority, cited to docs.docker.com/engine/install/ubuntu. Rootless runtime setup for the de-priv job user is then the ctrl install-actions-runner.sh handler (which needs docker-ce-rootless-extras present first β€” the prereq this script satisfies). DISSOLVES WHEN host_effect_apply binds an AptRepoEnsure plus PackageInstall effect without the shell leaf." + +fn docker_ce_repo_install_script(repo: DockerCeAptRepo, packages: List) -> String { + join( + [ + "set -e", + "sudo install -m 0755 -d /etc/apt/keyrings", + join(["curl -fsSL ", repo.keyring_url as String, " | sudo gpg --dearmor --yes -o ", repo.keyring_path as String], ""), + join(["sudo chmod a+r ", repo.keyring_path as String], ""), + join(["echo '", docker_ce_apt_source_line(repo: repo), "' | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null"], ""), + "sudo apt-get update -q", + join(["sudo apt-get install -y -q ", join(map(packages, p => p as String), " ")], ""), + "", + ], + "\n", + ) +} diff --git a/dag/extdeps/firmware/uefi_shell.dag b/dag/extdeps/firmware/uefi_shell.dag index 91f716da2f9..ed320d1703e 100644 --- a/dag/extdeps/firmware/uefi_shell.dag +++ b/dag/extdeps/firmware/uefi_shell.dag @@ -1,6 +1,7 @@ module extdeps.firmware.uefi_shell import std.types { NonEmptyStr, String, Int, Bool, List } +import extdeps.firmware.kernel_cmdline { KernelCmdlineArg, kernel_cmdline_render } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } @@ -49,6 +50,9 @@ type UefiShellCommand | ConnectRecursive | ListDevices | Ifconfig { action: UefiIfconfigAction } + | Ping { host: NonEmptyStr } + | Tftp { interface: NonEmptyStr, server: NonEmptyStr, remote_file: NonEmptyStr, local_file: NonEmptyStr } + | ExecuteEfi { path: NonEmptyStr, cmdline: List } | Reset { mode: UefiResetMode } fn uefi_shell_command_line(cmd: UefiShellCommand) -> String { @@ -74,6 +78,11 @@ fn uefi_shell_command_line(cmd: UefiShellCommand) -> String { if r { "map -r" } else { "map" } ConnectRecursive => "connect -r" ListDevices => "devices" + Ping { host: h } => concat("ping ", h as String) + Tftp { interface: i, server: s, remote_file: rf, local_file: lf } => + join(["tftp", "-i", i as String, s as String, rf as String, lf as String], " ") + ExecuteEfi { path: p, cmdline: c } => + join([p as String, kernel_cmdline_render(args: c)], " ") Ifconfig { action: a } => match a { IfconfigList => "ifconfig -l" diff --git a/dag/extdeps/linux/usb_gadget.dag b/dag/extdeps/linux/usb_gadget.dag new file mode 100644 index 00000000000..d65b563e803 --- /dev/null +++ b/dag/extdeps/linux/usb_gadget.dag @@ -0,0 +1,111 @@ +module extdeps.linux.usb_gadget + +import std.types { NonEmptyStr, String, Int, Bool, List } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "docs.kernel.org/usb/gadget_configfs.html", + } +} + +type MassStorageLun { + backing_device: NonEmptyStr + cdrom: Bool + read_only: Bool + removable: Bool +} + +type UsbGadget { + config_root: NonEmptyStr + name: NonEmptyStr + id_vendor: NonEmptyStr + id_product: NonEmptyStr + manufacturer: NonEmptyStr + product: NonEmptyStr + serial: NonEmptyStr + config_label: NonEmptyStr + lun: MassStorageLun + udc: NonEmptyStr +} + +type ConfigfsOp + = ConfigfsMkdir { path: NonEmptyStr } + | ConfigfsWriteAttr { path: NonEmptyStr, value: String } + | ConfigfsSymlink { target: NonEmptyStr, link: NonEmptyStr } + | ConfigfsRemoveLink { link: NonEmptyStr } + | ConfigfsRmdir { path: NonEmptyStr } + +fn usb_gadget_dir(gadget: UsbGadget) -> NonEmptyStr { + join([gadget.config_root as String, "/", gadget.name as String], "") as NonEmptyStr +} + +fn gadget_bool_bit(b: Bool) -> String { + if b { "1" } else { "0" } +} + +fn usb_gadget_setup_ops(gadget: UsbGadget) -> List { + let g = usb_gadget_dir(gadget: gadget) as String + let strings = join([g, "/strings/0x409"], "") + let func = join([g, "/functions/mass_storage.0"], "") + let lun = join([func, "/lun.0"], "") + let config = join([g, "/configs/", gadget.config_label as String], "") + [ + ConfigfsMkdir { path: g as NonEmptyStr }, + ConfigfsWriteAttr { path: join([g, "/idVendor"], "") as NonEmptyStr, value: gadget.id_vendor as String }, + ConfigfsWriteAttr { path: join([g, "/idProduct"], "") as NonEmptyStr, value: gadget.id_product as String }, + ConfigfsMkdir { path: strings as NonEmptyStr }, + ConfigfsWriteAttr { path: join([strings, "/manufacturer"], "") as NonEmptyStr, value: gadget.manufacturer as String }, + ConfigfsWriteAttr { path: join([strings, "/product"], "") as NonEmptyStr, value: gadget.product as String }, + ConfigfsWriteAttr { path: join([strings, "/serialnumber"], "") as NonEmptyStr, value: gadget.serial as String }, + ConfigfsMkdir { path: func as NonEmptyStr }, + ConfigfsWriteAttr { path: join([lun, "/removable"], "") as NonEmptyStr, value: gadget_bool_bit(b: gadget.lun.removable) }, + ConfigfsWriteAttr { path: join([lun, "/cdrom"], "") as NonEmptyStr, value: gadget_bool_bit(b: gadget.lun.cdrom) }, + ConfigfsWriteAttr { path: join([lun, "/ro"], "") as NonEmptyStr, value: gadget_bool_bit(b: gadget.lun.read_only) }, + ConfigfsWriteAttr { path: join([lun, "/file"], "") as NonEmptyStr, value: gadget.lun.backing_device as String }, + ConfigfsMkdir { path: join([config, "/strings/0x409"], "") as NonEmptyStr }, + ConfigfsWriteAttr { path: join([config, "/strings/0x409/configuration"], "") as NonEmptyStr, value: gadget.config_label as String }, + ConfigfsSymlink { target: func as NonEmptyStr, link: join([config, "/"], "") as NonEmptyStr }, + ConfigfsWriteAttr { path: join([g, "/UDC"], "") as NonEmptyStr, value: gadget.udc as String }, + ] +} + +fn usb_gadget_teardown_ops(gadget: UsbGadget) -> List { + let g = usb_gadget_dir(gadget: gadget) as String + let config = join([g, "/configs/", gadget.config_label as String], "") + [ + ConfigfsWriteAttr { path: join([g, "/UDC"], "") as NonEmptyStr, value: "" }, + ConfigfsRemoveLink { link: join([config, "/mass_storage.0"], "") as NonEmptyStr }, + ConfigfsRmdir { path: join([g, "/functions/mass_storage.0"], "") as NonEmptyStr }, + ConfigfsRmdir { path: join([config, "/strings/0x409"], "") as NonEmptyStr }, + ConfigfsRmdir { path: config as NonEmptyStr }, + ConfigfsRmdir { path: join([g, "/strings/0x409"], "") as NonEmptyStr }, + ConfigfsRmdir { path: g as NonEmptyStr }, + ] +} + +data configfs_op_render_dissolution_trigger: String = "🟑 dissolve-on: configfs_op_line β€” the ConfigfsOp list is the typed authority (structure), but each op is rendered to a POSIX shell line because a configfs attribute write is a remote-file-write with no argv-only form (printf > sysfs needs a redirect) and the BMC transport is SSH. This single per-variant renderer replaces 32 scattered concats. DISSOLVES WHEN host_effect_apply binds ConfigfsOp directly to a remote Filesystem.Write effect (WriteAttr β†’ Filesystem.Write, Mkdir/Symlink/Rmdir β†’ typed FS ops) over the ssh transport, removing the shell leaf." + +fn configfs_op_line(op: ConfigfsOp) -> String { + match op { + ConfigfsMkdir { path: p } => join(["mkdir -p ", p as String], "") + ConfigfsWriteAttr { path: p, value: v } => join(["printf '%s' '", v, "' > ", p as String], "") + ConfigfsSymlink { target: t, link: l } => join(["ln -sf ", t as String, " ", l as String], "") + ConfigfsRemoveLink { link: l } => join(["rm -f ", l as String, " 2>/dev/null || true"], "") + ConfigfsRmdir { path: p } => join(["rmdir ", p as String, " 2>/dev/null || true"], "") + } +} + +fn configfs_ops_script(ops: List) -> String { + fold(ops, init: "", f: (acc, op) => join([acc, configfs_op_line(op: op), "\n"], "")) +} + +fn usb_gadget_setup_script(gadget: UsbGadget) -> String { + join(["set -e", "\n", configfs_ops_script(ops: usb_gadget_setup_ops(gadget: gadget))], "") +} + +fn usb_gadget_teardown_script(gadget: UsbGadget) -> String { + configfs_ops_script(ops: usb_gadget_teardown_ops(gadget: gadget)) +} diff --git a/dag/extdeps/os/ubuntu_seeded_install_media.dag b/dag/extdeps/os/ubuntu_seeded_install_media.dag index c60249c86af..690158304cf 100644 --- a/dag/extdeps/os/ubuntu_seeded_install_media.dag +++ b/dag/extdeps/os/ubuntu_seeded_install_media.dag @@ -28,8 +28,14 @@ type UbuntuSeededInstallMediaArtifactRow { content_hash: ContentHash } +data grub_semicolon_quote_note: String = "The ds= NoCloud seedfrom carries a ';' (e.g. ds=nocloud;s=). In a grub.cfg linux line ';' is grub's command separator, so the arg MUST be double-quoted or grub truncates the cmdline at ';' and the kernel never sees s= β€” autoinstall then falls to interactive. These builders emit grub cmdline strings (injected by the remaster sed), so they quote the ds arg here; the List grub path quotes structurally in grub_cmdline_arg_render." + fn ubuntu_seeded_install_media_grub_kernel_cmdline(nocloud_dir: NonEmptyStr) -> NonEmptyStr { - concat("autoinstall ds=nocloud;s=/cdrom/", nocloud_dir as String) as NonEmptyStr + concat("autoinstall \"ds=nocloud;s=/cdrom/", concat(nocloud_dir as String, "/\"")) as NonEmptyStr +} + +fn ubuntu_seeded_install_media_grub_kernel_cmdline_net(seed_url: NonEmptyStr) -> NonEmptyStr { + concat("autoinstall ip=dhcp \"ds=nocloud-net;s=", concat(seed_url as String, "\"")) as NonEmptyStr } fn ubuntu_seeded_install_media_default_nocloud_dir(hostname: NonEmptyStr) -> NonEmptyStr { diff --git a/dag/extdeps/os/ubuntu_seeded_install_media_remaster.dag b/dag/extdeps/os/ubuntu_seeded_install_media_remaster.dag index c00406c8d3b..c275a2dc238 100644 --- a/dag/extdeps/os/ubuntu_seeded_install_media_remaster.dag +++ b/dag/extdeps/os/ubuntu_seeded_install_media_remaster.dag @@ -117,14 +117,14 @@ fn install_media_remaster_program(artifact: UbuntuSeededInstallMediaArtifactRow) RawLine { text: "chmod -R u+w \"$work\"" }, RawLine { text: "mkdir -p \"$work/$NO_CLOUD\"" }, RawLine { text: "printf '%s' \"$USER_DATA\" > \"$work/$NO_CLOUD/user-data\"" }, - RawLine { text: "printf 'instance-id: gunbc-srv3-seeded\\n' > \"$work/$NO_CLOUD/meta-data\"" }, + RawLine { text: "printf 'instance-id: gunbc-%s-seeded\\n' \"$(basename \"$NO_CLOUD\")\" > \"$work/$NO_CLOUD/meta-data\"" }, RawLine { text: "grub_cfg=\"$work/boot/grub/grub.cfg\"" }, RawLine { text: "test -f \"$grub_cfg\" || { echo \"InstallMediaRemasterRefusal: missing grub.cfg at $grub_cfg\" >&2; exit 1; }" }, RawLine { text: "if ! grep -qF \"$GRUB_CMD\" \"$grub_cfg\"; then" }, RawLine { text: " sed -i \"s|\\(linux[[:space:]].*\\) ---|\\1 $GRUB_CMD ---|g\" \"$grub_cfg\"" }, RawLine { text: "fi" }, RawLine { text: "grep -qF \"$GRUB_CMD\" \"$grub_cfg\" || { echo \"InstallMediaRemasterRefusal: grub kernel cmdline not applied\" >&2; exit 1; }" }, - RawLine { text: "xorriso -as mkisofs -r -V \"UBUNTU_SEEDED\" -o \"$PART\" -graft-points /=\"$work\" >/dev/null" }, + RawLine { text: "xorriso -indev \"$STOCK\" -outdev \"$PART\" -boot_image any replay -volid \"UBUNTU_SEEDED\" -map \"$work/$NO_CLOUD\" \"/$NO_CLOUD\" -map \"$grub_cfg\" /boot/grub/grub.cfg -commit >/dev/null" }, RawLine { text: "mv \"$PART\" \"$FINAL\"" }, RawLine { text: "printf '%s' \"$HASH\" > \"$FINAL.gunbc-content-hash\"" }, RawLine { text: "echo \"InstallMediaRemasterReceipt: outcome=Remastered path=$FINAL content_hash=$HASH stock=$STOCK\"" }, diff --git a/dag/extdeps/storage/nbd.dag b/dag/extdeps/storage/nbd.dag new file mode 100644 index 00000000000..9d1647ab103 --- /dev/null +++ b/dag/extdeps/storage/nbd.dag @@ -0,0 +1,72 @@ +module extdeps.storage.nbd + +import std.types { NonEmptyStr, String, Int, Bool, List } +import extdeps.external_authority { ExternalAuthority } +import extdeps.uri { Uri, Https } +import extdeps.exec.command { ShellCommand } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https, + locator: "manpages.debian.org/bookworm/nbd-server/nbd-server.5.en.html", + } +} + +type NbdExport { + export_name: NonEmptyStr + backing_path: NonEmptyStr + port: Int + read_only: Bool +} + +data nbd_server_config_render_dissolution_trigger: String = "🟑 dissolve-on: nbd_server_config_text β€” the NbdExport is the typed authority; this renders it to nbd-server's INI config-file format (a rendered artifact like grub_config_render, not shell). The section/key layout is joined inline. DISSOLVES WHEN a shared std ini/config-file grammar (emit direction of extdeps.languages) renders NbdExport structurally instead of the inline join." + +fn nbd_bool_word(b: Bool) -> String { + if b { "true" } else { "false" } +} + +fn nbd_server_config_text(export: NbdExport) -> String { + join( + [ + "[generic]", + join([" port = ", to_string(export.port)], ""), + " allowlist = false", + join(["[", export.export_name as String, "]"], ""), + join([" exportname = ", export.backing_path as String], ""), + join([" readonly = ", nbd_bool_word(b: export.read_only)], ""), + "", + ], + "\n", + ) +} + +fn nbd_server_command(config_path: NonEmptyStr) -> ShellCommand { + ShellCommand { + argv: ["nbd-server", "-C", config_path as String], + } +} + +type NbdClientAttach { + server: NonEmptyStr + port: Int + export_name: NonEmptyStr + device: NonEmptyStr +} + +fn nbd_client_attach_command(attach: NbdClientAttach) -> ShellCommand { + ShellCommand { + argv: [ + "nbd-client", + "-N", attach.export_name as String, + attach.server as String, + to_string(attach.port), + attach.device as String, + ], + } +} + +fn nbd_client_disconnect_command(device: NonEmptyStr) -> ShellCommand { + ShellCommand { + argv: ["nbd-client", "-d", device as String], + } +} diff --git a/dag/extdeps/storage/samsung.dag b/dag/extdeps/storage/samsung.dag index 03533244f2d..a789bcc3ccf 100644 --- a/dag/extdeps/storage/samsung.dag +++ b/dag/extdeps/storage/samsung.dag @@ -30,3 +30,20 @@ data samsung_970_evo_plus_2tb_catalog: NvmeDeviceCatalogRow = samsung_970_evo_pl part_number: "MZ-V7S2T0BW", capacity_bytes: byte_size(2000398934016), ) + +fn samsung_970_evo(part_number: NonEmptyStr, capacity_bytes: ByteSize) -> NvmeDeviceCatalogRow { + NvmeDeviceCatalogRow { + part_number: part_number, + manufacturer: samsung, + capacity_bytes: capacity_bytes, + pcie_link: PcieLink { generation: Gen3, lane_count: 4 }, + form_factor: M2_2280, + sequential_read: bandwidth(3400000000), + sequential_write: bandwidth(2300000000), + } +} + +data samsung_970_evo_500gb_catalog: NvmeDeviceCatalogRow = samsung_970_evo( + part_number: "MZ-V7E500BW", + capacity_bytes: byte_size(500107862016), +) diff --git a/dag/gunbc/bmc_netboot_shell_boot.dag b/dag/gunbc/bmc_netboot_shell_boot.dag new file mode 100644 index 00000000000..ea72b61fa44 --- /dev/null +++ b/dag/gunbc/bmc_netboot_shell_boot.dag @@ -0,0 +1,55 @@ +module gunbc.bmc_netboot_shell_boot + +import std.types { NonEmptyStr, String, List } +import extdeps.uri { uri_wire } +import extdeps.firmware.uefi_shell { + UefiShellCommand, + Ifconfig, + IfconfigSetDhcp, + Ping, + Tftp, + ExecuteEfi, +} +import extdeps.firmware.kernel_cmdline { KernelCmdlineArg, KernelFlag, KernelKeyValue } +import extdeps.exec.command { ShellCommand } +import gunbc.bmc_netboot_serve { + BmcNetbootServePlan, + srv4_bmc_netboot_serve_plan, + bmc_netboot_seed_datasource, +} + +data shell_boot_local_kernel: NonEmptyStr = "fs0:\\vmlinuz" +data shell_boot_local_initrd: NonEmptyStr = "fs0:\\initrd" + +fn bmc_tftpd_serve_command(plan: BmcNetbootServePlan) -> ShellCommand { + ShellCommand { + argv: [ + join([plan.netboot_dir as String, "/busybox"], ""), + "udpsvd", "-E", "0.0.0.0", "69", + join([plan.netboot_dir as String, "/busybox"], ""), + "tftpd", "-r", plan.netboot_dir as String, + ], + } +} + +fn shell_boot_kernel_cmdline(plan: BmcNetbootServePlan) -> List { + [ + KernelKeyValue { key: "initrd", value: "\\initrd" }, + KernelKeyValue { key: "ip", value: "dhcp" }, + KernelKeyValue { key: "url", value: uri_wire(uri: plan.bulk_iso_url) }, + KernelFlag { name: "autoinstall" }, + KernelKeyValue { key: "ds", value: bmc_netboot_seed_datasource(plan: plan) }, + ] +} + +fn bmc_netboot_shell_boot_commands(plan: BmcNetbootServePlan) -> List { + [ + Ifconfig { action: IfconfigSetDhcp { interface: plan.boot_interface } }, + Ping { host: plan.bmc_host }, + Tftp { interface: plan.boot_interface, server: plan.bmc_host, remote_file: "vmlinuz", local_file: shell_boot_local_kernel }, + Tftp { interface: plan.boot_interface, server: plan.bmc_host, remote_file: "initrd", local_file: shell_boot_local_initrd }, + ExecuteEfi { path: shell_boot_local_kernel, cmdline: shell_boot_kernel_cmdline(plan: plan) }, + ] +} + +data srv4_shell_boot_commands: List = bmc_netboot_shell_boot_commands(plan: srv4_bmc_netboot_serve_plan) diff --git a/dag/gunbc/bmc_virtual_media.dag b/dag/gunbc/bmc_virtual_media.dag new file mode 100644 index 00000000000..f68512df65f --- /dev/null +++ b/dag/gunbc/bmc_virtual_media.dag @@ -0,0 +1,142 @@ +module gunbc.bmc_virtual_media + +import std.types { NonEmptyStr, String, Int, Bool, List } +import extdeps.exec.command { ShellCommand, shell_command_render } +import extdeps.storage.nbd { + NbdExport, + NbdClientAttach, + nbd_server_config_text, + nbd_server_command, + nbd_client_attach_command, + nbd_client_disconnect_command, +} +import extdeps.linux.usb_gadget { + UsbGadget, + MassStorageLun, + ConfigfsOp, + usb_gadget_setup_ops, + usb_gadget_setup_script, + usb_gadget_teardown_script, +} +import extdeps.bmc.virtual_media { + VirtualMediaIntent, + VirtualMediaSource, + VirtualMediaTransport, + ConfigfsGadgetParams, + ConfigfsGadgetTransport, +} + +type BmcVirtualMediaPlan { + bmc_host: NonEmptyStr + bmc_ssh_target: NonEmptyStr + srv1_host: NonEmptyStr + iso_path: NonEmptyStr + export_name: NonEmptyStr + nbd_port: Int + nbd_device: NonEmptyStr + udc_port: NonEmptyStr + nbd_config_path: NonEmptyStr +} + +data srv4_virtual_media_source: VirtualMediaSource = VirtualMediaSource { + iso_path: "/var/lib/gunbc/artifacts/ubuntu-24.04.3-live-server-srv4-seeded.iso", +} + +data srv4_virtual_media_intent: VirtualMediaIntent = VirtualMediaIntent { + bmc_host: "192.168.1.195", + source: srv4_virtual_media_source, + read_only: true, +} + +data srv4_configfs_params: ConfigfsGadgetParams = ConfigfsGadgetParams { + nbd_server_host: "192.168.1.185", + nbd_port: 10809, + export_name: "srv4iso", + nbd_device: "/dev/nbd0", + udc_port: "1e6a0000.usb-vhub:p3", +} + +data srv4_configfs_transport: VirtualMediaTransport = ConfigfsGadgetTransport { + params: srv4_configfs_params, +} + +fn bmc_virtual_media_plan_of(intent: VirtualMediaIntent, params: ConfigfsGadgetParams) -> BmcVirtualMediaPlan { + BmcVirtualMediaPlan { + bmc_host: intent.bmc_host, + bmc_ssh_target: join(["root@", intent.bmc_host as String], "") as NonEmptyStr, + srv1_host: params.nbd_server_host, + iso_path: intent.source.iso_path, + export_name: params.export_name, + nbd_port: params.nbd_port, + nbd_device: params.nbd_device, + udc_port: params.udc_port, + nbd_config_path: "/tmp/gunbc-nbd-srv4.conf", + } +} + +data srv4_bmc_virtual_media_plan: BmcVirtualMediaPlan = bmc_virtual_media_plan_of( + intent: srv4_virtual_media_intent, + params: srv4_configfs_params, +) + +fn plan_nbd_export(plan: BmcVirtualMediaPlan) -> NbdExport { + NbdExport { + export_name: plan.export_name, + backing_path: plan.iso_path, + port: plan.nbd_port, + read_only: true, + } +} + +fn plan_nbd_client_attach(plan: BmcVirtualMediaPlan) -> NbdClientAttach { + NbdClientAttach { + server: plan.srv1_host, + port: plan.nbd_port, + export_name: plan.export_name, + device: plan.nbd_device, + } +} + +fn plan_vm_gadget(plan: BmcVirtualMediaPlan) -> UsbGadget { + UsbGadget { + config_root: "/sys/kernel/config/usb_gadget", + name: "gunbcvm", + id_vendor: "0x1d6b", + id_product: "0x0104", + manufacturer: "gunbc", + product: "gunbc-virtual-media", + serial: "srv4-0001", + config_label: "c.1", + lun: MassStorageLun { + backing_device: plan.nbd_device, + cdrom: false, + read_only: true, + removable: true, + }, + udc: plan.udc_port, + } +} + +fn srv4_nbd_server_config_text() -> String { + nbd_server_config_text(export: plan_nbd_export(plan: srv4_bmc_virtual_media_plan)) +} + +fn srv4_nbd_server_command_text() -> String { + shell_command_render(command: nbd_server_command(config_path: srv4_bmc_virtual_media_plan.nbd_config_path)) +} + +fn srv4_nbd_client_attach_command_text() -> String { + shell_command_render(command: nbd_client_attach_command(attach: plan_nbd_client_attach(plan: srv4_bmc_virtual_media_plan))) +} + +fn srv4_vm_gadget_setup_ops() -> List { + usb_gadget_setup_ops(gadget: plan_vm_gadget(plan: srv4_bmc_virtual_media_plan)) +} + +fn srv4_vm_gadget_setup_script() -> String { + usb_gadget_setup_script(gadget: plan_vm_gadget(plan: srv4_bmc_virtual_media_plan)) +} + +fn srv4_vm_gadget_teardown_script() -> String { + usb_gadget_teardown_script(gadget: plan_vm_gadget(plan: srv4_bmc_virtual_media_plan)) +} diff --git a/dag/gunbc/ci_fleet.dag b/dag/gunbc/ci_fleet.dag index 4c165c95f71..e39e9bb4474 100644 --- a/dag/gunbc/ci_fleet.dag +++ b/dag/gunbc/ci_fleet.dag @@ -11,6 +11,7 @@ import gunbc.fleet_intent { srv1_host, srv2_host, srv3_host, + srv4_host, } import gunbc.fleet_intent { IsolationBoundary, PerJobFilesystem } import extdeps.cpu.ampere { altra_q6430_catalog } @@ -97,7 +98,19 @@ data srv3_offer: ComputeOffer = ComputeOffer { cost_quote: none, } -data fleet_intent_offers: List = [srv1_offer, srv2_offer, srv3_offer] +data srv4_offer: ComputeOffer = ComputeOffer { + provider: "gunbc-operator-srv4", + supply: ComputeSupplyFacts { + physical: srv4_host, + execution: fleet_intent_execution_surface(), + cost: Present { value: operator_owned_cost() }, + observed_performance: [], + }, + available_window: AvailabilityWindow { open: "always-open", close: none }, + cost_quote: none, +} + +data fleet_intent_offers: List = [srv1_offer, srv2_offer, srv3_offer, srv4_offer] data gunbc_ci_host_identity: HostIdentity = "gunbc-ci-fleet" diff --git a/dag/gunbc/deployed_intent_v1.dag b/dag/gunbc/deployed_intent_v1.dag index a3d55cbeaa9..f3822316dc2 100644 --- a/dag/gunbc/deployed_intent_v1.dag +++ b/dag/gunbc/deployed_intent_v1.dag @@ -3,8 +3,8 @@ module gunbc.deployed_intent_v1 import std.types { String } import std.disposition { Disposition, Terminal } import product.placement_supply { HostIdentity } -import gunbc.fleet_intent { ComputeHost, srv1_host, srv2_host, srv3_host } -import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3 } +import gunbc.fleet_intent { ComputeHost, srv1_host, srv2_host, srv3_host, srv4_host } +import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3, operator_host_srv4 } import extdeps.dns.domain_name { DnsLabel, parse_dns_label_raw } import std.error_primitives { Result, Ok, Err } @@ -63,6 +63,14 @@ fn deployed_intent_v1_srv3() -> DeployedIntentV1? { ) } +fn deployed_intent_v1_srv4() -> DeployedIntentV1? { + deployed_intent_v1_from_fleet_slot( + fleet_slot: operator_host_srv4, + target_host: srv4_host, + ) +} + data gunbc_srv1_deployed_intent_v1: DeployedIntentV1? = deployed_intent_v1_srv1() data gunbc_srv2_deployed_intent_v1: DeployedIntentV1? = deployed_intent_v1_srv2() data gunbc_srv3_deployed_intent_v1: DeployedIntentV1? = deployed_intent_v1_srv3() +data gunbc_srv4_deployed_intent_v1: DeployedIntentV1? = deployed_intent_v1_srv4() diff --git a/dag/gunbc/fleet_intent.dag b/dag/gunbc/fleet_intent.dag index be042a20488..33350fee520 100644 --- a/dag/gunbc/fleet_intent.dag +++ b/dag/gunbc/fleet_intent.dag @@ -1,8 +1,8 @@ module gunbc.fleet_intent -import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3 } +import gunbc.fleet_intent_network { operator_host_srv1, operator_host_srv2, operator_host_srv3, operator_host_srv4 } import extdeps.storage.western_digital { wd_black_sn850x_2tb_catalog } -import extdeps.storage.samsung { samsung_970_evo_plus_2tb_catalog } +import extdeps.storage.samsung { samsung_970_evo_plus_2tb_catalog, samsung_970_evo_500gb_catalog } import extdeps.storage.types { StorageDevice, nvme_storage_device, @@ -17,6 +17,7 @@ import extdeps.boards.asrock_rack { asrock_altrad8ud_1l2t_catalog } import extdeps.memory.types { MemoryKind, Dram } import extdeps.gpu.types { GpuFacts, AcceleratorFacts, GpuRuntime } import extdeps.toolchain.types { ToolchainCapability } +import extdeps.cache.sccache { sccache_toolchain_capability } import product.network_topology { NetworkLocality, NetworkInterface, NetworkAddressability, LatencyClass } import product.placement_supply { HostIdentity, PlacementSupplyRow } import std.baseboard { ServerBaseboard } @@ -200,6 +201,10 @@ data srv3_baseboard: ServerBaseboard = ServerBaseboard { catalog: asrock_altrad8ud_1l2t_catalog, bmc: BmcEndpoint { host: "192.168.1.192", protocol: Redfish }, } +data srv4_baseboard: ServerBaseboard = ServerBaseboard { + catalog: asrock_altrad8ud_1l2t_catalog, + bmc: BmcEndpoint { host: "192.168.1.195", protocol: Redfish }, +} data srv1_nvme0: StorageDevice = nvme_storage_device( catalog: wd_black_sn850x_2tb_catalog, @@ -213,6 +218,10 @@ data srv3_nvme0: StorageDevice = nvme_storage_device( catalog: samsung_970_evo_plus_2tb_catalog, persistence: Persistent, ) +data srv4_nvme0: StorageDevice = nvme_storage_device( + catalog: samsung_970_evo_500gb_catalog, + persistence: Persistent, +) data srv1_host: ComputeHost = ComputeHost { identity: operator_host_srv1, @@ -238,15 +247,23 @@ data srv3_host: ComputeHost = ComputeHost { storage: [srv3_nvme0], network_interfaces: [], } +data srv4_host: ComputeHost = ComputeHost { + identity: operator_host_srv4, + baseboard: Present { value: srv4_baseboard }, + processors: [CpuProcessor { cpu: fleet_intent_cpu }], + memory: [MemoryDevice { capacity: gibibyte_to_byte_size(g: gibibyte(128)), memory_kind: Dram, bandwidth: none }], + storage: [srv4_nvme0], + network_interfaces: [], +} -data fleet_intent_known_hosts: List = [srv1_host, srv2_host, srv3_host] +data fleet_intent_known_hosts: List = [srv1_host, srv2_host, srv3_host, srv4_host] fn fleet_intent_execution_surface() -> ExecutionSurface { ExecutionSurface { os: fleet_intent_os_surface, isolation: PerJobFilesystem, - container_runtime: none, - toolchains: [], + container_runtime: Present { value: Docker }, + toolchains: [sccache_toolchain_capability], mounted_storage: [], network: [], } diff --git a/dag/gunbc/fleet_intent_network.dag b/dag/gunbc/fleet_intent_network.dag index ec5588d780c..c42ea824a16 100644 --- a/dag/gunbc/fleet_intent_network.dag +++ b/dag/gunbc/fleet_intent_network.dag @@ -88,6 +88,14 @@ data srv4_bmc_endpoint: NetworkEndpoint = NetworkEndpoint { locality: Lan, } +data srv4_host_lan_endpoint: NetworkEndpoint = NetworkEndpoint { + label: "srv4-host-lan", + host: Present { value: operator_host_srv4 }, + address: "192.168.1.196", + role: HostOperatingSystem, + locality: Lan, +} + data srv1_tailscale_endpoint: NetworkEndpoint = NetworkEndpoint { label: "srv1-tailscale", host: Present { value: operator_host_srv1 }, @@ -123,6 +131,7 @@ data fleet_intent_network: NetworkTopology = NetworkTopology { operator_lan_gateway_endpoint, srv1_host_lan_endpoint, srv2_host_lan_endpoint, + srv4_host_lan_endpoint, srv1_bmc_endpoint, srv2_bmc_endpoint, srv3_bmc_endpoint, diff --git a/dag/gunbc/runner_host_deploy.dag b/dag/gunbc/runner_host_deploy.dag new file mode 100644 index 00000000000..6d28e135eed --- /dev/null +++ b/dag/gunbc/runner_host_deploy.dag @@ -0,0 +1,109 @@ +module gunbc.runner_host_deploy + +import std.types { NonEmptyStr, String, Int, List } +import extdeps.exec.command { ShellCommand, shell_command_render } +import extdeps.cloud.gcp.secret_ref { SecretRef, HashPending } +import gunbc.fleet_ssh_access { fleet_ssh_secrets_project } +import extdeps.cache.sccache { SccacheBinaryRelease, sccache_release_0_15_0, sccache_install_script } +import extdeps.container.docker_ce { docker_ce_apt_repo_noble_arm64, docker_ce_packages, docker_ce_repo_install_script } + +data runner_host_deploy_citation: String = "Β§3 cite-upstream, do-not-re-coin: GitHub Actions runner host provisioning is realized by the ctrl installer at ctrl:scripts/session-dashboard/host/ (deploy-runner-hosts.sh and install-actions-runner.sh) β€” the single battle-tested rotation point (GitHub App key distribution, jit-runner.sh ephemeral wrapper, actions-runner service template, ghrunner de-priv job user plus rootless docker for public-PR isolation, per-slot cgroup caps). RunnerHostDeploy models the typed INTERFACE (which host, label, runner user, de-priv job user, App-key secret, slot count); the installer is the bound realization HANDLER, one of N transports, NOT re-implemented in .dag. DISSOLVES WHEN host_effect_apply binds RunnerHostDeploy through a transport handler that invokes the cited installer plus per-slot provisioning without a hand-run shell step." + +data runner_host_deploy_upstream_installer: NonEmptyStr = "ctrl:scripts/session-dashboard/host/install-actions-runner.sh" +data runner_host_deploy_upstream_fanout: NonEmptyStr = "ctrl:scripts/session-dashboard/host/deploy-runner-hosts.sh" + +type GithubAppRunnerConfig { + app_id: NonEmptyStr + installation_id: NonEmptyStr + org: NonEmptyStr + app_pem_secret: SecretRef +} + +data gunbc_ci_github_app: GithubAppRunnerConfig = GithubAppRunnerConfig { + app_id: "2653532", + installation_id: "104134109", + org: "gunb-ai", + app_pem_secret: SecretRef { + project: fleet_ssh_secrets_project, + secret: "ci-github-app-private-key", + version: "latest", + hash_state: HashPending, + }, +} + +type RunnerHostDeploy { + host_label: NonEmptyStr + ssh_target: NonEmptyStr + runner_user: NonEmptyStr + job_user: NonEmptyStr + app: GithubAppRunnerConfig + runner_count: Int +} + +data srv4_runner_count_disk_cap_note: String = "srv4 caps at 5 runners (not the RAM-derived 10 the placement solver gives it): its NVMe is a Samsung 970 EVO 500GB vs 2TB on srv1-3, and each ephemeral runner keeps a _work tree + caches. Disk, not RAM, is srv4's binding runner-count constraint. dissolve-on: fold a per-host disk-headroom term into runner_deployment_plan so the cap is solver-derived, not a hand-pin here." + +data srv4_runner_host_deploy: RunnerHostDeploy = RunnerHostDeploy { + host_label: "srv4", + ssh_target: "briansrls@192.168.1.196", + runner_user: "briansrls", + job_user: "ghrunner", + app: gunbc_ci_github_app, + runner_count: 5, +} + +data runner_host_prereq_note: String = "Execution-surface prereqs the runner host must satisfy (gunbc.fleet_intent.fleet_intent_execution_surface now declares container_runtime: Docker + toolchains: [sccache], so the surface no longer lies). These two scripts provision that surface on a fresh host BEFORE install-actions-runner.sh: docker_ce (the rootless-docker apt prereq the ctrl installer assumes) then sccache (the pinned binary install-ctrl-sccache.sh assumes on PATH). Each cites its upstream; the shell leaves carry dissolution triggers in their extdeps modules." + +fn runner_host_docker_provision_script() -> String { + docker_ce_repo_install_script(repo: docker_ce_apt_repo_noble_arm64, packages: docker_ce_packages) +} + +fn runner_host_sccache_provision_script(arch: NonEmptyStr, sha256: NonEmptyStr) -> String { + sccache_install_script(rel: sccache_release_0_15_0, arch: arch, sha256: sha256) +} + +fn runner_host_sccache_provision_aarch64() -> String { + sccache_install_script( + rel: sccache_release_0_15_0, + arch: "aarch64", + sha256: sccache_release_0_15_0.sha256_aarch64, + ) +} + +fn runner_host_installer_command(deploy: RunnerHostDeploy, installer_path: NonEmptyStr, pem_src_path: NonEmptyStr) -> ShellCommand { + ShellCommand { + argv: [ + "env", + join(["CTRL_RUNNER_HOST_LABEL=", deploy.host_label as String], ""), + join(["CTRL_RUNNER_USER=", deploy.runner_user as String], ""), + join(["CTRL_RUNNER_JOB_USER=", deploy.job_user as String], ""), + join(["CTRL_RUNNER_APP_PEM_SRC=", pem_src_path as String], ""), + "sudo", "-E", "bash", installer_path as String, + ], + } +} + +fn runner_instance_name(deploy: RunnerHostDeploy, index: Int) -> String { + let two = if index < 10 { join(["0", to_string(index)], "") } else { to_string(index) } + join([deploy.host_label as String, "-", two], "") +} + +fn runner_index_seq(n: Int) -> List { + if n <= 0 { + [] + } else { + append(runner_index_seq(n: n - 1), items: [n]) + } +} + +fn runner_instance_names(deploy: RunnerHostDeploy) -> List { + map(runner_index_seq(n: deploy.runner_count), i => runner_instance_name(deploy: deploy, index: i)) +} + +fn runner_enable_command(deploy: RunnerHostDeploy) -> ShellCommand { + ShellCommand { + argv: append( + ["sudo", "systemctl", "enable", "--now"], + items: map(runner_instance_names(deploy: deploy), n => join(["actions-runner@", n, ".service"], "")), + ), + } +} diff --git a/dag/test/claim/bmc_netboot_serve_witness_test.dag b/dag/test/claim/bmc_netboot_serve_witness_test.dag index a34da706734..44ce8c6144a 100644 --- a/dag/test/claim/bmc_netboot_serve_witness_test.dag +++ b/dag/test/claim/bmc_netboot_serve_witness_test.dag @@ -19,7 +19,7 @@ import gunbc.bmc_netboot_serve { test fn srv4_grub_cfg_boots_mirror_bulk_with_bmc_seed() -> Bool { let cfg = bmc_netboot_grub_cfg_text(plan: srv4_bmc_netboot_serve_plan) - string_contains(s: cfg, pattern: "linux /vmlinuz ip=dhcp url=https://cdimage.ubuntu.com/ubuntu/releases/24.04/release/ubuntu-24.04.3-live-server-arm64.iso autoinstall ds=nocloud-net;s=http://192.168.1.195:8080/nocloud/ ---") + string_contains(s: cfg, pattern: "linux /vmlinuz ip=dhcp url=https://cdimage.ubuntu.com/ubuntu/releases/24.04/release/ubuntu-24.04.3-live-server-arm64.iso autoinstall \"ds=nocloud-net;s=http://192.168.1.195:8080/nocloud/\" ---") && string_contains(s: cfg, pattern: "initrd /initrd") && string_contains(s: cfg, pattern: "set timeout=1") } diff --git a/dag/test/claim/bmc_netboot_shell_boot_witness_test.dag b/dag/test/claim/bmc_netboot_shell_boot_witness_test.dag new file mode 100644 index 00000000000..5724482ef31 --- /dev/null +++ b/dag/test/claim/bmc_netboot_shell_boot_witness_test.dag @@ -0,0 +1,26 @@ +module test.claim.bmc_netboot_shell_boot + +import std.logic { Bool } +import std.types { list_length } +import extdeps.firmware.uefi_shell { uefi_shell_command_line } +import extdeps.exec.command { shell_command_render } +import gunbc.bmc_netboot_shell_boot { + srv4_shell_boot_commands, + bmc_tftpd_serve_command, +} +import gunbc.bmc_netboot_serve { srv4_bmc_netboot_serve_plan } + +test fn srv4_shell_boot_sequence_direct_kernel() -> Bool { + let all = fold(srv4_shell_boot_commands, init: "", f: (acc, c) => join([acc, uefi_shell_command_line(cmd: c), "\n"], "")) + string_contains(s: all, pattern: "ifconfig -s eth2 dhcp") + && string_contains(s: all, pattern: "ping 192.168.1.195") + && string_contains(s: all, pattern: "tftp -i eth2 192.168.1.195 vmlinuz fs0:\\vmlinuz") + && string_contains(s: all, pattern: "tftp -i eth2 192.168.1.195 initrd fs0:\\initrd") + && string_contains(s: all, pattern: "fs0:\\vmlinuz initrd=\\initrd ip=dhcp url=https://cdimage.ubuntu.com") + && string_contains(s: all, pattern: "autoinstall ds=nocloud-net;s=http://192.168.1.195:8080/nocloud/") + && list_length(items: srv4_shell_boot_commands) == 5 +} + +test fn srv4_tftpd_serve_command_grounded() -> Bool { + string_contains(s: shell_command_render(command: bmc_tftpd_serve_command(plan: srv4_bmc_netboot_serve_plan)), pattern: "'/tmp/netboot/busybox' 'udpsvd' '-E' '0.0.0.0' '69' '/tmp/netboot/busybox' 'tftpd' '-r' '/tmp/netboot'") +} diff --git a/dag/test/claim/bmc_virtual_media_witness_test.dag b/dag/test/claim/bmc_virtual_media_witness_test.dag new file mode 100644 index 00000000000..c6b9a444bd9 --- /dev/null +++ b/dag/test/claim/bmc_virtual_media_witness_test.dag @@ -0,0 +1,51 @@ +module test.claim.bmc_virtual_media + +import std.logic { Bool } +import std.types { list_length } +import extdeps.linux.usb_gadget { ConfigfsOp, ConfigfsWriteAttr, ConfigfsMkdir, ConfigfsSymlink, ConfigfsRemoveLink, ConfigfsRmdir } +import gunbc.bmc_virtual_media { + srv4_nbd_server_config_text, + srv4_nbd_client_attach_command_text, + srv4_vm_gadget_setup_script, + srv4_vm_gadget_setup_ops, +} + +fn configfs_write_value_for(ops: List, path_suffix: String) -> String { + fold(ops, init: "", f: (acc, op) => match op { + ConfigfsWriteAttr { path: p, value: v } => + if string_contains(s: p as String, pattern: path_suffix) { v } else { acc } + ConfigfsMkdir { path: _ } => acc + ConfigfsSymlink { target: _, link: _ } => acc + ConfigfsRemoveLink { link: _ } => acc + ConfigfsRmdir { path: _ } => acc + }) +} + +test fn srv4_nbd_export_read_only_seeded_iso() -> Bool { + let cfg = srv4_nbd_server_config_text() + string_contains(s: cfg, pattern: "port = 10809") + && string_contains(s: cfg, pattern: "[srv4iso]") + && string_contains(s: cfg, pattern: "exportname = /var/lib/gunbc/artifacts/ubuntu-24.04.3-live-server-srv4-seeded.iso") + && string_contains(s: cfg, pattern: "readonly = true") +} + +test fn srv4_nbd_client_attaches_named_export() -> Bool { + string_contains(s: srv4_nbd_client_attach_command_text(), pattern: "'nbd-client' '-N' 'srv4iso' '192.168.1.185' '10809' '/dev/nbd0'") +} + +test fn srv4_gadget_is_readonly_cdrom_on_nbd0() -> Bool { + let s = srv4_vm_gadget_setup_script() + string_contains(s: s, pattern: "mkdir -p /sys/kernel/config/usb_gadget/gunbcvm") + && string_contains(s: s, pattern: "printf '%s' '/dev/nbd0' > /sys/kernel/config/usb_gadget/gunbcvm/functions/mass_storage.0/lun.0/file") + && string_contains(s: s, pattern: "printf '%s' '0' > /sys/kernel/config/usb_gadget/gunbcvm/functions/mass_storage.0/lun.0/cdrom") + && string_contains(s: s, pattern: "printf '%s' '1e6a0000.usb-vhub:p3' > /sys/kernel/config/usb_gadget/gunbcvm/UDC") +} + +test fn srv4_gadget_ops_are_typed_not_scraped_from_shell() -> Bool { + let ops = srv4_vm_gadget_setup_ops() + configfs_write_value_for(ops: ops, path_suffix: "/lun.0/file") == "/dev/nbd0" + && configfs_write_value_for(ops: ops, path_suffix: "/lun.0/cdrom") == "0" + && configfs_write_value_for(ops: ops, path_suffix: "/lun.0/ro") == "1" + && configfs_write_value_for(ops: ops, path_suffix: "gunbcvm/UDC") == "1e6a0000.usb-vhub:p3" + && list_length(items: ops) == 16 +} diff --git a/dag/test/claim/grub_cmdline_quoting_witness_test.dag b/dag/test/claim/grub_cmdline_quoting_witness_test.dag new file mode 100644 index 00000000000..56b623a06a4 --- /dev/null +++ b/dag/test/claim/grub_cmdline_quoting_witness_test.dag @@ -0,0 +1,22 @@ +module test.claim.grub_cmdline_quoting + +import std.logic { Bool } +import extdeps.bootloader.grub { GrubMenuEntry, grub_menuentry_render } +import extdeps.firmware.kernel_cmdline { KernelKeyValue, KernelFlag } + +test fn grub_quotes_semicolon_cmdline_args_only() -> Bool { + let entry = GrubMenuEntry { + title: "gunbc autoinstall", + linux_image: "/casper/vmlinuz", + linux_cmdline: [ + KernelKeyValue { key: "ip", value: "dhcp" }, + KernelKeyValue { key: "ds", value: "nocloud;s=/cdrom/nocloud/srv4/" }, + KernelFlag { name: "autoinstall" }, + ], + initrd_image: "/casper/initrd", + } + let r = grub_menuentry_render(entry: entry) + string_contains(s: r, pattern: "\"ds=nocloud;s=/cdrom/nocloud/srv4/\"") + && string_contains(s: r, pattern: " ip=dhcp ") + && (string_contains(s: r, pattern: "\"ip=dhcp\"") == false) +} diff --git a/dag/test/claim/runner_host_deploy_witness_test.dag b/dag/test/claim/runner_host_deploy_witness_test.dag new file mode 100644 index 00000000000..faccea2ef1e --- /dev/null +++ b/dag/test/claim/runner_host_deploy_witness_test.dag @@ -0,0 +1,62 @@ +module test.claim.runner_host_deploy + +import std.logic { Bool } +import std.types { list_length } +import extdeps.exec.command { shell_command_render } +import gunbc.runner_host_deploy { + srv4_runner_host_deploy, + gunbc_ci_github_app, + runner_host_installer_command, + runner_instance_names, + runner_enable_command, + runner_host_docker_provision_script, + runner_host_sccache_provision_script, +} +import gunbc.fleet_intent { fleet_intent_execution_surface } +import extdeps.toolchain.types { ToolchainCapability } + +test fn srv4_docker_prereq_cites_official_repo() -> Bool { + let s = runner_host_docker_provision_script() + string_contains(s: s, pattern: "download.docker.com/linux/ubuntu/gpg") + && string_contains(s: s, pattern: "deb [arch=arm64 signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu noble stable") + && string_contains(s: s, pattern: "docker-ce-rootless-extras") +} + +test fn srv4_sccache_prereq_pins_version_and_digest() -> Bool { + let s = runner_host_sccache_provision_script(arch: "aarch64", sha256: "3a6a3712b49da3d263bf2d30d702de4302793016019e800bfb81c0c69401d8f8") + string_contains(s: s, pattern: "github.com/mozilla/sccache/releases/download/v0.15.0/sccache-v0.15.0-aarch64-unknown-linux-musl.tar.gz") + && string_contains(s: s, pattern: "3a6a3712b49da3d263bf2d30d702de4302793016019e800bfb81c0c69401d8f8 /tmp/sccache.tar.gz") + && string_contains(s: s, pattern: "/tmp/sccache-v0.15.0-aarch64-unknown-linux-musl/sccache' /usr/local/bin/sccache") +} + +test fn fleet_surface_no_longer_lies_declares_sccache() -> Bool { + let surf = fleet_intent_execution_surface() + fold(surf.toolchains, init: false, f: (acc, t) => acc || (t.name as String == "sccache")) +} + +test fn srv4_runner_installer_command_cites_installer_with_env() -> Bool { + let c = shell_command_render(command: runner_host_installer_command( + deploy: srv4_runner_host_deploy, + installer_path: "/tmp/install-actions-runner.sh", + pem_src_path: "/tmp/app.pem", + )) + string_contains(s: c, pattern: "'CTRL_RUNNER_HOST_LABEL=srv4'") + && string_contains(s: c, pattern: "'CTRL_RUNNER_JOB_USER=ghrunner'") + && string_contains(s: c, pattern: "'CTRL_RUNNER_APP_PEM_SRC=/tmp/app.pem'") + && string_contains(s: c, pattern: "'sudo' '-E' 'bash' '/tmp/install-actions-runner.sh'") +} + +test fn srv4_runner_app_from_secret_manager() -> Bool { + gunbc_ci_github_app.org as String == "gunb-ai" + && gunbc_ci_github_app.app_pem_secret.secret as String == "ci-github-app-private-key" +} + +test fn srv4_enables_five_named_runner_instances() -> Bool { + let names = runner_instance_names(deploy: srv4_runner_host_deploy) + let enable = shell_command_render(command: runner_enable_command(deploy: srv4_runner_host_deploy)) + list_length(items: names) == 5 + && string_contains(s: enable, pattern: "'actions-runner@srv4-01.service'") + && string_contains(s: enable, pattern: "'actions-runner@srv4-05.service'") + && (string_contains(s: enable, pattern: "srv4-06") == false) + && string_contains(s: enable, pattern: "'sudo' 'systemctl' 'enable' '--now'") +} diff --git a/dag/test/claim/virtual_media_unification_witness_test.dag b/dag/test/claim/virtual_media_unification_witness_test.dag new file mode 100644 index 00000000000..a2f0e281937 --- /dev/null +++ b/dag/test/claim/virtual_media_unification_witness_test.dag @@ -0,0 +1,28 @@ +module test.claim.virtual_media_unification + +import std.logic { Bool } +import std.types { String } +import extdeps.bmc.virtual_media { + VirtualMediaIntent, + NbdProxyWebsocatParams, + NbdProxyWebsocatTransport, + virtual_media_transport_kind, +} +import gunbc.bmc_virtual_media { + srv4_virtual_media_intent, + srv4_configfs_transport, + srv4_configfs_params, + bmc_virtual_media_plan_of, +} +import extdeps.bmc.webui.nbd_proxy_serve { nbd_proxy_serve_intent_of_virtual_media } + +test fn one_intent_two_transports_single_authority() -> Bool { + let intent = srv4_virtual_media_intent + let cfg_plan = bmc_virtual_media_plan_of(intent: intent, params: srv4_configfs_params) + let proxy_params = NbdProxyWebsocatParams { session_token: "TOK", ws_path: "/nbd/0", local_nbd_port: 10809 } + let proxy_intent = nbd_proxy_serve_intent_of_virtual_media(vm: intent, params: proxy_params) + (cfg_plan.iso_path as String == proxy_intent.iso_path as String) + && (cfg_plan.bmc_host as String == proxy_intent.bmc_host as String) + && (virtual_media_transport_kind(transport: srv4_configfs_transport) == "configfs-gadget") + && (virtual_media_transport_kind(transport: NbdProxyWebsocatTransport { params: proxy_params }) == "nbdproxy-websocat") +} From a11696a34b53283a66545beb67d42b3f47d5c2be Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 22 Jul 2026 22:39:05 +0000 Subject: [PATCH 26/58] plans: fleet subsumption manual-gaps receipt (srv4 hand-steps -> modeling backlog) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modeled plan doc capturing every step of srv4 install+subsumption that was done by hand β€” each a modeling gap. Two families: (A) credential handling (every temp credential file = a missing MaterializedSecret lifecycle; reach-secrets ADC; SecretRef liveness after the stale App-key 401) and (B) provisioning (runner SLOT provisioning = the width-INCREASE gap fleet-converge.sh already names, documented in full: pinned ActionsRunnerRelease + per-slot dir seed + count reconcile, all hand-unrolled this time; fleet runner-user; fresh-host prereqs; configfs virtual-media install actuator + media-detach lifecycle). Each item carries acceptance tier + RED control; the plan's dissolution trigger retires item-by-item, fully gone when srv5 subsumes with zero hand-run shell. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/fleet_subsumption_manual_gaps.dag | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 dag/gunbc/plans/fleet_subsumption_manual_gaps.dag diff --git a/dag/gunbc/plans/fleet_subsumption_manual_gaps.dag b/dag/gunbc/plans/fleet_subsumption_manual_gaps.dag new file mode 100644 index 00000000000..dbcf7e2bf19 --- /dev/null +++ b/dag/gunbc/plans/fleet_subsumption_manual_gaps.dag @@ -0,0 +1,157 @@ +module gunbc.plans.fleet_subsumption_manual_gaps + +import std.types { String, List } +import std.markdown { MarkdownBlock } +import gunbc.plan { Plan, DissolutionTrigger, HasTrigger } +import gunbc.plans.md_helpers { h2, h3, p, li, ul, ol, quote } + +fn fsmg_intro() -> List { + [ + p(text: "Home: fleet subsumption (compute fabric). This doc is the receipt of the MANUAL steps taken to install and subsume srv4 (2026-07-22) that the modeling workflow did NOT cover. Each is a modeling gap: a step done by hand is a step the fleet cannot yet reproduce, verify, or retract through the model. The bar (per fleet_acceptance_criteria): a host must be install-able and subsume-able through the model with a read-back receipt, with zero hand-run shell. Until every item here dissolves, subsuming the next host (srv5) repeats the manual toil."), + quote(blocks: [p(text: "A step performed by hand is not modeled. It is not that the model is wrong; it is that the model is SILENT, and silence is the Β§5 fail-open of a workflow: the operator performs the missing step and the gap never surfaces as a diagnostic.")]), + p(text: "Two families: (A) credential handling β€” every temp credential file materialized on disk maps to a missing modeled secret-lifecycle effect; and (B) provisioning steps the model does not yet drive. Priority order at the end."), + ] +} + +fn fsmg_secret_materialization() -> List { + [ + h2(text: "A1 β€” Secret materialization lifecycle (the systemic credential gap)"), + p(text: "The model has SecretRef (a typed REFERENCE to a Secret Manager secret) but NOT the runtime effect that turns a reference into a usable credential and cleans it up. That effect, done by hand for every credential this session, is: fetch the SecretRef from Secret Manager, materialize it to a 0600 temp file or an env var, bind it to the transport that needs it, and shred it on completion."), + p(text: "Why materialization is unavoidable at point-of-use: ssh -i needs a key FILE; sudo -S and sshpass need the secret as a STRING; ipmitool needs a password. None can read a credential straight from Secret Manager. So the file/env is inherent β€” but its LIFECYCLE (fetch, scope, shred) must be modeled, not improvised. The ctrl deploy-runner-hosts.sh already does exactly this pattern (scp the PEM to a temp, use it, shred) β€” evidence the shape is known; it is just not a first-class modeled effect."), + h3(text: "Manual instances this session (each is one missing MaterializedSecret)"), + ul(items: [ + li(text: "fleet-automation-ssh-key (Secret Manager) -> /tmp/fleet_key (0600) -> ssh -i, for every ssh/scp to srv4."), + li(text: "host-srv4-console (Secret Manager) -> /tmp/srv4_console_pw -> sudo -S, to bootstrap briansrls before passwordless sudo existed."), + li(text: "bmc-srv4-admin (Secret Manager) -> /tmp/bmcpw.env -> ipmitool / sshpass, for BMC + SOL."), + li(text: "ci-github-app-private-key (Secret Manager) -> /tmp/app.pem -> scp to host -> install-actions-runner.sh, for runner registration."), + ]), + h3(text: "Acceptance (target tier T2 wired, then T4 live)"), + ol(items: [ + li(text: "A MaterializedSecret effect: given a SecretRef and a transport binding (SshKeyFile, EnvVar, StdinPassword), it fetches, writes 0600 to a scoped temp, yields a handle, and shreds on scope exit β€” shred is by construction (the handle drop), not a remembered step."), + li(text: "Every fleet transport that needs a secret (ssh, sudo, ipmitool, scp-of-key) consumes a MaterializedSecret, never a raw path β€” proof by consumption: remove the effect and the transport witness fails to resolve."), + li(text: "RED: a scope that exits without shred is a type error or a failing hygiene witness; a transport handed a raw temp path (not a MaterializedSecret handle) reds."), + ]), + ] +} + +fn fsmg_reach_secrets() -> List { + [ + h2(text: "A2 β€” Reach-secrets bootstrap (ADC / workload identity)"), + p(text: "The GCP access token is pasted by the operator every ~1h because the workflow has NO modeled way to authenticate to Secret Manager itself. This is the credential that unlocks A1, so it is the deeper enabler. It is already a DeclaredGap in host_standup (P0:reach-secrets-network; note: full WIF SecretAccessGrant modeling is follow-up) and in fleet_ssh_access (secretAccessor bound out-of-band 2026-07-21; full WIF SecretAccessGrant modeling is follow-up)."), + h3(text: "Acceptance"), + ol(items: [ + li(text: "A modeled SecretAccessGrant / ADC identity the workflow assumes on the host it runs from (the dashboard/orchestrator), so reaching Secret Manager needs no operator-pasted token."), + li(text: "The grant is least-privilege and per-secret (mirrors the fleet-automation SA secretAccessor binding already done by hand), modeled as the accessor of each SecretRef."), + li(text: "RED: a workflow step that reads a SecretRef the running identity is not granted reds at plan time, not at a runtime 403."), + ]), + ] +} + +fn fsmg_secret_liveness() -> List { + [ + h2(text: "A3 β€” SecretRef liveness (existence is not validity)"), + p(text: "The model cited ci-github-app-private-key as truth, but Secret Manager held a REVOKED key (fingerprint 0c7a15..., deleted from the GitHub App) while the live key (64a8af...) lived only on the hosts. srv4 registration 401'd; I diagnosed and reconciled by hand (added the live key as a new SM version, disabled the dead one). The model verifies a SecretRef EXISTS but never that it is the credential the upstream (GitHub) actually ACCEPTS."), + p(text: "This is a Β§5 gap: a rotation that updates the hosts but skips Secret Manager leaves the cited authority stale, and nothing catches it until the next host deploy 401s at runtime."), + h3(text: "Acceptance"), + ol(items: [ + li(text: "A liveness assertion on the App-key SecretRef: the key must mint a valid GitHub App JWT / installation token (a cheap, read-only probe), fail-closed if it does not."), + li(text: "The single rotation point is the SecretRef itself: hosts materialize FROM the SecretRef (A1), so host and Secret Manager cannot diverge by construction. Retire the canonical-key-on-dashboard-host copy that deploy-runner-hosts.sh uses today."), + li(text: "RED: point the SecretRef at a revoked key -> the liveness probe reds before any host is touched."), + ]), + ] +} + +fn fsmg_slot_provisioning() -> List { + [ + h2(text: "B1 β€” Runner slot provisioning (the width-INCREASE gap) β€” DONE ENTIRELY BY HAND"), + p(text: "This is the one that should never have been hand-work. RunnerHostDeploy (this PR) models the runner ENABLE step (runner_enable_command -> systemctl enable --now actions-runner@srv4-01..05) but NOT the provisioning of the per-slot runner directories those units point at. The whole slot install was manual. fleet-converge.sh already names this exact gap: width INCREASE $current->N needs slot provisioning (rsync from -01); run the install path -- converge only drains for DECREASE. So DECREASE (drain runners) is modeled/converged; INCREASE (add runners) is not."), + h3(text: "Exactly what I did by hand (the receipt)"), + ol(items: [ + li(text: "Located the pinned runner artifact on srv1: /opt/actions-runner-cache/actions-runner-linux-arm64-2.334.0.tar.gz (runner v2.334.0). Its version + digest are NOT modeled β€” there is no ActionsRunnerRelease row analogous to the SccacheBinaryRelease this PR added (github.com/actions/runner/releases)."), + li(text: "sudo cp the tarball to /tmp, scp'd it srv1 -> srv4 (~132M). No modeled artifact staging."), + li(text: "sudo install -d -m 755 -o briansrls -g briansrls /opt/actions-runner and /opt/actions-runner-cache β€” the install-actions-runner.sh installer ASSUMES /opt/actions-runner exists (it writes jit-runner.sh into it) and fails on a fresh host without it. The base directory + its ownership are unmodeled."), + li(text: "For each of srv4-01..05: sudo rm -rf the dir, sudo mkdir, sudo tar xzf the tarball into it, sudo chown -R ghrunner:ghrunner. Five hand-unrolled extractions. The count (5) came from a decision, not from the modeled runner_deployment_plan feeding provisioning β€” the plan derives 10 (RAM-based) and I hand-capped to 5 for the 500G disk. The ownership split (briansrls owns /opt/actions-runner, ghrunner owns each slot dir) is the de-priv posture install-actions-runner.sh chown loop expects, but the DIR CREATION it does not do."), + li(text: "Only THEN did the modeled part run: systemctl daemon-reload + runner_enable_command."), + ]), + h3(text: "What the model is missing"), + ul(items: [ + li(text: "ActionsRunnerRelease: a cited row (version 2.334.0, per-arch download URL + sha256, github.com/actions/runner/releases) β€” the runner tarball is a pinned upstream artifact and must be modeled + verified exactly like SccacheBinaryRelease, not copied from a neighbor host by hand."), + li(text: "RunnerSlot provisioning as a modeled effect: fetch/verify the release, ensure the /opt/actions-runner base dir + ownership, seed N slot dirs (extract per instance), set the ghrunner ownership, THEN enable. RunnerHostDeploy currently starts at enable and assumes the dirs exist."), + li(text: "Slot-COUNT reconcile (three-way, like host_converge): desired N vs observed slot dirs -> add the delta (seed new dirs) or drain the delta. Today only drain (DECREASE) is converged; INCREASE is the hand step. runner_deployment_plan derives the count but nothing consumes it to PROVISION β€” only to enable."), + li(text: "The disk-headroom term that made me hand-cap srv4 to 5: runner_deployment_plan derives count from RAM only, so it gave srv4 10; a per-host disk-headroom fold would make the 5 solver-derived, not a hand-pin (this PR leaves a note in runner_host_deploy but the solver still says 10)."), + ]), + h3(text: "Acceptance (target tier T4 β€” live apply + independent read-back)"), + ol(items: [ + li(text: "A modeled RunnerSlotProvision consumes runner_deployment_plan's count for a host, ensures the ActionsRunnerRelease is present+verified, seeds exactly N slot dirs from it with correct ownership, and enables them β€” one fold, no hand-unrolled per-slot shell."), + li(text: "Live: run it on a real host; an INDEPENDENT read (ls -d /opt/actions-runner/