diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index b1b491ee729..e4b7537965d 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -13,7 +13,7 @@ on: mode: description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; runner_browser_toolchain_converge installs the declared Playwright/Chromium toolchain (apt host libraries as the administrator, digest-pinned node, Playwright and Chromium archives into the job user's root) on the selected host, which must be in the pool that runs the floor job, and refuses unless every digest, version and host library reads back and headless Chromium renders a local page; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; spark_v41_engram_differential compares upstream's Engram lookup kernel with the design-B file-backed lookup over sampled real rows of every row store, byte for byte, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; spark_v41_group_a_launch reads the production image back under its tag on every Group A host, reads its registry inside its digest and every host's occupancy, and only when Group A is suspended for this candidate with every host held and vacant stages the Engram manifest and applies the V4.1 four-rank arm as one transaction at the capacity measurement's shape -- the target names only the session host; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit and the gunbc-microvm-slot-reserve broker unit on srv1 and starts neither; microvm_slot_reserve starts that broker unit once on srv1, which reserves the shakedown cell through the fabric broker route from inside the root process (slot, demand and offer derived from the model, never inputs), and uploads that invocation's reservation receipt, failing unless the reservation committed; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, spark_v41_group_a_launch, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_reserve, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, runner_browser_toolchain_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, mtcollins1_census_member_readback, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, spark_v41_group_a_launch, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_reserve, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, runner_browser_toolchain_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_ui_bundle_observe, mtcollins1_census_image_publish, mtcollins1_census_member_readback, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -858,6 +858,38 @@ jobs: retention-days: 30 if: github.event.inputs.mode == 'mtcollins1_fan_observe' timeout-minutes: 10 + - name: "Mt. Collins UI bundle: firmware revision, then the served source.min.js and its sha256 (reads only)" + id: mtcollins1_ui_bundle_observe + run: |- + # 🟡 dissolve-on: gunbc_ci_mtcollins1_fan_observe_invoke - orch-emitted foreign-executor (GitHub Actions run:) credential runner: WIF access token by env, ONE pinned BMC secret version fetched over curl through the shared gunbc_ci_mtcollins1_bmc_credential_fetch_steps, a 0600 file under RUNNER_TEMP with a trap armed BEFORE the credential touches disk. The pipeline steps are modeled but each step body is a shell string built by concat, so the transport itself remains hand-shell; DISSOLVES WHEN the runner is authored as v2.extdeps.languages.bash_build nodes (typed, grammar-quoted words; command substitution; assignment; pipe; if/test; redirect; || true) emitted by v2.workflow.bash_emit bash_emit_stmts -- a route AVAILABLE today (#12422 retired the sibling pack runner on it), so this migration is unauthored, not blocked on a missing capability; set -euo pipefail, umask, export and trap are ordinary commands on that route, and the trap body is itself built from nodes and passed as one grammar-quoted word, so no new orchestration intent is a prerequisite. + set -euo pipefail + umask 077 + HDR_FILE="$RUNNER_TEMP/mtcollins1-fan-auth-header" + RESP_FILE="$RUNNER_TEMP/mtcollins1-fan-sm.json" + DEST_FILE="$RUNNER_TEMP/mtcollins1-bmc-credential" + trap 'rm -f "$HDR_FILE" "$RESP_FILE" "$DEST_FILE"' EXIT + printf 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE" + curl -sSf --connect-timeout 10 --max-time 60 -o "$RESP_FILE" -H @"$HDR_FILE" "https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/bmc-mtcollins1-gunbc/versions/2:access" && python3 -c 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$DEST_FILE" + chmod 600 "$DEST_FILE" + export GUNBC_MTCOLLINS1_BMC_CREDENTIAL_FILE="$DEST_FILE" + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag --function mtcollins1_ui_bundle_observe_wet + cat "$ROOT/target/mtcollins1-ui-bundle-receipt.txt" + env: + WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} + if: github.event.inputs.mode == 'mtcollins1_ui_bundle_observe' + timeout-minutes: 5 + - name: Upload Mt. Collins UI bundle bytes and receipt + id: mtcollins1_ui_bundle_observe_upload + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: mtcollins1-ui-bundle + path: target/mtcollins1-ui-bundle-* + if-no-files-found: error + retention-days: 90 + compression-level: 0 + if: always() && github.event.inputs.mode == 'mtcollins1_ui_bundle_observe' + timeout-minutes: 10 - name: "Fabric writer identity: the login this host presents at the fabric DB's served door (reads only)" id: fabric_writer_identity_observe run: |- diff --git a/dag/extdeps/bmc/megarac.dag b/dag/extdeps/bmc/megarac.dag index 79cb1eeed65..f5b2b2ad756 100644 --- a/dag/extdeps/bmc/megarac.dag +++ b/dag/extdeps/bmc/megarac.dag @@ -615,3 +615,31 @@ service megarac.Media { } } } + +// THE FIRMWARE'S OWN UI BUNDLE, SERVED AT A FIXED PATH. The SP-X web UI is one minified bundle the +// login page loads before any session exists, so the GET carries no cookie and no token and opens no +// session to release. The codes this module cites as "recovered from source.min.js" (start-media's +// 13410/13460, the served api/ surface, the KVM wire) were read from these bytes; the operation is the +// route that re-reads them instead of a hand-run probe. +// +// THE BYTES ARE WRITTEN AS SERVED. `Accept-Encoding: gzip` without --compressed asks for the encoding +// the 2026-09-27 read recorded (gzip) and leaves it undecoded, so a digest of dest_file is a digest of +// the served bytes, comparable with that read. stdout carries only the write-out: the HTTP status and +// the byte count, space-separated on one line. +data megarac_ui_bundle_path: NonEmptyStr = "/source.min.js" + +service megarac.Ui { + + operation GetServedBundle { + input { bmc_host: NonEmptyStr, bundle_path: NonEmptyStr, dest_file: NonEmptyStr, max_time: NonEmptyStr } + output { writeout: String from "stdout", stderr: String from "stderr", exit_code: Int from "exit_code", success: Bool from "exit_success" } + readonly + transport shell { + argv: ["curl", "--fail-with-body", "-sS", "-k", "--max-time", "{max_time}", "-H", "Accept-Encoding: gzip", "-o", "{dest_file}", "-w", "%\{http_code} %\{size_download}", "https://{bmc_host}{bundle_path}"] + } + exit { + 0 => Unit + nonzero => String "megarac UI bundle GET failed" + } + } +} diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 9c2d4952a9a..94d522a1390 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -75,6 +75,7 @@ import gunbc.machine_intake_mtcollins1_fan_observe { mtcollins1_fan_credential_p import gunbc.fabric_writer_identity_observe { fabric_writer_identity_receipt_path } import gunbc.machine_intake_mtcollins1_census_image_publish { mtcollins1_census_image_receipt_path } import gunbc.machine_intake_mtcollins1_census_member_readback { census_member_readback_receipt_path } +import gunbc.machine_intake_mtcollins1_ui_bundle_observe { mtcollins1_ui_bundle_receipt_path } import std.dissolution { DissolutionCondition, dissolution_description, unbound_dissolution } @@ -995,6 +996,11 @@ data gunbc_ci_mtcollins1_fan_observe_target: GunbcRunStepTarget = GunbcRunStepTa function: "mtcollins1_fan_observe_wet", } +data gunbc_ci_mtcollins1_ui_bundle_observe_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag", + function: "mtcollins1_ui_bundle_observe_wet", +} + data gunbc_ci_mtcollins1_census_image_publish_target: GunbcRunStepTarget = GunbcRunStepTarget { entry: "dag/gunbc/machine_intake/mtcollins1_census_image_publish.dag", function: "mtcollins1_census_image_publish_wet", @@ -1281,6 +1287,7 @@ fn gunbc_run_step_targets() -> List { gunbc_ci_host_reset_return_target, gunbc_ci_host_reset_dispatch_admission_target, gunbc_ci_mtcollins1_fan_observe_target, + gunbc_ci_mtcollins1_ui_bundle_observe_target, gunbc_ci_mtcollins1_census_image_publish_target, gunbc_ci_mtcollins1_census_member_readback_target, gunbc_ci_spark_grant_install_target, @@ -1826,6 +1833,33 @@ fn gunbc_ci_mtcollins1_fan_observe_invoke() -> String { ) } +// THE UI BUNDLE LANE RUNS ON THE FAN LANE'S PRELUDE, NOT A COPY OF IT. It needs exactly what the fan +// series needs -- the one IPMI credential, for `mc info` -- and the bundle GET itself carries none, so +// a second prelude would be a second authority for the same materialization. It is the same +// hand-shell transport and carries the same marker and trigger. +data gunbc_ci_mtcollins1_ui_bundle_observe_shell_emit_scaffold: Disposition = Scaffold { + dissolves_to: SingleAuthority, + bind: DeclarationRef { + module_path: "gunbc.ci_spec", + decl_name: "gunbc_ci_mtcollins1_ui_bundle_observe_invoke", + field: WholeDeclaration + } +} + +fn gunbc_ci_mtcollins1_ui_bundle_observe_invoke() -> String { + concat( + concat("# ", concat(dissolution_description(condition: gunbc_ci_mtcollins1_fan_observe_shell_emit_dissolution_trigger), "\n")), + gunbc_run_step_script_with_prelude( + prelude: gunbc_ci_mtcollins1_fan_observe_credential_prelude(), + source_roots: witness_layer_roots, + entry: gunbc_ci_mtcollins1_ui_bundle_observe_target.entry, + function: gunbc_ci_mtcollins1_ui_bundle_observe_target.function, + claim_run: false, + receipt_rel: mtcollins1_ui_bundle_receipt_path + ) + ) +} + // The in-run ssh-agent teardown — a FRESH per-job effect (each isolated job starts and must kill // its own agent; nothing is memoizable across jobs), named so the workflow step and the // materialization gate's fresh-effect list reference ONE authority (§3), rather than an inline diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index 1f5e905826f..66e621be2c4 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -27,6 +27,7 @@ import gunbc.fleet.organization_runner_roster_read { organization_runner_roster_ import gunbc.cloudflare.r2_permission_group_observe { r2_mint_preflight_receipt_path } import gunbc.cloudflare.r2_bucket_ensure { r2_bucket_ensure_receipt_path } import gunbc.machine_intake_mtcollins1_fan_observe { mtcollins1_fan_observation_receipt_path } +import gunbc.machine_intake_mtcollins1_ui_bundle_observe { mtcollins1_ui_bundle_bytes_path, mtcollins1_ui_bundle_receipt_path } import gunbc.fabric_writer_identity_observe { fabric_writer_identity_receipt_path } import gunbc.machine_intake_mtcollins1_census_image_publish { mtcollins1_census_image_receipt_path } import gunbc.machine_intake_mtcollins1_census_member_readback { census_member_readback_receipt_path } @@ -123,6 +124,7 @@ import gunbc.ci_spec { gunbc_ci_app_key_version_verify_mint_script, gunbc_ci_app_key_version_verify_verdict_invoke, gunbc_ci_mtcollins1_fan_observe_invoke, + gunbc_ci_mtcollins1_ui_bundle_observe_invoke, gunbc_ci_fabric_writer_identity_observe_invoke, gunbc_ci_mtcollins1_census_image_publish_invoke, gunbc_ci_mtcollins1_census_member_readback_invoke, @@ -296,6 +298,7 @@ type FleetConvergeWorkflowMode | MtCollins1Boot | PairServingD0 | MtCollins1FanObserve + | MtCollins1UiBundleObserve | FabricWriterIdentityObserve | MtCollins1CensusImagePublish | MtCollins1CensusMemberReadback @@ -356,6 +359,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String NamecheapObserve => "namecheap_observe" PairServingD0 => "pair_serving_d0" MtCollins1FanObserve => "mtcollins1_fan_observe" + MtCollins1UiBundleObserve => "mtcollins1_ui_bundle_observe" FabricWriterIdentityObserve => "fabric_writer_identity_observe" MtCollins1CensusImagePublish => "mtcollins1_census_image_publish" MtCollins1CensusMemberReadback => "mtcollins1_census_member_readback" @@ -385,7 +389,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, SparkV41GroupALaunch, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotReserve, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, RunnerBrowserToolchainConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, MtCollins1CensusMemberReadback, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, SparkV41GroupALaunch, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotReserve, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, RunnerBrowserToolchainConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1UiBundleObserve, MtCollins1CensusImagePublish, MtCollins1CensusMemberReadback, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // // EVERY MODE IS NAMED, AND THE WILDCARD IS DELIBERATELY ABSENT. A `_ => none` would read the same @@ -458,6 +462,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc NamecheapObserve => none PairServingD0 => none MtCollins1FanObserve => none + MtCollins1UiBundleObserve => none FabricWriterIdentityObserve => none MtCollins1CensusImagePublish => none MtCollins1CensusMemberReadback => none @@ -586,6 +591,7 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> MtCollins1Boot => FleetSshKeyConsumed PairServingD0 => FleetSshKeyConsumed MtCollins1FanObserve => FleetSshKeyNotConsumed + MtCollins1UiBundleObserve => FleetSshKeyNotConsumed FabricWriterIdentityObserve => FleetSshKeyNotConsumed MtCollins1CensusImagePublish => FleetSshKeyConsumed MtCollins1CensusMemberReadback => FleetSshKeyConsumed @@ -1208,6 +1214,7 @@ fn fleet_converge_mode_mutation_domain(mode: FleetConvergeWorkflowMode) -> Fleet MtCollins1Boot => ExecutorDomain PairServingD0 => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } MtCollins1FanObserve => ExecutorDomain + MtCollins1UiBundleObserve => ExecutorDomain FabricWriterIdentityObserve => ExecutorDomain MtCollins1CensusImagePublish => ExecutorDomain MtCollins1CensusMemberReadback => ExecutorDomain @@ -2071,6 +2078,39 @@ fn fleet_converge_mtcollins1_fan_observe_step() -> Step { } } +// MT. COLLINS' OWN UI BUNDLE AND ITS DIGEST (gunbc.machine_intake_mtcollins1_ui_bundle_observe): the +// firmware revision over IPMI, then the served source.min.js bytes, only under revision 0.32. Reads +// only, no web session. The receipt and the bytes are uploaded whatever the outcome, because a refused +// run's receipt is the one that says why it holds no bytes; a step condition without a status +// function is implicitly success() and would drop exactly that receipt. +data fleet_converge_mtcollins1_ui_bundle_observe_step_if: String = fleet_converge_mode_step_if(mode: MtCollins1UiBundleObserve) + +fn fleet_converge_mtcollins1_ui_bundle_observe_step() -> Step { + RunStep { + name: Present { value: "Mt. Collins UI bundle: firmware revision, then the served source.min.js and its sha256 (reads only)" }, + id: Present { value: "mtcollins1_ui_bundle_observe" }, + run: gunbc_ci_mtcollins1_ui_bundle_observe_invoke(), + shell: none, + env: Present { value: [ + kv(key: "WIF_ACCESS_TOKEN", value: yaml_string(s: "${{ steps.wif_auth.outputs.access_token }}")), + ] }, + working_directory: none, + if_condition: Present { value: fleet_converge_mtcollins1_ui_bundle_observe_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_aux_step_timeout_minutes } + } +} + +fn fleet_converge_mtcollins1_ui_bundle_observe_upload_step() -> Step { + fleet_converge_receipt_upload_step( + id: "mtcollins1_ui_bundle_observe_upload", + name: "Upload Mt. Collins UI bundle bytes and receipt", + artifact: "mtcollins1-ui-bundle", + path: "target/mtcollins1-ui-bundle-*", + if_condition: join(["always() && ", fleet_converge_mtcollins1_ui_bundle_observe_step_if], "") + ) +} + // THE READING THE WRITER-ROSTER WALL WAITS FOR (gunbc.fabric_writer_identity_observe): from the // fleet host the dispatch names, through tailscale serve, what Tailscale-User-Login the fabric DB's // door sees. Reads only; the receipt is the roster's ground. Dispatched once per fleet host. @@ -3423,6 +3463,8 @@ fn fleet_converge_job() -> Job { fleet_converge_app_key_version_verify_receipt_upload_step(), fleet_converge_mtcollins1_fan_observe_step(), fleet_converge_mtcollins1_fan_observe_receipt_upload_step(), + fleet_converge_mtcollins1_ui_bundle_observe_step(), + fleet_converge_mtcollins1_ui_bundle_observe_upload_step(), fleet_converge_fabric_writer_identity_observe_step(), fleet_converge_fabric_writer_identity_observe_receipt_upload_step(), fleet_converge_pair_serving_d0_step(), @@ -3667,6 +3709,7 @@ fn fleet_converge_mode_job_id(mode: FleetConvergeWorkflowMode) -> NonEmptyStr { ApprovalDeviceEnrolmentCodeIssue => fleet_converge_shared_job_id MtCollins1Boot => "mtcollins1-boot" as NonEmptyStr MtCollins1FanObserve => fleet_converge_shared_job_id + MtCollins1UiBundleObserve => fleet_converge_shared_job_id MtCollins1CensusImagePublish => fleet_converge_shared_job_id MtCollins1CensusMemberReadback => fleet_converge_shared_job_id HostCredentialCustodyConverge => fleet_converge_shared_job_id diff --git a/dag/gunbc/machine_intake/megarac_served_ui_catalog_observation.dag b/dag/gunbc/machine_intake/megarac_served_ui_catalog_observation.dag index 95116b93386..2bbb9e9c70a 100644 --- a/dag/gunbc/machine_intake/megarac_served_ui_catalog_observation.dag +++ b/dag/gunbc/machine_intake/megarac_served_ui_catalog_observation.dag @@ -11,3 +11,10 @@ import std.types { NonEmptyStr } data megarac_spx_ui_surface_path: NonEmptyStr = "artifacts/bmc/mtcollins1-megarac-api-surface.txt" as NonEmptyStr data megarac_spx_ui_surface_sha256: NonEmptyStr = "4fc53fea3fe49bc3ed3b967511654223578d59d963546117ab524df11b419bcd" as NonEmptyStr + +// THE 2026-09-27 READ OF THE BUNDLE ITSELF, as its digest. extdeps.bmc.megarac cites this read for the +// screen-capture surfaces and the KVM wire; the bytes were not committed, so the digest is the only +// thing a later read can be compared with. gunbc.machine_intake_mtcollins1_ui_bundle_observe reports +// match or drift against it and does not refuse on drift: a different digest is a finding about the +// firmware's served UI, not a failed read. +data megarac_source_min_js_sha256_2026_09_27: NonEmptyStr = "5029fae278f7fc8f4dd4ad5eb8e4457e49fa25d4d0158d4bf9fbab09f57d4bf3" as NonEmptyStr diff --git a/dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag b/dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag new file mode 100644 index 00000000000..1e469bc5b32 --- /dev/null +++ b/dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag @@ -0,0 +1,163 @@ +module gunbc.machine_intake_mtcollins1_ui_bundle_observe + +import extdeps.shell +import extdeps.bmc.ipmi +import extdeps.bmc.megarac +import extdeps.tools.sha256sum +import std.algebra { trim } +import std.types { Bool, Int, List, NonEmptyStr, String } +import std.process { ExitSuccess, ProcessExit, exit_failure } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.tools.gnu_coreutils { coreutils_duration_operand } +import extdeps.bmc.megarac { megarac_ui_bundle_path } +import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint, mtcollins1_firmware } +import gunbc.machine_intake_mtcollins1_bmc_secure_observation { mtcollins1_secured_account } +import gunbc.machine_intake_mtcollins1_fan_observe { FanCredentialPathReady, FanCredentialPathUnset, FanCredentialPathEmpty, mtcollins1_fan_credential_path, mtcollins1_fan_credential_path_env } +import gunbc.machine_intake_mtcollins1_boot_diagnostic_bundle { mtcollins1_boot_ipmi_read_deadline, mtcollins1_boot_ipmi_kill_after } +import gunbc.machine_intake_megarac_media_attach { megarac_media_read_max_time_operand } +import gunbc.machine_intake_megarac_served_ui_catalog_observation { megarac_source_min_js_sha256_2026_09_27 } + +// ONE READ OF MT. COLLINS' OWN UI BUNDLE, SO A VENDOR CODE IS INTERPRETED FROM THE VENDOR'S SOURCE +// (operator, amended brief msg_20e45239 and decision msg_b3c77f62, 2026-09-30). The settings route +// answers cd_error_code 16 and nothing in the tree says what 16 names; the firmware's UI names its +// own codes in source.min.js, which is how start-media's 13410/13460 were recovered. This mode is the +// modeled route to those bytes: the controller's firmware revision is read over IPMI (`mc info`, +// read-only), the bundle is fetched only when that revision is the one this unit's facts are keyed +// on, and the served bytes, their sha256 and the comparison with the 2026-09-27 read are the receipt. +// +// NOTHING IS WRITTEN TO THE CONTROLLER AND NO WEB SESSION IS OPENED. The bundle is what the login page +// loads before a session exists, so the GET carries no credential (extdeps.bmc.megarac +// megarac.Ui.GetServedBundle); the only credential this run materializes is the IPMI one `mc info` +// needs, through the same prelude the fan observation uses. No unit hold is taken: like the fan +// series, nothing here changes chassis, media or boot state. + +data mtcollins1_ui_bundle_bytes_path: String = "target/mtcollins1-ui-bundle-source.min.js" + +data mtcollins1_ui_bundle_receipt_path: String = "target/mtcollins1-ui-bundle-receipt.txt" + +// THE REVISION AS `mc info` PRINTS IT: the value after the colon on the `Firmware Revision` line. +fn firmware_revision_of_mc_info(stdout: String) -> String? { + let lines = filter(split(s: stdout, delimiter: "\n"), l => starts_with(s: trim(s: l), prefix: "Firmware Revision")) + match lines.first() { + Absent => none + Present { value: line } => { + let after = split(s: line, delimiter: ":").skip(n: 1) + let v = trim(s: join(after, ":")) + if v == "" { none } else { Present { value: v } } + } + } +} + +// THE BUNDLE IS FETCHED ONLY UNDER THE REVISION THE UNIT'S FACTS ARE KEYED ON. Every code this +// module exists to recover is a 0.32 fact (mtcollins1_firmware); bytes from another build would be +// read as 0.32's table. +type UiBundleFirmwareStanding + = UiBundleFirmwareExpected { revision: NonEmptyStr } + | UiBundleFirmwareDiffers { revision: NonEmptyStr, expected: NonEmptyStr } + | UiBundleFirmwareUnread { detail: String } + +fn ui_bundle_firmware_standing(success: Bool, stdout: String, exit_code: Int, stderr: String) -> UiBundleFirmwareStanding { + if !success { + UiBundleFirmwareUnread { detail: join(["ipmitool mc info exited ", to_string(exit_code), ": ", trim(s: stderr)], "") } + } else { + match firmware_revision_of_mc_info(stdout: stdout) { + Absent => UiBundleFirmwareUnread { detail: concat("mc info printed no Firmware Revision line: ", stdout) } + Present { value: r } => + if r == (mtcollins1_firmware.raw_version as String) { UiBundleFirmwareExpected { revision: r as NonEmptyStr } } + else { UiBundleFirmwareDiffers { revision: r as NonEmptyStr, expected: mtcollins1_firmware.raw_version } } + } + } +} + +// MATCH OR DRIFT AGAINST THE 2026-09-27 READ, REPORTED AND NEVER A REFUSAL: a different digest is a +// finding about the served UI, not a failed read. +type UiBundleDigestComparison + = UiBundleDigestMatches + | UiBundleDigestDrifted { cited: NonEmptyStr } + +fn ui_bundle_digest_comparison(observed: String) -> UiBundleDigestComparison { + if observed == (megarac_source_min_js_sha256_2026_09_27 as String) { UiBundleDigestMatches } + else { UiBundleDigestDrifted { cited: megarac_source_min_js_sha256_2026_09_27 } } +} + +fn ui_bundle_digest_comparison_text(c: UiBundleDigestComparison) -> String { + match c { + UiBundleDigestMatches => "MATCH with the 2026-09-27 read" + UiBundleDigestDrifted { cited: d } => concat("DRIFT from the 2026-09-27 read, which was ", d as String) + } +} + +fn ui_bundle_firmware_text(f: UiBundleFirmwareStanding) -> String { + match f { + UiBundleFirmwareExpected { revision: r } => concat("firmware revision ", concat(r as String, " (expected)")) + UiBundleFirmwareDiffers { revision: r, expected: e } => join(["firmware revision ", r as String, " is NOT the expected ", e as String, "; the bundle was not fetched"], "") + UiBundleFirmwareUnread { detail: d } => concat("firmware revision UNREAD, so the bundle was not fetched: ", d) + } +} + +// THE RECEIPT IS WRITTEN ON EVERY PATH THAT REACHED THE CONTROLLER, and the exit carries the same +// sentence, so a refused run's artifact says why it holds no bytes. +fn ui_bundle_conclude(lines: List, outcome: ProcessExit) -> ProcessExit { + let body = join(concat([concat("subject=", mtcollins1_endpoint.host as String), concat("bundle_path=", megarac_ui_bundle_path as String)], lines), "\n") + let kept = Filesystem.Write(path: mtcollins1_ui_bundle_receipt_path, content: concat(body, "\n")) + if kept.success { outcome } else { + match outcome { + ExitSuccess => exit_failure(reason: concat("mtcollins1 ui bundle: the bundle was read and its receipt could not be written: ", kept.error)) + other => other + } + } +} + +fn ui_bundle_firmware_refused(firmware: UiBundleFirmwareStanding) -> ProcessExit { + ui_bundle_conclude(lines: [concat("firmware=", ui_bundle_firmware_text(f: firmware))], outcome: exit_failure(reason: concat("mtcollins1 ui bundle: ", ui_bundle_firmware_text(f: firmware)))) +} + +fn mtcollins1_ui_bundle_observe_with(username: NonEmptyStr, password_file: NonEmptyStr) -> ProcessExit { + let mc = diagnostic.ipmi.Tool.McInfo( + bmc_host: mtcollins1_endpoint.host, + username: username, + password_file: password_file, + deadline: coreutils_duration_operand(d: mtcollins1_boot_ipmi_read_deadline) as NonEmptyStr, + kill_after: coreutils_duration_operand(d: mtcollins1_boot_ipmi_kill_after) as NonEmptyStr, + ) + let firmware = ui_bundle_firmware_standing(success: mc.success, stdout: mc.stdout, exit_code: mc.exit_code, stderr: mc.transport_stderr) + let fw_line = concat("firmware=", ui_bundle_firmware_text(f: firmware)) + match firmware { + UiBundleFirmwareExpected { revision: _ } => {} + UiBundleFirmwareDiffers { revision: _, expected: _ } => return ui_bundle_firmware_refused(firmware: firmware) + UiBundleFirmwareUnread { detail: _ } => return ui_bundle_firmware_refused(firmware: firmware) + } + let got = megarac.Ui.GetServedBundle( + bmc_host: mtcollins1_endpoint.host, + bundle_path: megarac_ui_bundle_path, + dest_file: mtcollins1_ui_bundle_bytes_path as NonEmptyStr, + max_time: megarac_media_read_max_time_operand(), + ) + let fetch_line = join(["fetch=exit ", to_string(got.exit_code), " writeout(http_code size_download)=", trim(s: got.writeout), if trim(s: got.stderr) == "" { "" } else { concat(" stderr=", trim(s: got.stderr)) }], "") + if !got.success { + return ui_bundle_conclude(lines: [fw_line, fetch_line], outcome: exit_failure(reason: concat("mtcollins1 ui bundle: the bundle GET did not succeed: ", fetch_line))) + } + let digest = sha256sum.Sha256.DigestFile(path: mtcollins1_ui_bundle_bytes_path as NonEmptyStr) + let observed = match split(s: trim(s: digest.line), delimiter: " ").first() { Present { value: d } => d Absent => "" } + if !digest.success || observed == "" { + return ui_bundle_conclude(lines: [fw_line, fetch_line], outcome: exit_failure(reason: concat("mtcollins1 ui bundle: the fetched bytes could not be digested: ", digest.line))) + } + ui_bundle_conclude( + lines: [fw_line, fetch_line, concat("sha256=", observed), concat("comparison=", ui_bundle_digest_comparison_text(c: ui_bundle_digest_comparison(observed: observed))), concat("bytes=", mtcollins1_ui_bundle_bytes_path)], + outcome: ExitSuccess, + ) +} + +// REFUSALS BEFORE THE FIRST REQUEST, EACH NAMING WHAT IS MISSING, exactly as the fan observation's. +fn mtcollins1_ui_bundle_observe_wet() -> ProcessExit +{ + match mtcollins1_secured_account() { + Absent => exit_failure(reason: "mtcollins1 ui bundle: the BMC is not BmcSecured, so this run holds no account to read the firmware revision with; contacting NO host") + Present { value: username } => + match mtcollins1_fan_credential_path() { + FanCredentialPathUnset => exit_failure(reason: join(["mtcollins1 ui bundle: no ", mtcollins1_fan_credential_path_env as String, " in the environment; contacting NO host"], "")) + FanCredentialPathEmpty => exit_failure(reason: join(["mtcollins1 ui bundle: ", mtcollins1_fan_credential_path_env as String, " is set but empty; contacting NO host"], "")) + FanCredentialPathReady { path: p } => mtcollins1_ui_bundle_observe_with(username: username, password_file: p) + } + } +} diff --git a/dag/test/claim/machine_intake/mtcollins1_ui_bundle_observe_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_ui_bundle_observe_witness_test.dag new file mode 100644 index 00000000000..e50d8a45a68 --- /dev/null +++ b/dag/test/claim/machine_intake/mtcollins1_ui_bundle_observe_witness_test.dag @@ -0,0 +1,68 @@ +module test.claim.machine_intake.mtcollins1_ui_bundle_observe_witness_test + +import std.types { Bool, String } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import gunbc.machine_intake_megarac_served_ui_catalog_observation { megarac_source_min_js_sha256_2026_09_27 } +import gunbc.machine_intake_mtcollins1_ui_bundle_observe { + UiBundleDigestDrifted, + UiBundleDigestMatches, + UiBundleFirmwareDiffers, + UiBundleFirmwareExpected, + UiBundleFirmwareUnread, + firmware_revision_of_mc_info, + ui_bundle_digest_comparison, + ui_bundle_firmware_standing, +} + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// `ipmitool mc info` as it prints, with the revision a parameter so the pair differs in one value. +fn mc_info_with(revision: String) -> String { + join([ + "Device ID : 32", + "Device Revision : 1", + concat("Firmware Revision : ", revision), + "IPMI Version : 2.0", + "Manufacturer ID : 10876", + ], "\n") +} + +test fn the_revision_is_the_value_after_the_colon() -> Bool { + match firmware_revision_of_mc_info(stdout: mc_info_with(revision: "0.32")) { + Present { value: r } => r == "0.32" + Absent => false + } +} + +// THE DISCRIMINATING PAIR: one value differs, and only 0.32 admits the fetch. +test fn only_the_expected_revision_admits_the_fetch() -> Bool { + (match ui_bundle_firmware_standing(success: true, stdout: mc_info_with(revision: "0.32"), exit_code: 0, stderr: "") { + UiBundleFirmwareExpected { revision: r } => (r as String) == "0.32" + _ => false + }) + && (match ui_bundle_firmware_standing(success: true, stdout: mc_info_with(revision: "0.33"), exit_code: 0, stderr: "") { + UiBundleFirmwareDiffers { revision: r, expected: e } => (r as String) == "0.33" && (e as String) == "0.32" + _ => false + }) +} + +// An unanswered read and an answer with no revision are both unread, never a revision. +test fn an_unread_revision_never_admits_the_fetch() -> Bool { + (match ui_bundle_firmware_standing(success: false, stdout: "", exit_code: 124, stderr: "") { + UiBundleFirmwareUnread { detail: d } => string_contains(s: d, pattern: "124") + _ => false + }) + && (match ui_bundle_firmware_standing(success: true, stdout: "Device ID : 32", exit_code: 0, stderr: "") { + UiBundleFirmwareUnread { detail: _ } => true + _ => false + }) +} + +// DRIFT IS REPORTED, AND THE CITED DIGEST RIDES WITH IT. +test fn the_cited_digest_matches_and_any_other_drifts() -> Bool { + (match ui_bundle_digest_comparison(observed: megarac_source_min_js_sha256_2026_09_27 as String) { UiBundleDigestMatches => true _ => false }) + && (match ui_bundle_digest_comparison(observed: "0000000000000000000000000000000000000000000000000000000000000000") { + UiBundleDigestDrifted { cited: c } => c == megarac_source_min_js_sha256_2026_09_27 + _ => false + }) +}