diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 4fab01f5693..2d8579eccff 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -13,7 +13,7 @@ on: mode: description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; runner_browser_toolchain_converge installs the declared Playwright/Chromium toolchain (apt host libraries as the administrator, digest-pinned node, Playwright and Chromium archives into the job user's root) on the selected host, which must be in the pool that runs the floor job, and refuses unless every digest, version and host library reads back and headless Chromium renders a local page; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; spark_v41_engram_differential compares upstream's Engram lookup kernel with the design-B file-backed lookup over sampled real rows of every row store, byte for byte, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; spark_v41_group_a_launch reads the production image back under its tag on every Group A host, reads its registry inside its digest and every host's occupancy, and only when Group A is suspended for this candidate with every host held and vacant stages the Engram manifest and applies the V4.1 four-rank arm as one transaction at the capacity measurement's shape -- the target names only the session host; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit and the gunbc-microvm-slot-reserve broker unit on srv1 and starts neither; microvm_slot_reserve starts that broker unit once on srv1, which reserves the shakedown cell through the fabric broker route from inside the root process (slot, demand and offer derived from the model, never inputs), and uploads that invocation's reservation receipt, failing unless the reservation committed; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction; mtcollins1_census_qemu_host_observe reads the selected host's KVM device, qemu-system-aarch64 build and AAVMF images against their pins and writes nothing required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, spark_v41_group_a_launch, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_reserve, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, runner_browser_toolchain_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, mtcollins1_census_member_readback, mtcollins1_census_qemu_host_observe, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, spark_v41_group_a_launch, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_reserve, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, runner_browser_toolchain_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_ui_bundle_observe, mtcollins1_kvm_observer_observe, mtcollins1_census_image_publish, mtcollins1_census_member_readback, mtcollins1_census_qemu_host_observe, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -859,6 +859,80 @@ jobs: retention-days: 30 if: github.event.inputs.mode == 'mtcollins1_fan_observe' timeout-minutes: 10 + - name: "Mt. Collins served UI: firmware revision, then source.min.js, viewer.html and every script they load, each with its sha256 (reads only)" + id: mtcollins1_ui_bundle_observe + run: | + 'set' '-euo' 'pipefail' + 'umask' '077' + HDR_FILE="$RUNNER_TEMP"'/mtcollins1-ui-bundle-auth-header' + RESP_FILE="$RUNNER_TEMP"'/mtcollins1-ui-bundle-sm.json' + DEST_FILE="$RUNNER_TEMP"'/mtcollins1-ui-bundle-bmc-credential' + 'trap' ''\''rm'\'' '\''-f'\'' "$HDR_FILE" "$RESP_FILE" "$DEST_FILE" + ' 'EXIT' + 'printf' 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE" + 'curl' '-sSf' '--connect-timeout' '10' '--max-time' '60' '-o' "$RESP_FILE" '-H' '@'"$HDR_FILE" 'https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/bmc-mtcollins1-gunbc/versions/2:access' && 'python3' '-c' 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$DEST_FILE" + 'chmod' '600' "$DEST_FILE" + 'export' 'GUNBC_MTCOLLINS1_BMC_CREDENTIAL_FILE='"$DEST_FILE" + ROOT=$('git' 'rev-parse' '--show-toplevel' 2>/dev/null || 'pwd') + + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag --function mtcollins1_ui_bundle_observe_wet + env: + WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} + if: github.event.inputs.mode == 'mtcollins1_ui_bundle_observe' + timeout-minutes: 25 + - name: Upload Mt. Collins served UI files and receipt + id: mtcollins1_ui_bundle_observe_upload + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: mtcollins1-ui-bundle + path: target/mtcollins1-ui-bundle-* + if-no-files-found: error + retention-days: 90 + compression-level: 0 + if: always() && github.event.inputs.mode == 'mtcollins1_ui_bundle_observe' + timeout-minutes: 10 + - name: "Mt. Collins KVM observer, no boot: under the unit hold, log in, establish the viewer, one still, release" + id: mtcollins1_kvm_observer_observe + run: | + 'set' '-euo' 'pipefail' + 'umask' '077' + HDR_FILE="$RUNNER_TEMP"'/mtcollins1-kvm-observer-auth-header' + RESP_FILE="$RUNNER_TEMP"'/mtcollins1-kvm-observer-sm.json' + DEST_FILE="$RUNNER_TEMP"'/mtcollins1-kvm-observer-bmc-credential' + 'trap' ''\''rm'\'' '\''-f'\'' "$HDR_FILE" "$RESP_FILE" "$DEST_FILE" + ' 'EXIT' + 'printf' 'Authorization: Bearer %s\n' "$WIF_ACCESS_TOKEN" > "$HDR_FILE" + 'curl' '-sSf' '--connect-timeout' '10' '--max-time' '60' '-o' "$RESP_FILE" '-H' '@'"$HDR_FILE" 'https://secretmanager.googleapis.com/v1/projects/gunbai-secrets/secrets/bmc-mtcollins1-gunbc/versions/2:access' && 'python3' '-c' 'import sys,json,base64; sys.stdout.buffer.write(base64.b64decode(json.load(open(sys.argv[1]))["payload"]["data"]))' "$RESP_FILE" > "$DEST_FILE" + 'chmod' '600' "$DEST_FILE" + 'export' 'GUNBC_MTCOLLINS1_BMC_CREDENTIAL_FILE='"$DEST_FILE" + ROOT=$('git' 'rev-parse' '--show-toplevel' 2>/dev/null || 'pwd') + + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag --function mtcollins1_kvm_observer_observe_wet + env: + WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} + if: github.event.inputs.mode == 'mtcollins1_kvm_observer_observe' + timeout-minutes: 18 + - name: "Mt. Collins KVM observer: release this run's observer and its unit hold (always; a no-op if nothing is held)" + id: mtcollins1_kvm_observer_release + run: | + 'set' '-euo' 'pipefail' + ROOT=$('git' 'rev-parse' '--show-toplevel' 2>/dev/null || 'pwd') + + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag --function mtcollins1_kvm_observer_release_wet + if: always() && github.event.inputs.mode == 'mtcollins1_kvm_observer_observe' + timeout-minutes: 16 + - name: Upload Mt. Collins KVM observer journal, stills and receipt + id: mtcollins1_kvm_observer_observe_upload + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: mtcollins1-kvm-observer + path: artifacts/mtcollins1-boot-kvm-probe-* + if-no-files-found: error + retention-days: 90 + compression-level: 0 + if: always() && github.event.inputs.mode == 'mtcollins1_kvm_observer_observe' + timeout-minutes: 10 - name: "Fabric writer identity: the login this host presents at the fabric DB's served door (reads only)" id: fabric_writer_identity_observe run: |- diff --git a/dag/extdeps/bmc/megarac.dag b/dag/extdeps/bmc/megarac.dag index 79cb1eeed65..3f5f922a5a4 100644 --- a/dag/extdeps/bmc/megarac.dag +++ b/dag/extdeps/bmc/megarac.dag @@ -15,7 +15,7 @@ import extdeps.uri { Uri, Https } import extdeps.vendor { Vendor } import extdeps.hardware { Hardware } import extdeps.vendor.ami { ami } -import extdeps.languages.json.emit { JsonValue, json_object, json_array, json_kv, json_int, json_string } +import extdeps.languages.json.emit { JsonValue, json_object, json_array, json_kv, json_int, json_string, serialize_json_array } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -615,3 +615,40 @@ service megarac.Media { } } } + +// THE VALUE THE SP-X UI STORES AS sessionStorage "features": JSON.stringify of its project-configurations +// collection, an array of {"feature": NAME} objects; the viewer's KVM view reads it with unguarded +// indexOf calls. Which names a given build carries is an OBSERVATION this repository made of that +// build's served bundle, so it is a receipt in the observing layer +// (gunbc.machine_intake_megarac_ui_features megarac_ui_features_0_32_receipt), not a fact owned here. +fn megarac_ui_features_json(names: List) -> String { + serialize_json_array(elements: map(names, n => json_object(members: [json_kv(key: "feature", value: json_string(s: n as String))]))) +} + +// THE FIRMWARE'S OWN UI BUNDLE, SERVED AT A FIXED PATH. The SP-X web UI is one minified bundle the +// login page loads before any session exists, so the GET carries no cookie and no token and opens no +// session to release. The codes this module cites as "recovered from source.min.js" (start-media's +// 13410/13460, the served api/ surface, the KVM wire) were read from these bytes; the operation is the +// route that re-reads them instead of a hand-run probe. +// +// THE BYTES ARE WRITTEN AS SERVED. `Accept-Encoding: gzip` without --compressed asks for the encoding +// the 2026-09-27 read recorded (gzip) and leaves it undecoded, so a digest of dest_file is a digest of +// the served bytes, comparable with that read. stdout carries only the write-out: the HTTP status and +// the byte count, space-separated on one line. +data megarac_ui_bundle_path: NonEmptyStr = "/source.min.js" + +service megarac.Ui { + + operation GetServedBundle { + input { bmc_host: NonEmptyStr, bundle_path: NonEmptyStr, dest_file: NonEmptyStr, max_time: NonEmptyStr } + output { writeout: String from "stdout", stderr: String from "stderr", exit_code: Int from "exit_code", success: Bool from "exit_success" } + readonly + transport shell { + argv: ["curl", "--fail-with-body", "-sS", "-k", "--max-time", "{max_time}", "-H", "Accept-Encoding: gzip", "-o", "{dest_file}", "-w", "%\{http_code} %\{size_download}", "https://{bmc_host}{bundle_path}"] + } + exit { + 0 => Unit + nonzero => String "megarac UI bundle GET failed" + } + } +} diff --git a/dag/extdeps/exec/program.dag b/dag/extdeps/exec/program.dag index c8ad6e2b4e4..795a0944e81 100644 --- a/dag/extdeps/exec/program.dag +++ b/dag/extdeps/exec/program.dag @@ -75,6 +75,7 @@ fn cataloged_program(invocation: NonEmptyStr) -> ProgramIdentity decl_ref(module_path: "extdeps.tools.wc", decl_name: "wc_program"), decl_ref(module_path: "extdeps.tools.nbdkit", decl_name: "nbdkit_program"), decl_ref(module_path: "extdeps.tools.sha512sum", decl_name: "sha512sum_program"), + decl_ref(module_path: "extdeps.tools.gzip", decl_name: "gzip_program"), decl_ref(module_path: "extdeps.tools.rustfmt", decl_name: "rustfmt_program"), decl_ref(module_path: "extdeps.tools.npm", decl_name: "npm_program"), decl_ref(module_path: "extdeps.tools.node", decl_name: "node_program"), diff --git a/dag/extdeps/tools/gzip.dag b/dag/extdeps/tools/gzip.dag new file mode 100644 index 00000000000..69a461d08c4 --- /dev/null +++ b/dag/extdeps/tools/gzip.dag @@ -0,0 +1,60 @@ +module extdeps.tools.gzip + +import extdeps.exec.program { ProgramIdentity, cataloged_program } +import std.types { Bool, Int, NonEmptyStr, Unit } +import std.string_type { String } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import extdeps.uri { Uri, Https } +import extdeps.tools { CliTool, SourceApt } + +// GNU gzip, as its manual documents it. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "www.gnu.org/software/gzip/manual/gzip.html" + } +} + +fn gzip_program() -> ProgramIdentity = cataloged_program(invocation: "gzip") + +data gzip_cli_tool: CliTool = CliTool { + program: gzip_program(), + min_version: none, + installable_via: [SourceApt { package: "gzip", bin_dir: "/usr/bin" }], +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.tools.gzip", + decl_name: "gzip_cli_tool", + field: WholeDeclaration + } + }, + first_citation: extdeps.tools.gzip.extdeps_external_authority_anchor, + further_citations: [] +} + +// THE FILE'S CONTENT, DECODED IF IT IS GZIP AND UNCHANGED IF IT IS NOT: `--decompress --stdout --force`. +// The manual's --force: "If the input data is not in a format recognized by gzip, and if the option +// --stdout is also given, copy the input data without change to the standard output: let zcat behave as +// cat." So one read answers for a served file whether or not the server compressed it, and the file on +// disk is never modified (--stdout). +service gzip.Gzip { + operation DecodeToStdout { + input { path: NonEmptyStr } + output { + text: String from "stdout" + stderr: String from "stderr" + exit_code: Int from "exit_code" + success: Bool from "exit_success" + } + readonly + transport shell { argv: ["gzip", "--decompress", "--stdout", "--force", "{path}"] } + exit { + 0 => Unit + nonzero => String "gzip decode failed" + } + } +} diff --git a/dag/gunbc/bmc_megarac_web_transport.dag b/dag/gunbc/bmc_megarac_web_transport.dag index 4d54c0cff66..a2c890ba370 100644 --- a/dag/gunbc/bmc_megarac_web_transport.dag +++ b/dag/gunbc/bmc_megarac_web_transport.dag @@ -27,6 +27,9 @@ import gunbc.owned_process { // // The stand-in viewer page is narrower than the vendor's: a canvas#kvm painted from the /kvm stream // once a frame arrives, drawn from a cross-origin image when the world's canvas_readable flag is false. +// The stand-in viewer reads sessionStorage.features.indexOf(...) UNGUARDED before it opens /kvm, as the +// served viewer.min.js does (crawl run 36787375313): a viewer opened without the feature list throws and +// never connects, which is what KVM probe run 36788605665 did on the hardware. // The stand-in root page replaces itself on DOMContentLoaded and the transport logs that it was served: // the shape that destroyed the observer's in-page login on fleet-converge run 36721915217. An observer // that loads "/" and then evaluates in it fails here exactly as it did there. @@ -44,7 +47,7 @@ data megarac_web_transport_scaffold: Disposition = Scaffold { data megarac_web_transport_dissolution_trigger: DissolutionCondition = unbound_dissolution(description: "๐ŸŸก dissolve-on: gunbc.bmc_megarac_web_transport megarac_web_transport_script -- the loopback HTTP+WebSocket transport is a node -e string that enforces a model-derived transition table. RETIRED BY THE CAPABILITY: gunbc serve WebSocket support SUFFICIENT TO SERVE THE gunbc.bmc_model KVM STREAM -- accept the /kvm upgrade, hold the connection across requests, deliver frames, and close it when a model transition leaves the stream closed, with every HTTP and upgrade request decoded, answered by the gunbc.bmc_model transition and encoded in .dag per request. HTTP-only serve support does not retire it. Admission: operator escalation msg_d5eebfff, default-approved on recommendation A, 2026-09-28") -data megarac_web_transport_script: String = "const http=require('http');const crypto=require('crypto');const fs=require('fs');const [tablePath,logPath]=process.argv.slice(1);function log(m)\{fs.appendFileSync(logPath,Date.now()+' pid='+process.pid+' '+m+'\\n')}const T=JSON.parse(fs.readFileSync(tablePath,'utf8'));let current=T.initial;const canvasReadable=T.canvas_readable===true;const kvmPath=T.kvm_path;function key(op,args)\{return JSON.stringify([op].concat(args))}const rows=new Map();const advances=new Set();for(const r of T.rows)\{const k=r.from+'\\u0000'+key(r.request.op,r.request.args);if(rows.has(k))\{log('refused: duplicate table row '+JSON.stringify(k));process.exit(3)}rows.set(k,\{status:r.status,stream:r.stream_open?'open':'none',to:r.to,body:r.body});if(r.request.op==='advance')advances.add(+r.request.args[0])}let selfStart='unknown';try\{const st=fs.readFileSync('/proc/self/stat','utf8');selfStart=st.slice(st.lastIndexOf(')')+2).split(' ')[19]}catch(e)\{}let socket=null;function apply(k)\{const r=rows.get(current+'\\u0000'+k);if(!r)\{log('unmodeled state='+JSON.stringify(current)+' request='+k);return null} log('transition request='+k+' status='+r.status+' to='+JSON.stringify(r.to));current=r.to; if(socket&&r.stream!=='open')\{log('stream closed by the model');const s=socket;socket=null;s.destroy()}return r}const png=Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==','base64');const img=http.createServer((q,r)=>\{r.writeHead(200,\{'Content-Type':'image/png'});r.end(png)});function viewer()\{const port2=img.address().port;const taint=canvasReadable?'':\"const i=new Image();i.onload=()=>x.drawImage(i,0,0);i.src='http://localhost:\"+port2+\"/x.png';\"; return \"\"}function refuse(r,key)\{r.writeHead(599,\{'Content-Type':'text/plain'});r.end('unmodeled request: '+key)}function answer(r,key)\{const t=apply(key);if(!t)return refuse(r,key);r.writeHead(t.status,\{'Content-Type':'application/json'});r.end(t.body)}const s=http.createServer((q,r)=>\{let body='';q.on('data',d=>body+=d);q.on('end',()=>\{ if(q.url==='/')\{log('spa-root served');r.writeHead(200,\{'Content-Type':'text/html'});r.end(\"\");return} if(q.url==='/login.html')\{r.writeHead(200,\{'Content-Type':'text/html'});r.end('');return} if(q.url==='/viewer.html')\{r.writeHead(200,\{'Content-Type':'text/html'});r.end(viewer());return} if(q.url==='/api/session'&&q.method==='POST')\{const p=new URLSearchParams(body);return answer(r,key('login',[p.get('username')||'',p.get('password')||'']))} if(q.url==='/api/session'&&q.method==='DELETE')\{return answer(r,key('logout',[q.headers['x-csrftoken']||'']))} if(q.url==='/api/settings/services')\{return answer(r,key('services',[]))} r.writeHead(404);r.end()})});s.on('upgrade',(q,sock)=>\{const u=new URL(q.url,'http://x');if(u.pathname!==kvmPath)\{sock.destroy();return} const t=apply(key('connect',[u.searchParams.get('token')||'']));if(!t||t.status!==101)\{log('kvm refused');sock.destroy();return} const k=crypto.createHash('sha1').update(q.headers['sec-websocket-key']+'258EAFA5-E914-47DA-95CA-C5AB0DC85B11').digest('base64'); sock.write('HTTP/1.1 101 Switching Protocols\\r\\nUpgrade: websocket\\r\\nConnection: Upgrade\\r\\nSec-WebSocket-Accept: '+k+'\\r\\n\\r\\n'); sock.write(Buffer.from([0x81,0x05,0x66,0x72,0x61,0x6d,0x65]));socket=sock;log('kvm-open');sock.on('close',()=>\{if(socket===sock)socket=null})});const started=Date.now();for(const a of advances)\{setTimeout(()=>apply(key('advance',[String(a)])),a*1000)}img.listen(0,'127.0.0.1',()=>s.listen(0,'127.0.0.1',()=>log('listening start='+selfStart+' port='+s.address().port)));" +data megarac_web_transport_script: String = "const http=require('http');const crypto=require('crypto');const fs=require('fs');const [tablePath,logPath]=process.argv.slice(1);function log(m)\{fs.appendFileSync(logPath,Date.now()+' pid='+process.pid+' '+m+'\\n')}const T=JSON.parse(fs.readFileSync(tablePath,'utf8'));let current=T.initial;const canvasReadable=T.canvas_readable===true;const kvmPath=T.kvm_path;function key(op,args)\{return JSON.stringify([op].concat(args))}const rows=new Map();const advances=new Set();for(const r of T.rows)\{const k=r.from+'\\u0000'+key(r.request.op,r.request.args);if(rows.has(k))\{log('refused: duplicate table row '+JSON.stringify(k));process.exit(3)}rows.set(k,\{status:r.status,stream:r.stream_open?'open':'none',to:r.to,body:r.body});if(r.request.op==='advance')advances.add(+r.request.args[0])}let selfStart='unknown';try\{const st=fs.readFileSync('/proc/self/stat','utf8');selfStart=st.slice(st.lastIndexOf(')')+2).split(' ')[19]}catch(e)\{}let socket=null;function apply(k)\{const r=rows.get(current+'\\u0000'+k);if(!r)\{log('unmodeled state='+JSON.stringify(current)+' request='+k);return null} log('transition request='+k+' status='+r.status+' to='+JSON.stringify(r.to));current=r.to; if(socket&&r.stream!=='open')\{log('stream closed by the model');const s=socket;socket=null;s.destroy()}return r}const png=Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==','base64');const img=http.createServer((q,r)=>\{r.writeHead(200,\{'Content-Type':'image/png'});r.end(png)});function viewer()\{const port2=img.address().port;const taint=canvasReadable?'':\"const i=new Image();i.onload=()=>x.drawImage(i,0,0);i.src='http://localhost:\"+port2+\"/x.png';\"; return \"\"}function refuse(r,key)\{r.writeHead(599,\{'Content-Type':'text/plain'});r.end('unmodeled request: '+key)}function answer(r,key)\{const t=apply(key);if(!t)return refuse(r,key);r.writeHead(t.status,\{'Content-Type':'application/json'});r.end(t.body)}const s=http.createServer((q,r)=>\{let body='';q.on('data',d=>body+=d);q.on('end',()=>\{ if(q.url==='/')\{log('spa-root served');r.writeHead(200,\{'Content-Type':'text/html'});r.end(\"\");return} if(q.url==='/login.html')\{r.writeHead(200,\{'Content-Type':'text/html'});r.end('');return} if(q.url==='/viewer.html')\{r.writeHead(200,\{'Content-Type':'text/html'});r.end(viewer());return} if(q.url==='/api/session'&&q.method==='POST')\{const p=new URLSearchParams(body);return answer(r,key('login',[p.get('username')||'',p.get('password')||'']))} if(q.url==='/api/session'&&q.method==='DELETE')\{return answer(r,key('logout',[q.headers['x-csrftoken']||'']))} if(q.url==='/api/settings/services')\{return answer(r,key('services',[]))} r.writeHead(404);r.end()})});s.on('upgrade',(q,sock)=>\{const u=new URL(q.url,'http://x');if(u.pathname!==kvmPath)\{sock.destroy();return} const t=apply(key('connect',[u.searchParams.get('token')||'']));if(!t||t.status!==101)\{log('kvm refused');sock.destroy();return} const k=crypto.createHash('sha1').update(q.headers['sec-websocket-key']+'258EAFA5-E914-47DA-95CA-C5AB0DC85B11').digest('base64'); sock.write('HTTP/1.1 101 Switching Protocols\\r\\nUpgrade: websocket\\r\\nConnection: Upgrade\\r\\nSec-WebSocket-Accept: '+k+'\\r\\n\\r\\n'); sock.write(Buffer.from([0x81,0x05,0x66,0x72,0x61,0x6d,0x65]));socket=sock;log('kvm-open');sock.on('close',()=>\{if(socket===sock)socket=null})});const started=Date.now();for(const a of advances)\{setTimeout(()=>apply(key('advance',[String(a)])),a*1000)}img.listen(0,'127.0.0.1',()=>s.listen(0,'127.0.0.1',()=>log('listening start='+selfStart+' port='+s.address().port)));" // WHETHER THE TRANSPORT IS SERVING, AND IF NOT WHY. Readiness is bound to the whole launch INSTANCE: // only a listening line carrying BOTH the pid and the start time the owned launch recorded counts, so a diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index f4678f485e6..d98058af31b 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -997,6 +997,21 @@ data gunbc_ci_mtcollins1_fan_observe_target: GunbcRunStepTarget = GunbcRunStepTa function: "mtcollins1_fan_observe_wet", } +data gunbc_ci_mtcollins1_ui_bundle_observe_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag", + function: "mtcollins1_ui_bundle_observe_wet", +} + +data gunbc_ci_mtcollins1_kvm_observer_observe_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag", + function: "mtcollins1_kvm_observer_observe_wet", +} + +data gunbc_ci_mtcollins1_kvm_observer_release_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag", + function: "mtcollins1_kvm_observer_release_wet", +} + data gunbc_ci_mtcollins1_census_image_publish_target: GunbcRunStepTarget = GunbcRunStepTarget { entry: "dag/gunbc/machine_intake/mtcollins1_census_image_publish.dag", function: "mtcollins1_census_image_publish_wet", @@ -1288,6 +1303,9 @@ fn gunbc_run_step_targets() -> List { gunbc_ci_host_reset_return_target, gunbc_ci_host_reset_dispatch_admission_target, gunbc_ci_mtcollins1_fan_observe_target, + gunbc_ci_mtcollins1_ui_bundle_observe_target, + gunbc_ci_mtcollins1_kvm_observer_observe_target, + gunbc_ci_mtcollins1_kvm_observer_release_target, gunbc_ci_mtcollins1_census_image_publish_target, gunbc_ci_mtcollins1_census_member_readback_target, gunbc_ci_mtcollins1_census_qemu_host_observe_target, @@ -1834,6 +1852,73 @@ fn gunbc_ci_mtcollins1_fan_observe_invoke() -> String { ) } +// THE READ-ONLY MT. COLLINS PROBES' STEPS ARE bash_build NODES RENDERED BY v2.workflow.bash_emit +// bash_emit_stmts (review 73415): the modeled route the fan lane's marker names as available, so these +// steps carry no hand-spelled shell and no scaffold marker. Each probe materializes the one pinned BMC +// credential the fan series and the boot read -- the shared auth-header format, ONE bounded Secret +// Manager read (gunbc_ci_sm_connect_timeout / gunbc_ci_sm_max_time) and the shared payload decode -- +// into its own 0600 file under RUNNER_TEMP, with the EXIT trap armed before the credential touches +// disk; the run line is v2.workflow.gunbc_invoke_step_emit gunbc_run_command, the modeled invocation. +fn gunbc_ci_mtcollins1_probe_rendered(stmts: List, set_e: Bool) -> String { + match bash_emit_stmts(statements: stmts, set_e: set_e) { + Accepted { value: text, diagnostics: _ } => text + Rejected { diagnostics: _ } => gunbc_invoke_step_orch_refused_poison + } +} + +fn gunbc_ci_mtcollins1_probe_root_statement() -> Node { + bash_build_assign(name: "ROOT", value: bash_build_word_cmdsubst(body: bash_build_or_else(left: bquiet(stmt: bcmdl(lits: ["git", "rev-parse", "--show-toplevel"])), right: bcmdl(lits: ["pwd"])))) +} + +fn gunbc_ci_mtcollins1_probe_credential_statements(stem: String) -> List { + let trap_body = gunbc_ci_mtcollins1_probe_rendered(stmts: [bcmd(words: [bl(text: "rm"), bl(text: "-f"), bv(name: "HDR_FILE"), bv(name: "RESP_FILE"), bv(name: "DEST_FILE")])], set_e: false) + [ + bcmdl(lits: ["set", "-euo", "pipefail"]), + bcmdl(lits: ["umask", "077"]), + bash_build_assign(name: "HDR_FILE", value: bcat(parts: [bv(name: "RUNNER_TEMP"), bl(text: concat(concat("/", stem), "-auth-header"))])), + bash_build_assign(name: "RESP_FILE", value: bcat(parts: [bv(name: "RUNNER_TEMP"), bl(text: concat(concat("/", stem), "-sm.json"))])), + bash_build_assign(name: "DEST_FILE", value: bcat(parts: [bv(name: "RUNNER_TEMP"), bl(text: concat(concat("/", stem), "-bmc-credential"))])), + bcmd(words: [bl(text: "trap"), bl(text: trap_body), bl(text: "EXIT")]), + bash_build_with_redir_to_file(inner: bcmd(words: [bl(text: "printf"), bl(text: gunbc_ci_auth_header_format), bv(name: "WIF_ACCESS_TOKEN")]), path: bv(name: "HDR_FILE")), + bash_build_and_then( + left: bcmd(words: [ + bl(text: "curl"), bl(text: "-sSf"), bl(text: "--connect-timeout"), bl(text: to_string(second_count(s: gunbc_ci_sm_connect_timeout))), + bl(text: "--max-time"), bl(text: to_string(second_count(s: gunbc_ci_sm_max_time))), bl(text: "-o"), bv(name: "RESP_FILE"), + bl(text: "-H"), bcat(parts: [bl(text: "@"), bv(name: "HDR_FILE")]), bl(text: secret_ref_access_url(ref: mtcollins1_bmc_gunbc_secret_ref)), + ]), + right: bash_build_with_redir_to_file(inner: bcmd(words: [bl(text: "python3"), bl(text: "-c"), bl(text: gunbc_ci_sm_decode_payload_program), bv(name: "RESP_FILE")]), path: bv(name: "DEST_FILE")), + ), + bcmd(words: [bl(text: "chmod"), bl(text: "600"), bv(name: "DEST_FILE")]), + bcmd(words: [bl(text: "export"), bcat(parts: [bl(text: concat(mtcollins1_fan_credential_path_env as String, "=")), bv(name: "DEST_FILE")])]), + gunbc_ci_mtcollins1_probe_root_statement(), + ] +} + +fn gunbc_ci_mtcollins1_probe_invoke(stem: String, target: GunbcRunStepTarget) -> String { + join([ + gunbc_ci_mtcollins1_probe_rendered(stmts: gunbc_ci_mtcollins1_probe_credential_statements(stem: stem), set_e: false), "\n", + gunbc_run_command(source_roots: witness_layer_roots, entry: target.entry, function: target.function, claim_run: false), "\n", + ], "") +} + +fn gunbc_ci_mtcollins1_ui_bundle_observe_invoke() -> String { + gunbc_ci_mtcollins1_probe_invoke(stem: "mtcollins1-ui-bundle", target: gunbc_ci_mtcollins1_ui_bundle_observe_target) +} + +fn gunbc_ci_mtcollins1_kvm_observer_observe_invoke() -> String { + gunbc_ci_mtcollins1_probe_invoke(stem: "mtcollins1-kvm-observer", target: gunbc_ci_mtcollins1_kvm_observer_observe_target) +} + +// THE PROBE'S always() RELEASE needs no credential: it releases the observer process it started and the +// unit hold only if this run holds it (gunbc.machine_intake_mtcollins1_kvm_observer_observe +// mtcollins1_kvm_observer_release_wet). +fn gunbc_ci_mtcollins1_kvm_observer_release_invoke() -> String { + join([ + gunbc_ci_mtcollins1_probe_rendered(stmts: [bcmdl(lits: ["set", "-euo", "pipefail"]), gunbc_ci_mtcollins1_probe_root_statement()], set_e: false), "\n", + gunbc_run_command(source_roots: witness_layer_roots, entry: gunbc_ci_mtcollins1_kvm_observer_release_target.entry, function: gunbc_ci_mtcollins1_kvm_observer_release_target.function, claim_run: false), "\n", + ], "") +} + // The in-run ssh-agent teardown โ€” a FRESH per-job effect (each isolated job starts and must kill // its own agent; nothing is memoizable across jobs), named so the workflow step and the // materialization gate's fresh-effect list reference ONE authority (ยง3), rather than an inline diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index 766c896ab03..6847a44eb17 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -3,10 +3,10 @@ module gunbc.fleet_converge_workflow import gunbc.namecheap.observation_artifact { namecheap_observation_receipt_path } import gunbc.namecheap.federation { namecheap_environment, namecheap_provider_resource, namecheap_service_account } -import gunbc.ci_spec { gunbc_ci_namecheap_observe_invoke } +import gunbc.ci_spec { gunbc_ci_namecheap_observe_invoke, gunbc_ci_sm_max_time } import gunbc.runner_microvm_slot_unit { microvm_slot_unit_timeout_stop, microvm_slot_start_receipt_path, microvm_slot_reserve_receipt_path, microvm_slot_reserve_wait_bound } -import std.measure { minute_count, second_count } +import std.measure { Second, minute_count, second_count } import extdeps.languages.yaml.types { YamlKeyValue, yaml_string, yaml_int, yaml_bool, kv } import extdeps.github.actions { @@ -27,6 +27,9 @@ import gunbc.fleet.organization_runner_roster_read { organization_runner_roster_ import gunbc.cloudflare.r2_permission_group_observe { r2_mint_preflight_receipt_path } import gunbc.cloudflare.r2_bucket_ensure { r2_bucket_ensure_receipt_path } import gunbc.machine_intake_mtcollins1_fan_observe { mtcollins1_fan_observation_receipt_path } +import gunbc.machine_intake_mtcollins1_kvm_observer_observe { mtcollins1_kvm_observer_probe_bound, mtcollins1_kvm_observer_probe_artifact_glob } +import gunbc.machine_intake_mtcollins1_kvm_still { mtcollins1_kvm_observer_release_allowance } +import gunbc.machine_intake_mtcollins1_ui_bundle_observe { mtcollins1_ui_bundle_artifact_glob, mtcollins1_ui_bundle_read_bound } import gunbc.fabric_writer_identity_observe { fabric_writer_identity_receipt_path } import gunbc.machine_intake_mtcollins1_census_image_publish { mtcollins1_census_image_receipt_path } import gunbc.machine_intake_mtcollins1_census_member_readback { census_member_readback_receipt_path } @@ -124,6 +127,9 @@ import gunbc.ci_spec { gunbc_ci_app_key_version_verify_mint_script, gunbc_ci_app_key_version_verify_verdict_invoke, gunbc_ci_mtcollins1_fan_observe_invoke, + gunbc_ci_mtcollins1_ui_bundle_observe_invoke, + gunbc_ci_mtcollins1_kvm_observer_observe_invoke, + gunbc_ci_mtcollins1_kvm_observer_release_invoke, gunbc_ci_fabric_writer_identity_observe_invoke, gunbc_ci_mtcollins1_census_image_publish_invoke, gunbc_ci_mtcollins1_census_member_readback_invoke, @@ -298,6 +304,8 @@ type FleetConvergeWorkflowMode | MtCollins1Boot | PairServingD0 | MtCollins1FanObserve + | MtCollins1UiBundleObserve + | MtCollins1KvmObserverObserve | FabricWriterIdentityObserve | MtCollins1CensusImagePublish | MtCollins1CensusMemberReadback @@ -359,6 +367,8 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String NamecheapObserve => "namecheap_observe" PairServingD0 => "pair_serving_d0" MtCollins1FanObserve => "mtcollins1_fan_observe" + MtCollins1UiBundleObserve => "mtcollins1_ui_bundle_observe" + MtCollins1KvmObserverObserve => "mtcollins1_kvm_observer_observe" FabricWriterIdentityObserve => "fabric_writer_identity_observe" MtCollins1CensusImagePublish => "mtcollins1_census_image_publish" MtCollins1CensusMemberReadback => "mtcollins1_census_member_readback" @@ -389,7 +399,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, SparkV41GroupALaunch, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotReserve, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, RunnerBrowserToolchainConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, MtCollins1CensusMemberReadback, MtCollins1CensusQemuHostObserve, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, SparkV41GroupALaunch, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotReserve, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, RunnerBrowserToolchainConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1UiBundleObserve, MtCollins1KvmObserverObserve, MtCollins1CensusImagePublish, MtCollins1CensusMemberReadback, MtCollins1CensusQemuHostObserve, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // // EVERY MODE IS NAMED, AND THE WILDCARD IS DELIBERATELY ABSENT. A `_ => none` would read the same @@ -462,6 +472,8 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc NamecheapObserve => none PairServingD0 => none MtCollins1FanObserve => none + MtCollins1UiBundleObserve => none + MtCollins1KvmObserverObserve => none FabricWriterIdentityObserve => none MtCollins1CensusImagePublish => none MtCollins1CensusMemberReadback => none @@ -591,6 +603,8 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> MtCollins1Boot => FleetSshKeyConsumed PairServingD0 => FleetSshKeyConsumed MtCollins1FanObserve => FleetSshKeyNotConsumed + MtCollins1UiBundleObserve => FleetSshKeyNotConsumed + MtCollins1KvmObserverObserve => FleetSshKeyNotConsumed FabricWriterIdentityObserve => FleetSshKeyNotConsumed MtCollins1CensusImagePublish => FleetSshKeyConsumed MtCollins1CensusMemberReadback => FleetSshKeyConsumed @@ -1214,6 +1228,8 @@ fn fleet_converge_mode_mutation_domain(mode: FleetConvergeWorkflowMode) -> Fleet MtCollins1Boot => ExecutorDomain PairServingD0 => ArmDomain { group: fleet_arm_mutation_group_deepseek_group_a } MtCollins1FanObserve => ExecutorDomain + MtCollins1UiBundleObserve => ExecutorDomain + MtCollins1KvmObserverObserve => ExecutorDomain FabricWriterIdentityObserve => ExecutorDomain MtCollins1CensusImagePublish => ExecutorDomain MtCollins1CensusMemberReadback => ExecutorDomain @@ -2078,6 +2094,105 @@ fn fleet_converge_mtcollins1_fan_observe_step() -> Step { } } +// MT. COLLINS' OWN UI BUNDLE AND ITS DIGEST (gunbc.machine_intake_mtcollins1_ui_bundle_observe): the +// firmware revision over IPMI, then the served source.min.js bytes, only under revision 0.32. Reads +// only, no web session. The receipt and the bytes are uploaded whatever the outcome, because a refused +// run's receipt is the one that says why it holds no bytes; a step condition without a status +// function is implicitly success() and would drop exactly that receipt. +// A READ-ONLY MT. COLLINS PROBE STEP'S BUDGET IS A FOLD, NEVER A LITERAL, over the step's phases in +// order, each counted once and rounded once: gunbc run's load of the probe's closure, whose authority is +// the measured resolve budget (gunbc.fleet_workflow_steps gunbc_ci_gunbc_run_resolve_step_timeout_minutes), +// then the step's own phases -- for a probe, the credential prelude's one bounded Secret Manager read +// (gunbc.ci_spec gunbc_ci_sm_max_time; the prelude's other lines are local file writes) and the probe's +// declared bound; for the always() release, the owned release alone. The flat aux budget +// is what killed the UI bundle's first wet run (36759368077) while it was still typechecking. +fn fleet_converge_mtcollins1_probe_step_timeout_minutes(phases: List) -> Int { + let secs = fold(phases, gunbc_ci_gunbc_run_resolve_step_timeout_minutes * 60, (acc, p) => acc + (second_count(s: p) as Int)) + (secs + 59) / 60 +} + +data fleet_converge_mtcollins1_ui_bundle_observe_step_if: String = fleet_converge_mode_step_if(mode: MtCollins1UiBundleObserve) + +fn fleet_converge_mtcollins1_ui_bundle_observe_step() -> Step { + RunStep { + name: Present { value: "Mt. Collins served UI: firmware revision, then source.min.js, viewer.html and every script they load, each with its sha256 (reads only)" }, + id: Present { value: "mtcollins1_ui_bundle_observe" }, + run: gunbc_ci_mtcollins1_ui_bundle_observe_invoke(), + shell: none, + env: Present { value: [ + kv(key: "WIF_ACCESS_TOKEN", value: yaml_string(s: "${{ steps.wif_auth.outputs.access_token }}")), + ] }, + working_directory: none, + if_condition: Present { value: fleet_converge_mtcollins1_ui_bundle_observe_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: fleet_converge_mtcollins1_probe_step_timeout_minutes(phases: [gunbc_ci_sm_max_time, mtcollins1_ui_bundle_read_bound()]) } + } +} + +fn fleet_converge_mtcollins1_ui_bundle_observe_upload_step() -> Step { + fleet_converge_receipt_upload_step( + id: "mtcollins1_ui_bundle_observe_upload", + name: "Upload Mt. Collins served UI files and receipt", + artifact: "mtcollins1-ui-bundle", + path: mtcollins1_ui_bundle_artifact_glob, + if_condition: join(["always() && ", fleet_converge_mtcollins1_ui_bundle_observe_step_if], "") + ) +} + +// THE REAL KVM OBSERVER AGAINST THE REAL BMC, NO BOOT (gunbc.machine_intake_mtcollins1_kvm_observer_observe): +// under the unit hold, log in, establish the viewer, take one still, release the web session and the +// browser. Nothing is written to media, power, boot device or SEL. The journal, its pre-teardown +// snapshot, the observer log, every still and the receipt are uploaded whatever the outcome: a refused +// or fallen run's journal is the record of where the page navigated before it fell. Its closure still +// loads the fleet's host effects through gunbc.runner_browser_toolchain (the declared host_effect +// frontier), which the resolve budget carries. +data fleet_converge_mtcollins1_kvm_observer_observe_step_if: String = fleet_converge_mode_step_if(mode: MtCollins1KvmObserverObserve) + +fn fleet_converge_mtcollins1_kvm_observer_observe_step() -> Step { + RunStep { + name: Present { value: "Mt. Collins KVM observer, no boot: under the unit hold, log in, establish the viewer, one still, release" }, + id: Present { value: "mtcollins1_kvm_observer_observe" }, + run: gunbc_ci_mtcollins1_kvm_observer_observe_invoke(), + shell: none, + env: Present { value: [ + kv(key: "WIF_ACCESS_TOKEN", value: yaml_string(s: "${{ steps.wif_auth.outputs.access_token }}")), + kv(key: fleet_converge_expected_host_env_name, value: yaml_string(s: "${{ github.event.inputs.host }}")), + ] }, + working_directory: none, + if_condition: Present { value: fleet_converge_mtcollins1_kvm_observer_observe_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: fleet_converge_mtcollins1_probe_step_timeout_minutes(phases: [gunbc_ci_sm_max_time, mtcollins1_kvm_observer_probe_bound()]) } + } +} + +// THE PROBE'S always() RELEASE (gunbc.machine_intake_mtcollins1_kvm_observer_observe +// mtcollins1_kvm_observer_release_wet): after the probe step whatever it did -- success, failure, cancel, +// or its own timeout -- release the observer process this run started and the unit hold only if this run +// holds it. It needs no credential, so it has no prelude and no token. +fn fleet_converge_mtcollins1_kvm_observer_release_step() -> Step { + RunStep { + name: Present { value: "Mt. Collins KVM observer: release this run's observer and its unit hold (always; a no-op if nothing is held)" }, + id: Present { value: "mtcollins1_kvm_observer_release" }, + run: gunbc_ci_mtcollins1_kvm_observer_release_invoke(), + shell: none, + env: none, + working_directory: none, + if_condition: Present { value: join(["always() && ", fleet_converge_mtcollins1_kvm_observer_observe_step_if], "") }, + continue_on_error: none, + timeout_minutes: Present { value: fleet_converge_mtcollins1_probe_step_timeout_minutes(phases: [mtcollins1_kvm_observer_release_allowance]) } + } +} + +fn fleet_converge_mtcollins1_kvm_observer_observe_upload_step() -> Step { + fleet_converge_receipt_upload_step( + id: "mtcollins1_kvm_observer_observe_upload", + name: "Upload Mt. Collins KVM observer journal, stills and receipt", + artifact: "mtcollins1-kvm-observer", + path: mtcollins1_kvm_observer_probe_artifact_glob, + if_condition: join(["always() && ", fleet_converge_mtcollins1_kvm_observer_observe_step_if], "") + ) +} + // THE READING THE WRITER-ROSTER WALL WAITS FOR (gunbc.fabric_writer_identity_observe): from the // fleet host the dispatch names, through tailscale serve, what Tailscale-User-Login the fabric DB's // door sees. Reads only; the receipt is the roster's ground. Dispatched once per fleet host. @@ -3469,6 +3584,11 @@ fn fleet_converge_job() -> Job { fleet_converge_app_key_version_verify_receipt_upload_step(), fleet_converge_mtcollins1_fan_observe_step(), fleet_converge_mtcollins1_fan_observe_receipt_upload_step(), + fleet_converge_mtcollins1_ui_bundle_observe_step(), + fleet_converge_mtcollins1_ui_bundle_observe_upload_step(), + fleet_converge_mtcollins1_kvm_observer_observe_step(), + fleet_converge_mtcollins1_kvm_observer_release_step(), + fleet_converge_mtcollins1_kvm_observer_observe_upload_step(), fleet_converge_fabric_writer_identity_observe_step(), fleet_converge_fabric_writer_identity_observe_receipt_upload_step(), fleet_converge_pair_serving_d0_step(), @@ -3715,6 +3835,8 @@ fn fleet_converge_mode_job_id(mode: FleetConvergeWorkflowMode) -> NonEmptyStr { ApprovalDeviceEnrolmentCodeIssue => fleet_converge_shared_job_id MtCollins1Boot => "mtcollins1-boot" as NonEmptyStr MtCollins1FanObserve => fleet_converge_shared_job_id + MtCollins1UiBundleObserve => fleet_converge_shared_job_id + MtCollins1KvmObserverObserve => fleet_converge_shared_job_id MtCollins1CensusImagePublish => fleet_converge_shared_job_id MtCollins1CensusMemberReadback => fleet_converge_shared_job_id MtCollins1CensusQemuHostObserve => fleet_converge_shared_job_id diff --git a/dag/gunbc/machine_intake/megarac_served_ui_catalog_observation.dag b/dag/gunbc/machine_intake/megarac_served_ui_catalog_observation.dag index 95116b93386..2bbb9e9c70a 100644 --- a/dag/gunbc/machine_intake/megarac_served_ui_catalog_observation.dag +++ b/dag/gunbc/machine_intake/megarac_served_ui_catalog_observation.dag @@ -11,3 +11,10 @@ import std.types { NonEmptyStr } data megarac_spx_ui_surface_path: NonEmptyStr = "artifacts/bmc/mtcollins1-megarac-api-surface.txt" as NonEmptyStr data megarac_spx_ui_surface_sha256: NonEmptyStr = "4fc53fea3fe49bc3ed3b967511654223578d59d963546117ab524df11b419bcd" as NonEmptyStr + +// THE 2026-09-27 READ OF THE BUNDLE ITSELF, as its digest. extdeps.bmc.megarac cites this read for the +// screen-capture surfaces and the KVM wire; the bytes were not committed, so the digest is the only +// thing a later read can be compared with. gunbc.machine_intake_mtcollins1_ui_bundle_observe reports +// match or drift against it and does not refuse on drift: a different digest is a finding about the +// firmware's served UI, not a failed read. +data megarac_source_min_js_sha256_2026_09_27: NonEmptyStr = "5029fae278f7fc8f4dd4ad5eb8e4457e49fa25d4d0158d4bf9fbab09f57d4bf3" as NonEmptyStr diff --git a/dag/gunbc/machine_intake/megarac_ui_features.dag b/dag/gunbc/machine_intake/megarac_ui_features.dag new file mode 100644 index 00000000000..3bfab3d80ea --- /dev/null +++ b/dag/gunbc/machine_intake/megarac_ui_features.dag @@ -0,0 +1,224 @@ +module gunbc.machine_intake_megarac_ui_features + +import extdeps.tools.sha256sum +import extdeps.tools.gzip +import std.algebra { trim } +import std.types { Bool, FilePath, Int, List, NonEmptyStr, String } +import v2.std.optional { Present, Absent } +import std.process { ExitSuccess, ProcessExit, exit_failure } +import extdeps.filesystem.filesystem_io { Filesystem } +import gunbc.machine_intake_megarac_served_ui_catalog_observation { megarac_source_min_js_sha256_2026_09_27 } +import extdeps.external_authority { CitedFigureStanding, TranscribedUncited } +import extdeps.bmc.capability { BmcFirmwareReleaseIdentity, bmc_firmware_release_identity_eq } +import extdeps.bmc.megarac { megarac_firmware_0_32 } + +// THE FEATURE LIST THE VENDOR UI HANDS ITS KVM VIEWER, READ FROM THE UI'S OWN BYTES (eager-owl-205, +// 2026-09-30; operator rule: derived, never hand-typed). The served viewer.min.js (fleet-converge crawl +// run 36787375313) initialises its KVM view with UNGUARDED sessionStorage.features.indexOf(...) reads +// (KVM_SESSION_RECONNECT, VMEDIA_MAX_COUNT_FOR_KVM, HOST_CURSOR_ENABLED_DEFAULT): with `features` unset +// it throws, the view never initialises, and /kvm is never opened -- KVM probe run 36788605665 journalled +// exactly that page error and no WebSocket. The real UI stores JSON.stringify of its +// project-configurations collection, which is seeded from the STATIC `features` module compiled into +// source.min.js: +// o("features",[],function(){var e=[{feature:"RMEDIA_MULTI_IMAGE_SUPPORT"},...];return e}) +// so the list is parsed out of those bytes, and only out of the build this unit's facts are keyed on: +// the served source.min.js must digest to the 2026-09-27 read's sha256, or nothing is read. +// +// THE LIST IS AN OBSERVATION RECEIPT, NOT A DERIVED FACT IN GIT (review 73613; operator decision via +// eager-owl-205, 2026-10-01). The bundle it is read from is AMI's proprietary MegaRAC UI and is not +// redistributable, so it is not committed and no CI job can reach it: a CI-diffed gate is unavailable, +// and that is a DECLARED BOUNDARY, not a below-ceiling choice (DESIGN 4b, outside the modeled guarantee; +// 4d, the typed bet). The receipt names what was read -- the firmware build, the bundle's sha256, the +// run that retained it -- and carries TranscribedUncited with its read obligation: the route that +// re-derives it. What gates it is that route: megarac_ui_features_row_emit refuses any bundle whose +// digest is not the pinned one, so a later fetch either reproduces these names from these bytes or +// refuses, and never silently diverges from them. +data megarac_ui_features_module_open: String = "o(\"features\",[],function()\{var e=[" + +data megarac_ui_features_module_close: String = "];return e}" + +data megarac_ui_feature_key: String = "feature:\"" + +// THE NAMES IN THE MODULE, IN ITS ORDER: every `feature:"NAME"` between the module's opening and its +// `];return e}`. Absent when the module is not in the text, or holds no name. +fn megarac_ui_features_of_bundle(text: String) -> List? { + match split(s: text, delimiter: megarac_ui_features_module_open).skip(n: 1).first() { + Absent => none + Present { value: after } => { + let body = match split(s: after, delimiter: megarac_ui_features_module_close).first() { Present { value: b } => b Absent => "" } + let names = map(split(s: body, delimiter: megarac_ui_feature_key).skip(n: 1), rest => match split(s: rest, delimiter: "\"").first() { Present { value: n } => n Absent => "" }) + if count(names) == 0 { none } else { Present { value: names } } + } + } +} + +// A FEATURE NAME IS THE VENDOR'S IDENTIFIER SHAPE: upper-case letters, digits and underscores, nonempty. +// Anything else is refused rather than written into a JSON string it could escape from. +data megarac_ui_feature_name_characters: String = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" + +fn megarac_ui_feature_name_admitted(name: String) -> Bool { + name != "" && all(split(s: name, delimiter: ""), c => c == "" || string_contains(s: megarac_ui_feature_name_characters, pattern: c)) +} + + +type MegaRacUiFeatures + = MegaRacUiFeaturesRead { names: List, source_sha256: NonEmptyStr } + | MegaRacUiFeaturesUnread { cause: NonEmptyStr } + +// THE READING FROM ALREADY-DECODED BUNDLE TEXT AND ITS DIGEST: refused unless the digest is the pinned +// build's and the module yields only admitted names. +fn megarac_ui_features_reading(observed_sha256: String, text: String) -> MegaRacUiFeatures { + if observed_sha256 != (megarac_source_min_js_sha256_2026_09_27 as String) { + MegaRacUiFeaturesUnread { cause: concat(concat("source.min.js digests to ", observed_sha256), concat(", not the pinned ", megarac_source_min_js_sha256_2026_09_27 as String)) as NonEmptyStr } + } else { + match megarac_ui_features_of_bundle(text: text) { + Absent => MegaRacUiFeaturesUnread { cause: "the features module was not found in source.min.js, or holds no name" } + Present { value: names } => + if all(names, n => megarac_ui_feature_name_admitted(name: n)) { + MegaRacUiFeaturesRead { names: names, source_sha256: megarac_source_min_js_sha256_2026_09_27 } + } else { + MegaRacUiFeaturesUnread { cause: concat("the features module holds a name outside the vendor identifier shape: ", join(filter(names, n => !megarac_ui_feature_name_admitted(name: n)), " ")) as NonEmptyStr } + } + } + } +} + +type MegaRacUiFeaturesReceipt { + firmware: BmcFirmwareReleaseIdentity + bundle_sha256: NonEmptyStr + producing_run: NonEmptyStr + names: List + standing: CitedFigureStanding +} + +// The names, as megarac_ui_features_row_emit wrote them from run 36787375313's retained source.min.js. +data megarac_ui_features_0_32: List = [ + "RMEDIA_MULTI_IMAGE_SUPPORT", + "BIOS_SUPPORT", + "KVM_SESSION_RECONNECT", + "SYSTEM_FIREWALL", + "OEM_PROPRIETARY_LEVEL_SUPPORT", + "HTML5_KVM", + "DUAL_BIOS_FLASH_SUPPORT", + "IPV6_SUPPORT", + "ALLOW_NON_SECURE_COMMUNICATION", + "SET_SENSOR_THRESHOLDS", + "REPLACE_OEM_LOGO", + "EXTENDEDLOG", + "AUTO_RESIZE_KVM_CLIENT_WINDOW", + "SERVICES", + "RMEDIA", + "AUTOVDORECORD_REMOTE", + "CAPTURE_BSOD_JPEG", + "RADIUS_SUPPORT", + "SYSTEM_FIREWALL_TIMEOUT", + "UARTLOG", + "RUNTIME_HOST_LOCK", + "SOL_SUPPORT", + "HPM_SUPPORT", + "ADVISER_SUPPORT", + "AD_SUPPORT", + "MULTIPLE_USER_VMEDIA", + "LDAP_SUPPORT", + "KVM_SESSION_TIMEOUT", + "FIRMWARE_FLASH_SUPPORT", + "CPLD_SUPPORT", + "PTP_SERVER_SUPPORT", + "BOOTFW_SUPPORT", + "HOST_LOCK_AUTO", + "BACKUP_CONFIG", + "SET_SENSOR_THRESHOLDS_RESET", + "SAVE_SELLOG", + "SESSION_MANAGEMENT", + "RMEDIA_MULTI_CD_SUPPORT", + "TSIG", + "IPV6_COMPLIANCE", + "CD_SERVER_APP", + "WEB_DASHBOARD_WIDGETS", + "FOLDER_REDIRECTION", + "CIRCULAR_SEL", + "FW_IMAGE", + "DISABLE_EMPTY_PASSWORD", + "KB_LANG_SELECT_SUPPORT", + "DYNAMIC_KCS_LAN_SUPPORT", + "LMEDIA", + "DUAL_BOOTFW_EEPROM_SUPPORT", + "BLOCK_ICMP_TIMESTAMP", + "SINGLE_PORT_APP", + "MDNS", + "NCSI_SUPPORT", + "IMG_REDIRECTION", + "HD_SERVER_APP", + "PAM_REORDERING", + "EXTENDED_PRIV", + "SNMP", + "WEB_APP_PROXY", + "LMEDIA_MULTI_IMAGE_SUPPORT", + "VMEDIA_MAX_COUNT_FOR_KVM", + "MEDIA_REDIR_READ_WRITE_ONLY", + "RMEDIA_SESSION_RECONNECT", + "PRESERVECONF", + "TIMEZONE_SUPPORT", + "RMEDIA_MULTI_HD_SUPPORT", + "AUTOVDORECORD", + "YES", + "VERSION_CMP_FLASH", + "BMC_RECOVERY", + "NWLINK", + "KVM_OR_MEDIA_AVAILABLE", +] + +data megarac_ui_features_0_32_receipt: MegaRacUiFeaturesReceipt = MegaRacUiFeaturesReceipt { + firmware: megarac_firmware_0_32, + bundle_sha256: megarac_source_min_js_sha256_2026_09_27, + producing_run: "36787375313" as NonEmptyStr, + names: megarac_ui_features_0_32, + standing: TranscribedUncited { read_obligation: "re-derive by the fleet-converge mtcollins1_ui_bundle_observe mode, which retains the served source.min.js with its sha256 as the mtcollins1-ui-bundle artifact, then gunbc.machine_intake_megarac_ui_features megarac_ui_features_row_emit over that file: it refuses unless the bytes digest to megarac_source_min_js_sha256_2026_09_27, and otherwise writes these names" as NonEmptyStr }, +} + +data megarac_ui_features_receipts: List = [megarac_ui_features_0_32_receipt] + +// THE LIST FOR A FIRMWARE BUILD: the receipt observed for that build, or none -- a build no receipt +// covers has no list, and the observer does not drive its viewer. +fn megarac_ui_features(firmware: BmcFirmwareReleaseIdentity) -> List? { + match filter(megarac_ui_features_receipts, r => bmc_firmware_release_identity_eq(a: r.firmware, b: firmware)).first() { + Present { value: r } => Present { value: r.names } + Absent => none + } +} + +// THE ROW'S EMITTER (the entry megarac_ui_features_0_32_receipt's read obligation names): read a RETAINED +// source.min.js as served -- the mtcollins1-ui-bundle artifact of a crawl run -- refuse unless it +// digests to the pinned 2026-09-27 build, decode it, parse the features module, and write the +// extdeps row's text to `out`. Run: +// gunbc run --source-root dag --source-root src/v2 --entry dag/gunbc/machine_intake/megarac_ui_features.dag +// --function megarac_ui_features_row_emit --arg bundle= --arg out= +fn megarac_ui_features_row_text(names: List) -> String { + concat(concat("data megarac_ui_features_0_32: List = [\n", join(map(names, n => concat(concat(" \"", n), "\",")), "\n")), "\n]\n") +} + +fn megarac_ui_features_row_emit(bundle: String, out: String) -> ProcessExit { + let digest = sha256sum.Sha256.DigestFile(path: bundle as NonEmptyStr) + let observed = match split(s: trim(s: digest.line), delimiter: " ").first() { Present { value: d } => d Absent => "" } + if !digest.success || observed == "" { + return exit_failure(reason: concat("megarac ui features: the retained bundle could not be digested: ", digest.line)) + } + let decoded = gzip.Gzip.DecodeToStdout(path: bundle as NonEmptyStr) + if !decoded.success { + return exit_failure(reason: concat("megarac ui features: the retained bundle could not be decoded: ", trim(s: decoded.stderr))) + } + match megarac_ui_features_reading(observed_sha256: observed, text: decoded.text) { + MegaRacUiFeaturesUnread { cause: c } => exit_failure(reason: concat("megarac ui features: ", c as String)) + MegaRacUiFeaturesRead { names: n, source_sha256: _ } => { + let w = Filesystem.Write(path: out, content: megarac_ui_features_row_text(names: n)) + if w.success { ExitSuccess } else { exit_failure(reason: concat("megarac ui features: the row could not be written: ", w.error)) } + } + } +} + +fn megarac_ui_features_text(f: MegaRacUiFeatures) -> String { + match f { + MegaRacUiFeaturesRead { names: n, source_sha256: s } => join([to_string(count(n)), " features read from source.min.js sha256 ", s as String], "") + MegaRacUiFeaturesUnread { cause: c } => concat("features UNREAD: ", c as String) + } +} diff --git a/dag/gunbc/machine_intake/mtcollins1_access_observation.dag b/dag/gunbc/machine_intake/mtcollins1_access_observation.dag index 1579a55b6b1..2f81043a7cc 100644 --- a/dag/gunbc/machine_intake/mtcollins1_access_observation.dag +++ b/dag/gunbc/machine_intake/mtcollins1_access_observation.dag @@ -1,6 +1,7 @@ module gunbc.machine_intake_mtcollins1_access_observation import std.types { EpochMs, Int, NonEmptyStr, String } +import std.measure { Second, second } import std.content_hash { ContentHash, Sha256Digest, Sha256Hash } import extdeps.boards.types { AmpereMtCollins } import extdeps.bmc.endpoint { AmiMegaRac, BmcControllerEndpoint } @@ -283,3 +284,11 @@ data mtcollins1_capability_row: BmcFirmwareReleaseCapabilityRow = BmcFirmwareRel CapabilityOemRemoteMedia, ], } + +// THE BOUND ON ONE READ-ONLY ipmitool EXCHANGE WITH THIS UNIT'S CONTROLLER, as `timeout --kill-after` +// operands. A fact about reaching this BMC, not about any one run that reads it: the boot's diagnostic +// bundle, its media attach and the UI bundle observation all bound their IPMI reads by it. It lives +// here, with the endpoint it bounds, so a read-only mode does not import the boot closure to learn it. +data mtcollins1_boot_ipmi_read_deadline: Second = second(count: 20) + +data mtcollins1_boot_ipmi_kill_after: Second = second(count: 5) diff --git a/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag b/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag index fe8f4db3705..d9d513870e5 100644 --- a/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag +++ b/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag @@ -55,7 +55,7 @@ import gunbc.machine_intake_ampere_socket_console_observation { } import gunbc.machine_intake_host_capture_envelope { HostCaptureLiveStream, host_capture_lines } import gunbc.machine_intake_mtcollins1_memory_census_observation { mtcollins1_slots, mtcollins1_census_sol_artifact_path } -import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint } +import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint, mtcollins1_boot_ipmi_read_deadline, mtcollins1_boot_ipmi_kill_after } import gunbc.machine_intake_mtcollins1_kvm_still { KvmObserverRecord, kvm_observer_record_text } import gunbc.machine_intake_mount_establishment { RemoteMediaMountStanding } import gunbc.machine_intake_megarac_media_attach { @@ -1514,10 +1514,6 @@ fn mtcollins1_boot_bundle_text(bundle: MtCollins1BootDiagnosticBundle) -> String // extra pages; a firmware that pages more finely spends the rest and says so. data mtcollins1_boot_redfish_read_deadline: Second = second(count: 15) -data mtcollins1_boot_ipmi_read_deadline: Second = second(count: 20) - -data mtcollins1_boot_ipmi_kill_after: Second = second(count: 5) - // The two SEL snapshots, the one SDR dump, the one sensor read and every SMpro probe the post-attempt // pass can send // (gunbc.machine_intake_mtcollins1_smpro_observation mtcollins1_smpro_probe_count) each run under diff --git a/dag/gunbc/machine_intake/mtcollins1_boot_dry_realization.dag b/dag/gunbc/machine_intake/mtcollins1_boot_dry_realization.dag index d94f731c5ae..05945e3b829 100644 --- a/dag/gunbc/machine_intake/mtcollins1_boot_dry_realization.dag +++ b/dag/gunbc/machine_intake/mtcollins1_boot_dry_realization.dag @@ -36,6 +36,7 @@ import gunbc.machine_intake_mtcollins1_kvm_still { KvmInstant, KvmJournalEvent, KvmJournalRendered, KvmJournalRenderRefused, kvm_journal_line, KvmJournalConnectionRequested, KvmJournalConnectionOpen, KvmJournalEstablished, KvmJournalStill, KvmJournalStopRequested, KvmJournalSessionRelease, KvmJournalBrowserClose, KvmJournalStopped, + KvmJournalRefused, KvmNoFrame, } // THE DRY REALIZATION OF ONE mtcollins1 BOOT ATTEMPT'S WHOLE EFFECT DEMAND. The scenario world @@ -483,8 +484,14 @@ fn operation_input_list(call: OperationCall, name: String) -> List? { } // kvm_observer_launch_in's operands end the node command: base URL, user, credential path, dir, -// attempt, cadence, frame wait, periodic limit, playwright module. -data kvm_observer_operand_count: Int = 9 +// attempt, cadence, frame wait, periodic limit, playwright module, viewer feature-list path. +data kvm_observer_operand_count: Int = 10 + +// THE VIEWER READS ITS FEATURE LIST UNGUARDED (KVM probe run 36788605665): the healthy viewer reaches +// established only when the feature-list operand names a file the launch wrote; without one the vendor's +// viewer throws in its KVM view's initialize() and never opens /kvm, which the observer journals as a +// refusal for want of a frame. +data kvm_observer_features_operand: Int = 9 fn kvm_operand(command: List, i: Int) -> String { match command.skip(n: count(command) - kvm_observer_operand_count + i).first() { Present { value: v } => v Absent => "" } @@ -511,11 +518,17 @@ fn owned_launch_handler(w: MtCollins1BootWorld, call: OperationCall) -> Operatio Present { value: at } => { let dir = kvm_operand(command: command, i: 3) let p = ModeledProcess { pid: w.worker.next_pid, start_time: start_ticks(w: w, now: call.now), comm: "node", cmdline: join(command, " "), capture_path: "", diagnostic_path: "", alive: true } - let journal = kvm_lines(events: [ - KvmJournalConnectionRequested { at: at }, - KvmJournalConnectionOpen { at: at, gen: kvm_observer_generation }, - KvmJournalEstablished { at: at, host: concat("\"", concat(url_host(base_url: kvm_operand(command: command, i: 0)), "\"")), session: w.screen.session, attempt: kvm_operand(command: command, i: 4), gen: kvm_observer_generation }, - ]) + let features = kvm_operand(command: command, i: kvm_observer_features_operand) + let seeded = features != "" && match fs_file(fs: w.fs, path: features) { Present { value: _ } => true Absent => false } + let journal = if seeded { + kvm_lines(events: [ + KvmJournalConnectionRequested { at: at }, + KvmJournalConnectionOpen { at: at, gen: kvm_observer_generation }, + KvmJournalEstablished { at: at, host: concat("\"", concat(url_host(base_url: kvm_operand(command: command, i: 0)), "\"")), session: w.screen.session, attempt: kvm_operand(command: command, i: 4), gen: kvm_observer_generation }, + ]) + } else { + kvm_lines(events: [KvmJournalRefused { at: at, refusal: KvmNoFrame { detail: "\"the viewer was opened without its feature list\"" } }]) + } match journal { KvmModeledRefused { event_word: word, would_write: line } => OperationHarnessFault { reason: concat("kvm_journal_line refused the modeled ", concat(word, concat(" line: ", line))) as NonEmptyStr } KvmModeledRendered { text: t } => { diff --git a/dag/gunbc/machine_intake/mtcollins1_boot_run.dag b/dag/gunbc/machine_intake/mtcollins1_boot_run.dag index 38df1e95d15..400709684ec 100644 --- a/dag/gunbc/machine_intake/mtcollins1_boot_run.dag +++ b/dag/gunbc/machine_intake/mtcollins1_boot_run.dag @@ -62,7 +62,7 @@ import extdeps.bootloader.grub { GrubConsoleReport, GrubNothingStated, GrubBannerOnly, GrubStillTrying, GrubBootFailed, grub_console_report, GrubError, grub_error_text, } -import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint, mtcollins1_capability_row } +import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint, mtcollins1_capability_row, mtcollins1_boot_ipmi_read_deadline, mtcollins1_boot_ipmi_kill_after } import gunbc.host_reset_return_run { host_reset_bmc_credential_path_env } import v2.std.optional { Present } import gunbc.actions_run_binding { ActionsVariableRead, actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent, actions_variable_trimmed } @@ -81,7 +81,6 @@ import gunbc.machine_intake_mtcollins1_boot_diagnostic_bundle { MtCollins1BootDiagnosticBundle, SelSnapshotReading, SelSnapshotNotTaken, inventory_not_taken, SdrCacheReading, SdrCacheNotTaken, mtcollins1_boot_sdr_cache, SensorsNotTaken, mtcollins1_boot_sensors, SmproProbed, SmproNotTaken, - mtcollins1_boot_ipmi_read_deadline, mtcollins1_boot_ipmi_kill_after, mtcollins1_boot_sel_snapshot, mtcollins1_boot_inventory, mtcollins1_boot_console, mtcollins1_boot_write_bundle, mtcollins1_boot_outcome_after_bundle, mtcollins1_boot_bundle_findings, bundle_write_receipt_line, mtcollins1_boot_diagnostics_allowance, AttemptIdentityReading, AttemptRunId, AttemptIdentityUnavailable, attempt_identity_text, diff --git a/dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag b/dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag new file mode 100644 index 00000000000..308a0063bce --- /dev/null +++ b/dag/gunbc/machine_intake/mtcollins1_kvm_observer_observe.dag @@ -0,0 +1,276 @@ +module gunbc.machine_intake_mtcollins1_kvm_observer_observe + +import std.types { Bool, Int, List, NonEmptyStr } +import std.string_type { String } +import std.algebra { trim } +import std.measure { Second, second, second_count } +import std.process { ExitSuccess, ExitFailure, ProcessExit, exit_failure } +import v2.std.optional { Present, Absent } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.github.actions_environment { github_run_id_variable_name } +import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent } +import gunbc.runner_browser_toolchain { runner_browser_toolchain_here_wet } +import gunbc.owned_process { OwnedRelease, OwnedReleased, OwnedAlreadyGone, OwnedReleaseNotOwned, OwnedNotRecorded, OwnedForeign, owned_process_release, owned_release_text } +import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint } +import gunbc.machine_intake_mtcollins1_bmc_secure_observation { mtcollins1_secured_account } +import gunbc.machine_intake_mtcollins1_fan_observe { FanCredentialPathReady, FanCredentialPathUnset, FanCredentialPathEmpty, mtcollins1_fan_credential_path, mtcollins1_fan_credential_path_env } +import gunbc.machine_intake_mtcollins1_boot_admission { mtcollins1_boot_host_admission } +import gunbc.machine_intake_mtcollins1_maintenance_hold { + UnitHeld, UnitHoldRefused, mtcollins1_kvm_observer_acquire_unit_hold, unit_hold_release, + ProbeHoldReleased, ProbeHoldAlreadyFree, ProbeHoldNotOurs, ProbeHoldStale, ProbeHoldUnreleased, + mtcollins1_kvm_observer_release_unit_hold, probe_hold_release_outcome_text, +} +import gunbc.machine_intake_mtcollins1_kvm_still { + KvmObserverRecord, KvmObserverStart, KvmObserverStarted, KvmObserverNotStarted, + KvmHandoffAdmitted, KvmHandoffRefused, kvm_handoff_admission, + KvmStillHashed, KvmTeardownRecorded, KvmTeardownNotNeeded, KvmTerminalStillAnswered, + KvmSessionRelease, KvmSessionReleased, KvmSessionNoSession, KvmBrowserClose, KvmBrowserClosed, KvmBrowserNotLaunched, kvm_session_release_of, kvm_browser_close_of, + KvmJournalRead, KvmJournalText, KvmJournalAbsent, KvmJournalUnread, kvm_journal_read_at, kvm_journal_events_of, kvm_last_result, + mtcollins1_kvm_observer_start, mtcollins1_kvm_observer_finish, mtcollins1_kvm_observer_dir, + mtcollins1_bmc_firmware, mtcollins1_kvm_still_allowance, + kvm_observer_record_text, +} + +// THE REAL KVM OBSERVER AGAINST THE REAL BMC, WITH NO BOOT (operator-approved brief, 2026-09-30). +// The observer (gunbc.machine_intake_mtcollins1_kvm_still) is the boot's; this mode drives the SAME +// functions the boot drives -- mtcollins1_kvm_observer_start, the handoff admission that waits for +// establishment, and mtcollins1_kvm_observer_finish, which asks one terminal still, requests the stop, +// releases the web session and the browser and hashes the still -- and nothing else. So it exercises, +// on the hardware, exactly the login and the MegaRAC SPA navigation after DOMContentLoaded that ended +// run 36721915217, and the journal records where the page went (`navigated` lines, written and read +// only through kvm_journal_line / KvmJournalWord). +// +// NOTHING IS WRITTEN TO THE CONTROLLER'S MEDIA, POWER, BOOT DEVICE OR SEL. The web session it opens is +// its own and is DELETEd on every reachable exit; the only other effect is the KVM viewer seat, which +// is why the run holds the unit (gunbc.machine_intake_mtcollins1_maintenance_hold +// mtcollins1_kvm_observer_acquire_unit_hold): the boot takes that seat under the same hold and refuses +// its handoff when the seat is busy, so an unheld probe could make a concurrent boot refuse. +// +// THE ARTIFACTS are the observer's own directory -- journal, journal.pre-teardown, observer.log and +// every still-N.jpg -- under mtcollins1_kvm_observer_dir(attempt), plus this mode's receipt, which is +// kvm_observer_record_text of the record, written on every path that started the observer. + +// The attempt names the observer's directory (mtcollins1_kvm_observer_dir); a probe's carries this +// prefix, so its directories, and the receipt beside them, are one glob apart from every boot's. +data mtcollins1_kvm_observer_probe_prefix: String = "probe" + +fn mtcollins1_kvm_observer_probe_attempt(run_id: NonEmptyStr) -> String { + join([mtcollins1_kvm_observer_probe_prefix, "-", run_id as String], "") +} + +data mtcollins1_kvm_observer_observe_receipt_path: String = concat(mtcollins1_kvm_observer_dir(attempt: mtcollins1_kvm_observer_probe_prefix), "-receipt.txt") + +// Everything the probe keeps: every probe's observer directory and its receipt. +data mtcollins1_kvm_observer_probe_artifact_glob: String = concat(mtcollins1_kvm_observer_dir(attempt: mtcollins1_kvm_observer_probe_prefix), "-*") + +// WHAT THE PROBE'S OWN BOUNDED WAITS COST AT MOST is the observer's own allowance +// (mtcollins1_kvm_still_allowance: the establishment wait, the terminal-still wait, the stop wait and +// the owned release), counted once. The step budget adds gunbc run's load and the credential prelude to +// this on the workflow side, which owns those authorities. +fn mtcollins1_kvm_observer_probe_bound() -> Second { + mtcollins1_kvm_still_allowance() +} + +// THE VERDICT IS A CONJUNCTION OF OBSERVED FACTS, EACH READ FROM THE RECORD: established for this +// host and attempt (the handoff admission's own test), the one terminal still answered and hashed, and +// the teardown seen through -- stopped line written, session released, browser closed, process gone. +type KvmProbeVerdict + = KvmProbeHeld + | KvmProbeFell { reason: NonEmptyStr } + +// THE TEARDOWN IS JUDGED BY ITS WORDS, NOT BY SUBSTRINGS: the session release must read exactly +// `released` and the browser close exactly `closed` (kvm_session_release_of / kvm_browser_close_of, the +// journal's own RS rows), the stopped line must have been seen, and the owned process released or +// already gone. A "release-failed detail=..." whose detail mentions "released" is a failed release. +// Absent means seen through; Present names the first fact that was not. +fn kvm_probe_teardown_gap(rec: KvmObserverRecord) -> NonEmptyStr? { + match rec.teardown { + KvmTeardownRecorded { terminal: _, snapshot: _, requested_at: _, stopped_seen: st, session_release: sr, browser_close: bc, process: p } => + if !st { Present { value: "the observer's stopped line was not seen" as NonEmptyStr } } + else if !(match kvm_session_release_of(result: sr) { KvmSessionReleased => true _ => false }) { Present { value: concat("the web session was not released: ", sr) as NonEmptyStr } } + else if !(match kvm_browser_close_of(result: bc) { KvmBrowserClosed => true _ => false }) { Present { value: concat("the browser was not closed: ", bc) as NonEmptyStr } } + else if !(match p { OwnedReleased { pid: _ } => true OwnedAlreadyGone { pid: _ } => true _ => false }) { Present { value: "the observer process was not seen released" as NonEmptyStr } } + else { none } + KvmTeardownNotNeeded { reason: r } => Present { value: concat("no teardown was recorded: ", r as String) as NonEmptyStr } + } +} + +fn kvm_probe_still_answered(rec: KvmObserverRecord) -> Bool { + match rec.teardown { + KvmTeardownRecorded { terminal: t, snapshot: _, requested_at: _, stopped_seen: _, session_release: _, browser_close: _, process: _ } => + match t { KvmTerminalStillAnswered { line: _ } => true _ => false } + _ => false + } +} + +fn kvm_probe_hashed_stills(rec: KvmObserverRecord) -> Int { + count(filter(rec.stills, s => match s { KvmStillHashed { at: _, seq: _, reason: _, path: _, digest: _ } => true _ => false })) +} + +fn kvm_probe_verdict(admission: String, established: Bool, rec: KvmObserverRecord) -> KvmProbeVerdict { + if !established { KvmProbeFell { reason: concat("the observer was not established: ", admission) as NonEmptyStr } } + else if !kvm_probe_still_answered(rec: rec) { KvmProbeFell { reason: "the terminal still was not answered" } } + else if kvm_probe_hashed_stills(rec: rec) < 1 { KvmProbeFell { reason: "no still was retained with its digest" } } + else { + match kvm_probe_teardown_gap(rec: rec) { + Present { value: g } => KvmProbeFell { reason: g } + Absent => KvmProbeHeld + } + } +} + +fn kvm_probe_conclude(rec: KvmObserverRecord, verdict: KvmProbeVerdict) -> ProcessExit { + let verdict_line = match verdict { KvmProbeHeld => "verdict=held" KvmProbeFell { reason: r } => concat("verdict=FELL: ", r as String) } + let body = join([concat("subject=", mtcollins1_endpoint.host as String), verdict_line, kvm_observer_record_text(rec: rec)], "\n") + let kept = Filesystem.Write(path: mtcollins1_kvm_observer_observe_receipt_path, content: concat(body, "\n")) + let outcome = match verdict { + KvmProbeHeld => ExitSuccess + KvmProbeFell { reason: r } => exit_failure(reason: concat("mtcollins1 kvm observer: ", r as String)) + } + if kept.success { outcome } else { + match outcome { + ExitSuccess => exit_failure(reason: concat("mtcollins1 kvm observer: the probe held and its receipt could not be written: ", kept.error)) + other => other + } + } +} + +// WHAT THE HELD PROBE LEAVES: its exit, and the record its seat decision is read from. +type KvmProbeHeldRun { + outcome: ProcessExit + record: KvmObserverRecord +} + +fn mtcollins1_kvm_observer_observe_held(username: NonEmptyStr, password_file: NonEmptyStr, run_id: NonEmptyStr) -> KvmProbeHeldRun { + let attempt = mtcollins1_kvm_observer_probe_attempt(run_id: run_id) + let start = mtcollins1_kvm_observer_start( + firmware: mtcollins1_bmc_firmware, + bmc_host: mtcollins1_endpoint.host, + username: username, + password_file: password_file as String, + toolchain: runner_browser_toolchain_here_wet(), + attempt: attempt, + ) + let admission = kvm_handoff_admission(start: start) + let established = match admission { KvmHandoffAdmitted { session: _ } => true KvmHandoffRefused { cause: _ } => false } + let admission_text = match admission { KvmHandoffAdmitted { session: s } => concat("established, web session ", s) KvmHandoffRefused { cause: c } => c } + let rec = mtcollins1_kvm_observer_finish(start: start, attempt: attempt, terminal_order: 1) + KvmProbeHeldRun { outcome: kvm_probe_conclude(rec: rec, verdict: kvm_probe_verdict(admission: admission_text, established: established, rec: rec)), record: rec } +} + +// REFUSALS BEFORE THE FIRST REQUEST, EACH NAMING WHAT IS MISSING: off srv1 (the hold store's host), +// no secured account, no credential file, no run id, or the unit already held. +fn mtcollins1_kvm_observer_observe_wet() -> ProcessExit +{ + match mtcollins1_boot_host_admission() { + ExitSuccess => {} + other => return other + } + let run_id = match actions_variable_read(name: github_run_id_variable_name) { + ActionsVariablePresent { value: v } => trim(s: v) + ActionsVariableAbsent { variable: _, detail: _ } => "" + } + if run_id == "" { + return exit_failure(reason: concat(concat("mtcollins1 kvm observer: ", github_run_id_variable_name as String), " is unset or empty, so the unit hold has no owner to name; contacting NO host")) + } + match mtcollins1_secured_account() { + Absent => exit_failure(reason: "mtcollins1 kvm observer: the BMC is not BmcSecured, so this run holds no account to log in to the viewer with; contacting NO host") + Present { value: username } => + match mtcollins1_fan_credential_path() { + FanCredentialPathUnset => exit_failure(reason: join(["mtcollins1 kvm observer: no ", mtcollins1_fan_credential_path_env as String, " in the environment; contacting NO host"], "")) + FanCredentialPathEmpty => exit_failure(reason: join(["mtcollins1 kvm observer: ", mtcollins1_fan_credential_path_env as String, " is set but empty; contacting NO host"], "")) + FanCredentialPathReady { path: p } => + match mtcollins1_kvm_observer_acquire_unit_hold(run_id: run_id as NonEmptyStr) { + UnitHoldRefused { reason: r } => exit_failure(reason: concat("mtcollins1 kvm observer: ", r as String)) + UnitHeld { proof: proof } => { + let run = mtcollins1_kvm_observer_observe_held(username: username, password_file: p, run_id: run_id as NonEmptyStr) + match kvm_probe_seat_hold_gap(rec: run.record) { + Absent => unit_hold_release(proof: proof, outcome: run.outcome) + Present { value: gap } => exit_failure(reason: kvm_probe_hold_kept_text(gap: gap, outcome: run.outcome)) + } + } + } + } + } +} + +// THE UNIT HOLD IS FREED ONLY OVER A SEAT OBSERVED LET GO (side-chat review, blocker 4a). The hold is +// what keeps a boot from starting while the probe may still own the BMC's one KVM viewer seat, so it is +// released only when every part of that ownership is observed ended, by its typed word: +// - the web session reads `released`, or `no-session` (no login was ever accepted, so none exists); +// - the browser reads `closed`, or `no-browser` (none was launched); +// - the observer process is observed exited (released, already gone, or its pid now names another +// process), or was never recorded because the observer never launched. +// `login-unobserved` (a login may have been accepted), any failed, http- or unrecognised word, an +// unresolved process or an unreadable journal KEEPS the hold and says why; the BMC's own session +// timeout frees the seat, and the hold is then cleared by the operator or, once it lands, by +// observed-dead recovery. Absent is "let go"; Present names the first part not observed ended. +fn kvm_probe_seat_parts_gap(session: KvmSessionRelease, browser: KvmBrowserClose, process: OwnedRelease, launched: Bool) -> NonEmptyStr? { + let session_ended = match session { KvmSessionReleased => true KvmSessionNoSession => true _ => false } + let browser_ended = match browser { KvmBrowserClosed => true KvmBrowserNotLaunched => true _ => false } + let process_ended = match process { + OwnedReleased { pid: _ } => true + OwnedAlreadyGone { pid: _ } => true + OwnedReleaseNotOwned { standing: s } => match s { OwnedForeign { pid: _, observed_start: _ } => true OwnedNotRecorded { detail: _ } => !launched _ => false } + _ => false + } + if !session_ended { Present { value: "the web session was not observed released" as NonEmptyStr } } + else if !browser_ended { Present { value: "the browser was not observed closed" as NonEmptyStr } } + else if !process_ended { Present { value: concat("the observer process was not observed exited: ", owned_release_text(r: process)) as NonEmptyStr } } + else { none } +} + +fn kvm_probe_seat_hold_gap(rec: KvmObserverRecord) -> NonEmptyStr? { + match rec.teardown { + KvmTeardownNotNeeded { reason: _ } => none + KvmTeardownRecorded { terminal: _, snapshot: _, requested_at: _, stopped_seen: _, session_release: sr, browser_close: bc, process: p } => + kvm_probe_seat_parts_gap(session: kvm_session_release_of(result: sr), browser: kvm_browser_close_of(result: bc), process: p, launched: true) + } +} + +fn kvm_probe_hold_kept_text(gap: NonEmptyStr, outcome: ProcessExit) -> String { + join(["mtcollins1 kvm observer: unit hold KEPT, because ", gap as String, "; the probe's own outcome was: ", match outcome { ExitSuccess => "success" ExitFailure { code: _, reason: r } => r }], "") +} + +// THE SAME RULE FROM THE ARTIFACTS ALONE, for the always() step: the journal's last session-release and +// browser-close words and the owned release it just performed. A journal that cannot be read keeps the +// hold; no journal and no pid record means the observer never launched. +fn kvm_probe_seat_gap_from_journal(journal: KvmJournalRead, process: OwnedRelease) -> NonEmptyStr? { + match journal { + KvmJournalUnread { cause: c } => Present { value: concat("the observer journal could not be read: ", c) as NonEmptyStr } + KvmJournalAbsent => kvm_probe_seat_parts_gap(session: KvmSessionNoSession, browser: KvmBrowserNotLaunched, process: process, launched: false) + KvmJournalText { text: _ } => { + let events = kvm_journal_events_of(read: journal) + kvm_probe_seat_parts_gap(session: kvm_session_release_of(result: kvm_last_result(events: events, session: true)), browser: kvm_browser_close_of(result: kvm_last_result(events: events, session: false)), process: process, launched: true) + } + } +} + +// THE always() RELEASE (a separate step after the probe's, run whatever the probe step did: success, +// failure, cancel or its own timeout). First the observer process this run started, by its recorded +// (pid, start time) -- so the viewer seat and the web session it holds are let go -- then the unit hold, +// only if THIS run holds it (gunbc.machine_intake_mtcollins1_maintenance_hold +// mtcollins1_kvm_observer_release_unit_hold). A probe that finished cleanly left nothing, and both +// halves report that as a no-op. It fails loudly only when something this run owns stays held. +fn mtcollins1_kvm_observer_release_wet() -> ProcessExit +{ + let run_id = match actions_variable_read(name: github_run_id_variable_name) { + ActionsVariablePresent { value: v } => trim(s: v) + ActionsVariableAbsent { variable: _, detail: _ } => "" + } + if run_id == "" { + return exit_failure(reason: concat(concat("mtcollins1 kvm observer release: ", github_run_id_variable_name as String), " is unset or empty, so this run cannot name the hold it may release")) + } + let dir = mtcollins1_kvm_observer_dir(attempt: mtcollins1_kvm_observer_probe_attempt(run_id: run_id as NonEmptyStr)) + let process = owned_process_release(pid_path: concat(dir, "/observer.pid")) + match kvm_probe_seat_gap_from_journal(journal: kvm_journal_read_at(dir: dir), process: process) { + Present { value: gap } => return exit_failure(reason: join(["mtcollins1 kvm observer release: process ", owned_release_text(r: process), "; unit hold KEPT, because ", gap as String], "")) + Absent => {} + } + let hold = mtcollins1_kvm_observer_release_unit_hold(run_id: run_id as NonEmptyStr) + let line = join(["mtcollins1 kvm observer release: process ", owned_release_text(r: process), "; unit hold ", probe_hold_release_outcome_text(o: hold)], "") + match hold { + ProbeHoldUnreleased { cause: _ } => exit_failure(reason: line) + _ => ExitSuccess + } +} diff --git a/dag/gunbc/machine_intake/mtcollins1_kvm_still.dag b/dag/gunbc/machine_intake/mtcollins1_kvm_still.dag index f7f7caf8bec..490296f9226 100644 --- a/dag/gunbc/machine_intake/mtcollins1_kvm_still.dag +++ b/dag/gunbc/machine_intake/mtcollins1_kvm_still.dag @@ -15,7 +15,9 @@ import gunbc.runner_browser_toolchain { RunnerBrowserToolchainResolution, BrowserToolchainReady, BrowserToolchainNotReady, BrowserToolchainUnresolved, RunnerBrowserToolchainLocation, runner_browser_toolchain_node_command, runner_browser_toolchain_standing_text, } +import gunbc.machine_intake_megarac_ui_features { megarac_ui_features } import extdeps.bmc.megarac { + megarac_ui_features_json, ScreenCaptureFactKnown, ScreenCaptureFirmwareUncatalogued, ScreenCaptureClientSideOnly, KvmClientCanvasJpeg, megarac_screen_capture_standing, megarac_firmware_0_32, } @@ -27,7 +29,6 @@ import v2.std.algebra { filter } import extdeps.shell import extdeps.filesystem.filesystem_io { Filesystem } import extdeps.tools.sleep { sleep_delay_seconds_second_carrier_projection } -import gunbc.machine_intake_sol_hold // THE BMC KVM SCREEN AS BOOT EVIDENCE (operator ruling 2026-09-27; route A, eager-owl-205 the same // day). On run 36335369059 the SOL collector died about 12 s into Linux and the watch went blind, @@ -76,9 +77,9 @@ data mtcollins1_bmc_firmware: BmcFirmwareReleaseIdentity = megarac_firmware_0_32 // UI's root page. It used to: it loaded "/" to borrow an origin and ran the login inside it, and on // that run the journal read "page.evaluate: Execution context was destroyed, most likely because of a // navigation" with session-release login-unobserved -- the root document was replaced under the login. -// That the vendor SPA redirects itself after DOMContentLoaded is INFERRED from that message, not read -// from its bundles; the repair does not depend on it, because no step now runs in a document the -// observer did not navigate to itself. The only in-page evaluations left are on viewer.html: the +// That the vendor SPA reloads itself to #login is read from its bundle (see the navigation note below); +// the repair does not depend on it, because no step now runs in a document the observer did not +// navigate to itself. The only in-page evaluations left are on viewer.html: the // canvas wait and the toDataURL. // // ONE OBSERVER, HELD FOR THE WHOLE BOOT (side-chat acceptance (a), 2026-09-28). The observer is a @@ -96,6 +97,20 @@ data mtcollins1_bmc_firmware: BmcFirmwareReleaseIdentity = megarac_firmware_0_32 // that exists but cannot be read (toDataURL throws) is acquisition-failed cause=canvas-unreadable. An // unchanged picture is never read as a stale one: the observer does not compare pixels at all. // +// EVERY MAIN-FRAME NAVIGATION IS JOURNALLED (operator, 2026-09-30: the navigation is MODELED, not +// avoided). What the SPA does is now read from its own bundle, not inferred: the UI fetched by +// fleet-converge run 36778446928 (gunbc.machine_intake_mtcollins1_ui_bundle_observe, firmware 0.32, +// digest matching the 2026-09-27 read) routes by Backbone hash routing, and on session expiry, on an +// unauthorized answer and on its logout paths sets location.href "#login" and then calls +// location.reload() -- a FULL document reload, which is what destroys an evaluate in flight. The only +// document the observer loads is viewer.html; every commit of its main frame is a `navigated` line (seq, +// the observer's phase -- viewer while it opens the viewer, held after establishment -- and the URL the +// frame committed to), so a reload to #login under the held viewer is on the journal with the instant it +// happened, before the connection-lost or failed line it causes. A navigation is recorded, never retried +// into a different answer. The viewer's own console errors and warnings, and any uncaught page error, +// are journalled too (page-console / page-error, with the phase): fleet-converge run 36779479938 loaded +// viewer.html and never requested /kvm, and what the viewer said about why is the first thing to read. +// // THE SEAT. A held seat (viewer_count > 0) refuses with its count -- an operator's session is never // evicted -- and every exit the process reaches (stop file, SIGTERM, refusal, failure) DELETEs its web // session and closes the browser, journalling stopped session=. The @@ -141,6 +156,9 @@ data kvm_event_failed: KvmJournalWord = KvmJournalWord { id: "failed", word: "fa data kvm_event_stop_requested: KvmJournalWord = KvmJournalWord { id: "stopRequested", word: "stop-requested" } data kvm_event_stopped: KvmJournalWord = KvmJournalWord { id: "stopped", word: "stopped" } data kvm_event_trigger_scan_failed: KvmJournalWord = KvmJournalWord { id: "triggerScanFailed", word: "trigger-scan-failed" } +data kvm_event_navigated: KvmJournalWord = KvmJournalWord { id: "navigated", word: "navigated" } +data kvm_event_page_console: KvmJournalWord = KvmJournalWord { id: "pageConsole", word: "page-console" } +data kvm_event_page_error: KvmJournalWord = KvmJournalWord { id: "pageError", word: "page-error" } data kvm_cause_seat_busy: KvmJournalWord = KvmJournalWord { id: "seatBusy", word: "seat-busy" } data kvm_cause_login: KvmJournalWord = KvmJournalWord { id: "login", word: "login" } @@ -153,9 +171,25 @@ data kvm_cause_credential_unreadable: KvmJournalWord = KvmJournalWord { id: "cre data kvm_cause_not_connected: KvmJournalWord = KvmJournalWord { id: "notConnected", word: "not-connected" } data kvm_cause_connection_changed_during_acquisition: KvmJournalWord = KvmJournalWord { id: "connectionChangedDuringAcquisition", word: "connection-changed-during-acquisition" } data kvm_cause_canvas_unreadable: KvmJournalWord = KvmJournalWord { id: "canvasUnreadable", word: "canvas-unreadable" } - -data kvm_journal_event_words: List = [kvm_event_established, kvm_event_connection_requested, kvm_event_connection_refused, kvm_event_connection_open, kvm_event_connection_lost, kvm_event_connection_error, kvm_event_still, kvm_event_acquisition_failed, kvm_event_session_release, kvm_event_browser_close, kvm_event_refused, kvm_event_failed, kvm_event_stop_requested, kvm_event_stopped, kvm_event_trigger_scan_failed] -data kvm_journal_cause_words: List = [kvm_cause_seat_busy, kvm_cause_login, kvm_cause_no_kvm_service, kvm_cause_no_frame, kvm_cause_no_kvm_frame, kvm_cause_no_kvm_websocket, kvm_cause_toolchain, kvm_cause_credential_unreadable, kvm_cause_not_connected, kvm_cause_connection_changed_during_acquisition, kvm_cause_canvas_unreadable] +data kvm_cause_features_unreadable: KvmJournalWord = KvmJournalWord { id: "featuresUnreadable", word: "features-unreadable" } + +// THE TEARDOWN'S RESULT WORDS: the leading token of a session-release or browser-close result. The +// observer script spells them only through the RS vocabulary below, and kvm_session_release_of / +// kvm_browser_close_of read them back through the same rows, so a teardown is judged by its word and +// never by a substring of its detail. +data kvm_result_released: KvmJournalWord = KvmJournalWord { id: "released", word: "released" } +data kvm_result_http_prefix: KvmJournalWord = KvmJournalWord { id: "httpPrefix", word: "http-" } +data kvm_result_release_failed: KvmJournalWord = KvmJournalWord { id: "releaseFailed", word: "release-failed" } +data kvm_result_login_unobserved: KvmJournalWord = KvmJournalWord { id: "loginUnobserved", word: "login-unobserved" } +data kvm_result_no_session: KvmJournalWord = KvmJournalWord { id: "noSession", word: "no-session" } +data kvm_result_closed: KvmJournalWord = KvmJournalWord { id: "closed", word: "closed" } +data kvm_result_no_browser: KvmJournalWord = KvmJournalWord { id: "noBrowser", word: "no-browser" } +data kvm_result_close_failed: KvmJournalWord = KvmJournalWord { id: "closeFailed", word: "close-failed" } + +data kvm_journal_result_words: List = [kvm_result_released, kvm_result_http_prefix, kvm_result_release_failed, kvm_result_login_unobserved, kvm_result_no_session, kvm_result_closed, kvm_result_no_browser, kvm_result_close_failed] + +data kvm_journal_event_words: List = [kvm_event_established, kvm_event_connection_requested, kvm_event_connection_refused, kvm_event_connection_open, kvm_event_connection_lost, kvm_event_connection_error, kvm_event_still, kvm_event_acquisition_failed, kvm_event_session_release, kvm_event_browser_close, kvm_event_refused, kvm_event_failed, kvm_event_stop_requested, kvm_event_stopped, kvm_event_trigger_scan_failed, kvm_event_navigated, kvm_event_page_console, kvm_event_page_error] +data kvm_journal_cause_words: List = [kvm_cause_seat_busy, kvm_cause_login, kvm_cause_no_kvm_service, kvm_cause_no_frame, kvm_cause_no_kvm_frame, kvm_cause_no_kvm_websocket, kvm_cause_toolchain, kvm_cause_credential_unreadable, kvm_cause_not_connected, kvm_cause_connection_changed_during_acquisition, kvm_cause_canvas_unreadable, kvm_cause_features_unreadable] // THE LINE LAYOUT, DECLARED ONCE PER EVENT: the key=value fields an event word carries, in the order // the observer script writes them. kvm_journal_event reads its fields through these rows and @@ -180,6 +214,9 @@ data kvm_layout_session_release: KvmJournalLayout = KvmJournalLayout { event: kv data kvm_layout_browser_close: KvmJournalLayout = KvmJournalLayout { event: kvm_event_browser_close, keys: ["result"] } data kvm_layout_stopped: KvmJournalLayout = KvmJournalLayout { event: kvm_event_stopped, keys: ["session", "browser"] } data kvm_layout_trigger_scan_failed: KvmJournalLayout = KvmJournalLayout { event: kvm_event_trigger_scan_failed, keys: ["detail"] } +data kvm_layout_navigated: KvmJournalLayout = KvmJournalLayout { event: kvm_event_navigated, keys: ["seq", "phase", "url"] } +data kvm_layout_page_console: KvmJournalLayout = KvmJournalLayout { event: kvm_event_page_console, keys: ["phase", "level", "detail"] } +data kvm_layout_page_error: KvmJournalLayout = KvmJournalLayout { event: kvm_event_page_error, keys: ["phase", "detail"] } // THE REFUSED LINE'S KEYS, PER CAUSE: `cause` first, then the cause's own field. type KvmRefusalLayout { @@ -204,10 +241,10 @@ fn kvm_journal_words_js(name: String, words: List) -> String { } fn mtcollins1_kvm_observer_vocabulary_js() -> String { - concat(kvm_journal_words_js(name: "EV", words: kvm_journal_event_words), kvm_journal_words_js(name: "CA", words: kvm_journal_cause_words)) + concat(concat(kvm_journal_words_js(name: "EV", words: kvm_journal_event_words), kvm_journal_words_js(name: "CA", words: kvm_journal_cause_words)), kvm_journal_words_js(name: "RS", words: kvm_journal_result_words)) } -data mtcollins1_kvm_observer_script_body: String = "const fs=require('fs');const path=require('path');const [base,user,pwFile,dir,attempt,cadenceMs,frameMs,maxPeriodic,pwModule]=process.argv.slice(1);const J=path.join(dir,'journal');function j(ev,kv)\{const t=Date.now();try\{fs.appendFileSync(J,t+' '+new Date(t).toISOString()+' '+ev+(kv?' '+kv:'')+'\\n')}catch(e)\{try\{process.stderr.write('journal write failed: '+ev+' '+String(e&&e.message)+'\\n')}catch(e2)\{}}}function one(e)\{return String(e&&e.message||e).replace(/[^ -~]+/g,' ').replace(/ +/g,' ').slice(0,300)}let browser=null,page=null,api=null,stopping=false,login='not-attempted',token=null,seq=0,periodic=0;let gen=0,live=null;async function teardown(why,code)\{if(stopping)return;stopping=true; j(EV.stopRequested,'by='+why); let rel; if(login==='accepted'&&api&&token)\{try\{const r=await api.delete(base+'/api/session',\{headers:\{'X-CSRFTOKEN':token}});rel=r.ok()?'released':'http-'+r.status()}catch(e)\{rel='release-failed detail='+JSON.stringify(one(e))}} else if(login==='unobserved')\{rel='login-unobserved'} else\{rel='no-session'} j(EV.sessionRelease,'result='+rel); let closed='closed';try\{if(browser)await browser.close();else closed='no-browser'}catch(e)\{closed='close-failed detail='+JSON.stringify(one(e))} j(EV.browserClose,'result='+closed); j(EV.stopped,'session='+rel.split(' ')[0]+' browser='+closed.split(' ')[0]); process.exit(code)}process.on('SIGTERM',()=>teardown('sigterm',0));process.on('SIGINT',()=>teardown('sigint',0));async function still(reason)\{seq+=1;const n=seq;const g=gen;const startedMs=Date.now(); if(!live||live.gen!==g)\{j(EV.acquisitionFailed,'seq='+n+' reason='+reason+' gen='+g+' cause='+CA.notConnected+'');return} try\{const url=await page.evaluate(()=>\{const c=document.getElementById('kvm');if(!c)throw new Error('no canvas#kvm');return c.toDataURL('image/jpeg')}); if(!live||live.gen!==g)\{j(EV.acquisitionFailed,'seq='+n+' reason='+reason+' gen='+g+' started_ms='+startedMs+' cause='+CA.connectionChangedDuringAcquisition+'');return} if(!url.startsWith('data:image/jpeg;base64,'))throw new Error('toDataURL answered '+url.slice(0,40)); const b=Buffer.from(url.slice(url.indexOf(',')+1),'base64');const f='still-'+n+'.jpg';fs.writeFileSync(path.join(dir,f),b); j(EV.still,'seq='+n+' reason='+reason+' gen='+g+' started_ms='+startedMs+' file='+f+' bytes='+b.length)} catch(e)\{j(EV.acquisitionFailed,'seq='+n+' reason='+reason+' gen='+g+' started_ms='+startedMs+' cause='+CA.canvasUnreadable+' detail='+JSON.stringify(one(e)))}}(async()=>\{ let pw;try\{pw=fs.readFileSync(pwFile,'utf8')}catch(e)\{j(EV.refused,'cause='+CA.credentialUnreadable+' detail='+JSON.stringify(one(e)));return teardown('refused',14)} let chromium;try\{chromium=require(pwModule).chromium}catch(e)\{j(EV.refused,'cause='+CA.toolchain+' detail='+JSON.stringify(one(e)));return teardown('refused',13)} try\{browser=await chromium.launch(\{headless:true})}catch(e)\{j(EV.refused,'cause='+CA.toolchain+' detail='+JSON.stringify(one(e)));return teardown('refused',13)} try\{const ctx=await browser.newContext(\{ignoreHTTPSErrors:true,viewport:\{width:1280,height:1024}});api=ctx.request;page=await ctx.newPage(); const expectedHost=new URL(base).host; login='unobserved'; const lr=await api.post(base+'/api/session',\{form:\{username:user,password:pw}});let lb=null;try\{lb=await lr.json()}catch(e)\{}const reply=\{status:lr.status(),b:lb}; if(reply.status!==200||!reply.b||reply.b.ok!==0)\{login='refused';j(EV.refused,'cause='+CA.login+' http='+reply.status);return teardown('refused',11)} login='accepted';token=reply.b.CSRFToken; await ctx.addInitScript(([h,b,u])=>\{if(location.host!==h)return;sessionStorage.setItem('garc',b.CSRFToken);sessionStorage.setItem('privilege_id',String(b.privilege));sessionStorage.setItem('extended_privilege',String(b.extendedpriv));sessionStorage.setItem('session_id',String(b.racsession_id));sessionStorage.setItem('username',u);sessionStorage.setItem('kvm_access','1');sessionStorage.setItem('vmedia_access','1')},[expectedHost,reply.b,user]); const sa=await(await api.get(base+'/api/settings/services',\{headers:\{'X-CSRFTOKEN':token}})).json();const kr=Array.isArray(sa)?sa.find(s=>s.service_name==='kvm'):null;const seat=kr?kr.viewer_count:null; if(seat===null||seat===undefined)\{j(EV.refused,'cause='+CA.noKvmService+'');return teardown('refused',15)} if(seat!==0)\{j(EV.refused,'cause='+CA.seatBusy+' viewer_count='+seat);return teardown('refused',10)} page.on('websocket',ws=>\{let u;try\{u=new URL(ws.url())}catch(e)\{return} if(u.host!==expectedHost||u.pathname!=='/kvm')return; j(EV.connectionRequested,'url='+JSON.stringify(ws.url()));let mine=null; ws.on('framereceived',()=>\{if(!mine)\{gen+=1;mine=\{gen:gen};live=mine;j(EV.connectionOpen,'gen='+mine.gen+' url='+JSON.stringify(ws.url()))}}); ws.on('socketerror',e=>\{j(EV.connectionError,'gen='+(mine?mine.gen:0)+' detail='+JSON.stringify(one(e)));if(mine&&live===mine)live=null}); ws.on('close',()=>\{if(mine&&live===mine)\{live=null;j(EV.connectionLost,'gen='+mine.gen)}else if(!mine)\{j(EV.connectionRefused,'url='+JSON.stringify(ws.url()))}})}); await page.goto(base+'/viewer.html',\{waitUntil:'domcontentloaded'}); try\{await page.waitForFunction(()=>\{const c=document.getElementById('kvm');return !!c&&c.width>1&&c.height>1},null,\{timeout:Number(frameMs)})} catch(e)\{j(EV.refused,'cause='+CA.noFrame+' detail='+JSON.stringify(one(e)));return teardown('refused',12)} const deadline=Date.now()+Number(frameMs);while(!live&&Date.now()setTimeout(r,100))} if(!live)\{j(EV.refused,'cause='+CA.noKvmFrame+'');return teardown('refused',12)} j(EV.established,'host='+JSON.stringify(expectedHost)+' session='+String(reply.b.racsession_id)+' attempt='+attempt+' gen='+live.gen); let next=Date.now()+Number(cadenceMs); while(!stopping)\{ if(fs.existsSync(path.join(dir,'stop')))\{return teardown('stop-file',0)} let tf=[];try\{tf=fs.readdirSync(dir).filter(n=>n.startsWith('trigger-')).sort()}catch(e)\{j(EV.triggerScanFailed,'detail='+JSON.stringify(one(e)))} for(const n of tf)\{const p=path.join(dir,n);try\{fs.unlinkSync(p)}catch(e)\{continue}await still('trigger:'+n.slice(8))} if(Date.now()>=next&&periodicsetTimeout(r,250))} }catch(e)\{j(EV.failed,'detail='+JSON.stringify(one(e)));return teardown('failed',1)}})();" +data mtcollins1_kvm_observer_script_body: String = "const fs=require('fs');const path=require('path');const [base,user,pwFile,dir,attempt,cadenceMs,frameMs,maxPeriodic,pwModule,featuresFile]=process.argv.slice(1);const J=path.join(dir,'journal');function j(ev,kv)\{const t=Date.now();try\{fs.appendFileSync(J,t+' '+new Date(t).toISOString()+' '+ev+(kv?' '+kv:'')+'\\n')}catch(e)\{try\{process.stderr.write('journal write failed: '+ev+' '+String(e&&e.message)+'\\n')}catch(e2)\{}}}function one(e)\{return String(e&&e.message||e).replace(/[^ -~]+/g,' ').replace(/ +/g,' ').slice(0,300)}let browser=null,page=null,api=null,stopping=false,login='not-attempted',token=null,seq=0,periodic=0;let gen=0,live=null;let nav=0,phase='launch';async function teardown(why,code)\{if(stopping)return;stopping=true; j(EV.stopRequested,'by='+why); let rel; if(login==='accepted'&&api&&token)\{try\{const r=await api.delete(base+'/api/session',\{headers:\{'X-CSRFTOKEN':token}});rel=r.ok()?RS.released:RS.httpPrefix+r.status()}catch(e)\{rel=RS.releaseFailed+' detail='+JSON.stringify(one(e))}} else if(login==='unobserved')\{rel=RS.loginUnobserved} else\{rel=RS.noSession} j(EV.sessionRelease,'result='+rel); let closed=RS.closed;try\{if(browser)await browser.close();else closed=RS.noBrowser}catch(e)\{closed=RS.closeFailed+' detail='+JSON.stringify(one(e))} j(EV.browserClose,'result='+closed); j(EV.stopped,'session='+rel.split(' ')[0]+' browser='+closed.split(' ')[0]); process.exit(code)}process.on('SIGTERM',()=>teardown('sigterm',0));process.on('SIGINT',()=>teardown('sigint',0));async function still(reason)\{seq+=1;const n=seq;const g=gen;const startedMs=Date.now(); if(!live||live.gen!==g)\{j(EV.acquisitionFailed,'seq='+n+' reason='+reason+' gen='+g+' cause='+CA.notConnected+'');return} try\{const url=await page.evaluate(()=>\{const c=document.getElementById('kvm');if(!c)throw new Error('no canvas#kvm');return c.toDataURL('image/jpeg')}); if(!live||live.gen!==g)\{j(EV.acquisitionFailed,'seq='+n+' reason='+reason+' gen='+g+' started_ms='+startedMs+' cause='+CA.connectionChangedDuringAcquisition+'');return} if(!url.startsWith('data:image/jpeg;base64,'))throw new Error('toDataURL answered '+url.slice(0,40)); const b=Buffer.from(url.slice(url.indexOf(',')+1),'base64');const f='still-'+n+'.jpg';fs.writeFileSync(path.join(dir,f),b); j(EV.still,'seq='+n+' reason='+reason+' gen='+g+' started_ms='+startedMs+' file='+f+' bytes='+b.length)} catch(e)\{j(EV.acquisitionFailed,'seq='+n+' reason='+reason+' gen='+g+' started_ms='+startedMs+' cause='+CA.canvasUnreadable+' detail='+JSON.stringify(one(e)))}}(async()=>\{ let pw;try\{pw=fs.readFileSync(pwFile,'utf8')}catch(e)\{j(EV.refused,'cause='+CA.credentialUnreadable+' detail='+JSON.stringify(one(e)));return teardown('refused',14)} let chromium;try\{chromium=require(pwModule).chromium}catch(e)\{j(EV.refused,'cause='+CA.toolchain+' detail='+JSON.stringify(one(e)));return teardown('refused',13)} try\{browser=await chromium.launch(\{headless:true})}catch(e)\{j(EV.refused,'cause='+CA.toolchain+' detail='+JSON.stringify(one(e)));return teardown('refused',13)} try\{const ctx=await browser.newContext(\{ignoreHTTPSErrors:true,viewport:\{width:1280,height:1024}});api=ctx.request;page=await ctx.newPage(); page.on('framenavigated',f=>\{if(f===page.mainFrame())\{nav+=1;j(EV.navigated,'seq='+nav+' phase='+phase+' url='+JSON.stringify(f.url()))}}); page.on('console',m=>\{const l=m.type();if(l==='error'||l==='warning')j(EV.pageConsole,'phase='+phase+' level='+l+' detail='+JSON.stringify(one(m.text())))}); page.on('pageerror',e=>\{j(EV.pageError,'phase='+phase+' detail='+JSON.stringify(one(e)))}); const expectedHost=new URL(base).host; login='unobserved'; const lr=await api.post(base+'/api/session',\{form:\{username:user,password:pw}});let lb=null;try\{lb=await lr.json()}catch(e)\{}const reply=\{status:lr.status(),b:lb}; if(reply.status!==200||!reply.b||reply.b.ok!==0)\{login='refused';j(EV.refused,'cause='+CA.login+' http='+reply.status);return teardown('refused',11)} login='accepted';token=reply.b.CSRFToken; let feats='';if(featuresFile)\{try\{feats=fs.readFileSync(featuresFile,'utf8')}catch(e)\{j(EV.refused,'cause='+CA.featuresUnreadable+' detail='+JSON.stringify(one(e)));return teardown('refused',14)}} await ctx.addInitScript(([h,b,u,f])=>\{if(location.host!==h)return;if(f)sessionStorage.setItem('features',f);sessionStorage.setItem('garc',b.CSRFToken);sessionStorage.setItem('privilege_id',String(b.privilege));sessionStorage.setItem('extended_privilege',String(b.extendedpriv));sessionStorage.setItem('session_id',String(b.racsession_id));sessionStorage.setItem('username',u);sessionStorage.setItem('kvm_access','1');sessionStorage.setItem('vmedia_access','1')},[expectedHost,reply.b,user,feats]); const sa=await(await api.get(base+'/api/settings/services',\{headers:\{'X-CSRFTOKEN':token}})).json();const kr=Array.isArray(sa)?sa.find(s=>s.service_name==='kvm'):null;const seat=kr?kr.viewer_count:null; if(seat===null||seat===undefined)\{j(EV.refused,'cause='+CA.noKvmService+'');return teardown('refused',15)} if(seat!==0)\{j(EV.refused,'cause='+CA.seatBusy+' viewer_count='+seat);return teardown('refused',10)} page.on('websocket',ws=>\{let u;try\{u=new URL(ws.url())}catch(e)\{return} if(u.host!==expectedHost||u.pathname!=='/kvm')return; j(EV.connectionRequested,'url='+JSON.stringify(ws.url()));let mine=null; ws.on('framereceived',()=>\{if(!mine)\{gen+=1;mine=\{gen:gen};live=mine;j(EV.connectionOpen,'gen='+mine.gen+' url='+JSON.stringify(ws.url()))}}); ws.on('socketerror',e=>\{j(EV.connectionError,'gen='+(mine?mine.gen:0)+' detail='+JSON.stringify(one(e)));if(mine&&live===mine)live=null}); ws.on('close',()=>\{if(mine&&live===mine)\{live=null;j(EV.connectionLost,'gen='+mine.gen)}else if(!mine)\{j(EV.connectionRefused,'url='+JSON.stringify(ws.url()))}})}); phase='viewer';await page.goto(base+'/viewer.html',\{waitUntil:'domcontentloaded'}); try\{await page.waitForFunction(()=>\{const c=document.getElementById('kvm');return !!c&&c.width>1&&c.height>1},null,\{timeout:Number(frameMs)})} catch(e)\{j(EV.refused,'cause='+CA.noFrame+' detail='+JSON.stringify(one(e)));return teardown('refused',12)} const deadline=Date.now()+Number(frameMs);while(!live&&Date.now()setTimeout(r,100))} if(!live)\{j(EV.refused,'cause='+CA.noKvmFrame+'');return teardown('refused',12)} j(EV.established,'host='+JSON.stringify(expectedHost)+' session='+String(reply.b.racsession_id)+' attempt='+attempt+' gen='+live.gen); phase='held';let next=Date.now()+Number(cadenceMs); while(!stopping)\{ if(fs.existsSync(path.join(dir,'stop')))\{return teardown('stop-file',0)} let tf=[];try\{tf=fs.readdirSync(dir).filter(n=>n.startsWith('trigger-')).sort()}catch(e)\{j(EV.triggerScanFailed,'detail='+JSON.stringify(one(e)))} for(const n of tf)\{const p=path.join(dir,n);try\{fs.unlinkSync(p)}catch(e)\{continue}await still('trigger:'+n.slice(8))} if(Date.now()>=next&&periodicsetTimeout(r,250))} }catch(e)\{j(EV.failed,'detail='+JSON.stringify(one(e)));return teardown('failed',1)}})();" data mtcollins1_kvm_observer_script: String = concat(mtcollins1_kvm_observer_vocabulary_js(), mtcollins1_kvm_observer_script_body) @@ -293,6 +330,9 @@ type KvmJournalEvent | KvmJournalBrowserClose { at: KvmInstant, result: String } | KvmJournalStopped { at: KvmInstant, session: String, browser: String } | KvmJournalTriggerScanFailed { at: KvmInstant, detail: String } + | KvmJournalNavigated { at: KvmInstant, seq: String, phase: String, url: String } + | KvmJournalPageConsole { at: KvmInstant, phase: String, level: String, detail: String } + | KvmJournalPageError { at: KvmInstant, phase: String, detail: String } | KvmJournalUnparsed { line: String } // A field's value: the word after " key=", or for the free-text keys (detail, url), the rest of the @@ -353,6 +393,9 @@ fn kvm_journal_event(line: String) -> KvmJournalEvent { else if event == kvm_event_stop_requested.word { KvmJournalStopRequested { at: at, by: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_stop_requested.keys, i: 0)) } } else if event == kvm_event_stopped.word { KvmJournalStopped { at: at, session: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_stopped.keys, i: 0)), browser: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_stopped.keys, i: 1)) } } else if event == kvm_event_trigger_scan_failed.word { KvmJournalTriggerScanFailed { at: at, detail: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_trigger_scan_failed.keys, i: 0)) } } + else if event == kvm_event_navigated.word { KvmJournalNavigated { at: at, seq: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_navigated.keys, i: 0)), phase: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_navigated.keys, i: 1)), url: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_navigated.keys, i: 2)) } } + else if event == kvm_event_page_console.word { KvmJournalPageConsole { at: at, phase: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_page_console.keys, i: 0)), level: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_page_console.keys, i: 1)), detail: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_page_console.keys, i: 2)) } } + else if event == kvm_event_page_error.word { KvmJournalPageError { at: at, phase: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_page_error.keys, i: 0)), detail: kvm_field(rest: rest, key: kvm_key(keys: kvm_layout_page_error.keys, i: 1)) } } else { KvmJournalUnparsed { line: line } } } } @@ -447,6 +490,9 @@ fn kvm_journal_candidate_line(event: KvmJournalEvent) -> String { KvmJournalBrowserClose { at: a, result: r } => kvm_layout_line(at: a, layout: kvm_layout_browser_close, values: [r]) KvmJournalStopped { at: a, session: se, browser: b } => kvm_layout_line(at: a, layout: kvm_layout_stopped, values: [se, b]) KvmJournalTriggerScanFailed { at: a, detail: d } => kvm_layout_line(at: a, layout: kvm_layout_trigger_scan_failed, values: [d]) + KvmJournalNavigated { at: a, seq: n, phase: p, url: u } => kvm_layout_line(at: a, layout: kvm_layout_navigated, values: [n, p, u]) + KvmJournalPageConsole { at: a, phase: p, level: l, detail: d } => kvm_layout_line(at: a, layout: kvm_layout_page_console, values: [p, l, d]) + KvmJournalPageError { at: a, phase: p, detail: d } => kvm_layout_line(at: a, layout: kvm_layout_page_error, values: [p, d]) KvmJournalUnparsed { line: l } => l } } @@ -769,7 +815,11 @@ fn mtcollins1_kvm_observer_start( else if attempt == "" { KvmObserverNotStarted { reason: "no attempt identity is bound, so the observer could not be named for it" } } else { match toolchain { - BrowserToolchainReady { location: loc, standing: _ } => kvm_observer_launch(base_url: concat("https://", bmc_host as String), host: bmc_host as String, username: username, password_file: password_file, toolchain: loc, attempt: attempt) + BrowserToolchainReady { location: loc, standing: _ } => + match megarac_ui_features(firmware: firmware) { + Absent => KvmObserverNotStarted { reason: "no viewer feature list is observed for this firmware (gunbc.machine_intake_megarac_ui_features megarac_ui_features), and the viewer's KVM view reads it unguarded; no viewer was driven" } + Present { value: names } => kvm_observer_launch(base_url: concat("https://", bmc_host as String), host: bmc_host as String, username: username, password_file: password_file, toolchain: loc, attempt: attempt, features_json: megarac_ui_features_json(names: names)) + } BrowserToolchainNotReady { standing: st } => KvmObserverNotStarted { reason: concat("the runner browser toolchain is not ready: ", runner_browser_toolchain_standing_text(s: st)) as NonEmptyStr } BrowserToolchainUnresolved { cause: c } => KvmObserverNotStarted { reason: concat("the runner browser toolchain is unresolved: ", c) as NonEmptyStr } } @@ -781,13 +831,13 @@ fn mtcollins1_kvm_observer_start( // THE LAUNCH OVER A BASE URL. The boot passes https://; the protocol controls pass a local // transport. `host` is the host:port the established line must name. -fn kvm_observer_launch(base_url: String, host: String, username: NonEmptyStr, password_file: String, toolchain: RunnerBrowserToolchainLocation, attempt: String) -> KvmObserverStart { +fn kvm_observer_launch(base_url: String, host: String, username: NonEmptyStr, password_file: String, toolchain: RunnerBrowserToolchainLocation, attempt: String, features_json: String) -> KvmObserverStart { let dir = mtcollins1_kvm_observer_dir(attempt: attempt) let made = shell.Mkdir.Parents(path: dir as FilePath) if !made.success { return KvmObserverNotStarted { reason: concat("the observer directory could not be made: ", dir) as NonEmptyStr } } - kvm_observer_launch_in(dir_given: dir, base_url: base_url, host: host, username: username, password_file: password_file, toolchain: toolchain, attempt: attempt) + kvm_observer_launch_in(dir_given: dir, base_url: base_url, host: host, username: username, password_file: password_file, toolchain: toolchain, attempt: attempt, features_json: features_json) } // ONE ABSOLUTE LOCATION (hold on gunbc#12492): the directory is canonicalised once, here, and that @@ -795,12 +845,16 @@ fn kvm_observer_launch(base_url: String, host: String, username: NonEmptyStr, pa // reads the journal, stills and triggers from and writes the stop request to. A relative directory // passed as both working directory and operand would be resolved twice -- //journal -- and the // boot and the observer would address different files. -fn kvm_observer_launch_in(dir_given: String, base_url: String, host: String, username: NonEmptyStr, password_file: String, toolchain: RunnerBrowserToolchainLocation, attempt: String) -> KvmObserverStart { +fn kvm_observer_launch_in(dir_given: String, base_url: String, host: String, username: NonEmptyStr, password_file: String, toolchain: RunnerBrowserToolchainLocation, attempt: String, features_json: String) -> KvmObserverStart { let resolved = shell.Path.Canonical(path: dir_given as FilePath) if !resolved.success { return KvmObserverNotStarted { reason: concat("the observer directory could not be resolved: ", concat(dir_given, concat(": ", trim(s: resolved.stderr)))) as NonEmptyStr } } let dir = trim(s: resolved.stdout) + let features_path = concat(dir, "/features.json") + if features_json != "" && !Filesystem.Write(path: features_path, content: features_json).success { + return KvmObserverNotStarted { reason: concat("the viewer feature list could not be written: ", features_path) as NonEmptyStr } + } let node = runner_browser_toolchain_node_command( location: toolchain, directory: dir, @@ -811,6 +865,7 @@ fn kvm_observer_launch_in(dir_given: String, base_url: String, host: String, use to_string(millisecond_count(m: second_to_millisecond(s: mtcollins1_kvm_observer_frame_wait)) as Int), to_string(mtcollins1_kvm_observer_periodic_limit), toolchain.playwright_module as String, + if features_json == "" { "" } else { features_path }, ], ) let launched = gunbc.owned_process.launch.LaunchOwned(pid_path: concat(dir, "/observer.pid") as NonEmptyStr, log_path: concat(dir, "/observer.log") as NonEmptyStr, command: argv_words(command: node)) @@ -884,6 +939,51 @@ fn kvm_last_result(events: List, session: Bool) -> String { } } +// THE TEARDOWN RESULTS, TYPED: the leading token of a result, read against the RS rows. Anything +// after it -- a detail, an " at " suffix -- is carried, never matched, so a failure whose detail +// happens to contain "released" or "closed" is a failure. +type KvmSessionRelease + = KvmSessionReleased + | KvmSessionReleaseHttp { status: String } + | KvmSessionReleaseFailed { detail: String } + | KvmSessionLoginUnobserved + | KvmSessionNoSession + | KvmSessionReleaseUnrecognized { raw: String } + +type KvmBrowserClose + = KvmBrowserClosed + | KvmBrowserNotLaunched + | KvmBrowserCloseFailed { detail: String } + | KvmBrowserCloseUnrecognized { raw: String } + +fn kvm_result_leading(result: String) -> String { + match split(s: trim(s: result), delimiter: " ").first() { Present { value: w } => w Absent => "" } +} + +fn kvm_result_rest(result: String) -> String { + join(split(s: trim(s: result), delimiter: " ").skip(n: 1), " ") +} + +fn kvm_session_release_of(result: String) -> KvmSessionRelease { + let w = kvm_result_leading(result: result) + if w == kvm_result_released.word { KvmSessionReleased } + else if w == kvm_result_release_failed.word { KvmSessionReleaseFailed { detail: kvm_result_rest(result: result) } } + else if w == kvm_result_login_unobserved.word { KvmSessionLoginUnobserved } + else if w == kvm_result_no_session.word { KvmSessionNoSession } + else if starts_with(s: w, prefix: kvm_result_http_prefix.word) && string_length(w) > string_length(kvm_result_http_prefix.word) { + KvmSessionReleaseHttp { status: substring(s: w, start: string_length(kvm_result_http_prefix.word), end: string_length(w)) } + } + else { KvmSessionReleaseUnrecognized { raw: result } } +} + +fn kvm_browser_close_of(result: String) -> KvmBrowserClose { + let w = kvm_result_leading(result: result) + if w == kvm_result_closed.word { KvmBrowserClosed } + else if w == kvm_result_no_browser.word { KvmBrowserNotLaunched } + else if w == kvm_result_close_failed.word { KvmBrowserCloseFailed { detail: kvm_result_rest(result: result) } } + else { KvmBrowserCloseUnrecognized { raw: result } } +} + // THE END: the terminal still, the pre-teardown snapshot from a journal actually read, the boot's // timestamped stop request, the bounded wait for the observer's stopped line, then the owned-process // release -- always attempted and observed, so an observer that journalled "stopped" but is still @@ -980,6 +1080,9 @@ fn kvm_event_text(e: KvmJournalEvent) -> String { KvmJournalBrowserClose { at: a, result: r } => concat(a.iso, " browser close: ", r) KvmJournalStopped { at: a, session: s, browser: b } => concat(a.iso, " stopped; session ", s, "; browser ", b) KvmJournalTriggerScanFailed { at: a, detail: d } => concat(a.iso, concat(" trigger scan failed: ", d)) + KvmJournalNavigated { at: a, seq: n, phase: p, url: u } => concat(a.iso, " main frame navigated #", n, " during ", p, " to ", u) + KvmJournalPageConsole { at: a, phase: p, level: l, detail: d } => concat(a.iso, " viewer console ", l, " during ", p, ": ", d) + KvmJournalPageError { at: a, phase: p, detail: d } => concat(a.iso, " VIEWER PAGE ERROR during ", p, ": ", d) KvmJournalUnparsed { line: l } => concat("unparsed journal line: ", l) } } diff --git a/dag/gunbc/machine_intake/mtcollins1_maintenance_hold.dag b/dag/gunbc/machine_intake/mtcollins1_maintenance_hold.dag index 6ff1983dbc5..c66a1a9d4cd 100644 --- a/dag/gunbc/machine_intake/mtcollins1_maintenance_hold.dag +++ b/dag/gunbc/machine_intake/mtcollins1_maintenance_hold.dag @@ -5,7 +5,9 @@ import std.process { ProcessExit, ExitSuccess, ExitFailure, exit_failure } import std.durable_compare_and_set { CasGeneration } import v2.std.optional { Present, Absent } import std.durable_exclusive_hold { - DurableHoldOwnerRef, DurableHoldReleaseRef, + DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldObservation, DurableHoldReleaseRefusal, + HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, + HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, DurableHoldHolderLiveness, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, DurableHoldHolderReport, DurableHoldRecoveryRefusal, @@ -31,7 +33,7 @@ import extdeps.filesystem.filesystem_io { import gunbc.durable_exclusive_hold_file_store { FileHoldAcquireOutcome, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable, - FileHoldReleasePlanned, FileHoldReleaseNotEligible, + FileHoldReleasePlanned, FileHoldReleaseNotEligible, FileHoldReleaseOutcome, FileHoldReleased, FileHoldReleaseLost, FileHoldReleaseStoreUnavailable, FileHoldReleaseKeyNotSlotAddressable, file_hold_acquire, file_hold_release_assess, file_hold_release_commit, observe_file_hold, FileHoldRecoveryPlanned, FileHoldRecoveryNotEligible, file_hold_recovery_assess, @@ -89,12 +91,14 @@ type UnitHoldOwner = OperatorMaintenance { reason: NonEmptyStr } | BootRun { run_id: NonEmptyStr, process: ProcessIdentity, pid_namespace: NonEmptyStr } | HostResetReturn { attempt: NonEmptyStr } + | KvmObserverProbe { run_id: NonEmptyStr, process: ProcessIdentity, pid_namespace: NonEmptyStr } fn unit_hold_owner_tag(o: UnitHoldOwner) -> String { match o { OperatorMaintenance { reason: _ } => "operator-maintenance:" BootRun { run_id: _, process: _, pid_namespace: _ } => "mtcollins1-boot:" HostResetReturn { attempt: _ } => "host-reset-return:" + KvmObserverProbe { run_id: _, process: _, pid_namespace: _ } => "mtcollins1-kvm-observer:" } } @@ -151,6 +155,7 @@ fn unit_hold_owner_detail(o: UnitHoldOwner) -> String { OperatorMaintenance { reason: r } => r as String BootRun { run_id: r, process: p, pid_namespace: n } => concat(r as String, unit_hold_process_text(p: p, pid_namespace: n)) HostResetReturn { attempt: a } => a as String + KvmObserverProbe { run_id: r, process: p, pid_namespace: n } => concat(r as String, unit_hold_process_text(p: p, pid_namespace: n)) } } @@ -163,10 +168,11 @@ fn unit_hold_owner_label(o: UnitHoldOwner) -> String { OperatorMaintenance { reason: r } => concat("operator maintenance: ", r as String) BootRun { run_id: r, process: p, pid_namespace: _ } => join(["the mtcollins1 boot run ", r as String, " (process ", to_string(p.pid), " on boot ", p.boot_id as String, ")"], "") HostResetReturn { attempt: a } => concat("host reset-return attempt ", a as String) + KvmObserverProbe { run_id: r, process: p, pid_namespace: _ } => join(["the mtcollins1 KVM observer probe run ", r as String, " (process ", to_string(p.pid), " on boot ", p.boot_id as String, ")"], "") } } -// THE DECODE IS TOTAL OVER THE THREE TAGS AND REFUSES ANYTHING ELSE: an owner string written by +// THE DECODE IS TOTAL OVER THE FOUR TAGS AND REFUSES ANYTHING ELSE: an owner string written by // something other than this module is not guessed into a kind. type UnitHoldOwnerDecode = UnitHoldOwnerDecoded { owner: UnitHoldOwner } @@ -182,6 +188,7 @@ fn decode_unit_hold_owner(owner: DurableHoldOwnerRef) -> UnitHoldOwnerDecode { let op = unit_hold_owner_tag(o: OperatorMaintenance { reason: "x" as NonEmptyStr }) let boot = unit_hold_owner_tag(o: BootRun { run_id: "x" as NonEmptyStr, process: ProcessIdentity { boot_id: "x" as NonEmptyStr, pid: 1, start_time: "x" as NonEmptyStr }, pid_namespace: "x" as NonEmptyStr }) let reset = unit_hold_owner_tag(o: HostResetReturn { attempt: "x" as NonEmptyStr }) + let probe = unit_hold_owner_tag(o: KvmObserverProbe { run_id: "x" as NonEmptyStr, process: ProcessIdentity { boot_id: "x" as NonEmptyStr, pid: 1, start_time: "x" as NonEmptyStr }, pid_namespace: "x" as NonEmptyStr }) if starts_with(s: raw, prefix: op) { match unit_hold_owner_detail_after(raw: raw, tag: op) { Present { value: d } => UnitHoldOwnerDecoded { owner: OperatorMaintenance { reason: d } } Absent => UnitHoldOwnerUnrecognized { raw: raw } } } else if starts_with(s: raw, prefix: boot) { @@ -199,6 +206,21 @@ fn decode_unit_hold_owner(owner: DurableHoldOwnerRef) -> UnitHoldOwnerDecode { } } } + } else if starts_with(s: raw, prefix: probe) { + match unit_hold_owner_detail_after(raw: raw, tag: probe) { + Absent => UnitHoldOwnerUnrecognized { raw: raw } + Present { value: d } => { + let halves = split(s: d as String, delimiter: unit_hold_process_marker) + if count(halves) != 2 || text_at(xs: halves, i: 0) == "" { + UnitHoldOwnerUnrecognized { raw: raw } + } else { + match unit_hold_process_decode(text: concat("boot=", text_at(xs: halves, i: 1))) { + Present { value: h } => UnitHoldOwnerDecoded { owner: KvmObserverProbe { run_id: text_at(xs: halves, i: 0) as NonEmptyStr, process: h.process, pid_namespace: h.pid_namespace } } + Absent => UnitHoldOwnerUnrecognized { raw: raw } + } + } + } + } } else if starts_with(s: raw, prefix: reset) { match unit_hold_owner_detail_after(raw: raw, tag: reset) { Present { value: d } => UnitHoldOwnerDecoded { owner: HostResetReturn { attempt: d } } Absent => UnitHoldOwnerUnrecognized { raw: raw } } } else { @@ -406,19 +428,34 @@ fn self_process_identity() -> SelfProcessIdentity { } // โ”€โ”€ RECOVERING A HOLD WHOSE HOLDER WAS OBSERVED DEAD โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ -// Only a holder whose owner names a process can be observed, so only a decoded BootRun is looked at; +// Only a holder whose owner names a process can be observed, so only a decoded BootRun or +// KvmObserverProbe is looked at; // every other owner is reported as not recoverable. The liveness verdict is bound to the holder and // generation the refused acquire saw, and the recovery re-reads the slot, so a hold that changed hands // in between is not the one freed. Age is never an input. -fn holder_report(holder: DurableHoldOwnerRef, generation: CasGeneration) -> DurableHoldHolderReport { - let liveness = match decode_unit_hold_owner(owner: holder) { - UnitHoldOwnerDecoded { owner: BootRun { run_id: _, process: p, pid_namespace: n } } => holder_process_liveness(p: p, pid_namespace: n) +// WHICH HOLDERS ARE OBSERVED, AS A PURE ROUTE: a holder whose decoded owner names a process -- a boot run +// or a KVM observer probe -- is observed by that process; every other owner is unobservable, with why. +type HolderLivenessRoute + = HolderObserveProcess { process: ProcessIdentity, pid_namespace: NonEmptyStr } + | HolderNotObservable { cause: NonEmptyStr } + +fn holder_liveness_route(holder: DurableHoldOwnerRef) -> HolderLivenessRoute { + match decode_unit_hold_owner(owner: holder) { + UnitHoldOwnerDecoded { owner: BootRun { run_id: _, process: p, pid_namespace: n } } => HolderObserveProcess { process: p, pid_namespace: n } + UnitHoldOwnerDecoded { owner: KvmObserverProbe { run_id: _, process: p, pid_namespace: n } } => HolderObserveProcess { process: p, pid_namespace: n } UnitHoldOwnerDecoded { owner: OperatorMaintenance { reason: _ } } => - HolderLivenessUnobservable { cause: "an operator maintenance hold is not bound to a process; only the operator releases it" as NonEmptyStr } + HolderNotObservable { cause: "an operator maintenance hold is not bound to a process; only the operator releases it" as NonEmptyStr } UnitHoldOwnerDecoded { owner: HostResetReturn { attempt: _ } } => - HolderLivenessUnobservable { cause: "a host reset-return hold names no process, so its holder cannot be observed" as NonEmptyStr } + HolderNotObservable { cause: "a host reset-return hold names no process, so its holder cannot be observed" as NonEmptyStr } UnitHoldOwnerUnrecognized { raw: _ } => - HolderLivenessUnobservable { cause: "the holder's owner names no process this module can observe (it was written without one, or not by this module)" as NonEmptyStr } + HolderNotObservable { cause: "the holder's owner names no process this module can observe (it was written without one, or not by this module)" as NonEmptyStr } + } +} + +fn holder_report(holder: DurableHoldOwnerRef, generation: CasGeneration) -> DurableHoldHolderReport { + let liveness = match holder_liveness_route(holder: holder) { + HolderObserveProcess { process: p, pid_namespace: n } => holder_process_liveness(p: p, pid_namespace: n) + HolderNotObservable { cause: c } => HolderLivenessUnobservable { cause: c } } DurableHoldHolderReport { holder: holder, generation: generation, liveness: liveness } } @@ -493,6 +530,7 @@ fn unit_hold_acquire(root: NonEmptyStr, key: NonEmptyStr, owner: DurableHoldOwne admit_callers: [ decl_ref(module_path: "gunbc.machine_intake_mtcollins1_maintenance_hold", decl_name: "mtcollins1_boot_acquire_unit_hold"), decl_ref(module_path: "gunbc.host_reset_return_run", decl_name: "host_reset_return_wet"), + decl_ref(module_path: "gunbc.machine_intake_mtcollins1_maintenance_hold", decl_name: "mtcollins1_kvm_observer_acquire_unit_hold"), ] { let outcome = file_hold_acquire(root: root, slot_key: key, requested_owner: owner) @@ -524,6 +562,115 @@ fn mtcollins1_boot_acquire_unit_hold(run_id: NonEmptyStr) -> UnitHoldAcquisition } } +// THE KVM OBSERVER PROBE TAKES THE SAME SLOT (gunbc.machine_intake_mtcollins1_kvm_observer_observe). +// It writes nothing to the controller, but it takes the BMC's one KVM viewer seat, and the boot takes +// that seat under this hold and REFUSES its handoff when the seat is busy -- so a probe that did not +// hold the unit could make a concurrent boot refuse, and an operator's maintenance would see a viewer +// nobody accounted for. Holding the unit makes the probe, the boot and maintenance exclude each other. +fn mtcollins1_kvm_observer_acquire_unit_hold(run_id: NonEmptyStr) -> UnitHoldAcquisition + admit_callers: [ + decl_ref(module_path: "gunbc.machine_intake_mtcollins1_kvm_observer_observe", decl_name: "mtcollins1_kvm_observer_observe_wet"), + ] +{ + match self_process_identity() { + SelfUnidentified { cause: c } => + UnitHoldRefused { reason: join(["this probe cannot name its own process (", c as String, "), and a hold it could never be shown dead in would lock the unit out if it died; controller untouched"], "") as NonEmptyStr } + SelfIdentified { process: p, pid_namespace: n } => + unit_hold_acquire(root: unit_hold_store_root, key: mtcollins1_unit_hold_key, owner: unit_hold_owner_ref(o: KvmObserverProbe { run_id: run_id, process: p, pid_namespace: n })) + } +} + +// THE PROBE'S always() RELEASE, SCOPED TO ITS OWN RUN (eager-owl-205, 2026-09-30). A cancel or a step +// timeout after the probe acquired leaves the hold with no proof in hand, so a separate step that runs +// under always() observes the slot and frees it ONLY when the holder decodes to +// KvmObserverProbe { run_id } for THIS run, releasing at the generation it observed (the store's +// compare-and-set). A free slot, another owner -- an operator, a boot, another probe run -- or an +// unreadable slot is a reported no-op, never a release; a slot that moved between the observation and +// the commit is stale, and is left as it is. +// +// A runner that dies, or a job cancelled at its backstop, runs no always() step. Its probe hold names +// the holding process (KvmObserverProbe's ProcessIdentity and pid namespace), so the next acquirer of +// the unit recovers it once that process is OBSERVED dead (unit_hold_acquire's observed-dead recovery, +// gunbc#12555), exactly as it recovers a dead boot run's. +type ProbeHoldReleaseDecision + = ProbeHoldReleaseAt { owner: DurableHoldOwnerRef, generation: CasGeneration } + | ProbeHoldFreeNoop + | ProbeHoldForeignNoop { holder: String } + | ProbeHoldUnobservable { cause: String } + +fn probe_hold_release_decision(observed: DurableHoldObservation, run_id: NonEmptyStr) -> ProbeHoldReleaseDecision { + match observed { + HoldSlotAbsent => ProbeHoldFreeNoop + HoldSlotFree { generation: _, released_by: _ } => ProbeHoldFreeNoop + HoldSlotUndecodable { generation: _, detail: d } => ProbeHoldUnobservable { cause: concat("slot undecodable: ", d as String) } + HoldObservationUnavailable { cause: _ } => ProbeHoldUnobservable { cause: "the store could not be read" } + HoldSlotHeld { generation: g, owner: o } => + match decode_unit_hold_owner(owner: o) { + UnitHoldOwnerDecoded { owner: KvmObserverProbe { run_id: r, process: p, pid_namespace: n } } => + if (r as String) == (run_id as String) { ProbeHoldReleaseAt { owner: o, generation: g } } + else { ProbeHoldForeignNoop { holder: unit_hold_owner_label(o: KvmObserverProbe { run_id: r, process: p, pid_namespace: n }) } } + UnitHoldOwnerDecoded { owner: other } => ProbeHoldForeignNoop { holder: unit_hold_owner_label(o: other) } + UnitHoldOwnerUnrecognized { raw: raw } => ProbeHoldForeignNoop { holder: raw } + } + } +} + +type ProbeHoldReleaseOutcome + = ProbeHoldReleased + | ProbeHoldAlreadyFree + | ProbeHoldNotOurs { holder: String } + | ProbeHoldStale { detail: String } + | ProbeHoldUnreleased { cause: String } + +// A RELEASE ASSESSED AGAINST THE OBSERVATION AND REFUSED: every refusal but the slot having been freed +// meanwhile means the slot moved since this run looked, so it is stale and left alone. +fn probe_hold_release_refused(refusal: DurableHoldReleaseRefusal) -> ProbeHoldReleaseOutcome { + match refusal { + HoldReleaseSlotAbsent => ProbeHoldAlreadyFree + HoldReleaseAlreadyFree { generation: _, released_by: _ } => ProbeHoldAlreadyFree + HoldReleaseHeldByAnotherOwner { generation: _, holder: h } => ProbeHoldNotOurs { holder: h as String } + HoldReleaseAgainstAnotherGeneration { acquired: _, observed: _ } => ProbeHoldStale { detail: "the slot moved to another generation after it was observed" } + HoldReleaseSlotUndecodable { generation: _, detail: d } => ProbeHoldUnreleased { cause: concat("slot undecodable: ", d as String) } + HoldReleaseObservationUnavailable { cause: _ } => ProbeHoldUnreleased { cause: "the store could not be read" } + } +} + +fn probe_hold_release_committed(outcome: FileHoldReleaseOutcome) -> ProbeHoldReleaseOutcome { + match outcome { + FileHoldReleased { slot_key: _, generation: _ } => ProbeHoldReleased + FileHoldReleaseLost { slot_key: _ } => ProbeHoldStale { detail: "the slot moved before the release committed" } + FileHoldReleaseStoreUnavailable { slot_key: _, cause: _ } => ProbeHoldUnreleased { cause: "the store refused the release; the unit is STILL held" } + FileHoldReleaseKeyNotSlotAddressable { slot_key: _ } => ProbeHoldUnreleased { cause: "the key is not slot-addressable" } + } +} + +fn probe_hold_release_outcome_text(o: ProbeHoldReleaseOutcome) -> String { + match o { + ProbeHoldReleased => "released this run's probe hold" + ProbeHoldAlreadyFree => "no hold to release (the slot was free)" + ProbeHoldNotOurs { holder: h } => concat("left alone: the unit is held by ", h) + ProbeHoldStale { detail: d } => concat("left alone, stale: ", d) + ProbeHoldUnreleased { cause: c } => concat("NOT released: ", c) + } +} + +fn mtcollins1_kvm_observer_release_unit_hold(run_id: NonEmptyStr) -> ProbeHoldReleaseOutcome + admit_callers: [ + decl_ref(module_path: "gunbc.machine_intake_mtcollins1_kvm_observer_observe", decl_name: "mtcollins1_kvm_observer_release_wet"), + ] +{ + match probe_hold_release_decision(observed: observe_file_hold(root: unit_hold_store_root, key: mtcollins1_unit_hold_key), run_id: run_id) { + ProbeHoldFreeNoop => ProbeHoldAlreadyFree + ProbeHoldForeignNoop { holder: h } => ProbeHoldNotOurs { holder: h } + ProbeHoldUnobservable { cause: c } => ProbeHoldUnreleased { cause: c } + ProbeHoldReleaseAt { owner: o, generation: g } => + match file_hold_release_assess(root: unit_hold_store_root, slot_key: mtcollins1_unit_hold_key, owner: o, acquired_generation: g, released_by: concat("mtcollins1-kvm-observer-always-release:", run_id as String) as DurableHoldReleaseRef) { + FileHoldReleaseNotEligible { slot_key: _, refusal: r } => probe_hold_release_refused(refusal: r) + FileHoldReleasePlanned { plan: p } => probe_hold_release_committed(outcome: file_hold_release_commit(plan: p)) + } + } +} + // THE RELEASE CONSUMES THE PROOF AND KEEPS THE STEP'S OWN OUTCOME, saying when the hold was left: a // run that actuated and then failed to free the unit has still actuated, and an operator reading a // bare success would not know the next run will refuse on a stale holder. @@ -598,6 +745,8 @@ fn mtcollins1_maintenance_hold_release() -> ProcessExit { exit_failure(reason: concat("mtcollins1 maintenance hold: held by ", concat(unit_hold_owner_label(o: BootRun { run_id: r, process: p, pid_namespace: n }), ", not by maintenance; that run frees its own hold, and the next boot recovers it if that run is observed dead"))) UnitHoldOwnerDecoded { owner: HostResetReturn { attempt: a } } => exit_failure(reason: concat("mtcollins1 maintenance hold: held by ", concat(unit_hold_owner_label(o: HostResetReturn { attempt: a }), ", not by maintenance; that run frees its own hold"))) + UnitHoldOwnerDecoded { owner: KvmObserverProbe { run_id: r, process: p, pid_namespace: n } } => + exit_failure(reason: concat("mtcollins1 maintenance hold: held by ", concat(unit_hold_owner_label(o: KvmObserverProbe { run_id: r, process: p, pid_namespace: n }), ", not by maintenance; that run frees its own hold, and the next acquirer recovers it if that run is observed dead"))) UnitHoldOwnerDecoded { owner: OperatorMaintenance { reason: _ } } => match file_hold_release_assess(root: unit_hold_store_root, slot_key: mtcollins1_unit_hold_key, owner: o, acquired_generation: g, released_by: "operator-maintenance-release" as DurableHoldReleaseRef) { FileHoldReleaseNotEligible { slot_key: _, refusal: _ } => exit_failure(reason: "mtcollins1 maintenance hold: slot moved before release; re-run") diff --git a/dag/gunbc/machine_intake/mtcollins1_media_attach.dag b/dag/gunbc/machine_intake/mtcollins1_media_attach.dag index 3abcdd72563..19b1cc3e018 100644 --- a/dag/gunbc/machine_intake/mtcollins1_media_attach.dag +++ b/dag/gunbc/machine_intake/mtcollins1_media_attach.dag @@ -6,7 +6,7 @@ import extdeps.tools.sleep { sleep_delay_seconds_second_carrier_projection } import extdeps.bmc.ipmi import std.decl_ref { decl_ref } import gunbc.machine_intake_mtcollins1_maintenance_hold { UnitHoldProof, unit_hold_proof_refusal, mtcollins1_unit_hold_key } -import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint, mtcollins1_firmware } +import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint, mtcollins1_firmware, mtcollins1_boot_ipmi_read_deadline, mtcollins1_boot_ipmi_kill_after } import gunbc.machine_intake_mtcollins1_bmc_secure_observation { mtcollins1_secured_account } import gunbc.machine_intake_mtcollins1_boot_authorization { MtCollins1BootSubject, mtcollins1_boot_medium, mtcollins1_boot_medium_image_name, mtcollins1_boot_medium_image_sha256, @@ -21,8 +21,6 @@ import gunbc.machine_intake_mtcollins1_boot_diagnostic_bundle { MtCollins1MediaAttachRecord, media_attach_outcome_reason, mtcollins1_boot_chassis_power, - mtcollins1_boot_ipmi_read_deadline, - mtcollins1_boot_ipmi_kill_after, session_release_text, StaleMediumPowerOff, StalePowerOffNotNeeded, diff --git a/dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag b/dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag new file mode 100644 index 00000000000..5f649d338c1 --- /dev/null +++ b/dag/gunbc/machine_intake/mtcollins1_ui_bundle_observe.dag @@ -0,0 +1,365 @@ +module gunbc.machine_intake_mtcollins1_ui_bundle_observe + +import extdeps.shell +import extdeps.bmc.ipmi +import extdeps.bmc.megarac +import extdeps.tools.sha256sum +import extdeps.tools.gzip +import gunbc.rust_item_scan { ends_with } +import std.algebra { trim } +import std.types { Bool, FilePath, Int, List, NonEmptyStr, String } +import std.process { ExitSuccess, ProcessExit, exit_failure } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.tools.gnu_coreutils { coreutils_duration_operand } +import extdeps.bmc.megarac { megarac_ui_bundle_path } +import gunbc.machine_intake_mtcollins1_access_observation { mtcollins1_endpoint, mtcollins1_firmware, mtcollins1_boot_ipmi_read_deadline, mtcollins1_boot_ipmi_kill_after } +import gunbc.machine_intake_mtcollins1_bmc_secure_observation { mtcollins1_secured_account } +import gunbc.machine_intake_mtcollins1_fan_observe { FanCredentialPathReady, FanCredentialPathUnset, FanCredentialPathEmpty, mtcollins1_fan_credential_path, mtcollins1_fan_credential_path_env } +import gunbc.machine_intake_megarac_media_attach { megarac_media_read_max_time, megarac_media_read_max_time_operand } +import std.measure { Second, second, second_count } +import std.nat { Nat } +import v2.std.optional { Present, Absent } +import v2.std.algebra { any } +import gunbc.machine_intake_megarac_served_ui_catalog_observation { megarac_source_min_js_sha256_2026_09_27 } + +// ONE READ OF MT. COLLINS' OWN UI BUNDLE, SO A VENDOR CODE IS INTERPRETED FROM THE VENDOR'S SOURCE +// (operator, amended brief msg_20e45239 and decision msg_b3c77f62, 2026-09-30). The settings route +// answers cd_error_code 16 and nothing in the tree says what 16 names; the firmware's UI names its +// own codes in source.min.js, which is how start-media's 13410/13460 were recovered. This mode is the +// modeled route to those bytes: the controller's firmware revision is read over IPMI (`mc info`, +// read-only), the bundle is fetched only when that revision is the one this unit's facts are keyed +// on, and the served bytes, their sha256 and the comparison with the 2026-09-27 read are the receipt. +// +// ONE READ-ONLY SERVED-FILE OBSERVATION, NOT ONE FILE (eager-owl-205, 2026-09-30). The KVM viewer's +// /kvm video socket is not in source.min.js -- that bundle's one WebSocket is SOL -- so the mode also +// reads viewer.html and every script it loads, derived by parsing what was served and followed to +// closure within a declared budget (served_crawl). Each file is its own receipt row. +// +// NOTHING IS WRITTEN TO THE CONTROLLER AND NO WEB SESSION IS OPENED. Every file read is served to the +// login page's origin before a session exists, so each GET carries no credential (extdeps.bmc.megarac +// megarac.Ui.GetServedBundle); the only credential this run materializes is the IPMI one `mc info` +// needs, through the same prelude the fan observation uses. No unit hold is taken: like the fan +// series, nothing here changes chassis, media or boot state. + +// EVERYTHING THE MODE KEEPS SHARES ONE PREFIX, so the upload is one glob: the receipt, and one directory +// holding every served file, named by its served path with "/" read as "__". +data mtcollins1_ui_bundle_prefix: String = "target/mtcollins1-ui-bundle-" + +data mtcollins1_ui_bundle_artifact_glob: String = concat(mtcollins1_ui_bundle_prefix, "*") + +data mtcollins1_ui_bundle_dir: String = concat(mtcollins1_ui_bundle_prefix, "files") + +fn mtcollins1_ui_bundle_file_path(served: String) -> String { + concat(concat(mtcollins1_ui_bundle_dir, "/"), join(split(s: served, delimiter: "/"), "__")) +} + +data mtcollins1_ui_bundle_receipt_path: String = concat(mtcollins1_ui_bundle_prefix, "receipt.txt") + +// THE CRAWL'S BUDGET, A DECLARED POLICY BOUND (DESIGN ยง5: an explicit budget, never a silent cut): at +// most this many served files are fetched in one run. The require config the 2026-09-30 read of +// source.min.js carries names 23 ./libs/kvm/ modules; the budget is room for roughly twice that plus the +// two seeds. A run that reaches it records a ServedFileBudgetReached row naming every path left unvisited. +data mtcollins1_ui_bundle_file_budget: Nat = 48 + +// WHAT THE PROBE'S OWN READS CAN COST AT MOST: the one `mc info` under its timeout and kill-after, then +// every served GET the budget admits under curl --max-time. The step budget adds gunbc run's load and the +// credential prelude on the workflow side (gunbc.fleet_converge_workflow), which owns those authorities. +fn mtcollins1_ui_bundle_read_bound() -> Second { + second(count: second_count(s: mtcollins1_boot_ipmi_read_deadline) + second_count(s: mtcollins1_boot_ipmi_kill_after) + + mtcollins1_ui_bundle_file_budget * second_count(s: megarac_media_read_max_time)) +} + +// THE REVISION AS `mc info` PRINTS IT: the value after the colon on the `Firmware Revision` line. +fn firmware_revision_of_mc_info(stdout: String) -> String? { + let lines = filter(split(s: stdout, delimiter: "\n"), l => starts_with(s: trim(s: l), prefix: "Firmware Revision")) + match lines.first() { + Absent => none + Present { value: line } => { + let after = split(s: line, delimiter: ":").skip(n: 1) + let v = trim(s: join(after, ":")) + if v == "" { none } else { Present { value: v } } + } + } +} + +// THE BUNDLE IS FETCHED ONLY UNDER THE REVISION THE UNIT'S FACTS ARE KEYED ON. Every code this +// module exists to recover is a 0.32 fact (mtcollins1_firmware); bytes from another build would be +// read as 0.32's table. +type UiBundleFirmwareStanding + = UiBundleFirmwareExpected { revision: NonEmptyStr } + | UiBundleFirmwareDiffers { revision: NonEmptyStr, expected: NonEmptyStr } + | UiBundleFirmwareUnread { detail: String } + +fn ui_bundle_firmware_standing(success: Bool, stdout: String, exit_code: Int, stderr: String) -> UiBundleFirmwareStanding { + if !success { + UiBundleFirmwareUnread { detail: join(["ipmitool mc info exited ", to_string(exit_code), ": ", trim(s: stderr)], "") } + } else { + match firmware_revision_of_mc_info(stdout: stdout) { + Absent => UiBundleFirmwareUnread { detail: concat("mc info printed no Firmware Revision line: ", stdout) } + Present { value: r } => + if r == (mtcollins1_firmware.raw_version as String) { UiBundleFirmwareExpected { revision: r as NonEmptyStr } } + else { UiBundleFirmwareDiffers { revision: r as NonEmptyStr, expected: mtcollins1_firmware.raw_version } } + } + } +} + +// MATCH OR DRIFT AGAINST THE 2026-09-27 READ, REPORTED AND NEVER A REFUSAL: a different digest is a +// finding about the served UI, not a failed read. +type UiBundleDigestComparison + = UiBundleDigestMatches + | UiBundleDigestDrifted { cited: NonEmptyStr } + +fn ui_bundle_digest_comparison(observed: String) -> UiBundleDigestComparison { + if observed == (megarac_source_min_js_sha256_2026_09_27 as String) { UiBundleDigestMatches } + else { UiBundleDigestDrifted { cited: megarac_source_min_js_sha256_2026_09_27 } } +} + +fn ui_bundle_digest_comparison_text(c: UiBundleDigestComparison) -> String { + match c { + UiBundleDigestMatches => "MATCH with the 2026-09-27 read" + UiBundleDigestDrifted { cited: d } => concat("DRIFT from the 2026-09-27 read, which was ", d as String) + } +} + +fn ui_bundle_firmware_text(f: UiBundleFirmwareStanding) -> String { + match f { + UiBundleFirmwareExpected { revision: r } => concat("firmware revision ", concat(r as String, " (expected)")) + UiBundleFirmwareDiffers { revision: r, expected: e } => join(["firmware revision ", r as String, " is NOT the expected ", e as String, "; the bundle was not fetched"], "") + UiBundleFirmwareUnread { detail: d } => concat("firmware revision UNREAD, so the bundle was not fetched: ", d) + } +} + +// THE RECEIPT IS WRITTEN ON EVERY PATH THAT REACHED THE CONTROLLER, and the exit carries the same +// sentence, so a refused run's artifact says why it holds no bytes. +fn ui_bundle_conclude(lines: List, outcome: ProcessExit) -> ProcessExit { + let body = join(concat([concat("subject=", mtcollins1_endpoint.host as String), concat("bundle_path=", megarac_ui_bundle_path as String)], lines), "\n") + let kept = Filesystem.Write(path: mtcollins1_ui_bundle_receipt_path, content: concat(body, "\n")) + if kept.success { outcome } else { + match outcome { + ExitSuccess => exit_failure(reason: concat("mtcollins1 ui bundle: the bundle was read and its receipt could not be written: ", kept.error)) + other => other + } + } +} + +fn ui_bundle_firmware_refused(firmware: UiBundleFirmwareStanding) -> ProcessExit { + ui_bundle_conclude(lines: [concat("firmware=", ui_bundle_firmware_text(f: firmware))], outcome: exit_failure(reason: concat("mtcollins1 ui bundle: ", ui_bundle_firmware_text(f: firmware)))) +} + +// โ”€โ”€ THE SERVED FILES, DERIVED RATHER THAN LISTED (operator rule: no hardcoded lists) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +// Two seeds: source.min.js, the UI bundle (still compared with the 2026-09-27 read), and viewer.html, +// the H5 viewer the UI opens with window.open. Every further path is PARSED out of a file already +// fetched: an HTML file's " + served_html_references(html: html) == ["libs/require.js", "viewer-main.js", "./js/viewer.min.js?v=12"] +} + +// A script's KVM libraries, as the 2026-09-30 source.min.js spells them: a require-config path (a module +// id, loaded as .js) and a Worker constructed from a file name, in either quote. +test fn a_script_yields_its_kvm_libraries_as_files() -> Bool { + let js = "paths:\{videosocket:\"./libs/kvm/videosocket\",decode_worker:\"./libs/kvm/ast/decode_worker.min\"},this.decodeWorker=new Worker('./libs/kvm/ast/decode_worker.js');var s=\"./libs/media/cdimage\"" + served_js_references(js: js) == ["./libs/kvm/videosocket.js", "./libs/kvm/ast/decode_worker.min.js", "./libs/kvm/ast/decode_worker.js"] +} + +// Only HTML and scripts are parsed; anything else yields no references. +test fn only_pages_and_scripts_are_parsed_for_references() -> Bool { + served_references(path: "viewer.html", text: "") == ["a.js"] + && served_references(path: "libs/kvm/x.js", text: "\"./libs/kvm/y\"") == ["./libs/kvm/y.js"] + && served_references(path: "images/logo.png", text: "\"./libs/kvm/y\"") == [] +} + +// A reference becomes a path under the controller's root; another origin is kept, never fetched. +test fn a_reference_reads_as_a_root_path_or_another_origin() -> Bool { + (match served_ref_reading(reference: "./js/viewer.min.js?v=12#x") { ServedRefLocal { path: p } => p == "js/viewer.min.js" _ => false }) + && (match served_ref_reading(reference: "/libs/require.js") { ServedRefLocal { path: p } => p == "libs/require.js" _ => false }) + && (match served_ref_reading(reference: "https://cdn.example/jquery.js") { ServedRefOtherOrigin { reference: r } => r == "https://cdn.example/jquery.js" _ => false }) + && (match served_ref_reading(reference: "//cdn.example/x.js") { ServedRefOtherOrigin { reference: _ } => true _ => false }) + && (match served_ref_reading(reference: "?v=1") { ServedRefEmpty => true _ => false }) +} + +// THE BUDGET'S EDGE IS A ROW, NOT A SILENT CUT: with nothing left to spend, the crawl fetches nothing and +// records every distinct unvisited path. (No GET is made on this path, so it runs without a controller.) +test fn the_budget_edge_names_every_unvisited_path() -> Bool { + let queue = [ + ServedRef { path: "viewer.html", found_by: "seed" }, + ServedRef { path: "libs/kvm/videosocket.js", found_by: "viewer.html" }, + ServedRef { path: "viewer.html", found_by: "libs/kvm/x.js" }, + ] + match served_crawl(queue: queue, seen: [], rows: [], remaining: 0).first() { + Present { value: row } => + match row { + ServedFileBudgetReached { budget: b, unvisited: u } => b == (mtcollins1_ui_bundle_file_budget as Int) && u == ["viewer.html", "libs/kvm/videosocket.js"] + _ => false + } + Absent => false + } +} + +// ONE GLOB CARRIES THE RECEIPT AND EVERY KEPT FILE. +test fn the_upload_glob_covers_the_receipt_and_every_kept_file() -> Bool { + let prefix = substring(s: mtcollins1_ui_bundle_artifact_glob, start: 0, end: string_length(mtcollins1_ui_bundle_artifact_glob) - 1) + starts_with(s: mtcollins1_ui_bundle_receipt_path, prefix: prefix) + && starts_with(s: mtcollins1_ui_bundle_file_path(served: "libs/kvm/ast/decode_worker.js"), prefix: concat(mtcollins1_ui_bundle_dir, "/")) + && starts_with(s: mtcollins1_ui_bundle_dir, prefix: prefix) + && mtcollins1_ui_bundle_file_path(served: "libs/kvm/ast/decode_worker.js") == concat(mtcollins1_ui_bundle_dir, "/libs__kvm__ast__decode_worker.js") +} diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 25fd3c2a1c2..316a396d34f 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1938,8 +1938,10 @@ fn floor_route_gap_expectation_chunk_29() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.machine_intake.mtcollins1_kvm_observer_protocol_wet_witness.a_root_page_that_navigates_cannot_destroy_the_login", operation: "Dir", ground: NoMockResponse {} }, tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.machine_intake.mtcollins1_kvm_observer_protocol_wet_witness.a_viewer_without_its_feature_list_never_opens_kvm", operation: "Dir", ground: NoMockResponse {} }, + tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.machine_intake.sol_hold_stdin_wet_witness.a_channel_loss_asks_the_screen_and_the_ask_does_not_change_the_verdict", operation: "DirWithTemplate", ground: NoMockResponse {} }, - tail: Empty {} } } } } } } } } } } } } } + tail: Empty {} } } } } } } } } } } } } } } } fn floor_route_gap_expectation_chunks() -> List> { diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index ebbc1b38822..e950220a04f 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -2240,6 +2240,12 @@ fn local_repo_wet_schedule() -> List { function: "a_root_page_that_navigates_cannot_destroy_the_login", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.machine_intake.mtcollins1_kvm_observer_protocol_wet_witness", function: "a_viewer_without_its_feature_list_never_opens_kvm" }, + entry: "dag/test/claim/machine_intake/mtcollins1_kvm_observer_protocol_wet_witness_test.dag", + function: "a_viewer_without_its_feature_list_never_opens_kvm", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.machine_intake.mtcollins1_kvm_observer_protocol_wet_witness", function: "a_busy_viewer_slot_refuses_the_observer_and_the_handoff" }, entry: "dag/test/claim/machine_intake/mtcollins1_kvm_observer_protocol_wet_witness_test.dag",