diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 014355dabd8..627f9557387 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -13,7 +13,7 @@ on: mode: description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; spark_v41_engram_differential compares upstream's Engram lookup kernel with the design-B file-backed lookup over sampled real rows of every row store, byte for byte, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, mtcollins1_census_member_readback, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -252,7 +252,7 @@ jobs: echo "fleet-key: agent loaded (identity fleet-automation@gunbc; secret versions/1 pinned; fingerprint verified against modeled authority; key file wiped)" env: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} - if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_observe' || github.event.inputs.mode == 'microvm_network_apply' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_native_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' || github.event.inputs.mode == 'spark_v41_row_store_encode' || github.event.inputs.mode == 'spark_v41_row_store_readback' || github.event.inputs.mode == 'spark_v41_engram_differential' || github.event.inputs.mode == 'spark_v41_runtime_image_build' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'approval_broker_dark_install' || github.event.inputs.mode == 'microvm_controller_install' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'approval_device_enrolment_code_issue' || github.event.inputs.mode == 'mtcollins1_boot' || github.event.inputs.mode == 'host_credential_custody_converge' || github.event.inputs.mode == 'pair_serving_d0' + if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_observe' || github.event.inputs.mode == 'microvm_network_apply' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_native_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' || github.event.inputs.mode == 'spark_v41_row_store_encode' || github.event.inputs.mode == 'spark_v41_row_store_readback' || github.event.inputs.mode == 'spark_v41_engram_differential' || github.event.inputs.mode == 'spark_v41_runtime_image_build' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'approval_broker_dark_install' || github.event.inputs.mode == 'microvm_controller_install' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'approval_device_enrolment_code_issue' || github.event.inputs.mode == 'mtcollins1_boot' || github.event.inputs.mode == 'mtcollins1_census_member_readback' || github.event.inputs.mode == 'host_credential_custody_converge' || github.event.inputs.mode == 'pair_serving_d0' timeout-minutes: 5 - name: Fleet converge plan (membership_reconcile → artifact) id: plan @@ -642,6 +642,24 @@ jobs: retention-days: 30 if: github.event.inputs.mode == 'mtcollins1_census_image_publish' timeout-minutes: 10 + - name: "Mt. Collins census image: read the published kernel and initrd back against the stock medium" + id: mtcollins1_census_member_readback + run: |- + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag --function mtcollins1_census_member_readback_wet + cat "$ROOT/target/mtcollins1-census-member-readback.txt" + if: github.event.inputs.mode == 'mtcollins1_census_member_readback' + timeout-minutes: 30 + - name: Upload Mt. Collins census member readback receipt + id: mtcollins1_census_member_readback_receipt_upload + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: mtcollins1-census-member-readback + path: target/mtcollins1-census-member-readback.txt + if-no-files-found: error + retention-days: 30 + if: always() && github.event.inputs.mode == 'mtcollins1_census_member_readback' + timeout-minutes: 10 - name: "R2 mint preflight: bootstrap read + account permission-group listing (reachability, no mutation)" id: r2_mint_preflight run: |- diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index ee8aba9cdf3..6a1357228ff 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -115,8 +115,10 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.cadquery.cadquery", decl_name: "cadquery_run_program_command"), decl_ref(module_path: "extdeps.cargo_build", decl_name: "cargo_clippy_command"), decl_ref(module_path: "extdeps.tools.sed", decl_name: "sed_in_place_command"), + decl_ref(module_path: "extdeps.tools.xorriso", decl_name: "xorriso_extract_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "cp_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "cat_command"), + decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "mktemp_directory_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "od_hex_span_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "rm_force_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "rm_recursive_force_command"), diff --git a/dag/extdeps/kmod/libkmod.dag b/dag/extdeps/kmod/libkmod.dag new file mode 100644 index 00000000000..77d16f8ab75 --- /dev/null +++ b/dag/extdeps/kmod/libkmod.dag @@ -0,0 +1,45 @@ +module extdeps.kmod.libkmod + +import std.types { NonEmptyStr, String } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Https, Uri } + +// libkmod v31 (the version in the Ubuntu 24.04.3 live-server initrd, `kmod version 31`). +// kmod_module_insert_module logs a failed insertion at INFO with the module FILE PATH: +// INFO(mod->ctx, "Failed to insert module '%s': %s\n", path, strerror(-err)); +// and a context's log priority defaults to LOG_ERR unless the KMOD_LOG environment variable names +// another (libkmod.c kmod_new: secure_getenv("KMOD_LOG") -> kmod_set_log_priority). So by default the +// path-bearing line is filtered inside libkmod before any caller's log function sees it. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/kmod-project/kmod/blob/v31/libkmod/libkmod-module.c" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.kmod.libkmod", + decl_name: "libkmod_insert_failed_prefix", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +data libkmod_insert_failed_prefix: NonEmptyStr = "Failed to insert module '" + +// The stem without the opening quote: a line carrying it but not a complete quoted path and errno is +// a truncated record of this INFO line. +data libkmod_insert_failed_stem: NonEmptyStr = "Failed to insert module" + +// The separator after the quoted path, before strerror(-err). +data libkmod_insert_failed_path_end: NonEmptyStr = "': " + +data libkmod_log_priority_env_name: NonEmptyStr = "KMOD_LOG" + +// log_priority() accepts a syslog name or a number; "info" selects LOG_INFO, the INFO() level. +data libkmod_log_priority_info: NonEmptyStr = "info" diff --git a/dag/extdeps/linux/kernel.dag b/dag/extdeps/linux/kernel.dag index 9045a105b26..b63a99538bb 100644 --- a/dag/extdeps/linux/kernel.dag +++ b/dag/extdeps/linux/kernel.dag @@ -37,3 +37,8 @@ type LinuxKernelRelease = NonEmptyStr where brand("LinuxKernelRelease") // /sys/module all key on. Branded rather than left a bare String so a module name cannot be // interchanged with the arbitrary text beside it in a driver-binding row. type LinuxKernelModuleName = NonEmptyStr where brand("LinuxKernelModuleName") + +// Documentation/admin-guide/kernel-parameters.txt, printk.devkmsg=: "ratelimit" (the default) limits +// records userspace writes to /dev/kmsg; "on" admits them unlimited. A userspace logger that writes a +// burst of records at boot loses all but the first few under the default. +data linux_printk_devkmsg_on_cmdline_arg: NonEmptyStr = "printk.devkmsg=on" diff --git a/dag/extdeps/linux/module_decompress.dag b/dag/extdeps/linux/module_decompress.dag new file mode 100644 index 00000000000..39639ebb17c --- /dev/null +++ b/dag/extdeps/linux/module_decompress.dag @@ -0,0 +1,57 @@ +module extdeps.linux.module_decompress + +import std.types { Int, NonEmptyStr, String } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Https, Uri } + +// IN-KERNEL MODULE DECOMPRESSION (CONFIG_MODULE_DECOMPRESS), Linux v6.8 kernel/module/decompress.c. +// finit_module(2) with MODULE_INIT_COMPRESSED_FILE hands the kernel a compressed module file; a zstd +// decoder error is printed by module_zstd_decompress as the line below and returned to the caller as +// -EINVAL. THE LINE NAMES NO MODULE: the printk carries the decoder status and nothing else, so the +// kernel log alone never identifies which file failed -- that identity is a userspace fact (the +// loader that issued the finit_module call). +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/torvalds/linux/blob/v6.8/kernel/module/decompress.c" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.linux.module_decompress", + decl_name: "linux_module_zstd_decompress_failed_prefix", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +// pr_err("ZSTD-decompression failed with status %d\n", retval), retval = zstd_get_error_code(ret). +data linux_module_zstd_decompress_failed_prefix: NonEmptyStr = "ZSTD-decompression failed with status " + +// The stem of that line without its status clause: a line carrying the stem but no parseable status +// is still a record of this printk, truncated or garbled, never an unrelated line. +data linux_module_zstd_decompress_failed_stem: NonEmptyStr = "ZSTD-decompression failed" + +// zstd_errors.h ZSTD_ErrorCode: the status values a decoder can return are the library's own enum. +// Only the rows this corpus has observed are carried; any other status is kept as its integer. +type ZstdErrorCode + = ZstdCorruptionDetected + | ZstdErrorCodeOther { code: Int } + +data zstd_error_corruption_detected_code: Int = 20 + +fn zstd_error_code_of(code: Int) -> ZstdErrorCode { + if code == zstd_error_corruption_detected_code { ZstdCorruptionDetected } else { ZstdErrorCodeOther { code: code } } +} + +fn zstd_error_code_name(code: ZstdErrorCode) -> String { + match code { + ZstdCorruptionDetected => "corruption_detected" + ZstdErrorCodeOther { code: c } => concat("zstd error ", to_string(c)) + } +} diff --git a/dag/extdeps/ssh/session.dag b/dag/extdeps/ssh/session.dag index 5bf23a7981c..be7f161da95 100644 --- a/dag/extdeps/ssh/session.dag +++ b/dag/extdeps/ssh/session.dag @@ -41,6 +41,12 @@ fn shape_scp_copy_to_argv( [local_path as String, join([host as String, ":", remote_path as String], "")] } +// ssh(1) EXIT STATUS (OpenBSD 7.6): "ssh exits with the exit status of the remote command or with 255 +// if an error occurred." 255 is the CLIENT's status: it does not establish that the remote command +// did not run -- a connection lost after the command started, or before its status came back, also +// ends here. A command that itself exits 255 is indistinguishable from it at this boundary. +data ssh_client_error_exit_status: Int = 255 + type SshSessionExecResult { exit_code: Int success: Bool diff --git a/dag/extdeps/systemd/udevd.dag b/dag/extdeps/systemd/udevd.dag new file mode 100644 index 00000000000..db46aaf7794 --- /dev/null +++ b/dag/extdeps/systemd/udevd.dag @@ -0,0 +1,41 @@ +module extdeps.systemd.udevd + +import std.types { NonEmptyStr, String } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Https, Uri } + +// systemd-udevd v255 (255.4-1ubuntu8.10 in the Ubuntu 24.04.3 live-server initrd). +// run_udevd parses the environment (SYSTEMD_LOG_LEVEL, SYSTEMD_LOG_TARGET) and THEN the kernel +// command line, so `udev.log_level=` overrides a SYSTEMD_LOG_LEVEL set by the caller -- which is what +// initramfs-tools' scripts/init-top/udev does (SYSTEMD_LOG_LEVEL=info). The kmod builtin routes +// libkmod's log function into udev's own log (udev_kmod_log -> log_internalv), and module_load_and_warn +// logs its own failures at DEBUG (verbose=false). With the default AUTO target and no journal, a +// daemonized udevd writes to its inherited stderr, i.e. /dev/console -- the LAST console= on the +// kernel command line -- so SYSTEMD_LOG_TARGET=kmsg is what puts its records in the kernel log. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/systemd/systemd/blob/v255/src/udev/udevd.c" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.systemd.udevd", + decl_name: "udevd_log_level_cmdline_key", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +data udevd_log_level_cmdline_key: NonEmptyStr = "udev.log_level" + +data systemd_log_target_env_name: NonEmptyStr = "SYSTEMD_LOG_TARGET" + +data systemd_log_target_kmsg: NonEmptyStr = "kmsg" + +data systemd_log_level_debug: NonEmptyStr = "debug" diff --git a/dag/extdeps/tools/gnu_coreutils.dag b/dag/extdeps/tools/gnu_coreutils.dag index 12bf6e9c880..de6d5206400 100644 --- a/dag/extdeps/tools/gnu_coreutils.dag +++ b/dag/extdeps/tools/gnu_coreutils.dag @@ -36,6 +36,7 @@ data min_coreutils_version: VersionConstraint = ">= 8.0" // criterion for climbing to an absolute row, and the counter-example that makes the distinction // load-bearing are recorded once at extdeps.exec.command program_spelling_is_an_observation_claim. data cat_program: NonEmptyStr = "cat" +data mktemp_program: NonEmptyStr = "mktemp" data cp_program: NonEmptyStr = "cp" data printf_program: NonEmptyStr = "printf" data true_program: NonEmptyStr = "true" @@ -58,6 +59,17 @@ fn ls_one_per_line_command(path: String) -> ArgvCommand { argv_command(program: ls_program, arguments: ["-1", path]) } +// mktemp -d WITH A TEMPLATE: coreutils creates a NEW directory named from the template (trailing +// X's replaced), mode 0700, and prints its path; it refuses rather than reuse an existing name +// (coreutils manual, "mktemp invocation"). That exclusive creation is what makes the directory one +// caller's own. This is the argv for a remote typed-argv transport; extdeps.shell shell.Mktemp +// DirWithTemplate is the local realization of the same invocation -- the op-plus-builder seam +// extdeps.crypto.hash records for sha256sum, folding when a transport row can be read from the +// operation itself. +fn mktemp_directory_command(template: NonEmptyStr) -> ArgvCommand { + argv_command(program: mktemp_program, arguments: ["-d", template as String]) +} + fn cat_command(path: String) -> ArgvCommand { argv_command(program: cat_program, arguments: [path]) } diff --git a/dag/extdeps/tools/xorriso.dag b/dag/extdeps/tools/xorriso.dag index a0897197459..cebbffd62e5 100644 --- a/dag/extdeps/tools/xorriso.dag +++ b/dag/extdeps/tools/xorriso.dag @@ -7,6 +7,7 @@ import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import extdeps.tools { CliTool, SourceApt } import extdeps.apt { apt_bin_dir, package_xorriso } +import extdeps.exec.command { ArgvCommand, argv_command } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -24,19 +25,6 @@ data xorriso_cli_tool: CliTool = CliTool { } service xorriso.Iso { - operation ExtractPathBestEffort { - input { iso_path: NonEmptyStr, iso_internal_path: NonEmptyStr, destination: NonEmptyStr } - output { success: Bool from "exit_success" } - readonly - transport shell { - argv: ["xorriso", "-osirrox", "on", "-indev", "{iso_path}", "-extract", "{iso_internal_path}", "{destination}"] - } - exit { - 0 => Unit "extracted" - nonzero => Unit "path not present on ISO" - } - } - operation ExtractTree { input { iso_path: NonEmptyStr, destination: NonEmptyStr } output { success: Bool from "exit_success" } @@ -84,3 +72,14 @@ service xorriso.Iso { } } } + +// ONE ISO MEMBER COPIED OUT TO ONE DESTINATION PATH, read-only on the ISO: `xorriso -osirrox on +// -indev ISO -extract MEMBER DEST`. xorriso exits nonzero when the member is absent or the +// destination cannot be written. This is the one spelling of the invocation: its only consumer runs +// it over a remote typed-argv transport (gunbc.machine_intake_mtcollins1_census_member_readback), and +// the local ExtractPathBestEffort operation that restated the same argv had no caller and was deleted +// rather than kept as a second copy (DESIGN §3). A local consumer binds this builder to its local +// transport; it does not re-author the word list. +fn xorriso_extract_command(iso_path: NonEmptyStr, iso_internal_path: NonEmptyStr, destination: NonEmptyStr) -> ArgvCommand { + argv_command(program: xorriso_cli_tool.name, arguments: ["-osirrox", "on", "-indev", iso_path as String, "-extract", iso_internal_path as String, destination as String]) +} diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 46d77a8137d..bab3a4ae917 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -58,6 +58,7 @@ import gunbc.spark.bootstrap_provision { spark_bootstrap_cloud_principal_member import gunbc.machine_intake_mtcollins1_fan_observe { mtcollins1_fan_credential_path_env, mtcollins1_fan_observation_receipt_path } import gunbc.fabric_writer_identity_observe { fabric_writer_identity_receipt_path } import gunbc.machine_intake_mtcollins1_census_image { mtcollins1_census_image_receipt_path } +import gunbc.machine_intake_mtcollins1_census_member_readback { census_member_readback_receipt_path } import std.dissolution { DissolutionCondition, dissolution_description, unbound_dissolution } @@ -999,6 +1000,11 @@ data gunbc_ci_mtcollins1_census_image_publish_target: GunbcRunStepTarget = Gunbc function: "mtcollins1_census_image_publish_wet", } +data gunbc_ci_mtcollins1_census_member_readback_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag", + function: "mtcollins1_census_member_readback_wet", +} + data gunbc_ci_spark_grant_install_target: GunbcRunStepTarget = GunbcRunStepTarget { entry: "dag/gunbc/spark/managed_grant_install.dag", function: "spark_grant_install_ci_wet", @@ -1266,6 +1272,7 @@ fn gunbc_run_step_targets() -> List { gunbc_ci_host_reset_dispatch_admission_target, gunbc_ci_mtcollins1_fan_observe_target, gunbc_ci_mtcollins1_census_image_publish_target, + gunbc_ci_mtcollins1_census_member_readback_target, gunbc_ci_spark_grant_install_target, gunbc_ci_spark_bootstrap_target, gunbc_ci_spark_serving_apply_target, @@ -2582,6 +2589,18 @@ fn gunbc_ci_mtcollins1_census_image_publish_invoke() -> String { ) } +// The member readback reaches srv2 over fleet SSH (the mode consumes the fleet key, see +// gunbc.fleet_converge_workflow) and touches no controller or credential: no prelude. +fn gunbc_ci_mtcollins1_census_member_readback_invoke() -> String { + gunbc_run_step_script( + source_roots: witness_layer_roots, + entry: gunbc_ci_mtcollins1_census_member_readback_target.entry, + function: gunbc_ci_mtcollins1_census_member_readback_target.function, + claim_run: false, + receipt_rel: census_member_readback_receipt_path + ) +} + fn gunbc_ci_runner_guest_image_converge_invoke() -> String { gunbc_run_step_script( source_roots: witness_layer_roots, diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index c28dae01e8c..fb14fdc3c1c 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -29,6 +29,7 @@ import gunbc.cloudflare.r2_bucket_ensure { r2_bucket_ensure_receipt_path } import gunbc.machine_intake_mtcollins1_fan_observe { mtcollins1_fan_observation_receipt_path } import gunbc.fabric_writer_identity_observe { fabric_writer_identity_receipt_path } import gunbc.machine_intake_mtcollins1_census_image { mtcollins1_census_image_receipt_path } +import gunbc.machine_intake_mtcollins1_census_member_readback { census_member_readback_receipt_path } import gunbc.cloudflare.r2_token_mint_run { r2_mint_receipt_glob_in, r2_mint_receipt_scope, R2MintReceiptAttemptScoped, R2OriginObjectWrite, R2AccountBucketAdmin, } @@ -114,6 +115,7 @@ import gunbc.ci_spec { gunbc_ci_mtcollins1_fan_observe_invoke, gunbc_ci_fabric_writer_identity_observe_invoke, gunbc_ci_mtcollins1_census_image_publish_invoke, + gunbc_ci_mtcollins1_census_member_readback_invoke, gunbc_ci_r2_mint_preflight_invoke, gunbc_ci_r2_bucket_ensure_invoke, gunbc_ci_r2_bucket_admin_mint_invoke, @@ -280,6 +282,7 @@ type FleetConvergeWorkflowMode | MtCollins1FanObserve | FabricWriterIdentityObserve | MtCollins1CensusImagePublish + | MtCollins1CensusMemberReadback | MicrovmRunnerGroupEnsure | OrgRunnerRosterObserve | GcpIamConverge @@ -336,6 +339,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String MtCollins1FanObserve => "mtcollins1_fan_observe" FabricWriterIdentityObserve => "fabric_writer_identity_observe" MtCollins1CensusImagePublish => "mtcollins1_census_image_publish" + MtCollins1CensusMemberReadback => "mtcollins1_census_member_readback" MicrovmRunnerGroupEnsure => "microvm_runner_group_ensure" OrgRunnerRosterObserve => "org_runner_roster_observe" } @@ -362,7 +366,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, MtCollins1CensusMemberReadback, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // // EVERY MODE IS NAMED, AND THE WILDCARD IS DELIBERATELY ABSENT. A `_ => none` would read the same @@ -434,6 +438,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc MtCollins1FanObserve => none FabricWriterIdentityObserve => none MtCollins1CensusImagePublish => none + MtCollins1CensusMemberReadback => none MicrovmRunnerGroupEnsure => none OrgRunnerRosterObserve => none } @@ -558,6 +563,7 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> MtCollins1FanObserve => FleetSshKeyNotConsumed FabricWriterIdentityObserve => FleetSshKeyNotConsumed MtCollins1CensusImagePublish => FleetSshKeyNotConsumed + MtCollins1CensusMemberReadback => FleetSshKeyConsumed MicrovmRunnerGroupEnsure => FleetSshKeyNotConsumed GcpIamConverge => FleetSshKeyNotConsumed NamecheapObserve => FleetSshKeyNotConsumed @@ -1116,6 +1122,7 @@ fn fleet_converge_mode_mutation_domain(mode: FleetConvergeWorkflowMode) -> Fleet MtCollins1FanObserve => ExecutorDomain FabricWriterIdentityObserve => ExecutorDomain MtCollins1CensusImagePublish => ExecutorDomain + MtCollins1CensusMemberReadback => ExecutorDomain MicrovmRunnerGroupEnsure => ExecutorDomain OrgRunnerRosterObserve => ExecutorDomain GcpIamConverge => ExecutorDomain @@ -2046,6 +2053,45 @@ fn fleet_converge_mtcollins1_census_image_publish_receipt_upload_step() -> Step } } +// THE PUBLISHED CENSUS IMAGE'S BOOT CHAIN, READ BACK AGAINST THE STOCK MEDIUM on the host that serves +// both (gunbc.machine_intake_mtcollins1_census_member_readback), over fleet SSH from any runner: srv2 +// has no runner slots. It writes only scratch files under srv2's /tmp, removed after each measure. +// The receipt is uploaded whatever the verdict, because a refusing reading is the observation the +// run exists to retain. +data fleet_converge_mtcollins1_census_member_readback_step_if: String = fleet_converge_mode_step_if(mode: MtCollins1CensusMemberReadback) + +fn fleet_converge_mtcollins1_census_member_readback_step() -> Step { + RunStep { + name: Present { value: "Mt. Collins census image: read the published kernel and initrd back against the stock medium" }, + id: Present { value: "mtcollins1_census_member_readback" }, + run: gunbc_ci_mtcollins1_census_member_readback_invoke(), + shell: none, + env: none, + working_directory: none, + if_condition: Present { value: fleet_converge_mtcollins1_census_member_readback_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_mtcollins1_census_image_step_timeout_minutes } + } +} + +fn fleet_converge_mtcollins1_census_member_readback_receipt_upload_step() -> Step { + UsesStep { + name: Present { value: "Upload Mt. Collins census member readback receipt" }, + id: Present { value: "mtcollins1_census_member_readback_receipt_upload" }, + uses: upload_artifact_action, + with: Present { value: [ + kv(key: "name", value: yaml_string(s: "mtcollins1-census-member-readback")), + kv(key: "path", value: yaml_string(s: census_member_readback_receipt_path)), + kv(key: "if-no-files-found", value: yaml_string(s: "error")), + kv(key: "retention-days", value: yaml_int(n: 30)), + ] }, + env: none, + if_condition: Present { value: concat("always() && ", fleet_converge_mtcollins1_census_member_readback_step_if) }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_artifact_transfer_step_timeout_minutes } + } +} + fn fleet_converge_r2_mint_preflight_step() -> Step { RunStep { name: Present { value: "R2 mint preflight: bootstrap read + account permission-group listing (reachability, no mutation)" }, @@ -3219,6 +3265,8 @@ fn fleet_converge_job() -> Job { fleet_converge_pair_serving_d0_step(), fleet_converge_mtcollins1_census_image_publish_step(), fleet_converge_mtcollins1_census_image_publish_receipt_upload_step(), + fleet_converge_mtcollins1_census_member_readback_step(), + fleet_converge_mtcollins1_census_member_readback_receipt_upload_step(), fleet_converge_r2_mint_preflight_step(), fleet_converge_r2_mint_preflight_receipt_upload_step(), fleet_converge_r2_bucket_admin_mint_step(), diff --git a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag new file mode 100644 index 00000000000..c64b33b53d0 --- /dev/null +++ b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag @@ -0,0 +1,260 @@ +module gunbc.machine_intake_kernel_module_decompression_observation + +import std.types { Bool, Int, List, NonEmptyStr, String } +import std.algebra { trim } +import v2.std.algebra { filter } +import v2.std.optional { Present } +import extdeps.linux.module_decompress { linux_module_zstd_decompress_failed_prefix, linux_module_zstd_decompress_failed_stem, ZstdErrorCode, zstd_error_code_of, zstd_error_code_name } +import extdeps.kmod.libkmod { libkmod_insert_failed_prefix, libkmod_insert_failed_stem, libkmod_insert_failed_path_end } + +// WHAT A RETAINED CONSOLE CAPTURE SAYS ABOUT IN-KERNEL MODULE DECOMPRESSION, AT ITS REAL STRENGTH. +// +// Two populations, observed independently and kept apart: +// 1. the kernel's decompression REFUSALS (extdeps.linux.module_decompress): a decoder status and +// nothing else -- the printk names no module; +// 2. the loader's named INSERT FAILURES (extdeps.kmod.libkmod, reaching the capture under the +// module-identity instrument's KMOD_LOG / kmsg settings): a path and the errno the insertion +// returned. +// NOTHING IN THE CAPTURE LINKS ONE INVOCATION TO THE OTHER. A zstd refusal returns -EINVAL, but EINVAL +// does not establish a decompression failure -- the v6.8 loader returns it for other conditions too +// (symbol-version and namespace checks, invalid flags), and libkmod reports the insertion's errno, +// not the kernel's failing stage. So an EINVAL insert failure is a CANDIDATE for a refusal's +// identity, never its known identity, and no nearest-line or time-window rule promotes it. A known +// identity needs an invocation-level link this capture does not carry (the declared next rung). +// +// A RECOGNISED RECORD THAT CANNOT BE PARSED IS KEPT, never dropped: a truncated refusal line is not +// evidence that no refusal occurred, and an incomplete loader record is not evidence that no module +// failed. Each is retained with its line and the field that could not be read, and a refusal count +// is never stated as complete while an unparsed refusal record stands beside it. +// +// This observation says nothing about device availability, live-root acceptance, SOL health or the +// hardware. The xhci renesas_usb_fw.mem "fallback to ROM" warning is not read here at all -- it +// appears in the census-reaching boot too, so it is not a refusal of anything. + +type ModuleDecompressionRefusal { + at: String + status: ZstdErrorCode +} + +// `line` is the record's message text after the timestamp and CR/NUL are stripped; `at` is that +// timestamp ("" when the line had none). The retained console capture stays the verbatim reference. +type UnparsedRecord { + at: String + line: String + field: String +} + +type LaterDriverReading + = DriverPrintedLater { at: String, line: String } + | DriverNotSeenInCapture + +type ModuleInsertFailure { + at: String + loader: String + path: NonEmptyStr + error: String + module_name: String + later: LaterDriverReading +} + +// The candidates are the named insert failures whose errno is the one a refusal returns. Their +// relationship to any particular refusal stays unresolved. +type RefusalIdentityReading + = RefusalIdentityUnread + | RefusalIdentityCandidates { candidates: List } + +type KernelModuleDecompressionObservation + = NoRefusalRecorded { insert_failures: List, unparsed_insert_records: List } + | RefusalRecorded { + refusals: List, + unparsed_refusal_records: List, + insert_failures: List, + unparsed_insert_records: List, + identity: RefusalIdentityReading, + } + +// The errno a decompression refusal returns to finit_module (decompress.c: retval = -EINVAL), as +// strerror renders it -- necessary for a candidate, not sufficient for an identity. +data refused_decompression_strerror: String = "Invalid argument" + +data request_identity_on_sol_text: String = "requesting device/modalias: unread on SOL (kmsg ring only)" + +fn first_or_empty(xs: List) -> String { + match first(xs) { + Absent => "" + Present { value: x } => x + } +} + +// A console line as written: CR and NUL removed. +fn console_line(raw: String) -> String { + join(split(s: join(split(s: raw, delimiter: "\r"), ""), delimiter: "\0"), "") +} + +type StampedLine { + at: String + rest: String +} + +// "[ 10.374590] text" -> at "10.374590", rest "text". An unstamped line keeps at "". +fn stamped(raw: String) -> StampedLine { + let line = console_line(raw: raw) + let closed = split(s: line, delimiter: "] ") + if starts_with(s: trim(line), prefix: "[") && count(closed) >= 2 { + StampedLine { + at: trim(join(split(s: first_or_empty(xs: closed), delimiter: "["), "")), + rest: join(closed.skip(n: 1), "] "), + } + } else { + StampedLine { at: "", rest: line } + } +} + +type RefusalLineReading + = NotARefusalLine + | RefusalLineParsed { refusal: ModuleDecompressionRefusal } + | RefusalLineUnparsed { record: UnparsedRecord } + +fn refusal_line_reading(l: StampedLine) -> RefusalLineReading { + if !string_contains(s: l.rest, pattern: linux_module_zstd_decompress_failed_stem as String) { NotARefusalLine } else { + let prefix = linux_module_zstd_decompress_failed_prefix as String + let status_text = trim(join(split(s: l.rest, delimiter: prefix).skip(n: 1), prefix)) + if !string_contains(s: l.rest, pattern: prefix) || status_text == "" { + RefusalLineUnparsed { record: UnparsedRecord { at: l.at, line: l.rest, field: "status" } } + } else { + match parse_int(s: status_text) { + Absent => RefusalLineUnparsed { record: UnparsedRecord { at: l.at, line: l.rest, field: "status" } } + Present { value: code } => RefusalLineParsed { refusal: ModuleDecompressionRefusal { at: l.at, status: zstd_error_code_of(code: code) } } + } + } + } +} + +type InsertLineReading + = NotAnInsertLine + | InsertLineParsed { at: String, loader: String, path: NonEmptyStr, error: String } + | InsertLineUnparsed { record: UnparsedRecord } + +fn insert_line_reading(l: StampedLine) -> InsertLineReading { + if !string_contains(s: l.rest, pattern: libkmod_insert_failed_stem as String) { NotAnInsertLine } else { + let prefix = libkmod_insert_failed_prefix as String + let end = libkmod_insert_failed_path_end as String + let parts = split(s: l.rest, delimiter: prefix) + let path_and_error = split(s: join(parts.skip(n: 1), prefix), delimiter: end) + let path = first_or_empty(xs: path_and_error) + let error = trim(join(path_and_error.skip(n: 1), end)) + if count(parts) < 2 || count(path_and_error) < 2 || path == "" { + InsertLineUnparsed { record: UnparsedRecord { at: l.at, line: l.rest, field: "path" } } + } else if error == "" { + InsertLineUnparsed { record: UnparsedRecord { at: l.at, line: l.rest, field: "errno" } } + } else { + InsertLineParsed { at: l.at, loader: trim(join(split(s: trim(first_or_empty(xs: parts)), delimiter: ":"), "")), path: path as NonEmptyStr, error: error } + } + } +} + +// ".../xhci-pci-renesas.ko.zst" -> "xhci_pci_renesas": the name the kernel prints a module under. +fn module_name_of_path(path: String) -> String { + let segments = split(s: path, delimiter: "/") + let file = first_or_empty(xs: segments.skip(n: count(segments) - 1)) + join(split(s: first_or_empty(xs: split(s: file, delimiter: ".ko")), delimiter: "-"), "_") +} + +fn later_driver(lines: List, after_index: Int, module_name: String) -> LaterDriverReading { + let later: List = filter(lines.skip(n: after_index + 1), l => starts_with(s: l.rest, prefix: concat(module_name, " ")) || starts_with(s: l.rest, prefix: concat(module_name, ":"))) + match first(later) { + Absent => DriverNotSeenInCapture + Present { value: l } => DriverPrintedLater { at: l.at, line: l.rest } + } +} + +fn kernel_module_decompression_observation(lines: List) -> KernelModuleDecompressionObservation { + let stamped_lines = map(lines, raw => stamped(raw: raw)) + let refusal_readings = map(stamped_lines, l => refusal_line_reading(l: l)) + let refusals = flat_map(refusal_readings, r => match r { RefusalLineParsed { refusal: x } => [x] _ => [] }) + let unparsed_refusals = flat_map(refusal_readings, r => match r { RefusalLineUnparsed { record: x } => [x] _ => [] }) + let insert_readings = map(stamped_lines, l => insert_line_reading(l: l)) + let insert_failures = flat_map(enumerate(insert_readings), ir => match ir.second { + InsertLineParsed { at: a, loader: w, path: p, error: e } => + [ModuleInsertFailure { + at: a, loader: w, path: p, error: e, + module_name: module_name_of_path(path: p as String), + later: later_driver(lines: stamped_lines, after_index: ir.first, module_name: module_name_of_path(path: p as String)), + }] + _ => [] + }) + let unparsed_inserts = flat_map(insert_readings, r => match r { InsertLineUnparsed { record: x } => [x] _ => [] }) + if count(refusals) == 0 && count(unparsed_refusals) == 0 { + NoRefusalRecorded { insert_failures: insert_failures, unparsed_insert_records: unparsed_inserts } + } else { + let candidates: List = filter(insert_failures, f => f.error == refused_decompression_strerror) + RefusalRecorded { + refusals: refusals, + unparsed_refusal_records: unparsed_refusals, + insert_failures: insert_failures, + unparsed_insert_records: unparsed_inserts, + identity: if count(candidates) == 0 { RefusalIdentityUnread } else { RefusalIdentityCandidates { candidates: candidates } }, + } + } +} + +fn refusal_text(r: ModuleDecompressionRefusal) -> String { + concat("[", r.at, "] status ", zstd_error_code_name(code: r.status)) +} + +fn unparsed_text(u: UnparsedRecord) -> String { + concat(u.field, " unread in [", u.at, "] \"", u.line, "\"") +} + +fn later_text(l: LaterDriverReading) -> String { + match l { + DriverPrintedLater { at: a, line: t } => concat("printed later at ", a, ": ", t) + DriverNotSeenInCapture => "not seen later in this capture" + } +} + +fn insert_failure_text(f: ModuleInsertFailure) -> String { + concat("[", f.at, "] ", f.loader, " ", f.path as String, ": ", f.error, "; ", f.module_name, " ", later_text(l: f.later)) +} + +fn insert_population_text(failures: List, unparsed: List) -> String { + concat( + "named insert failures: ", if count(failures) == 0 { "none" } else { join(map(failures, f => insert_failure_text(f: f)), "; ") }, + if count(unparsed) == 0 { "" } else { concat("; UNPARSED insert records: ", join(map(unparsed, u => unparsed_text(u: u)), "; ")) }, + ) +} + +fn refusal_count_text(refusals: List, unparsed: List) -> String { + if count(unparsed) == 0 { + concat(to_string(count(refusals)), " time(s)") + } else { + concat(to_string(count(refusals)), " parsed record(s) plus ", to_string(count(unparsed)), " UNPARSED; the total is not stated") + } +} + +fn kernel_module_decompression_text(o: KernelModuleDecompressionObservation) -> String { + match o { + NoRefusalRecorded { insert_failures: fs, unparsed_insert_records: us } => + concat("module decompression: no refusal record in this capture; ", insert_population_text(failures: fs, unparsed: us)) + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: fs, unparsed_insert_records: us, identity: id } => + concat( + "module decompression REFUSED ", refusal_count_text(refusals: rs, unparsed: urs), ": ", join(map(rs, r => refusal_text(r: r)), ", "), + if count(urs) == 0 { "" } else { concat("; UNPARSED refusal records: ", join(map(urs, u => unparsed_text(u: u)), "; ")) }, + "; ", + match id { + RefusalIdentityUnread => "refused module identity UNREAD (no named insert failure with the refusal's errno)" + RefusalIdentityCandidates { candidates: cs } => + concat("refused module identity UNRESOLVED; EINVAL candidates, not established by this capture: ", join(map(cs, f => concat(f.path as String, " [", f.at, "]")), ", ")) + }, + "; ", insert_population_text(failures: fs, unparsed: us), + "; ", request_identity_on_sol_text, + ) + } +} + +fn kernel_module_decompression_findings(o: KernelModuleDecompressionObservation) -> List { + match o { + NoRefusalRecorded { insert_failures: _, unparsed_insert_records: _ } => [] + RefusalRecorded { refusals: _, unparsed_refusal_records: _, insert_failures: _, unparsed_insert_records: _, identity: _ } => [kernel_module_decompression_text(o: o)] + } +} diff --git a/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag b/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag index 61522ef06d6..46afa8c8f93 100644 --- a/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag +++ b/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag @@ -47,6 +47,7 @@ import gunbc.machine_intake_pre_os_bringup_verdict { import gunbc.machine_intake_ampere_dram_console_observation { AmpereDramConsoleObservation, ampere_dram_console_observation, ampere_dram_findings, ampere_dram_console_text, } +import gunbc.machine_intake_kernel_module_decompression_observation { KernelModuleDecompressionObservation, kernel_module_decompression_observation, kernel_module_decompression_text, kernel_module_decompression_findings } import gunbc.machine_intake_ampere_socket_console_observation { AmpereSocketConsoleObservation, AmpereSummaryObserved, AmpereSummaryNotPrinted, AmpereSocketReferenceComparison, ampere_socket_console_observation, ampere_socket_console_text, ampere_socket_reference_comparison, ampere_socket_reference_comparison_text, @@ -258,7 +259,7 @@ type InventoryReading { // those bytes would under-count multi-byte console output. The size is taken before the content is // read, so a capture over the bound is reported without being loaded. type ConsoleReading - = ConsoleRetained { path: NonEmptyStr, size: ByteSize, lines: Int, firmware: FirmwareConsoleReport, dram: AmpereDramConsoleObservation, sockets: AmpereSocketConsoleObservation, against_reference: AmpereSocketReferenceComparison } + = ConsoleRetained { path: NonEmptyStr, size: ByteSize, lines: Int, firmware: FirmwareConsoleReport, dram: AmpereDramConsoleObservation, sockets: AmpereSocketConsoleObservation, against_reference: AmpereSocketReferenceComparison, modules: KernelModuleDecompressionObservation } | ConsoleExceedsBound { path: NonEmptyStr, size: ByteSize, bound: ByteSize } | ConsoleNotRetained { path: NonEmptyStr } | ConsoleUnread { path: NonEmptyStr, detail: String } @@ -576,7 +577,7 @@ fn socket_summary_findings(o: AmpereSocketConsoleObservation) -> List { fn console_findings(c: ConsoleReading) -> List { match c { - ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: d, sockets: o, against_reference: _ } => concat(ampere_dram_findings(observation: d, expected_sockets: mtcollins1_census_sockets()), socket_summary_findings(o: o)) + ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: d, sockets: o, against_reference: _, modules: m } => concat(ampere_dram_findings(observation: d, expected_sockets: mtcollins1_census_sockets()), concat(socket_summary_findings(o: o), kernel_module_decompression_findings(o: m))) _ => [] } } @@ -898,8 +899,8 @@ fn firmware_report_text(report: FirmwareConsoleReport) -> String { fn console_text(c: ConsoleReading) -> String { match c { - ConsoleRetained { path: p, size: b, lines: n, firmware: f, dram: d, sockets: o, against_reference: cmp } => - concat("capture ", p as String, " retained in this artifact: ", to_string(byte_size_count(b: b)), " bytes (stat), ", to_string(n), " line(s), within the ", to_string(byte_size_count(b: mtcollins1_boot_console_bound)), "-byte bound; ", firmware_report_text(report: f), "\n", ampere_dram_console_text(observation: d, expected_sockets: mtcollins1_census_sockets()), "\n", ampere_socket_console_text(o: o), ampere_socket_reference_comparison_text(c: cmp)) + ConsoleRetained { path: p, size: b, lines: n, firmware: f, dram: d, sockets: o, against_reference: cmp, modules: m } => + concat("capture ", p as String, " retained in this artifact: ", to_string(byte_size_count(b: b)), " bytes (stat), ", to_string(n), " line(s), within the ", to_string(byte_size_count(b: mtcollins1_boot_console_bound)), "-byte bound; ", firmware_report_text(report: f), "\n", ampere_dram_console_text(observation: d, expected_sockets: mtcollins1_census_sockets()), "\n", ampere_socket_console_text(o: o), ampere_socket_reference_comparison_text(c: cmp), "\n", kernel_module_decompression_text(o: m)) ConsoleExceedsBound { path: p, size: b, bound: bound } => concat("capture ", p as String, " is ", to_string(byte_size_count(b: b)), " bytes (stat), OVER the ", to_string(byte_size_count(b: bound)), "-byte bound; the collector does not truncate, so the artifact carries it whole") ConsoleNotRetained { path: p } => @@ -1553,6 +1554,7 @@ fn mtcollins1_boot_console_reading(path: String, size: ByteSize, capture: String path: path as NonEmptyStr, size: size, lines: count(lines), firmware: firmware_console_report(lines: lines), dram: ampere_dram_console_observation(lines: lines), sockets: sockets, against_reference: ampere_socket_reference_comparison(reference_path: mtcollins1_census_sol_artifact_path, reference_text: reference.text, read_ok: reference.ok, read_error: reference.error, subject: sockets), + modules: kernel_module_decompression_observation(lines: lines), } } } diff --git a/dag/gunbc/machine_intake/mtcollins1_census_image.dag b/dag/gunbc/machine_intake/mtcollins1_census_image.dag index 7c882e3b9d5..2ef694234e9 100644 --- a/dag/gunbc/machine_intake/mtcollins1_census_image.dag +++ b/dag/gunbc/machine_intake/mtcollins1_census_image.dag @@ -6,6 +6,9 @@ import extdeps.filesystem.filesystem_io { Filesystem } import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent, fleet_converge_expected_host_env_name } import gunbc.fleet_intent_network { operator_host_srv2 } import std.measure { Bandwidth, bandwidth, bandwidth_count } +import extdeps.kmod.libkmod { libkmod_log_priority_env_name, libkmod_log_priority_info } +import extdeps.systemd.udevd { udevd_log_level_cmdline_key, systemd_log_level_debug, systemd_log_target_env_name, systemd_log_target_kmsg } +import extdeps.linux.kernel { linux_printk_devkmsg_on_cmdline_arg } import extdeps.provisioning.ubuntu_install_media { noble_numbat_2404_3_live_server_arm64 } import extdeps.provisioning.ubuntu_seeded_install_media { NoCloudSeedFile, @@ -70,6 +73,29 @@ data mtcollins1_census_console_arg: NonEmptyStr = concat( bandwidth_count(b: mtcollins1_census_console_baud) as String, ) as NonEmptyStr +// THE MODULE-IDENTITY INSTRUMENT (gunbc.machine_intake_kernel_module_decompression_observation). +// The kernel's "ZSTD-decompression failed" line names no module; the loader's record does, and in +// this initrd (systemd-udevd 255.4, libkmod 31) it reaches the serial capture only when all four hold: +// KMOD_LOG=info libkmod's path-bearing "Failed to insert module" line is INFO, and its +// default priority is ERR (extdeps.kmod.libkmod). The kernel hands an +// unrecognised KEY=value to /init's environment and initramfs-tools' +// init does not scrub it, so udevd inherits it. +// udev.log_level=debug overrides the SYSTEMD_LOG_LEVEL=info that init-top/udev sets. +// SYSTEMD_LOG_TARGET=kmsg without it udevd writes to /dev/console, which is the LAST console= -- +// tty0 here, the KVM, not the SOL line (extdeps.systemd.udevd). +// printk.devkmsg=on the default rate limit drops a burst of userspace kmsg records. +// The INFO record reaches SOL at the default console loglevel; udev's per-device DEBUG records stay +// in the kmsg ring, so the requesting device is unread on SOL by design (eager-owl-205, option A: +// ignore_loglevel would flood the 115200 line and perturb the timing of a possible race). +// It is a build input like any other: changing it changes the build key and the digest, so the +// image is republished, read back and repinned; nothing edits GRUB by hand. +data mtcollins1_census_module_identity_args: NonEmptyStr = join([ + concat(libkmod_log_priority_env_name as String, "=", libkmod_log_priority_info as String), + concat(udevd_log_level_cmdline_key as String, "=", systemd_log_level_debug as String), + concat(systemd_log_target_env_name as String, "=", systemd_log_target_kmsg as String), + linux_printk_devkmsg_on_cmdline_arg as String, +], " ") as NonEmptyStr + // The seed directory the earlier census booted with (its /proc/cmdline in attempt=3 reads // ds=nocloud;s=/cdrom/gunbc-census/), kept so the two captures are comparable. data mtcollins1_census_nocloud_dir: NonEmptyStr = "gunbc-census" @@ -131,6 +157,8 @@ fn mtcollins1_seeded_image_input( ubuntu_seeded_install_media_grub_kernel_cmdline(nocloud_dir: mtcollins1_census_nocloud_dir) as String, " ", mtcollins1_census_console_arg as String, + " ", + mtcollins1_census_module_identity_args as String, ) as NonEmptyStr, volume_id: volume_id, pinned_dates: seeded_install_media_pinned_dates, diff --git a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag new file mode 100644 index 00000000000..0c7d03b8664 --- /dev/null +++ b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag @@ -0,0 +1,523 @@ +module gunbc.machine_intake_mtcollins1_census_member_readback + +import std.types { Bool, Int, List, NonEmptyStr, String } +import std.process { ExitSuccess, ProcessExit, exit_failure } +import std.algebra { trim } +import v2.std.algebra { filter } +import std.content_hash { sha256_hex_digest } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.tools.xorriso { xorriso_extract_command } +import extdeps.tools.gnu_coreutils { mktemp_directory_command, rm_recursive_force_one_file_system_command } +import extdeps.crypto.hash { sha256sum_file_command } +import extdeps.exec.command { ArgvCommand, argv_words } +import gunbc.fleet_intent_network { operator_host_srv2 } +import gunbc.fleet_reach_endpoint { fleet_probe_endpoint_for } +import gunbc.fleet_bootstrap_principal { executor_bootstrap_principal } +import gunbc.fleet_known_hosts_anchor { SshTarget, FleetSshExecutionContext } +import gunbc.fleet_ssh_locus { prepare_fleet_ssh_agent_context, FleetSshContextReady, FleetSshContextRefused } +import gunbc.typed_argv_exec { typed_argv_exec_over_fleet_ssh, TypedArgvExecOutcome, TypedArgvExecConverged, TypedArgvExecRefused } +import extdeps.ssh.session { ssh_client_error_exit_status } +import extdeps.provisioning.ubuntu_install_media { noble_numbat_2404_3_live_server_arm64 } +import gunbc.machine_intake_mtcollins1_boot_authorization { MtCollins1BootMedium, MtCollins1CensusMedium, MtCollins1StockInstallerMedium, mtcollins1_boot_medium } +import gunbc.machine_intake_mtcollins1_census_image { mtcollins1_census_image_stock_path } +import gunbc.machine_intake_mtcollins1_census_medium_readback { mtcollins1_census_medium_served_path, RemoteAnswer, remote_answer_of } + +// THE PUBLISHED IMAGE'S BOOT CHAIN, READ BACK MEMBER BY MEMBER AGAINST THE STOCK MEDIUM IT WAS +// REMASTERED FROM. The census remaster (extdeps.provisioning.ubuntu_seeded_install_media_remaster) +// maps in the NoCloud seed and rewrites grub.cfg; every other member is supposed to pass through the +// replay unchanged. That is a claim about the builder, and a boot that failed in-kernel module +// decompression makes it worth measuring rather than assuming: this reads the kernel and the initrd +// out of BOTH served ISOs on the host that serves them and compares digests. +// +// SCOPE OF THE COMPARISON. If an initrd digest is equal, every member inside it -- every compressed +// module -- is equal by construction, so the module members are not unpacked here: a per-module +// comparison over identical bytes would re-derive a fact the outer digest already establishes. Only +// a differing initrd owes a descent into its layers, and that descent is the declared frontier +// named by BootChainFirstDifference below, not a guess made here. +// +// THE CONTROL IS IN THE SAME RUN. grub.cfg is the member the builder DOES rewrite (the census kernel +// cmdline lives there), so a reading in which grub.cfg compares equal means the difference control +// FAILED, and it refuses rather than reporting the boot chain identical. It does not choose the +// cause: the census file may not have been the one read, or the published artifact may lack the +// intended rewrite. Likewise equal kernel and initrd bytes establish the scoped stock-versus-published +// comparison and nothing about what reached host memory or why a decoder refused. + +type CensusIsoMember + = CasperKernel + | CasperInitrd + | GrubConfig + +fn census_iso_member_path(member: CensusIsoMember) -> NonEmptyStr { + match member { + CasperKernel => "/casper/vmlinuz" + CasperInitrd => "/casper/initrd" + GrubConfig => "/boot/grub/grub.cfg" + } +} + +fn census_iso_member_slug(member: CensusIsoMember) -> NonEmptyStr { + match member { + CasperKernel => "vmlinuz" + CasperInitrd => "initrd" + GrubConfig => "grub.cfg" + } +} + +// Boot order: the loader reads grub.cfg, then the kernel, then the initrd. The first differing +// boundary is the first member in this order that differs outside the builder's declared rewrite; +// boot_chain_verdict walks them in that order. + +type IsoSide + = StockSide + | CensusSide + +fn iso_side_slug(side: IsoSide) -> NonEmptyStr { + match side { + StockSide => "stock" + CensusSide => "census" + } +} + +type MemberReading + = MemberRead { digest: NonEmptyStr } + | MemberUnread { reason: String } + +type IsoReading + = IsoMeasured { digest: NonEmptyStr } + | IsoUnmeasured { reason: String } + +type MemberComparison + = MemberIdentical { member: CensusIsoMember, digest: NonEmptyStr } + | MemberDiffers { member: CensusIsoMember, stock: NonEmptyStr, census: NonEmptyStr } + | MemberNotCompared { member: CensusIsoMember, side: IsoSide, reason: String } + +fn compare_member(member: CensusIsoMember, stock: MemberReading, census: MemberReading) -> MemberComparison { + match stock { + MemberUnread { reason: r } => MemberNotCompared { member: member, side: StockSide, reason: r } + MemberRead { digest: s } => + match census { + MemberUnread { reason: r } => MemberNotCompared { member: member, side: CensusSide, reason: r } + MemberRead { digest: c } => + if (s as String) == (c as String) { + MemberIdentical { member: member, digest: s } + } else { + MemberDiffers { member: member, stock: s, census: c } + } + } + } +} + +// THE OBSERVATION. Each ISO is first measured and admitted against the digest it is pinned to -- +// the stock against its SHA256SUMS row, the census against the boot medium row -- because a member +// read out of an unpinned file is a reading of some other image. +type CensusMemberReadback + = BootChainIdenticalToStock { stock_iso: NonEmptyStr, census_iso: NonEmptyStr, kernel: NonEmptyStr, initrd: NonEmptyStr } + | BootChainFirstDifference { stock_iso: NonEmptyStr, census_iso: NonEmptyStr, member: CensusIsoMember, stock: NonEmptyStr, census: NonEmptyStr } + | BootChainControlDidNotDiscriminate { grub_digest: NonEmptyStr } + | BootChainIsoNotPinned { side: IsoSide, expected: NonEmptyStr, observed: String } + | BootChainUnread { member: CensusIsoMember, side: IsoSide, reason: String } + | BootChainMediumNotCensus + | BootChainHostUnreached { reason: String } + | BootChainWorkspaceUnavailable { reason: String } + +fn iso_pin_refusal(side: IsoSide, expected: NonEmptyStr, reading: IsoReading) -> CensusMemberReadback? { + match reading { + IsoUnmeasured { reason: r } => Present { value: BootChainIsoNotPinned { side: side, expected: expected, observed: concat("unmeasured: ", r) } } + IsoMeasured { digest: d } => + if (d as String) == (expected as String) { + none + } else { + Present { value: BootChainIsoNotPinned { side: side, expected: expected, observed: d as String } } + } + } +} + +fn boot_chain_verdict(stock_iso: NonEmptyStr, census_iso: NonEmptyStr, grub: MemberComparison, kernel: MemberComparison, initrd: MemberComparison) -> CensusMemberReadback { + match grub { + MemberNotCompared { member: m, side: s, reason: r } => BootChainUnread { member: m, side: s, reason: r } + MemberIdentical { member: _, digest: d } => BootChainControlDidNotDiscriminate { grub_digest: d } + MemberDiffers { member: _, stock: _, census: _ } => + match kernel { + MemberNotCompared { member: m, side: s, reason: r } => BootChainUnread { member: m, side: s, reason: r } + MemberDiffers { member: m, stock: s, census: c } => + BootChainFirstDifference { stock_iso: stock_iso, census_iso: census_iso, member: m, stock: s, census: c } + MemberIdentical { member: _, digest: k } => + match initrd { + MemberNotCompared { member: m, side: s, reason: r } => BootChainUnread { member: m, side: s, reason: r } + MemberDiffers { member: m, stock: s, census: c } => + BootChainFirstDifference { stock_iso: stock_iso, census_iso: census_iso, member: m, stock: s, census: c } + MemberIdentical { member: _, digest: i } => + BootChainIdenticalToStock { stock_iso: stock_iso, census_iso: census_iso, kernel: k, initrd: i } + } + } + } +} + +// THE PIN GATE, evaluated BEFORE any member is extracted: the producer measures both ISOs, and only +// PinsAdmitted lets it acquire a workspace and read members. A refusal here is final. +type CensusPinGate + = PinsAdmitted { stock_iso: NonEmptyStr, census_iso: NonEmptyStr } + | PinsRefused { readback: CensusMemberReadback } + +fn census_pin_gate(medium: MtCollins1BootMedium, stock_iso: IsoReading, census_iso: IsoReading) -> CensusPinGate { + match medium { + MtCollins1StockInstallerMedium => PinsRefused { readback: BootChainMediumNotCensus } + MtCollins1CensusMedium { output_digest: census_pin } => + match iso_pin_refusal(side: StockSide, expected: noble_numbat_2404_3_live_server_arm64.content_sha256, reading: stock_iso) { + Present { value: refused } => PinsRefused { readback: refused } + Absent => + match iso_pin_refusal(side: CensusSide, expected: census_pin, reading: census_iso) { + Present { value: refused } => PinsRefused { readback: refused } + Absent => PinsAdmitted { stock_iso: noble_numbat_2404_3_live_server_arm64.content_sha256, census_iso: census_pin } + } + } + } +} + +fn census_member_verdict(stock_iso: NonEmptyStr, census_iso: NonEmptyStr, readings: CensusMemberReadings) -> CensusMemberReadback { + boot_chain_verdict( + stock_iso: stock_iso, + census_iso: census_iso, + grub: compare_member(member: GrubConfig, stock: readings.stock_grub, census: readings.census_grub), + kernel: compare_member(member: CasperKernel, stock: readings.stock_kernel, census: readings.census_kernel), + initrd: compare_member(member: CasperInitrd, stock: readings.stock_initrd, census: readings.census_initrd), + ) +} + +type CensusMemberReadings { + stock_grub: MemberReading + census_grub: MemberReading + stock_kernel: MemberReading + census_kernel: MemberReading + stock_initrd: MemberReading + census_initrd: MemberReading +} + +// The pure join over pins and six readings, composed as the producer composes it. +fn census_member_readback_from_readings( + medium: MtCollins1BootMedium, + stock_iso: IsoReading, + census_iso: IsoReading, + stock_grub: MemberReading, census_grub: MemberReading, + stock_kernel: MemberReading, census_kernel: MemberReading, + stock_initrd: MemberReading, census_initrd: MemberReading, +) -> CensusMemberReadback { + match census_pin_gate(medium: medium, stock_iso: stock_iso, census_iso: census_iso) { + PinsRefused { readback: r } => r + PinsAdmitted { stock_iso: s, census_iso: c } => + census_member_verdict(stock_iso: s, census_iso: c, readings: CensusMemberReadings { + stock_grub: stock_grub, census_grub: census_grub, + stock_kernel: stock_kernel, census_kernel: census_kernel, + stock_initrd: stock_initrd, census_initrd: census_initrd, + }) + } +} + +fn census_member_readback_holds(readback: CensusMemberReadback) -> Bool { + match readback { + BootChainIdenticalToStock { stock_iso: _, census_iso: _, kernel: _, initrd: _ } => true + _ => false + } +} + +fn render_census_member_readback(readback: CensusMemberReadback) -> String { + match readback { + BootChainIdenticalToStock { stock_iso: s, census_iso: c, kernel: k, initrd: i } => + concat("verdict=boot_chain_identical_to_stock\nstock_iso=", s as String, "\ncensus_iso=", c as String, + "\nvmlinuz=", k as String, "\ninitrd=", i as String, "\ngrub.cfg=differs (declared rewrite, control discriminated)\n") + BootChainFirstDifference { stock_iso: s, census_iso: c, member: m, stock: sd, census: cd } => + concat("verdict=first_difference\nstock_iso=", s as String, "\ncensus_iso=", c as String, + "\nmember=", census_iso_member_path(member: m) as String, "\nstock_member=", sd as String, "\ncensus_member=", cd as String, "\n") + BootChainControlDidNotDiscriminate { grub_digest: g } => + concat("verdict=control_did_not_discriminate\ngrub.cfg=", g as String, " on both sides: the difference control failed; cause not chosen (census file not the one read, or the published artifact lacks the rewrite)\n") + BootChainIsoNotPinned { side: s, expected: e, observed: o } => + concat("verdict=iso_not_pinned\nside=", iso_side_slug(side: s) as String, "\nexpected=", e as String, "\nobserved=", o, "\n") + BootChainUnread { member: m, side: s, reason: r } => + concat("verdict=unread\nmember=", census_iso_member_path(member: m) as String, "\nside=", iso_side_slug(side: s) as String, "\nreason=", r, "\n") + BootChainMediumNotCensus => "verdict=medium_not_census\n" + BootChainHostUnreached { reason: r } => concat("verdict=host_unreached\nreason=", r, "\n") + BootChainWorkspaceUnavailable { reason: r } => concat("verdict=workspace_unavailable (nothing extracted)\nreason=", r, "\n") + } +} + +// ---- the one producer: read-only legs on srv2 over fleet SSH, like the medium readback ---- +// +// srv2 has had no runner slots since 2026-09-19 (gunbc.machine_intake_mtcollins1_boot_authorization), +// so this reaches the host that serves /srv/bmc the way the boot's medium readback does, as the fleet +// principal over typed argv. ONE TRANSACTION PER ATTEMPT: +// 1. measure both ISOs; a pin refusal ends the attempt before any extraction; +// 2. create an EXCLUSIVELY OWNED workspace with mktemp -d (coreutils creates a new 0700 directory +// and never reuses an existing name), so concurrent dispatches -- the mode's concurrency key is +// the runner host, while the target is always srv2 -- never share a path; no workspace, no +// extraction; +// 3. per member, extract under that workspace and hash only what extracted; +// 4. remove that workspace, and report the cleanup outcome BESIDE the readback, never in place of +// it: an initiating failure and a cleanup failure are both retained. A creation or removal whose +// reply was lost is reported UNCONFIRMED with residue outstanding, never as "never created" or +// "not removed" (WorkspaceOutcome). +// Nothing under /srv/bmc is written. + +data census_member_readback_attempt_raw: String = "mtcollins1-census-member-readback" + +data census_member_readback_receipt_path: String = "target/mtcollins1-census-member-readback.txt" + +data census_member_workspace_prefix: String = "/tmp/gunbc-mtcollins1-census-member." + +data census_member_workspace_template: NonEmptyStr = concat(census_member_workspace_prefix, "XXXXXXXX") as NonEmptyStr + +// ONE REMOTE LEG AS THE TRANSPORT ANSWERED IT, before any collapse to a RemoteAnswer. Refused means +// the invocation was never made (gunbc.typed_argv_exec, FleetPrincipalLegRefused); Ran carries what +// came back. A Ran leg with ssh's client status 255 (extdeps.ssh.session ssh_client_error_exit_status) +// establishes neither that the remote command ran nor that it did not. +type RemoteLeg + = LegNotDispatched { reason: String } + | LegAnswered { exit_code: Int, stdout: String, stderr: String } + +fn remote_leg_of(outcome: TypedArgvExecOutcome) -> RemoteLeg { + match outcome { + TypedArgvExecRefused { reason: r } => LegNotDispatched { reason: r } + TypedArgvExecConverged { result: r } => LegAnswered { exit_code: r.exit_code, stdout: r.stdout, stderr: r.stderr } + } +} + +// THE WORKSPACE'S WHOLE HISTORY, each arm naming what is established and nothing more. +// CreationNotRequested a gate stopped the attempt before mktemp was issued; +// CreationNotDispatched mktemp was refused by the transport before it was sent; +// CreationFailed mktemp itself reported failure (a non-255 status): nothing was created; +// CreationUnconfirmed mktemp was sent and its result was not established -- ssh's 255, or a +// zero exit whose output is not exactly one path under the prefix. A +// directory MAY remain; whatever came back is retained, and no name that +// was not admitted is ever used to extract or to delete; +// Removed / RemovalFailed / RemovalUnconfirmed / RemovalNotDispatched the acquired workspace's end: +// rm reported success; rm reported failure (the path may remain); +// the reply was lost (removal unconfirmed, NOT proof it remains); +// rm was never sent (the workspace remains). +// Only Removed lets an attempt hold. +type WorkspaceOutcome + = CreationNotRequested + | CreationNotDispatched { reason: String } + | CreationFailed { exit_code: Int, stderr: String } + | CreationUnconfirmed { exit_code: Int, stdout: String, stderr: String } + | Removed { dir: NonEmptyStr } + | RemovalFailed { dir: NonEmptyStr, exit_code: Int, stderr: String } + | RemovalUnconfirmed { dir: NonEmptyStr, exit_code: Int, stderr: String } + | RemovalNotDispatched { dir: NonEmptyStr, reason: String } + +type WorkspaceAcquisition + = WorkspaceAcquired { dir: NonEmptyStr } + | WorkspaceNotAcquired { outcome: WorkspaceOutcome } + +fn admitted_workspace_path(stdout: String) -> String? { + let dir = trim(s: stdout) + if !starts_with(s: dir, prefix: census_member_workspace_prefix) || dir == census_member_workspace_prefix + || string_contains(s: dir, pattern: "\n") || string_contains(s: dir, pattern: " ") || string_contains(s: dir, pattern: "/..") { + none + } else { + Present { value: dir } + } +} + +fn workspace_acquisition_of(leg: RemoteLeg) -> WorkspaceAcquisition { + match leg { + LegNotDispatched { reason: r } => WorkspaceNotAcquired { outcome: CreationNotDispatched { reason: r } } + LegAnswered { exit_code: c, stdout: o, stderr: e } => + if c == ssh_client_error_exit_status { + WorkspaceNotAcquired { outcome: CreationUnconfirmed { exit_code: c, stdout: o, stderr: e } } + } else if c != 0 { + WorkspaceNotAcquired { outcome: CreationFailed { exit_code: c, stderr: e } } + } else { + match admitted_workspace_path(stdout: o) { + Absent => WorkspaceNotAcquired { outcome: CreationUnconfirmed { exit_code: c, stdout: o, stderr: e } } + Present { value: d } => WorkspaceAcquired { dir: d as NonEmptyStr } + } + } + } +} + +fn workspace_removal_of(dir: NonEmptyStr, leg: RemoteLeg) -> WorkspaceOutcome { + match leg { + LegNotDispatched { reason: r } => RemovalNotDispatched { dir: dir, reason: r } + LegAnswered { exit_code: c, stdout: _, stderr: e } => + if c == 0 { Removed { dir: dir } } + else if c == ssh_client_error_exit_status { RemovalUnconfirmed { dir: dir, exit_code: c, stderr: e } } + else { RemovalFailed { dir: dir, exit_code: c, stderr: e } } + } +} + +// THE ATTEMPT: the readback and the workspace's outcome, both retained. It holds only when the boot +// chain is identical to stock AND this attempt's workspace was removed. +type CensusMemberReadbackAttempt { + readback: CensusMemberReadback + workspace: WorkspaceOutcome +} + +fn census_member_attempt_holds(attempt: CensusMemberReadbackAttempt) -> Bool { + census_member_readback_holds(readback: attempt.readback) && match attempt.workspace { + Removed { dir: _ } => true + _ => false + } +} + +fn render_workspace_outcome(w: WorkspaceOutcome) -> String { + match w { + CreationNotRequested => "workspace=never created: creation was not requested (nothing extracted)\n" + CreationNotDispatched { reason: r } => concat("workspace=never created: mktemp was not dispatched (nothing extracted): ", r, "\n") + CreationFailed { exit_code: c, stderr: e } => concat("workspace=never created: mktemp reported failure exit=", to_string(c), " ", trim(s: e), " (nothing extracted)\n") + CreationUnconfirmed { exit_code: c, stdout: o, stderr: e } => + concat("workspace=CREATION UNCONFIRMED exit=", to_string(c), ": a directory under ", census_member_workspace_prefix, " MAY REMAIN on srv2 (residue outstanding; not removed, nothing extracted); returned stdout=\"", o, "\" stderr=\"", trim(s: e), "\"\n") + Removed { dir: d } => concat("workspace=", d as String, " removed\n") + RemovalFailed { dir: d, exit_code: c, stderr: e } => concat("workspace=", d as String, " REMOVAL FAILED exit=", to_string(c), " ", trim(s: e), " (the path may remain)\n") + RemovalUnconfirmed { dir: d, exit_code: c, stderr: e } => concat("workspace=", d as String, " REMOVAL UNCONFIRMED exit=", to_string(c), " ", trim(s: e), " (the reply was lost; the path may or may not remain)\n") + RemovalNotDispatched { dir: d, reason: r } => concat("workspace=", d as String, " NOT REMOVED: rm was not dispatched (residue outstanding): ", r, "\n") + } +} + +fn render_census_member_attempt(attempt: CensusMemberReadbackAttempt) -> String { + concat(render_census_member_readback(readback: attempt.readback), render_workspace_outcome(w: attempt.workspace)) +} + +// THE ATTEMPT'S DECISIONS AS ONE FOLD over the legs that were issued, so the consumer-level controls +// exercise the same composition the producer runs. `readings` is consulted only when a workspace was +// acquired; `cleanup` is consulted only then too. +fn census_member_attempt_of(gate: CensusPinGate, creation: RemoteLeg, readings: fn(NonEmptyStr) -> CensusMemberReadings, cleanup: fn(NonEmptyStr) -> RemoteLeg) -> CensusMemberReadbackAttempt { + match gate { + PinsRefused { readback: r } => CensusMemberReadbackAttempt { readback: r, workspace: CreationNotRequested } + PinsAdmitted { stock_iso: s, census_iso: c } => + match workspace_acquisition_of(leg: creation) { + WorkspaceNotAcquired { outcome: o } => + CensusMemberReadbackAttempt { readback: BootChainWorkspaceUnavailable { reason: render_workspace_outcome(w: o) }, workspace: o } + WorkspaceAcquired { dir: dir } => { + let readback = census_member_verdict(stock_iso: s, census_iso: c, readings: readings(dir)) + CensusMemberReadbackAttempt { readback: readback, workspace: workspace_removal_of(dir: dir, leg: cleanup(dir)) } + } + } + } +} + +fn member_scratch_path(dir: NonEmptyStr, side: IsoSide, member: CensusIsoMember) -> NonEmptyStr { + concat(dir as String, "/", iso_side_slug(side: side) as String, "-", census_iso_member_slug(member: member) as String) as NonEmptyStr +} + +fn remote_first_field(stdout: String) -> String { + let tokens: List = filter(split(s: trim(stdout), delimiter: " "), t => t != "") + match first(tokens) { + Absent => "" + Present { value: t } => t + } +} + +fn digest_of_sha256sum(measure: RemoteAnswer) -> String? { + if measure.exit_code != 0 { none } else { + match sha256_hex_digest(hex: remote_first_field(stdout: measure.stdout)) { + Absent => none + Present { value: d } => Present { value: d.hex as String } + } + } +} + +fn sha256sum_unread_reason(measure: RemoteAnswer) -> String { + if measure.exit_code != 0 { + concat("sha256sum exit=", to_string(measure.exit_code), " ", trim(s: measure.stderr)) + } else { + concat("sha256sum printed no digest: ", trim(s: measure.stdout)) + } +} + +// A failed extraction is never hashed: the measure leg is not run, so there is no answer to join. +fn member_reading_after_extract(extract: RemoteAnswer) -> MemberReading? { + if extract.exit_code != 0 { + Present { value: MemberUnread { reason: concat("xorriso extract exit=", to_string(extract.exit_code), " ", trim(s: extract.stderr)) } } + } else { none } +} + +fn member_reading_of_measure(measure: RemoteAnswer) -> MemberReading { + match digest_of_sha256sum(measure: measure) { + Absent => MemberUnread { reason: sha256sum_unread_reason(measure: measure) } + Present { value: d } => MemberRead { digest: d as NonEmptyStr } + } +} + +fn iso_reading_of(measure: RemoteAnswer) -> IsoReading { + match digest_of_sha256sum(measure: measure) { + Absent => IsoUnmeasured { reason: sha256sum_unread_reason(measure: measure) } + Present { value: d } => IsoMeasured { digest: d as NonEmptyStr } + } +} + +fn census_member_readback_target() -> SshTarget { + SshTarget { + endpoint: fleet_probe_endpoint_for(host: operator_host_srv2 as String) as NonEmptyStr, + principal: executor_bootstrap_principal.login, + } +} + +fn remote(target: SshTarget, context: FleetSshExecutionContext, command: ArgvCommand) -> RemoteAnswer { + remote_answer_of(outcome: typed_argv_exec_over_fleet_ssh(target: target, context: context, argv: argv_words(command: command))) +} + +fn read_member_remote(target: SshTarget, context: FleetSshExecutionContext, dir: NonEmptyStr, iso_path: NonEmptyStr, side: IsoSide, member: CensusIsoMember) -> MemberReading { + let scratch = member_scratch_path(dir: dir, side: side, member: member) + let extract = remote(target: target, context: context, command: xorriso_extract_command(iso_path: iso_path, iso_internal_path: census_iso_member_path(member: member), destination: scratch)) + match member_reading_after_extract(extract: extract) { + Present { value: refused } => refused + Absent => member_reading_of_measure(measure: remote(target: target, context: context, command: sha256sum_file_command(path: scratch as String))) + } +} + +fn census_member_readings_in(target: SshTarget, context: FleetSshExecutionContext, dir: NonEmptyStr, stock_path: NonEmptyStr, census_path: NonEmptyStr) -> CensusMemberReadings { + CensusMemberReadings { + stock_grub: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: GrubConfig), + census_grub: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: GrubConfig), + stock_kernel: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: CasperKernel), + census_kernel: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: CasperKernel), + stock_initrd: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: CasperInitrd), + census_initrd: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: CasperInitrd), + } +} + +fn remote_leg(target: SshTarget, context: FleetSshExecutionContext, command: ArgvCommand) -> RemoteLeg { + remote_leg_of(outcome: typed_argv_exec_over_fleet_ssh(target: target, context: context, argv: argv_words(command: command))) +} + +fn census_member_readback(medium: MtCollins1BootMedium) -> CensusMemberReadbackAttempt { + match prepare_fleet_ssh_agent_context(attempt_raw: census_member_readback_attempt_raw) { + FleetSshContextRefused { cause: c } => + CensusMemberReadbackAttempt { readback: BootChainHostUnreached { reason: c }, workspace: CreationNotRequested } + FleetSshContextReady { context: context, receipt: _ } => { + let target = census_member_readback_target() + let stock_path = mtcollins1_census_image_stock_path() + let census_path = mtcollins1_census_medium_served_path(medium: medium) + let gate = census_pin_gate( + medium: medium, + stock_iso: iso_reading_of(measure: remote(target: target, context: context, command: sha256sum_file_command(path: stock_path as String))), + census_iso: iso_reading_of(measure: remote(target: target, context: context, command: sha256sum_file_command(path: census_path as String))), + ) + match gate { + PinsRefused { readback: r } => CensusMemberReadbackAttempt { readback: r, workspace: CreationNotRequested } + PinsAdmitted { stock_iso: _, census_iso: _ } => + census_member_attempt_of( + gate: gate, + creation: remote_leg(target: target, context: context, command: mktemp_directory_command(template: census_member_workspace_template)), + readings: dir => census_member_readings_in(target: target, context: context, dir: dir, stock_path: stock_path, census_path: census_path), + cleanup: dir => remote_leg(target: target, context: context, command: rm_recursive_force_one_file_system_command(path: dir as String)), + ) + } + } + } +} + +// CONSUMPTION (DESIGN §3c): invoked by the fleet-converge mode mtcollins1_census_member_readback, +// which consumes the fleet SSH key; the receipt is uploaded as that run's artifact whatever the +// verdict. The run exits nonzero unless the chain is identical AND the workspace was removed. +fn mtcollins1_census_member_readback_wet() -> ProcessExit { + let attempt = census_member_readback(medium: mtcollins1_boot_medium) + let rendered = render_census_member_attempt(attempt: attempt) + let written = Filesystem.Write(path: census_member_readback_receipt_path, content: rendered) + if written.success == false { + exit_failure(reason: concat("mtcollins1 census member readback: the receipt could not be written: ", written.error, "\n", rendered)) + } else if census_member_attempt_holds(attempt: attempt) { + ExitSuccess + } else { + exit_failure(reason: concat("mtcollins1 census member readback did not hold:\n", rendered)) + } +} diff --git a/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag b/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag new file mode 100644 index 00000000000..cbce7165301 --- /dev/null +++ b/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag @@ -0,0 +1,210 @@ +module test.claim.machine_intake.kernel_module_decompression_observation_witness_test + +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import std.types { Bool, List, String } +import std.measure { byte_size } +import extdeps.linux.module_decompress { ZstdCorruptionDetected, ZstdErrorCodeOther } +import gunbc.machine_intake_kernel_module_decompression_observation { + kernel_module_decompression_observation, kernel_module_decompression_text, kernel_module_decompression_findings, + KernelModuleDecompressionObservation, NoRefusalRecorded, RefusalRecorded, + RefusalIdentityUnread, RefusalIdentityCandidates, ModuleInsertFailure, DriverPrintedLater, DriverNotSeenInCapture, +} +import gunbc.machine_intake_mtcollins1_boot_diagnostic_bundle { mtcollins1_boot_console_reading, ReferenceCaptureRead, ConsoleRetained } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// EXCERPT OF THE FAILED ATTEMPT'S RETAINED SOL CAPTURE, byte-for-byte lines 1893-1897 of +// mtcollins1-boot-sol.capture (sha256 761e79ca42b7ac955a3a4186a1cc5c2bcc1fde0422340d5be02cde2690fcee87) +// in artifact mtcollins1-boot-receipts of fleet-converge run 36335369059 +// (artifact digest sha256:16dcc1b34233cea3b338104a6cff5d88a46f8b6c2e815443653da5271062dd9b). +// That boot ran without the module-identity instrument, so no loader record exists. +data failed_attempt_excerpt: List = [ + "[ 10.149389] Run /init as init process\r", + "[ 10.374590] ZSTD-decompression failed with status 20\r", + "[ 10.380475] ZSTD-decompression failed with status 20\r", + "[ 10.392987] igb: Intel(R) Gigabit Ethernet Network Driver\r", + "[ 10.398716] igb: Copyright (c) 2007-2014 Intel Corporation.\r", +] + +// EXCERPT OF THE CENSUS-REACHING ATTEMPT'S RETAINED SOL CAPTURE, lines 1895-1900 of +// mtcollins1-boot-sol.capture (sha256 83e818fb2be55383966f5e98c69c7853625d429941192062530da5f5d21f8161) +// in artifact mtcollins1-boot-receipts of fleet-converge run 36253081549 +// (artifact digest sha256:ca38324ebcad6477911b0f56a86f91a0785a77dc31163ede6b9afc616fde3496). +// It carries the renesas "fallback to ROM" warning, which is not a refusal. +data census_reaching_excerpt: List = [ + "[ 10.154211] Run /init as init process\r", + "[ 10.391272] xhci_hcd 0004:03:00.0: Adding to iommu group 17\r", + "[ 10.391751] igb: Intel(R) Gigabit Ethernet Network Driver\r", + "[ 10.397592] xhci_hcd 0004:03:00.0: failed to load firmware renesas_usb_fw.mem, fallback to ROM\r", + "[ 10.402897] igb: Copyright (c) 2007-2014 Intel Corporation.\r", + "[ 10.403241] xhci_hcd 0004:03:00.0: xHCI Host Controller\r", +] + +// THE OFFLINE CONTROL'S INSTRUMENTED CAPTURE, lines 323-325 of its ttyAMA0 log (sha256 +// b8b333934516a1172baabb11cc2b8b649e83d367578aa63773582e556daf6bcf): qemu-system-aarch64 10.0.13 -M virt +// with a qemu-xhci device, booting the stock 24.04.3 /casper/vmlinuz and /casper/initrd with ONE edit -- +// 64 bytes inverted inside the zstd body of xhci-pci.ko.zst, frame header intact (initrd sha256 +// 83fd2a47a6449584e5062d5238ad84915da7d095c5ebdee406469046a8e40f96) -- under the census cmdline plus +// mtcollins1_census_module_identity_args, "--- console=tty0" kept. The same boot WITHOUT the args +// (log sha256 7e8db2f8d916ea009b2085c65e5a2348d77c06d27ad4eeac60f03988874b5552) printed the status-20 +// line and no loader record: the instrument is what names the file. The uas and xhci_pci lines are +// DESIGNED, added to exercise the errno filter and the later-driver reading. +data instrumented_excerpt: List = [ + "[ 27.190394] ZSTD-decompression failed with status 20\r", + "[ 27.209606] (udev-worker)[100]: Failed to insert module '/lib/modules/6.8.0-71-generic/kernel/drivers/usb/host/xhci-pci.ko.zst': Invalid argument\r", + "[ 27.300000] (udev-worker)[101]: Failed to insert module '/lib/modules/6.8.0-71-generic/kernel/drivers/usb/storage/uas.ko.zst': No such device\r", + "[ 28.916926] input: gpio-keys as /devices/platform/gpio-keys/input/input0\r", + "[ 29.000000] xhci_pci: printed later\r", +] + +// POSITIVE CONTROL: the census-reaching boot is not a refusal, renesas warning included. +test fn the_census_reaching_capture_reads_as_not_refused() -> Bool { + let o = kernel_module_decompression_observation(lines: census_reaching_excerpt) + count(kernel_module_decompression_findings(o: o)) == 0 + && match o { + NoRefusalRecorded { insert_failures: fs, unparsed_insert_records: us } => count(fs) == 0 && count(us) == 0 + _ => false + } +} + +// RED, ANONYMOUS: the failed boot is a refusal with both statuses retained, and identity UNREAD. +test fn the_failed_capture_is_a_refusal_whose_identity_is_unread() -> Bool { + match kernel_module_decompression_observation(lines: failed_attempt_excerpt) { + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: RefusalIdentityUnread } => + count(rs) == 2 && count(urs) == 0 + && match first(rs) { + Absent => false + Present { value: r } => r.at == "10.374590" && match r.status { ZstdCorruptionDetected => true _ => false } + } + _ => false + } +} + +test fn the_failed_capture_text_names_no_module_and_leaves_the_request_unread_on_sol() -> Bool { + let text = kernel_module_decompression_text(o: kernel_module_decompression_observation(lines: failed_attempt_excerpt)) + string_contains(s: text, pattern: "module identity UNREAD") + && string_contains(s: text, pattern: "REFUSED 2 time(s)") + && string_contains(s: text, pattern: "status corruption_detected") + && string_contains(s: text, pattern: "requesting device/modalias: unread on SOL") +} + +// THE INSTRUMENTED CAPTURE: the EINVAL insert failure is a CANDIDATE, identity UNRESOLVED -- the +// capture carries no invocation-level link, however close the two lines sit. Every named failure, +// candidate or not, is retained as its own observation. +test fn an_instrumented_capture_keeps_the_named_failure_as_an_unresolved_candidate() -> Bool { + match kernel_module_decompression_observation(lines: instrumented_excerpt) { + RefusalRecorded { refusals: rs, unparsed_refusal_records: _, insert_failures: all, unparsed_insert_records: _, identity: RefusalIdentityCandidates { candidates: cs } } => + count(rs) == 1 && count(all) == 2 && count(cs) == 1 + && match first(cs) { Absent => false Present { value: f } => named_xhci_pci(f: f) } + _ => false + } +} + +test fn the_instrumented_text_says_unresolved_not_known() -> Bool { + let text = kernel_module_decompression_text(o: kernel_module_decompression_observation(lines: instrumented_excerpt)) + string_contains(s: text, pattern: "identity UNRESOLVED") + && string_contains(s: text, pattern: "not established by this capture") + && string_contains(s: text, pattern: "uas.ko.zst: No such device") +} + +fn named_xhci_pci(f: ModuleInsertFailure) -> Bool { + (f.path as String) == "/lib/modules/6.8.0-71-generic/kernel/drivers/usb/host/xhci-pci.ko.zst" + && f.loader == "(udev-worker)[100]" + && f.at == "27.209606" + && f.module_name == "xhci_pci" + && match f.later { DriverPrintedLater { at: a, line: _ } => a == "29.000000" DriverNotSeenInCapture => false } +} + +// REVIEW COUNTEREXAMPLE (eager-owl-205): an unrelated EINVAL failure BEFORE the refusal is not the +// refusal's identity; it is at most a candidate, and the verdict never reads "known". +test fn an_unrelated_earlier_einval_failure_is_only_a_candidate() -> Bool { + match kernel_module_decompression_observation(lines: [ + "[ 9.000000] (udev-worker)[99]: Failed to insert module '/unrelated.ko.zst': Invalid argument", + "[ 10.374590] ZSTD-decompression failed with status 20", + ]) { + RefusalRecorded { refusals: _, unparsed_refusal_records: _, insert_failures: _, unparsed_insert_records: _, identity: RefusalIdentityCandidates { candidates: cs } } => count(cs) == 1 + _ => false + } +} + +// A loader record with no refusal line is not a decompression refusal, and is still retained. +test fn a_loader_failure_without_a_kernel_refusal_is_not_a_refusal_but_is_kept() -> Bool { + match kernel_module_decompression_observation(lines: instrumented_excerpt.skip(n: 1).take(n: 1)) { + NoRefusalRecorded { insert_failures: fs, unparsed_insert_records: _ } => count(fs) == 1 + _ => false + } +} + +// MALFORMED-ONLY: a recognised refusal record whose status cannot be read is a refusal record, never +// "no refusal", and no status is invented for it. +test fn a_malformed_refusal_record_alone_is_still_a_refusal_with_its_line_kept() -> Bool { + match kernel_module_decompression_observation(lines: [ + "[ 10.374590] ZSTD-decompression failed with status", + "[ 10.374590] ZSTD-decompression failed with status unreadable", + ]) { + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: _ } => + count(rs) == 0 && count(urs) == 2 + && match first(urs) { Absent => false Present { value: u } => u.field == "status" && u.at == "10.374590" && string_contains(s: u.line, pattern: "failed with status") } + _ => false + } +} + +// VALID PLUS MALFORMED: the parsed refusal is kept, the unparsed one is not dropped, and the text +// does not state a complete count. +test fn a_valid_plus_malformed_refusal_does_not_state_a_complete_count() -> Bool { + let o = kernel_module_decompression_observation(lines: [ + "[ 10.374590] ZSTD-decompression failed with status 20", + "[ 10.380475] ZSTD-decompression failed with sta", + ]) + let text = kernel_module_decompression_text(o: o) + match o { + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: _ } => count(rs) == 1 && count(urs) == 1 + _ => false + } + && string_contains(s: text, pattern: "the total is not stated") + && !string_contains(s: text, pattern: "REFUSED 1 time(s)") +} + +// An uncatalogued numeric status stays a parsed refusal carrying its integer. +test fn an_uncatalogued_numeric_status_is_parsed_not_unread() -> Bool { + match kernel_module_decompression_observation(lines: ["[ 1.0] ZSTD-decompression failed with status 42"]) { + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: _ } => + count(urs) == 0 && match first(rs) { Absent => false Present { value: r } => match r.status { ZstdErrorCodeOther { code: c } => c == 42 _ => false } } + _ => false + } +} + +// INCOMPLETE LOADER RECORDS: truncated before the path end, or with no errno, are kept as unparsed +// insert records with the field that failed -- neither dropped nor turned into a candidate. +test fn incomplete_loader_records_are_kept_as_unparsed() -> Bool { + match kernel_module_decompression_observation(lines: [ + "[ 10.374590] ZSTD-decompression failed with status 20", + "[ 10.374801] (udev-worker)[412]: Failed to insert module '/lib/modules/6.8.0-71-generic/kernel/drivers/usb/host/xhci-pci.ko.z", + "[ 10.374802] (udev-worker)[413]: Failed to insert module '/lib/modules/x.ko.zst': ", + ]) { + RefusalRecorded { refusals: _, unparsed_refusal_records: _, insert_failures: fs, unparsed_insert_records: us, identity: RefusalIdentityUnread } => + count(fs) == 0 && count(us) == 2 + && match first(us) { Absent => false Present { value: u } => u.field == "path" } + && match first(us.skip(n: 1)) { Absent => false Present { value: u } => u.field == "errno" } + _ => false + } +} + +// UNRELATED TEXT is neither a refusal nor a loader record. +test fn unrelated_text_is_not_a_record() -> Bool { + match kernel_module_decompression_observation(lines: ["[ 1.0] zstd: something else", "[ 1.1] Failed to allocate memory pressure watch"]) { + NoRefusalRecorded { insert_failures: fs, unparsed_insert_records: us } => count(fs) == 0 && count(us) == 0 + _ => false + } +} + +// INHABITANCE: the attempt-bound console reading carries this observation from the capture text it +// retains -- the real route the boot bundle renders, not a supplied observation. +test fn the_retained_console_reading_carries_the_refusal() -> Bool { + let capture = join(failed_attempt_excerpt, "\n") + match mtcollins1_boot_console_reading(path: "artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 10), capture: concat(capture, "\n"), read_ok: true, read_error: "", reference: ReferenceCaptureRead { text: "", ok: false, error: "not supplied" }) { + ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: _, sockets: _, against_reference: _, modules: m } => + match m { RefusalRecorded { refusals: rs, unparsed_refusal_records: _, insert_failures: _, unparsed_insert_records: _, identity: RefusalIdentityUnread } => count(rs) == 2 _ => false } + _ => false + } +} diff --git a/dag/test/claim/machine_intake/mtcollins1_boot_diagnostic_bundle_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_boot_diagnostic_bundle_witness_test.dag index 329fd0a3555..576b72acd32 100644 --- a/dag/test/claim/machine_intake/mtcollins1_boot_diagnostic_bundle_witness_test.dag +++ b/dag/test/claim/machine_intake/mtcollins1_boot_diagnostic_bundle_witness_test.dag @@ -454,7 +454,7 @@ test fn a_capture_outside_the_uploaded_glob_is_reported_not_retained() -> Bool { test fn a_retained_capture_reports_its_firmware_statement() -> Bool { let capture = "ERROR: Non-identical DIMM mixture NOT supported!\n" match mtcollins1_boot_console_reading(path: "/w/artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 51), capture: capture, read_ok: true, read_error: "", reference: no_reference) { - ConsoleRetained { path: _, size: b, lines: _, firmware: f, dram: _, sockets: _, against_reference: _ } => (byte_size_count(b: b) as Int) == 51 && match f { TrainingRefusedByFirmware { statement: _, locus: _ } => true _ => false } + ConsoleRetained { path: _, size: b, lines: _, firmware: f, dram: _, sockets: _, against_reference: _, modules: _ } => (byte_size_count(b: b) as Int) == 51 && match f { TrainingRefusedByFirmware { statement: _, locus: _ } => true _ => false } _ => false } } @@ -550,7 +550,7 @@ test fn a_supplied_reference_renders_its_differences() -> Bool { let reference = "UEFI RC version: 1.08\r\n Number of active sockets : 2\r\n Inter Socket Connection 0 : Width: x16 / Speed 25 GT/s\r\n Socket[0]: Core voltage : 1065\r\n Socket[1]: Core voltage : 1035\r\n" let capture = "UEFI RC version: 1.08\r\n Number of active sockets : 1\r\n Socket[0]: Core voltage : 1065\r\n" match mtcollins1_boot_console_reading(path: "artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 100), capture: capture, read_ok: true, read_error: "", reference: reference_of(text: reference)) { - ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: _, sockets: _, against_reference: _ } => { + ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: _, sockets: _, against_reference: _, modules: _ } => { let b = bundle_socket1_absent() let text = mtcollins1_boot_bundle_text(bundle: MtCollins1BootDiagnosticBundle { attempt: b.attempt, outcome: b.outcome, sel_before: b.sel_before, sel_after: b.sel_after, sdr_cache: b.sdr_cache, sensors: b.sensors, smpro: b.smpro, inventory: b.inventory, console: mtcollins1_boot_console_reading(path: "artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 100), capture: capture, read_ok: true, read_error: "", reference: reference_of(text: reference)), run: no_run() }) diff --git a/dag/test/claim/machine_intake/mtcollins1_census_medium_readback_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_census_medium_readback_witness_test.dag index 8a438addd02..c721141e3e4 100644 --- a/dag/test/claim/machine_intake/mtcollins1_census_medium_readback_witness_test.dag +++ b/dag/test/claim/machine_intake/mtcollins1_census_medium_readback_witness_test.dag @@ -57,14 +57,14 @@ test fn the_served_path_and_the_record_path_derive_from_the_census_input() -> Bo Mtcollins1CensusImageDerived { input: input } => (mtcollins1_census_medium_served_path(medium: census) as String) == "/srv/bmc/gunbc-mtcollins1-census-ubuntu-24.04.3-arm64-30ca88d43f891399.iso" && (mtcollins1_census_record_path(input: input) as String) == concat("/srv/bmc/.gunbc-mtcollins1-census-ubuntu-24.04.3-arm64.", mtcollins1_census_build_key(input: input) as String, ".built") - && (mtcollins1_census_build_key(input: input) as String) == "8d357a525f078cb4" + && (mtcollins1_census_build_key(input: input) as String) == "5828991ca3dcace9" } } test fn a_record_and_a_file_that_both_repeat_the_selected_digest_agree_and_retain_every_value() -> Bool { match mtcollins1_census_medium_readback_from_answers(medium: census, host: srv2, record: record_ok(), measure: measure_ok()) { CensusMediumReadbackAgreed { host: h, build_key: k, recorded_digest: r, image_name: n, served_path: p, measured_digest: m } => - (h as String) == "srv2" && (k as String) == "8d357a525f078cb4" && (r as String) == (recorded as String) && (m as String) == (recorded as String) + (h as String) == "srv2" && (k as String) == "5828991ca3dcace9" && (r as String) == (recorded as String) && (m as String) == (recorded as String) && (n as String) == "gunbc-mtcollins1-census-ubuntu-24.04.3-arm64-30ca88d43f891399.iso" && (p as String) == "/srv/bmc/gunbc-mtcollins1-census-ubuntu-24.04.3-arm64-30ca88d43f891399.iso" _ => false diff --git a/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag new file mode 100644 index 00000000000..137d946d20f --- /dev/null +++ b/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag @@ -0,0 +1,254 @@ +module test.claim.machine_intake.mtcollins1_census_member_readback_witness_test + +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import std.types { Bool, Int, NonEmptyStr, String } +import gunbc.machine_intake_mtcollins1_boot_authorization { MtCollins1CensusMedium, MtCollins1StockInstallerMedium } +import gunbc.machine_intake_mtcollins1_census_member_readback { + census_member_readback_from_readings, census_member_readback_holds, render_census_member_readback, + MemberReading, MemberRead, MemberUnread, IsoReading, IsoMeasured, IsoUnmeasured, + CensusMemberReadback, BootChainIdenticalToStock, BootChainFirstDifference, BootChainControlDidNotDiscriminate, + BootChainIsoNotPinned, BootChainUnread, BootChainMediumNotCensus, + CasperKernel, CasperInitrd, GrubConfig, StockSide, CensusSide, + census_pin_gate, PinsAdmitted, PinsRefused, CensusPinGate, CensusMemberReadings, + RemoteLeg, remote_leg_of, census_member_attempt_of, + WorkspaceOutcome, CreationNotRequested, CreationNotDispatched, CreationFailed, CreationUnconfirmed, + Removed, RemovalFailed, RemovalUnconfirmed, RemovalNotDispatched, + CensusMemberReadbackAttempt, census_member_attempt_holds, render_census_member_attempt, + member_reading_after_extract, member_reading_of_measure, BootChainWorkspaceUnavailable, +} +import gunbc.typed_argv_exec { TypedArgvExecConverged, TypedArgvExecRefused } +import extdeps.ssh.session { SshSessionExecResult } +import gunbc.machine_intake_mtcollins1_census_medium_readback { RemoteAnswer } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +data stock_iso: NonEmptyStr = "2ee2163c9b901ff5926400e80759088ff3b879982a3956c02100495b489fd555" +data census_iso: NonEmptyStr = "30ca88d43f8913995078f6c2c917d5049842f927f3fc886a103f460dd146a706" +data census: MtCollins1CensusMedium = MtCollins1CensusMedium { output_digest: census_iso } + +fn read(d: NonEmptyStr) -> MemberReading { MemberRead { digest: d } } + +// Supplied member digests: the fold is over readings, the producer is the srv2 run. +fn over(stock_grub: MemberReading, census_grub: MemberReading, census_kernel: MemberReading, census_initrd: MemberReading) -> CensusMemberReadback { + census_member_readback_from_readings( + medium: census, + stock_iso: IsoMeasured { digest: stock_iso }, + census_iso: IsoMeasured { digest: census_iso }, + stock_grub: stock_grub, census_grub: census_grub, + stock_kernel: read(d: "k0"), census_kernel: census_kernel, + stock_initrd: read(d: "i0"), census_initrd: census_initrd, + ) +} + +// POSITIVE: the declared rewrite differs, kernel and initrd pass through -- the only verdict that holds. +test fn a_rewritten_grub_and_an_unchanged_kernel_and_initrd_read_as_identical_boot_chain() -> Bool { + let r = over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: read(d: "i0")) + census_member_readback_holds(readback: r) + && match r { + BootChainIdenticalToStock { stock_iso: _, census_iso: _, kernel: k, initrd: i } => (k as String) == "k0" && (i as String) == "i0" + _ => false + } +} + +// RED: a differing initrd is located as the first differing boundary, with both digests retained. +test fn a_differing_initrd_is_the_first_difference_and_does_not_hold() -> Bool { + let r = over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: read(d: "i1")) + census_member_readback_holds(readback: r) == false + && match r { + BootChainFirstDifference { stock_iso: _, census_iso: _, member: CasperInitrd, stock: s, census: c } => (s as String) == "i0" && (c as String) == "i1" + _ => false + } +} + +// RED: the kernel precedes the initrd in boot order, so a differing kernel is named first. +test fn a_differing_kernel_is_named_before_a_differing_initrd() -> Bool { + match over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k1"), census_initrd: read(d: "i1")) { + BootChainFirstDifference { stock_iso: _, census_iso: _, member: CasperKernel, stock: _, census: _ } => true + _ => false + } +} + +// RED: an equal grub.cfg means the difference control failed (cause not chosen); identical kernel and initrd do not rescue it. +test fn an_equal_grub_config_refuses_as_a_control_that_did_not_discriminate() -> Bool { + match over(stock_grub: read(d: "g0"), census_grub: read(d: "g0"), census_kernel: read(d: "k0"), census_initrd: read(d: "i0")) { + BootChainControlDidNotDiscriminate { grub_digest: _ } => true + _ => false + } +} + +test fn an_unread_census_initrd_is_unread_not_identical() -> Bool { + match over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: MemberUnread { reason: "xorriso" }) { + BootChainUnread { member: CasperInitrd, side: CensusSide, reason: _ } => true + _ => false + } +} + +test fn a_census_iso_off_its_pin_refuses_before_any_member_is_compared() -> Bool { + let r = census_member_readback_from_readings( + medium: census, + stock_iso: IsoMeasured { digest: stock_iso }, + census_iso: IsoMeasured { digest: stock_iso }, + stock_grub: read(d: "g0"), census_grub: read(d: "g1"), + stock_kernel: read(d: "k0"), census_kernel: read(d: "k0"), + stock_initrd: read(d: "i0"), census_initrd: read(d: "i0"), + ) + match r { + BootChainIsoNotPinned { side: CensusSide, expected: _, observed: _ } => true + _ => false + } +} + +test fn the_stock_medium_row_has_no_census_boot_chain_to_read() -> Bool { + let r = census_member_readback_from_readings( + medium: MtCollins1StockInstallerMedium, + stock_iso: IsoMeasured { digest: stock_iso }, census_iso: IsoMeasured { digest: census_iso }, + stock_grub: read(d: "g0"), census_grub: read(d: "g1"), + stock_kernel: read(d: "k0"), census_kernel: read(d: "k0"), + stock_initrd: read(d: "i0"), census_initrd: read(d: "i0"), + ) + match r { + BootChainMediumNotCensus => true + _ => false + } +} + +fn answer(code: Int, stdout: String, stderr: String) -> RemoteAnswer { RemoteAnswer { exit_code: code, stdout: stdout, stderr: stderr } } + +data stock_member_digest: NonEmptyStr = "deb2c288c12e6f35c2b9242134116ea03f7d4eb91f0c354ecd993680ffabbbf7" + +// THE PIN GATE PRECEDES EXTRACTION: an off-pin census ISO is refused by the gate the producer +// evaluates before it acquires a workspace, so the gate's refusal carries no member reading at all. +test fn an_off_pin_iso_is_refused_by_the_gate_that_precedes_extraction() -> Bool { + match census_pin_gate(medium: census, stock_iso: IsoMeasured { digest: stock_iso }, census_iso: IsoMeasured { digest: stock_iso }) { + PinsRefused { readback: BootChainIsoNotPinned { side: CensusSide, expected: _, observed: _ } } => true + _ => false + } + && match census_pin_gate(medium: census, stock_iso: IsoMeasured { digest: stock_iso }, census_iso: IsoMeasured { digest: census_iso }) { + PinsAdmitted { stock_iso: _, census_iso: _ } => true + _ => false + } +} + +// A FAILED EXTRACTION IS NEVER HASHED: its reading is decided from the extract leg alone. +test fn a_failed_extraction_is_unread_without_a_measure() -> Bool { + match member_reading_after_extract(extract: answer(code: 1, stdout: "", stderr: "xorriso : FAILURE")) { + Present { value: MemberUnread { reason: r } } => string_contains(s: r, pattern: "xorriso extract exit=1") + _ => false + } + && match member_reading_after_extract(extract: answer(code: 0, stdout: "", stderr: "")) { Absent => true _ => false } + && match member_reading_of_measure(measure: answer(code: 0, stdout: concat(stock_member_digest as String, " /tmp/x/stock-initrd\n"), stderr: "")) { + MemberRead { digest: d } => (d as String) == (stock_member_digest as String) + _ => false + } +} + +// ---- the attempt consumer: every workspace outcome, composed as the producer composes it ---- + +fn ran(code: Int, stdout: String, stderr: String) -> RemoteLeg { + remote_leg_of(outcome: TypedArgvExecConverged { result: SshSessionExecResult { exit_code: code, success: code == 0, stdout: stdout, stderr: stderr } }) +} + +fn not_dispatched(reason: String) -> RemoteLeg { remote_leg_of(outcome: TypedArgvExecRefused { reason: reason }) } + +data admitted: CensusPinGate = PinsAdmitted { stock_iso: stock_iso, census_iso: census_iso } + +data workspace_dir: String = "/tmp/gunbc-mtcollins1-census-member.Ab12Cd34" + +// Supplied member readings for an acquired workspace: an identical chain, so a non-holding attempt is +// attributable to its workspace outcome alone. +fn identical_readings(dir: NonEmptyStr) -> CensusMemberReadings { + CensusMemberReadings { + stock_grub: read(d: "g0"), census_grub: read(d: "g1"), + stock_kernel: read(d: "k0"), census_kernel: read(d: "k0"), + stock_initrd: read(d: "i0"), census_initrd: read(d: "i0"), + } +} + +fn attempt(gate: CensusPinGate, creation: RemoteLeg, cleanup: RemoteLeg) -> CensusMemberReadbackAttempt { + census_member_attempt_of(gate: gate, creation: creation, readings: d => identical_readings(dir: d), cleanup: d => cleanup) +} + +fn says(a: CensusMemberReadbackAttempt, pattern: String) -> Bool { string_contains(s: render_census_member_attempt(attempt: a), pattern: pattern) } + +// POSITIVE: an admitted path, an identical chain, a confirmed removal -- the only holding attempt. +test fn an_acquired_workspace_that_is_removed_holds() -> Bool { + let a = attempt(gate: admitted, creation: ran(code: 0, stdout: concat(workspace_dir, "\n"), stderr: ""), cleanup: ran(code: 0, stdout: "", stderr: "")) + census_member_attempt_holds(attempt: a) + && says(a: a, pattern: "verdict=boot_chain_identical_to_stock") + && says(a: a, pattern: concat("workspace=", concat(workspace_dir, " removed"))) +} + +// A PIN REFUSAL ISSUES NO CREATION: the creation leg is never consulted, and "never created" is +// justified because it was never requested. +test fn a_pin_refusal_requests_no_creation() -> Bool { + let refused = census_pin_gate(medium: census, stock_iso: IsoMeasured { digest: stock_iso }, census_iso: IsoMeasured { digest: stock_iso }) + let a = attempt(gate: refused, creation: ran(code: 0, stdout: workspace_dir, stderr: ""), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: a) + && match a.workspace { CreationNotRequested => true _ => false } + && says(a: a, pattern: "creation was not requested") +} + +// ssh 255 WITH NO PATH: mktemp was sent and its result is unknown -- residue outstanding, never +// "never created", and nothing extracted. +test fn an_unacknowledged_creation_without_a_path_is_unconfirmed_not_never_created() -> Bool { + let a = attempt(gate: admitted, creation: ran(code: 255, stdout: "", stderr: "Connection reset by peer"), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: a) + && match a.workspace { CreationUnconfirmed { exit_code: c, stdout: _, stderr: _ } => c == 255 _ => false } + && match a.readback { BootChainWorkspaceUnavailable { reason: _ } => true _ => false } + && says(a: a, pattern: "CREATION UNCONFIRMED") && says(a: a, pattern: "MAY REMAIN") + && !says(a: a, pattern: "never created") +} + +// ssh 255 WITH A PARTIAL PATH: the returned text is retained as evidence and is NOT admitted as a +// workspace -- nothing is extracted under it and nothing deletes it. +test fn an_unacknowledged_creation_with_a_partial_path_keeps_the_text_and_admits_nothing() -> Bool { + let a = attempt(gate: admitted, creation: ran(code: 255, stdout: "/tmp/gunbc-mtcollins1-census-mem", stderr: "client_loop: send disconnect"), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: a) + && match a.workspace { CreationUnconfirmed { exit_code: _, stdout: o, stderr: _ } => o == "/tmp/gunbc-mtcollins1-census-mem" _ => false } + && says(a: a, pattern: "returned stdout=\"/tmp/gunbc-mtcollins1-census-mem\"") + && says(a: a, pattern: "not removed, nothing extracted") + && !says(a: a, pattern: "workspace=/tmp") +} + +// EXIT 0 WITH UNUSABLE OUTPUT: two paths, or a path outside the prefix, establish no owned +// workspace and no non-creation either. +test fn a_zero_exit_with_unusable_output_is_unconfirmed() -> Bool { + let two = attempt(gate: admitted, creation: ran(code: 0, stdout: concat(workspace_dir, concat("\n", workspace_dir)), stderr: ""), cleanup: ran(code: 0, stdout: "", stderr: "")) + let elsewhere = attempt(gate: admitted, creation: ran(code: 0, stdout: "/srv/bmc/x", stderr: ""), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: two) && !census_member_attempt_holds(attempt: elsewhere) + && match two.workspace { CreationUnconfirmed { exit_code: c, stdout: _, stderr: _ } => c == 0 _ => false } + && match elsewhere.workspace { CreationUnconfirmed { exit_code: _, stdout: _, stderr: _ } => true _ => false } +} + +// A LEGITIMATE mktemp FAILURE (its own non-255 status) established that nothing was created; a leg +// the transport refused before dispatch is kept as that, not as a lost reply. +test fn a_reported_mktemp_failure_and_an_undispatched_creation_are_never_created() -> Bool { + let failed = attempt(gate: admitted, creation: ran(code: 1, stdout: "", stderr: "mktemp: failed to create directory via template"), cleanup: ran(code: 0, stdout: "", stderr: "")) + let undispatched = attempt(gate: admitted, creation: not_dispatched(reason: "remote words are not portable"), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: failed) && !census_member_attempt_holds(attempt: undispatched) + && match failed.workspace { CreationFailed { exit_code: c, stderr: _ } => c == 1 _ => false } + && says(a: failed, pattern: "never created: mktemp reported failure") + && match undispatched.workspace { CreationNotDispatched { reason: _ } => true _ => false } + && says(a: undispatched, pattern: "mktemp was not dispatched") +} + +// CLEANUP REPLY LOSS BESIDE A RETAINED COMPARISON: the identical-chain verdict survives, the removal +// is UNCONFIRMED (not "remains"), and the attempt does not hold. +test fn a_lost_cleanup_reply_is_unconfirmed_beside_the_retained_verdict() -> Bool { + let a = attempt(gate: admitted, creation: ran(code: 0, stdout: workspace_dir, stderr: ""), cleanup: ran(code: 255, stdout: "", stderr: "Connection closed")) + !census_member_attempt_holds(attempt: a) + && census_member_readback_holds(readback: a.readback) + && says(a: a, pattern: "verdict=boot_chain_identical_to_stock") + && says(a: a, pattern: "REMOVAL UNCONFIRMED") && says(a: a, pattern: "may or may not remain") +} + +// rm's OWN failure and an undispatched rm are distinct from a lost reply. +test fn a_reported_rm_failure_and_an_undispatched_rm_are_distinct_from_a_lost_reply() -> Bool { + let failed = attempt(gate: admitted, creation: ran(code: 0, stdout: workspace_dir, stderr: ""), cleanup: ran(code: 1, stdout: "", stderr: "Device or resource busy")) + let undispatched = attempt(gate: admitted, creation: ran(code: 0, stdout: workspace_dir, stderr: ""), cleanup: not_dispatched(reason: "no transport")) + !census_member_attempt_holds(attempt: failed) && !census_member_attempt_holds(attempt: undispatched) + && match failed.workspace { RemovalFailed { dir: _, exit_code: c, stderr: _ } => c == 1 _ => false } + && says(a: failed, pattern: "REMOVAL FAILED") + && match undispatched.workspace { RemovalNotDispatched { dir: _, reason: _ } => true _ => false } + && says(a: undispatched, pattern: "rm was not dispatched") +}