From d22aa3340d77ccfcbdb09a8e65ae58403b328d15 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 28 Sep 2026 01:52:31 +0000 Subject: [PATCH 1/7] mtcollins1 census: published-ISO boot-chain readback, module-identity instrument, decompression-refusal observation Deliverable 1: gunbc.machine_intake_mtcollins1_census_member_readback reads vmlinuz, initrd and grub.cfg out of the pinned stock and census ISOs on srv2 over fleet SSH (fleet-converge mode mtcollins1_census_member_readback), with grub.cfg as the in-run discriminating control. Deliverable 2: mtcollins1_census_module_identity_args (KMOD_LOG=info udev.log_level=debug SYSTEMD_LOG_TARGET=kmsg printk.devkmsg=on) as a census build input; option A per eager-owl-205. Upstream facts homed in extdeps.linux.module_decompress, extdeps.kmod.libkmod, extdeps.systemd.udevd. Deliverables 3-4: gunbc.machine_intake_kernel_module_decompression_observation rides ConsoleRetained; controls from runs 36335369059 / 36253081549 captures and an offline qemu receipt. Co-Authored-By: Claude Opus 5.5 (1M context) --- dag/extdeps/exec/command.dag | 1 + dag/extdeps/kmod/libkmod.dag | 41 +++ dag/extdeps/linux/kernel.dag | 5 + dag/extdeps/linux/module_decompress.dag | 53 +++ dag/extdeps/systemd/udevd.dag | 41 +++ dag/extdeps/tools/xorriso.dag | 8 + dag/gunbc/ci/ci_spec.dag | 19 ++ dag/gunbc/fleet/fleet_converge_workflow.dag | 50 ++- ...ernel_module_decompression_observation.dag | 195 +++++++++++ .../mtcollins1_boot_diagnostic_bundle.dag | 10 +- .../mtcollins1_census_image.dag | 28 ++ .../mtcollins1_census_member_readback.dag | 318 ++++++++++++++++++ ...decompression_observation_witness_test.dag | 125 +++++++ ...s1_boot_diagnostic_bundle_witness_test.dag | 4 +- ...s1_census_medium_readback_witness_test.dag | 4 +- ...s1_census_member_readback_witness_test.dag | 104 ++++++ 16 files changed, 997 insertions(+), 9 deletions(-) create mode 100644 dag/extdeps/kmod/libkmod.dag create mode 100644 dag/extdeps/linux/module_decompress.dag create mode 100644 dag/extdeps/systemd/udevd.dag create mode 100644 dag/gunbc/machine_intake/kernel_module_decompression_observation.dag create mode 100644 dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag create mode 100644 dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag create mode 100644 dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index ee8aba9cdf3..a0177240acc 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -115,6 +115,7 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.cadquery.cadquery", decl_name: "cadquery_run_program_command"), decl_ref(module_path: "extdeps.cargo_build", decl_name: "cargo_clippy_command"), decl_ref(module_path: "extdeps.tools.sed", decl_name: "sed_in_place_command"), + decl_ref(module_path: "extdeps.tools.xorriso", decl_name: "xorriso_extract_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "cp_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "cat_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "od_hex_span_command"), diff --git a/dag/extdeps/kmod/libkmod.dag b/dag/extdeps/kmod/libkmod.dag new file mode 100644 index 00000000000..f48bda10473 --- /dev/null +++ b/dag/extdeps/kmod/libkmod.dag @@ -0,0 +1,41 @@ +module extdeps.kmod.libkmod + +import std.types { NonEmptyStr, String } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Https, Uri } + +// libkmod v31 (the version in the Ubuntu 24.04.3 live-server initrd, `kmod version 31`). +// kmod_module_insert_module logs a failed insertion at INFO with the module FILE PATH: +// INFO(mod->ctx, "Failed to insert module '%s': %s\n", path, strerror(-err)); +// and a context's log priority defaults to LOG_ERR unless the KMOD_LOG environment variable names +// another (libkmod.c kmod_new: secure_getenv("KMOD_LOG") -> kmod_set_log_priority). So by default the +// path-bearing line is filtered inside libkmod before any caller's log function sees it. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/kmod-project/kmod/blob/v31/libkmod/libkmod-module.c" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.kmod.libkmod", + decl_name: "libkmod_insert_failed_prefix", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +data libkmod_insert_failed_prefix: NonEmptyStr = "Failed to insert module '" + +// The separator after the quoted path, before strerror(-err). +data libkmod_insert_failed_path_end: NonEmptyStr = "': " + +data libkmod_log_priority_env_name: NonEmptyStr = "KMOD_LOG" + +// log_priority() accepts a syslog name or a number; "info" selects LOG_INFO, the INFO() level. +data libkmod_log_priority_info: NonEmptyStr = "info" diff --git a/dag/extdeps/linux/kernel.dag b/dag/extdeps/linux/kernel.dag index 9045a105b26..b63a99538bb 100644 --- a/dag/extdeps/linux/kernel.dag +++ b/dag/extdeps/linux/kernel.dag @@ -37,3 +37,8 @@ type LinuxKernelRelease = NonEmptyStr where brand("LinuxKernelRelease") // /sys/module all key on. Branded rather than left a bare String so a module name cannot be // interchanged with the arbitrary text beside it in a driver-binding row. type LinuxKernelModuleName = NonEmptyStr where brand("LinuxKernelModuleName") + +// Documentation/admin-guide/kernel-parameters.txt, printk.devkmsg=: "ratelimit" (the default) limits +// records userspace writes to /dev/kmsg; "on" admits them unlimited. A userspace logger that writes a +// burst of records at boot loses all but the first few under the default. +data linux_printk_devkmsg_on_cmdline_arg: NonEmptyStr = "printk.devkmsg=on" diff --git a/dag/extdeps/linux/module_decompress.dag b/dag/extdeps/linux/module_decompress.dag new file mode 100644 index 00000000000..65490dfb498 --- /dev/null +++ b/dag/extdeps/linux/module_decompress.dag @@ -0,0 +1,53 @@ +module extdeps.linux.module_decompress + +import std.types { Int, NonEmptyStr, String } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Https, Uri } + +// IN-KERNEL MODULE DECOMPRESSION (CONFIG_MODULE_DECOMPRESS), Linux v6.8 kernel/module/decompress.c. +// finit_module(2) with MODULE_INIT_COMPRESSED_FILE hands the kernel a compressed module file; a zstd +// decoder error is printed by module_zstd_decompress as the line below and returned to the caller as +// -EINVAL. THE LINE NAMES NO MODULE: the printk carries the decoder status and nothing else, so the +// kernel log alone never identifies which file failed -- that identity is a userspace fact (the +// loader that issued the finit_module call). +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/torvalds/linux/blob/v6.8/kernel/module/decompress.c" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.linux.module_decompress", + decl_name: "linux_module_zstd_decompress_failed_prefix", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +// pr_err("ZSTD-decompression failed with status %d\n", retval), retval = zstd_get_error_code(ret). +data linux_module_zstd_decompress_failed_prefix: NonEmptyStr = "ZSTD-decompression failed with status " + +// zstd_errors.h ZSTD_ErrorCode: the status values a decoder can return are the library's own enum. +// Only the rows this corpus has observed are carried; any other status is kept as its integer. +type ZstdErrorCode + = ZstdCorruptionDetected + | ZstdErrorCodeOther { code: Int } + +data zstd_error_corruption_detected_code: Int = 20 + +fn zstd_error_code_of(code: Int) -> ZstdErrorCode { + if code == zstd_error_corruption_detected_code { ZstdCorruptionDetected } else { ZstdErrorCodeOther { code: code } } +} + +fn zstd_error_code_name(code: ZstdErrorCode) -> String { + match code { + ZstdCorruptionDetected => "corruption_detected" + ZstdErrorCodeOther { code: c } => concat("zstd error ", to_string(c)) + } +} diff --git a/dag/extdeps/systemd/udevd.dag b/dag/extdeps/systemd/udevd.dag new file mode 100644 index 00000000000..db46aaf7794 --- /dev/null +++ b/dag/extdeps/systemd/udevd.dag @@ -0,0 +1,41 @@ +module extdeps.systemd.udevd + +import std.types { NonEmptyStr, String } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Https, Uri } + +// systemd-udevd v255 (255.4-1ubuntu8.10 in the Ubuntu 24.04.3 live-server initrd). +// run_udevd parses the environment (SYSTEMD_LOG_LEVEL, SYSTEMD_LOG_TARGET) and THEN the kernel +// command line, so `udev.log_level=` overrides a SYSTEMD_LOG_LEVEL set by the caller -- which is what +// initramfs-tools' scripts/init-top/udev does (SYSTEMD_LOG_LEVEL=info). The kmod builtin routes +// libkmod's log function into udev's own log (udev_kmod_log -> log_internalv), and module_load_and_warn +// logs its own failures at DEBUG (verbose=false). With the default AUTO target and no journal, a +// daemonized udevd writes to its inherited stderr, i.e. /dev/console -- the LAST console= on the +// kernel command line -- so SYSTEMD_LOG_TARGET=kmsg is what puts its records in the kernel log. +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "github.com/systemd/systemd/blob/v255/src/udev/udevd.c" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.systemd.udevd", + decl_name: "udevd_log_level_cmdline_key", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +data udevd_log_level_cmdline_key: NonEmptyStr = "udev.log_level" + +data systemd_log_target_env_name: NonEmptyStr = "SYSTEMD_LOG_TARGET" + +data systemd_log_target_kmsg: NonEmptyStr = "kmsg" + +data systemd_log_level_debug: NonEmptyStr = "debug" diff --git a/dag/extdeps/tools/xorriso.dag b/dag/extdeps/tools/xorriso.dag index a0897197459..b5fe90be257 100644 --- a/dag/extdeps/tools/xorriso.dag +++ b/dag/extdeps/tools/xorriso.dag @@ -7,6 +7,7 @@ import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import extdeps.tools { CliTool, SourceApt } import extdeps.apt { apt_bin_dir, package_xorriso } +import extdeps.exec.command { ArgvCommand, argv_command } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -84,3 +85,10 @@ service xorriso.Iso { } } } + +// THE SAME -extract ExtractPathBestEffort runs, as an argv for a remote typed-argv transport: one ISO +// member copied out to one destination path, read-only on the ISO. xorriso exits nonzero when the +// member is absent or the destination cannot be written. +fn xorriso_extract_command(iso_path: String, iso_internal_path: String, destination: String) -> ArgvCommand { + argv_command(program: xorriso_cli_tool.name, arguments: ["-osirrox", "on", "-indev", iso_path, "-extract", iso_internal_path, destination]) +} diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 46d77a8137d..bab3a4ae917 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -58,6 +58,7 @@ import gunbc.spark.bootstrap_provision { spark_bootstrap_cloud_principal_member import gunbc.machine_intake_mtcollins1_fan_observe { mtcollins1_fan_credential_path_env, mtcollins1_fan_observation_receipt_path } import gunbc.fabric_writer_identity_observe { fabric_writer_identity_receipt_path } import gunbc.machine_intake_mtcollins1_census_image { mtcollins1_census_image_receipt_path } +import gunbc.machine_intake_mtcollins1_census_member_readback { census_member_readback_receipt_path } import std.dissolution { DissolutionCondition, dissolution_description, unbound_dissolution } @@ -999,6 +1000,11 @@ data gunbc_ci_mtcollins1_census_image_publish_target: GunbcRunStepTarget = Gunbc function: "mtcollins1_census_image_publish_wet", } +data gunbc_ci_mtcollins1_census_member_readback_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag", + function: "mtcollins1_census_member_readback_wet", +} + data gunbc_ci_spark_grant_install_target: GunbcRunStepTarget = GunbcRunStepTarget { entry: "dag/gunbc/spark/managed_grant_install.dag", function: "spark_grant_install_ci_wet", @@ -1266,6 +1272,7 @@ fn gunbc_run_step_targets() -> List { gunbc_ci_host_reset_dispatch_admission_target, gunbc_ci_mtcollins1_fan_observe_target, gunbc_ci_mtcollins1_census_image_publish_target, + gunbc_ci_mtcollins1_census_member_readback_target, gunbc_ci_spark_grant_install_target, gunbc_ci_spark_bootstrap_target, gunbc_ci_spark_serving_apply_target, @@ -2582,6 +2589,18 @@ fn gunbc_ci_mtcollins1_census_image_publish_invoke() -> String { ) } +// The member readback reaches srv2 over fleet SSH (the mode consumes the fleet key, see +// gunbc.fleet_converge_workflow) and touches no controller or credential: no prelude. +fn gunbc_ci_mtcollins1_census_member_readback_invoke() -> String { + gunbc_run_step_script( + source_roots: witness_layer_roots, + entry: gunbc_ci_mtcollins1_census_member_readback_target.entry, + function: gunbc_ci_mtcollins1_census_member_readback_target.function, + claim_run: false, + receipt_rel: census_member_readback_receipt_path + ) +} + fn gunbc_ci_runner_guest_image_converge_invoke() -> String { gunbc_run_step_script( source_roots: witness_layer_roots, diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index c28dae01e8c..fb14fdc3c1c 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -29,6 +29,7 @@ import gunbc.cloudflare.r2_bucket_ensure { r2_bucket_ensure_receipt_path } import gunbc.machine_intake_mtcollins1_fan_observe { mtcollins1_fan_observation_receipt_path } import gunbc.fabric_writer_identity_observe { fabric_writer_identity_receipt_path } import gunbc.machine_intake_mtcollins1_census_image { mtcollins1_census_image_receipt_path } +import gunbc.machine_intake_mtcollins1_census_member_readback { census_member_readback_receipt_path } import gunbc.cloudflare.r2_token_mint_run { r2_mint_receipt_glob_in, r2_mint_receipt_scope, R2MintReceiptAttemptScoped, R2OriginObjectWrite, R2AccountBucketAdmin, } @@ -114,6 +115,7 @@ import gunbc.ci_spec { gunbc_ci_mtcollins1_fan_observe_invoke, gunbc_ci_fabric_writer_identity_observe_invoke, gunbc_ci_mtcollins1_census_image_publish_invoke, + gunbc_ci_mtcollins1_census_member_readback_invoke, gunbc_ci_r2_mint_preflight_invoke, gunbc_ci_r2_bucket_ensure_invoke, gunbc_ci_r2_bucket_admin_mint_invoke, @@ -280,6 +282,7 @@ type FleetConvergeWorkflowMode | MtCollins1FanObserve | FabricWriterIdentityObserve | MtCollins1CensusImagePublish + | MtCollins1CensusMemberReadback | MicrovmRunnerGroupEnsure | OrgRunnerRosterObserve | GcpIamConverge @@ -336,6 +339,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String MtCollins1FanObserve => "mtcollins1_fan_observe" FabricWriterIdentityObserve => "fabric_writer_identity_observe" MtCollins1CensusImagePublish => "mtcollins1_census_image_publish" + MtCollins1CensusMemberReadback => "mtcollins1_census_member_readback" MicrovmRunnerGroupEnsure => "microvm_runner_group_ensure" OrgRunnerRosterObserve => "org_runner_roster_observe" } @@ -362,7 +366,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, MicrovmNetworkApply, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkNativeServingApply, SparkRuntimeImageProbe, SparkV41CheckpointMaterialize, SparkV41RowStoreEncode, SparkV41RowStoreReadback, SparkV41EngramDifferential, SparkV41RuntimeImageBuild, SparkV41RuntimeImageDistribute, DashboardDeploy, ApprovalBrokerDarkInstall, MicrovmControllerInstall, MicrovmSlotStart, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, ApprovalDeviceEnrolmentCodeIssue, MtCollins1Boot, MtCollins1FanObserve, MtCollins1CensusImagePublish, MtCollins1CensusMemberReadback, HostCredentialCustodyConverge, AppKeyVersionVerify, R2BucketEnsure, R2BucketAdminMint, FabricWriterIdentityObserve, PairServingD0, MicrovmRunnerGroupEnsure, GcpIamConverge, NamecheapObserve] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // // EVERY MODE IS NAMED, AND THE WILDCARD IS DELIBERATELY ABSENT. A `_ => none` would read the same @@ -434,6 +438,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc MtCollins1FanObserve => none FabricWriterIdentityObserve => none MtCollins1CensusImagePublish => none + MtCollins1CensusMemberReadback => none MicrovmRunnerGroupEnsure => none OrgRunnerRosterObserve => none } @@ -558,6 +563,7 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> MtCollins1FanObserve => FleetSshKeyNotConsumed FabricWriterIdentityObserve => FleetSshKeyNotConsumed MtCollins1CensusImagePublish => FleetSshKeyNotConsumed + MtCollins1CensusMemberReadback => FleetSshKeyConsumed MicrovmRunnerGroupEnsure => FleetSshKeyNotConsumed GcpIamConverge => FleetSshKeyNotConsumed NamecheapObserve => FleetSshKeyNotConsumed @@ -1116,6 +1122,7 @@ fn fleet_converge_mode_mutation_domain(mode: FleetConvergeWorkflowMode) -> Fleet MtCollins1FanObserve => ExecutorDomain FabricWriterIdentityObserve => ExecutorDomain MtCollins1CensusImagePublish => ExecutorDomain + MtCollins1CensusMemberReadback => ExecutorDomain MicrovmRunnerGroupEnsure => ExecutorDomain OrgRunnerRosterObserve => ExecutorDomain GcpIamConverge => ExecutorDomain @@ -2046,6 +2053,45 @@ fn fleet_converge_mtcollins1_census_image_publish_receipt_upload_step() -> Step } } +// THE PUBLISHED CENSUS IMAGE'S BOOT CHAIN, READ BACK AGAINST THE STOCK MEDIUM on the host that serves +// both (gunbc.machine_intake_mtcollins1_census_member_readback), over fleet SSH from any runner: srv2 +// has no runner slots. It writes only scratch files under srv2's /tmp, removed after each measure. +// The receipt is uploaded whatever the verdict, because a refusing reading is the observation the +// run exists to retain. +data fleet_converge_mtcollins1_census_member_readback_step_if: String = fleet_converge_mode_step_if(mode: MtCollins1CensusMemberReadback) + +fn fleet_converge_mtcollins1_census_member_readback_step() -> Step { + RunStep { + name: Present { value: "Mt. Collins census image: read the published kernel and initrd back against the stock medium" }, + id: Present { value: "mtcollins1_census_member_readback" }, + run: gunbc_ci_mtcollins1_census_member_readback_invoke(), + shell: none, + env: none, + working_directory: none, + if_condition: Present { value: fleet_converge_mtcollins1_census_member_readback_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_mtcollins1_census_image_step_timeout_minutes } + } +} + +fn fleet_converge_mtcollins1_census_member_readback_receipt_upload_step() -> Step { + UsesStep { + name: Present { value: "Upload Mt. Collins census member readback receipt" }, + id: Present { value: "mtcollins1_census_member_readback_receipt_upload" }, + uses: upload_artifact_action, + with: Present { value: [ + kv(key: "name", value: yaml_string(s: "mtcollins1-census-member-readback")), + kv(key: "path", value: yaml_string(s: census_member_readback_receipt_path)), + kv(key: "if-no-files-found", value: yaml_string(s: "error")), + kv(key: "retention-days", value: yaml_int(n: 30)), + ] }, + env: none, + if_condition: Present { value: concat("always() && ", fleet_converge_mtcollins1_census_member_readback_step_if) }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_artifact_transfer_step_timeout_minutes } + } +} + fn fleet_converge_r2_mint_preflight_step() -> Step { RunStep { name: Present { value: "R2 mint preflight: bootstrap read + account permission-group listing (reachability, no mutation)" }, @@ -3219,6 +3265,8 @@ fn fleet_converge_job() -> Job { fleet_converge_pair_serving_d0_step(), fleet_converge_mtcollins1_census_image_publish_step(), fleet_converge_mtcollins1_census_image_publish_receipt_upload_step(), + fleet_converge_mtcollins1_census_member_readback_step(), + fleet_converge_mtcollins1_census_member_readback_receipt_upload_step(), fleet_converge_r2_mint_preflight_step(), fleet_converge_r2_mint_preflight_receipt_upload_step(), fleet_converge_r2_bucket_admin_mint_step(), diff --git a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag new file mode 100644 index 00000000000..5c757666947 --- /dev/null +++ b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag @@ -0,0 +1,195 @@ +module gunbc.machine_intake_kernel_module_decompression_observation + +import std.types { Bool, Int, List, NonEmptyStr, String } +import std.algebra { trim } +import v2.std.optional { Present } +import extdeps.linux.module_decompress { linux_module_zstd_decompress_failed_prefix, ZstdErrorCode, zstd_error_code_of, zstd_error_code_name } +import extdeps.kmod.libkmod { libkmod_insert_failed_prefix, libkmod_insert_failed_path_end } + +// WHAT A RETAINED CONSOLE CAPTURE SAYS ABOUT IN-KERNEL MODULE DECOMPRESSION, AT ITS REAL STRENGTH. +// +// Three facts, kept apart because they have different producers: +// 1. the kernel REFUSED to decompress some module (extdeps.linux.module_decompress), with the +// decoder status it printed -- a decoder result, not a memory report, and no module name; +// 2. WHICH FILE was refused, known only when the loader's own record reached the capture +// (extdeps.kmod.libkmod, under the module-identity instrument's KMOD_LOG / kmsg settings); +// 3. whether that driver LATER printed anything in the capture. +// Refusal without a loader record is IDENTITY UNREAD, never a guessed module: adjacent kernel lines +// are not evidence of which finit_module call failed. The requesting device / modalias is not on the +// serial console at the instrument's chosen levels (it stays in the kmsg ring), so it is reported +// as unread on SOL and never inferred. +// +// This observation says nothing about device availability, live-root acceptance, SOL health or the +// hardware: those are other readings of the same capture with other producers. In particular the +// xhci renesas_usb_fw.mem "fallback to ROM" warning is not read here at all -- it appears in the +// census-reaching boot too, so it is not a refusal of anything. + +type ModuleDecompressionRefusal { + at: String + status: ZstdErrorCode +} + +type ModuleInsertFailure { + at: String + loader: String + path: NonEmptyStr + error: String +} + +type LaterDriverReading + = DriverPrintedLater { at: String, line: String } + | DriverNotSeenInCapture + +type FailedModuleIdentity { + failure: ModuleInsertFailure + module_name: String + later: LaterDriverReading +} + +type ModuleIdentityReading + = ModuleIdentityKnown { failures: List } + | ModuleIdentityUnread + +type KernelModuleDecompressionObservation + = ModuleDecompressionNotRefused + | ModuleDecompressionRefused { refusals: List, identity: ModuleIdentityReading } + +// The errno a decompression refusal returns to finit_module (decompress.c: retval = -EINVAL), as +// strerror renders it. An insert failure with another errno is a different failure and is not +// attributed to the refusal. +data refused_decompression_strerror: String = "Invalid argument" + +data request_identity_on_sol_text: String = "requesting device/modalias: unread on SOL (kmsg ring only)" + +fn first_or_empty(xs: List) -> String { + match first(xs) { + Absent => "" + Present { value: x } => x + } +} + +// A console line as written: CR and NUL removed. +fn console_line(raw: String) -> String { + join(split(s: join(split(s: raw, delimiter: "\r"), ""), delimiter: "\0"), "") +} + +type StampedLine { + at: String + rest: String +} + +// "[ 10.374590] text" -> at "10.374590", rest "text". An unstamped line keeps at "". +fn stamped(raw: String) -> StampedLine { + let line = console_line(raw: raw) + let closed = split(s: line, delimiter: "] ") + if starts_with(s: trim(line), prefix: "[") && count(closed) >= 2 { + StampedLine { + at: trim(join(split(s: first_or_empty(xs: closed), delimiter: "["), "")), + rest: join(closed.skip(n: 1), "] "), + } + } else { + StampedLine { at: "", rest: line } + } +} + +fn decompression_refusal(l: StampedLine) -> ModuleDecompressionRefusal? { + let prefix = linux_module_zstd_decompress_failed_prefix as String + if !string_contains(s: l.rest, pattern: prefix) { none } else { + match parse_int(s: trim(join(split(s: l.rest, delimiter: prefix).skip(n: 1), prefix))) { + Absent => none + Present { value: code } => Present { value: ModuleDecompressionRefusal { at: l.at, status: zstd_error_code_of(code: code) } } + } + } +} + +fn insert_failure(l: StampedLine) -> ModuleInsertFailure? { + let prefix = libkmod_insert_failed_prefix as String + let parts = split(s: l.rest, delimiter: prefix) + if count(parts) < 2 { none } else { + let tail = join(parts.skip(n: 1), prefix) + let path_and_error = split(s: tail, delimiter: libkmod_insert_failed_path_end as String) + let path = first_or_empty(xs: path_and_error) + if count(path_and_error) < 2 || path == "" { none } else { + Present { value: ModuleInsertFailure { + at: l.at, + loader: trim(join(split(s: trim(first_or_empty(xs: parts)), delimiter: ":"), "")), + path: path as NonEmptyStr, + error: trim(join(path_and_error.skip(n: 1), libkmod_insert_failed_path_end as String)), + } } + } + } +} + +// ".../xhci-pci-renesas.ko.zst" -> "xhci_pci_renesas": the name the kernel prints a module under. +fn module_name_of_path(path: String) -> String { + let segments = split(s: path, delimiter: "/") + let file = first_or_empty(xs: segments.skip(n: count(segments) - 1)) + join(split(s: first_or_empty(xs: split(s: file, delimiter: ".ko")), delimiter: "-"), "_") +} + +fn later_driver(lines: List, after_index: Int, module_name: String) -> LaterDriverReading { + let later = filter(lines.skip(n: after_index + 1), l => starts_with(s: l.rest, prefix: concat(module_name, " ")) || starts_with(s: l.rest, prefix: concat(module_name, ":"))) + match first(later) { + Absent => DriverNotSeenInCapture + Present { value: l } => DriverPrintedLater { at: l.at, line: l.rest } + } +} + +fn kernel_module_decompression_observation(lines: List) -> KernelModuleDecompressionObservation { + let stamped_lines = map(lines, raw => stamped(raw: raw)) + let refusals = flat_map(stamped_lines, l => match decompression_refusal(l: l) { Absent => [] Present { value: r } => [r] }) + if count(refusals) == 0 { ModuleDecompressionNotRefused } else { + let failures = flat_map(enumerate(stamped_lines), il => match insert_failure(l: il.second) { + Absent => [] + Present { value: f } => + if f.error == refused_decompression_strerror { + [FailedModuleIdentity { + failure: f, + module_name: module_name_of_path(path: f.path as String), + later: later_driver(lines: stamped_lines, after_index: il.first, module_name: module_name_of_path(path: f.path as String)), + }] + } else { [] } + }) + ModuleDecompressionRefused { + refusals: refusals, + identity: if count(failures) == 0 { ModuleIdentityUnread } else { ModuleIdentityKnown { failures: failures } }, + } + } +} + +fn refusal_text(r: ModuleDecompressionRefusal) -> String { + concat("[", r.at, "] status ", zstd_error_code_name(code: r.status)) +} + +fn later_text(l: LaterDriverReading) -> String { + match l { + DriverPrintedLater { at: a, line: t } => concat("printed later at ", a, ": ", t) + DriverNotSeenInCapture => "not seen later in this capture" + } +} + +fn failed_module_text(f: FailedModuleIdentity) -> String { + concat("[", f.failure.at, "] ", f.failure.loader, " ", f.failure.path as String, ": ", f.failure.error, "; ", f.module_name, " ", later_text(l: f.later)) +} + +fn kernel_module_decompression_text(o: KernelModuleDecompressionObservation) -> String { + match o { + ModuleDecompressionNotRefused => "module decompression: no refusal in this capture" + ModuleDecompressionRefused { refusals: rs, identity: id } => + concat( + "module decompression REFUSED ", to_string(count(rs)), " time(s): ", join(map(rs, r => refusal_text(r: r)), ", "), "; ", + match id { + ModuleIdentityUnread => "module identity UNREAD (no loader record reached this capture)" + ModuleIdentityKnown { failures: fs } => concat("module identity: ", join(map(fs, f => failed_module_text(f: f)), "; ")) + }, + "; ", request_identity_on_sol_text, + ) + } +} + +fn kernel_module_decompression_findings(o: KernelModuleDecompressionObservation) -> List { + match o { + ModuleDecompressionNotRefused => [] + ModuleDecompressionRefused { refusals: _, identity: _ } => [kernel_module_decompression_text(o: o)] + } +} diff --git a/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag b/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag index 61522ef06d6..46afa8c8f93 100644 --- a/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag +++ b/dag/gunbc/machine_intake/mtcollins1_boot_diagnostic_bundle.dag @@ -47,6 +47,7 @@ import gunbc.machine_intake_pre_os_bringup_verdict { import gunbc.machine_intake_ampere_dram_console_observation { AmpereDramConsoleObservation, ampere_dram_console_observation, ampere_dram_findings, ampere_dram_console_text, } +import gunbc.machine_intake_kernel_module_decompression_observation { KernelModuleDecompressionObservation, kernel_module_decompression_observation, kernel_module_decompression_text, kernel_module_decompression_findings } import gunbc.machine_intake_ampere_socket_console_observation { AmpereSocketConsoleObservation, AmpereSummaryObserved, AmpereSummaryNotPrinted, AmpereSocketReferenceComparison, ampere_socket_console_observation, ampere_socket_console_text, ampere_socket_reference_comparison, ampere_socket_reference_comparison_text, @@ -258,7 +259,7 @@ type InventoryReading { // those bytes would under-count multi-byte console output. The size is taken before the content is // read, so a capture over the bound is reported without being loaded. type ConsoleReading - = ConsoleRetained { path: NonEmptyStr, size: ByteSize, lines: Int, firmware: FirmwareConsoleReport, dram: AmpereDramConsoleObservation, sockets: AmpereSocketConsoleObservation, against_reference: AmpereSocketReferenceComparison } + = ConsoleRetained { path: NonEmptyStr, size: ByteSize, lines: Int, firmware: FirmwareConsoleReport, dram: AmpereDramConsoleObservation, sockets: AmpereSocketConsoleObservation, against_reference: AmpereSocketReferenceComparison, modules: KernelModuleDecompressionObservation } | ConsoleExceedsBound { path: NonEmptyStr, size: ByteSize, bound: ByteSize } | ConsoleNotRetained { path: NonEmptyStr } | ConsoleUnread { path: NonEmptyStr, detail: String } @@ -576,7 +577,7 @@ fn socket_summary_findings(o: AmpereSocketConsoleObservation) -> List { fn console_findings(c: ConsoleReading) -> List { match c { - ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: d, sockets: o, against_reference: _ } => concat(ampere_dram_findings(observation: d, expected_sockets: mtcollins1_census_sockets()), socket_summary_findings(o: o)) + ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: d, sockets: o, against_reference: _, modules: m } => concat(ampere_dram_findings(observation: d, expected_sockets: mtcollins1_census_sockets()), concat(socket_summary_findings(o: o), kernel_module_decompression_findings(o: m))) _ => [] } } @@ -898,8 +899,8 @@ fn firmware_report_text(report: FirmwareConsoleReport) -> String { fn console_text(c: ConsoleReading) -> String { match c { - ConsoleRetained { path: p, size: b, lines: n, firmware: f, dram: d, sockets: o, against_reference: cmp } => - concat("capture ", p as String, " retained in this artifact: ", to_string(byte_size_count(b: b)), " bytes (stat), ", to_string(n), " line(s), within the ", to_string(byte_size_count(b: mtcollins1_boot_console_bound)), "-byte bound; ", firmware_report_text(report: f), "\n", ampere_dram_console_text(observation: d, expected_sockets: mtcollins1_census_sockets()), "\n", ampere_socket_console_text(o: o), ampere_socket_reference_comparison_text(c: cmp)) + ConsoleRetained { path: p, size: b, lines: n, firmware: f, dram: d, sockets: o, against_reference: cmp, modules: m } => + concat("capture ", p as String, " retained in this artifact: ", to_string(byte_size_count(b: b)), " bytes (stat), ", to_string(n), " line(s), within the ", to_string(byte_size_count(b: mtcollins1_boot_console_bound)), "-byte bound; ", firmware_report_text(report: f), "\n", ampere_dram_console_text(observation: d, expected_sockets: mtcollins1_census_sockets()), "\n", ampere_socket_console_text(o: o), ampere_socket_reference_comparison_text(c: cmp), "\n", kernel_module_decompression_text(o: m)) ConsoleExceedsBound { path: p, size: b, bound: bound } => concat("capture ", p as String, " is ", to_string(byte_size_count(b: b)), " bytes (stat), OVER the ", to_string(byte_size_count(b: bound)), "-byte bound; the collector does not truncate, so the artifact carries it whole") ConsoleNotRetained { path: p } => @@ -1553,6 +1554,7 @@ fn mtcollins1_boot_console_reading(path: String, size: ByteSize, capture: String path: path as NonEmptyStr, size: size, lines: count(lines), firmware: firmware_console_report(lines: lines), dram: ampere_dram_console_observation(lines: lines), sockets: sockets, against_reference: ampere_socket_reference_comparison(reference_path: mtcollins1_census_sol_artifact_path, reference_text: reference.text, read_ok: reference.ok, read_error: reference.error, subject: sockets), + modules: kernel_module_decompression_observation(lines: lines), } } } diff --git a/dag/gunbc/machine_intake/mtcollins1_census_image.dag b/dag/gunbc/machine_intake/mtcollins1_census_image.dag index 7c882e3b9d5..2ef694234e9 100644 --- a/dag/gunbc/machine_intake/mtcollins1_census_image.dag +++ b/dag/gunbc/machine_intake/mtcollins1_census_image.dag @@ -6,6 +6,9 @@ import extdeps.filesystem.filesystem_io { Filesystem } import gunbc.actions_run_binding { actions_variable_read, ActionsVariablePresent, ActionsVariableAbsent, fleet_converge_expected_host_env_name } import gunbc.fleet_intent_network { operator_host_srv2 } import std.measure { Bandwidth, bandwidth, bandwidth_count } +import extdeps.kmod.libkmod { libkmod_log_priority_env_name, libkmod_log_priority_info } +import extdeps.systemd.udevd { udevd_log_level_cmdline_key, systemd_log_level_debug, systemd_log_target_env_name, systemd_log_target_kmsg } +import extdeps.linux.kernel { linux_printk_devkmsg_on_cmdline_arg } import extdeps.provisioning.ubuntu_install_media { noble_numbat_2404_3_live_server_arm64 } import extdeps.provisioning.ubuntu_seeded_install_media { NoCloudSeedFile, @@ -70,6 +73,29 @@ data mtcollins1_census_console_arg: NonEmptyStr = concat( bandwidth_count(b: mtcollins1_census_console_baud) as String, ) as NonEmptyStr +// THE MODULE-IDENTITY INSTRUMENT (gunbc.machine_intake_kernel_module_decompression_observation). +// The kernel's "ZSTD-decompression failed" line names no module; the loader's record does, and in +// this initrd (systemd-udevd 255.4, libkmod 31) it reaches the serial capture only when all four hold: +// KMOD_LOG=info libkmod's path-bearing "Failed to insert module" line is INFO, and its +// default priority is ERR (extdeps.kmod.libkmod). The kernel hands an +// unrecognised KEY=value to /init's environment and initramfs-tools' +// init does not scrub it, so udevd inherits it. +// udev.log_level=debug overrides the SYSTEMD_LOG_LEVEL=info that init-top/udev sets. +// SYSTEMD_LOG_TARGET=kmsg without it udevd writes to /dev/console, which is the LAST console= -- +// tty0 here, the KVM, not the SOL line (extdeps.systemd.udevd). +// printk.devkmsg=on the default rate limit drops a burst of userspace kmsg records. +// The INFO record reaches SOL at the default console loglevel; udev's per-device DEBUG records stay +// in the kmsg ring, so the requesting device is unread on SOL by design (eager-owl-205, option A: +// ignore_loglevel would flood the 115200 line and perturb the timing of a possible race). +// It is a build input like any other: changing it changes the build key and the digest, so the +// image is republished, read back and repinned; nothing edits GRUB by hand. +data mtcollins1_census_module_identity_args: NonEmptyStr = join([ + concat(libkmod_log_priority_env_name as String, "=", libkmod_log_priority_info as String), + concat(udevd_log_level_cmdline_key as String, "=", systemd_log_level_debug as String), + concat(systemd_log_target_env_name as String, "=", systemd_log_target_kmsg as String), + linux_printk_devkmsg_on_cmdline_arg as String, +], " ") as NonEmptyStr + // The seed directory the earlier census booted with (its /proc/cmdline in attempt=3 reads // ds=nocloud;s=/cdrom/gunbc-census/), kept so the two captures are comparable. data mtcollins1_census_nocloud_dir: NonEmptyStr = "gunbc-census" @@ -131,6 +157,8 @@ fn mtcollins1_seeded_image_input( ubuntu_seeded_install_media_grub_kernel_cmdline(nocloud_dir: mtcollins1_census_nocloud_dir) as String, " ", mtcollins1_census_console_arg as String, + " ", + mtcollins1_census_module_identity_args as String, ) as NonEmptyStr, volume_id: volume_id, pinned_dates: seeded_install_media_pinned_dates, diff --git a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag new file mode 100644 index 00000000000..b3e5456340a --- /dev/null +++ b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag @@ -0,0 +1,318 @@ +module gunbc.machine_intake_mtcollins1_census_member_readback + +import std.types { Bool, List, NonEmptyStr, String } +import std.process { ExitSuccess, ProcessExit, exit_failure } +import std.algebra { trim } +import std.content_hash { sha256_hex_digest } +import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.tools.xorriso { xorriso_extract_command } +import extdeps.tools.gnu_coreutils { rm_force_command } +import extdeps.crypto.hash { sha256sum_file_command } +import extdeps.exec.command { ArgvCommand, argv_words } +import gunbc.fleet_intent_network { operator_host_srv2 } +import gunbc.fleet_reach_endpoint { fleet_probe_endpoint_for } +import gunbc.fleet_bootstrap_principal { executor_bootstrap_principal } +import gunbc.fleet_known_hosts_anchor { SshTarget, FleetSshExecutionContext } +import gunbc.fleet_ssh_locus { prepare_fleet_ssh_agent_context, FleetSshContextReady, FleetSshContextRefused } +import gunbc.typed_argv_exec { typed_argv_exec_over_fleet_ssh } +import extdeps.provisioning.ubuntu_install_media { noble_numbat_2404_3_live_server_arm64 } +import gunbc.machine_intake_mtcollins1_boot_authorization { MtCollins1BootMedium, MtCollins1CensusMedium, MtCollins1StockInstallerMedium, mtcollins1_boot_medium } +import gunbc.machine_intake_mtcollins1_census_image { mtcollins1_census_image_stock_path } +import gunbc.machine_intake_mtcollins1_census_medium_readback { mtcollins1_census_medium_served_path, RemoteAnswer, remote_answer_of } + +// THE PUBLISHED IMAGE'S BOOT CHAIN, READ BACK MEMBER BY MEMBER AGAINST THE STOCK MEDIUM IT WAS +// REMASTERED FROM. The census remaster (extdeps.provisioning.ubuntu_seeded_install_media_remaster) +// maps in the NoCloud seed and rewrites grub.cfg; every other member is supposed to pass through the +// replay unchanged. That is a claim about the builder, and a boot that failed in-kernel module +// decompression makes it worth measuring rather than assuming: this reads the kernel and the initrd +// out of BOTH served ISOs on the host that serves them and compares digests. +// +// SCOPE OF THE COMPARISON. If an initrd digest is equal, every member inside it -- every compressed +// module -- is equal by construction, so the module members are not unpacked here: a per-module +// comparison over identical bytes would re-derive a fact the outer digest already establishes. Only +// a differing initrd owes a descent into its layers, and that descent is the declared frontier +// named by BootChainFirstDifference below, not a guess made here. +// +// THE CONTROL IS IN THE SAME RUN. grub.cfg is the member the builder DOES rewrite (the census kernel +// cmdline lives there), so a reading in which grub.cfg compares equal did not read the census image +// at all -- a wrong path, a stale extraction, both sides extracted from one file -- and it refuses +// rather than reporting the boot chain identical. + +type CensusIsoMember + = CasperKernel + | CasperInitrd + | GrubConfig + +fn census_iso_member_path(member: CensusIsoMember) -> NonEmptyStr { + match member { + CasperKernel => "/casper/vmlinuz" + CasperInitrd => "/casper/initrd" + GrubConfig => "/boot/grub/grub.cfg" + } +} + +fn census_iso_member_slug(member: CensusIsoMember) -> NonEmptyStr { + match member { + CasperKernel => "vmlinuz" + CasperInitrd => "initrd" + GrubConfig => "grub.cfg" + } +} + +// Boot order: the loader reads grub.cfg, then the kernel, then the initrd. The first differing +// boundary is the first member in this order that differs outside the builder's declared rewrite. +data census_boot_chain_members: List = [CasperKernel, CasperInitrd] + +type IsoSide + = StockSide + | CensusSide + +fn iso_side_slug(side: IsoSide) -> NonEmptyStr { + match side { + StockSide => "stock" + CensusSide => "census" + } +} + +type MemberReading + = MemberRead { digest: NonEmptyStr } + | MemberUnread { reason: String } + +type IsoReading + = IsoMeasured { digest: NonEmptyStr } + | IsoUnmeasured { reason: String } + +type MemberComparison + = MemberIdentical { member: CensusIsoMember, digest: NonEmptyStr } + | MemberDiffers { member: CensusIsoMember, stock: NonEmptyStr, census: NonEmptyStr } + | MemberNotCompared { member: CensusIsoMember, side: IsoSide, reason: String } + +fn compare_member(member: CensusIsoMember, stock: MemberReading, census: MemberReading) -> MemberComparison { + match stock { + MemberUnread { reason: r } => MemberNotCompared { member: member, side: StockSide, reason: r } + MemberRead { digest: s } => + match census { + MemberUnread { reason: r } => MemberNotCompared { member: member, side: CensusSide, reason: r } + MemberRead { digest: c } => + if (s as String) == (c as String) { + MemberIdentical { member: member, digest: s } + } else { + MemberDiffers { member: member, stock: s, census: c } + } + } + } +} + +// THE OBSERVATION. Each ISO is first measured and admitted against the digest it is pinned to -- +// the stock against its SHA256SUMS row, the census against the boot medium row -- because a member +// read out of an unpinned file is a reading of some other image. +type CensusMemberReadback + = BootChainIdenticalToStock { stock_iso: NonEmptyStr, census_iso: NonEmptyStr, kernel: NonEmptyStr, initrd: NonEmptyStr } + | BootChainFirstDifference { stock_iso: NonEmptyStr, census_iso: NonEmptyStr, member: CensusIsoMember, stock: NonEmptyStr, census: NonEmptyStr } + | BootChainControlDidNotDiscriminate { grub_digest: NonEmptyStr } + | BootChainIsoNotPinned { side: IsoSide, expected: NonEmptyStr, observed: String } + | BootChainUnread { member: CensusIsoMember, side: IsoSide, reason: String } + | BootChainMediumNotCensus + | BootChainHostUnreached { reason: String } + +fn iso_pin_refusal(side: IsoSide, expected: NonEmptyStr, reading: IsoReading) -> CensusMemberReadback? { + match reading { + IsoUnmeasured { reason: r } => Present { value: BootChainIsoNotPinned { side: side, expected: expected, observed: concat("unmeasured: ", r) } } + IsoMeasured { digest: d } => + if (d as String) == (expected as String) { + none + } else { + Present { value: BootChainIsoNotPinned { side: side, expected: expected, observed: d as String } } + } + } +} + +fn boot_chain_verdict(stock_iso: NonEmptyStr, census_iso: NonEmptyStr, grub: MemberComparison, kernel: MemberComparison, initrd: MemberComparison) -> CensusMemberReadback { + match grub { + MemberNotCompared { member: m, side: s, reason: r } => BootChainUnread { member: m, side: s, reason: r } + MemberIdentical { member: _, digest: d } => BootChainControlDidNotDiscriminate { grub_digest: d } + MemberDiffers { member: _, stock: _, census: _ } => + match kernel { + MemberNotCompared { member: m, side: s, reason: r } => BootChainUnread { member: m, side: s, reason: r } + MemberDiffers { member: m, stock: s, census: c } => + BootChainFirstDifference { stock_iso: stock_iso, census_iso: census_iso, member: m, stock: s, census: c } + MemberIdentical { member: _, digest: k } => + match initrd { + MemberNotCompared { member: m, side: s, reason: r } => BootChainUnread { member: m, side: s, reason: r } + MemberDiffers { member: m, stock: s, census: c } => + BootChainFirstDifference { stock_iso: stock_iso, census_iso: census_iso, member: m, stock: s, census: c } + MemberIdentical { member: _, digest: i } => + BootChainIdenticalToStock { stock_iso: stock_iso, census_iso: census_iso, kernel: k, initrd: i } + } + } + } +} + +// The pure join over the six readings. The wet producer below supplies them from srv2. +fn census_member_readback_from_readings( + medium: MtCollins1BootMedium, + stock_iso: IsoReading, + census_iso: IsoReading, + stock_grub: MemberReading, census_grub: MemberReading, + stock_kernel: MemberReading, census_kernel: MemberReading, + stock_initrd: MemberReading, census_initrd: MemberReading, +) -> CensusMemberReadback { + match medium { + MtCollins1StockInstallerMedium => BootChainMediumNotCensus + MtCollins1CensusMedium { output_digest: census_pin } => + match iso_pin_refusal(side: StockSide, expected: noble_numbat_2404_3_live_server_arm64.content_sha256, reading: stock_iso) { + Present { value: refused } => refused + Absent => + match iso_pin_refusal(side: CensusSide, expected: census_pin, reading: census_iso) { + Present { value: refused } => refused + Absent => + boot_chain_verdict( + stock_iso: noble_numbat_2404_3_live_server_arm64.content_sha256, + census_iso: census_pin, + grub: compare_member(member: GrubConfig, stock: stock_grub, census: census_grub), + kernel: compare_member(member: CasperKernel, stock: stock_kernel, census: census_kernel), + initrd: compare_member(member: CasperInitrd, stock: stock_initrd, census: census_initrd), + ) + } + } + } +} + +fn census_member_readback_holds(readback: CensusMemberReadback) -> Bool { + match readback { + BootChainIdenticalToStock { stock_iso: _, census_iso: _, kernel: _, initrd: _ } => true + _ => false + } +} + +fn render_census_member_readback(readback: CensusMemberReadback) -> String { + match readback { + BootChainIdenticalToStock { stock_iso: s, census_iso: c, kernel: k, initrd: i } => + concat("verdict=boot_chain_identical_to_stock\nstock_iso=", s as String, "\ncensus_iso=", c as String, + "\nvmlinuz=", k as String, "\ninitrd=", i as String, "\ngrub.cfg=differs (declared rewrite, control discriminated)\n") + BootChainFirstDifference { stock_iso: s, census_iso: c, member: m, stock: sd, census: cd } => + concat("verdict=first_difference\nstock_iso=", s as String, "\ncensus_iso=", c as String, + "\nmember=", census_iso_member_path(member: m) as String, "\nstock_member=", sd as String, "\ncensus_member=", cd as String, "\n") + BootChainControlDidNotDiscriminate { grub_digest: g } => + concat("verdict=control_did_not_discriminate\ngrub.cfg=", g as String, " on both sides; the census image was not read\n") + BootChainIsoNotPinned { side: s, expected: e, observed: o } => + concat("verdict=iso_not_pinned\nside=", iso_side_slug(side: s) as String, "\nexpected=", e as String, "\nobserved=", o, "\n") + BootChainUnread { member: m, side: s, reason: r } => + concat("verdict=unread\nmember=", census_iso_member_path(member: m) as String, "\nside=", iso_side_slug(side: s) as String, "\nreason=", r, "\n") + BootChainMediumNotCensus => "verdict=medium_not_census\n" + BootChainHostUnreached { reason: r } => concat("verdict=host_unreached\nreason=", r, "\n") + } +} + +// ---- the one producer: read-only legs on srv2 over fleet SSH, like the medium readback ---- +// +// srv2 has had no runner slots since 2026-09-19 (gunbc.machine_intake_mtcollins1_boot_authorization), +// so this does not run ON srv2: it reaches the host that serves /srv/bmc the way the boot's own +// medium readback does, as the fleet principal over typed argv. Per member and side, three legs: +// xorriso extracts the member to a path under srv2's /tmp, sha256sum measures it, rm -f removes it. +// Nothing under /srv/bmc is written. + +data census_member_readback_attempt_raw: String = "mtcollins1-census-member-readback" + +data census_member_readback_receipt_path: String = "target/mtcollins1-census-member-readback.txt" + +fn member_scratch_path(side: IsoSide, member: CensusIsoMember) -> String { + concat("/tmp/gunbc-mtcollins1-census-member-", iso_side_slug(side: side) as String, "-", census_iso_member_slug(member: member) as String) +} + +fn remote_first_field(stdout: String) -> String { + match first(filter(split(s: trim(stdout), delimiter: " "), t => t != "")) { + Absent => "" + Present { value: t } => t + } +} + +// The join over one member's legs. A scratch file that could not be cleared first or removed after +// is not a reading: the first means a stale file may be what was measured, the second leaves bytes +// on srv2, and both refuse rather than pass unremarked. +fn member_reading_of(cleared: RemoteAnswer, extract: RemoteAnswer, measure: RemoteAnswer, removed: RemoteAnswer) -> MemberReading { + if cleared.exit_code != 0 { + MemberUnread { reason: concat("rm -f of the scratch path before extraction exit=", to_string(cleared.exit_code), " ", trim(s: cleared.stderr)) } + } else if removed.exit_code != 0 { + MemberUnread { reason: concat("rm -f of the scratch path after measuring exit=", to_string(removed.exit_code), " ", trim(s: removed.stderr)) } + } else if extract.exit_code != 0 { + MemberUnread { reason: concat("xorriso extract exit=", to_string(extract.exit_code), " ", trim(s: extract.stderr)) } + } else if measure.exit_code != 0 { + MemberUnread { reason: concat("sha256sum exit=", to_string(measure.exit_code), " ", trim(s: measure.stderr)) } + } else { + match sha256_hex_digest(hex: remote_first_field(stdout: measure.stdout)) { + Absent => MemberUnread { reason: concat("sha256sum printed no digest: ", trim(s: measure.stdout)) } + Present { value: d } => MemberRead { digest: d.hex as String as NonEmptyStr } + } + } +} + +fn iso_reading_of(measure: RemoteAnswer) -> IsoReading { + if measure.exit_code != 0 { + IsoUnmeasured { reason: concat("sha256sum exit=", to_string(measure.exit_code), " ", trim(s: measure.stderr)) } + } else { + match sha256_hex_digest(hex: remote_first_field(stdout: measure.stdout)) { + Absent => IsoUnmeasured { reason: concat("sha256sum printed no digest: ", trim(s: measure.stdout)) } + Present { value: d } => IsoMeasured { digest: d.hex as String as NonEmptyStr } + } + } +} + +fn census_member_readback_target() -> SshTarget { + SshTarget { + endpoint: fleet_probe_endpoint_for(host: operator_host_srv2 as String) as NonEmptyStr, + principal: executor_bootstrap_principal.login, + } +} + +fn remote(target: SshTarget, context: FleetSshExecutionContext, command: ArgvCommand) -> RemoteAnswer { + remote_answer_of(outcome: typed_argv_exec_over_fleet_ssh(target: target, context: context, argv: argv_words(command: command))) +} + +fn read_member_remote(target: SshTarget, context: FleetSshExecutionContext, iso_path: NonEmptyStr, side: IsoSide, member: CensusIsoMember) -> MemberReading { + let scratch = member_scratch_path(side: side, member: member) + let cleared = remote(target: target, context: context, command: rm_force_command(path: scratch)) + let extract = remote(target: target, context: context, command: xorriso_extract_command(iso_path: iso_path as String, iso_internal_path: census_iso_member_path(member: member) as String, destination: scratch)) + let measure = remote(target: target, context: context, command: sha256sum_file_command(path: scratch)) + let removed = remote(target: target, context: context, command: rm_force_command(path: scratch)) + member_reading_of(cleared: cleared, extract: extract, measure: measure, removed: removed) +} + +fn census_member_readback(medium: MtCollins1BootMedium) -> CensusMemberReadback { + match prepare_fleet_ssh_agent_context(attempt_raw: census_member_readback_attempt_raw) { + FleetSshContextRefused { cause: c } => + BootChainHostUnreached { reason: c } + FleetSshContextReady { context: context, receipt: _ } => { + let target = census_member_readback_target() + let stock_path = mtcollins1_census_image_stock_path() + let census_path = mtcollins1_census_medium_served_path(medium: medium) + census_member_readback_from_readings( + medium: medium, + stock_iso: iso_reading_of(measure: remote(target: target, context: context, command: sha256sum_file_command(path: stock_path as String))), + census_iso: iso_reading_of(measure: remote(target: target, context: context, command: sha256sum_file_command(path: census_path as String))), + stock_grub: read_member_remote(target: target, context: context, iso_path: stock_path, side: StockSide, member: GrubConfig), + census_grub: read_member_remote(target: target, context: context, iso_path: census_path, side: CensusSide, member: GrubConfig), + stock_kernel: read_member_remote(target: target, context: context, iso_path: stock_path, side: StockSide, member: CasperKernel), + census_kernel: read_member_remote(target: target, context: context, iso_path: census_path, side: CensusSide, member: CasperKernel), + stock_initrd: read_member_remote(target: target, context: context, iso_path: stock_path, side: StockSide, member: CasperInitrd), + census_initrd: read_member_remote(target: target, context: context, iso_path: census_path, side: CensusSide, member: CasperInitrd), + ) + } + } +} + +// CONSUMPTION (DESIGN §3c): invoked by the fleet-converge mode mtcollins1_census_member_readback, +// which consumes the fleet SSH key; the receipt is uploaded as that run's artifact whatever the +// verdict. The run exits nonzero on every verdict but BootChainIdenticalToStock. +fn mtcollins1_census_member_readback_wet() -> ProcessExit { + let readback = census_member_readback(medium: mtcollins1_boot_medium) + let rendered = render_census_member_readback(readback: readback) + let written = Filesystem.Write(path: census_member_readback_receipt_path, content: rendered) + if written.success == false { + exit_failure(reason: concat("mtcollins1 census member readback: the receipt could not be written: ", written.error)) + } else if census_member_readback_holds(readback: readback) { + ExitSuccess + } else { + exit_failure(reason: concat("mtcollins1 census member readback did not hold:\n", rendered)) + } +} diff --git a/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag b/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag new file mode 100644 index 00000000000..711735e580f --- /dev/null +++ b/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag @@ -0,0 +1,125 @@ +module test.claim.machine_intake.kernel_module_decompression_observation_witness_test + +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import std.types { Bool, List, String } +import std.measure { byte_size } +import extdeps.linux.module_decompress { ZstdCorruptionDetected } +import gunbc.machine_intake_kernel_module_decompression_observation { + kernel_module_decompression_observation, kernel_module_decompression_text, kernel_module_decompression_findings, + KernelModuleDecompressionObservation, ModuleDecompressionNotRefused, ModuleDecompressionRefused, + ModuleIdentityKnown, ModuleIdentityUnread, FailedModuleIdentity, DriverPrintedLater, DriverNotSeenInCapture, +} +import gunbc.machine_intake_mtcollins1_boot_diagnostic_bundle { mtcollins1_boot_console_reading, ReferenceCaptureRead, ConsoleRetained } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// EXCERPT OF THE FAILED ATTEMPT'S RETAINED SOL CAPTURE, byte-for-byte lines 1893-1897 of +// mtcollins1-boot-sol.capture (sha256 761e79ca42b7ac955a3a4186a1cc5c2bcc1fde0422340d5be02cde2690fcee87) +// in artifact mtcollins1-boot-receipts of fleet-converge run 36335369059 +// (artifact digest sha256:16dcc1b34233cea3b338104a6cff5d88a46f8b6c2e815443653da5271062dd9b). +// That boot ran without the module-identity instrument, so no loader record exists. +data failed_attempt_excerpt: List = [ + "[ 10.149389] Run /init as init process\r", + "[ 10.374590] ZSTD-decompression failed with status 20\r", + "[ 10.380475] ZSTD-decompression failed with status 20\r", + "[ 10.392987] igb: Intel(R) Gigabit Ethernet Network Driver\r", + "[ 10.398716] igb: Copyright (c) 2007-2014 Intel Corporation.\r", +] + +// EXCERPT OF THE CENSUS-REACHING ATTEMPT'S RETAINED SOL CAPTURE, lines 1895-1900 of +// mtcollins1-boot-sol.capture (sha256 83e818fb2be55383966f5e98c69c7853625d429941192062530da5f5d21f8161) +// in artifact mtcollins1-boot-receipts of fleet-converge run 36253081549 +// (artifact digest sha256:ca38324ebcad6477911b0f56a86f91a0785a77dc31163ede6b9afc616fde3496). +// It carries the renesas "fallback to ROM" warning, which is not a refusal. +data census_reaching_excerpt: List = [ + "[ 10.154211] Run /init as init process\r", + "[ 10.391272] xhci_hcd 0004:03:00.0: Adding to iommu group 17\r", + "[ 10.391751] igb: Intel(R) Gigabit Ethernet Network Driver\r", + "[ 10.397592] xhci_hcd 0004:03:00.0: failed to load firmware renesas_usb_fw.mem, fallback to ROM\r", + "[ 10.402897] igb: Copyright (c) 2007-2014 Intel Corporation.\r", + "[ 10.403241] xhci_hcd 0004:03:00.0: xHCI Host Controller\r", +] + +// THE OFFLINE CONTROL'S INSTRUMENTED CAPTURE, lines 323-325 of its ttyAMA0 log (sha256 +// b8b333934516a1172baabb11cc2b8b649e83d367578aa63773582e556daf6bcf): qemu-system-aarch64 10.0.13 -M virt +// with a qemu-xhci device, booting the stock 24.04.3 /casper/vmlinuz and /casper/initrd with ONE edit -- +// 64 bytes inverted inside the zstd body of xhci-pci.ko.zst, frame header intact (initrd sha256 +// 83fd2a47a6449584e5062d5238ad84915da7d095c5ebdee406469046a8e40f96) -- under the census cmdline plus +// mtcollins1_census_module_identity_args, "--- console=tty0" kept. The same boot WITHOUT the args +// (log sha256 7e8db2f8d916ea009b2085c65e5a2348d77c06d27ad4eeac60f03988874b5552) printed the status-20 +// line and no loader record: the instrument is what names the file. The uas and xhci_pci lines are +// DESIGNED, added to exercise the errno filter and the later-driver reading. +data instrumented_excerpt: List = [ + "[ 27.190394] ZSTD-decompression failed with status 20\r", + "[ 27.209606] (udev-worker)[100]: Failed to insert module '/lib/modules/6.8.0-71-generic/kernel/drivers/usb/host/xhci-pci.ko.zst': Invalid argument\r", + "[ 27.300000] (udev-worker)[101]: Failed to insert module '/lib/modules/6.8.0-71-generic/kernel/drivers/usb/storage/uas.ko.zst': No such device\r", + "[ 28.916926] input: gpio-keys as /devices/platform/gpio-keys/input/input0\r", + "[ 29.000000] xhci_pci: printed later\r", +] + +// POSITIVE CONTROL: the census-reaching boot is not a refusal, renesas warning included. +test fn the_census_reaching_capture_reads_as_not_refused() -> Bool { + match kernel_module_decompression_observation(lines: census_reaching_excerpt) { + ModuleDecompressionNotRefused => true + _ => false + } + && count(kernel_module_decompression_findings(o: kernel_module_decompression_observation(lines: census_reaching_excerpt))) == 0 +} + +// RED, ANONYMOUS: the failed boot is a refusal with both statuses retained, and identity UNREAD. +test fn the_failed_capture_is_a_refusal_whose_identity_is_unread() -> Bool { + match kernel_module_decompression_observation(lines: failed_attempt_excerpt) { + ModuleDecompressionRefused { refusals: rs, identity: ModuleIdentityUnread } => + count(rs) == 2 + && match first(rs) { + Absent => false + Present { value: r } => r.at == "10.374590" && match r.status { ZstdCorruptionDetected => true _ => false } + } + _ => false + } +} + +test fn the_failed_capture_text_names_no_module_and_leaves_the_request_unread_on_sol() -> Bool { + let text = kernel_module_decompression_text(o: kernel_module_decompression_observation(lines: failed_attempt_excerpt)) + string_contains(s: text, pattern: "module identity UNREAD") + && string_contains(s: text, pattern: "status corruption_detected") + && string_contains(s: text, pattern: "requesting device/modalias: unread on SOL") +} + +// RED, NAMED: a loader record with the refusal's errno names the file, the worker and the later +// printing; an insert failure with another errno is NOT attributed to the refusal. +test fn an_instrumented_capture_names_the_refused_module_path_and_worker() -> Bool { + match kernel_module_decompression_observation(lines: instrumented_excerpt) { + ModuleDecompressionRefused { refusals: rs, identity: ModuleIdentityKnown { failures: fs } } => + count(rs) == 1 && count(fs) == 1 + && match first(fs) { Absent => false Present { value: f } => named_xhci_pci(f: f) } + _ => false + } +} + +fn named_xhci_pci(f: FailedModuleIdentity) -> Bool { + (f.failure.path as String) == "/lib/modules/6.8.0-71-generic/kernel/drivers/usb/host/xhci-pci.ko.zst" + && f.failure.loader == "(udev-worker)[100]" + && f.failure.at == "27.209606" + && f.module_name == "xhci_pci" + && match f.later { DriverPrintedLater { at: a, line: _ } => a == "29.000000" DriverNotSeenInCapture => false } +} + +// A loader record with no refusal line is not a decompression refusal. +test fn a_loader_failure_without_a_kernel_refusal_is_not_a_refusal() -> Bool { + match kernel_module_decompression_observation(lines: instrumented_excerpt.skip(n: 1).take(n: 1)) { + ModuleDecompressionNotRefused => true + _ => false + } +} + +// INHABITANCE: the attempt-bound console reading carries this observation from the capture text it +// retains -- the real route the boot bundle renders, not a supplied observation. +test fn the_retained_console_reading_carries_the_refusal() -> Bool { + let capture = join(failed_attempt_excerpt, "\n") + match mtcollins1_boot_console_reading(path: "artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 10), capture: concat(capture, "\n"), read_ok: true, read_error: "", reference: ReferenceCaptureRead { text: "", ok: false, error: "not supplied" }) { + ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: _, sockets: _, against_reference: _, modules: m } => + match m { ModuleDecompressionRefused { refusals: _, identity: ModuleIdentityUnread } => true _ => false } + _ => false + } +} diff --git a/dag/test/claim/machine_intake/mtcollins1_boot_diagnostic_bundle_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_boot_diagnostic_bundle_witness_test.dag index 329fd0a3555..576b72acd32 100644 --- a/dag/test/claim/machine_intake/mtcollins1_boot_diagnostic_bundle_witness_test.dag +++ b/dag/test/claim/machine_intake/mtcollins1_boot_diagnostic_bundle_witness_test.dag @@ -454,7 +454,7 @@ test fn a_capture_outside_the_uploaded_glob_is_reported_not_retained() -> Bool { test fn a_retained_capture_reports_its_firmware_statement() -> Bool { let capture = "ERROR: Non-identical DIMM mixture NOT supported!\n" match mtcollins1_boot_console_reading(path: "/w/artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 51), capture: capture, read_ok: true, read_error: "", reference: no_reference) { - ConsoleRetained { path: _, size: b, lines: _, firmware: f, dram: _, sockets: _, against_reference: _ } => (byte_size_count(b: b) as Int) == 51 && match f { TrainingRefusedByFirmware { statement: _, locus: _ } => true _ => false } + ConsoleRetained { path: _, size: b, lines: _, firmware: f, dram: _, sockets: _, against_reference: _, modules: _ } => (byte_size_count(b: b) as Int) == 51 && match f { TrainingRefusedByFirmware { statement: _, locus: _ } => true _ => false } _ => false } } @@ -550,7 +550,7 @@ test fn a_supplied_reference_renders_its_differences() -> Bool { let reference = "UEFI RC version: 1.08\r\n Number of active sockets : 2\r\n Inter Socket Connection 0 : Width: x16 / Speed 25 GT/s\r\n Socket[0]: Core voltage : 1065\r\n Socket[1]: Core voltage : 1035\r\n" let capture = "UEFI RC version: 1.08\r\n Number of active sockets : 1\r\n Socket[0]: Core voltage : 1065\r\n" match mtcollins1_boot_console_reading(path: "artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 100), capture: capture, read_ok: true, read_error: "", reference: reference_of(text: reference)) { - ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: _, sockets: _, against_reference: _ } => { + ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: _, sockets: _, against_reference: _, modules: _ } => { let b = bundle_socket1_absent() let text = mtcollins1_boot_bundle_text(bundle: MtCollins1BootDiagnosticBundle { attempt: b.attempt, outcome: b.outcome, sel_before: b.sel_before, sel_after: b.sel_after, sdr_cache: b.sdr_cache, sensors: b.sensors, smpro: b.smpro, inventory: b.inventory, console: mtcollins1_boot_console_reading(path: "artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 100), capture: capture, read_ok: true, read_error: "", reference: reference_of(text: reference)), run: no_run() }) diff --git a/dag/test/claim/machine_intake/mtcollins1_census_medium_readback_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_census_medium_readback_witness_test.dag index 8a438addd02..c721141e3e4 100644 --- a/dag/test/claim/machine_intake/mtcollins1_census_medium_readback_witness_test.dag +++ b/dag/test/claim/machine_intake/mtcollins1_census_medium_readback_witness_test.dag @@ -57,14 +57,14 @@ test fn the_served_path_and_the_record_path_derive_from_the_census_input() -> Bo Mtcollins1CensusImageDerived { input: input } => (mtcollins1_census_medium_served_path(medium: census) as String) == "/srv/bmc/gunbc-mtcollins1-census-ubuntu-24.04.3-arm64-30ca88d43f891399.iso" && (mtcollins1_census_record_path(input: input) as String) == concat("/srv/bmc/.gunbc-mtcollins1-census-ubuntu-24.04.3-arm64.", mtcollins1_census_build_key(input: input) as String, ".built") - && (mtcollins1_census_build_key(input: input) as String) == "8d357a525f078cb4" + && (mtcollins1_census_build_key(input: input) as String) == "5828991ca3dcace9" } } test fn a_record_and_a_file_that_both_repeat_the_selected_digest_agree_and_retain_every_value() -> Bool { match mtcollins1_census_medium_readback_from_answers(medium: census, host: srv2, record: record_ok(), measure: measure_ok()) { CensusMediumReadbackAgreed { host: h, build_key: k, recorded_digest: r, image_name: n, served_path: p, measured_digest: m } => - (h as String) == "srv2" && (k as String) == "8d357a525f078cb4" && (r as String) == (recorded as String) && (m as String) == (recorded as String) + (h as String) == "srv2" && (k as String) == "5828991ca3dcace9" && (r as String) == (recorded as String) && (m as String) == (recorded as String) && (n as String) == "gunbc-mtcollins1-census-ubuntu-24.04.3-arm64-30ca88d43f891399.iso" && (p as String) == "/srv/bmc/gunbc-mtcollins1-census-ubuntu-24.04.3-arm64-30ca88d43f891399.iso" _ => false diff --git a/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag new file mode 100644 index 00000000000..9e3eeaa6304 --- /dev/null +++ b/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag @@ -0,0 +1,104 @@ +module test.claim.machine_intake.mtcollins1_census_member_readback_witness_test + +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import std.types { Bool, NonEmptyStr, String } +import gunbc.machine_intake_mtcollins1_boot_authorization { MtCollins1CensusMedium, MtCollins1StockInstallerMedium } +import gunbc.machine_intake_mtcollins1_census_member_readback { + census_member_readback_from_readings, census_member_readback_holds, render_census_member_readback, + MemberReading, MemberRead, MemberUnread, IsoReading, IsoMeasured, IsoUnmeasured, + CensusMemberReadback, BootChainIdenticalToStock, BootChainFirstDifference, BootChainControlDidNotDiscriminate, + BootChainIsoNotPinned, BootChainUnread, BootChainMediumNotCensus, + CasperKernel, CasperInitrd, GrubConfig, StockSide, CensusSide, +} + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +data stock_iso: NonEmptyStr = "2ee2163c9b901ff5926400e80759088ff3b879982a3956c02100495b489fd555" +data census_iso: NonEmptyStr = "30ca88d43f8913995078f6c2c917d5049842f927f3fc886a103f460dd146a706" +data census: MtCollins1CensusMedium = MtCollins1CensusMedium { output_digest: census_iso } + +fn read(d: NonEmptyStr) -> MemberReading { MemberRead { digest: d } } + +// Supplied member digests: the fold is over readings, the producer is the srv2 run. +fn over(stock_grub: MemberReading, census_grub: MemberReading, census_kernel: MemberReading, census_initrd: MemberReading) -> CensusMemberReadback { + census_member_readback_from_readings( + medium: census, + stock_iso: IsoMeasured { digest: stock_iso }, + census_iso: IsoMeasured { digest: census_iso }, + stock_grub: stock_grub, census_grub: census_grub, + stock_kernel: read(d: "k0"), census_kernel: census_kernel, + stock_initrd: read(d: "i0"), census_initrd: census_initrd, + ) +} + +// POSITIVE: the declared rewrite differs, kernel and initrd pass through -- the only verdict that holds. +test fn a_rewritten_grub_and_an_unchanged_kernel_and_initrd_read_as_identical_boot_chain() -> Bool { + let r = over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: read(d: "i0")) + census_member_readback_holds(readback: r) + && match r { + BootChainIdenticalToStock { stock_iso: _, census_iso: _, kernel: k, initrd: i } => (k as String) == "k0" && (i as String) == "i0" + _ => false + } +} + +// RED: a differing initrd is located as the first differing boundary, with both digests retained. +test fn a_differing_initrd_is_the_first_difference_and_does_not_hold() -> Bool { + let r = over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: read(d: "i1")) + census_member_readback_holds(readback: r) == false + && match r { + BootChainFirstDifference { stock_iso: _, census_iso: _, member: CasperInitrd, stock: s, census: c } => (s as String) == "i0" && (c as String) == "i1" + _ => false + } +} + +// RED: the kernel precedes the initrd in boot order, so a differing kernel is named first. +test fn a_differing_kernel_is_named_before_a_differing_initrd() -> Bool { + match over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k1"), census_initrd: read(d: "i1")) { + BootChainFirstDifference { stock_iso: _, census_iso: _, member: CasperKernel, stock: _, census: _ } => true + _ => false + } +} + +// RED: an equal grub.cfg means the census image was not read; identical kernel and initrd do not rescue it. +test fn an_equal_grub_config_refuses_as_a_control_that_did_not_discriminate() -> Bool { + match over(stock_grub: read(d: "g0"), census_grub: read(d: "g0"), census_kernel: read(d: "k0"), census_initrd: read(d: "i0")) { + BootChainControlDidNotDiscriminate { grub_digest: _ } => true + _ => false + } +} + +test fn an_unread_census_initrd_is_unread_not_identical() -> Bool { + match over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: MemberUnread { reason: "xorriso" }) { + BootChainUnread { member: CasperInitrd, side: CensusSide, reason: _ } => true + _ => false + } +} + +test fn a_census_iso_off_its_pin_refuses_before_any_member_is_compared() -> Bool { + let r = census_member_readback_from_readings( + medium: census, + stock_iso: IsoMeasured { digest: stock_iso }, + census_iso: IsoMeasured { digest: stock_iso }, + stock_grub: read(d: "g0"), census_grub: read(d: "g1"), + stock_kernel: read(d: "k0"), census_kernel: read(d: "k0"), + stock_initrd: read(d: "i0"), census_initrd: read(d: "i0"), + ) + match r { + BootChainIsoNotPinned { side: CensusSide, expected: _, observed: _ } => true + _ => false + } +} + +test fn the_stock_medium_row_has_no_census_boot_chain_to_read() -> Bool { + let r = census_member_readback_from_readings( + medium: MtCollins1StockInstallerMedium, + stock_iso: IsoMeasured { digest: stock_iso }, census_iso: IsoMeasured { digest: census_iso }, + stock_grub: read(d: "g0"), census_grub: read(d: "g1"), + stock_kernel: read(d: "k0"), census_kernel: read(d: "k0"), + stock_initrd: read(d: "i0"), census_initrd: read(d: "i0"), + ) + match r { + BootChainMediumNotCensus => true + _ => false + } +} From 1846778cb9b8f0ad6280aa457ecd2c682f45e81f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 28 Sep 2026 02:02:48 +0000 Subject: [PATCH 2/7] regen fleet-converge.yml (tools.generated_artifact_gate main_wet_one) Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/fleet-converge.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 014355dabd8..627f9557387 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -13,7 +13,7 @@ on: mode: description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; app_key_version_verify reads the gunbai-ci App private key at the EXACT Secret Manager version named by app_key_version, mints an installation token with it, and refuses unless GitHub accepts it and the key's rotation deadline has not passed -- no add, disable or destroy; microvm_host_converge installs the cited Firecracker release on the selected host and reads the kvm grant back, refusing by name when the grant has not landed (the grant itself is applied by the full-host apply spine); microvm_network_apply stages the slot and host network files the model renders at the named expected_revision as root:root 0600 in a root-only directory over the fleet SSH edge as the host's ADMINISTRATOR, installs them with the modeled operations, reloads networkd, systemd-sysctl and the nft loader unit, and reads the ruleset back -- the job user is granted none of it, because install plus systemctl over content that principal can write is arbitrary root for any pull request; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' pair vLLM units over the password session, workers before heads; spark_native_serving_apply relaunches group B's native four-rank arm as ONE BOUNDED TRANSACTION -- an all-host preflight that mutates nothing and refuses on any unresolved artifact from an earlier run, the incumbent UNIT preserved (plus a diagnostic inspect of the incumbent container -- the container itself cannot outlive its unit, because these run --rm in the foreground under systemd), the head applied BEFORE the workers so every rank joins the head's new rendezvous store, then a readback of the COMPLETE realization through each container's own inspect -- every field the create spec states, the effective --node-rank off that container's argv, and NCCL's typed transport verdict on that incarnation -- plus the head's front door answering with this arm's served model, and then commit or a rollback of the whole arm -- and it is a separate mode because it reloads a 328 GB arm and nobody should reach it by asking for the pair units; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; spark_v41_checkpoint_materialize fetches the admitted published DeepSeek V4.1 files onto the selected Group A Spark (about 510 GB; spark_v41_row_store_encode encodes the eight Engram row stores from the verified shards on the selected Group A Spark and reads each store's sha256; spark_v41_row_store_readback reads those stores back at their header, first and last record and every rank seam, with the published source rows at the same rows, and writes nothing; spark_v41_engram_differential compares upstream's Engram lookup kernel with the design-B file-backed lookup over sampled real rows of every row store, byte for byte, and writes nothing; it states the requirement and refuses before fetching when the disk cannot hold it), publishes each only after its sha256 matches the manifest, leaves a present file with the right digest alone and refuses one with the wrong digest, and reads the storage-backed Engram spans from the verified shards; a transfer runs detached and a rerun reattaches; spark_v41_runtime_image_build PRODUCES the DeepSeek V4.1 image on the selected Spark -- it verifies the candidate's three FlashInfer wheels against the digests the candidate keys, converges the patched source tree, builds from it, reads the produced configuration digest back from inside the image through the probe route, and admits that digest against the candidate's own recipe, refusing a digest that does not recompute from it -- and it is a separate mode from the probe because it occupies one host for hours where the probe occupies it for minutes; spark_v41_runtime_image_distribute moves that produced image, named by the configuration digest its production receipt read back, from the host that receipt names to the selected Group A Spark -- save, scp through the executor, load -- after stating its size against every filesystem a copy lands on, leaves a target already holding the digest untouched, refuses a target holding a different image under the tag, and refuses unless the target's image inspect Id reads back as that digest; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown; microvm_controller_install writes the microVM slot controller's root-owned release locus (gunbc + sources + Firecracker + jailer) and the gunbc-microvm-slot@ template unit on srv1 and starts no instance; microvm_slot_start starts the shakedown slot's controller unit once on srv1 (the instance is derived from the model, never an input), waits for it bounded by the unit's own stop timeout, and uploads that invocation's controller receipt; microvm_runner_group_ensure (srv1 only) reads the organization's runner groups and, only when the microvm-shakedown group is absent, files ONE operator approval, creates it restricted to the shakedown workflow on the default branch, and refuses unless the readback holds that restriction required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, microvm_network_apply, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_native_serving_apply, spark_runtime_image_probe, spark_v41_checkpoint_materialize, spark_v41_row_store_encode, spark_v41_row_store_readback, spark_v41_engram_differential, spark_v41_runtime_image_build, spark_v41_runtime_image_distribute, dashboard_deploy, approval_broker_dark_install, microvm_controller_install, microvm_slot_start, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, approval_device_enrolment_code_issue, mtcollins1_boot, mtcollins1_fan_observe, mtcollins1_census_image_publish, mtcollins1_census_member_readback, host_credential_custody_converge, app_key_version_verify, r2_bucket_ensure, r2_bucket_admin_mint, fabric_writer_identity_observe, pair_serving_d0, microvm_runner_group_ensure, gcp_iam_converge, namecheap_observe] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -252,7 +252,7 @@ jobs: echo "fleet-key: agent loaded (identity fleet-automation@gunbc; secret versions/1 pinned; fingerprint verified against modeled authority; key file wiped)" env: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} - if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_observe' || github.event.inputs.mode == 'microvm_network_apply' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_native_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' || github.event.inputs.mode == 'spark_v41_row_store_encode' || github.event.inputs.mode == 'spark_v41_row_store_readback' || github.event.inputs.mode == 'spark_v41_engram_differential' || github.event.inputs.mode == 'spark_v41_runtime_image_build' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'approval_broker_dark_install' || github.event.inputs.mode == 'microvm_controller_install' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'approval_device_enrolment_code_issue' || github.event.inputs.mode == 'mtcollins1_boot' || github.event.inputs.mode == 'host_credential_custody_converge' || github.event.inputs.mode == 'pair_serving_d0' + if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_observe' || github.event.inputs.mode == 'microvm_network_apply' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_native_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'spark_v41_checkpoint_materialize' || github.event.inputs.mode == 'spark_v41_row_store_encode' || github.event.inputs.mode == 'spark_v41_row_store_readback' || github.event.inputs.mode == 'spark_v41_engram_differential' || github.event.inputs.mode == 'spark_v41_runtime_image_build' || github.event.inputs.mode == 'spark_v41_runtime_image_distribute' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'approval_broker_dark_install' || github.event.inputs.mode == 'microvm_controller_install' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'approval_device_enrolment_code_issue' || github.event.inputs.mode == 'mtcollins1_boot' || github.event.inputs.mode == 'mtcollins1_census_member_readback' || github.event.inputs.mode == 'host_credential_custody_converge' || github.event.inputs.mode == 'pair_serving_d0' timeout-minutes: 5 - name: Fleet converge plan (membership_reconcile → artifact) id: plan @@ -642,6 +642,24 @@ jobs: retention-days: 30 if: github.event.inputs.mode == 'mtcollins1_census_image_publish' timeout-minutes: 10 + - name: "Mt. Collins census image: read the published kernel and initrd back against the stock medium" + id: mtcollins1_census_member_readback + run: |- + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag --function mtcollins1_census_member_readback_wet + cat "$ROOT/target/mtcollins1-census-member-readback.txt" + if: github.event.inputs.mode == 'mtcollins1_census_member_readback' + timeout-minutes: 30 + - name: Upload Mt. Collins census member readback receipt + id: mtcollins1_census_member_readback_receipt_upload + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: mtcollins1-census-member-readback + path: target/mtcollins1-census-member-readback.txt + if-no-files-found: error + retention-days: 30 + if: always() && github.event.inputs.mode == 'mtcollins1_census_member_readback' + timeout-minutes: 10 - name: "R2 mint preflight: bootstrap read + account permission-group listing (reachability, no mutation)" id: r2_mint_preflight run: |- From 9ece70aa040a780341a34a3b4131fdc6747b54d3 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 28 Sep 2026 02:37:54 +0000 Subject: [PATCH 3/7] import v2.std.algebra filter explicitly (floor UnimportedBareProvider) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../machine_intake/kernel_module_decompression_observation.dag | 1 + dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag | 1 + 2 files changed, 2 insertions(+) diff --git a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag index 5c757666947..7715c2c7784 100644 --- a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag +++ b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag @@ -2,6 +2,7 @@ module gunbc.machine_intake_kernel_module_decompression_observation import std.types { Bool, Int, List, NonEmptyStr, String } import std.algebra { trim } +import v2.std.algebra { filter } import v2.std.optional { Present } import extdeps.linux.module_decompress { linux_module_zstd_decompress_failed_prefix, ZstdErrorCode, zstd_error_code_of, zstd_error_code_name } import extdeps.kmod.libkmod { libkmod_insert_failed_prefix, libkmod_insert_failed_path_end } diff --git a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag index b3e5456340a..93fbd9445be 100644 --- a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag +++ b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag @@ -3,6 +3,7 @@ module gunbc.machine_intake_mtcollins1_census_member_readback import std.types { Bool, List, NonEmptyStr, String } import std.process { ExitSuccess, ProcessExit, exit_failure } import std.algebra { trim } +import v2.std.algebra { filter } import std.content_hash { sha256_hex_digest } import extdeps.filesystem.filesystem_io { Filesystem } import extdeps.tools.xorriso { xorriso_extract_command } From 157517f1db8458eaac61a2985a0b22ed95931d86 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 28 Sep 2026 03:04:01 +0000 Subject: [PATCH 4/7] annotate filtered lists so first() has a resolved scrutinee (#12398 checker) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../machine_intake/kernel_module_decompression_observation.dag | 2 +- dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag index 7715c2c7784..02a10e6600d 100644 --- a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag +++ b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag @@ -129,7 +129,7 @@ fn module_name_of_path(path: String) -> String { } fn later_driver(lines: List, after_index: Int, module_name: String) -> LaterDriverReading { - let later = filter(lines.skip(n: after_index + 1), l => starts_with(s: l.rest, prefix: concat(module_name, " ")) || starts_with(s: l.rest, prefix: concat(module_name, ":"))) + let later: List = filter(lines.skip(n: after_index + 1), l => starts_with(s: l.rest, prefix: concat(module_name, " ")) || starts_with(s: l.rest, prefix: concat(module_name, ":"))) match first(later) { Absent => DriverNotSeenInCapture Present { value: l } => DriverPrintedLater { at: l.at, line: l.rest } diff --git a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag index 93fbd9445be..addf99a8a7e 100644 --- a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag +++ b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag @@ -222,7 +222,8 @@ fn member_scratch_path(side: IsoSide, member: CensusIsoMember) -> String { } fn remote_first_field(stdout: String) -> String { - match first(filter(split(s: trim(stdout), delimiter: " "), t => t != "")) { + let tokens: List = filter(split(s: trim(stdout), delimiter: " "), t => t != "") + match first(tokens) { Absent => "" Present { value: t } => t } From f0350b0cab831639fa3d242b69426cc7d056d017 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 28 Sep 2026 03:47:45 +0000 Subject: [PATCH 5/7] xorriso: delete the uncalled ExtractPathBestEffort op; xorriso_extract_command is the one -extract spelling (review 72051) Co-Authored-By: Claude Opus 5.5 (1M context) --- dag/extdeps/tools/xorriso.dag | 27 +++++++------------ .../mtcollins1_census_member_readback.dag | 2 +- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/dag/extdeps/tools/xorriso.dag b/dag/extdeps/tools/xorriso.dag index b5fe90be257..cebbffd62e5 100644 --- a/dag/extdeps/tools/xorriso.dag +++ b/dag/extdeps/tools/xorriso.dag @@ -25,19 +25,6 @@ data xorriso_cli_tool: CliTool = CliTool { } service xorriso.Iso { - operation ExtractPathBestEffort { - input { iso_path: NonEmptyStr, iso_internal_path: NonEmptyStr, destination: NonEmptyStr } - output { success: Bool from "exit_success" } - readonly - transport shell { - argv: ["xorriso", "-osirrox", "on", "-indev", "{iso_path}", "-extract", "{iso_internal_path}", "{destination}"] - } - exit { - 0 => Unit "extracted" - nonzero => Unit "path not present on ISO" - } - } - operation ExtractTree { input { iso_path: NonEmptyStr, destination: NonEmptyStr } output { success: Bool from "exit_success" } @@ -86,9 +73,13 @@ service xorriso.Iso { } } -// THE SAME -extract ExtractPathBestEffort runs, as an argv for a remote typed-argv transport: one ISO -// member copied out to one destination path, read-only on the ISO. xorriso exits nonzero when the -// member is absent or the destination cannot be written. -fn xorriso_extract_command(iso_path: String, iso_internal_path: String, destination: String) -> ArgvCommand { - argv_command(program: xorriso_cli_tool.name, arguments: ["-osirrox", "on", "-indev", iso_path, "-extract", iso_internal_path, destination]) +// ONE ISO MEMBER COPIED OUT TO ONE DESTINATION PATH, read-only on the ISO: `xorriso -osirrox on +// -indev ISO -extract MEMBER DEST`. xorriso exits nonzero when the member is absent or the +// destination cannot be written. This is the one spelling of the invocation: its only consumer runs +// it over a remote typed-argv transport (gunbc.machine_intake_mtcollins1_census_member_readback), and +// the local ExtractPathBestEffort operation that restated the same argv had no caller and was deleted +// rather than kept as a second copy (DESIGN §3). A local consumer binds this builder to its local +// transport; it does not re-author the word list. +fn xorriso_extract_command(iso_path: NonEmptyStr, iso_internal_path: NonEmptyStr, destination: NonEmptyStr) -> ArgvCommand { + argv_command(program: xorriso_cli_tool.name, arguments: ["-osirrox", "on", "-indev", iso_path as String, "-extract", iso_internal_path as String, destination as String]) } diff --git a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag index addf99a8a7e..7227c90f20a 100644 --- a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag +++ b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag @@ -274,7 +274,7 @@ fn remote(target: SshTarget, context: FleetSshExecutionContext, command: ArgvCom fn read_member_remote(target: SshTarget, context: FleetSshExecutionContext, iso_path: NonEmptyStr, side: IsoSide, member: CensusIsoMember) -> MemberReading { let scratch = member_scratch_path(side: side, member: member) let cleared = remote(target: target, context: context, command: rm_force_command(path: scratch)) - let extract = remote(target: target, context: context, command: xorriso_extract_command(iso_path: iso_path as String, iso_internal_path: census_iso_member_path(member: member) as String, destination: scratch)) + let extract = remote(target: target, context: context, command: xorriso_extract_command(iso_path: iso_path, iso_internal_path: census_iso_member_path(member: member), destination: scratch as NonEmptyStr)) let measure = remote(target: target, context: context, command: sha256sum_file_command(path: scratch)) let removed = remote(target: target, context: context, command: rm_force_command(path: scratch)) member_reading_of(cleared: cleared, extract: extract, measure: measure, removed: removed) From 6fee01d67beae9bf3eec254a9b1c7aa4516fcb3c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 28 Sep 2026 05:36:33 +0000 Subject: [PATCH 6/7] Repair review 5334296282: identity candidates not known; unparsed records kept; owned srv2 workspace 1. EINVAL insert failures are unresolved candidates; every named insert failure is kept as its own observation; no nearest-line rule promotes one to the refusal's identity. 2. A recognised refusal or loader record that cannot be parsed is retained with its line and the failed field; the refusal count is not stated complete beside an unparsed record. 3. The readback gates extraction on the ISO pins, acquires an exclusively created mktemp -d workspace on srv2, extracts under it, never hashes a failed extraction, and removes that workspace once, reporting cleanup beside the verdict. Co-Authored-By: Claude Opus 5.5 (1M context) --- dag/extdeps/exec/command.dag | 1 + dag/extdeps/kmod/libkmod.dag | 4 + dag/extdeps/linux/module_decompress.dag | 4 + dag/extdeps/tools/gnu_coreutils.dag | 12 + ...ernel_module_decompression_observation.dag | 205 ++++++++----- .../mtcollins1_census_member_readback.dag | 271 +++++++++++++----- ...decompression_observation_witness_test.dag | 133 +++++++-- ...s1_census_member_readback_witness_test.dag | 74 ++++- 8 files changed, 534 insertions(+), 170 deletions(-) diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index a0177240acc..6a1357228ff 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -118,6 +118,7 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.tools.xorriso", decl_name: "xorriso_extract_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "cp_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "cat_command"), + decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "mktemp_directory_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "od_hex_span_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "rm_force_command"), decl_ref(module_path: "extdeps.tools.gnu_coreutils", decl_name: "rm_recursive_force_command"), diff --git a/dag/extdeps/kmod/libkmod.dag b/dag/extdeps/kmod/libkmod.dag index f48bda10473..77d16f8ab75 100644 --- a/dag/extdeps/kmod/libkmod.dag +++ b/dag/extdeps/kmod/libkmod.dag @@ -32,6 +32,10 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope { data libkmod_insert_failed_prefix: NonEmptyStr = "Failed to insert module '" +// The stem without the opening quote: a line carrying it but not a complete quoted path and errno is +// a truncated record of this INFO line. +data libkmod_insert_failed_stem: NonEmptyStr = "Failed to insert module" + // The separator after the quoted path, before strerror(-err). data libkmod_insert_failed_path_end: NonEmptyStr = "': " diff --git a/dag/extdeps/linux/module_decompress.dag b/dag/extdeps/linux/module_decompress.dag index 65490dfb498..39639ebb17c 100644 --- a/dag/extdeps/linux/module_decompress.dag +++ b/dag/extdeps/linux/module_decompress.dag @@ -33,6 +33,10 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope { // pr_err("ZSTD-decompression failed with status %d\n", retval), retval = zstd_get_error_code(ret). data linux_module_zstd_decompress_failed_prefix: NonEmptyStr = "ZSTD-decompression failed with status " +// The stem of that line without its status clause: a line carrying the stem but no parseable status +// is still a record of this printk, truncated or garbled, never an unrelated line. +data linux_module_zstd_decompress_failed_stem: NonEmptyStr = "ZSTD-decompression failed" + // zstd_errors.h ZSTD_ErrorCode: the status values a decoder can return are the library's own enum. // Only the rows this corpus has observed are carried; any other status is kept as its integer. type ZstdErrorCode diff --git a/dag/extdeps/tools/gnu_coreutils.dag b/dag/extdeps/tools/gnu_coreutils.dag index 12bf6e9c880..de6d5206400 100644 --- a/dag/extdeps/tools/gnu_coreutils.dag +++ b/dag/extdeps/tools/gnu_coreutils.dag @@ -36,6 +36,7 @@ data min_coreutils_version: VersionConstraint = ">= 8.0" // criterion for climbing to an absolute row, and the counter-example that makes the distinction // load-bearing are recorded once at extdeps.exec.command program_spelling_is_an_observation_claim. data cat_program: NonEmptyStr = "cat" +data mktemp_program: NonEmptyStr = "mktemp" data cp_program: NonEmptyStr = "cp" data printf_program: NonEmptyStr = "printf" data true_program: NonEmptyStr = "true" @@ -58,6 +59,17 @@ fn ls_one_per_line_command(path: String) -> ArgvCommand { argv_command(program: ls_program, arguments: ["-1", path]) } +// mktemp -d WITH A TEMPLATE: coreutils creates a NEW directory named from the template (trailing +// X's replaced), mode 0700, and prints its path; it refuses rather than reuse an existing name +// (coreutils manual, "mktemp invocation"). That exclusive creation is what makes the directory one +// caller's own. This is the argv for a remote typed-argv transport; extdeps.shell shell.Mktemp +// DirWithTemplate is the local realization of the same invocation -- the op-plus-builder seam +// extdeps.crypto.hash records for sha256sum, folding when a transport row can be read from the +// operation itself. +fn mktemp_directory_command(template: NonEmptyStr) -> ArgvCommand { + argv_command(program: mktemp_program, arguments: ["-d", template as String]) +} + fn cat_command(path: String) -> ArgvCommand { argv_command(program: cat_program, arguments: [path]) } diff --git a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag index 02a10e6600d..c2dec8b0049 100644 --- a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag +++ b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag @@ -4,60 +4,74 @@ import std.types { Bool, Int, List, NonEmptyStr, String } import std.algebra { trim } import v2.std.algebra { filter } import v2.std.optional { Present } -import extdeps.linux.module_decompress { linux_module_zstd_decompress_failed_prefix, ZstdErrorCode, zstd_error_code_of, zstd_error_code_name } -import extdeps.kmod.libkmod { libkmod_insert_failed_prefix, libkmod_insert_failed_path_end } +import extdeps.linux.module_decompress { linux_module_zstd_decompress_failed_prefix, linux_module_zstd_decompress_failed_stem, ZstdErrorCode, zstd_error_code_of, zstd_error_code_name } +import extdeps.kmod.libkmod { libkmod_insert_failed_prefix, libkmod_insert_failed_stem, libkmod_insert_failed_path_end } // WHAT A RETAINED CONSOLE CAPTURE SAYS ABOUT IN-KERNEL MODULE DECOMPRESSION, AT ITS REAL STRENGTH. // -// Three facts, kept apart because they have different producers: -// 1. the kernel REFUSED to decompress some module (extdeps.linux.module_decompress), with the -// decoder status it printed -- a decoder result, not a memory report, and no module name; -// 2. WHICH FILE was refused, known only when the loader's own record reached the capture -// (extdeps.kmod.libkmod, under the module-identity instrument's KMOD_LOG / kmsg settings); -// 3. whether that driver LATER printed anything in the capture. -// Refusal without a loader record is IDENTITY UNREAD, never a guessed module: adjacent kernel lines -// are not evidence of which finit_module call failed. The requesting device / modalias is not on the -// serial console at the instrument's chosen levels (it stays in the kmsg ring), so it is reported -// as unread on SOL and never inferred. +// Two populations, observed independently and kept apart: +// 1. the kernel's decompression REFUSALS (extdeps.linux.module_decompress): a decoder status and +// nothing else -- the printk names no module; +// 2. the loader's named INSERT FAILURES (extdeps.kmod.libkmod, reaching the capture under the +// module-identity instrument's KMOD_LOG / kmsg settings): a path and the errno the insertion +// returned. +// NOTHING IN THE CAPTURE LINKS ONE INVOCATION TO THE OTHER. A zstd refusal returns -EINVAL, but EINVAL +// does not establish a decompression failure -- the v6.8 loader returns it for other conditions too +// (symbol-version and namespace checks, invalid flags), and libkmod reports the insertion's errno, +// not the kernel's failing stage. So an EINVAL insert failure is a CANDIDATE for a refusal's +// identity, never its known identity, and no nearest-line or time-window rule promotes it. A known +// identity needs an invocation-level link this capture does not carry (the declared next rung). +// +// A RECOGNISED RECORD THAT CANNOT BE PARSED IS KEPT, never dropped: a truncated refusal line is not +// evidence that no refusal occurred, and an incomplete loader record is not evidence that no module +// failed. Each is retained with its line and the field that could not be read, and a refusal count +// is never stated as complete while an unparsed refusal record stands beside it. // // This observation says nothing about device availability, live-root acceptance, SOL health or the -// hardware: those are other readings of the same capture with other producers. In particular the -// xhci renesas_usb_fw.mem "fallback to ROM" warning is not read here at all -- it appears in the -// census-reaching boot too, so it is not a refusal of anything. +// hardware. The xhci renesas_usb_fw.mem "fallback to ROM" warning is not read here at all -- it +// appears in the census-reaching boot too, so it is not a refusal of anything. type ModuleDecompressionRefusal { at: String status: ZstdErrorCode } -type ModuleInsertFailure { - at: String - loader: String - path: NonEmptyStr - error: String +type UnparsedRecord { + line: String + field: String } type LaterDriverReading = DriverPrintedLater { at: String, line: String } | DriverNotSeenInCapture -type FailedModuleIdentity { - failure: ModuleInsertFailure +type ModuleInsertFailure { + at: String + loader: String + path: NonEmptyStr + error: String module_name: String later: LaterDriverReading } -type ModuleIdentityReading - = ModuleIdentityKnown { failures: List } - | ModuleIdentityUnread +// The candidates are the named insert failures whose errno is the one a refusal returns. Their +// relationship to any particular refusal stays unresolved. +type RefusalIdentityReading + = RefusalIdentityUnread + | RefusalIdentityCandidates { candidates: List } type KernelModuleDecompressionObservation - = ModuleDecompressionNotRefused - | ModuleDecompressionRefused { refusals: List, identity: ModuleIdentityReading } + = NoRefusalRecorded { insert_failures: List, unparsed_insert_records: List } + | RefusalRecorded { + refusals: List, + unparsed_refusal_records: List, + insert_failures: List, + unparsed_insert_records: List, + identity: RefusalIdentityReading, + } // The errno a decompression refusal returns to finit_module (decompress.c: retval = -EINVAL), as -// strerror renders it. An insert failure with another errno is a different failure and is not -// attributed to the refusal. +// strerror renders it -- necessary for a candidate, not sufficient for an identity. data refused_decompression_strerror: String = "Invalid argument" data request_identity_on_sol_text: String = "requesting device/modalias: unread on SOL (kmsg ring only)" @@ -93,30 +107,45 @@ fn stamped(raw: String) -> StampedLine { } } -fn decompression_refusal(l: StampedLine) -> ModuleDecompressionRefusal? { - let prefix = linux_module_zstd_decompress_failed_prefix as String - if !string_contains(s: l.rest, pattern: prefix) { none } else { - match parse_int(s: trim(join(split(s: l.rest, delimiter: prefix).skip(n: 1), prefix))) { - Absent => none - Present { value: code } => Present { value: ModuleDecompressionRefusal { at: l.at, status: zstd_error_code_of(code: code) } } +type RefusalLineReading + = NotARefusalLine + | RefusalLineParsed { refusal: ModuleDecompressionRefusal } + | RefusalLineUnparsed { record: UnparsedRecord } + +fn refusal_line_reading(l: StampedLine) -> RefusalLineReading { + if !string_contains(s: l.rest, pattern: linux_module_zstd_decompress_failed_stem as String) { NotARefusalLine } else { + let prefix = linux_module_zstd_decompress_failed_prefix as String + let status_text = trim(join(split(s: l.rest, delimiter: prefix).skip(n: 1), prefix)) + if !string_contains(s: l.rest, pattern: prefix) || status_text == "" { + RefusalLineUnparsed { record: UnparsedRecord { line: l.rest, field: "status" } } + } else { + match parse_int(s: status_text) { + Absent => RefusalLineUnparsed { record: UnparsedRecord { line: l.rest, field: "status" } } + Present { value: code } => RefusalLineParsed { refusal: ModuleDecompressionRefusal { at: l.at, status: zstd_error_code_of(code: code) } } + } } } } -fn insert_failure(l: StampedLine) -> ModuleInsertFailure? { - let prefix = libkmod_insert_failed_prefix as String - let parts = split(s: l.rest, delimiter: prefix) - if count(parts) < 2 { none } else { - let tail = join(parts.skip(n: 1), prefix) - let path_and_error = split(s: tail, delimiter: libkmod_insert_failed_path_end as String) +type InsertLineReading + = NotAnInsertLine + | InsertLineParsed { at: String, loader: String, path: NonEmptyStr, error: String } + | InsertLineUnparsed { record: UnparsedRecord } + +fn insert_line_reading(l: StampedLine) -> InsertLineReading { + if !string_contains(s: l.rest, pattern: libkmod_insert_failed_stem as String) { NotAnInsertLine } else { + let prefix = libkmod_insert_failed_prefix as String + let end = libkmod_insert_failed_path_end as String + let parts = split(s: l.rest, delimiter: prefix) + let path_and_error = split(s: join(parts.skip(n: 1), prefix), delimiter: end) let path = first_or_empty(xs: path_and_error) - if count(path_and_error) < 2 || path == "" { none } else { - Present { value: ModuleInsertFailure { - at: l.at, - loader: trim(join(split(s: trim(first_or_empty(xs: parts)), delimiter: ":"), "")), - path: path as NonEmptyStr, - error: trim(join(path_and_error.skip(n: 1), libkmod_insert_failed_path_end as String)), - } } + let error = trim(join(path_and_error.skip(n: 1), end)) + if count(parts) < 2 || count(path_and_error) < 2 || path == "" { + InsertLineUnparsed { record: UnparsedRecord { line: l.rest, field: "path" } } + } else if error == "" { + InsertLineUnparsed { record: UnparsedRecord { line: l.rest, field: "errno" } } + } else { + InsertLineParsed { at: l.at, loader: trim(join(split(s: trim(first_or_empty(xs: parts)), delimiter: ":"), "")), path: path as NonEmptyStr, error: error } } } } @@ -138,22 +167,30 @@ fn later_driver(lines: List, after_index: Int, module_name: String) fn kernel_module_decompression_observation(lines: List) -> KernelModuleDecompressionObservation { let stamped_lines = map(lines, raw => stamped(raw: raw)) - let refusals = flat_map(stamped_lines, l => match decompression_refusal(l: l) { Absent => [] Present { value: r } => [r] }) - if count(refusals) == 0 { ModuleDecompressionNotRefused } else { - let failures = flat_map(enumerate(stamped_lines), il => match insert_failure(l: il.second) { - Absent => [] - Present { value: f } => - if f.error == refused_decompression_strerror { - [FailedModuleIdentity { - failure: f, - module_name: module_name_of_path(path: f.path as String), - later: later_driver(lines: stamped_lines, after_index: il.first, module_name: module_name_of_path(path: f.path as String)), - }] - } else { [] } - }) - ModuleDecompressionRefused { + let refusal_readings = map(stamped_lines, l => refusal_line_reading(l: l)) + let refusals = flat_map(refusal_readings, r => match r { RefusalLineParsed { refusal: x } => [x] _ => [] }) + let unparsed_refusals = flat_map(refusal_readings, r => match r { RefusalLineUnparsed { record: x } => [x] _ => [] }) + let insert_readings = map(stamped_lines, l => insert_line_reading(l: l)) + let insert_failures = flat_map(enumerate(insert_readings), ir => match ir.second { + InsertLineParsed { at: a, loader: w, path: p, error: e } => + [ModuleInsertFailure { + at: a, loader: w, path: p, error: e, + module_name: module_name_of_path(path: p as String), + later: later_driver(lines: stamped_lines, after_index: ir.first, module_name: module_name_of_path(path: p as String)), + }] + _ => [] + }) + let unparsed_inserts = flat_map(insert_readings, r => match r { InsertLineUnparsed { record: x } => [x] _ => [] }) + if count(refusals) == 0 && count(unparsed_refusals) == 0 { + NoRefusalRecorded { insert_failures: insert_failures, unparsed_insert_records: unparsed_inserts } + } else { + let candidates: List = filter(insert_failures, f => f.error == refused_decompression_strerror) + RefusalRecorded { refusals: refusals, - identity: if count(failures) == 0 { ModuleIdentityUnread } else { ModuleIdentityKnown { failures: failures } }, + unparsed_refusal_records: unparsed_refusals, + insert_failures: insert_failures, + unparsed_insert_records: unparsed_inserts, + identity: if count(candidates) == 0 { RefusalIdentityUnread } else { RefusalIdentityCandidates { candidates: candidates } }, } } } @@ -162,6 +199,10 @@ fn refusal_text(r: ModuleDecompressionRefusal) -> String { concat("[", r.at, "] status ", zstd_error_code_name(code: r.status)) } +fn unparsed_text(u: UnparsedRecord) -> String { + concat(u.field, " unread in \"", u.line, "\"") +} + fn later_text(l: LaterDriverReading) -> String { match l { DriverPrintedLater { at: a, line: t } => concat("printed later at ", a, ": ", t) @@ -169,20 +210,40 @@ fn later_text(l: LaterDriverReading) -> String { } } -fn failed_module_text(f: FailedModuleIdentity) -> String { - concat("[", f.failure.at, "] ", f.failure.loader, " ", f.failure.path as String, ": ", f.failure.error, "; ", f.module_name, " ", later_text(l: f.later)) +fn insert_failure_text(f: ModuleInsertFailure) -> String { + concat("[", f.at, "] ", f.loader, " ", f.path as String, ": ", f.error, "; ", f.module_name, " ", later_text(l: f.later)) +} + +fn insert_population_text(failures: List, unparsed: List) -> String { + concat( + "named insert failures: ", if count(failures) == 0 { "none" } else { join(map(failures, f => insert_failure_text(f: f)), "; ") }, + if count(unparsed) == 0 { "" } else { concat("; UNPARSED insert records: ", join(map(unparsed, u => unparsed_text(u: u)), "; ")) }, + ) +} + +fn refusal_count_text(refusals: List, unparsed: List) -> String { + if count(unparsed) == 0 { + concat(to_string(count(refusals)), " time(s)") + } else { + concat(to_string(count(refusals)), " parsed record(s) plus ", to_string(count(unparsed)), " UNPARSED; the total is not stated") + } } fn kernel_module_decompression_text(o: KernelModuleDecompressionObservation) -> String { match o { - ModuleDecompressionNotRefused => "module decompression: no refusal in this capture" - ModuleDecompressionRefused { refusals: rs, identity: id } => + NoRefusalRecorded { insert_failures: fs, unparsed_insert_records: us } => + concat("module decompression: no refusal record in this capture; ", insert_population_text(failures: fs, unparsed: us)) + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: fs, unparsed_insert_records: us, identity: id } => concat( - "module decompression REFUSED ", to_string(count(rs)), " time(s): ", join(map(rs, r => refusal_text(r: r)), ", "), "; ", + "module decompression REFUSED ", refusal_count_text(refusals: rs, unparsed: urs), ": ", join(map(rs, r => refusal_text(r: r)), ", "), + if count(urs) == 0 { "" } else { concat("; UNPARSED refusal records: ", join(map(urs, u => unparsed_text(u: u)), "; ")) }, + "; ", match id { - ModuleIdentityUnread => "module identity UNREAD (no loader record reached this capture)" - ModuleIdentityKnown { failures: fs } => concat("module identity: ", join(map(fs, f => failed_module_text(f: f)), "; ")) + RefusalIdentityUnread => "refused module identity UNREAD (no named insert failure with the refusal's errno)" + RefusalIdentityCandidates { candidates: cs } => + concat("refused module identity UNRESOLVED; EINVAL candidates, not established by this capture: ", join(map(cs, f => concat(f.path as String, " [", f.at, "]")), ", ")) }, + "; ", insert_population_text(failures: fs, unparsed: us), "; ", request_identity_on_sol_text, ) } @@ -190,7 +251,7 @@ fn kernel_module_decompression_text(o: KernelModuleDecompressionObservation) -> fn kernel_module_decompression_findings(o: KernelModuleDecompressionObservation) -> List { match o { - ModuleDecompressionNotRefused => [] - ModuleDecompressionRefused { refusals: _, identity: _ } => [kernel_module_decompression_text(o: o)] + NoRefusalRecorded { insert_failures: _, unparsed_insert_records: _ } => [] + RefusalRecorded { refusals: _, unparsed_refusal_records: _, insert_failures: _, unparsed_insert_records: _, identity: _ } => [kernel_module_decompression_text(o: o)] } } diff --git a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag index 7227c90f20a..df0aec9dadf 100644 --- a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag +++ b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag @@ -7,7 +7,7 @@ import v2.std.algebra { filter } import std.content_hash { sha256_hex_digest } import extdeps.filesystem.filesystem_io { Filesystem } import extdeps.tools.xorriso { xorriso_extract_command } -import extdeps.tools.gnu_coreutils { rm_force_command } +import extdeps.tools.gnu_coreutils { mktemp_directory_command, rm_recursive_force_one_file_system_command } import extdeps.crypto.hash { sha256sum_file_command } import extdeps.exec.command { ArgvCommand, argv_words } import gunbc.fleet_intent_network { operator_host_srv2 } @@ -35,9 +35,11 @@ import gunbc.machine_intake_mtcollins1_census_medium_readback { mtcollins1_censu // named by BootChainFirstDifference below, not a guess made here. // // THE CONTROL IS IN THE SAME RUN. grub.cfg is the member the builder DOES rewrite (the census kernel -// cmdline lives there), so a reading in which grub.cfg compares equal did not read the census image -// at all -- a wrong path, a stale extraction, both sides extracted from one file -- and it refuses -// rather than reporting the boot chain identical. +// cmdline lives there), so a reading in which grub.cfg compares equal means the difference control +// FAILED, and it refuses rather than reporting the boot chain identical. It does not choose the +// cause: the census file may not have been the one read, or the published artifact may lack the +// intended rewrite. Likewise equal kernel and initrd bytes establish the scoped stock-versus-published +// comparison and nothing about what reached host memory or why a decoder refused. type CensusIsoMember = CasperKernel @@ -61,8 +63,8 @@ fn census_iso_member_slug(member: CensusIsoMember) -> NonEmptyStr { } // Boot order: the loader reads grub.cfg, then the kernel, then the initrd. The first differing -// boundary is the first member in this order that differs outside the builder's declared rewrite. -data census_boot_chain_members: List = [CasperKernel, CasperInitrd] +// boundary is the first member in this order that differs outside the builder's declared rewrite; +// boot_chain_verdict walks them in that order. type IsoSide = StockSide @@ -115,6 +117,7 @@ type CensusMemberReadback | BootChainUnread { member: CensusIsoMember, side: IsoSide, reason: String } | BootChainMediumNotCensus | BootChainHostUnreached { reason: String } + | BootChainWorkspaceUnavailable { reason: String } fn iso_pin_refusal(side: IsoSide, expected: NonEmptyStr, reading: IsoReading) -> CensusMemberReadback? { match reading { @@ -149,7 +152,47 @@ fn boot_chain_verdict(stock_iso: NonEmptyStr, census_iso: NonEmptyStr, grub: Mem } } -// The pure join over the six readings. The wet producer below supplies them from srv2. +// THE PIN GATE, evaluated BEFORE any member is extracted: the producer measures both ISOs, and only +// PinsAdmitted lets it acquire a workspace and read members. A refusal here is final. +type CensusPinGate + = PinsAdmitted { stock_iso: NonEmptyStr, census_iso: NonEmptyStr } + | PinsRefused { readback: CensusMemberReadback } + +fn census_pin_gate(medium: MtCollins1BootMedium, stock_iso: IsoReading, census_iso: IsoReading) -> CensusPinGate { + match medium { + MtCollins1StockInstallerMedium => PinsRefused { readback: BootChainMediumNotCensus } + MtCollins1CensusMedium { output_digest: census_pin } => + match iso_pin_refusal(side: StockSide, expected: noble_numbat_2404_3_live_server_arm64.content_sha256, reading: stock_iso) { + Present { value: refused } => PinsRefused { readback: refused } + Absent => + match iso_pin_refusal(side: CensusSide, expected: census_pin, reading: census_iso) { + Present { value: refused } => PinsRefused { readback: refused } + Absent => PinsAdmitted { stock_iso: noble_numbat_2404_3_live_server_arm64.content_sha256, census_iso: census_pin } + } + } + } +} + +fn census_member_verdict(stock_iso: NonEmptyStr, census_iso: NonEmptyStr, readings: CensusMemberReadings) -> CensusMemberReadback { + boot_chain_verdict( + stock_iso: stock_iso, + census_iso: census_iso, + grub: compare_member(member: GrubConfig, stock: readings.stock_grub, census: readings.census_grub), + kernel: compare_member(member: CasperKernel, stock: readings.stock_kernel, census: readings.census_kernel), + initrd: compare_member(member: CasperInitrd, stock: readings.stock_initrd, census: readings.census_initrd), + ) +} + +type CensusMemberReadings { + stock_grub: MemberReading + census_grub: MemberReading + stock_kernel: MemberReading + census_kernel: MemberReading + stock_initrd: MemberReading + census_initrd: MemberReading +} + +// The pure join over pins and six readings, composed as the producer composes it. fn census_member_readback_from_readings( medium: MtCollins1BootMedium, stock_iso: IsoReading, @@ -158,24 +201,14 @@ fn census_member_readback_from_readings( stock_kernel: MemberReading, census_kernel: MemberReading, stock_initrd: MemberReading, census_initrd: MemberReading, ) -> CensusMemberReadback { - match medium { - MtCollins1StockInstallerMedium => BootChainMediumNotCensus - MtCollins1CensusMedium { output_digest: census_pin } => - match iso_pin_refusal(side: StockSide, expected: noble_numbat_2404_3_live_server_arm64.content_sha256, reading: stock_iso) { - Present { value: refused } => refused - Absent => - match iso_pin_refusal(side: CensusSide, expected: census_pin, reading: census_iso) { - Present { value: refused } => refused - Absent => - boot_chain_verdict( - stock_iso: noble_numbat_2404_3_live_server_arm64.content_sha256, - census_iso: census_pin, - grub: compare_member(member: GrubConfig, stock: stock_grub, census: census_grub), - kernel: compare_member(member: CasperKernel, stock: stock_kernel, census: census_kernel), - initrd: compare_member(member: CasperInitrd, stock: stock_initrd, census: census_initrd), - ) - } - } + match census_pin_gate(medium: medium, stock_iso: stock_iso, census_iso: census_iso) { + PinsRefused { readback: r } => r + PinsAdmitted { stock_iso: s, census_iso: c } => + census_member_verdict(stock_iso: s, census_iso: c, readings: CensusMemberReadings { + stock_grub: stock_grub, census_grub: census_grub, + stock_kernel: stock_kernel, census_kernel: census_kernel, + stock_initrd: stock_initrd, census_initrd: census_initrd, + }) } } @@ -195,30 +228,96 @@ fn render_census_member_readback(readback: CensusMemberReadback) -> String { concat("verdict=first_difference\nstock_iso=", s as String, "\ncensus_iso=", c as String, "\nmember=", census_iso_member_path(member: m) as String, "\nstock_member=", sd as String, "\ncensus_member=", cd as String, "\n") BootChainControlDidNotDiscriminate { grub_digest: g } => - concat("verdict=control_did_not_discriminate\ngrub.cfg=", g as String, " on both sides; the census image was not read\n") + concat("verdict=control_did_not_discriminate\ngrub.cfg=", g as String, " on both sides: the difference control failed; cause not chosen (census file not the one read, or the published artifact lacks the rewrite)\n") BootChainIsoNotPinned { side: s, expected: e, observed: o } => concat("verdict=iso_not_pinned\nside=", iso_side_slug(side: s) as String, "\nexpected=", e as String, "\nobserved=", o, "\n") BootChainUnread { member: m, side: s, reason: r } => concat("verdict=unread\nmember=", census_iso_member_path(member: m) as String, "\nside=", iso_side_slug(side: s) as String, "\nreason=", r, "\n") BootChainMediumNotCensus => "verdict=medium_not_census\n" BootChainHostUnreached { reason: r } => concat("verdict=host_unreached\nreason=", r, "\n") + BootChainWorkspaceUnavailable { reason: r } => concat("verdict=workspace_unavailable (nothing extracted)\nreason=", r, "\n") } } // ---- the one producer: read-only legs on srv2 over fleet SSH, like the medium readback ---- // // srv2 has had no runner slots since 2026-09-19 (gunbc.machine_intake_mtcollins1_boot_authorization), -// so this does not run ON srv2: it reaches the host that serves /srv/bmc the way the boot's own -// medium readback does, as the fleet principal over typed argv. Per member and side, three legs: -// xorriso extracts the member to a path under srv2's /tmp, sha256sum measures it, rm -f removes it. +// so this reaches the host that serves /srv/bmc the way the boot's medium readback does, as the fleet +// principal over typed argv. ONE TRANSACTION PER ATTEMPT: +// 1. measure both ISOs; a pin refusal ends the attempt before any extraction; +// 2. create an EXCLUSIVELY OWNED workspace with mktemp -d (coreutils creates a new 0700 directory +// and never reuses an existing name), so concurrent dispatches -- the mode's concurrency key is +// the runner host, while the target is always srv2 -- never share a path; no workspace, no +// extraction; +// 3. per member, extract under that workspace and hash only what extracted; +// 4. remove that workspace, and report the cleanup outcome BESIDE the readback, never in place of +// it: an initiating failure and a cleanup failure are both retained. // Nothing under /srv/bmc is written. data census_member_readback_attempt_raw: String = "mtcollins1-census-member-readback" data census_member_readback_receipt_path: String = "target/mtcollins1-census-member-readback.txt" -fn member_scratch_path(side: IsoSide, member: CensusIsoMember) -> String { - concat("/tmp/gunbc-mtcollins1-census-member-", iso_side_slug(side: side) as String, "-", census_iso_member_slug(member: member) as String) +data census_member_workspace_prefix: String = "/tmp/gunbc-mtcollins1-census-member." + +data census_member_workspace_template: NonEmptyStr = concat(census_member_workspace_prefix, "XXXXXXXX") as NonEmptyStr + +type MemberWorkspace + = WorkspaceAcquired { dir: NonEmptyStr } + | WorkspaceRefused { reason: String } + +// mktemp prints the created path. Anything but exit 0 and one path under the template's prefix is a +// refusal: a path elsewhere is not the directory this attempt asked to own. +fn member_workspace_of(answer: RemoteAnswer) -> MemberWorkspace { + let dir = trim(s: answer.stdout) + if answer.exit_code != 0 { + WorkspaceRefused { reason: concat("mktemp -d exit=", to_string(answer.exit_code), " ", trim(s: answer.stderr)) } + } else if !starts_with(s: dir, prefix: census_member_workspace_prefix) || string_contains(s: dir, pattern: "\n") || string_contains(s: dir, pattern: "/..") { + WorkspaceRefused { reason: concat("mktemp -d printed a path outside ", census_member_workspace_prefix, ": ", dir) } + } else { + WorkspaceAcquired { dir: dir as NonEmptyStr } + } +} + +type WorkspaceCleanup + = WorkspaceRemoved { dir: NonEmptyStr } + | WorkspaceNotRemoved { dir: NonEmptyStr, reason: String } + | WorkspaceNeverCreated + +fn workspace_cleanup_of(dir: NonEmptyStr, answer: RemoteAnswer) -> WorkspaceCleanup { + if answer.exit_code == 0 { WorkspaceRemoved { dir: dir } } else { + WorkspaceNotRemoved { dir: dir, reason: concat("rm -rf --one-file-system exit=", to_string(answer.exit_code), " ", trim(s: answer.stderr)) } + } +} + +// THE ATTEMPT: the readback and its cleanup, both retained. It holds only when the boot chain is +// identical to stock AND this attempt's workspace was removed. +type CensusMemberReadbackAttempt { + readback: CensusMemberReadback + cleanup: WorkspaceCleanup +} + +fn census_member_attempt_holds(attempt: CensusMemberReadbackAttempt) -> Bool { + census_member_readback_holds(readback: attempt.readback) && match attempt.cleanup { + WorkspaceRemoved { dir: _ } => true + _ => false + } +} + +fn render_workspace_cleanup(c: WorkspaceCleanup) -> String { + match c { + WorkspaceRemoved { dir: d } => concat("workspace=", d as String, " removed\n") + WorkspaceNotRemoved { dir: d, reason: r } => concat("workspace=", d as String, " NOT REMOVED: ", r, "\n") + WorkspaceNeverCreated => "workspace=never created (nothing extracted)\n" + } +} + +fn render_census_member_attempt(attempt: CensusMemberReadbackAttempt) -> String { + concat(render_census_member_readback(readback: attempt.readback), render_workspace_cleanup(c: attempt.cleanup)) +} + +fn member_scratch_path(dir: NonEmptyStr, side: IsoSide, member: CensusIsoMember) -> NonEmptyStr { + concat(dir as String, "/", iso_side_slug(side: side) as String, "-", census_iso_member_slug(member: member) as String) as NonEmptyStr } fn remote_first_field(stdout: String) -> String { @@ -229,34 +328,41 @@ fn remote_first_field(stdout: String) -> String { } } -// The join over one member's legs. A scratch file that could not be cleared first or removed after -// is not a reading: the first means a stale file may be what was measured, the second leaves bytes -// on srv2, and both refuse rather than pass unremarked. -fn member_reading_of(cleared: RemoteAnswer, extract: RemoteAnswer, measure: RemoteAnswer, removed: RemoteAnswer) -> MemberReading { - if cleared.exit_code != 0 { - MemberUnread { reason: concat("rm -f of the scratch path before extraction exit=", to_string(cleared.exit_code), " ", trim(s: cleared.stderr)) } - } else if removed.exit_code != 0 { - MemberUnread { reason: concat("rm -f of the scratch path after measuring exit=", to_string(removed.exit_code), " ", trim(s: removed.stderr)) } - } else if extract.exit_code != 0 { - MemberUnread { reason: concat("xorriso extract exit=", to_string(extract.exit_code), " ", trim(s: extract.stderr)) } - } else if measure.exit_code != 0 { - MemberUnread { reason: concat("sha256sum exit=", to_string(measure.exit_code), " ", trim(s: measure.stderr)) } - } else { +fn digest_of_sha256sum(measure: RemoteAnswer) -> String? { + if measure.exit_code != 0 { none } else { match sha256_hex_digest(hex: remote_first_field(stdout: measure.stdout)) { - Absent => MemberUnread { reason: concat("sha256sum printed no digest: ", trim(s: measure.stdout)) } - Present { value: d } => MemberRead { digest: d.hex as String as NonEmptyStr } + Absent => none + Present { value: d } => Present { value: d.hex as String } } } } -fn iso_reading_of(measure: RemoteAnswer) -> IsoReading { +fn sha256sum_unread_reason(measure: RemoteAnswer) -> String { if measure.exit_code != 0 { - IsoUnmeasured { reason: concat("sha256sum exit=", to_string(measure.exit_code), " ", trim(s: measure.stderr)) } + concat("sha256sum exit=", to_string(measure.exit_code), " ", trim(s: measure.stderr)) } else { - match sha256_hex_digest(hex: remote_first_field(stdout: measure.stdout)) { - Absent => IsoUnmeasured { reason: concat("sha256sum printed no digest: ", trim(s: measure.stdout)) } - Present { value: d } => IsoMeasured { digest: d.hex as String as NonEmptyStr } - } + concat("sha256sum printed no digest: ", trim(s: measure.stdout)) + } +} + +// A failed extraction is never hashed: the measure leg is not run, so there is no answer to join. +fn member_reading_after_extract(extract: RemoteAnswer) -> MemberReading? { + if extract.exit_code != 0 { + Present { value: MemberUnread { reason: concat("xorriso extract exit=", to_string(extract.exit_code), " ", trim(s: extract.stderr)) } } + } else { none } +} + +fn member_reading_of_measure(measure: RemoteAnswer) -> MemberReading { + match digest_of_sha256sum(measure: measure) { + Absent => MemberUnread { reason: sha256sum_unread_reason(measure: measure) } + Present { value: d } => MemberRead { digest: d as NonEmptyStr } + } +} + +fn iso_reading_of(measure: RemoteAnswer) -> IsoReading { + match digest_of_sha256sum(measure: measure) { + Absent => IsoUnmeasured { reason: sha256sum_unread_reason(measure: measure) } + Present { value: d } => IsoMeasured { digest: d as NonEmptyStr } } } @@ -271,48 +377,67 @@ fn remote(target: SshTarget, context: FleetSshExecutionContext, command: ArgvCom remote_answer_of(outcome: typed_argv_exec_over_fleet_ssh(target: target, context: context, argv: argv_words(command: command))) } -fn read_member_remote(target: SshTarget, context: FleetSshExecutionContext, iso_path: NonEmptyStr, side: IsoSide, member: CensusIsoMember) -> MemberReading { - let scratch = member_scratch_path(side: side, member: member) - let cleared = remote(target: target, context: context, command: rm_force_command(path: scratch)) - let extract = remote(target: target, context: context, command: xorriso_extract_command(iso_path: iso_path, iso_internal_path: census_iso_member_path(member: member), destination: scratch as NonEmptyStr)) - let measure = remote(target: target, context: context, command: sha256sum_file_command(path: scratch)) - let removed = remote(target: target, context: context, command: rm_force_command(path: scratch)) - member_reading_of(cleared: cleared, extract: extract, measure: measure, removed: removed) +fn read_member_remote(target: SshTarget, context: FleetSshExecutionContext, dir: NonEmptyStr, iso_path: NonEmptyStr, side: IsoSide, member: CensusIsoMember) -> MemberReading { + let scratch = member_scratch_path(dir: dir, side: side, member: member) + let extract = remote(target: target, context: context, command: xorriso_extract_command(iso_path: iso_path, iso_internal_path: census_iso_member_path(member: member), destination: scratch)) + match member_reading_after_extract(extract: extract) { + Present { value: refused } => refused + Absent => member_reading_of_measure(measure: remote(target: target, context: context, command: sha256sum_file_command(path: scratch as String))) + } } -fn census_member_readback(medium: MtCollins1BootMedium) -> CensusMemberReadback { +fn census_member_readback_in_workspace(target: SshTarget, context: FleetSshExecutionContext, dir: NonEmptyStr, stock_path: NonEmptyStr, census_path: NonEmptyStr, stock_iso: NonEmptyStr, census_iso: NonEmptyStr) -> CensusMemberReadbackAttempt { + let readback = census_member_verdict(stock_iso: stock_iso, census_iso: census_iso, readings: CensusMemberReadings { + stock_grub: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: GrubConfig), + census_grub: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: GrubConfig), + stock_kernel: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: CasperKernel), + census_kernel: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: CasperKernel), + stock_initrd: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: CasperInitrd), + census_initrd: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: CasperInitrd), + }) + CensusMemberReadbackAttempt { + readback: readback, + cleanup: workspace_cleanup_of(dir: dir, answer: remote(target: target, context: context, command: rm_recursive_force_one_file_system_command(path: dir as String))), + } +} + +fn census_member_readback(medium: MtCollins1BootMedium) -> CensusMemberReadbackAttempt { match prepare_fleet_ssh_agent_context(attempt_raw: census_member_readback_attempt_raw) { FleetSshContextRefused { cause: c } => - BootChainHostUnreached { reason: c } + CensusMemberReadbackAttempt { readback: BootChainHostUnreached { reason: c }, cleanup: WorkspaceNeverCreated } FleetSshContextReady { context: context, receipt: _ } => { let target = census_member_readback_target() let stock_path = mtcollins1_census_image_stock_path() let census_path = mtcollins1_census_medium_served_path(medium: medium) - census_member_readback_from_readings( + let gate = census_pin_gate( medium: medium, stock_iso: iso_reading_of(measure: remote(target: target, context: context, command: sha256sum_file_command(path: stock_path as String))), census_iso: iso_reading_of(measure: remote(target: target, context: context, command: sha256sum_file_command(path: census_path as String))), - stock_grub: read_member_remote(target: target, context: context, iso_path: stock_path, side: StockSide, member: GrubConfig), - census_grub: read_member_remote(target: target, context: context, iso_path: census_path, side: CensusSide, member: GrubConfig), - stock_kernel: read_member_remote(target: target, context: context, iso_path: stock_path, side: StockSide, member: CasperKernel), - census_kernel: read_member_remote(target: target, context: context, iso_path: census_path, side: CensusSide, member: CasperKernel), - stock_initrd: read_member_remote(target: target, context: context, iso_path: stock_path, side: StockSide, member: CasperInitrd), - census_initrd: read_member_remote(target: target, context: context, iso_path: census_path, side: CensusSide, member: CasperInitrd), ) + match gate { + PinsRefused { readback: r } => CensusMemberReadbackAttempt { readback: r, cleanup: WorkspaceNeverCreated } + PinsAdmitted { stock_iso: s, census_iso: c } => + match member_workspace_of(answer: remote(target: target, context: context, command: mktemp_directory_command(template: census_member_workspace_template))) { + WorkspaceRefused { reason: r } => + CensusMemberReadbackAttempt { readback: BootChainWorkspaceUnavailable { reason: r }, cleanup: WorkspaceNeverCreated } + WorkspaceAcquired { dir: dir } => + census_member_readback_in_workspace(target: target, context: context, dir: dir, stock_path: stock_path, census_path: census_path, stock_iso: s, census_iso: c) + } + } } } } // CONSUMPTION (DESIGN §3c): invoked by the fleet-converge mode mtcollins1_census_member_readback, // which consumes the fleet SSH key; the receipt is uploaded as that run's artifact whatever the -// verdict. The run exits nonzero on every verdict but BootChainIdenticalToStock. +// verdict. The run exits nonzero unless the chain is identical AND the workspace was removed. fn mtcollins1_census_member_readback_wet() -> ProcessExit { - let readback = census_member_readback(medium: mtcollins1_boot_medium) - let rendered = render_census_member_readback(readback: readback) + let attempt = census_member_readback(medium: mtcollins1_boot_medium) + let rendered = render_census_member_attempt(attempt: attempt) let written = Filesystem.Write(path: census_member_readback_receipt_path, content: rendered) if written.success == false { - exit_failure(reason: concat("mtcollins1 census member readback: the receipt could not be written: ", written.error)) - } else if census_member_readback_holds(readback: readback) { + exit_failure(reason: concat("mtcollins1 census member readback: the receipt could not be written: ", written.error, "\n", rendered)) + } else if census_member_attempt_holds(attempt: attempt) { ExitSuccess } else { exit_failure(reason: concat("mtcollins1 census member readback did not hold:\n", rendered)) diff --git a/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag b/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag index 711735e580f..0b5a88dbe7e 100644 --- a/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag +++ b/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag @@ -3,11 +3,11 @@ module test.claim.machine_intake.kernel_module_decompression_observation_witness import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import std.types { Bool, List, String } import std.measure { byte_size } -import extdeps.linux.module_decompress { ZstdCorruptionDetected } +import extdeps.linux.module_decompress { ZstdCorruptionDetected, ZstdErrorCodeOther } import gunbc.machine_intake_kernel_module_decompression_observation { kernel_module_decompression_observation, kernel_module_decompression_text, kernel_module_decompression_findings, - KernelModuleDecompressionObservation, ModuleDecompressionNotRefused, ModuleDecompressionRefused, - ModuleIdentityKnown, ModuleIdentityUnread, FailedModuleIdentity, DriverPrintedLater, DriverNotSeenInCapture, + KernelModuleDecompressionObservation, NoRefusalRecorded, RefusalRecorded, + RefusalIdentityUnread, RefusalIdentityCandidates, ModuleInsertFailure, DriverPrintedLater, DriverNotSeenInCapture, } import gunbc.machine_intake_mtcollins1_boot_diagnostic_bundle { mtcollins1_boot_console_reading, ReferenceCaptureRead, ConsoleRetained } @@ -59,18 +59,19 @@ data instrumented_excerpt: List = [ // POSITIVE CONTROL: the census-reaching boot is not a refusal, renesas warning included. test fn the_census_reaching_capture_reads_as_not_refused() -> Bool { - match kernel_module_decompression_observation(lines: census_reaching_excerpt) { - ModuleDecompressionNotRefused => true - _ => false - } - && count(kernel_module_decompression_findings(o: kernel_module_decompression_observation(lines: census_reaching_excerpt))) == 0 + let o = kernel_module_decompression_observation(lines: census_reaching_excerpt) + count(kernel_module_decompression_findings(o: o)) == 0 + && match o { + NoRefusalRecorded { insert_failures: fs, unparsed_insert_records: us } => count(fs) == 0 && count(us) == 0 + _ => false + } } // RED, ANONYMOUS: the failed boot is a refusal with both statuses retained, and identity UNREAD. test fn the_failed_capture_is_a_refusal_whose_identity_is_unread() -> Bool { match kernel_module_decompression_observation(lines: failed_attempt_excerpt) { - ModuleDecompressionRefused { refusals: rs, identity: ModuleIdentityUnread } => - count(rs) == 2 + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: RefusalIdentityUnread } => + count(rs) == 2 && count(urs) == 0 && match first(rs) { Absent => false Present { value: r } => r.at == "10.374590" && match r.status { ZstdCorruptionDetected => true _ => false } @@ -82,33 +83,117 @@ test fn the_failed_capture_is_a_refusal_whose_identity_is_unread() -> Bool { test fn the_failed_capture_text_names_no_module_and_leaves_the_request_unread_on_sol() -> Bool { let text = kernel_module_decompression_text(o: kernel_module_decompression_observation(lines: failed_attempt_excerpt)) string_contains(s: text, pattern: "module identity UNREAD") + && string_contains(s: text, pattern: "REFUSED 2 time(s)") && string_contains(s: text, pattern: "status corruption_detected") && string_contains(s: text, pattern: "requesting device/modalias: unread on SOL") } -// RED, NAMED: a loader record with the refusal's errno names the file, the worker and the later -// printing; an insert failure with another errno is NOT attributed to the refusal. -test fn an_instrumented_capture_names_the_refused_module_path_and_worker() -> Bool { +// THE INSTRUMENTED CAPTURE: the EINVAL insert failure is a CANDIDATE, identity UNRESOLVED -- the +// capture carries no invocation-level link, however close the two lines sit. Every named failure, +// candidate or not, is retained as its own observation. +test fn an_instrumented_capture_keeps_the_named_failure_as_an_unresolved_candidate() -> Bool { match kernel_module_decompression_observation(lines: instrumented_excerpt) { - ModuleDecompressionRefused { refusals: rs, identity: ModuleIdentityKnown { failures: fs } } => - count(rs) == 1 && count(fs) == 1 - && match first(fs) { Absent => false Present { value: f } => named_xhci_pci(f: f) } + RefusalRecorded { refusals: rs, unparsed_refusal_records: _, insert_failures: all, unparsed_insert_records: _, identity: RefusalIdentityCandidates { candidates: cs } } => + count(rs) == 1 && count(all) == 2 && count(cs) == 1 + && match first(cs) { Absent => false Present { value: f } => named_xhci_pci(f: f) } _ => false } } -fn named_xhci_pci(f: FailedModuleIdentity) -> Bool { - (f.failure.path as String) == "/lib/modules/6.8.0-71-generic/kernel/drivers/usb/host/xhci-pci.ko.zst" - && f.failure.loader == "(udev-worker)[100]" - && f.failure.at == "27.209606" +test fn the_instrumented_text_says_unresolved_not_known() -> Bool { + let text = kernel_module_decompression_text(o: kernel_module_decompression_observation(lines: instrumented_excerpt)) + string_contains(s: text, pattern: "identity UNRESOLVED") + && string_contains(s: text, pattern: "not established by this capture") + && string_contains(s: text, pattern: "uas.ko.zst: No such device") +} + +fn named_xhci_pci(f: ModuleInsertFailure) -> Bool { + (f.path as String) == "/lib/modules/6.8.0-71-generic/kernel/drivers/usb/host/xhci-pci.ko.zst" + && f.loader == "(udev-worker)[100]" + && f.at == "27.209606" && f.module_name == "xhci_pci" && match f.later { DriverPrintedLater { at: a, line: _ } => a == "29.000000" DriverNotSeenInCapture => false } } -// A loader record with no refusal line is not a decompression refusal. -test fn a_loader_failure_without_a_kernel_refusal_is_not_a_refusal() -> Bool { +// REVIEW COUNTEREXAMPLE (eager-owl-205): an unrelated EINVAL failure BEFORE the refusal is not the +// refusal's identity; it is at most a candidate, and the verdict never reads "known". +test fn an_unrelated_earlier_einval_failure_is_only_a_candidate() -> Bool { + match kernel_module_decompression_observation(lines: [ + "[ 9.000000] (udev-worker)[99]: Failed to insert module '/unrelated.ko.zst': Invalid argument", + "[ 10.374590] ZSTD-decompression failed with status 20", + ]) { + RefusalRecorded { refusals: _, unparsed_refusal_records: _, insert_failures: _, unparsed_insert_records: _, identity: RefusalIdentityCandidates { candidates: cs } } => count(cs) == 1 + _ => false + } +} + +// A loader record with no refusal line is not a decompression refusal, and is still retained. +test fn a_loader_failure_without_a_kernel_refusal_is_not_a_refusal_but_is_kept() -> Bool { match kernel_module_decompression_observation(lines: instrumented_excerpt.skip(n: 1).take(n: 1)) { - ModuleDecompressionNotRefused => true + NoRefusalRecorded { insert_failures: fs, unparsed_insert_records: _ } => count(fs) == 1 + _ => false + } +} + +// MALFORMED-ONLY: a recognised refusal record whose status cannot be read is a refusal record, never +// "no refusal", and no status is invented for it. +test fn a_malformed_refusal_record_alone_is_still_a_refusal_with_its_line_kept() -> Bool { + match kernel_module_decompression_observation(lines: [ + "[ 10.374590] ZSTD-decompression failed with status", + "[ 10.374590] ZSTD-decompression failed with status unreadable", + ]) { + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: _ } => + count(rs) == 0 && count(urs) == 2 + && match first(urs) { Absent => false Present { value: u } => u.field == "status" && string_contains(s: u.line, pattern: "failed with status") } + _ => false + } +} + +// VALID PLUS MALFORMED: the parsed refusal is kept, the unparsed one is not dropped, and the text +// does not state a complete count. +test fn a_valid_plus_malformed_refusal_does_not_state_a_complete_count() -> Bool { + let o = kernel_module_decompression_observation(lines: [ + "[ 10.374590] ZSTD-decompression failed with status 20", + "[ 10.380475] ZSTD-decompression failed with sta", + ]) + let text = kernel_module_decompression_text(o: o) + match o { + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: _ } => count(rs) == 1 && count(urs) == 1 + _ => false + } + && string_contains(s: text, pattern: "the total is not stated") + && !string_contains(s: text, pattern: "REFUSED 1 time(s)") +} + +// An uncatalogued numeric status stays a parsed refusal carrying its integer. +test fn an_uncatalogued_numeric_status_is_parsed_not_unread() -> Bool { + match kernel_module_decompression_observation(lines: ["[ 1.0] ZSTD-decompression failed with status 42"]) { + RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: _ } => + count(urs) == 0 && match first(rs) { Absent => false Present { value: r } => match r.status { ZstdErrorCodeOther { code: c } => c == 42 _ => false } } + _ => false + } +} + +// INCOMPLETE LOADER RECORDS: truncated before the path end, or with no errno, are kept as unparsed +// insert records with the field that failed -- neither dropped nor turned into a candidate. +test fn incomplete_loader_records_are_kept_as_unparsed() -> Bool { + match kernel_module_decompression_observation(lines: [ + "[ 10.374590] ZSTD-decompression failed with status 20", + "[ 10.374801] (udev-worker)[412]: Failed to insert module '/lib/modules/6.8.0-71-generic/kernel/drivers/usb/host/xhci-pci.ko.z", + "[ 10.374802] (udev-worker)[413]: Failed to insert module '/lib/modules/x.ko.zst': ", + ]) { + RefusalRecorded { refusals: _, unparsed_refusal_records: _, insert_failures: fs, unparsed_insert_records: us, identity: RefusalIdentityUnread } => + count(fs) == 0 && count(us) == 2 + && match first(us) { Absent => false Present { value: u } => u.field == "path" } + && match first(us.skip(n: 1)) { Absent => false Present { value: u } => u.field == "errno" } + _ => false + } +} + +// UNRELATED TEXT is neither a refusal nor a loader record. +test fn unrelated_text_is_not_a_record() -> Bool { + match kernel_module_decompression_observation(lines: ["[ 1.0] zstd: something else", "[ 1.1] Failed to allocate memory pressure watch"]) { + NoRefusalRecorded { insert_failures: fs, unparsed_insert_records: us } => count(fs) == 0 && count(us) == 0 _ => false } } @@ -119,7 +204,7 @@ test fn the_retained_console_reading_carries_the_refusal() -> Bool { let capture = join(failed_attempt_excerpt, "\n") match mtcollins1_boot_console_reading(path: "artifacts/mtcollins1-boot-sol.capture", size: byte_size(count: 10), capture: concat(capture, "\n"), read_ok: true, read_error: "", reference: ReferenceCaptureRead { text: "", ok: false, error: "not supplied" }) { ConsoleRetained { path: _, size: _, lines: _, firmware: _, dram: _, sockets: _, against_reference: _, modules: m } => - match m { ModuleDecompressionRefused { refusals: _, identity: ModuleIdentityUnread } => true _ => false } + match m { RefusalRecorded { refusals: rs, unparsed_refusal_records: _, insert_failures: _, unparsed_insert_records: _, identity: RefusalIdentityUnread } => count(rs) == 2 _ => false } _ => false } } diff --git a/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag index 9e3eeaa6304..41f7daae6b2 100644 --- a/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag +++ b/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag @@ -1,7 +1,7 @@ module test.claim.machine_intake.mtcollins1_census_member_readback_witness_test import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } -import std.types { Bool, NonEmptyStr, String } +import std.types { Bool, Int, NonEmptyStr, String } import gunbc.machine_intake_mtcollins1_boot_authorization { MtCollins1CensusMedium, MtCollins1StockInstallerMedium } import gunbc.machine_intake_mtcollins1_census_member_readback { census_member_readback_from_readings, census_member_readback_holds, render_census_member_readback, @@ -9,7 +9,13 @@ import gunbc.machine_intake_mtcollins1_census_member_readback { CensusMemberReadback, BootChainIdenticalToStock, BootChainFirstDifference, BootChainControlDidNotDiscriminate, BootChainIsoNotPinned, BootChainUnread, BootChainMediumNotCensus, CasperKernel, CasperInitrd, GrubConfig, StockSide, CensusSide, + census_pin_gate, PinsAdmitted, PinsRefused, + member_workspace_of, WorkspaceAcquired, WorkspaceRefused, + workspace_cleanup_of, WorkspaceRemoved, WorkspaceNotRemoved, WorkspaceNeverCreated, + CensusMemberReadbackAttempt, census_member_attempt_holds, render_census_member_attempt, + member_reading_after_extract, member_reading_of_measure, member_scratch_path, BootChainWorkspaceUnavailable, } +import gunbc.machine_intake_mtcollins1_census_medium_readback { RemoteAnswer } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -102,3 +108,69 @@ test fn the_stock_medium_row_has_no_census_boot_chain_to_read() -> Bool { _ => false } } + +fn answer(code: Int, stdout: String, stderr: String) -> RemoteAnswer { RemoteAnswer { exit_code: code, stdout: stdout, stderr: stderr } } + +data stock_member_digest: NonEmptyStr = "deb2c288c12e6f35c2b9242134116ea03f7d4eb91f0c354ecd993680ffabbbf7" + +// THE PIN GATE PRECEDES EXTRACTION: an off-pin census ISO is refused by the gate the producer +// evaluates before it acquires a workspace, so the gate's refusal carries no member reading at all. +test fn an_off_pin_iso_is_refused_by_the_gate_that_precedes_extraction() -> Bool { + match census_pin_gate(medium: census, stock_iso: IsoMeasured { digest: stock_iso }, census_iso: IsoMeasured { digest: stock_iso }) { + PinsRefused { readback: BootChainIsoNotPinned { side: CensusSide, expected: _, observed: _ } } => true + _ => false + } + && match census_pin_gate(medium: census, stock_iso: IsoMeasured { digest: stock_iso }, census_iso: IsoMeasured { digest: census_iso }) { + PinsAdmitted { stock_iso: _, census_iso: _ } => true + _ => false + } +} + +// THE WORKSPACE IS OWNED OR ABSENT: mktemp's printed path under the template prefix is admitted; +// a failed mktemp, or a path anywhere else, is a refusal -- and no extraction path is derived from it. +test fn a_workspace_is_admitted_only_from_a_successful_mktemp_under_the_prefix() -> Bool { + match member_workspace_of(answer: answer(code: 0, stdout: "/tmp/gunbc-mtcollins1-census-member.Ab12Cd34\n", stderr: "")) { + WorkspaceAcquired { dir: d } => (d as String) == "/tmp/gunbc-mtcollins1-census-member.Ab12Cd34" + && (member_scratch_path(dir: d, side: CensusSide, member: CasperInitrd) as String) == "/tmp/gunbc-mtcollins1-census-member.Ab12Cd34/census-initrd" + _ => false + } + && match member_workspace_of(answer: answer(code: 1, stdout: "", stderr: "mktemp: failed to create directory")) { WorkspaceRefused { reason: _ } => true _ => false } + && match member_workspace_of(answer: answer(code: 0, stdout: "/srv/bmc/x", stderr: "")) { WorkspaceRefused { reason: _ } => true _ => false } + && match member_workspace_of(answer: answer(code: 0, stdout: "/tmp/gunbc-mtcollins1-census-member.X/../../srv/bmc", stderr: "")) { WorkspaceRefused { reason: _ } => true _ => false } +} + +// A FAILED EXTRACTION IS NEVER HASHED: its reading is decided from the extract leg alone. +test fn a_failed_extraction_is_unread_without_a_measure() -> Bool { + match member_reading_after_extract(extract: answer(code: 1, stdout: "", stderr: "xorriso : FAILURE")) { + Present { value: MemberUnread { reason: r } } => string_contains(s: r, pattern: "xorriso extract exit=1") + _ => false + } + && match member_reading_after_extract(extract: answer(code: 0, stdout: "", stderr: "")) { Absent => true _ => false } + && match member_reading_of_measure(measure: answer(code: 0, stdout: concat(stock_member_digest as String, " /tmp/x/stock-initrd\n"), stderr: "")) { + MemberRead { digest: d } => (d as String) == (stock_member_digest as String) + _ => false + } +} + +// THE ATTEMPT HOLDS ONLY WITH ITS WORKSPACE REMOVED, and a cleanup failure is reported BESIDE the +// readback, never in place of it: both the verdict and the cleanup outcome survive in the receipt. +test fn a_cleanup_failure_is_retained_beside_the_verdict_and_blocks_holding() -> Bool { + let identical = over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: read(d: "i0")) + let removed = CensusMemberReadbackAttempt { readback: identical, cleanup: workspace_cleanup_of(dir: "/tmp/gunbc-mtcollins1-census-member.A", answer: answer(code: 0, stdout: "", stderr: "")) } + let stuck = CensusMemberReadbackAttempt { readback: identical, cleanup: workspace_cleanup_of(dir: "/tmp/gunbc-mtcollins1-census-member.A", answer: answer(code: 1, stdout: "", stderr: "Device or resource busy")) } + let text = render_census_member_attempt(attempt: stuck) + census_member_attempt_holds(attempt: removed) + && census_member_attempt_holds(attempt: stuck) == false + && string_contains(s: text, pattern: "verdict=boot_chain_identical_to_stock") + && string_contains(s: text, pattern: "NOT REMOVED") +} + +// AN INITIATING FAILURE AND ITS CLEANUP ARE BOTH KEPT: an unread member with a removed workspace +// renders the unread verdict and the removal; no workspace means "never created", not "removed". +test fn an_initiating_failure_keeps_its_verdict_and_its_cleanup_outcome() -> Bool { + let unread = over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: MemberUnread { reason: "xorriso extract exit=1" }) + let text = render_census_member_attempt(attempt: CensusMemberReadbackAttempt { readback: unread, cleanup: WorkspaceRemoved { dir: "/tmp/gunbc-mtcollins1-census-member.B" } }) + let never = render_census_member_attempt(attempt: CensusMemberReadbackAttempt { readback: BootChainWorkspaceUnavailable { reason: "mktemp exit=1" }, cleanup: WorkspaceNeverCreated }) + string_contains(s: text, pattern: "verdict=unread") && string_contains(s: text, pattern: "removed") + && string_contains(s: never, pattern: "workspace_unavailable (nothing extracted)") && string_contains(s: never, pattern: "never created") +} From b6ae2216e5d9df9e90a0949544bf395e89d6ee79 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 28 Sep 2026 06:41:34 +0000 Subject: [PATCH 7/7] Workspace outcomes at the remote acknowledgement boundary (review 5334663150) An issued mktemp whose reply was lost (ssh 255) or whose output admits no path is CreationUnconfirmed with residue outstanding and the returned text kept, never 'never created'; a lost rm reply is RemovalUnconfirmed, distinct from rm's own failure and from an undispatched rm. Legs keep the transport's not-dispatched arm instead of collapsing to exit 255. Controls run at the attempt/render consumer. UnparsedRecord carries its timestamp. Co-Authored-By: Claude Opus 5.5 (1M context) --- dag/extdeps/ssh/session.dag | 6 + ...ernel_module_decompression_observation.dag | 13 +- .../mtcollins1_census_member_readback.dag | 174 +++++++++++++----- ...decompression_observation_witness_test.dag | 2 +- ...s1_census_member_readback_witness_test.dag | 156 ++++++++++++---- 5 files changed, 258 insertions(+), 93 deletions(-) diff --git a/dag/extdeps/ssh/session.dag b/dag/extdeps/ssh/session.dag index 5bf23a7981c..be7f161da95 100644 --- a/dag/extdeps/ssh/session.dag +++ b/dag/extdeps/ssh/session.dag @@ -41,6 +41,12 @@ fn shape_scp_copy_to_argv( [local_path as String, join([host as String, ":", remote_path as String], "")] } +// ssh(1) EXIT STATUS (OpenBSD 7.6): "ssh exits with the exit status of the remote command or with 255 +// if an error occurred." 255 is the CLIENT's status: it does not establish that the remote command +// did not run -- a connection lost after the command started, or before its status came back, also +// ends here. A command that itself exits 255 is indistinguishable from it at this boundary. +data ssh_client_error_exit_status: Int = 255 + type SshSessionExecResult { exit_code: Int success: Bool diff --git a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag index c2dec8b0049..c64b33b53d0 100644 --- a/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag +++ b/dag/gunbc/machine_intake/kernel_module_decompression_observation.dag @@ -36,7 +36,10 @@ type ModuleDecompressionRefusal { status: ZstdErrorCode } +// `line` is the record's message text after the timestamp and CR/NUL are stripped; `at` is that +// timestamp ("" when the line had none). The retained console capture stays the verbatim reference. type UnparsedRecord { + at: String line: String field: String } @@ -117,10 +120,10 @@ fn refusal_line_reading(l: StampedLine) -> RefusalLineReading { let prefix = linux_module_zstd_decompress_failed_prefix as String let status_text = trim(join(split(s: l.rest, delimiter: prefix).skip(n: 1), prefix)) if !string_contains(s: l.rest, pattern: prefix) || status_text == "" { - RefusalLineUnparsed { record: UnparsedRecord { line: l.rest, field: "status" } } + RefusalLineUnparsed { record: UnparsedRecord { at: l.at, line: l.rest, field: "status" } } } else { match parse_int(s: status_text) { - Absent => RefusalLineUnparsed { record: UnparsedRecord { line: l.rest, field: "status" } } + Absent => RefusalLineUnparsed { record: UnparsedRecord { at: l.at, line: l.rest, field: "status" } } Present { value: code } => RefusalLineParsed { refusal: ModuleDecompressionRefusal { at: l.at, status: zstd_error_code_of(code: code) } } } } @@ -141,9 +144,9 @@ fn insert_line_reading(l: StampedLine) -> InsertLineReading { let path = first_or_empty(xs: path_and_error) let error = trim(join(path_and_error.skip(n: 1), end)) if count(parts) < 2 || count(path_and_error) < 2 || path == "" { - InsertLineUnparsed { record: UnparsedRecord { line: l.rest, field: "path" } } + InsertLineUnparsed { record: UnparsedRecord { at: l.at, line: l.rest, field: "path" } } } else if error == "" { - InsertLineUnparsed { record: UnparsedRecord { line: l.rest, field: "errno" } } + InsertLineUnparsed { record: UnparsedRecord { at: l.at, line: l.rest, field: "errno" } } } else { InsertLineParsed { at: l.at, loader: trim(join(split(s: trim(first_or_empty(xs: parts)), delimiter: ":"), "")), path: path as NonEmptyStr, error: error } } @@ -200,7 +203,7 @@ fn refusal_text(r: ModuleDecompressionRefusal) -> String { } fn unparsed_text(u: UnparsedRecord) -> String { - concat(u.field, " unread in \"", u.line, "\"") + concat(u.field, " unread in [", u.at, "] \"", u.line, "\"") } fn later_text(l: LaterDriverReading) -> String { diff --git a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag index df0aec9dadf..0c7d03b8664 100644 --- a/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag +++ b/dag/gunbc/machine_intake/mtcollins1_census_member_readback.dag @@ -1,6 +1,6 @@ module gunbc.machine_intake_mtcollins1_census_member_readback -import std.types { Bool, List, NonEmptyStr, String } +import std.types { Bool, Int, List, NonEmptyStr, String } import std.process { ExitSuccess, ProcessExit, exit_failure } import std.algebra { trim } import v2.std.algebra { filter } @@ -15,7 +15,8 @@ import gunbc.fleet_reach_endpoint { fleet_probe_endpoint_for } import gunbc.fleet_bootstrap_principal { executor_bootstrap_principal } import gunbc.fleet_known_hosts_anchor { SshTarget, FleetSshExecutionContext } import gunbc.fleet_ssh_locus { prepare_fleet_ssh_agent_context, FleetSshContextReady, FleetSshContextRefused } -import gunbc.typed_argv_exec { typed_argv_exec_over_fleet_ssh } +import gunbc.typed_argv_exec { typed_argv_exec_over_fleet_ssh, TypedArgvExecOutcome, TypedArgvExecConverged, TypedArgvExecRefused } +import extdeps.ssh.session { ssh_client_error_exit_status } import extdeps.provisioning.ubuntu_install_media { noble_numbat_2404_3_live_server_arm64 } import gunbc.machine_intake_mtcollins1_boot_authorization { MtCollins1BootMedium, MtCollins1CensusMedium, MtCollins1StockInstallerMedium, mtcollins1_boot_medium } import gunbc.machine_intake_mtcollins1_census_image { mtcollins1_census_image_stock_path } @@ -251,7 +252,9 @@ fn render_census_member_readback(readback: CensusMemberReadback) -> String { // extraction; // 3. per member, extract under that workspace and hash only what extracted; // 4. remove that workspace, and report the cleanup outcome BESIDE the readback, never in place of -// it: an initiating failure and a cleanup failure are both retained. +// it: an initiating failure and a cleanup failure are both retained. A creation or removal whose +// reply was lost is reported UNCONFIRMED with residue outstanding, never as "never created" or +// "not removed" (WorkspaceOutcome). // Nothing under /srv/bmc is written. data census_member_readback_attempt_raw: String = "mtcollins1-census-member-readback" @@ -262,58 +265,133 @@ data census_member_workspace_prefix: String = "/tmp/gunbc-mtcollins1-census-memb data census_member_workspace_template: NonEmptyStr = concat(census_member_workspace_prefix, "XXXXXXXX") as NonEmptyStr -type MemberWorkspace +// ONE REMOTE LEG AS THE TRANSPORT ANSWERED IT, before any collapse to a RemoteAnswer. Refused means +// the invocation was never made (gunbc.typed_argv_exec, FleetPrincipalLegRefused); Ran carries what +// came back. A Ran leg with ssh's client status 255 (extdeps.ssh.session ssh_client_error_exit_status) +// establishes neither that the remote command ran nor that it did not. +type RemoteLeg + = LegNotDispatched { reason: String } + | LegAnswered { exit_code: Int, stdout: String, stderr: String } + +fn remote_leg_of(outcome: TypedArgvExecOutcome) -> RemoteLeg { + match outcome { + TypedArgvExecRefused { reason: r } => LegNotDispatched { reason: r } + TypedArgvExecConverged { result: r } => LegAnswered { exit_code: r.exit_code, stdout: r.stdout, stderr: r.stderr } + } +} + +// THE WORKSPACE'S WHOLE HISTORY, each arm naming what is established and nothing more. +// CreationNotRequested a gate stopped the attempt before mktemp was issued; +// CreationNotDispatched mktemp was refused by the transport before it was sent; +// CreationFailed mktemp itself reported failure (a non-255 status): nothing was created; +// CreationUnconfirmed mktemp was sent and its result was not established -- ssh's 255, or a +// zero exit whose output is not exactly one path under the prefix. A +// directory MAY remain; whatever came back is retained, and no name that +// was not admitted is ever used to extract or to delete; +// Removed / RemovalFailed / RemovalUnconfirmed / RemovalNotDispatched the acquired workspace's end: +// rm reported success; rm reported failure (the path may remain); +// the reply was lost (removal unconfirmed, NOT proof it remains); +// rm was never sent (the workspace remains). +// Only Removed lets an attempt hold. +type WorkspaceOutcome + = CreationNotRequested + | CreationNotDispatched { reason: String } + | CreationFailed { exit_code: Int, stderr: String } + | CreationUnconfirmed { exit_code: Int, stdout: String, stderr: String } + | Removed { dir: NonEmptyStr } + | RemovalFailed { dir: NonEmptyStr, exit_code: Int, stderr: String } + | RemovalUnconfirmed { dir: NonEmptyStr, exit_code: Int, stderr: String } + | RemovalNotDispatched { dir: NonEmptyStr, reason: String } + +type WorkspaceAcquisition = WorkspaceAcquired { dir: NonEmptyStr } - | WorkspaceRefused { reason: String } - -// mktemp prints the created path. Anything but exit 0 and one path under the template's prefix is a -// refusal: a path elsewhere is not the directory this attempt asked to own. -fn member_workspace_of(answer: RemoteAnswer) -> MemberWorkspace { - let dir = trim(s: answer.stdout) - if answer.exit_code != 0 { - WorkspaceRefused { reason: concat("mktemp -d exit=", to_string(answer.exit_code), " ", trim(s: answer.stderr)) } - } else if !starts_with(s: dir, prefix: census_member_workspace_prefix) || string_contains(s: dir, pattern: "\n") || string_contains(s: dir, pattern: "/..") { - WorkspaceRefused { reason: concat("mktemp -d printed a path outside ", census_member_workspace_prefix, ": ", dir) } + | WorkspaceNotAcquired { outcome: WorkspaceOutcome } + +fn admitted_workspace_path(stdout: String) -> String? { + let dir = trim(s: stdout) + if !starts_with(s: dir, prefix: census_member_workspace_prefix) || dir == census_member_workspace_prefix + || string_contains(s: dir, pattern: "\n") || string_contains(s: dir, pattern: " ") || string_contains(s: dir, pattern: "/..") { + none } else { - WorkspaceAcquired { dir: dir as NonEmptyStr } + Present { value: dir } } } -type WorkspaceCleanup - = WorkspaceRemoved { dir: NonEmptyStr } - | WorkspaceNotRemoved { dir: NonEmptyStr, reason: String } - | WorkspaceNeverCreated +fn workspace_acquisition_of(leg: RemoteLeg) -> WorkspaceAcquisition { + match leg { + LegNotDispatched { reason: r } => WorkspaceNotAcquired { outcome: CreationNotDispatched { reason: r } } + LegAnswered { exit_code: c, stdout: o, stderr: e } => + if c == ssh_client_error_exit_status { + WorkspaceNotAcquired { outcome: CreationUnconfirmed { exit_code: c, stdout: o, stderr: e } } + } else if c != 0 { + WorkspaceNotAcquired { outcome: CreationFailed { exit_code: c, stderr: e } } + } else { + match admitted_workspace_path(stdout: o) { + Absent => WorkspaceNotAcquired { outcome: CreationUnconfirmed { exit_code: c, stdout: o, stderr: e } } + Present { value: d } => WorkspaceAcquired { dir: d as NonEmptyStr } + } + } + } +} -fn workspace_cleanup_of(dir: NonEmptyStr, answer: RemoteAnswer) -> WorkspaceCleanup { - if answer.exit_code == 0 { WorkspaceRemoved { dir: dir } } else { - WorkspaceNotRemoved { dir: dir, reason: concat("rm -rf --one-file-system exit=", to_string(answer.exit_code), " ", trim(s: answer.stderr)) } +fn workspace_removal_of(dir: NonEmptyStr, leg: RemoteLeg) -> WorkspaceOutcome { + match leg { + LegNotDispatched { reason: r } => RemovalNotDispatched { dir: dir, reason: r } + LegAnswered { exit_code: c, stdout: _, stderr: e } => + if c == 0 { Removed { dir: dir } } + else if c == ssh_client_error_exit_status { RemovalUnconfirmed { dir: dir, exit_code: c, stderr: e } } + else { RemovalFailed { dir: dir, exit_code: c, stderr: e } } } } -// THE ATTEMPT: the readback and its cleanup, both retained. It holds only when the boot chain is -// identical to stock AND this attempt's workspace was removed. +// THE ATTEMPT: the readback and the workspace's outcome, both retained. It holds only when the boot +// chain is identical to stock AND this attempt's workspace was removed. type CensusMemberReadbackAttempt { readback: CensusMemberReadback - cleanup: WorkspaceCleanup + workspace: WorkspaceOutcome } fn census_member_attempt_holds(attempt: CensusMemberReadbackAttempt) -> Bool { - census_member_readback_holds(readback: attempt.readback) && match attempt.cleanup { - WorkspaceRemoved { dir: _ } => true + census_member_readback_holds(readback: attempt.readback) && match attempt.workspace { + Removed { dir: _ } => true _ => false } } -fn render_workspace_cleanup(c: WorkspaceCleanup) -> String { - match c { - WorkspaceRemoved { dir: d } => concat("workspace=", d as String, " removed\n") - WorkspaceNotRemoved { dir: d, reason: r } => concat("workspace=", d as String, " NOT REMOVED: ", r, "\n") - WorkspaceNeverCreated => "workspace=never created (nothing extracted)\n" +fn render_workspace_outcome(w: WorkspaceOutcome) -> String { + match w { + CreationNotRequested => "workspace=never created: creation was not requested (nothing extracted)\n" + CreationNotDispatched { reason: r } => concat("workspace=never created: mktemp was not dispatched (nothing extracted): ", r, "\n") + CreationFailed { exit_code: c, stderr: e } => concat("workspace=never created: mktemp reported failure exit=", to_string(c), " ", trim(s: e), " (nothing extracted)\n") + CreationUnconfirmed { exit_code: c, stdout: o, stderr: e } => + concat("workspace=CREATION UNCONFIRMED exit=", to_string(c), ": a directory under ", census_member_workspace_prefix, " MAY REMAIN on srv2 (residue outstanding; not removed, nothing extracted); returned stdout=\"", o, "\" stderr=\"", trim(s: e), "\"\n") + Removed { dir: d } => concat("workspace=", d as String, " removed\n") + RemovalFailed { dir: d, exit_code: c, stderr: e } => concat("workspace=", d as String, " REMOVAL FAILED exit=", to_string(c), " ", trim(s: e), " (the path may remain)\n") + RemovalUnconfirmed { dir: d, exit_code: c, stderr: e } => concat("workspace=", d as String, " REMOVAL UNCONFIRMED exit=", to_string(c), " ", trim(s: e), " (the reply was lost; the path may or may not remain)\n") + RemovalNotDispatched { dir: d, reason: r } => concat("workspace=", d as String, " NOT REMOVED: rm was not dispatched (residue outstanding): ", r, "\n") } } fn render_census_member_attempt(attempt: CensusMemberReadbackAttempt) -> String { - concat(render_census_member_readback(readback: attempt.readback), render_workspace_cleanup(c: attempt.cleanup)) + concat(render_census_member_readback(readback: attempt.readback), render_workspace_outcome(w: attempt.workspace)) +} + +// THE ATTEMPT'S DECISIONS AS ONE FOLD over the legs that were issued, so the consumer-level controls +// exercise the same composition the producer runs. `readings` is consulted only when a workspace was +// acquired; `cleanup` is consulted only then too. +fn census_member_attempt_of(gate: CensusPinGate, creation: RemoteLeg, readings: fn(NonEmptyStr) -> CensusMemberReadings, cleanup: fn(NonEmptyStr) -> RemoteLeg) -> CensusMemberReadbackAttempt { + match gate { + PinsRefused { readback: r } => CensusMemberReadbackAttempt { readback: r, workspace: CreationNotRequested } + PinsAdmitted { stock_iso: s, census_iso: c } => + match workspace_acquisition_of(leg: creation) { + WorkspaceNotAcquired { outcome: o } => + CensusMemberReadbackAttempt { readback: BootChainWorkspaceUnavailable { reason: render_workspace_outcome(w: o) }, workspace: o } + WorkspaceAcquired { dir: dir } => { + let readback = census_member_verdict(stock_iso: s, census_iso: c, readings: readings(dir)) + CensusMemberReadbackAttempt { readback: readback, workspace: workspace_removal_of(dir: dir, leg: cleanup(dir)) } + } + } + } } fn member_scratch_path(dir: NonEmptyStr, side: IsoSide, member: CensusIsoMember) -> NonEmptyStr { @@ -386,25 +464,25 @@ fn read_member_remote(target: SshTarget, context: FleetSshExecutionContext, dir: } } -fn census_member_readback_in_workspace(target: SshTarget, context: FleetSshExecutionContext, dir: NonEmptyStr, stock_path: NonEmptyStr, census_path: NonEmptyStr, stock_iso: NonEmptyStr, census_iso: NonEmptyStr) -> CensusMemberReadbackAttempt { - let readback = census_member_verdict(stock_iso: stock_iso, census_iso: census_iso, readings: CensusMemberReadings { +fn census_member_readings_in(target: SshTarget, context: FleetSshExecutionContext, dir: NonEmptyStr, stock_path: NonEmptyStr, census_path: NonEmptyStr) -> CensusMemberReadings { + CensusMemberReadings { stock_grub: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: GrubConfig), census_grub: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: GrubConfig), stock_kernel: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: CasperKernel), census_kernel: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: CasperKernel), stock_initrd: read_member_remote(target: target, context: context, dir: dir, iso_path: stock_path, side: StockSide, member: CasperInitrd), census_initrd: read_member_remote(target: target, context: context, dir: dir, iso_path: census_path, side: CensusSide, member: CasperInitrd), - }) - CensusMemberReadbackAttempt { - readback: readback, - cleanup: workspace_cleanup_of(dir: dir, answer: remote(target: target, context: context, command: rm_recursive_force_one_file_system_command(path: dir as String))), } } +fn remote_leg(target: SshTarget, context: FleetSshExecutionContext, command: ArgvCommand) -> RemoteLeg { + remote_leg_of(outcome: typed_argv_exec_over_fleet_ssh(target: target, context: context, argv: argv_words(command: command))) +} + fn census_member_readback(medium: MtCollins1BootMedium) -> CensusMemberReadbackAttempt { match prepare_fleet_ssh_agent_context(attempt_raw: census_member_readback_attempt_raw) { FleetSshContextRefused { cause: c } => - CensusMemberReadbackAttempt { readback: BootChainHostUnreached { reason: c }, cleanup: WorkspaceNeverCreated } + CensusMemberReadbackAttempt { readback: BootChainHostUnreached { reason: c }, workspace: CreationNotRequested } FleetSshContextReady { context: context, receipt: _ } => { let target = census_member_readback_target() let stock_path = mtcollins1_census_image_stock_path() @@ -415,14 +493,14 @@ fn census_member_readback(medium: MtCollins1BootMedium) -> CensusMemberReadbackA census_iso: iso_reading_of(measure: remote(target: target, context: context, command: sha256sum_file_command(path: census_path as String))), ) match gate { - PinsRefused { readback: r } => CensusMemberReadbackAttempt { readback: r, cleanup: WorkspaceNeverCreated } - PinsAdmitted { stock_iso: s, census_iso: c } => - match member_workspace_of(answer: remote(target: target, context: context, command: mktemp_directory_command(template: census_member_workspace_template))) { - WorkspaceRefused { reason: r } => - CensusMemberReadbackAttempt { readback: BootChainWorkspaceUnavailable { reason: r }, cleanup: WorkspaceNeverCreated } - WorkspaceAcquired { dir: dir } => - census_member_readback_in_workspace(target: target, context: context, dir: dir, stock_path: stock_path, census_path: census_path, stock_iso: s, census_iso: c) - } + PinsRefused { readback: r } => CensusMemberReadbackAttempt { readback: r, workspace: CreationNotRequested } + PinsAdmitted { stock_iso: _, census_iso: _ } => + census_member_attempt_of( + gate: gate, + creation: remote_leg(target: target, context: context, command: mktemp_directory_command(template: census_member_workspace_template)), + readings: dir => census_member_readings_in(target: target, context: context, dir: dir, stock_path: stock_path, census_path: census_path), + cleanup: dir => remote_leg(target: target, context: context, command: rm_recursive_force_one_file_system_command(path: dir as String)), + ) } } } diff --git a/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag b/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag index 0b5a88dbe7e..cbce7165301 100644 --- a/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag +++ b/dag/test/claim/machine_intake/kernel_module_decompression_observation_witness_test.dag @@ -144,7 +144,7 @@ test fn a_malformed_refusal_record_alone_is_still_a_refusal_with_its_line_kept() ]) { RefusalRecorded { refusals: rs, unparsed_refusal_records: urs, insert_failures: _, unparsed_insert_records: _, identity: _ } => count(rs) == 0 && count(urs) == 2 - && match first(urs) { Absent => false Present { value: u } => u.field == "status" && string_contains(s: u.line, pattern: "failed with status") } + && match first(urs) { Absent => false Present { value: u } => u.field == "status" && u.at == "10.374590" && string_contains(s: u.line, pattern: "failed with status") } _ => false } } diff --git a/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag b/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag index 41f7daae6b2..137d946d20f 100644 --- a/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag +++ b/dag/test/claim/machine_intake/mtcollins1_census_member_readback_witness_test.dag @@ -9,12 +9,15 @@ import gunbc.machine_intake_mtcollins1_census_member_readback { CensusMemberReadback, BootChainIdenticalToStock, BootChainFirstDifference, BootChainControlDidNotDiscriminate, BootChainIsoNotPinned, BootChainUnread, BootChainMediumNotCensus, CasperKernel, CasperInitrd, GrubConfig, StockSide, CensusSide, - census_pin_gate, PinsAdmitted, PinsRefused, - member_workspace_of, WorkspaceAcquired, WorkspaceRefused, - workspace_cleanup_of, WorkspaceRemoved, WorkspaceNotRemoved, WorkspaceNeverCreated, + census_pin_gate, PinsAdmitted, PinsRefused, CensusPinGate, CensusMemberReadings, + RemoteLeg, remote_leg_of, census_member_attempt_of, + WorkspaceOutcome, CreationNotRequested, CreationNotDispatched, CreationFailed, CreationUnconfirmed, + Removed, RemovalFailed, RemovalUnconfirmed, RemovalNotDispatched, CensusMemberReadbackAttempt, census_member_attempt_holds, render_census_member_attempt, - member_reading_after_extract, member_reading_of_measure, member_scratch_path, BootChainWorkspaceUnavailable, + member_reading_after_extract, member_reading_of_measure, BootChainWorkspaceUnavailable, } +import gunbc.typed_argv_exec { TypedArgvExecConverged, TypedArgvExecRefused } +import extdeps.ssh.session { SshSessionExecResult } import gunbc.machine_intake_mtcollins1_census_medium_readback { RemoteAnswer } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -65,7 +68,7 @@ test fn a_differing_kernel_is_named_before_a_differing_initrd() -> Bool { } } -// RED: an equal grub.cfg means the census image was not read; identical kernel and initrd do not rescue it. +// RED: an equal grub.cfg means the difference control failed (cause not chosen); identical kernel and initrd do not rescue it. test fn an_equal_grub_config_refuses_as_a_control_that_did_not_discriminate() -> Bool { match over(stock_grub: read(d: "g0"), census_grub: read(d: "g0"), census_kernel: read(d: "k0"), census_initrd: read(d: "i0")) { BootChainControlDidNotDiscriminate { grub_digest: _ } => true @@ -126,19 +129,6 @@ test fn an_off_pin_iso_is_refused_by_the_gate_that_precedes_extraction() -> Bool } } -// THE WORKSPACE IS OWNED OR ABSENT: mktemp's printed path under the template prefix is admitted; -// a failed mktemp, or a path anywhere else, is a refusal -- and no extraction path is derived from it. -test fn a_workspace_is_admitted_only_from_a_successful_mktemp_under_the_prefix() -> Bool { - match member_workspace_of(answer: answer(code: 0, stdout: "/tmp/gunbc-mtcollins1-census-member.Ab12Cd34\n", stderr: "")) { - WorkspaceAcquired { dir: d } => (d as String) == "/tmp/gunbc-mtcollins1-census-member.Ab12Cd34" - && (member_scratch_path(dir: d, side: CensusSide, member: CasperInitrd) as String) == "/tmp/gunbc-mtcollins1-census-member.Ab12Cd34/census-initrd" - _ => false - } - && match member_workspace_of(answer: answer(code: 1, stdout: "", stderr: "mktemp: failed to create directory")) { WorkspaceRefused { reason: _ } => true _ => false } - && match member_workspace_of(answer: answer(code: 0, stdout: "/srv/bmc/x", stderr: "")) { WorkspaceRefused { reason: _ } => true _ => false } - && match member_workspace_of(answer: answer(code: 0, stdout: "/tmp/gunbc-mtcollins1-census-member.X/../../srv/bmc", stderr: "")) { WorkspaceRefused { reason: _ } => true _ => false } -} - // A FAILED EXTRACTION IS NEVER HASHED: its reading is decided from the extract leg alone. test fn a_failed_extraction_is_unread_without_a_measure() -> Bool { match member_reading_after_extract(extract: answer(code: 1, stdout: "", stderr: "xorriso : FAILURE")) { @@ -152,25 +142,113 @@ test fn a_failed_extraction_is_unread_without_a_measure() -> Bool { } } -// THE ATTEMPT HOLDS ONLY WITH ITS WORKSPACE REMOVED, and a cleanup failure is reported BESIDE the -// readback, never in place of it: both the verdict and the cleanup outcome survive in the receipt. -test fn a_cleanup_failure_is_retained_beside_the_verdict_and_blocks_holding() -> Bool { - let identical = over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: read(d: "i0")) - let removed = CensusMemberReadbackAttempt { readback: identical, cleanup: workspace_cleanup_of(dir: "/tmp/gunbc-mtcollins1-census-member.A", answer: answer(code: 0, stdout: "", stderr: "")) } - let stuck = CensusMemberReadbackAttempt { readback: identical, cleanup: workspace_cleanup_of(dir: "/tmp/gunbc-mtcollins1-census-member.A", answer: answer(code: 1, stdout: "", stderr: "Device or resource busy")) } - let text = render_census_member_attempt(attempt: stuck) - census_member_attempt_holds(attempt: removed) - && census_member_attempt_holds(attempt: stuck) == false - && string_contains(s: text, pattern: "verdict=boot_chain_identical_to_stock") - && string_contains(s: text, pattern: "NOT REMOVED") -} - -// AN INITIATING FAILURE AND ITS CLEANUP ARE BOTH KEPT: an unread member with a removed workspace -// renders the unread verdict and the removal; no workspace means "never created", not "removed". -test fn an_initiating_failure_keeps_its_verdict_and_its_cleanup_outcome() -> Bool { - let unread = over(stock_grub: read(d: "g0"), census_grub: read(d: "g1"), census_kernel: read(d: "k0"), census_initrd: MemberUnread { reason: "xorriso extract exit=1" }) - let text = render_census_member_attempt(attempt: CensusMemberReadbackAttempt { readback: unread, cleanup: WorkspaceRemoved { dir: "/tmp/gunbc-mtcollins1-census-member.B" } }) - let never = render_census_member_attempt(attempt: CensusMemberReadbackAttempt { readback: BootChainWorkspaceUnavailable { reason: "mktemp exit=1" }, cleanup: WorkspaceNeverCreated }) - string_contains(s: text, pattern: "verdict=unread") && string_contains(s: text, pattern: "removed") - && string_contains(s: never, pattern: "workspace_unavailable (nothing extracted)") && string_contains(s: never, pattern: "never created") +// ---- the attempt consumer: every workspace outcome, composed as the producer composes it ---- + +fn ran(code: Int, stdout: String, stderr: String) -> RemoteLeg { + remote_leg_of(outcome: TypedArgvExecConverged { result: SshSessionExecResult { exit_code: code, success: code == 0, stdout: stdout, stderr: stderr } }) +} + +fn not_dispatched(reason: String) -> RemoteLeg { remote_leg_of(outcome: TypedArgvExecRefused { reason: reason }) } + +data admitted: CensusPinGate = PinsAdmitted { stock_iso: stock_iso, census_iso: census_iso } + +data workspace_dir: String = "/tmp/gunbc-mtcollins1-census-member.Ab12Cd34" + +// Supplied member readings for an acquired workspace: an identical chain, so a non-holding attempt is +// attributable to its workspace outcome alone. +fn identical_readings(dir: NonEmptyStr) -> CensusMemberReadings { + CensusMemberReadings { + stock_grub: read(d: "g0"), census_grub: read(d: "g1"), + stock_kernel: read(d: "k0"), census_kernel: read(d: "k0"), + stock_initrd: read(d: "i0"), census_initrd: read(d: "i0"), + } +} + +fn attempt(gate: CensusPinGate, creation: RemoteLeg, cleanup: RemoteLeg) -> CensusMemberReadbackAttempt { + census_member_attempt_of(gate: gate, creation: creation, readings: d => identical_readings(dir: d), cleanup: d => cleanup) +} + +fn says(a: CensusMemberReadbackAttempt, pattern: String) -> Bool { string_contains(s: render_census_member_attempt(attempt: a), pattern: pattern) } + +// POSITIVE: an admitted path, an identical chain, a confirmed removal -- the only holding attempt. +test fn an_acquired_workspace_that_is_removed_holds() -> Bool { + let a = attempt(gate: admitted, creation: ran(code: 0, stdout: concat(workspace_dir, "\n"), stderr: ""), cleanup: ran(code: 0, stdout: "", stderr: "")) + census_member_attempt_holds(attempt: a) + && says(a: a, pattern: "verdict=boot_chain_identical_to_stock") + && says(a: a, pattern: concat("workspace=", concat(workspace_dir, " removed"))) +} + +// A PIN REFUSAL ISSUES NO CREATION: the creation leg is never consulted, and "never created" is +// justified because it was never requested. +test fn a_pin_refusal_requests_no_creation() -> Bool { + let refused = census_pin_gate(medium: census, stock_iso: IsoMeasured { digest: stock_iso }, census_iso: IsoMeasured { digest: stock_iso }) + let a = attempt(gate: refused, creation: ran(code: 0, stdout: workspace_dir, stderr: ""), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: a) + && match a.workspace { CreationNotRequested => true _ => false } + && says(a: a, pattern: "creation was not requested") +} + +// ssh 255 WITH NO PATH: mktemp was sent and its result is unknown -- residue outstanding, never +// "never created", and nothing extracted. +test fn an_unacknowledged_creation_without_a_path_is_unconfirmed_not_never_created() -> Bool { + let a = attempt(gate: admitted, creation: ran(code: 255, stdout: "", stderr: "Connection reset by peer"), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: a) + && match a.workspace { CreationUnconfirmed { exit_code: c, stdout: _, stderr: _ } => c == 255 _ => false } + && match a.readback { BootChainWorkspaceUnavailable { reason: _ } => true _ => false } + && says(a: a, pattern: "CREATION UNCONFIRMED") && says(a: a, pattern: "MAY REMAIN") + && !says(a: a, pattern: "never created") +} + +// ssh 255 WITH A PARTIAL PATH: the returned text is retained as evidence and is NOT admitted as a +// workspace -- nothing is extracted under it and nothing deletes it. +test fn an_unacknowledged_creation_with_a_partial_path_keeps_the_text_and_admits_nothing() -> Bool { + let a = attempt(gate: admitted, creation: ran(code: 255, stdout: "/tmp/gunbc-mtcollins1-census-mem", stderr: "client_loop: send disconnect"), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: a) + && match a.workspace { CreationUnconfirmed { exit_code: _, stdout: o, stderr: _ } => o == "/tmp/gunbc-mtcollins1-census-mem" _ => false } + && says(a: a, pattern: "returned stdout=\"/tmp/gunbc-mtcollins1-census-mem\"") + && says(a: a, pattern: "not removed, nothing extracted") + && !says(a: a, pattern: "workspace=/tmp") +} + +// EXIT 0 WITH UNUSABLE OUTPUT: two paths, or a path outside the prefix, establish no owned +// workspace and no non-creation either. +test fn a_zero_exit_with_unusable_output_is_unconfirmed() -> Bool { + let two = attempt(gate: admitted, creation: ran(code: 0, stdout: concat(workspace_dir, concat("\n", workspace_dir)), stderr: ""), cleanup: ran(code: 0, stdout: "", stderr: "")) + let elsewhere = attempt(gate: admitted, creation: ran(code: 0, stdout: "/srv/bmc/x", stderr: ""), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: two) && !census_member_attempt_holds(attempt: elsewhere) + && match two.workspace { CreationUnconfirmed { exit_code: c, stdout: _, stderr: _ } => c == 0 _ => false } + && match elsewhere.workspace { CreationUnconfirmed { exit_code: _, stdout: _, stderr: _ } => true _ => false } +} + +// A LEGITIMATE mktemp FAILURE (its own non-255 status) established that nothing was created; a leg +// the transport refused before dispatch is kept as that, not as a lost reply. +test fn a_reported_mktemp_failure_and_an_undispatched_creation_are_never_created() -> Bool { + let failed = attempt(gate: admitted, creation: ran(code: 1, stdout: "", stderr: "mktemp: failed to create directory via template"), cleanup: ran(code: 0, stdout: "", stderr: "")) + let undispatched = attempt(gate: admitted, creation: not_dispatched(reason: "remote words are not portable"), cleanup: ran(code: 0, stdout: "", stderr: "")) + !census_member_attempt_holds(attempt: failed) && !census_member_attempt_holds(attempt: undispatched) + && match failed.workspace { CreationFailed { exit_code: c, stderr: _ } => c == 1 _ => false } + && says(a: failed, pattern: "never created: mktemp reported failure") + && match undispatched.workspace { CreationNotDispatched { reason: _ } => true _ => false } + && says(a: undispatched, pattern: "mktemp was not dispatched") +} + +// CLEANUP REPLY LOSS BESIDE A RETAINED COMPARISON: the identical-chain verdict survives, the removal +// is UNCONFIRMED (not "remains"), and the attempt does not hold. +test fn a_lost_cleanup_reply_is_unconfirmed_beside_the_retained_verdict() -> Bool { + let a = attempt(gate: admitted, creation: ran(code: 0, stdout: workspace_dir, stderr: ""), cleanup: ran(code: 255, stdout: "", stderr: "Connection closed")) + !census_member_attempt_holds(attempt: a) + && census_member_readback_holds(readback: a.readback) + && says(a: a, pattern: "verdict=boot_chain_identical_to_stock") + && says(a: a, pattern: "REMOVAL UNCONFIRMED") && says(a: a, pattern: "may or may not remain") +} + +// rm's OWN failure and an undispatched rm are distinct from a lost reply. +test fn a_reported_rm_failure_and_an_undispatched_rm_are_distinct_from_a_lost_reply() -> Bool { + let failed = attempt(gate: admitted, creation: ran(code: 0, stdout: workspace_dir, stderr: ""), cleanup: ran(code: 1, stdout: "", stderr: "Device or resource busy")) + let undispatched = attempt(gate: admitted, creation: ran(code: 0, stdout: workspace_dir, stderr: ""), cleanup: not_dispatched(reason: "no transport")) + !census_member_attempt_holds(attempt: failed) && !census_member_attempt_holds(attempt: undispatched) + && match failed.workspace { RemovalFailed { dir: _, exit_code: c, stderr: _ } => c == 1 _ => false } + && says(a: failed, pattern: "REMOVAL FAILED") + && match undispatched.workspace { RemovalNotDispatched { dir: _, reason: _ } => true _ => false } + && says(a: undispatched, pattern: "rm was not dispatched") }