From 6a0d5c6cafb3ea57aade98d56db1f188448f33b1 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 02:32:20 +0000 Subject: [PATCH 01/36] microVM guest network: tap-grain default-deny egress, slot network readback, workspace staging gate - runner_microvm_network: tap-scoped egress rules in the forward chain (established; v6 drop; private/link-local/multicast denials above the grants; DNS and TCP 443; final tap drop). Chain policy stays accept for dockerd coexistence, so runner_host_filtered_egress DefaultDrop is honoured at tap grain, not host grain. - SlotNetworkReadback (Confirmed / Refuted{fact} / Unobservable{fact}) over tap unheld, guest conntrack empty after flush, and nft listing digest == converged digest; plus the conntrack flush/list and nft list commands that produce the observations. - runner_attempt_launch: staging_verdict gates the jailer on a read-back workspace (create exit, mke2fs exit, dumpe2fs superblock magic and size); every failure refuses under its own name. - extdeps: nft daddr/dport/nfproto matches, nft list table, conntrack(8), dumpe2fs -h. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/exec/command.dag | 4 + dag/extdeps/nftables/ruleset.dag | 38 ++- dag/extdeps/tools/conntrack.dag | 46 ++++ dag/extdeps/tools/e2fsprogs.dag | 12 + dag/gunbc/runner/runner_attempt_launch.dag | 106 +++++++- dag/gunbc/runner/runner_microvm_network.dag | 251 +++++++++++++++++- .../runner_attempt_launch_witness_test.dag | 71 +++++ .../runner_microvm_network_witness_test.dag | 131 +++++++++ 8 files changed, 647 insertions(+), 12 deletions(-) create mode 100644 dag/extdeps/tools/conntrack.dag diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index eac4dd24b28..b9073b3daac 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -91,6 +91,10 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.tools.squashfs_tools", decl_name: "unsquashfs_extract_command"), decl_ref(module_path: "extdeps.tools.e2fsprogs", decl_name: "mke2fs_populate_command"), decl_ref(module_path: "extdeps.tools.e2fsprogs", decl_name: "mke2fs_format_command"), + decl_ref(module_path: "extdeps.tools.e2fsprogs", decl_name: "dumpe2fs_superblock_command"), + decl_ref(module_path: "extdeps.nftables.ruleset", decl_name: "nft_list_table_command"), + decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_delete_by_source_command"), + decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_list_by_source_command"), decl_ref(module_path: "extdeps.tools.make", decl_name: "make_command"), decl_ref(module_path: "extdeps.package_managers.pip", decl_name: "pip_install_pinned_command"), decl_ref(module_path: "extdeps.package_managers.conda_forge", decl_name: "micromamba_create_command"), diff --git a/dag/extdeps/nftables/ruleset.dag b/dag/extdeps/nftables/ruleset.dag index ac859ca3cf1..883f0ca92ba 100644 --- a/dag/extdeps/nftables/ruleset.dag +++ b/dag/extdeps/nftables/ruleset.dag @@ -1,8 +1,9 @@ module extdeps.nftables.ruleset -import std.types { String, NonEmptyStr, List } +import std.types { String, NonEmptyStr, List, Int } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } +import extdeps.exec.command { ArgvCommand, argv_command } data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -105,16 +106,36 @@ fn nft_verdict_wire(v: NftVerdict) -> String { } } +// THE TRANSPORT IS A CLOSED PAIR because `dport` is a field of the transport header, not of the +// packet: nft refuses `dport 443` with no protocol, so the protocol travels with the port. +type NftTransport + = NftTcp + | NftUdp + +fn nft_transport_wire(t: NftTransport) -> String { + match t { + NftTcp => "tcp" + NftUdp => "udp" + } +} + // A MATCH IS A LIST OF EXPRESSIONS AND THE ORDER IS THE WIRE ORDER, because nftables evaluates them // left to right and a rule reads as its own sentence. The expressions here are the ones this fleet // matches on; each carries its operand rather than a pre-rendered string, so a device name or a // subnet cannot arrive already quoted or already wrong. +// +// `meta nfproto ipv6` EXISTS FOR THE inet FAMILY, where one chain sees both address families and an +// `ip daddr` match is simply false on a v6 packet -- so a rule list built only from v4 matches lets +// v6 fall through to whatever follows. Naming the family is how a chain refuses it outright. type NftMatch = NftInputInterface { device: NonEmptyStr } | NftOutputInterface { device: NonEmptyStr } | NftInputInterfaceWildcard { prefix: NonEmptyStr } | NftOutputInterfaceWildcard { prefix: NonEmptyStr } | NftSourceAddressPrefix { cidr: NonEmptyStr } + | NftDestinationAddressPrefix { cidr: NonEmptyStr } + | NftTransportDestinationPort { transport: NftTransport, port: Int } + | NftNetworkProtocolIpv6 | NftConnectionTrackingEstablished fn nft_match_wire(m: NftMatch) -> String { @@ -124,6 +145,10 @@ fn nft_match_wire(m: NftMatch) -> String { NftInputInterfaceWildcard { prefix: p } => join(["iifname \"", p as String, "*\""], "") NftOutputInterfaceWildcard { prefix: p } => join(["oifname \"", p as String, "*\""], "") NftSourceAddressPrefix { cidr: c } => join(["ip saddr ", c as String], "") + NftDestinationAddressPrefix { cidr: c } => join(["ip daddr ", c as String], "") + NftTransportDestinationPort { transport: t, port: n } => + join([nft_transport_wire(t: t), " dport ", to_string(n)], "") + NftNetworkProtocolIpv6 => "meta nfproto ipv6" NftConnectionTrackingEstablished => "ct state established,related" } } @@ -187,3 +212,14 @@ fn nft_chain_wire(family: NftFamily, c: NftChain) -> String { " }\n", ], "") } + +data nft_program: NonEmptyStr = "nft" + +// THE LISTING IS THE READBACK, NOT THE FILE WE APPLIED. The kernel holds the ruleset in its own +// representation and `nft list table` renders that, normalized: a rule written as `ip saddr +// 172.30.0.0/16` may come back with its operands reordered or its defaults spelled. So an applied +// file and a listing differ in bytes while agreeing in meaning, and a digest comparison is only +// honest between two LISTINGS. +fn nft_list_table_command(family: NftFamily, name: NonEmptyStr) -> ArgvCommand { + argv_command(program: nft_program, arguments: ["list", "table", nft_family_wire(f: family), name as String]) +} diff --git a/dag/extdeps/tools/conntrack.dag b/dag/extdeps/tools/conntrack.dag new file mode 100644 index 00000000000..5d640e7c5a5 --- /dev/null +++ b/dag/extdeps/tools/conntrack.dag @@ -0,0 +1,46 @@ +module extdeps.tools.conntrack + +import std.types { String, NonEmptyStr, List } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Uri, Https } +import extdeps.exec.command { ArgvCommand, argv_command } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "manpages.debian.org/bookworm/conntrack/conntrack.8.en.html" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.tools.conntrack", + decl_name: "conntrack_program", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +// conntrack(8) from conntrack-tools, the userspace reader and writer of the kernel's connection +// tracking table. Only the two operations a table keyed by ORIGINAL SOURCE needs are modeled: delete +// the flows a source opened, and list them. `-s` filters on the original-direction source, which for +// a masqueraded guest is the guest's own address -- the translated address appears only on the +// reply tuple, so filtering on it would miss every flow a guest opened. +data conntrack_program: NonEmptyStr = "conntrack" + +// `-D` WITH NO MATCHING ENTRY EXITS NONZERO ("0 flow entries have been deleted"), so its exit status +// is not a success signal a caller may gate on. The deletion's outcome is established by the list +// that follows it, never by this command's exit. +fn conntrack_delete_by_source_command(source: String) -> ArgvCommand { + argv_command(program: conntrack_program, arguments: ["-D", "-s", source]) +} + +// `-L` prints one line per flow on stdout and a summary on stderr, so the count of stdout lines is +// the number of surviving flows and an empty stdout with exit zero is the empty reading. +fn conntrack_list_by_source_command(source: String) -> ArgvCommand { + argv_command(program: conntrack_program, arguments: ["-L", "-s", source]) +} diff --git a/dag/extdeps/tools/e2fsprogs.dag b/dag/extdeps/tools/e2fsprogs.dag index da788a28294..a690745bec7 100644 --- a/dag/extdeps/tools/e2fsprogs.dag +++ b/dag/extdeps/tools/e2fsprogs.dag @@ -53,6 +53,18 @@ fn mke2fs_format_command(fs: Ext2FilesystemType, image_path: String) -> ArgvComm ) } +data dumpe2fs_program: NonEmptyStr = "dumpe2fs" + +// `-h` PRINTS THE SUPERBLOCK AND STOPS, which is the whole readback a freshly formatted image needs: +// the magic number that says an ext filesystem is there, and the block count and block size whose +// product is its size. It reads the image file directly, so no loop device or mount is reached -- +// the same property that makes mke2fs usable unprivileged here. A nonzero exit does NOT by itself +// separate "no ext superblock" from "file unreadable", so a caller reports it as a failed reading and +// never as a filesystem-type fact. +fn dumpe2fs_superblock_command(image_path: String) -> ArgvCommand { + argv_command(program: dumpe2fs_program, arguments: ["-h", image_path]) +} + // `-d` IS WHAT MAKES A ROOT FILESYSTEM BUILDABLE WITHOUT MOUNTING ONE, and that is the reason this // builder exists rather than a mount-copy-unmount sequence. mke2fs populates the new filesystem from // a directory tree in userspace, so the image is produced by one process with no loop device, no diff --git a/dag/gunbc/runner/runner_attempt_launch.dag b/dag/gunbc/runner/runner_attempt_launch.dag index 6d41be2e0f5..87d9eac2042 100644 --- a/dag/gunbc/runner/runner_attempt_launch.dag +++ b/dag/gunbc/runner/runner_attempt_launch.dag @@ -3,9 +3,9 @@ module gunbc.runner_attempt_launch import std.types { String, NonEmptyStr, Bool, Int, List, FilePath } import std.disposition { Disposition, SingleAuthority } import std.dissolution { DissolutionCondition, unbound_dissolution } -import std.measure { Second, second, ByteSize, byte_size, measure_le, Mebibyte, mebibyte } +import std.measure { Second, second, ByteSize, byte_size, byte_size_count, measure_le, Mebibyte, mebibyte, mebibyte_to_byte_size } import extdeps.exec.command { ArgvCommand } -import extdeps.tools.e2fsprogs { Ext2, mke2fs_format_command } +import extdeps.tools.e2fsprogs { Ext2, mke2fs_format_command, dumpe2fs_superblock_command } import extdeps.tools.gnu_coreutils { timeout_command, cp_command } import extdeps.tools.mkdir { mkdir_parents_command } import extdeps.tools.busybox { busybox_dd_sparse_image_command } @@ -462,3 +462,105 @@ type WorkNotEstablished { fn work_standing_from_vmm_exit(exit_code: Int) -> WorkNotEstablished { WorkNotEstablished { vmm_exit: exit_code } } + +// --- the workspace readback: staging admits the jailer or refuses it --- + +// THE PLAN SAYS WHAT TO RUN; THIS SAYS WHETHER WHAT RAN LEFT A WORKSPACE. They are separate +// because they are separated in time: plan_attempt_launch is decided before any host mutation, and +// the dd and mke2fs it plans can fail after it has answered LaunchAuthorized. A guest launched onto +// a workspace that failed to format boots, mounts nothing, and fails the job far from its cause -- +// so the jailer is gated on a reading of the image, not on the plan that asked for it. +// +// THE READING IS INDEPENDENT OF THE EXIT CODES. mke2fs exiting zero is the formatter's claim; the +// superblock read back from the image is the host's observation of the result. Both are required, +// and the readback also carries the size, because a sparse `seek` onto a full disk can leave a file +// shorter than asked without dd failing. +// +// The image lives under the attempt's jail root (runner_microvm_attempt_jail_base), so teardown's +// one subtree to prove absent already contains it; nothing here adds a second location. +type WorkspaceReadback + = WorkspaceSuperblockRead { ext_magic: Bool, bytes: ByteSize } + | WorkspaceReadbackFailed { reason: NonEmptyStr } + +type WorkspaceStagingObservation { + create_exit: Int + format_exit: Int + readback: WorkspaceReadback +} + +type WorkspaceStanding + = WorkspaceReady + | WorkspaceCreateFailed { exit: Int } + | WorkspaceFormatFailed { exit: Int } + | WorkspaceNotExtFilesystem + | WorkspaceSizeMismatch { observed: ByteSize, expected: ByteSize } + | WorkspaceUnreadable { reason: NonEmptyStr } + +fn runner_attempt_workspace_bytes() -> ByteSize { + mebibyte_to_byte_size(m: runner_attempt_workspace_size) +} + +// THE ORDER IS CAUSAL: a failed create makes the format's exit meaningless, and a failed format +// makes the superblock reading meaningless, so the earliest failure is the one reported. +fn workspace_standing(obs: WorkspaceStagingObservation) -> WorkspaceStanding { + if obs.create_exit != 0 { + WorkspaceCreateFailed { exit: obs.create_exit } + } else if obs.format_exit != 0 { + WorkspaceFormatFailed { exit: obs.format_exit } + } else { + match obs.readback { + WorkspaceReadbackFailed { reason: r } => WorkspaceUnreadable { reason: r } + WorkspaceSuperblockRead { ext_magic: m, bytes: b } => + if !m { + WorkspaceNotExtFilesystem + } else if byte_size_count(b: b) != byte_size_count(b: runner_attempt_workspace_bytes()) { + WorkspaceSizeMismatch { observed: b, expected: runner_attempt_workspace_bytes() } + } else { + WorkspaceReady + } + } + } +} + +type AttemptStagingVerdict + = StagingAdmitsJailer { jailer: ArgvCommand } + | StagingRefusedWorkspace { standing: WorkspaceStanding } + | StagingNotAuthorized + +// THE ONLY ROUTE TO THE JAILER COMMAND AFTER STAGING. A refused plan has no jailer to admit, and a +// workspace that is anything but WorkspaceReady refuses -- there is no arm that launches anyway. +fn staging_verdict(plan: AttemptLaunchPlan, obs: WorkspaceStagingObservation) -> AttemptStagingVerdict { + match plan { + LaunchAuthorized { + authorization: _, + tap_device: _, + guest_ip_boot_arg: _, + nft_ruleset: _, + slice: _, + jail_root: _, + stage_jail_root: _, + stage_kernel: _, + stage_rootfs: _, + workspace_path: _, + workspace_create: _, + workspace_format: _, + kvm_device: _, + vm_config: _, + jailer: j, + } => match workspace_standing(obs: obs) { + WorkspaceReady => StagingAdmitsJailer { jailer: j } + WorkspaceCreateFailed { exit: _ } => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } + WorkspaceFormatFailed { exit: _ } => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } + WorkspaceNotExtFilesystem => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } + WorkspaceSizeMismatch { observed: _, expected: _ } => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } + WorkspaceUnreadable { reason: _ } => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } + } + LaunchRefusedUnauthorized { authorization: _ } => StagingNotAuthorized + LaunchRefusedEnvelope { standing: _ } => StagingNotAuthorized + LaunchRefusedPriorResources => StagingNotAuthorized + } +} + +fn workspace_readback_command(workspace_path: String) -> ArgvCommand { + dumpe2fs_superblock_command(image_path: workspace_path) +} diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 74ecbbc6d9f..25a6137257d 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -3,6 +3,7 @@ module gunbc.runner_microvm_network import std.types { String, NonEmptyStr, Bool, Int, List } import std.nat { Nat } import std.disposition { Disposition, SingleAuthority } +import std.content_hash { Sha256Digest } import extdeps.network.ipv4 { Ipv4Address, ipv4_address, render_ipv4_address } import extdeps.network.mac { Eui48Address, mac_address, render_mac_address, MacAddressScope, LocallyAdministered, mac_address_scope } import gunbc.build_cache_instance { RunnerSlotIdentity } @@ -17,6 +18,12 @@ import extdeps.nftables.ruleset { NftVerdict, NftAccept, NftDrop, NftMasquerade, NftInputInterfaceWildcard, NftOutputInterfaceWildcard, NftSourceAddressPrefix, NftConnectionTrackingEstablished, nft_table_wire, + NftDestinationAddressPrefix, NftTransportDestinationPort, NftNetworkProtocolIpv6, + NftTransport, NftTcp, NftUdp, nft_list_table_command, +} +import extdeps.exec.command { ArgvCommand } +import extdeps.tools.conntrack { + conntrack_delete_by_source_command, conntrack_list_by_source_command, } data runner_microvm_network_disposition: Disposition = SingleAuthority @@ -233,18 +240,91 @@ fn runner_microvm_forward_chain() -> NftChain { hook: NftForward, priority: NftPriorityFilter, policy: NftPolicyAccept, - rules: [ - NftRule { - matches: [ - NftInputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, - NftOutputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, - ], - verdict: NftDrop, - }, - ], + rules: concat( + [ + NftRule { + matches: [ + NftInputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, + NftOutputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, + ], + verdict: NftDrop, + }, + ], + runner_microvm_guest_egress_rules(), + ), + } +} + +// --- guest egress: default-deny at TAP grain, not host grain --- + +// THIS IS WHERE gunbc.runner.runner_host_filtered_egress's DefaultDrop STANCE IS HONOURED, AND ITS +// GRAIN IS THE TAP. That module states forward: DefaultDrop for a host this fleet owns entirely. +// srvN is not that host (dockerd's tables, see the chain policy above), so the chain policy stays +// accept and the default-deny is the LAST RULE OF A TAP-SCOPED LIST: every packet a guest sends +// matches a rule written for it or reaches the final drop. Traffic that did not enter through a +// gunbc tap matches none of these rows and keeps whatever verdict the host's other tables give it. +// So the stance holds for guests -- the only principal it was ever about -- and not for the host. +// +// THE ORDER IS THE POLICY. Established first, so replies to admitted flows are never re-judged. The +// v6 refusal before any v4 grant, because in the inet family an `ip daddr` match is false on a v6 +// packet and a v6 flow would otherwise reach the port grants. The private-range denials ABOVE the +// port grants, because the 443 grant is a destination class and 192.168.x.y:443 is in it -- the +// Mt. Collins run (gunbc.runner.runner_filtered_egress_receipt denied_paths) is the receipt that +// this ordering is what stopped a guest reaching its own BMC. The grants are the ones that run +// admitted: name resolution, and TCP 443 for github.com and the Actions endpoints. +// +// The resolver the guest uses must therefore be PUBLIC: a private resolver is denied by the rows +// above the DNS grant, and that is the intended reading of "guest to the selected resolver", not an +// accident of ordering. +data runner_microvm_guest_denied_destinations: List = [ + "10.0.0.0/8" as NonEmptyStr, + "172.16.0.0/12" as NonEmptyStr, + "192.168.0.0/16" as NonEmptyStr, + "100.64.0.0/10" as NonEmptyStr, + "169.254.0.0/16" as NonEmptyStr, + "127.0.0.0/8" as NonEmptyStr, + "224.0.0.0/4" as NonEmptyStr, +] + +data runner_microvm_guest_dns_port: Int = 53 + +data runner_microvm_guest_tls_port: Int = 443 + +fn from_guest_tap(m: List, verdict: NftVerdict) -> NftRule { + NftRule { + matches: concat([NftInputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }], m), + verdict: verdict, } } +fn runner_microvm_guest_egress_rules() -> List { + concat( + concat( + [ + from_guest_tap(m: [NftConnectionTrackingEstablished], verdict: NftAccept), + from_guest_tap(m: [NftNetworkProtocolIpv6], verdict: NftDrop), + ], + map(runner_microvm_guest_denied_destinations, c => + from_guest_tap(m: [NftDestinationAddressPrefix { cidr: c }], verdict: NftDrop)), + ), + [ + from_guest_tap( + m: [NftTransportDestinationPort { transport: NftUdp, port: runner_microvm_guest_dns_port }], + verdict: NftAccept, + ), + from_guest_tap( + m: [NftTransportDestinationPort { transport: NftTcp, port: runner_microvm_guest_dns_port }], + verdict: NftAccept, + ), + from_guest_tap( + m: [NftTransportDestinationPort { transport: NftTcp, port: runner_microvm_guest_tls_port }], + verdict: NftAccept, + ), + from_guest_tap(m: [], verdict: NftDrop), + ], + ) +} + // THE MASQUERADE MATCHES ON SOURCE AND NOT ON OUTPUT INTERFACE, which removes an observation this // module would otherwise have to make and keep correct. srv2's uplink is enP3p3s0f1 and srv1's is // not; naming it would make the ruleset host-specific and would rot the moment an interface is @@ -379,3 +459,156 @@ fn runner_slot_network_install_operations( SystemdReloadUnit { unit: runner_microvm_networkd_unit }, ] } + +// --- the slot network readback: what sanitation reads after an attempt --- + +// THE TAP AND THE RULESET OUTLIVE THE ATTEMPT BY DESIGN, SO SANITATION CANNOT PROVE THEM ABSENT. +// They are slot and host state (the .netdev block above; parent ruling on the microVM program, +// 2026-09-19), and what an attempt can leave behind in them is not their existence but their USE: +// a process still holding the tap, a connection-tracking flow still keyed to the guest address the +// NEXT attempt in this slot will reuse, and a ruleset some process edited during the attempt. Those +// three facts are what this readback reads, and gunbc.runner_microvm_lifecycle consumes it as the +// network fact of its host teardown readback, mapped onto product.fabric.sanitation. +// +// CONNTRACK IS FLUSHED BEFORE IT IS READ, AND THE FLUSH IS NOT THE EVIDENCE. The flush is teardown's +// mutation; the reading after it is the fact. A flush that exited zero while a flow survived -- a +// packet from a still-running guest re-creates one -- is exactly the case the read exists to catch. +// +// THE RULESET DIGEST IS OF `nft list table`, NOT OF THE FILE WE APPLIED. nft normalizes what it is +// given, so the text above and the kernel's listing differ in bytes while agreeing in meaning. The +// converged digest is therefore the listing captured by the converge that installed the table, and +// it is supplied here from that receipt rather than recomputed from the rule rows. +type SlotNetworkFact + = TapUnheldFact + | GuestConntrackEmptyFact + | RulesetMatchesConvergedFact + +type TapHolderObservation + = TapPresentUnheld + | TapHeld { holders: Int } + | TapMissing + | TapHoldersUnreadable { reason: NonEmptyStr } + +type GuestConntrackObservation + = GuestConntrackEmptyAfterFlush + | GuestConntrackResidue { entries: Int } + | GuestConntrackUnreadable { reason: NonEmptyStr } + +type RulesetObservation + = RulesetListed { digest: Sha256Digest } + | RulesetUnreadable { reason: NonEmptyStr } + +// "COULD NOT READ" IS NOT "REFUTED", and the two arms stay separate because they call for different +// operators: a refutation is a fact about the host that needs remedying, an unobservable reading is +// a fact about the instrument. Both quarantine the cell -- neither is CellReady -- but collapsing +// them would send someone to clean a host that may be clean, or to fix a probe on a host that is not. +type SlotNetworkReadback + = SlotNetworkConfirmed + | SlotNetworkRefuted { fact: SlotNetworkFact, detail: NonEmptyStr } + | SlotNetworkUnobservable { fact: SlotNetworkFact, reason: NonEmptyStr } + +fn tap_readback(o: TapHolderObservation) -> SlotNetworkReadback { + match o { + TapPresentUnheld => SlotNetworkConfirmed + TapHeld { holders: n } => SlotNetworkRefuted { + fact: TapUnheldFact, + detail: join([to_string(n), " process(es) still hold the slot tap after the attempt"], "") as NonEmptyStr, + } + TapMissing => SlotNetworkRefuted { + fact: TapUnheldFact, + detail: "the slot tap is absent, so the converged network state no longer holds" as NonEmptyStr, + } + TapHoldersUnreadable { reason: r } => SlotNetworkUnobservable { fact: TapUnheldFact, reason: r } + } +} + +fn conntrack_readback(o: GuestConntrackObservation) -> SlotNetworkReadback { + match o { + GuestConntrackEmptyAfterFlush => SlotNetworkConfirmed + GuestConntrackResidue { entries: n } => SlotNetworkRefuted { + fact: GuestConntrackEmptyFact, + detail: join([to_string(n), " flow(s) keyed to the guest address survived the flush"], "") as NonEmptyStr, + } + GuestConntrackUnreadable { reason: r } => SlotNetworkUnobservable { fact: GuestConntrackEmptyFact, reason: r } + } +} + +fn ruleset_readback(o: RulesetObservation, converged: Sha256Digest) -> SlotNetworkReadback { + match o { + RulesetListed { digest: d } => + if (d.hex as String) == (converged.hex as String) { + SlotNetworkConfirmed + } else { + SlotNetworkRefuted { + fact: RulesetMatchesConvergedFact, + detail: join(["listed ruleset digest ", d.hex as String, " differs from converged ", converged.hex as String], "") as NonEmptyStr, + } + } + RulesetUnreadable { reason: r } => SlotNetworkUnobservable { fact: RulesetMatchesConvergedFact, reason: r } + } +} + +// A REFUTATION ANYWHERE OUTRANKS AN UNREADABLE FACT ELSEWHERE: it is the stronger statement about +// the host, and reporting the probe failure instead would hide a known defect behind an unknown one. +fn first_refuted(rs: List) -> List { + fold(rs, init: [], f: (acc, r) => + match acc { + Cons { head: _, tail: _ } => acc + Empty {} => match r { + SlotNetworkRefuted { fact: _, detail: _ } => [r] + SlotNetworkConfirmed => acc + SlotNetworkUnobservable { fact: _, reason: _ } => acc + } + } + ) +} + +fn first_unobservable(rs: List) -> List { + fold(rs, init: [], f: (acc, r) => + match acc { + Cons { head: _, tail: _ } => acc + Empty {} => match r { + SlotNetworkUnobservable { fact: _, reason: _ } => [r] + SlotNetworkConfirmed => acc + SlotNetworkRefuted { fact: _, detail: _ } => acc + } + } + ) +} + +fn slot_network_readback( + tap: TapHolderObservation, + conntrack: GuestConntrackObservation, + ruleset: RulesetObservation, + converged_ruleset: Sha256Digest, +) -> SlotNetworkReadback { + let facts = [ + tap_readback(o: tap), + conntrack_readback(o: conntrack), + ruleset_readback(o: ruleset, converged: converged_ruleset), + ] + match first_refuted(rs: facts) { + Cons { head: r, tail: _ } => r + Empty {} => match first_unobservable(rs: facts) { + Cons { head: u, tail: _ } => u + Empty {} => SlotNetworkConfirmed + } + } +} + +// --- the commands that produce the readback's observations --- + +// TEARDOWN FLUSHES; THE LIFECYCLE CONTROLLER READS. The flush is a mutation and belongs to the +// teardown step, which runs it before the list. Its exit status is not the evidence -- see +// extdeps.tools.conntrack -- the list is. +fn runner_slot_conntrack_flush_command(net: RunnerSlotNetwork) -> ArgvCommand { + conntrack_delete_by_source_command(source: render_ipv4_address(addr: net.guest_address) as String) +} + +fn runner_slot_conntrack_list_command(net: RunnerSlotNetwork) -> ArgvCommand { + conntrack_list_by_source_command(source: render_ipv4_address(addr: net.guest_address) as String) +} + +fn runner_microvm_nft_list_command() -> ArgvCommand { + nft_list_table_command(family: NftInet, name: runner_microvm_nft_table_name) +} diff --git a/dag/test/claim/runner/runner_attempt_launch_witness_test.dag b/dag/test/claim/runner/runner_attempt_launch_witness_test.dag index c548c9b3f7d..3d6ac0af4af 100644 --- a/dag/test/claim/runner/runner_attempt_launch_witness_test.dag +++ b/dag/test/claim/runner/runner_attempt_launch_witness_test.dag @@ -42,6 +42,11 @@ import gunbc.runner_attempt_launch { reported_attempt_launch_rung, host_boot_cutover_frontier, jit_mint_http_realization_frontier, jail_jit_device_staging_frontier, jail_workspace_path, + WorkspaceStagingObservation, WorkspaceSuperblockRead, WorkspaceReadbackFailed, + AttemptStagingVerdict, StagingAdmitsJailer, StagingRefusedWorkspace, StagingNotAuthorized, + WorkspaceStanding, WorkspaceReady, WorkspaceCreateFailed, WorkspaceFormatFailed, + WorkspaceNotExtFilesystem, WorkspaceSizeMismatch, WorkspaceUnreadable, + staging_verdict, runner_attempt_workspace_bytes, } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -333,3 +338,69 @@ test fn every_x_envelope_refusal_arm_yields_zero_jailer_launches() -> Bool { } } + +fn good_workspace() -> WorkspaceStagingObservation { + WorkspaceStagingObservation { + create_exit: 0, + format_exit: 0, + readback: WorkspaceSuperblockRead { ext_magic: true, bytes: runner_attempt_workspace_bytes() }, + } +} + +fn staging_label(v: AttemptStagingVerdict) -> String { + match v { + StagingAdmitsJailer { jailer: _ } => "admit" + StagingNotAuthorized => "unauthorized" + StagingRefusedWorkspace { standing: s } => match s { + WorkspaceReady => "ready-refused" + WorkspaceCreateFailed { exit: _ } => "create" + WorkspaceFormatFailed { exit: _ } => "format" + WorkspaceNotExtFilesystem => "not-ext" + WorkspaceSizeMismatch { observed: _, expected: _ } => "size" + WorkspaceUnreadable { reason: _ } => "unreadable" + } + } +} + +fn staging_for(obs: WorkspaceStagingObservation, ground: AttemptGround) -> String { + match issued_pairs() { + Cons { head: pair, tail: _ } => + match accepted_attempt(i: 2, id: "ws1") { + Cons { head: a, tail: _ } => staging_label(v: staging_verdict( + plan: plan_for(pair: pair, account: pair.account, attempt: a, envelope: admitted(n: 4096), ground: ground), + obs: obs, + )) + Empty {} => "no-attempt" + } + Empty {} => "no-grant" + } +} + +// THE JAILER IS REACHED ONLY THROUGH A READ-BACK WORKSPACE. The positive control admits; each +// failure refuses under its own name, and the earliest failure is the one named -- a failed create +// with a failed format reports the create. The red is any arm that launches onto a workspace the +// host did not read back as an ext filesystem of the planned size. +test fn the_jailer_is_admitted_only_onto_a_read_back_workspace() -> Bool { + staging_for(obs: good_workspace(), ground: GroundClear) == "admit" + && staging_for(obs: WorkspaceStagingObservation { + create_exit: 1, format_exit: 1, readback: WorkspaceReadbackFailed { reason: "absent" as NonEmptyStr }, + }, ground: GroundClear) == "create" + && staging_for(obs: WorkspaceStagingObservation { + create_exit: 0, format_exit: 1, readback: WorkspaceSuperblockRead { ext_magic: true, bytes: runner_attempt_workspace_bytes() }, + }, ground: GroundClear) == "format" + && staging_for(obs: WorkspaceStagingObservation { + create_exit: 0, format_exit: 0, readback: WorkspaceSuperblockRead { ext_magic: false, bytes: runner_attempt_workspace_bytes() }, + }, ground: GroundClear) == "not-ext" + && staging_for(obs: WorkspaceStagingObservation { + create_exit: 0, format_exit: 0, readback: WorkspaceSuperblockRead { ext_magic: true, bytes: byte_size(count: 1048576) }, + }, ground: GroundClear) == "size" + && staging_for(obs: WorkspaceStagingObservation { + create_exit: 0, format_exit: 0, readback: WorkspaceReadbackFailed { reason: "dumpe2fs exited 1" as NonEmptyStr }, + }, ground: GroundClear) == "unreadable" +} + +// A GOOD WORKSPACE CANNOT BUY A LAUNCH THE PLAN REFUSED. The red is a staging gate that reads only +// the observation: occupied ground would then launch because the image happened to format. +test fn a_good_workspace_does_not_admit_a_refused_plan() -> Bool { + staging_for(obs: good_workspace(), ground: GroundOccupied) == "unauthorized" +} diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index bcbc836d8d9..44da2de6bc5 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -15,7 +15,15 @@ import gunbc.runner_microvm_network { runner_slot_netdev_path, runner_slot_network_file_path, netmask_of_prefix, runner_microvm_network_supernet, runner_microvm_network_prefix_length, runner_microvm_network_supernet_prefix, + SlotNetworkReadback, SlotNetworkConfirmed, SlotNetworkRefuted, SlotNetworkUnobservable, + SlotNetworkFact, TapUnheldFact, GuestConntrackEmptyFact, RulesetMatchesConvergedFact, + TapPresentUnheld, TapHeld, TapMissing, TapHoldersUnreadable, + GuestConntrackEmptyAfterFlush, GuestConntrackResidue, GuestConntrackUnreadable, + RulesetListed, RulesetUnreadable, slot_network_readback, + runner_slot_conntrack_flush_command, runner_slot_conntrack_list_command, } +import std.content_hash { Sha256Digest, Sha256DigestHex } +import extdeps.exec.command { argv_words } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -176,3 +184,126 @@ test fn the_nat_chain_is_typed_nat_and_the_filter_chain_stays_filter() -> Bool { && string_contains(s: text, pattern: "type filter hook forward") && !string_contains(s: text, pattern: "type filter hook postrouting") } + +// THE GUEST EGRESS ORDER IS THE POLICY, so the control is the ORDER and not the presence of words. +// Two adjacencies are pinned. The head: established before anything, and the v6 refusal before any +// v4 grant, because an `ip daddr` match is false on a v6 packet in the inet family. The tail: the +// private-range denials end immediately above the port grants and the list ends in a tap-scoped +// drop. The reds are a 443 grant moved above the private denials -- a guest reaches its own BMC on +// 443, the ordering the Mt. Collins receipt showed holding -- and a missing final drop, which turns +// default-deny into default-accept for every port nobody named. +test fn guest_egress_is_default_deny_at_tap_grain_in_the_receipted_order() -> Bool { + let text = runner_microvm_nft_ruleset_text() + string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" meta nfproto ipv6 drop\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") + && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" udp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") + && string_contains(s: text, pattern: "policy accept") + && !string_contains(s: text, pattern: "policy drop") +} + +data converged_digest: Sha256Digest = Sha256Digest { + hex: "1111111111111111111111111111111111111111111111111111111111111111" as Sha256DigestHex, +} + +data drifted_digest: Sha256Digest = Sha256Digest { + hex: "2222222222222222222222222222222222222222222222222222222222222222" as Sha256DigestHex, +} + +fn refuted_fact(r: SlotNetworkReadback) -> String { + match r { + SlotNetworkRefuted { fact: f, detail: _ } => match f { + TapUnheldFact => "tap" + GuestConntrackEmptyFact => "conntrack" + RulesetMatchesConvergedFact => "ruleset" + } + SlotNetworkConfirmed => "confirmed" + SlotNetworkUnobservable { fact: _, reason: _ } => "unobservable" + } +} + +fn unobservable_fact(r: SlotNetworkReadback) -> String { + match r { + SlotNetworkUnobservable { fact: f, reason: _ } => match f { + TapUnheldFact => "tap" + GuestConntrackEmptyFact => "conntrack" + RulesetMatchesConvergedFact => "ruleset" + } + SlotNetworkConfirmed => "confirmed" + SlotNetworkRefuted { fact: _, detail: _ } => "refuted" + } +} + +// THE POSITIVE CONTROL: all three facts hold, and only then is the readback confirmed. +test fn a_clean_slot_network_is_confirmed_only_when_all_three_facts_hold() -> Bool { + refuted_fact(r: slot_network_readback( + tap: TapPresentUnheld, + conntrack: GuestConntrackEmptyAfterFlush, + ruleset: RulesetListed { digest: converged_digest }, + converged_ruleset: converged_digest, + )) == "confirmed" +} + +// EACH FACT REFUTES ON ITS OWN, AND THE REFUTATION NAMES IT. A VMM fd still on the tap, a flow that +// survived the flush (the next attempt reuses this guest address), and a ruleset some process edited +// are three different remedies, so the fact is carried rather than a bare false. +test fn each_fact_refutes_alone_and_names_itself() -> Bool { + refuted_fact(r: slot_network_readback( + tap: TapHeld { holders: 1 }, + conntrack: GuestConntrackEmptyAfterFlush, + ruleset: RulesetListed { digest: converged_digest }, + converged_ruleset: converged_digest, + )) == "tap" + && refuted_fact(r: slot_network_readback( + tap: TapMissing, + conntrack: GuestConntrackEmptyAfterFlush, + ruleset: RulesetListed { digest: converged_digest }, + converged_ruleset: converged_digest, + )) == "tap" + && refuted_fact(r: slot_network_readback( + tap: TapPresentUnheld, + conntrack: GuestConntrackResidue { entries: 2 }, + ruleset: RulesetListed { digest: converged_digest }, + converged_ruleset: converged_digest, + )) == "conntrack" + && refuted_fact(r: slot_network_readback( + tap: TapPresentUnheld, + conntrack: GuestConntrackEmptyAfterFlush, + ruleset: RulesetListed { digest: drifted_digest }, + converged_ruleset: converged_digest, + )) == "ruleset" +} + +// "COULD NOT READ" IS NOT "REFUTED" -- and a refutation elsewhere outranks it. The red for the first +// half is an unreadable conntrack table reported as residue; for the second, a drifted ruleset +// hidden behind a probe failure on another fact. +test fn an_unreadable_fact_is_unobservable_and_a_refutation_elsewhere_outranks_it() -> Bool { + unobservable_fact(r: slot_network_readback( + tap: TapPresentUnheld, + conntrack: GuestConntrackUnreadable { reason: "conntrack table not readable" as NonEmptyStr }, + ruleset: RulesetListed { digest: converged_digest }, + converged_ruleset: converged_digest, + )) == "conntrack" + && unobservable_fact(r: slot_network_readback( + tap: TapHoldersUnreadable { reason: "fd scan refused" as NonEmptyStr }, + conntrack: GuestConntrackEmptyAfterFlush, + ruleset: RulesetUnreadable { reason: "nft list refused" as NonEmptyStr }, + converged_ruleset: converged_digest, + )) == "tap" + && refuted_fact(r: slot_network_readback( + tap: TapPresentUnheld, + conntrack: GuestConntrackUnreadable { reason: "conntrack table not readable" as NonEmptyStr }, + ruleset: RulesetListed { digest: drifted_digest }, + converged_ruleset: converged_digest, + )) == "ruleset" +} + +// THE FLUSH AND THE LIST ARE KEYED BY THE GUEST'S ORIGINAL SOURCE. A masqueraded flow carries the +// host's address only on its reply tuple, so a flush keyed any other way leaves every flow the guest +// opened -- and the next attempt in this slot inherits them. The red is a slot's commands naming a +// different slot's guest address. +test fn conntrack_commands_are_keyed_by_this_slots_guest_source() -> Bool { + let net = runner_slot_network(slot: slot(i: 6)) + let guest = render_ipv4_address(addr: net.guest_address) as String + argv_words(command: runner_slot_conntrack_flush_command(net: net)) == ["conntrack", "-D", "-s", guest] + && argv_words(command: runner_slot_conntrack_list_command(net: net)) == ["conntrack", "-L", "-s", guest] + && guest == "172.30.6.2" +} From aaabe6b6c35aeb13353b95c298162674b7d8cdd8 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 03:13:56 +0000 Subject: [PATCH 02/36] Declare the readback and staging consumers as typed frontiers; drop the present-tense consumption claim Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_attempt_launch.dag | 8 ++++++++ dag/gunbc/runner/runner_microvm_network.dag | 16 ++++++++++++++-- .../runner_attempt_launch_witness_test.dag | 11 +++++++++++ .../runner_microvm_network_witness_test.dag | 13 +++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/dag/gunbc/runner/runner_attempt_launch.dag b/dag/gunbc/runner/runner_attempt_launch.dag index 87d9eac2042..2351da09ffd 100644 --- a/dag/gunbc/runner/runner_attempt_launch.dag +++ b/dag/gunbc/runner/runner_attempt_launch.dag @@ -564,3 +564,11 @@ fn staging_verdict(plan: AttemptLaunchPlan, obs: WorkspaceStagingObservation) -> fn workspace_readback_command(workspace_path: String) -> ArgvCommand { dumpe2fs_superblock_command(image_path: workspace_path) } + +// DECLARED FRONTIER (DESIGN 3c). NAMED CONSUMER: the lifecycle controller's staging step +// (gunbc#11670). TRIGGER: that step runs the plan's workspace_create and workspace_format, reads +// workspace_readback_command, and reaches the jailer only through staging_verdict. SUFFICIENT FOR +// deleting this row. NOT satisfied by a witness calling staging_verdict. +data workspace_staging_consumer_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: the microVM lifecycle controller's staging step (gunbc#11670). TRIGGER: that step executes workspace_create and workspace_format, reads workspace_readback_command, and reaches the jailer only through staging_verdict. SUFFICIENT FOR deleting this row. NOT satisfied by a witness calling staging_verdict." as NonEmptyStr, +) diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 25a6137257d..82dfb2c7d1b 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -3,6 +3,7 @@ module gunbc.runner_microvm_network import std.types { String, NonEmptyStr, Bool, Int, List } import std.nat { Nat } import std.disposition { Disposition, SingleAuthority } +import std.dissolution { DissolutionCondition, unbound_dissolution } import std.content_hash { Sha256Digest } import extdeps.network.ipv4 { Ipv4Address, ipv4_address, render_ipv4_address } import extdeps.network.mac { Eui48Address, mac_address, render_mac_address, MacAddressScope, LocallyAdministered, mac_address_scope } @@ -467,8 +468,8 @@ fn runner_slot_network_install_operations( // 2026-09-19), and what an attempt can leave behind in them is not their existence but their USE: // a process still holding the tap, a connection-tracking flow still keyed to the guest address the // NEXT attempt in this slot will reuse, and a ruleset some process edited during the attempt. Those -// three facts are what this readback reads, and gunbc.runner_microvm_lifecycle consumes it as the -// network fact of its host teardown readback, mapped onto product.fabric.sanitation. +// three facts are what this readback reads. Its consumer is not on main yet; see +// slot_network_readback_consumer_frontier below. // // CONNTRACK IS FLUSHED BEFORE IT IS READ, AND THE FLUSH IS NOT THE EVIDENCE. The flush is teardown's // mutation; the reading after it is the fact. A flush that exited zero while a flow survived -- a @@ -612,3 +613,14 @@ fn runner_slot_conntrack_list_command(net: RunnerSlotNetwork) -> ArgvCommand { fn runner_microvm_nft_list_command() -> ArgvCommand { nft_list_table_command(family: NftInet, name: runner_microvm_nft_table_name) } + +// DECLARED FRONTIER (DESIGN 3c). The readback and the three commands that feed it have no +// executing consumer in this tree yet. NAMED CONSUMER: the host teardown readback of the microVM +// slot lifecycle controller (gunbc#11670), whose network fact maps onto product.fabric.sanitation. +// TRIGGER: that controller executes teardown on a host -- runs runner_slot_conntrack_flush_command, +// reads runner_slot_conntrack_list_command and runner_microvm_nft_list_command, and derives +// CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT +// satisfied by #11670 importing the type, or by a witness calling the fold. +data slot_network_readback_consumer_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: the microVM slot lifecycle controller's host teardown readback (gunbc#11670). TRIGGER: that controller executes teardown on a host, runs runner_slot_conntrack_flush_command, reads runner_slot_conntrack_list_command and runner_microvm_nft_list_command, and derives CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT satisfied by an import of the type or by a witness calling the fold." as NonEmptyStr, +) diff --git a/dag/test/claim/runner/runner_attempt_launch_witness_test.dag b/dag/test/claim/runner/runner_attempt_launch_witness_test.dag index 3d6ac0af4af..1a4228274de 100644 --- a/dag/test/claim/runner/runner_attempt_launch_witness_test.dag +++ b/dag/test/claim/runner/runner_attempt_launch_witness_test.dag @@ -47,6 +47,7 @@ import gunbc.runner_attempt_launch { WorkspaceStanding, WorkspaceReady, WorkspaceCreateFailed, WorkspaceFormatFailed, WorkspaceNotExtFilesystem, WorkspaceSizeMismatch, WorkspaceUnreadable, staging_verdict, runner_attempt_workspace_bytes, + workspace_readback_command, workspace_staging_consumer_frontier, } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -404,3 +405,13 @@ test fn the_jailer_is_admitted_only_onto_a_read_back_workspace() -> Bool { test fn a_good_workspace_does_not_admit_a_refused_plan() -> Bool { staging_for(obs: good_workspace(), ground: GroundOccupied) == "unauthorized" } + +// THE READBACK READS THE PLANNED IMAGE'S SUPERBLOCK, AND ITS CONSUMER IS STILL A FRONTIER. The red +// for the first half is a readback of any path but the one the plan formatted. +test fn the_workspace_readback_reads_the_planned_image_and_its_consumer_is_a_frontier() -> Bool { + argv_words(command: workspace_readback_command(workspace_path: "/jail/work.img")) == ["dumpe2fs", "-h", "/jail/work.img"] + && match workspace_staging_consumer_frontier { + UnboundDissolution { description: _ } => true + BoundDissolution { trigger: _ } => false + } +} diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index 44da2de6bc5..cb3798a64bd 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -21,7 +21,9 @@ import gunbc.runner_microvm_network { GuestConntrackEmptyAfterFlush, GuestConntrackResidue, GuestConntrackUnreadable, RulesetListed, RulesetUnreadable, slot_network_readback, runner_slot_conntrack_flush_command, runner_slot_conntrack_list_command, + runner_microvm_nft_list_command, slot_network_readback_consumer_frontier, } +import std.dissolution { UnboundDissolution, BoundDissolution } import std.content_hash { Sha256Digest, Sha256DigestHex } import extdeps.exec.command { argv_words } @@ -307,3 +309,14 @@ test fn conntrack_commands_are_keyed_by_this_slots_guest_source() -> Bool { && argv_words(command: runner_slot_conntrack_list_command(net: net)) == ["conntrack", "-L", "-s", guest] && guest == "172.30.6.2" } + +// THE READBACK READS THE LISTING OF OUR OWN TABLE, AND ITS CONSUMER IS STILL A FRONTIER. The red for +// the first half is a listing of a different table (a digest of dockerd's rules would never match); +// for the second, the frontier row bound before the controller executes teardown. +test fn the_ruleset_readback_lists_our_table_and_its_consumer_is_a_frontier() -> Bool { + argv_words(command: runner_microvm_nft_list_command()) == ["nft", "list", "table", "inet", "gunbc_microvm"] + && match slot_network_readback_consumer_frontier { + UnboundDissolution { description: _ } => true + BoundDissolution { trigger: _ } => false + } +} From 2431ad6bb3869e33bb164d5e8d07ea5d22979330 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 03:58:52 +0000 Subject: [PATCH 03/36] Bind the workspace standing once; one optional-returning readback selector instead of two singleton-list folds Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_attempt_launch.dag | 17 +++++--- dag/gunbc/runner/runner_microvm_network.dag | 47 ++++++++++----------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/dag/gunbc/runner/runner_attempt_launch.dag b/dag/gunbc/runner/runner_attempt_launch.dag index 2351da09ffd..ae52c2d0e73 100644 --- a/dag/gunbc/runner/runner_attempt_launch.dag +++ b/dag/gunbc/runner/runner_attempt_launch.dag @@ -547,13 +547,16 @@ fn staging_verdict(plan: AttemptLaunchPlan, obs: WorkspaceStagingObservation) -> kvm_device: _, vm_config: _, jailer: j, - } => match workspace_standing(obs: obs) { - WorkspaceReady => StagingAdmitsJailer { jailer: j } - WorkspaceCreateFailed { exit: _ } => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } - WorkspaceFormatFailed { exit: _ } => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } - WorkspaceNotExtFilesystem => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } - WorkspaceSizeMismatch { observed: _, expected: _ } => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } - WorkspaceUnreadable { reason: _ } => StagingRefusedWorkspace { standing: workspace_standing(obs: obs) } + } => { + let s = workspace_standing(obs: obs) + match s { + WorkspaceReady => StagingAdmitsJailer { jailer: j } + WorkspaceCreateFailed { exit: _ } => StagingRefusedWorkspace { standing: s } + WorkspaceFormatFailed { exit: _ } => StagingRefusedWorkspace { standing: s } + WorkspaceNotExtFilesystem => StagingRefusedWorkspace { standing: s } + WorkspaceSizeMismatch { observed: _, expected: _ } => StagingRefusedWorkspace { standing: s } + WorkspaceUnreadable { reason: _ } => StagingRefusedWorkspace { standing: s } + } } LaunchRefusedUnauthorized { authorization: _ } => StagingNotAuthorized LaunchRefusedEnvelope { standing: _ } => StagingNotAuthorized diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 82dfb2c7d1b..3f3039b2015 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -551,28 +551,27 @@ fn ruleset_readback(o: RulesetObservation, converged: Sha256Digest) -> SlotNetwo // A REFUTATION ANYWHERE OUTRANKS AN UNREADABLE FACT ELSEWHERE: it is the stronger statement about // the host, and reporting the probe failure instead would hide a known defect behind an unknown one. -fn first_refuted(rs: List) -> List { - fold(rs, init: [], f: (acc, r) => - match acc { - Cons { head: _, tail: _ } => acc - Empty {} => match r { - SlotNetworkRefuted { fact: _, detail: _ } => [r] - SlotNetworkConfirmed => acc - SlotNetworkUnobservable { fact: _, reason: _ } => acc - } - } - ) +fn is_refuted(r: SlotNetworkReadback) -> Bool { + match r { + SlotNetworkRefuted { fact: _, detail: _ } => true + SlotNetworkConfirmed => false + SlotNetworkUnobservable { fact: _, reason: _ } => false + } } -fn first_unobservable(rs: List) -> List { - fold(rs, init: [], f: (acc, r) => +fn is_unobservable(r: SlotNetworkReadback) -> Bool { + match r { + SlotNetworkUnobservable { fact: _, reason: _ } => true + SlotNetworkConfirmed => false + SlotNetworkRefuted { fact: _, detail: _ } => false + } +} + +fn first_readback_where(rs: List, pick: fn(SlotNetworkReadback) -> Bool) -> SlotNetworkReadback? { + fold(rs, init: none, f: (acc, r) => match acc { - Cons { head: _, tail: _ } => acc - Empty {} => match r { - SlotNetworkUnobservable { fact: _, reason: _ } => [r] - SlotNetworkConfirmed => acc - SlotNetworkRefuted { fact: _, detail: _ } => acc - } + Present { value: _ } => acc + Absent => if pick(r) { Present { value: r } } else { acc } } ) } @@ -588,11 +587,11 @@ fn slot_network_readback( conntrack_readback(o: conntrack), ruleset_readback(o: ruleset, converged: converged_ruleset), ] - match first_refuted(rs: facts) { - Cons { head: r, tail: _ } => r - Empty {} => match first_unobservable(rs: facts) { - Cons { head: u, tail: _ } => u - Empty {} => SlotNetworkConfirmed + match first_readback_where(rs: facts, pick: is_refuted) { + Present { value: r } => r + Absent => match first_readback_where(rs: facts, pick: is_unobservable) { + Present { value: u } => u + Absent => SlotNetworkConfirmed } } } From e5830d13f90097198a74d6c70382bf4b90bc7dd3 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 04:46:03 +0000 Subject: [PATCH 04/36] Fold the guest egress rules from one typed policy; narrow DNS to the booted resolver; filtered-egress consumes the policy Co-Authored-By: Claude Opus 5 (1M context) --- .../runner/runner_host_filtered_egress.dag | 36 ++-- dag/gunbc/runner/runner_microvm_network.dag | 195 +++++++++++++----- .../runner_microvm_network_witness_test.dag | 20 +- 3 files changed, 169 insertions(+), 82 deletions(-) diff --git a/dag/gunbc/runner/runner_host_filtered_egress.dag b/dag/gunbc/runner/runner_host_filtered_egress.dag index 31cf977e101..38267256c3d 100644 --- a/dag/gunbc/runner/runner_host_filtered_egress.dag +++ b/dag/gunbc/runner/runner_host_filtered_egress.dag @@ -5,6 +5,8 @@ import extdeps.network.ipv4 { Ipv4Address, render_ipv4_address } import gunbc.runner_microvm_network { RunnerSlotNetwork, runner_slot_host_cidr, runner_slot_guest_ip_boot_arg, runner_microvm_network_supernet, + GuestEgressPolicy, runner_microvm_guest_egress_policy, + IpV6Standing, IpV6DisabledOnDataPath, IpV6DefaultDenyRuleset, IpV6Unaddressed, } // THE EGRESS PATH A RAW RUNNER HOST OWES, and why it is not the one the shared hosts run. @@ -76,10 +78,16 @@ data runner_host_uplink: HostUplinkConfiguration = StaticFromObservedOfferCandid // the BMC, the rest of the LAN, and any other attempt's tap. type ChainStance = DefaultDrop | DefaultAccept +// THE GUEST-FACING GRANTS AND DENIALS ARE CONSUMED, NOT RESTATED. They are +// gunbc.runner_microvm_network runner_microvm_guest_egress_policy, the same rows the shared hosts' +// ruleset is folded from; an earlier revision listed them here as prose and the two drifted (review +// 68173: a "selected resolver" here against any-host port 53 there). What this host adds is its OWN +// chain stances: it owns the whole forwarding policy, so its forward chain default-drops at host +// grain where the shared hosts cannot. type EgressPolicyShape { host_input: ChainStance forward: ChainStance - explicit_grants: List + guest: GuestEgressPolicy explicit_denials: List nat: NonEmptyStr } @@ -87,36 +95,16 @@ type EgressPolicyShape { data runner_host_egress_policy: EgressPolicyShape = EgressPolicyShape { host_input: DefaultDrop, forward: DefaultDrop, - explicit_grants: [ - "loopback on the host", - "established and related, both directions", - "guest to the selected resolver, for name resolution only", - "guest to the qualification destination class on TCP 443", - ], + guest: runner_microvm_guest_egress_policy, explicit_denials: [ "guest to the BMC and any management address", - "guest to the LAN, private, link-local and multicast ranges", "tap to tap -- no attempt may reach another attempt", - "everything not named above", ], nat: "masquerade the admitted guest prefix on the host uplink, and nothing else", } -// TCP 443 TO A DESTINATION CLASS IS NOT "GITHUB ONLY", and calling it that would be the kind of -// precision this lane keeps having to walk back. A static address list for a CDN-fronted service is -// stale the moment it is written, so the honest description is a default-deny policy with one broad -// allowance. Narrowing it properly means an authenticated egress proxy as the single permitted -// destination, which is its own design. -data egress_grant_precision: NonEmptyStr = "the TCP 443 grant is a destination CLASS, not a GitHub-only rule; a static address list cannot honestly express a CDN-fronted service and the narrow form is an egress proxy" - -// IPv6 IS AN EXPLICIT STANDING RATHER THAN AN OVERSIGHT. An IPv4-only ruleset on a dual-stack host -// is a default-allow wearing a default-deny's name, because everything the v4 policy denies remains -// reachable over v6. -type IpV6Standing - = IpV6DisabledOnDataPath - | IpV6DefaultDenyRuleset - | IpV6Unaddressed { risk: NonEmptyStr } - +// THIS HOST DISABLES v6 ON THE DATA PATH (read back as 1 on the filtered-egress-4 run); the shared +// hosts cannot, and filter it instead. One standing type, two host facts. data runner_host_ipv6: IpV6Standing = IpV6DisabledOnDataPath // ── what the kernel must carry, since nothing is modular ──────────────────────────────────────── diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 3f3039b2015..ff50013cd14 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -14,7 +14,7 @@ import gunbc.executor_privileged_operation { } import extdeps.nftables.ruleset { NftTable, NftChain, NftRule, NftMatch, NftFamily, NftInet, NftHook, NftForward, NftPostrouting, - NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPolicy, NftPolicyAccept, + NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPolicy, NftPolicyAccept, NftPolicyDrop, NftChainType, NftTypeFilter, NftTypeNat, NftVerdict, NftAccept, NftDrop, NftMasquerade, NftInputInterfaceWildcard, NftOutputInterfaceWildcard, NftSourceAddressPrefix, @@ -121,7 +121,7 @@ fn runner_slot_network_as_microvm_networking(net: RunnerSlotNetwork) -> RunnerMi // THE GUEST IS CONFIGURED BY THE KERNEL COMMAND LINE AND CARRIES NO DHCP CLIENT, which removes a // whole service from the guest and a whole failure mode from the boot. The kernel's own ip= syntax // (Documentation/admin-guide/nfs/nfsroot.rst) is -// ip=:::::: +// ip=::::::: // and the empty fields are REQUIRED separators rather than omissions -- a form with fewer colons is // a different, silently-misparsed directive. autoconf is "off" because every field is supplied // here; leaving it unset would have the kernel try DHCP anyway and stall the boot on a link where @@ -181,6 +181,8 @@ fn runner_slot_guest_ip_boot_arg(net: RunnerSlotNetwork) -> String { runner_microvm_network_guest_interface as String, ":", runner_microvm_network_autoconf_off as String, + ":", + render_ipv4_address(addr: runner_microvm_guest_resolver()) as String, ], "") } @@ -240,7 +242,7 @@ fn runner_microvm_forward_chain() -> NftChain { chain_type: NftTypeFilter, hook: NftForward, priority: NftPriorityFilter, - policy: NftPolicyAccept, + policy: forward_chain_policy(g: runner_microvm_guest_egress_policy.grain), rules: concat( [ NftRule { @@ -256,40 +258,79 @@ fn runner_microvm_forward_chain() -> NftChain { } } -// --- guest egress: default-deny at TAP grain, not host grain --- +// --- guest egress: ONE typed policy, and the rules are folded from it --- -// THIS IS WHERE gunbc.runner.runner_host_filtered_egress's DefaultDrop STANCE IS HONOURED, AND ITS -// GRAIN IS THE TAP. That module states forward: DefaultDrop for a host this fleet owns entirely. -// srvN is not that host (dockerd's tables, see the chain policy above), so the chain policy stays -// accept and the default-deny is the LAST RULE OF A TAP-SCOPED LIST: every packet a guest sends -// matches a rule written for it or reaches the final drop. Traffic that did not enter through a -// gunbc tap matches none of these rows and keeps whatever verdict the host's other tables give it. -// So the stance holds for guests -- the only principal it was ever about -- and not for the host. +// THE POLICY IS DATA AND THE RULESET IS ITS PROJECTION. An earlier revision of this change wrote the +// rule list by hand and joined it to gunbc.runner.runner_host_filtered_egress's policy with a +// comment, which let the two disagree (review 68173). The policy now lives here, typed, because that +// module already imports this one -- declaring it there would make this module's consumption an +// import cycle -- and runner_host_filtered_egress consumes these rows instead of restating them. // -// THE ORDER IS THE POLICY. Established first, so replies to admitted flows are never re-judged. The -// v6 refusal before any v4 grant, because in the inet family an `ip daddr` match is false on a v6 -// packet and a v6 flow would otherwise reach the port grants. The private-range denials ABOVE the -// port grants, because the 443 grant is a destination class and 192.168.x.y:443 is in it -- the -// Mt. Collins run (gunbc.runner.runner_filtered_egress_receipt denied_paths) is the receipt that -// this ordering is what stopped a guest reaching its own BMC. The grants are the ones that run -// admitted: name resolution, and TCP 443 for github.com and the Actions endpoints. -// -// The resolver the guest uses must therefore be PUBLIC: a private resolver is denied by the rows -// above the DNS grant, and that is the intended reading of "guest to the selected resolver", not an -// accident of ordering. -data runner_microvm_guest_denied_destinations: List = [ - "10.0.0.0/8" as NonEmptyStr, - "172.16.0.0/12" as NonEmptyStr, - "192.168.0.0/16" as NonEmptyStr, - "100.64.0.0/10" as NonEmptyStr, - "169.254.0.0/16" as NonEmptyStr, - "127.0.0.0/8" as NonEmptyStr, - "224.0.0.0/4" as NonEmptyStr, -] - -data runner_microvm_guest_dns_port: Int = 53 - -data runner_microvm_guest_tls_port: Int = 443 +// THE GRAIN IS A FIELD, NOT A COMMENT. A host that owns its whole forwarding policy can default-drop +// at the chain; a host where other tables hook forward (srv2's dockerd) cannot, because every table +// runs and any drop wins. There the default-deny is the LAST RULE OF A TAP-SCOPED LIST, and a packet +// that did not enter through a gunbc tap matches no row here. +type ForwardDenyGrain + = HostChainDefaultDrop + | TapScopedFinalDrop { chain_policy_reason: NonEmptyStr } + +// IPv6 IS AN EXPLICIT STANDING RATHER THAN AN OVERSIGHT. An IPv4-only ruleset on a dual-stack host is +// a default-allow wearing a default-deny's name. In the inet family an `ip daddr` match is false on +// a v6 packet, so under DefaultDenyRuleset the v6 drop must precede every v4 grant. +type IpV6Standing + = IpV6DisabledOnDataPath + | IpV6DefaultDenyRuleset + | IpV6Unaddressed { risk: NonEmptyStr } + +// A GRANT NAMES ITS DESTINATION. "Any destination the denials left" and "the one resolver" are +// materially different grants on the same port, so the difference is a constructor, not a reading. +type GuestEgressDestination + = AnyUndeniedDestination + | NamedDestination { address: Ipv4Address } + +type GuestEgressGrant { + transport: NftTransport + port: Int + destination: GuestEgressDestination +} + +type GuestEgressPolicy { + grain: ForwardDenyGrain + ipv6: IpV6Standing + denied_destinations: List + grants: List +} + +// THE RESOLVER IS ONE NAMED PUBLIC ADDRESS. 1.1.1.1 is the resolver the Mt. Collins filtered-egress +// run answered through (gunbc.runner.runner_filtered_egress_receipt admitted_paths, guest-dns); it is +// public, so the private-range denials never shadow it. The guest is told it at boot as the ip= +// directive's dns0 field, so the grant and the guest's configuration are one row. +fn runner_microvm_guest_resolver() -> Ipv4Address { + ipv4_address(octet1: 1, octet2: 1, octet3: 1, octet4: 1) +} + +// TCP 443 IS A DESTINATION CLASS, NOT "GITHUB ONLY": github.com and the Actions endpoints are +// CDN-fronted, so a static address list is stale when written. The narrow form is an egress proxy. +data runner_microvm_guest_egress_policy: GuestEgressPolicy = GuestEgressPolicy { + grain: TapScopedFinalDrop { + chain_policy_reason: "other tables hook forward on srvN (dockerd on srv2) and every table runs with any drop winning, so a chain-level drop would drop traffic this fleet does not own" as NonEmptyStr, + }, + ipv6: IpV6DefaultDenyRuleset, + denied_destinations: [ + "10.0.0.0/8" as NonEmptyStr, + "172.16.0.0/12" as NonEmptyStr, + "192.168.0.0/16" as NonEmptyStr, + "100.64.0.0/10" as NonEmptyStr, + "169.254.0.0/16" as NonEmptyStr, + "127.0.0.0/8" as NonEmptyStr, + "224.0.0.0/4" as NonEmptyStr, + ], + grants: [ + GuestEgressGrant { transport: NftUdp, port: 53, destination: NamedDestination { address: runner_microvm_guest_resolver() } }, + GuestEgressGrant { transport: NftTcp, port: 53, destination: NamedDestination { address: runner_microvm_guest_resolver() } }, + GuestEgressGrant { transport: NftTcp, port: 443, destination: AnyUndeniedDestination }, + ], +} fn from_guest_tap(m: List, verdict: NftVerdict) -> NftRule { NftRule { @@ -298,34 +339,64 @@ fn from_guest_tap(m: List, verdict: NftVerdict) -> NftRule { } } -fn runner_microvm_guest_egress_rules() -> List { +fn grant_destination_matches(d: GuestEgressDestination) -> List { + match d { + AnyUndeniedDestination => [] + NamedDestination { address: a } => + [NftDestinationAddressPrefix { cidr: join([render_ipv4_address(addr: a) as String, "/32"], "") as NonEmptyStr }] + } +} + +fn grant_rule(g: GuestEgressGrant) -> NftRule { + from_guest_tap( + m: concat(grant_destination_matches(d: g.destination), [NftTransportDestinationPort { transport: g.transport, port: g.port }]), + verdict: NftAccept, + ) +} + +fn ipv6_rules(v: IpV6Standing) -> List { + match v { + IpV6DefaultDenyRuleset => [from_guest_tap(m: [NftNetworkProtocolIpv6], verdict: NftDrop)] + IpV6DisabledOnDataPath => [] + IpV6Unaddressed { risk: _ } => [] + } +} + +fn final_drop_rules(g: ForwardDenyGrain) -> List { + match g { + TapScopedFinalDrop { chain_policy_reason: _ } => [from_guest_tap(m: [], verdict: NftDrop)] + HostChainDefaultDrop => [] + } +} + +fn forward_chain_policy(g: ForwardDenyGrain) -> NftPolicy { + match g { + TapScopedFinalDrop { chain_policy_reason: _ } => NftPolicyAccept + HostChainDefaultDrop => NftPolicyDrop + } +} + +// THE ORDER IS THE POLICY: established first, so replies to admitted flows are never re-judged; the +// v6 refusal before any v4 grant; the denials ABOVE the grants, because the 443 grant is a class and +// 192.168.x.y:443 is in it (the Mt. Collins receipt's denied_paths is this ordering holding); the +// grants; then the grain's final drop. +fn guest_egress_rules(p: GuestEgressPolicy) -> List { concat( concat( - [ - from_guest_tap(m: [NftConnectionTrackingEstablished], verdict: NftAccept), - from_guest_tap(m: [NftNetworkProtocolIpv6], verdict: NftDrop), - ], - map(runner_microvm_guest_denied_destinations, c => - from_guest_tap(m: [NftDestinationAddressPrefix { cidr: c }], verdict: NftDrop)), - ), - [ - from_guest_tap( - m: [NftTransportDestinationPort { transport: NftUdp, port: runner_microvm_guest_dns_port }], - verdict: NftAccept, - ), - from_guest_tap( - m: [NftTransportDestinationPort { transport: NftTcp, port: runner_microvm_guest_dns_port }], - verdict: NftAccept, + concat( + concat([from_guest_tap(m: [NftConnectionTrackingEstablished], verdict: NftAccept)], ipv6_rules(v: p.ipv6)), + map(p.denied_destinations, c => from_guest_tap(m: [NftDestinationAddressPrefix { cidr: c }], verdict: NftDrop)), ), - from_guest_tap( - m: [NftTransportDestinationPort { transport: NftTcp, port: runner_microvm_guest_tls_port }], - verdict: NftAccept, - ), - from_guest_tap(m: [], verdict: NftDrop), - ], + map(p.grants, g => grant_rule(g: g)), + ), + final_drop_rules(g: p.grain), ) } +fn runner_microvm_guest_egress_rules() -> List { + guest_egress_rules(p: runner_microvm_guest_egress_policy) +} + // THE MASQUERADE MATCHES ON SOURCE AND NOT ON OUTPUT INTERFACE, which removes an observation this // module would otherwise have to make and keep correct. srv2's uplink is enP3p3s0f1 and srv1's is // not; naming it would make the ruleset host-specific and would rot the moment an interface is @@ -623,3 +694,13 @@ fn runner_microvm_nft_list_command() -> ArgvCommand { data slot_network_readback_consumer_frontier: DissolutionCondition = unbound_dissolution( description: "NAMED CONSUMER: the microVM slot lifecycle controller's host teardown readback (gunbc#11670). TRIGGER: that controller executes teardown on a host, runs runner_slot_conntrack_flush_command, reads runner_slot_conntrack_list_command and runner_microvm_nft_list_command, and derives CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT satisfied by an import of the type or by a witness calling the fold." as NonEmptyStr, ) + +// DECLARED FRONTIER (DESIGN 3c). The kernel records dns0 in /proc/net/pnp; it does not write +// /etc/resolv.conf. NAMED CONSUMER: the guest image (gunbc.runner_guest_image), whose resolver +// configuration must read the booted dns0 so the guest resolves through the one address the DNS +// grant admits. TRIGGER: the guest image links /etc/resolv.conf to /proc/net/pnp (or derives it from +// dns0 at boot) and a booted guest resolves github.com through runner_microvm_guest_resolver. +// SUFFICIENT FOR deleting this row. NOT satisfied by the boot argument carrying dns0 alone. +data guest_resolver_configuration_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: gunbc.runner_guest_image resolver configuration. TRIGGER: the guest image makes /etc/resolv.conf follow the booted dns0 (/proc/net/pnp) and a booted guest resolves github.com through runner_microvm_guest_resolver. SUFFICIENT FOR deleting this row. NOT satisfied by the ip= boot argument carrying dns0 alone." as NonEmptyStr, +) diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index cb3798a64bd..2bf59b6dd6b 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -22,6 +22,7 @@ import gunbc.runner_microvm_network { RulesetListed, RulesetUnreadable, slot_network_readback, runner_slot_conntrack_flush_command, runner_slot_conntrack_list_command, runner_microvm_nft_list_command, slot_network_readback_consumer_frontier, + runner_microvm_guest_resolver, guest_resolver_configuration_frontier, } import std.dissolution { UnboundDissolution, BoundDissolution } import std.content_hash { Sha256Digest, Sha256DigestHex } @@ -197,7 +198,7 @@ test fn the_nat_chain_is_typed_nat_and_the_filter_chain_stays_filter() -> Bool { test fn guest_egress_is_default_deny_at_tap_grain_in_the_receipted_order() -> Bool { let text = runner_microvm_nft_ruleset_text() string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" meta nfproto ipv6 drop\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") - && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" udp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") + && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") && string_contains(s: text, pattern: "policy accept") && !string_contains(s: text, pattern: "policy drop") } @@ -320,3 +321,20 @@ test fn the_ruleset_readback_lists_our_table_and_its_consumer_is_a_frontier() -> BoundDissolution { trigger: _ } => false } } + +// THE DNS GRANT IS THE ONE RESOLVER THE GUEST IS TOLD ABOUT, AND NOTHING WIDER. The reds: a port-53 +// grant with no destination (every public host on 53, an exfiltration channel out of a default-deny +// guest -- review 68173), and a boot directive naming a resolver the ruleset does not admit. +test fn the_dns_grant_is_the_booted_resolver_and_no_other_host() -> Bool { + let resolver = render_ipv4_address(addr: runner_microvm_guest_resolver()) as String + let text = runner_microvm_nft_ruleset_text() + let arg = runner_slot_guest_ip_boot_arg(net: runner_slot_network(slot: slot(i: 7))) + !string_contains(s: text, pattern: "\" udp dport 53") + && !string_contains(s: text, pattern: "\" tcp dport 53") + && string_contains(s: text, pattern: join(["ip daddr ", resolver, "/32 udp dport 53 accept"], "")) + && string_contains(s: arg, pattern: join([":eth0:off:", resolver], "")) + && match guest_resolver_configuration_frontier { + UnboundDissolution { description: _ } => true + BoundDissolution { trigger: _ } => false + } +} From 4d701ddaec3071c1ab993f74ecf72b758b378b92 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 05:35:43 +0000 Subject: [PATCH 05/36] Split per-host egress grain from the shared guest policy; admit replies in their own direction Co-Authored-By: Claude Opus 5 (1M context) --- .../runner/runner_host_filtered_egress.dag | 13 ++-- dag/gunbc/runner/runner_microvm_network.dag | 65 ++++++++++++++----- .../runner_microvm_network_witness_test.dag | 18 ++++- 3 files changed, 70 insertions(+), 26 deletions(-) diff --git a/dag/gunbc/runner/runner_host_filtered_egress.dag b/dag/gunbc/runner/runner_host_filtered_egress.dag index 38267256c3d..0db8d224e08 100644 --- a/dag/gunbc/runner/runner_host_filtered_egress.dag +++ b/dag/gunbc/runner/runner_host_filtered_egress.dag @@ -6,7 +6,7 @@ import gunbc.runner_microvm_network { RunnerSlotNetwork, runner_slot_host_cidr, runner_slot_guest_ip_boot_arg, runner_microvm_network_supernet, GuestEgressPolicy, runner_microvm_guest_egress_policy, - IpV6Standing, IpV6DisabledOnDataPath, IpV6DefaultDenyRuleset, IpV6Unaddressed, + HostEgressGrain, HostChainDefaultDrop, IpV6DisabledOnDataPath, } // THE EGRESS PATH A RAW RUNNER HOST OWES, and why it is not the one the shared hosts run. @@ -83,10 +83,12 @@ type ChainStance = DefaultDrop | DefaultAccept // ruleset is folded from; an earlier revision listed them here as prose and the two drifted (review // 68173: a "selected resolver" here against any-host port 53 there). What this host adds is its OWN // chain stances: it owns the whole forwarding policy, so its forward chain default-drops at host -// grain where the shared hosts cannot. +// grain where the shared hosts cannot, and its v6 is disabled on the data path (read back as 1 on +// the filtered-egress-4 run) where theirs is filtered. Both are this host's HostEgressGrain, so the +// record carries one value for each fact. type EgressPolicyShape { host_input: ChainStance - forward: ChainStance + host: HostEgressGrain guest: GuestEgressPolicy explicit_denials: List nat: NonEmptyStr @@ -94,7 +96,7 @@ type EgressPolicyShape { data runner_host_egress_policy: EgressPolicyShape = EgressPolicyShape { host_input: DefaultDrop, - forward: DefaultDrop, + host: HostEgressGrain { forward: HostChainDefaultDrop, ipv6: IpV6DisabledOnDataPath }, guest: runner_microvm_guest_egress_policy, explicit_denials: [ "guest to the BMC and any management address", @@ -103,9 +105,6 @@ data runner_host_egress_policy: EgressPolicyShape = EgressPolicyShape { nat: "masquerade the admitted guest prefix on the host uplink, and nothing else", } -// THIS HOST DISABLES v6 ON THE DATA PATH (read back as 1 on the filtered-egress-4 run); the shared -// hosts cannot, and filter it instead. One standing type, two host facts. -data runner_host_ipv6: IpV6Standing = IpV6DisabledOnDataPath // ── what the kernel must carry, since nothing is modular ──────────────────────────────────────── diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index ff50013cd14..0242e2b6ffc 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -236,13 +236,13 @@ data runner_microvm_nft_table_name: NonEmptyStr = "gunbc_microvm" as NonEmptyStr // slot's guest at all (see the prefix row above), so this rule catches traffic that reached the // host and asked to be forwarded between taps. Both exist because they fail differently -- the // prefix is arithmetic and cannot be flushed, this rule survives a guest that sets its own routes. -fn runner_microvm_forward_chain() -> NftChain { +fn runner_microvm_forward_chain(host: HostEgressGrain) -> NftChain { NftChain { name: "forward" as NonEmptyStr, chain_type: NftTypeFilter, hook: NftForward, priority: NftPriorityFilter, - policy: forward_chain_policy(g: runner_microvm_guest_egress_policy.grain), + policy: forward_chain_policy(g: host.forward), rules: concat( [ NftRule { @@ -253,7 +253,7 @@ fn runner_microvm_forward_chain() -> NftChain { verdict: NftDrop, }, ], - runner_microvm_guest_egress_rules(), + guest_egress_rules(p: runner_microvm_guest_egress_policy, host: host), ), } } @@ -294,9 +294,16 @@ type GuestEgressGrant { destination: GuestEgressDestination } -type GuestEgressPolicy { - grain: ForwardDenyGrain +// WHAT A GUEST MAY REACH IS SHARED BY EVERY HOST; HOW A HOST ENFORCES IT IS NOT. The grants and +// denials are one row for the fleet. The forward grain and the IPv6 standing are facts about ONE +// host -- whether it owns its forward chain, whether v6 is off on its data path -- so each host +// supplies its own HostEgressGrain, and no host's row carries another host's values (review 68199). +type HostEgressGrain { + forward: ForwardDenyGrain ipv6: IpV6Standing +} + +type GuestEgressPolicy { denied_destinations: List grants: List } @@ -309,13 +316,19 @@ fn runner_microvm_guest_resolver() -> Ipv4Address { ipv4_address(octet1: 1, octet2: 1, octet3: 1, octet4: 1) } -// TCP 443 IS A DESTINATION CLASS, NOT "GITHUB ONLY": github.com and the Actions endpoints are -// CDN-fronted, so a static address list is stale when written. The narrow form is an egress proxy. -data runner_microvm_guest_egress_policy: GuestEgressPolicy = GuestEgressPolicy { - grain: TapScopedFinalDrop { +// THE SHARED RUNNER HOSTS (srvN). Other tables hook forward there (dockerd on srv2) and every table +// runs with any drop winning, so the default-deny is the tap-scoped final drop; v6 is not disabled on +// their data path, so it is filtered. +data runner_microvm_shared_host_grain: HostEgressGrain = HostEgressGrain { + forward: TapScopedFinalDrop { chain_policy_reason: "other tables hook forward on srvN (dockerd on srv2) and every table runs with any drop winning, so a chain-level drop would drop traffic this fleet does not own" as NonEmptyStr, }, ipv6: IpV6DefaultDenyRuleset, +} + +// TCP 443 IS A DESTINATION CLASS, NOT "GITHUB ONLY": github.com and the Actions endpoints are +// CDN-fronted, so a static address list is stale when written. The narrow form is an egress proxy. +data runner_microvm_guest_egress_policy: GuestEgressPolicy = GuestEgressPolicy { denied_destinations: [ "10.0.0.0/8" as NonEmptyStr, "172.16.0.0/12" as NonEmptyStr, @@ -376,26 +389,38 @@ fn forward_chain_policy(g: ForwardDenyGrain) -> NftPolicy { } } +// REPLIES ARE ADMITTED IN THEIR OWN DIRECTION. A reply to a guest's flow enters on the uplink and +// leaves on the tap, so an established rule matching only the tap as INPUT never sees it. Under a +// tap-scoped grain the chain's accept policy happened to carry replies; under HostChainDefaultDrop +// the chain policy drops them and no guest flow would complete (review 68199). So the reply rule is +// emitted on every grain, and nothing reaches a tap unsolicited under a dropping chain. +// // THE ORDER IS THE POLICY: established first, so replies to admitted flows are never re-judged; the // v6 refusal before any v4 grant; the denials ABOVE the grants, because the 443 grant is a class and // 192.168.x.y:443 is in it (the Mt. Collins receipt's denied_paths is this ordering holding); the // grants; then the grain's final drop. -fn guest_egress_rules(p: GuestEgressPolicy) -> List { +fn guest_egress_rules(p: GuestEgressPolicy, host: HostEgressGrain) -> List { concat( concat( concat( - concat([from_guest_tap(m: [NftConnectionTrackingEstablished], verdict: NftAccept)], ipv6_rules(v: p.ipv6)), + concat( + [ + NftRule { + matches: [NftOutputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, NftConnectionTrackingEstablished], + verdict: NftAccept, + }, + from_guest_tap(m: [NftConnectionTrackingEstablished], verdict: NftAccept), + ], + ipv6_rules(v: host.ipv6), + ), map(p.denied_destinations, c => from_guest_tap(m: [NftDestinationAddressPrefix { cidr: c }], verdict: NftDrop)), ), map(p.grants, g => grant_rule(g: g)), ), - final_drop_rules(g: p.grain), + final_drop_rules(g: host.forward), ) } -fn runner_microvm_guest_egress_rules() -> List { - guest_egress_rules(p: runner_microvm_guest_egress_policy) -} // THE MASQUERADE MATCHES ON SOURCE AND NOT ON OUTPUT INTERFACE, which removes an observation this // module would otherwise have to make and keep correct. srv2's uplink is enP3p3s0f1 and srv1's is @@ -418,16 +443,20 @@ fn runner_microvm_postrouting_chain() -> NftChain { } } -fn runner_microvm_nft_table() -> NftTable { +fn runner_microvm_nft_table(host: HostEgressGrain) -> NftTable { NftTable { family: NftInet, name: runner_microvm_nft_table_name, - chains: [runner_microvm_forward_chain(), runner_microvm_postrouting_chain()], + chains: [runner_microvm_forward_chain(host: host), runner_microvm_postrouting_chain()], } } +fn runner_microvm_nft_ruleset_text_for(host: HostEgressGrain) -> String { + nft_table_wire(t: runner_microvm_nft_table(host: host)) +} + fn runner_microvm_nft_ruleset_text() -> String { - nft_table_wire(t: runner_microvm_nft_table()) + runner_microvm_nft_ruleset_text_for(host: runner_microvm_shared_host_grain) } diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index 2bf59b6dd6b..50a4ca94007 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -23,7 +23,9 @@ import gunbc.runner_microvm_network { runner_slot_conntrack_flush_command, runner_slot_conntrack_list_command, runner_microvm_nft_list_command, slot_network_readback_consumer_frontier, runner_microvm_guest_resolver, guest_resolver_configuration_frontier, + runner_microvm_nft_ruleset_text_for, } +import gunbc.runner.runner_host_filtered_egress { runner_host_egress_policy } import std.dissolution { UnboundDissolution, BoundDissolution } import std.content_hash { Sha256Digest, Sha256DigestHex } import extdeps.exec.command { argv_words } @@ -197,7 +199,7 @@ test fn the_nat_chain_is_typed_nat_and_the_filter_chain_stays_filter() -> Bool { // default-deny into default-accept for every port nobody named. test fn guest_egress_is_default_deny_at_tap_grain_in_the_receipted_order() -> Bool { let text = runner_microvm_nft_ruleset_text() - string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" meta nfproto ipv6 drop\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") + string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" meta nfproto ipv6 drop\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") && string_contains(s: text, pattern: "policy accept") && !string_contains(s: text, pattern: "policy drop") @@ -338,3 +340,17 @@ test fn the_dns_grant_is_the_booted_resolver_and_no_other_host() -> Bool { BoundDissolution { trigger: _ } => false } } + +// A HOST THAT DROPS AT THE CHAIN STILL CARRIES REPLIES, AND ITS OWN GRAIN IS WHAT IS FOLDED. The raw +// host's row (runner_host_filtered_egress) is folded here: its chain policy is drop, it emits no +// tap-scoped final drop, it does not filter v6 (disabled on its data path), and the reply-direction +// established rule is present. The red is the previous fold, where the only established rule matched +// the tap as INPUT, so every reply to a guest's flow fell to the chain's drop policy (review 68199). +test fn a_chain_dropping_host_folds_its_own_grain_and_still_carries_replies() -> Bool { + let text = runner_microvm_nft_ruleset_text_for(host: runner_host_egress_policy.host) + string_contains(s: text, pattern: "hook forward priority 0; policy drop;") + && string_contains(s: text, pattern: "oifname \"gunbc-tap*\" ct state established,related accept") + && !string_contains(s: text, pattern: "meta nfproto ipv6") + && !string_contains(s: text, pattern: "iifname \"gunbc-tap*\" drop\n }") + && string_contains(s: text, pattern: "ip daddr 1.1.1.1/32 udp dport 53 accept") +} From c8fba5b5a7f911b60a6197d3712afcaf109f41c1 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 06:24:45 +0000 Subject: [PATCH 06/36] Consume std.types Port for nft dport and egress grants; compare digests through content_hash_eq_cryptographic Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/nftables/ruleset.dag | 4 ++-- dag/gunbc/runner/runner_microvm_network.dag | 18 +++++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/dag/extdeps/nftables/ruleset.dag b/dag/extdeps/nftables/ruleset.dag index 883f0ca92ba..31c69fca108 100644 --- a/dag/extdeps/nftables/ruleset.dag +++ b/dag/extdeps/nftables/ruleset.dag @@ -1,6 +1,6 @@ module extdeps.nftables.ruleset -import std.types { String, NonEmptyStr, List, Int } +import std.types { String, NonEmptyStr, List, Port } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } import extdeps.exec.command { ArgvCommand, argv_command } @@ -134,7 +134,7 @@ type NftMatch | NftOutputInterfaceWildcard { prefix: NonEmptyStr } | NftSourceAddressPrefix { cidr: NonEmptyStr } | NftDestinationAddressPrefix { cidr: NonEmptyStr } - | NftTransportDestinationPort { transport: NftTransport, port: Int } + | NftTransportDestinationPort { transport: NftTransport, port: Port } | NftNetworkProtocolIpv6 | NftConnectionTrackingEstablished diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 0242e2b6ffc..5343f9c5509 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -1,10 +1,10 @@ module gunbc.runner_microvm_network -import std.types { String, NonEmptyStr, Bool, Int, List } +import std.types { String, NonEmptyStr, Bool, Int, List, Port } import std.nat { Nat } import std.disposition { Disposition, SingleAuthority } import std.dissolution { DissolutionCondition, unbound_dissolution } -import std.content_hash { Sha256Digest } +import std.content_hash { Sha256Digest, content_hash_eq_cryptographic } import extdeps.network.ipv4 { Ipv4Address, ipv4_address, render_ipv4_address } import extdeps.network.mac { Eui48Address, mac_address, render_mac_address, MacAddressScope, LocallyAdministered, mac_address_scope } import gunbc.build_cache_instance { RunnerSlotIdentity } @@ -290,7 +290,7 @@ type GuestEgressDestination type GuestEgressGrant { transport: NftTransport - port: Int + port: Port destination: GuestEgressDestination } @@ -326,6 +326,10 @@ data runner_microvm_shared_host_grain: HostEgressGrain = HostEgressGrain { ipv6: IpV6DefaultDenyRuleset, } +data runner_microvm_guest_dns_port: Port = 53 + +data runner_microvm_guest_tls_port: Port = 443 + // TCP 443 IS A DESTINATION CLASS, NOT "GITHUB ONLY": github.com and the Actions endpoints are // CDN-fronted, so a static address list is stale when written. The narrow form is an egress proxy. data runner_microvm_guest_egress_policy: GuestEgressPolicy = GuestEgressPolicy { @@ -339,9 +343,9 @@ data runner_microvm_guest_egress_policy: GuestEgressPolicy = GuestEgressPolicy { "224.0.0.0/4" as NonEmptyStr, ], grants: [ - GuestEgressGrant { transport: NftUdp, port: 53, destination: NamedDestination { address: runner_microvm_guest_resolver() } }, - GuestEgressGrant { transport: NftTcp, port: 53, destination: NamedDestination { address: runner_microvm_guest_resolver() } }, - GuestEgressGrant { transport: NftTcp, port: 443, destination: AnyUndeniedDestination }, + GuestEgressGrant { transport: NftUdp, port: runner_microvm_guest_dns_port, destination: NamedDestination { address: runner_microvm_guest_resolver() } }, + GuestEgressGrant { transport: NftTcp, port: runner_microvm_guest_dns_port, destination: NamedDestination { address: runner_microvm_guest_resolver() } }, + GuestEgressGrant { transport: NftTcp, port: runner_microvm_guest_tls_port, destination: AnyUndeniedDestination }, ], } @@ -637,7 +641,7 @@ fn conntrack_readback(o: GuestConntrackObservation) -> SlotNetworkReadback { fn ruleset_readback(o: RulesetObservation, converged: Sha256Digest) -> SlotNetworkReadback { match o { RulesetListed { digest: d } => - if (d.hex as String) == (converged.hex as String) { + if content_hash_eq_cryptographic(left: d, right: converged) { SlotNetworkConfirmed } else { SlotNetworkRefuted { From 145f93f1927db06289dd17a7202fe25b0b7f6102 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 07:09:37 +0000 Subject: [PATCH 07/36] Take the nft ruleset out of the per-attempt plan; declare the host converge as its installer Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_attempt_launch.dag | 5 ----- .../runner/runner_host_filtered_egress.dag | 4 ++++ dag/gunbc/runner/runner_microvm_network.dag | 17 ++++++++++++++++- .../runner_attempt_launch_witness_test.dag | 5 +---- .../runner_microvm_network_witness_test.dag | 16 ++++++++-------- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/dag/gunbc/runner/runner_attempt_launch.dag b/dag/gunbc/runner/runner_attempt_launch.dag index ae52c2d0e73..3518d9a18fc 100644 --- a/dag/gunbc/runner/runner_attempt_launch.dag +++ b/dag/gunbc/runner/runner_attempt_launch.dag @@ -47,7 +47,6 @@ import gunbc.runner_microvm_network { runner_slot_network, runner_slot_network_as_microvm_networking, runner_slot_guest_ip_boot_arg, - runner_microvm_nft_ruleset_text, } import extdeps.systemd.systemd_run { SystemdRunProperty } @@ -171,7 +170,6 @@ type AttemptLaunchPlan authorization: GrantAuthorization tap_device: NonEmptyStr guest_ip_boot_arg: String - nft_ruleset: String slice: SystemdRunProperty jail_root: String stage_jail_root: ArgvCommand @@ -379,7 +377,6 @@ fn plan_authorized_steps( authorization: authorization, tap_device: net.tap_device, guest_ip_boot_arg: runner_slot_guest_ip_boot_arg(net: net), - nft_ruleset: runner_microvm_nft_ruleset_text(), slice: runner_microvm_attempt_slice_property(attempt: attempt), jail_root: jail_root, stage_jail_root: mkdir_parents_command(path: jail_root), @@ -436,7 +433,6 @@ fn planned_jailer_launches(plan: AttemptLaunchPlan) -> Int { authorization: _, tap_device: _, guest_ip_boot_arg: _, - nft_ruleset: _, slice: _, jail_root: _, stage_jail_root: _, @@ -535,7 +531,6 @@ fn staging_verdict(plan: AttemptLaunchPlan, obs: WorkspaceStagingObservation) -> authorization: _, tap_device: _, guest_ip_boot_arg: _, - nft_ruleset: _, slice: _, jail_root: _, stage_jail_root: _, diff --git a/dag/gunbc/runner/runner_host_filtered_egress.dag b/dag/gunbc/runner/runner_host_filtered_egress.dag index 0db8d224e08..f471f4d0963 100644 --- a/dag/gunbc/runner/runner_host_filtered_egress.dag +++ b/dag/gunbc/runner/runner_host_filtered_egress.dag @@ -86,6 +86,10 @@ type ChainStance = DefaultDrop | DefaultAccept // grain where the shared hosts cannot, and its v6 is disabled on the data path (read back as 1 on // the filtered-egress-4 run) where theirs is filtered. Both are this host's HostEgressGrain, so the // record carries one value for each fact. +// +// This host is NOT a fleet-converge target of the srvN microVM program: its grain has no installer in +// this tree, and the one that would install it is its host image (runner_attempt_launch +// host_boot_cutover_frontier), not the srvN converge named by host_ruleset_installer_frontier. type EgressPolicyShape { host_input: ChainStance host: HostEgressGrain diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 5343f9c5509..83a8e967cc6 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -459,7 +459,22 @@ fn runner_microvm_nft_ruleset_text_for(host: HostEgressGrain) -> String { nft_table_wire(t: runner_microvm_nft_table(host: host)) } -fn runner_microvm_nft_ruleset_text() -> String { +// THE RULESET IS HOST STATE AND ITS INSTALLER TAKES THE HOST'S OWN GRAIN. The per-attempt launch +// plan carried a ruleset until review 68256: an attempt is not the owner of a host-wide table (parent +// ruling A, 2026-09-19), and a plan with no host identity could only ever render one host's grain. +// The installer is the host converge, which knows which host it is converging. +// +// DECLARED FRONTIER (DESIGN 3c). NAMED CONSUMER: the microVM host converge (the microvm_host_converge +// fleet-converge mode) installing the table on srvN. TRIGGER: that converge installs +// runner_microvm_nft_ruleset_text_for(host: ), reads the +// listing back with runner_microvm_nft_list_command and records its digest as the converged digest +// slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by a +// witness folding a grain, and not by any per-attempt step installing a table. +data host_ruleset_installer_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: the microvm_host_converge fleet-converge mode on srvN. TRIGGER: it installs runner_microvm_nft_ruleset_text_for(host: the converged host's HostEgressGrain), reads the listing back with runner_microvm_nft_list_command, and records that digest as the converged digest slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by a witness folding a grain or by a per-attempt step installing a table." as NonEmptyStr, +) + +fn runner_microvm_shared_host_nft_ruleset_text() -> String { runner_microvm_nft_ruleset_text_for(host: runner_microvm_shared_host_grain) } diff --git a/dag/test/claim/runner/runner_attempt_launch_witness_test.dag b/dag/test/claim/runner/runner_attempt_launch_witness_test.dag index 1a4228274de..d9cdeae176f 100644 --- a/dag/test/claim/runner/runner_attempt_launch_witness_test.dag +++ b/dag/test/claim/runner/runner_attempt_launch_witness_test.dag @@ -24,7 +24,7 @@ import gunbc.runner_microvm_attempt { runner_microvm_attempt_jail_base, } import gunbc.runner_microvm_network { - runner_slot_tap_device_name, runner_microvm_nft_ruleset_text, + runner_slot_tap_device_name, } import std.dissolution { UnboundDissolution, BoundDissolution } import gunbc.guarantee_rung { Mitigatable, MechanicallyPreventable, OutsideTheLadder, StructurallyGuaranteed, StructurallyImpossible } @@ -160,7 +160,6 @@ test fn a_zero_byte_credential_does_not_start_the_vmm() -> Bool { authorization: _, tap_device: _, guest_ip_boot_arg: _, - nft_ruleset: _, slice: _, jail_root: _, stage_jail_root: _, @@ -231,7 +230,6 @@ test fn an_authorized_plan_carries_network_slice_workspace_and_jailer_command() authorization: _, tap_device: tap, guest_ip_boot_arg: ip, - nft_ruleset: nft, slice: _, jail_root: jail_root, stage_jail_root: stage_root, @@ -247,7 +245,6 @@ test fn an_authorized_plan_carries_network_slice_workspace_and_jailer_command() let words = join(argv_words(command: jailer), " ") let create_words = join(argv_words(command: create), " ") (tap as String) == (runner_slot_tap_device_name(slot: a.slot) as String) - && nft == runner_microvm_nft_ruleset_text() && jail_root == attempt_jail_chroot_root(attempt: a, firecracker_binary: "/bin/firecracker") && (stage_root.program as String) == (mkdir_binary_path as String) && (stage_k.program as String) == "cp" diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index 50a4ca94007..c2d289961b2 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -10,7 +10,7 @@ import gunbc.fleet_intent_network { operator_host_srv2 } import gunbc.runner_microvm_network { runner_slot_network, runner_slot_tap_device_name, runner_slot_guest_mac, runner_slot_host_address, runner_slot_guest_address, - runner_slot_guest_ip_boot_arg, runner_microvm_nft_ruleset_text, + runner_slot_guest_ip_boot_arg, runner_microvm_shared_host_nft_ruleset_text, runner_slot_netdev_text, runner_slot_network_file_text, runner_slot_netdev_path, runner_slot_network_file_path, netmask_of_prefix, runner_microvm_network_supernet, @@ -82,7 +82,7 @@ test fn the_guest_ip_directive_carries_every_positional_separator() -> Bool { // wins, so `policy drop` in our forward chain would drop packets dockerd forwards. The red here is // that exact regression: flip the chain's policy and this goes red before it reaches a host. test fn the_forward_chain_defaults_to_accept_and_drops_only_tap_to_tap() -> Bool { - let text = runner_microvm_nft_ruleset_text() + let text = runner_microvm_shared_host_nft_ruleset_text() string_contains(s: text, pattern: "hook forward") && string_contains(s: text, pattern: "policy accept") && !string_contains(s: text, pattern: "policy drop") @@ -93,7 +93,7 @@ test fn the_forward_chain_defaults_to_accept_and_drops_only_tap_to_tap() -> Bool // must converge rather than accumulate, and `delete table` on an absent table is an ERROR in // nftables rather than a no-op -- so the add must come FIRST. The red is either line missing. test fn the_ruleset_replaces_its_own_table_and_guards_the_delete() -> Bool { - let text = runner_microvm_nft_ruleset_text() + let text = runner_microvm_shared_host_nft_ruleset_text() string_contains(s: text, pattern: "add table inet gunbc_microvm") && string_contains(s: text, pattern: "delete table inet gunbc_microvm") } @@ -102,7 +102,7 @@ test fn the_ruleset_replaces_its_own_table_and_guards_the_delete() -> Bool { // one is host-specific and rots on a NIC rename. Source-matching the fleet supernet is exact and // identical everywhere, so the control asserts the masquerade is reached by source address. test fn the_masquerade_matches_the_supernet_and_names_no_interface() -> Bool { - let text = runner_microvm_nft_ruleset_text() + let text = runner_microvm_shared_host_nft_ruleset_text() string_contains(s: text, pattern: "ip saddr 172.30.0.0/16 masquerade") && !string_contains(s: text, pattern: "enP3p3s0f1") } @@ -173,7 +173,7 @@ test fn the_nat_supernet_contains_the_slot_addresses_it_claims_to() -> Bool { let supernet = runner_microvm_network_supernet() as String let host_octets = render_ipv4_address(addr: net.host_address) as String string_contains(s: supernet, pattern: to_string(runner_microvm_network_supernet_prefix)) - && string_contains(s: runner_microvm_nft_ruleset_text(), pattern: supernet) + && string_contains(s: runner_microvm_shared_host_nft_ruleset_text(), pattern: supernet) && string_contains(s: host_octets, pattern: "172.30.") && string_contains(s: supernet, pattern: "172.30.") } @@ -184,7 +184,7 @@ test fn the_nat_supernet_contains_the_slot_addresses_it_claims_to() -> Bool { // nat, so this pins the chain type that carries the masquerade AND the one that must stay filter -- // the red is the previous renderer, which hardcoded filter for both. test fn the_nat_chain_is_typed_nat_and_the_filter_chain_stays_filter() -> Bool { - let text = runner_microvm_nft_ruleset_text() + let text = runner_microvm_shared_host_nft_ruleset_text() string_contains(s: text, pattern: "type nat hook postrouting") && string_contains(s: text, pattern: "type filter hook forward") && !string_contains(s: text, pattern: "type filter hook postrouting") @@ -198,7 +198,7 @@ test fn the_nat_chain_is_typed_nat_and_the_filter_chain_stays_filter() -> Bool { // 443, the ordering the Mt. Collins receipt showed holding -- and a missing final drop, which turns // default-deny into default-accept for every port nobody named. test fn guest_egress_is_default_deny_at_tap_grain_in_the_receipted_order() -> Bool { - let text = runner_microvm_nft_ruleset_text() + let text = runner_microvm_shared_host_nft_ruleset_text() string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" meta nfproto ipv6 drop\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") && string_contains(s: text, pattern: "policy accept") @@ -329,7 +329,7 @@ test fn the_ruleset_readback_lists_our_table_and_its_consumer_is_a_frontier() -> // guest -- review 68173), and a boot directive naming a resolver the ruleset does not admit. test fn the_dns_grant_is_the_booted_resolver_and_no_other_host() -> Bool { let resolver = render_ipv4_address(addr: runner_microvm_guest_resolver()) as String - let text = runner_microvm_nft_ruleset_text() + let text = runner_microvm_shared_host_nft_ruleset_text() let arg = runner_slot_guest_ip_boot_arg(net: runner_slot_network(slot: slot(i: 7))) !string_contains(s: text, pattern: "\" udp dport 53") && !string_contains(s: text, pattern: "\" tcp dport 53") From a75964b468404416bf7e0a526aab0f82cf102225 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 07:55:41 +0000 Subject: [PATCH 08/36] Denied destinations are typed Ipv4Prefix rows rendered through the ipv4 authority Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/network/ipv4.dag | 16 +++++++++++++++ dag/gunbc/runner/runner_microvm_network.dag | 22 ++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/dag/extdeps/network/ipv4.dag b/dag/extdeps/network/ipv4.dag index 91e1bd6626a..17f28a369db 100644 --- a/dag/extdeps/network/ipv4.dag +++ b/dag/extdeps/network/ipv4.dag @@ -78,6 +78,22 @@ fn render_ipv4_cidr(addr: Ipv4Address, prefix: PrefixLength) -> NonEmptyStr { join([render_ipv4_address(addr: addr) as String, to_string(prefix)], "/") as NonEmptyStr } +// A NETWORK HELD AS A VALUE rather than as its rendering, for rosters of networks -- a denial list, +// a route table -- where each row must be a well-formed address and a prefix that exists. The CIDR +// text stays a projection of this, never a second spelling authored beside it. +type Ipv4Prefix { + network: Ipv4Address + length: PrefixLength +} + +fn ipv4_prefix(o1: Octet, o2: Octet, o3: Octet, o4: Octet, length: PrefixLength) -> Ipv4Prefix { + Ipv4Prefix { network: ipv4_address(octet1: o1, octet2: o2, octet3: o3, octet4: o4), length: length } +} + +fn render_ipv4_prefix(p: Ipv4Prefix) -> NonEmptyStr { + render_ipv4_cidr(addr: p.network, prefix: p.length) +} + // Whether two addresses share a network at a given prefix. Written as an octet-wise comparison over // the four whole octets a /8, /16, /24 or /32 fixes, plus the partial octet a /30 splits -- so it is // exact for the prefixes this repository actually uses and REFUSES to answer for the rest, rather diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 83a8e967cc6..1c99d2801e3 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -5,7 +5,7 @@ import std.nat { Nat } import std.disposition { Disposition, SingleAuthority } import std.dissolution { DissolutionCondition, unbound_dissolution } import std.content_hash { Sha256Digest, content_hash_eq_cryptographic } -import extdeps.network.ipv4 { Ipv4Address, ipv4_address, render_ipv4_address } +import extdeps.network.ipv4 { Ipv4Address, ipv4_address, render_ipv4_address, render_ipv4_cidr, Ipv4Prefix, ipv4_prefix, render_ipv4_prefix } import extdeps.network.mac { Eui48Address, mac_address, render_mac_address, MacAddressScope, LocallyAdministered, mac_address_scope } import gunbc.build_cache_instance { RunnerSlotIdentity } import gunbc.runner_microvm { RunnerMicroVmNetworking, GuestNetworkTap } @@ -304,7 +304,7 @@ type HostEgressGrain { } type GuestEgressPolicy { - denied_destinations: List + denied_destinations: List grants: List } @@ -334,13 +334,13 @@ data runner_microvm_guest_tls_port: Port = 443 // CDN-fronted, so a static address list is stale when written. The narrow form is an egress proxy. data runner_microvm_guest_egress_policy: GuestEgressPolicy = GuestEgressPolicy { denied_destinations: [ - "10.0.0.0/8" as NonEmptyStr, - "172.16.0.0/12" as NonEmptyStr, - "192.168.0.0/16" as NonEmptyStr, - "100.64.0.0/10" as NonEmptyStr, - "169.254.0.0/16" as NonEmptyStr, - "127.0.0.0/8" as NonEmptyStr, - "224.0.0.0/4" as NonEmptyStr, + ipv4_prefix(o1: 10, o2: 0, o3: 0, o4: 0, length: 8), + ipv4_prefix(o1: 172, o2: 16, o3: 0, o4: 0, length: 12), + ipv4_prefix(o1: 192, o2: 168, o3: 0, o4: 0, length: 16), + ipv4_prefix(o1: 100, o2: 64, o3: 0, o4: 0, length: 10), + ipv4_prefix(o1: 169, o2: 254, o3: 0, o4: 0, length: 16), + ipv4_prefix(o1: 127, o2: 0, o3: 0, o4: 0, length: 8), + ipv4_prefix(o1: 224, o2: 0, o3: 0, o4: 0, length: 4), ], grants: [ GuestEgressGrant { transport: NftUdp, port: runner_microvm_guest_dns_port, destination: NamedDestination { address: runner_microvm_guest_resolver() } }, @@ -360,7 +360,7 @@ fn grant_destination_matches(d: GuestEgressDestination) -> List { match d { AnyUndeniedDestination => [] NamedDestination { address: a } => - [NftDestinationAddressPrefix { cidr: join([render_ipv4_address(addr: a) as String, "/32"], "") as NonEmptyStr }] + [NftDestinationAddressPrefix { cidr: render_ipv4_cidr(addr: a, prefix: 32) }] } } @@ -417,7 +417,7 @@ fn guest_egress_rules(p: GuestEgressPolicy, host: HostEgressGrain) -> List from_guest_tap(m: [NftDestinationAddressPrefix { cidr: c }], verdict: NftDrop)), + map(p.denied_destinations, c => from_guest_tap(m: [NftDestinationAddressPrefix { cidr: render_ipv4_prefix(p: c) }], verdict: NftDrop)), ), map(p.grants, g => grant_rule(g: g)), ), From 9af5eae410a11ed21ee7b872851958b620163462 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 08:34:18 +0000 Subject: [PATCH 09/36] Drop prose denials that restate the typed guest policy and the forward chain Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_host_filtered_egress.dag | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/runner/runner_host_filtered_egress.dag b/dag/gunbc/runner/runner_host_filtered_egress.dag index f471f4d0963..6b9afc24471 100644 --- a/dag/gunbc/runner/runner_host_filtered_egress.dag +++ b/dag/gunbc/runner/runner_host_filtered_egress.dag @@ -90,11 +90,14 @@ type ChainStance = DefaultDrop | DefaultAccept // This host is NOT a fleet-converge target of the srvN microVM program: its grain has no installer in // this tree, and the one that would install it is its host image (runner_attempt_launch // host_boot_cutover_frontier), not the srvN converge named by host_ruleset_installer_frontier. +// THE DENIALS ARE NOT LISTED HERE BECAUSE THEY ARE ALREADY RULES. The BMC and management LAN fall in +// the guest policy's private-range rows (the BMC measured on the filtered-egress run is 192.168.1.228), +// and tap to tap is the forward chain's first rule in gunbc.runner_microvm_network; a prose list here +// would be a third spelling of both (review 68294). type EgressPolicyShape { host_input: ChainStance host: HostEgressGrain guest: GuestEgressPolicy - explicit_denials: List nat: NonEmptyStr } @@ -102,10 +105,6 @@ data runner_host_egress_policy: EgressPolicyShape = EgressPolicyShape { host_input: DefaultDrop, host: HostEgressGrain { forward: HostChainDefaultDrop, ipv6: IpV6DisabledOnDataPath }, guest: runner_microvm_guest_egress_policy, - explicit_denials: [ - "guest to the BMC and any management address", - "tap to tap -- no attempt may reach another attempt", - ], nat: "masquerade the admitted guest prefix on the host uplink, and nothing else", } From 25574f8e6d59fa3434955b20104d00e5cc7cdb93 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 17:49:20 +0000 Subject: [PATCH 10/36] Readbacks carry their subject; workspace readback bound to the planned image; denials precede every guest-side accept; tap v6 refused before conntrack Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/exec/command.dag | 4 +- dag/extdeps/nftables/ruleset.dag | 7 +- dag/extdeps/tools/conntrack.dag | 31 ++- dag/gunbc/runner/runner_attempt_launch.dag | 20 +- dag/gunbc/runner/runner_microvm_network.dag | 238 +++++++++++++----- .../runner_attempt_launch_witness_test.dag | 35 ++- .../runner_microvm_network_witness_test.dag | 215 ++++++++++------ 7 files changed, 388 insertions(+), 162 deletions(-) diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index b9073b3daac..49e9a9cc206 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -93,8 +93,8 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.tools.e2fsprogs", decl_name: "mke2fs_format_command"), decl_ref(module_path: "extdeps.tools.e2fsprogs", decl_name: "dumpe2fs_superblock_command"), decl_ref(module_path: "extdeps.nftables.ruleset", decl_name: "nft_list_table_command"), - decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_delete_by_source_command"), - decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_list_by_source_command"), + decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_delete_command"), + decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_list_command"), decl_ref(module_path: "extdeps.tools.make", decl_name: "make_command"), decl_ref(module_path: "extdeps.package_managers.pip", decl_name: "pip_install_pinned_command"), decl_ref(module_path: "extdeps.package_managers.conda_forge", decl_name: "micromamba_create_command"), diff --git a/dag/extdeps/nftables/ruleset.dag b/dag/extdeps/nftables/ruleset.dag index 31c69fca108..d70b904f843 100644 --- a/dag/extdeps/nftables/ruleset.dag +++ b/dag/extdeps/nftables/ruleset.dag @@ -48,19 +48,23 @@ fn nft_chain_type_wire(t: NftChainType) -> String { type NftHook = NftForward | NftPostrouting + | NftPrerouting // PRIORITY IS AN INTEGER WITH NAMED LANDMARKS, and the landmarks are the ones nftables itself -// documents: filter is 0, srcnat is 100. They are named rather than written as literals at the call +// documents: filter is 0, srcnat is 100, raw is -300. raw runs BEFORE connection tracking (-200), so +// a packet dropped there never creates a conntrack entry at all. They are named rather than written as literals at the call // site because a rule installed at the wrong priority still loads, still matches, and simply runs // on the wrong side of another subsystem's rules -- a silent misordering rather than a refusal. type NftChainPriority = NftPriorityFilter | NftPrioritySrcnat + | NftPriorityRaw fn nft_chain_priority_wire(p: NftChainPriority) -> String { match p { NftPriorityFilter => "0" NftPrioritySrcnat => "100" + NftPriorityRaw => "-300" } } @@ -75,6 +79,7 @@ fn nft_hook_wire(h: NftHook) -> String { match h { NftForward => "forward" NftPostrouting => "postrouting" + NftPrerouting => "prerouting" } } diff --git a/dag/extdeps/tools/conntrack.dag b/dag/extdeps/tools/conntrack.dag index 5d640e7c5a5..713574615cf 100644 --- a/dag/extdeps/tools/conntrack.dag +++ b/dag/extdeps/tools/conntrack.dag @@ -26,21 +26,36 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope { } // conntrack(8) from conntrack-tools, the userspace reader and writer of the kernel's connection -// tracking table. Only the two operations a table keyed by ORIGINAL SOURCE needs are modeled: delete -// the flows a source opened, and list them. `-s` filters on the original-direction source, which for -// a masqueraded guest is the guest's own address -- the translated address appears only on the -// reply tuple, so filtering on it would miss every flow a guest opened. +// tracking table. Only deleting and listing flows by an original-tuple address are modeled. For a +// masqueraded guest the original tuple carries the guest's own address -- the translated address +// appears only on the reply tuple, so filtering on it would miss every flow a guest opened. data conntrack_program: NonEmptyStr = "conntrack" +// A FLOW HAS TWO ADDRESSES IN ITS ORIGINAL TUPLE AND conntrack(8) FILTERS EACH SEPARATELY: `-s` is +// the original source, `-d` the original destination. A guest that opened a flow is its original +// source; a flow the HOST opened toward the guest has the guest as original destination. Flushing by +// one filter leaves every flow of the other kind, so the filter is a parameter rather than a choice +// baked into the command. +type ConntrackOriginalAddress + = ConntrackOriginalSource + | ConntrackOriginalDestination + +fn conntrack_original_address_flag(f: ConntrackOriginalAddress) -> String { + match f { + ConntrackOriginalSource => "-s" + ConntrackOriginalDestination => "-d" + } +} + // `-D` WITH NO MATCHING ENTRY EXITS NONZERO ("0 flow entries have been deleted"), so its exit status // is not a success signal a caller may gate on. The deletion's outcome is established by the list // that follows it, never by this command's exit. -fn conntrack_delete_by_source_command(source: String) -> ArgvCommand { - argv_command(program: conntrack_program, arguments: ["-D", "-s", source]) +fn conntrack_delete_command(filter: ConntrackOriginalAddress, address: String) -> ArgvCommand { + argv_command(program: conntrack_program, arguments: ["-D", conntrack_original_address_flag(f: filter), address]) } // `-L` prints one line per flow on stdout and a summary on stderr, so the count of stdout lines is // the number of surviving flows and an empty stdout with exit zero is the empty reading. -fn conntrack_list_by_source_command(source: String) -> ArgvCommand { - argv_command(program: conntrack_program, arguments: ["-L", "-s", source]) +fn conntrack_list_command(filter: ConntrackOriginalAddress, address: String) -> ArgvCommand { + argv_command(program: conntrack_program, arguments: ["-L", conntrack_original_address_flag(f: filter), address]) } diff --git a/dag/gunbc/runner/runner_attempt_launch.dag b/dag/gunbc/runner/runner_attempt_launch.dag index 3518d9a18fc..147794033bd 100644 --- a/dag/gunbc/runner/runner_attempt_launch.dag +++ b/dag/gunbc/runner/runner_attempt_launch.dag @@ -474,8 +474,12 @@ fn work_standing_from_vmm_exit(exit_code: Int) -> WorkNotEstablished { // // The image lives under the attempt's jail root (runner_microvm_attempt_jail_base), so teardown's // one subtree to prove absent already contains it; nothing here adds a second location. +// THE READING NAMES THE IMAGE IT READ. A superblock read of some other image -- the previous +// attempt's, another slot's -- says nothing about this plan's workspace, so the path travels with +// the reading and staging compares it against the plan's own workspace_path (sunny-ant-606 ruling on +// #11675). The path is attempt-bound by construction: it sits under the attempt's jail root. type WorkspaceReadback - = WorkspaceSuperblockRead { ext_magic: Bool, bytes: ByteSize } + = WorkspaceSuperblockRead { image_path: String, ext_magic: Bool, bytes: ByteSize } | WorkspaceReadbackFailed { reason: NonEmptyStr } type WorkspaceStagingObservation { @@ -491,6 +495,7 @@ type WorkspaceStanding | WorkspaceNotExtFilesystem | WorkspaceSizeMismatch { observed: ByteSize, expected: ByteSize } | WorkspaceUnreadable { reason: NonEmptyStr } + | WorkspaceReadbackOfDifferentImage { read: String, planned: String } fn runner_attempt_workspace_bytes() -> ByteSize { mebibyte_to_byte_size(m: runner_attempt_workspace_size) @@ -498,7 +503,7 @@ fn runner_attempt_workspace_bytes() -> ByteSize { // THE ORDER IS CAUSAL: a failed create makes the format's exit meaningless, and a failed format // makes the superblock reading meaningless, so the earliest failure is the one reported. -fn workspace_standing(obs: WorkspaceStagingObservation) -> WorkspaceStanding { +fn workspace_standing(obs: WorkspaceStagingObservation, planned_path: String) -> WorkspaceStanding { if obs.create_exit != 0 { WorkspaceCreateFailed { exit: obs.create_exit } } else if obs.format_exit != 0 { @@ -506,8 +511,10 @@ fn workspace_standing(obs: WorkspaceStagingObservation) -> WorkspaceStanding { } else { match obs.readback { WorkspaceReadbackFailed { reason: r } => WorkspaceUnreadable { reason: r } - WorkspaceSuperblockRead { ext_magic: m, bytes: b } => - if !m { + WorkspaceSuperblockRead { image_path: read, ext_magic: m, bytes: b } => + if read != planned_path { + WorkspaceReadbackOfDifferentImage { read: read, planned: planned_path } + } else if !m { WorkspaceNotExtFilesystem } else if byte_size_count(b: b) != byte_size_count(b: runner_attempt_workspace_bytes()) { WorkspaceSizeMismatch { observed: b, expected: runner_attempt_workspace_bytes() } @@ -536,14 +543,14 @@ fn staging_verdict(plan: AttemptLaunchPlan, obs: WorkspaceStagingObservation) -> stage_jail_root: _, stage_kernel: _, stage_rootfs: _, - workspace_path: _, + workspace_path: planned, workspace_create: _, workspace_format: _, kvm_device: _, vm_config: _, jailer: j, } => { - let s = workspace_standing(obs: obs) + let s = workspace_standing(obs: obs, planned_path: planned) match s { WorkspaceReady => StagingAdmitsJailer { jailer: j } WorkspaceCreateFailed { exit: _ } => StagingRefusedWorkspace { standing: s } @@ -551,6 +558,7 @@ fn staging_verdict(plan: AttemptLaunchPlan, obs: WorkspaceStagingObservation) -> WorkspaceNotExtFilesystem => StagingRefusedWorkspace { standing: s } WorkspaceSizeMismatch { observed: _, expected: _ } => StagingRefusedWorkspace { standing: s } WorkspaceUnreadable { reason: _ } => StagingRefusedWorkspace { standing: s } + WorkspaceReadbackOfDifferentImage { read: _, planned: _ } => StagingRefusedWorkspace { standing: s } } } LaunchRefusedUnauthorized { authorization: _ } => StagingNotAuthorized diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 1c99d2801e3..5caa820ad7e 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -5,16 +5,17 @@ import std.nat { Nat } import std.disposition { Disposition, SingleAuthority } import std.dissolution { DissolutionCondition, unbound_dissolution } import std.content_hash { Sha256Digest, content_hash_eq_cryptographic } -import extdeps.network.ipv4 { Ipv4Address, ipv4_address, render_ipv4_address, render_ipv4_cidr, Ipv4Prefix, ipv4_prefix, render_ipv4_prefix } +import extdeps.network.ipv4 { Ipv4Address, ipv4_address, render_ipv4_address, render_ipv4_cidr, Ipv4Prefix, ipv4_prefix, render_ipv4_prefix, ipv4_address_eq } import extdeps.network.mac { Eui48Address, mac_address, render_mac_address, MacAddressScope, LocallyAdministered, mac_address_scope } -import gunbc.build_cache_instance { RunnerSlotIdentity } +import gunbc.build_cache_instance { RunnerSlotIdentity, runner_slot_identity_equal } +import product.placement_supply { HostIdentity } import gunbc.runner_microvm { RunnerMicroVmNetworking, GuestNetworkTap } import gunbc.executor_privileged_operation { ExecutorPrivilegedOperation, InstallFile, SystemdReloadUnit, } import extdeps.nftables.ruleset { NftTable, NftChain, NftRule, NftMatch, NftFamily, NftInet, NftHook, NftForward, NftPostrouting, - NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPolicy, NftPolicyAccept, NftPolicyDrop, + NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPriorityRaw, NftPrerouting, NftPolicy, NftPolicyAccept, NftPolicyDrop, NftChainType, NftTypeFilter, NftTypeNat, NftVerdict, NftAccept, NftDrop, NftMasquerade, NftInputInterfaceWildcard, NftOutputInterfaceWildcard, NftSourceAddressPrefix, @@ -24,7 +25,7 @@ import extdeps.nftables.ruleset { } import extdeps.exec.command { ArgvCommand } import extdeps.tools.conntrack { - conntrack_delete_by_source_command, conntrack_list_by_source_command, + conntrack_delete_command, conntrack_list_command, ConntrackOriginalSource, ConntrackOriginalDestination, } data runner_microvm_network_disposition: Disposition = SingleAuthority @@ -371,14 +372,6 @@ fn grant_rule(g: GuestEgressGrant) -> NftRule { ) } -fn ipv6_rules(v: IpV6Standing) -> List { - match v { - IpV6DefaultDenyRuleset => [from_guest_tap(m: [NftNetworkProtocolIpv6], verdict: NftDrop)] - IpV6DisabledOnDataPath => [] - IpV6Unaddressed { risk: _ } => [] - } -} - fn final_drop_rules(g: ForwardDenyGrain) -> List { match g { TapScopedFinalDrop { chain_policy_reason: _ } => [from_guest_tap(m: [], verdict: NftDrop)] @@ -399,10 +392,13 @@ fn forward_chain_policy(g: ForwardDenyGrain) -> NftPolicy { // the chain policy drops them and no guest flow would complete (review 68199). So the reply rule is // emitted on every grain, and nothing reaches a tap unsolicited under a dropping chain. // -// THE ORDER IS THE POLICY: established first, so replies to admitted flows are never re-judged; the -// v6 refusal before any v4 grant; the denials ABOVE the grants, because the 443 grant is a class and -// 192.168.x.y:443 is in it (the Mt. Collins receipt's denied_paths is this ordering holding); the -// grants; then the grain's final drop. +// THE ORDER IS THE POLICY. Replies TO a tap first: they belong to flows a grant already admitted. +// Then the invariant denials, ABOVE every accept that starts from a tap -- including the guest-side +// established accept, because an established or related flow toward a denied range (a related ICMP, +// a flow that predates a ruleset change) must still be refused (sunny-ant-606 ruling on #11675). Then +// the guest-side established accept, the grants, and the grain's final drop. The denials sit above +// the 443 grant because that grant is a class and 192.168.x.y:443 is in it (the Mt. Collins receipt's +// denied_paths is this ordering holding). v6 never reaches this chain; see raw_ipv6_rules. fn guest_egress_rules(p: GuestEgressPolicy, host: HostEgressGrain) -> List { concat( concat( @@ -413,11 +409,10 @@ fn guest_egress_rules(p: GuestEgressPolicy, host: HostEgressGrain) -> List from_guest_tap(m: [NftDestinationAddressPrefix { cidr: render_ipv4_prefix(p: c) }], verdict: NftDrop)), ), - map(p.denied_destinations, c => from_guest_tap(m: [NftDestinationAddressPrefix { cidr: render_ipv4_prefix(p: c) }], verdict: NftDrop)), + [from_guest_tap(m: [NftConnectionTrackingEstablished], verdict: NftAccept)], ), map(p.grants, g => grant_rule(g: g)), ), @@ -425,6 +420,29 @@ fn guest_egress_rules(p: GuestEgressPolicy, host: HostEgressGrain) -> List List { + match v { + IpV6DefaultDenyRuleset => [from_guest_tap(m: [NftNetworkProtocolIpv6], verdict: NftDrop)] + IpV6DisabledOnDataPath => [] + IpV6Unaddressed { risk: _ } => [] + } +} + +fn runner_microvm_raw_prerouting_chain(host: HostEgressGrain) -> NftChain { + NftChain { + name: "raw_prerouting" as NonEmptyStr, + chain_type: NftTypeFilter, + hook: NftPrerouting, + priority: NftPriorityRaw, + policy: NftPolicyAccept, + rules: raw_ipv6_rules(v: host.ipv6), + } +} // THE MASQUERADE MATCHES ON SOURCE AND NOT ON OUTPUT INTERFACE, which removes an observation this // module would otherwise have to make and keep correct. srv2's uplink is enP3p3s0f1 and srv1's is @@ -451,7 +469,7 @@ fn runner_microvm_nft_table(host: HostEgressGrain) -> NftTable { NftTable { family: NftInet, name: runner_microvm_nft_table_name, - chains: [runner_microvm_forward_chain(host: host), runner_microvm_postrouting_chain()], + chains: [runner_microvm_raw_prerouting_chain(host: host), runner_microvm_forward_chain(host: host), runner_microvm_postrouting_chain()], } } @@ -598,9 +616,46 @@ fn runner_slot_network_install_operations( // given, so the text above and the kernel's listing differ in bytes while agreeing in meaning. The // converged digest is therefore the listing captured by the converge that installed the table, and // it is supplied here from that receipt rather than recomputed from the rule rows. +// EVERY READING NAMES WHAT IT IS ABOUT, AND A READING ABOUT SOMETHING ELSE ESTABLISHES NOTHING HERE. +// The three facts are about one slot on one host: its tap, the flows keyed to its guest address, and +// the table the host converge installed at one generation. A reading carries that subject, the fold +// is handed the subject it is sanitizing, and a reading whose subject differs is Unobservable for +// that fact -- never Confirmed -- so slot A's clean readings cannot release slot B (sunny-ant-606 +// ruling on #11675). The lifecycle controller joins the expected subject to the attempt's cell. +type SlotNetworkSubject { + slot: RunnerSlotIdentity + tap_device: NonEmptyStr + guest_address: Ipv4Address +} + +fn slot_network_subject(slot: RunnerSlotIdentity) -> SlotNetworkSubject { + let net = runner_slot_network(slot: slot) + SlotNetworkSubject { slot: slot, tap_device: net.tap_device, guest_address: net.guest_address } +} + +fn slot_network_subject_equal(a: SlotNetworkSubject, b: SlotNetworkSubject) -> Bool { + runner_slot_identity_equal(a: a.slot, b: b.slot) + && (a.tap_device as String) == (b.tap_device as String) + && ipv4_address_eq(a: a.guest_address, b: b.guest_address) +} + +// THE CONVERGED RULESET IS HOST STATE AT A GENERATION. The digest alone would let a listing taken +// after a re-converge be compared against the previous generation's receipt, or a listing from one +// host against another host's. +type ConvergedRuleset { + host: HostIdentity + generation: Int + digest: Sha256Digest +} + +// CONNTRACK IS READ IN BOTH ORIGINAL DIRECTIONS. A flow the guest opened has it as original source; a +// flow the host opened toward the guest has it as original destination. Both are keyed to an address +// the next attempt in this slot reuses. v6 needs no reading: raw_ipv6_rules refuses tap-originated v6 +// before connection tracking, so no v6 entry keyed to a tap can exist. type SlotNetworkFact = TapUnheldFact - | GuestConntrackEmptyFact + | GuestOriginatedConntrackEmptyFact + | GuestDestinedConntrackEmptyFact | RulesetMatchesConvergedFact type TapHolderObservation @@ -618,6 +673,22 @@ type RulesetObservation = RulesetListed { digest: Sha256Digest } | RulesetUnreadable { reason: NonEmptyStr } +type TapReading { + subject: SlotNetworkSubject + observation: TapHolderObservation +} + +type GuestConntrackReading { + subject: SlotNetworkSubject + observation: GuestConntrackObservation +} + +type RulesetReading { + host: HostIdentity + generation: Int + observation: RulesetObservation +} + // "COULD NOT READ" IS NOT "REFUTED", and the two arms stay separate because they call for different // operators: a refutation is a fact about the host that needs remedying, an unobservable reading is // a fact about the instrument. Both quarantine the cell -- neither is CellReady -- but collapsing @@ -627,44 +698,66 @@ type SlotNetworkReadback | SlotNetworkRefuted { fact: SlotNetworkFact, detail: NonEmptyStr } | SlotNetworkUnobservable { fact: SlotNetworkFact, reason: NonEmptyStr } -fn tap_readback(o: TapHolderObservation) -> SlotNetworkReadback { - match o { - TapPresentUnheld => SlotNetworkConfirmed - TapHeld { holders: n } => SlotNetworkRefuted { - fact: TapUnheldFact, - detail: join([to_string(n), " process(es) still hold the slot tap after the attempt"], "") as NonEmptyStr, - } - TapMissing => SlotNetworkRefuted { - fact: TapUnheldFact, - detail: "the slot tap is absent, so the converged network state no longer holds" as NonEmptyStr, +fn different_subject(fact: SlotNetworkFact) -> SlotNetworkReadback { + SlotNetworkUnobservable { + fact: fact, + reason: "the reading names a different slot, tap or guest address than the one being sanitized" as NonEmptyStr, + } +} + +fn tap_readback(expected: SlotNetworkSubject, r: TapReading) -> SlotNetworkReadback { + if !slot_network_subject_equal(a: expected, b: r.subject) { + different_subject(fact: TapUnheldFact) + } else { + match r.observation { + TapPresentUnheld => SlotNetworkConfirmed + TapHeld { holders: n } => SlotNetworkRefuted { + fact: TapUnheldFact, + detail: join([to_string(n), " process(es) still hold the slot tap after the attempt"], "") as NonEmptyStr, + } + TapMissing => SlotNetworkRefuted { + fact: TapUnheldFact, + detail: "the slot tap is absent, so the converged network state no longer holds" as NonEmptyStr, + } + TapHoldersUnreadable { reason: why } => SlotNetworkUnobservable { fact: TapUnheldFact, reason: why } } - TapHoldersUnreadable { reason: r } => SlotNetworkUnobservable { fact: TapUnheldFact, reason: r } } } -fn conntrack_readback(o: GuestConntrackObservation) -> SlotNetworkReadback { - match o { - GuestConntrackEmptyAfterFlush => SlotNetworkConfirmed - GuestConntrackResidue { entries: n } => SlotNetworkRefuted { - fact: GuestConntrackEmptyFact, - detail: join([to_string(n), " flow(s) keyed to the guest address survived the flush"], "") as NonEmptyStr, +fn conntrack_readback(expected: SlotNetworkSubject, fact: SlotNetworkFact, r: GuestConntrackReading) -> SlotNetworkReadback { + if !slot_network_subject_equal(a: expected, b: r.subject) { + different_subject(fact: fact) + } else { + match r.observation { + GuestConntrackEmptyAfterFlush => SlotNetworkConfirmed + GuestConntrackResidue { entries: n } => SlotNetworkRefuted { + fact: fact, + detail: join([to_string(n), " flow(s) keyed to the guest address survived the flush"], "") as NonEmptyStr, + } + GuestConntrackUnreadable { reason: why } => SlotNetworkUnobservable { fact: fact, reason: why } } - GuestConntrackUnreadable { reason: r } => SlotNetworkUnobservable { fact: GuestConntrackEmptyFact, reason: r } } } -fn ruleset_readback(o: RulesetObservation, converged: Sha256Digest) -> SlotNetworkReadback { - match o { - RulesetListed { digest: d } => - if content_hash_eq_cryptographic(left: d, right: converged) { - SlotNetworkConfirmed - } else { - SlotNetworkRefuted { - fact: RulesetMatchesConvergedFact, - detail: join(["listed ruleset digest ", d.hex as String, " differs from converged ", converged.hex as String], "") as NonEmptyStr, +fn ruleset_readback(expected: ConvergedRuleset, r: RulesetReading) -> SlotNetworkReadback { + if (expected.host as String) != (r.host as String) || expected.generation != r.generation { + SlotNetworkUnobservable { + fact: RulesetMatchesConvergedFact, + reason: "the listing was read on a different host or against a different converge generation than the receipt it would be compared with" as NonEmptyStr, + } + } else { + match r.observation { + RulesetListed { digest: d } => + if content_hash_eq_cryptographic(left: d, right: expected.digest) { + SlotNetworkConfirmed + } else { + SlotNetworkRefuted { + fact: RulesetMatchesConvergedFact, + detail: join(["listed ruleset digest ", d.hex as String, " differs from converged ", expected.digest.hex as String], "") as NonEmptyStr, + } } - } - RulesetUnreadable { reason: r } => SlotNetworkUnobservable { fact: RulesetMatchesConvergedFact, reason: r } + RulesetUnreadable { reason: why } => SlotNetworkUnobservable { fact: RulesetMatchesConvergedFact, reason: why } + } } } @@ -696,15 +789,18 @@ fn first_readback_where(rs: List, pick: fn(SlotNetworkReadb } fn slot_network_readback( - tap: TapHolderObservation, - conntrack: GuestConntrackObservation, - ruleset: RulesetObservation, - converged_ruleset: Sha256Digest, + expected: SlotNetworkSubject, + converged: ConvergedRuleset, + tap: TapReading, + guest_originated: GuestConntrackReading, + guest_destined: GuestConntrackReading, + ruleset: RulesetReading, ) -> SlotNetworkReadback { let facts = [ - tap_readback(o: tap), - conntrack_readback(o: conntrack), - ruleset_readback(o: ruleset, converged: converged_ruleset), + tap_readback(expected: expected, r: tap), + conntrack_readback(expected: expected, fact: GuestOriginatedConntrackEmptyFact, r: guest_originated), + conntrack_readback(expected: expected, fact: GuestDestinedConntrackEmptyFact, r: guest_destined), + ruleset_readback(expected: converged, r: ruleset), ] match first_readback_where(rs: facts, pick: is_refuted) { Present { value: r } => r @@ -717,15 +813,23 @@ fn slot_network_readback( // --- the commands that produce the readback's observations --- -// TEARDOWN FLUSHES; THE LIFECYCLE CONTROLLER READS. The flush is a mutation and belongs to the -// teardown step, which runs it before the list. Its exit status is not the evidence -- see -// extdeps.tools.conntrack -- the list is. -fn runner_slot_conntrack_flush_command(net: RunnerSlotNetwork) -> ArgvCommand { - conntrack_delete_by_source_command(source: render_ipv4_address(addr: net.guest_address) as String) +// TEARDOWN FLUSHES; THE LIFECYCLE CONTROLLER READS. The flushes are mutations and belong to the +// teardown step, which runs them before the lists. Their exit status is not the evidence -- see +// extdeps.tools.conntrack -- the lists are. One flush and one list per original direction. +fn runner_slot_conntrack_flush_commands(net: RunnerSlotNetwork) -> List { + let guest = render_ipv4_address(addr: net.guest_address) as String + [ + conntrack_delete_command(filter: ConntrackOriginalSource, address: guest), + conntrack_delete_command(filter: ConntrackOriginalDestination, address: guest), + ] +} + +fn runner_slot_guest_originated_conntrack_list_command(net: RunnerSlotNetwork) -> ArgvCommand { + conntrack_list_command(filter: ConntrackOriginalSource, address: render_ipv4_address(addr: net.guest_address) as String) } -fn runner_slot_conntrack_list_command(net: RunnerSlotNetwork) -> ArgvCommand { - conntrack_list_by_source_command(source: render_ipv4_address(addr: net.guest_address) as String) +fn runner_slot_guest_destined_conntrack_list_command(net: RunnerSlotNetwork) -> ArgvCommand { + conntrack_list_command(filter: ConntrackOriginalDestination, address: render_ipv4_address(addr: net.guest_address) as String) } fn runner_microvm_nft_list_command() -> ArgvCommand { @@ -735,12 +839,12 @@ fn runner_microvm_nft_list_command() -> ArgvCommand { // DECLARED FRONTIER (DESIGN 3c). The readback and the three commands that feed it have no // executing consumer in this tree yet. NAMED CONSUMER: the host teardown readback of the microVM // slot lifecycle controller (gunbc#11670), whose network fact maps onto product.fabric.sanitation. -// TRIGGER: that controller executes teardown on a host -- runs runner_slot_conntrack_flush_command, -// reads runner_slot_conntrack_list_command and runner_microvm_nft_list_command, and derives +// TRIGGER: that controller executes teardown on a host -- runs runner_slot_conntrack_flush_commands, +// reads both conntrack list commands and runner_microvm_nft_list_command, and derives // CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT // satisfied by #11670 importing the type, or by a witness calling the fold. data slot_network_readback_consumer_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: the microVM slot lifecycle controller's host teardown readback (gunbc#11670). TRIGGER: that controller executes teardown on a host, runs runner_slot_conntrack_flush_command, reads runner_slot_conntrack_list_command and runner_microvm_nft_list_command, and derives CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT satisfied by an import of the type or by a witness calling the fold." as NonEmptyStr, + description: "NAMED CONSUMER: the microVM slot lifecycle controller's host teardown readback (gunbc#11670). TRIGGER: that controller executes teardown on a host, runs runner_slot_conntrack_flush_commands, reads runner_slot_guest_originated_conntrack_list_command, runner_slot_guest_destined_conntrack_list_command and runner_microvm_nft_list_command, and derives CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT satisfied by an import of the type or by a witness calling the fold." as NonEmptyStr, ) // DECLARED FRONTIER (DESIGN 3c). The kernel records dns0 in /proc/net/pnp; it does not write diff --git a/dag/test/claim/runner/runner_attempt_launch_witness_test.dag b/dag/test/claim/runner/runner_attempt_launch_witness_test.dag index d9cdeae176f..0315f76259c 100644 --- a/dag/test/claim/runner/runner_attempt_launch_witness_test.dag +++ b/dag/test/claim/runner/runner_attempt_launch_witness_test.dag @@ -45,7 +45,7 @@ import gunbc.runner_attempt_launch { WorkspaceStagingObservation, WorkspaceSuperblockRead, WorkspaceReadbackFailed, AttemptStagingVerdict, StagingAdmitsJailer, StagingRefusedWorkspace, StagingNotAuthorized, WorkspaceStanding, WorkspaceReady, WorkspaceCreateFailed, WorkspaceFormatFailed, - WorkspaceNotExtFilesystem, WorkspaceSizeMismatch, WorkspaceUnreadable, + WorkspaceNotExtFilesystem, WorkspaceSizeMismatch, WorkspaceUnreadable, WorkspaceReadbackOfDifferentImage, staging_verdict, runner_attempt_workspace_bytes, workspace_readback_command, workspace_staging_consumer_frontier, } @@ -337,11 +337,19 @@ test fn every_x_envelope_refusal_arm_yields_zero_jailer_launches() -> Bool { } +fn planned_ws_path() -> String { + match accepted_attempt(i: 2, id: "ws1") { + Cons { head: a, tail: _ } => + join([attempt_jail_chroot_root(attempt: a, firecracker_binary: "/bin/firecracker"), jail_workspace_path as String], "") + Empty {} => "" + } +} + fn good_workspace() -> WorkspaceStagingObservation { WorkspaceStagingObservation { create_exit: 0, format_exit: 0, - readback: WorkspaceSuperblockRead { ext_magic: true, bytes: runner_attempt_workspace_bytes() }, + readback: WorkspaceSuperblockRead { image_path: planned_ws_path(), ext_magic: true, bytes: runner_attempt_workspace_bytes() }, } } @@ -356,6 +364,7 @@ fn staging_label(v: AttemptStagingVerdict) -> String { WorkspaceNotExtFilesystem => "not-ext" WorkspaceSizeMismatch { observed: _, expected: _ } => "size" WorkspaceUnreadable { reason: _ } => "unreadable" + WorkspaceReadbackOfDifferentImage { read: _, planned: _ } => "other-image" } } } @@ -384,13 +393,13 @@ test fn the_jailer_is_admitted_only_onto_a_read_back_workspace() -> Bool { create_exit: 1, format_exit: 1, readback: WorkspaceReadbackFailed { reason: "absent" as NonEmptyStr }, }, ground: GroundClear) == "create" && staging_for(obs: WorkspaceStagingObservation { - create_exit: 0, format_exit: 1, readback: WorkspaceSuperblockRead { ext_magic: true, bytes: runner_attempt_workspace_bytes() }, + create_exit: 0, format_exit: 1, readback: WorkspaceSuperblockRead { image_path: planned_ws_path(), ext_magic: true, bytes: runner_attempt_workspace_bytes() }, }, ground: GroundClear) == "format" && staging_for(obs: WorkspaceStagingObservation { - create_exit: 0, format_exit: 0, readback: WorkspaceSuperblockRead { ext_magic: false, bytes: runner_attempt_workspace_bytes() }, + create_exit: 0, format_exit: 0, readback: WorkspaceSuperblockRead { image_path: planned_ws_path(), ext_magic: false, bytes: runner_attempt_workspace_bytes() }, }, ground: GroundClear) == "not-ext" && staging_for(obs: WorkspaceStagingObservation { - create_exit: 0, format_exit: 0, readback: WorkspaceSuperblockRead { ext_magic: true, bytes: byte_size(count: 1048576) }, + create_exit: 0, format_exit: 0, readback: WorkspaceSuperblockRead { image_path: planned_ws_path(), ext_magic: true, bytes: byte_size(count: 1048576) }, }, ground: GroundClear) == "size" && staging_for(obs: WorkspaceStagingObservation { create_exit: 0, format_exit: 0, readback: WorkspaceReadbackFailed { reason: "dumpe2fs exited 1" as NonEmptyStr }, @@ -412,3 +421,19 @@ test fn the_workspace_readback_reads_the_planned_image_and_its_consumer_is_a_fro BoundDissolution { trigger: _ } => false } } + +// ANOTHER IMAGE'S READBACK DOES NOT AUTHORIZE THIS PLAN. A clean ext superblock of the right size, read +// from any path but the one this plan formatted -- the previous attempt's image, another slot's -- +// refuses. The red is a staging gate that reads only magic and size (sunny-ant-606 ruling on #11675). +test fn another_images_readback_does_not_authorize_this_plan() -> Bool { + planned_ws_path() != "" + && staging_for(obs: WorkspaceStagingObservation { + create_exit: 0, + format_exit: 0, + readback: WorkspaceSuperblockRead { + image_path: "/srv/jailer/firecracker/other-attempt/root/work.img", + ext_magic: true, + bytes: runner_attempt_workspace_bytes(), + }, + }, ground: GroundClear) == "other-image" +} diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index c2d289961b2..ee3dd1c71ca 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -6,7 +6,7 @@ import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import extdeps.network.ipv4 { render_ipv4_address } import extdeps.network.mac { MacAddressScope, LocallyAdministered, mac_address_scope, render_mac_address } import gunbc.build_cache_instance { RunnerSlotIdentity } -import gunbc.fleet_intent_network { operator_host_srv2 } +import gunbc.fleet_intent_network { operator_host_srv2, operator_host_srv1 } import gunbc.runner_microvm_network { runner_slot_network, runner_slot_tap_device_name, runner_slot_guest_mac, runner_slot_host_address, runner_slot_guest_address, @@ -16,11 +16,15 @@ import gunbc.runner_microvm_network { netmask_of_prefix, runner_microvm_network_supernet, runner_microvm_network_prefix_length, runner_microvm_network_supernet_prefix, SlotNetworkReadback, SlotNetworkConfirmed, SlotNetworkRefuted, SlotNetworkUnobservable, - SlotNetworkFact, TapUnheldFact, GuestConntrackEmptyFact, RulesetMatchesConvergedFact, + SlotNetworkFact, TapUnheldFact, GuestOriginatedConntrackEmptyFact, GuestDestinedConntrackEmptyFact, + RulesetMatchesConvergedFact, + SlotNetworkSubject, slot_network_subject, ConvergedRuleset, + TapReading, GuestConntrackReading, RulesetReading, TapPresentUnheld, TapHeld, TapMissing, TapHoldersUnreadable, GuestConntrackEmptyAfterFlush, GuestConntrackResidue, GuestConntrackUnreadable, RulesetListed, RulesetUnreadable, slot_network_readback, - runner_slot_conntrack_flush_command, runner_slot_conntrack_list_command, + runner_slot_conntrack_flush_commands, + runner_slot_guest_originated_conntrack_list_command, runner_slot_guest_destined_conntrack_list_command, runner_microvm_nft_list_command, slot_network_readback_consumer_frontier, runner_microvm_guest_resolver, guest_resolver_configuration_frontier, runner_microvm_nft_ruleset_text_for, @@ -28,6 +32,7 @@ import gunbc.runner_microvm_network { import gunbc.runner.runner_host_filtered_egress { runner_host_egress_policy } import std.dissolution { UnboundDissolution, BoundDissolution } import std.content_hash { Sha256Digest, Sha256DigestHex } +import gunbc.runner_microvm_network { TapHolderObservation, GuestConntrackObservation } import extdeps.exec.command { argv_words } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -199,12 +204,33 @@ test fn the_nat_chain_is_typed_nat_and_the_filter_chain_stays_filter() -> Bool { // default-deny into default-accept for every port nobody named. test fn guest_egress_is_default_deny_at_tap_grain_in_the_receipted_order() -> Bool { let text = runner_microvm_shared_host_nft_ruleset_text() - string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" meta nfproto ipv6 drop\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") - && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") + string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") + && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") && string_contains(s: text, pattern: "policy accept") && !string_contains(s: text, pattern: "policy drop") } +// THE GUEST-SIDE ESTABLISHED ACCEPT SITS BELOW EVERY INVARIANT DENIAL. The red is the previous order, +// where `iifname tap ct state established,related accept` came first: an established or related flow +// toward a denied range (a related ICMP, a flow predating a ruleset change) was accepted before the +// denial could see it. Only the reply-direction (oifname) established rule precedes the denials. +test fn no_guest_side_accept_precedes_an_invariant_denial() -> Bool { + let text = runner_microvm_shared_host_nft_ruleset_text() + !string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop") + && string_contains(s: text, pattern: "oifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop") +} + +// v6 FROM A TAP IS REFUSED BEFORE CONNECTION TRACKING, SO NO v6 STATE CAN OUTLIVE AN ATTEMPT. The drop +// is in a prerouting chain at raw priority (-300, below conntrack's -200), which also covers v6 the +// guest sends to the HOST, which forward never sees. The red is the v6 drop living in forward, after +// conntrack has already created the entry. +test fn tap_ipv6_is_refused_at_raw_before_conntrack_and_not_in_forward() -> Bool { + let text = runner_microvm_shared_host_nft_ruleset_text() + string_contains(s: text, pattern: "type filter hook prerouting priority -300; policy accept;\n iifname \"gunbc-tap*\" meta nfproto ipv6 drop\n }") + && !string_contains(s: text, pattern: "hook forward priority 0; policy accept;\n iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n iifname \"gunbc-tap*\" meta nfproto ipv6") + && length(split(s: text, delimiter: "meta nfproto ipv6")) == 2 +} + data converged_digest: Sha256Digest = Sha256Digest { hex: "1111111111111111111111111111111111111111111111111111111111111111" as Sha256DigestHex, } @@ -213,13 +239,56 @@ data drifted_digest: Sha256Digest = Sha256Digest { hex: "2222222222222222222222222222222222222222222222222222222222222222" as Sha256DigestHex, } +fn subject(i: Nat) -> SlotNetworkSubject { + slot_network_subject(slot: slot(i: i)) +} + +data converged_generation: Int = 7 + +fn converged() -> ConvergedRuleset { + ConvergedRuleset { host: operator_host_srv2, generation: converged_generation, digest: converged_digest } +} + +fn tap(i: Nat, o: TapHolderObservation) -> TapReading { + TapReading { subject: subject(i: i), observation: o } +} + +fn ct(i: Nat, o: GuestConntrackObservation) -> GuestConntrackReading { + GuestConntrackReading { subject: subject(i: i), observation: o } +} + +fn listed(d: Sha256Digest) -> RulesetReading { + RulesetReading { host: operator_host_srv2, generation: converged_generation, observation: RulesetListed { digest: d } } +} + +fn clean_ct(i: Nat) -> GuestConntrackReading { + ct(i: i, o: GuestConntrackEmptyAfterFlush) +} + +// Sanitizes slot 1 from the given readings. +fn readback_for_slot_1(t: TapReading, from_guest: GuestConntrackReading, to_guest: GuestConntrackReading, r: RulesetReading) -> SlotNetworkReadback { + slot_network_readback( + expected: subject(i: 1), + converged: converged(), + tap: t, + guest_originated: from_guest, + guest_destined: to_guest, + ruleset: r, + ) +} + +fn fact_name(f: SlotNetworkFact) -> String { + match f { + TapUnheldFact => "tap" + GuestOriginatedConntrackEmptyFact => "conntrack-from" + GuestDestinedConntrackEmptyFact => "conntrack-to" + RulesetMatchesConvergedFact => "ruleset" + } +} + fn refuted_fact(r: SlotNetworkReadback) -> String { match r { - SlotNetworkRefuted { fact: f, detail: _ } => match f { - TapUnheldFact => "tap" - GuestConntrackEmptyFact => "conntrack" - RulesetMatchesConvergedFact => "ruleset" - } + SlotNetworkRefuted { fact: f, detail: _ } => fact_name(f: f) SlotNetworkConfirmed => "confirmed" SlotNetworkUnobservable { fact: _, reason: _ } => "unobservable" } @@ -227,89 +296,89 @@ fn refuted_fact(r: SlotNetworkReadback) -> String { fn unobservable_fact(r: SlotNetworkReadback) -> String { match r { - SlotNetworkUnobservable { fact: f, reason: _ } => match f { - TapUnheldFact => "tap" - GuestConntrackEmptyFact => "conntrack" - RulesetMatchesConvergedFact => "ruleset" - } + SlotNetworkUnobservable { fact: f, reason: _ } => fact_name(f: f) SlotNetworkConfirmed => "confirmed" SlotNetworkRefuted { fact: _, detail: _ } => "refuted" } } -// THE POSITIVE CONTROL: all three facts hold, and only then is the readback confirmed. -test fn a_clean_slot_network_is_confirmed_only_when_all_three_facts_hold() -> Bool { - refuted_fact(r: slot_network_readback( - tap: TapPresentUnheld, - conntrack: GuestConntrackEmptyAfterFlush, - ruleset: RulesetListed { digest: converged_digest }, - converged_ruleset: converged_digest, +// THE POSITIVE CONTROL: every fact holds for the subject being sanitized, and only then is it confirmed. +test fn a_clean_slot_network_is_confirmed_only_when_every_fact_holds() -> Bool { + refuted_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), )) == "confirmed" } -// EACH FACT REFUTES ON ITS OWN, AND THE REFUTATION NAMES IT. A VMM fd still on the tap, a flow that -// survived the flush (the next attempt reuses this guest address), and a ruleset some process edited -// are three different remedies, so the fact is carried rather than a bare false. +// EACH FACT REFUTES ON ITS OWN, AND THE REFUTATION NAMES IT. A VMM fd still on the tap, a flow in +// either original direction that survived the flush, and an edited ruleset are different remedies. test fn each_fact_refutes_alone_and_names_itself() -> Bool { - refuted_fact(r: slot_network_readback( - tap: TapHeld { holders: 1 }, - conntrack: GuestConntrackEmptyAfterFlush, - ruleset: RulesetListed { digest: converged_digest }, - converged_ruleset: converged_digest, + refuted_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapHeld { holders: 1 }), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), )) == "tap" - && refuted_fact(r: slot_network_readback( - tap: TapMissing, - conntrack: GuestConntrackEmptyAfterFlush, - ruleset: RulesetListed { digest: converged_digest }, - converged_ruleset: converged_digest, + && refuted_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapMissing), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), )) == "tap" - && refuted_fact(r: slot_network_readback( - tap: TapPresentUnheld, - conntrack: GuestConntrackResidue { entries: 2 }, - ruleset: RulesetListed { digest: converged_digest }, - converged_ruleset: converged_digest, - )) == "conntrack" - && refuted_fact(r: slot_network_readback( - tap: TapPresentUnheld, - conntrack: GuestConntrackEmptyAfterFlush, - ruleset: RulesetListed { digest: drifted_digest }, - converged_ruleset: converged_digest, + && refuted_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), from_guest: ct(i: 1, o: GuestConntrackResidue { entries: 2 }), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), + )) == "conntrack-from" + && refuted_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: ct(i: 1, o: GuestConntrackResidue { entries: 1 }), r: listed(d: converged_digest), + )) == "conntrack-to" + && refuted_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: drifted_digest), )) == "ruleset" } -// "COULD NOT READ" IS NOT "REFUTED" -- and a refutation elsewhere outranks it. The red for the first -// half is an unreadable conntrack table reported as residue; for the second, a drifted ruleset -// hidden behind a probe failure on another fact. +// "COULD NOT READ" IS NOT "REFUTED" -- and a refutation elsewhere outranks it. test fn an_unreadable_fact_is_unobservable_and_a_refutation_elsewhere_outranks_it() -> Bool { - unobservable_fact(r: slot_network_readback( - tap: TapPresentUnheld, - conntrack: GuestConntrackUnreadable { reason: "conntrack table not readable" as NonEmptyStr }, - ruleset: RulesetListed { digest: converged_digest }, - converged_ruleset: converged_digest, - )) == "conntrack" - && unobservable_fact(r: slot_network_readback( - tap: TapHoldersUnreadable { reason: "fd scan refused" as NonEmptyStr }, - conntrack: GuestConntrackEmptyAfterFlush, - ruleset: RulesetUnreadable { reason: "nft list refused" as NonEmptyStr }, - converged_ruleset: converged_digest, - )) == "tap" - && refuted_fact(r: slot_network_readback( - tap: TapPresentUnheld, - conntrack: GuestConntrackUnreadable { reason: "conntrack table not readable" as NonEmptyStr }, - ruleset: RulesetListed { digest: drifted_digest }, - converged_ruleset: converged_digest, + unobservable_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), + from_guest: ct(i: 1, o: GuestConntrackUnreadable { reason: "conntrack table not readable" as NonEmptyStr }), + to_guest: clean_ct(i: 1), + r: listed(d: converged_digest), + )) == "conntrack-from" + && refuted_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), + from_guest: ct(i: 1, o: GuestConntrackUnreadable { reason: "conntrack table not readable" as NonEmptyStr }), + to_guest: clean_ct(i: 1), + r: listed(d: drifted_digest), + )) == "ruleset" +} + +// SLOT A'S CLEAN READINGS CANNOT SANITIZE SLOT B. Every reading below is clean, but one fact at a time +// is read about slot 2 -- or, for the ruleset, on another host or at another converge generation -- +// while slot 1 is the subject. Each yields Unobservable for exactly that fact, never Confirmed. The red +// is a fold that inspects only the observation (sunny-ant-606 ruling on #11675). +test fn readings_about_another_subject_never_sanitize_this_one() -> Bool { + unobservable_fact(r: readback_for_slot_1( + t: tap(i: 2, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), + )) == "tap" + && unobservable_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 2), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), + )) == "conntrack-from" + && unobservable_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 2), r: listed(d: converged_digest), + )) == "conntrack-to" + && unobservable_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), + r: RulesetReading { host: operator_host_srv1, generation: converged_generation, observation: RulesetListed { digest: converged_digest } }, + )) == "ruleset" + && unobservable_fact(r: readback_for_slot_1( + t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), + r: RulesetReading { host: operator_host_srv2, generation: 6, observation: RulesetListed { digest: converged_digest } }, )) == "ruleset" } -// THE FLUSH AND THE LIST ARE KEYED BY THE GUEST'S ORIGINAL SOURCE. A masqueraded flow carries the -// host's address only on its reply tuple, so a flush keyed any other way leaves every flow the guest -// opened -- and the next attempt in this slot inherits them. The red is a slot's commands naming a -// different slot's guest address. -test fn conntrack_commands_are_keyed_by_this_slots_guest_source() -> Bool { +// THE FLUSHES AND LISTS COVER BOTH ORIGINAL DIRECTIONS FOR THIS SLOT'S GUEST. A flow the guest opened +// has it as original source; a flow the host opened toward it has it as original destination. The +// reds: a single-direction flush, or commands naming another slot's guest. +test fn conntrack_commands_cover_both_directions_for_this_slots_guest() -> Bool { let net = runner_slot_network(slot: slot(i: 6)) let guest = render_ipv4_address(addr: net.guest_address) as String - argv_words(command: runner_slot_conntrack_flush_command(net: net)) == ["conntrack", "-D", "-s", guest] - && argv_words(command: runner_slot_conntrack_list_command(net: net)) == ["conntrack", "-L", "-s", guest] + map(runner_slot_conntrack_flush_commands(net: net), c => join(argv_words(command: c), " ")) + == [join(["conntrack -D -s ", guest], ""), join(["conntrack -D -d ", guest], "")] + && argv_words(command: runner_slot_guest_originated_conntrack_list_command(net: net)) == ["conntrack", "-L", "-s", guest] + && argv_words(command: runner_slot_guest_destined_conntrack_list_command(net: net)) == ["conntrack", "-L", "-d", guest] && guest == "172.30.6.2" } From f5be1845bc21e61fed77d614bf2efd91a53374e3 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 18:26:17 +0000 Subject: [PATCH 11/36] Sealed sanitation subject joined to a generation-bound ConvergedSlotNetwork receipt; v6 disabled on tap and guest; neighbour and expectation readbacks; helper standing over the effective host policy Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/exec/command.dag | 3 + dag/extdeps/iproute2/ip_neigh.dag | 41 ++ dag/extdeps/tools/conntrack.dag | 7 + dag/gunbc/runner/runner_attempt_launch.dag | 6 +- dag/gunbc/runner/runner_microvm_network.dag | 365 +++++++++++++++--- .../runner_microvm_network_witness_test.dag | 315 ++++++++++----- 6 files changed, 572 insertions(+), 165 deletions(-) create mode 100644 dag/extdeps/iproute2/ip_neigh.dag diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index 49e9a9cc206..f46f07a4c51 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -95,6 +95,9 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.nftables.ruleset", decl_name: "nft_list_table_command"), decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_delete_command"), decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_list_command"), + decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_list_expectations_command"), + decl_ref(module_path: "extdeps.iproute2.ip_neigh", decl_name: "ip_neigh_flush_command"), + decl_ref(module_path: "extdeps.iproute2.ip_neigh", decl_name: "ip_neigh_show_command"), decl_ref(module_path: "extdeps.tools.make", decl_name: "make_command"), decl_ref(module_path: "extdeps.package_managers.pip", decl_name: "pip_install_pinned_command"), decl_ref(module_path: "extdeps.package_managers.conda_forge", decl_name: "micromamba_create_command"), diff --git a/dag/extdeps/iproute2/ip_neigh.dag b/dag/extdeps/iproute2/ip_neigh.dag new file mode 100644 index 00000000000..d1f5b6c7b25 --- /dev/null +++ b/dag/extdeps/iproute2/ip_neigh.dag @@ -0,0 +1,41 @@ +module extdeps.iproute2.ip_neigh + +import std.types { String, NonEmptyStr, List } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Uri, Https } +import extdeps.exec.command { ArgvCommand, argv_command } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "man7.org/linux/man-pages/man8/ip-neighbour.8.html" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.iproute2.ip_neigh", + decl_name: "ip_program", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +data ip_program: NonEmptyStr = "ip" + +// `flush` REMOVES NON-PERMANENT ENTRIES MATCHING THE SELECTOR, and `to dev ` narrows it to +// one neighbour on one link, so no other interface's cache is touched. The flush's exit is not the +// evidence; the `show` that follows is. +fn ip_neigh_flush_command(address: String, device: String) -> ArgvCommand { + argv_command(program: ip_program, arguments: ["neigh", "flush", "to", address, "dev", device]) +} + +// `show` prints one line per entry in ANY state, including FAILED and INCOMPLETE, so an empty stdout +// with exit zero is the absent reading and any line is residue. +fn ip_neigh_show_command(address: String, device: String) -> ArgvCommand { + argv_command(program: ip_program, arguments: ["neigh", "show", "to", address, "dev", device]) +} diff --git a/dag/extdeps/tools/conntrack.dag b/dag/extdeps/tools/conntrack.dag index 713574615cf..6133e8b6210 100644 --- a/dag/extdeps/tools/conntrack.dag +++ b/dag/extdeps/tools/conntrack.dag @@ -59,3 +59,10 @@ fn conntrack_delete_command(filter: ConntrackOriginalAddress, address: String) - fn conntrack_list_command(filter: ConntrackOriginalAddress, address: String) -> ArgvCommand { argv_command(program: conntrack_program, arguments: ["-L", conntrack_original_address_flag(f: filter), address]) } + +// THE EXPECTATION TABLE, WHOLE. Expectations are the related-flow entries a conntrack helper creates +// (an FTP data channel, a SIP media stream); `-L expect` prints one line per entry. The table has no +// original-source filter in conntrack(8), so this lists every expectation on the host. +fn conntrack_list_expectations_command() -> ArgvCommand { + argv_command(program: conntrack_program, arguments: ["-L", "expect"]) +} diff --git a/dag/gunbc/runner/runner_attempt_launch.dag b/dag/gunbc/runner/runner_attempt_launch.dag index 147794033bd..f18c0bbe52b 100644 --- a/dag/gunbc/runner/runner_attempt_launch.dag +++ b/dag/gunbc/runner/runner_attempt_launch.dag @@ -46,7 +46,7 @@ import gunbc.runner_microvm_attempt { import gunbc.runner_microvm_network { runner_slot_network, runner_slot_network_as_microvm_networking, - runner_slot_guest_ip_boot_arg, + runner_slot_guest_network_boot_args, } import extdeps.systemd.systemd_run { SystemdRunProperty } @@ -371,12 +371,12 @@ fn plan_authorized_steps( ) let jailed = with_guest_ip_boot_arg( config: attach_workspace_drive(config: cfg, path: jail_workspace_path), - ip_arg: runner_slot_guest_ip_boot_arg(net: net), + ip_arg: runner_slot_guest_network_boot_args(net: net), ) LaunchAuthorized { authorization: authorization, tap_device: net.tap_device, - guest_ip_boot_arg: runner_slot_guest_ip_boot_arg(net: net), + guest_ip_boot_arg: runner_slot_guest_network_boot_args(net: net), slice: runner_microvm_attempt_slice_property(attempt: attempt), jail_root: jail_root, stage_jail_root: mkdir_parents_command(path: jail_root), diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 5caa820ad7e..ed823dad308 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -24,8 +24,10 @@ import extdeps.nftables.ruleset { NftTransport, NftTcp, NftUdp, nft_list_table_command, } import extdeps.exec.command { ArgvCommand } +import extdeps.iproute2.ip_neigh { ip_neigh_flush_command, ip_neigh_show_command } import extdeps.tools.conntrack { conntrack_delete_command, conntrack_list_command, ConntrackOriginalSource, ConntrackOriginalDestination, + conntrack_list_expectations_command, } data runner_microvm_network_disposition: Disposition = SingleAuthority @@ -603,9 +605,9 @@ fn runner_slot_network_install_operations( // THE TAP AND THE RULESET OUTLIVE THE ATTEMPT BY DESIGN, SO SANITATION CANNOT PROVE THEM ABSENT. // They are slot and host state (the .netdev block above; parent ruling on the microVM program, // 2026-09-19), and what an attempt can leave behind in them is not their existence but their USE: -// a process still holding the tap, a connection-tracking flow still keyed to the guest address the -// NEXT attempt in this slot will reuse, and a ruleset some process edited during the attempt. Those -// three facts are what this readback reads. Its consumer is not on main yet; see +// a process still holding the tap, connection-tracking flows, expectations or a neighbour entry +// keyed to the guest address the NEXT attempt in this slot will reuse, and a ruleset or tap setting +// that drifted from the converge receipt. Its consumer is not on main yet; see // slot_network_readback_consumer_frontier below. // // CONNTRACK IS FLUSHED BEFORE IT IS READ, AND THE FLUSH IS NOT THE EVIDENCE. The flush is teardown's @@ -616,46 +618,228 @@ fn runner_slot_network_install_operations( // given, so the text above and the kernel's listing differ in bytes while agreeing in meaning. The // converged digest is therefore the listing captured by the converge that installed the table, and // it is supplied here from that receipt rather than recomputed from the rule rows. -// EVERY READING NAMES WHAT IT IS ABOUT, AND A READING ABOUT SOMETHING ELSE ESTABLISHES NOTHING HERE. -// The three facts are about one slot on one host: its tap, the flows keyed to its guest address, and -// the table the host converge installed at one generation. A reading carries that subject, the fold -// is handed the subject it is sanitizing, and a reading whose subject differs is Unobservable for -// that fact -- never Confirmed -- so slot A's clean readings cannot release slot B (sunny-ant-606 -// ruling on #11675). The lifecycle controller joins the expected subject to the attempt's cell. + +// --- IPv6: disabled structurally on both ends of the link --- + +// v6 IS OFF ON THE TAP AND IN THE GUEST, SO THERE IS NO v6 PATH TO FILTER. A drop rule covers only the +// paths it names: the raw-prerouting drop below sees v6 ARRIVING from a tap, not v6 the host itself +// sends toward a tap nor v6 routed to a tap from another interface (sunny-ant-606 ruling on #11675). +// With disable_ipv6=1 on the tap the kernel assigns it no v6 address, installs no v6 route through +// it, and discards v6 received on it -- every direction at once, as a property of the link. The +// raw drop stays as the second wall, and the guest boots with ipv6.disable=1 so it never builds a v6 +// stack at all. +// +// THE HOST SETTING IS A sysctl.d FILE, not a command. systemd's udev rules re-run systemd-sysctl for +// each net device as it appears, so a per-device key applies when networkd creates the tap at boot +// and stays applied across reboots -- the same reason the tap itself is a .netdev. +data runner_microvm_sysctl_dir: NonEmptyStr = "/etc/sysctl.d" as NonEmptyStr + +fn runner_slot_ipv6_sysctl_path(net: RunnerSlotNetwork) -> String { + join([runner_microvm_sysctl_dir as String, "/50-", net.tap_device as String, "-ipv6.conf"], "") +} + +fn runner_slot_ipv6_sysctl_text(net: RunnerSlotNetwork) -> String { + join(["net.ipv6.conf.", net.tap_device as String, ".disable_ipv6 = 1\n"], "") +} + +// The kernel's live value, which is what a readback reads -- the file above is the intent. +fn runner_slot_ipv6_disabled_proc_path(net: RunnerSlotNetwork) -> String { + join(["/proc/sys/net/ipv6/conf/", net.tap_device as String, "/disable_ipv6"], "") +} + +data runner_microvm_guest_ipv6_disable_arg: NonEmptyStr = "ipv6.disable=1" as NonEmptyStr + +// Every network argument the guest kernel boots with: its address, gateway and resolver, and the +// v6 refusal. +fn runner_slot_guest_network_boot_args(net: RunnerSlotNetwork) -> String { + join([runner_slot_guest_ip_boot_arg(net: net), " ", runner_microvm_guest_ipv6_disable_arg as String], "") +} + +// DECLARED FRONTIER (DESIGN 3c). The host setting has a host readback (the per-attempt +// TapIpv6DisabledFact below); the guest's does not yet. NAMED CONSUMER: the guest boot observation +// (gunbc.runner_microvm_boot_probe). TRIGGER: a booted guest reports /proc/sys/net/ipv6 absent (the +// v6 stack never initialised) over the boot observation channel. SUFFICIENT FOR deleting this row. +// It is evidence of the guest's configuration, never of sanitation: a guest event may trigger +// teardown and may not establish sanitation. +data guest_ipv6_disabled_readback_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: gunbc.runner_microvm_boot_probe guest boot observation. TRIGGER: a booted guest reports its v6 stack absent (/proc/sys/net/ipv6 missing) under ipv6.disable=1 over the boot observation channel. SUFFICIENT FOR deleting this row. NOT satisfied by the boot argument being present, and never counted as sanitation evidence." as NonEmptyStr, +) + +// --- the converge receipt: persistent slot network state at one generation --- + +// PERSISTENT STATE IS A CONVERGE FACT; TRANSIENT RESIDUE IS AN ATTEMPT FACT. The tap, its address +// and route, its v6 standing, the host's helper standing and the installed ruleset are what the +// host converge establishes and reads back at one generation. The per-attempt readback below checks +// only what an attempt can leave behind, AGAINST this receipt -- it never re-derives the persistent +// facts itself (sunny-ant-606 ruling on #11675). +// +// THE SUBJECT IS DERIVED FROM THE SLOT, and it now carries the guest MAC: neighbour state is keyed +// to the guest's address on the tap and the MAC is the value that entry would hold. type SlotNetworkSubject { slot: RunnerSlotIdentity tap_device: NonEmptyStr guest_address: Ipv4Address + guest_mac: Eui48Address } fn slot_network_subject(slot: RunnerSlotIdentity) -> SlotNetworkSubject { let net = runner_slot_network(slot: slot) - SlotNetworkSubject { slot: slot, tap_device: net.tap_device, guest_address: net.guest_address } + SlotNetworkSubject { slot: slot, tap_device: net.tap_device, guest_address: net.guest_address, guest_mac: net.guest_mac } } fn slot_network_subject_equal(a: SlotNetworkSubject, b: SlotNetworkSubject) -> Bool { runner_slot_identity_equal(a: a.slot, b: b.slot) && (a.tap_device as String) == (b.tap_device as String) && ipv4_address_eq(a: a.guest_address, b: b.guest_address) + && (render_mac_address(addr: a.guest_mac) as String) == (render_mac_address(addr: b.guest_mac) as String) +} + +type TapIpv6Standing + = TapIpv6Disabled + | TapIpv6Enabled + | TapIpv6Unread { reason: NonEmptyStr } + +// ONE CONVERGED SLOT: the subject, and what the converge read back for its persistent state. +type ConvergedSlot { + subject: SlotNetworkSubject + host_address_cidr: NonEmptyStr + guest_route: NonEmptyStr + tap_ipv6: TapIpv6Standing } -// THE CONVERGED RULESET IS HOST STATE AT A GENERATION. The digest alone would let a listing taken -// after a re-converge be compared against the previous generation's receipt, or a listing from one -// host against another host's. -type ConvergedRuleset { +// HELPERS ARE EXCLUDED BY A PROOF OVER THE EFFECTIVE HOST POLICY, NOT OVER OUR TABLE. An expectation +// is created only by a conntrack helper, and a helper attaches to a flow in exactly three ways: +// automatic assignment by port, an explicit helper in ANY ruleset on the host (every nft table and +// any legacy xtables CT target -- dockerd's included), or a userspace helper. Each is a reading, and +// the expectation table is read empty at the same generation. +type HelperAutoAssignment + = AutoAssignmentAbsentFromKernel { kernel_release: NonEmptyStr } + | AutoAssignmentSysctlOff + | AutoAssignmentOn + | AutoAssignmentUnread { reason: NonEmptyStr } + +type HelperStanding { + auto_assignment: HelperAutoAssignment + nft_ct_helper_statements: Int + xtables_ct_helper_targets: Int + userspace_helpers: Int + expectations_at_generation: Int +} + +fn helpers_excluded(h: HelperStanding) -> Bool { + let auto_off = match h.auto_assignment { + AutoAssignmentAbsentFromKernel { kernel_release: _ } => true + AutoAssignmentSysctlOff => true + AutoAssignmentOn => false + AutoAssignmentUnread { reason: _ } => false + } + auto_off + && h.nft_ct_helper_statements == 0 + && h.xtables_ct_helper_targets == 0 + && h.userspace_helpers == 0 + && h.expectations_at_generation == 0 +} + +type ConvergedSlotNetwork { host: HostIdentity generation: Int - digest: Sha256Digest + slots: List + helpers: HelperStanding + nft_digest: Sha256Digest +} + +// DECLARED FRONTIER (DESIGN 3c). NAMED CONSUMER: the microvm_host_converge fleet-converge mode on +// srvN, which produces ConvergedSlotNetwork. TRIGGER: that converge installs each slot's .netdev, +// .network and v6 sysctl file and the ruleset from runner_microvm_nft_ruleset_text_for(the host's +// HostEgressGrain), then reads back every field of the receipt -- tap, address, route, disable_ipv6, +// helper standing, the nft listing digest -- and records it at a generation. SUFFICIENT FOR +// deleting this row and host_ruleset_installer_frontier together. NOT satisfied by a witness +// building a receipt. +data converged_slot_network_producer_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: microvm_host_converge on srvN, producing ConvergedSlotNetwork. TRIGGER: it installs each slot's .netdev, .network and v6 sysctl file and the ruleset for the host's HostEgressGrain, reads back tap, address, route, disable_ipv6, helper standing and the nft listing digest, and records them at a generation. SUFFICIENT FOR deleting this row and host_ruleset_installer_frontier. NOT satisfied by a witness building a receipt." as NonEmptyStr, +) + +// --- the sanitation subject: sealed, and joined to the receipt --- + +// THE SUBJECT OF A SANITATION IS A SLOT JOINED TO THE RECEIPT OF THE HOST IT RUNS ON. Taking the slot +// and the receipt as two unrelated arguments let a self-consistent receipt and listing from host B +// confirm a slot on host A (sunny-ant-606 ruling on #11675). This type has one constructor, and that +// constructor refuses unless the receipt is the slot's own host's, the slot is in the receipt's +// population with the subject the slot derives, its tap's v6 is disabled, and helpers are excluded. +// So the per-attempt fold can only be asked about a slot whose persistent state is established. +type SlotNetworkSanitationSubject sole_constructor { + subject: SlotNetworkSubject + converged: ConvergedSlotNetwork +} + +type SanitationSubjectRefusal + = ReceiptForDifferentHost { slot_host: HostIdentity, receipt_host: HostIdentity } + | SlotNotInConvergedPopulation + | ConvergedSlotDisagreesWithDerivation + | TapIpv6NotEstablishedDisabled + | HelpersNotExcluded + +type SanitationSubjectResult + = SanitationSubjectAdmitted { subject: SlotNetworkSanitationSubject } + | SanitationSubjectRefused { refusal: SanitationSubjectRefusal } + +fn converged_slot_of(slot: RunnerSlotIdentity, receipt: ConvergedSlotNetwork) -> ConvergedSlot? { + fold(receipt.slots, init: none, f: (acc, c) => + match acc { + Present { value: _ } => acc + Absent => if runner_slot_identity_equal(a: c.subject.slot, b: slot) { Present { value: c } } else { acc } + } + ) +} + +fn tap_ipv6_is_disabled(t: TapIpv6Standing) -> Bool { + match t { + TapIpv6Disabled => true + TapIpv6Enabled => false + TapIpv6Unread { reason: _ } => false + } } -// CONNTRACK IS READ IN BOTH ORIGINAL DIRECTIONS. A flow the guest opened has it as original source; a -// flow the host opened toward the guest has it as original destination. Both are keyed to an address -// the next attempt in this slot reuses. v6 needs no reading: raw_ipv6_rules refuses tap-originated v6 -// before connection tracking, so no v6 entry keyed to a tap can exist. +fn slot_network_sanitation_subject(slot: RunnerSlotIdentity, receipt: ConvergedSlotNetwork) -> SanitationSubjectResult { + if (slot.host as String) != (receipt.host as String) { + SanitationSubjectRefused { refusal: ReceiptForDifferentHost { slot_host: slot.host, receipt_host: receipt.host } } + } else { + match converged_slot_of(slot: slot, receipt: receipt) { + Absent => SanitationSubjectRefused { refusal: SlotNotInConvergedPopulation } + Present { value: c } => + if !slot_network_subject_equal(a: c.subject, b: slot_network_subject(slot: slot)) { + SanitationSubjectRefused { refusal: ConvergedSlotDisagreesWithDerivation } + } else if !tap_ipv6_is_disabled(t: c.tap_ipv6) { + SanitationSubjectRefused { refusal: TapIpv6NotEstablishedDisabled } + } else if !helpers_excluded(h: receipt.helpers) { + SanitationSubjectRefused { refusal: HelpersNotExcluded } + } else { + SanitationSubjectAdmitted { subject: SlotNetworkSanitationSubject { subject: c.subject, converged: receipt } } + } + } + } +} + +// --- the per-attempt readback: transient residue against the receipt --- + +// EVERY READING NAMES WHAT IT IS ABOUT, AND A READING ABOUT SOMETHING ELSE ESTABLISHES NOTHING HERE. +// Slot readings carry the slot subject; host readings carry the host and the generation they were +// taken against. A reading whose subject, host or generation differs from the sanitation subject's is +// Unobservable for that fact -- never Confirmed. +// +// CONNTRACK IS READ IN BOTH ORIGINAL DIRECTIONS, AND FLUSHED BEFORE IT IS READ. A flow the guest +// opened has it as original source; a flow the host opened toward it has it as original destination. +// The flush is teardown's mutation; the reading after it is the fact. +// +// THE RULESET DIGEST IS OF `nft list table`, NOT OF THE FILE WE APPLIED, and it is compared against +// the receipt's digest, which is a listing too. type SlotNetworkFact = TapUnheldFact + | TapIpv6DisabledFact | GuestOriginatedConntrackEmptyFact | GuestDestinedConntrackEmptyFact + | GuestNeighbourAbsentFact + | ExpectationTableEmptyFact | RulesetMatchesConvergedFact type TapHolderObservation @@ -664,10 +848,10 @@ type TapHolderObservation | TapMissing | TapHoldersUnreadable { reason: NonEmptyStr } -type GuestConntrackObservation - = GuestConntrackEmptyAfterFlush - | GuestConntrackResidue { entries: Int } - | GuestConntrackUnreadable { reason: NonEmptyStr } +type ResidueObservation + = ResidueAbsent + | ResiduePresent { entries: Int } + | ResidueUnreadable { reason: NonEmptyStr } type RulesetObservation = RulesetListed { digest: Sha256Digest } @@ -678,9 +862,20 @@ type TapReading { observation: TapHolderObservation } -type GuestConntrackReading { +type TapIpv6Reading { subject: SlotNetworkSubject - observation: GuestConntrackObservation + observation: TapIpv6Standing +} + +type SlotResidueReading { + subject: SlotNetworkSubject + observation: ResidueObservation +} + +type HostResidueReading { + host: HostIdentity + generation: Int + observation: ResidueObservation } type RulesetReading { @@ -689,6 +884,16 @@ type RulesetReading { observation: RulesetObservation } +type SlotNetworkReadings { + tap: TapReading + tap_ipv6: TapIpv6Reading + guest_originated_conntrack: SlotResidueReading + guest_destined_conntrack: SlotResidueReading + guest_neighbour: SlotResidueReading + expectations: HostResidueReading + ruleset: RulesetReading +} + // "COULD NOT READ" IS NOT "REFUTED", and the two arms stay separate because they call for different // operators: a refutation is a fact about the host that needs remedying, an unobservable reading is // a fact about the instrument. Both quarantine the cell -- neither is CellReady -- but collapsing @@ -701,12 +906,19 @@ type SlotNetworkReadback fn different_subject(fact: SlotNetworkFact) -> SlotNetworkReadback { SlotNetworkUnobservable { fact: fact, - reason: "the reading names a different slot, tap or guest address than the one being sanitized" as NonEmptyStr, + reason: "the reading names a different slot, tap, guest address or MAC than the one being sanitized" as NonEmptyStr, + } +} + +fn different_generation(fact: SlotNetworkFact) -> SlotNetworkReadback { + SlotNetworkUnobservable { + fact: fact, + reason: "the reading was taken on a different host or against a different converge generation than the receipt" as NonEmptyStr, } } -fn tap_readback(expected: SlotNetworkSubject, r: TapReading) -> SlotNetworkReadback { - if !slot_network_subject_equal(a: expected, b: r.subject) { +fn tap_readback(s: SlotNetworkSanitationSubject, r: TapReading) -> SlotNetworkReadback { + if !slot_network_subject_equal(a: s.subject, b: r.subject) { different_subject(fact: TapUnheldFact) } else { match r.observation { @@ -724,36 +936,64 @@ fn tap_readback(expected: SlotNetworkSubject, r: TapReading) -> SlotNetworkReadb } } -fn conntrack_readback(expected: SlotNetworkSubject, fact: SlotNetworkFact, r: GuestConntrackReading) -> SlotNetworkReadback { - if !slot_network_subject_equal(a: expected, b: r.subject) { - different_subject(fact: fact) +fn tap_ipv6_readback(s: SlotNetworkSanitationSubject, r: TapIpv6Reading) -> SlotNetworkReadback { + if !slot_network_subject_equal(a: s.subject, b: r.subject) { + different_subject(fact: TapIpv6DisabledFact) } else { match r.observation { - GuestConntrackEmptyAfterFlush => SlotNetworkConfirmed - GuestConntrackResidue { entries: n } => SlotNetworkRefuted { - fact: fact, - detail: join([to_string(n), " flow(s) keyed to the guest address survived the flush"], "") as NonEmptyStr, + TapIpv6Disabled => SlotNetworkConfirmed + TapIpv6Enabled => SlotNetworkRefuted { + fact: TapIpv6DisabledFact, + detail: "disable_ipv6 on the slot tap reads 0, so v6 has a path the receipt says does not exist" as NonEmptyStr, } - GuestConntrackUnreadable { reason: why } => SlotNetworkUnobservable { fact: fact, reason: why } + TapIpv6Unread { reason: why } => SlotNetworkUnobservable { fact: TapIpv6DisabledFact, reason: why } } } } -fn ruleset_readback(expected: ConvergedRuleset, r: RulesetReading) -> SlotNetworkReadback { - if (expected.host as String) != (r.host as String) || expected.generation != r.generation { - SlotNetworkUnobservable { - fact: RulesetMatchesConvergedFact, - reason: "the listing was read on a different host or against a different converge generation than the receipt it would be compared with" as NonEmptyStr, +fn residue_readback(fact: SlotNetworkFact, o: ResidueObservation) -> SlotNetworkReadback { + match o { + ResidueAbsent => SlotNetworkConfirmed + ResiduePresent { entries: n } => SlotNetworkRefuted { + fact: fact, + detail: join([to_string(n), " entry(ies) survived the attempt"], "") as NonEmptyStr, } + ResidueUnreadable { reason: why } => SlotNetworkUnobservable { fact: fact, reason: why } + } +} + +fn slot_residue_readback(s: SlotNetworkSanitationSubject, fact: SlotNetworkFact, r: SlotResidueReading) -> SlotNetworkReadback { + if !slot_network_subject_equal(a: s.subject, b: r.subject) { + different_subject(fact: fact) + } else { + residue_readback(fact: fact, o: r.observation) + } +} + +fn at_receipt(s: SlotNetworkSanitationSubject, host: HostIdentity, generation: Int) -> Bool { + (s.converged.host as String) == (host as String) && s.converged.generation == generation +} + +fn host_residue_readback(s: SlotNetworkSanitationSubject, fact: SlotNetworkFact, r: HostResidueReading) -> SlotNetworkReadback { + if !at_receipt(s: s, host: r.host, generation: r.generation) { + different_generation(fact: fact) + } else { + residue_readback(fact: fact, o: r.observation) + } +} + +fn ruleset_readback(s: SlotNetworkSanitationSubject, r: RulesetReading) -> SlotNetworkReadback { + if !at_receipt(s: s, host: r.host, generation: r.generation) { + different_generation(fact: RulesetMatchesConvergedFact) } else { match r.observation { RulesetListed { digest: d } => - if content_hash_eq_cryptographic(left: d, right: expected.digest) { + if content_hash_eq_cryptographic(left: d, right: s.converged.nft_digest) { SlotNetworkConfirmed } else { SlotNetworkRefuted { fact: RulesetMatchesConvergedFact, - detail: join(["listed ruleset digest ", d.hex as String, " differs from converged ", expected.digest.hex as String], "") as NonEmptyStr, + detail: join(["listed ruleset digest ", d.hex as String, " differs from converged ", s.converged.nft_digest.hex as String], "") as NonEmptyStr, } } RulesetUnreadable { reason: why } => SlotNetworkUnobservable { fact: RulesetMatchesConvergedFact, reason: why } @@ -788,19 +1028,15 @@ fn first_readback_where(rs: List, pick: fn(SlotNetworkReadb ) } -fn slot_network_readback( - expected: SlotNetworkSubject, - converged: ConvergedRuleset, - tap: TapReading, - guest_originated: GuestConntrackReading, - guest_destined: GuestConntrackReading, - ruleset: RulesetReading, -) -> SlotNetworkReadback { +fn slot_network_readback(s: SlotNetworkSanitationSubject, readings: SlotNetworkReadings) -> SlotNetworkReadback { let facts = [ - tap_readback(expected: expected, r: tap), - conntrack_readback(expected: expected, fact: GuestOriginatedConntrackEmptyFact, r: guest_originated), - conntrack_readback(expected: expected, fact: GuestDestinedConntrackEmptyFact, r: guest_destined), - ruleset_readback(expected: converged, r: ruleset), + tap_readback(s: s, r: readings.tap), + tap_ipv6_readback(s: s, r: readings.tap_ipv6), + slot_residue_readback(s: s, fact: GuestOriginatedConntrackEmptyFact, r: readings.guest_originated_conntrack), + slot_residue_readback(s: s, fact: GuestDestinedConntrackEmptyFact, r: readings.guest_destined_conntrack), + slot_residue_readback(s: s, fact: GuestNeighbourAbsentFact, r: readings.guest_neighbour), + host_residue_readback(s: s, fact: ExpectationTableEmptyFact, r: readings.expectations), + ruleset_readback(s: s, r: readings.ruleset), ] match first_readback_where(rs: facts, pick: is_refuted) { Present { value: r } => r @@ -815,12 +1051,13 @@ fn slot_network_readback( // TEARDOWN FLUSHES; THE LIFECYCLE CONTROLLER READS. The flushes are mutations and belong to the // teardown step, which runs them before the lists. Their exit status is not the evidence -- see -// extdeps.tools.conntrack -- the lists are. One flush and one list per original direction. -fn runner_slot_conntrack_flush_commands(net: RunnerSlotNetwork) -> List { +// extdeps.tools.conntrack and extdeps.iproute2.ip_neigh -- the lists are. +fn runner_slot_teardown_flush_commands(net: RunnerSlotNetwork) -> List { let guest = render_ipv4_address(addr: net.guest_address) as String [ conntrack_delete_command(filter: ConntrackOriginalSource, address: guest), conntrack_delete_command(filter: ConntrackOriginalDestination, address: guest), + ip_neigh_flush_command(address: guest, device: net.tap_device as String), ] } @@ -832,6 +1069,14 @@ fn runner_slot_guest_destined_conntrack_list_command(net: RunnerSlotNetwork) -> conntrack_list_command(filter: ConntrackOriginalDestination, address: render_ipv4_address(addr: net.guest_address) as String) } +fn runner_slot_guest_neighbour_show_command(net: RunnerSlotNetwork) -> ArgvCommand { + ip_neigh_show_command(address: render_ipv4_address(addr: net.guest_address) as String, device: net.tap_device as String) +} + +fn runner_host_expectation_list_command() -> ArgvCommand { + conntrack_list_expectations_command() +} + fn runner_microvm_nft_list_command() -> ArgvCommand { nft_list_table_command(family: NftInet, name: runner_microvm_nft_table_name) } @@ -839,12 +1084,12 @@ fn runner_microvm_nft_list_command() -> ArgvCommand { // DECLARED FRONTIER (DESIGN 3c). The readback and the three commands that feed it have no // executing consumer in this tree yet. NAMED CONSUMER: the host teardown readback of the microVM // slot lifecycle controller (gunbc#11670), whose network fact maps onto product.fabric.sanitation. -// TRIGGER: that controller executes teardown on a host -- runs runner_slot_conntrack_flush_commands, -// reads both conntrack list commands and runner_microvm_nft_list_command, and derives +// TRIGGER: that controller executes teardown on a host -- runs runner_slot_teardown_flush_commands, +// reads every observation SlotNetworkReadings names, and derives // CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT // satisfied by #11670 importing the type, or by a witness calling the fold. data slot_network_readback_consumer_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: the microVM slot lifecycle controller's host teardown readback (gunbc#11670). TRIGGER: that controller executes teardown on a host, runs runner_slot_conntrack_flush_commands, reads runner_slot_guest_originated_conntrack_list_command, runner_slot_guest_destined_conntrack_list_command and runner_microvm_nft_list_command, and derives CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT satisfied by an import of the type or by a witness calling the fold." as NonEmptyStr, + description: "NAMED CONSUMER: the microVM slot lifecycle controller's host teardown readback (gunbc#11670). TRIGGER: that controller executes teardown on a host, runs runner_slot_teardown_flush_commands, reads the two conntrack lists, runner_slot_guest_neighbour_show_command, runner_host_expectation_list_command, the tap disable_ipv6 value and runner_microvm_nft_list_command, and derives CellReady or CellQuarantined from slot_network_readback. SUFFICIENT FOR deleting this row. NOT satisfied by an import of the type or by a witness calling the fold." as NonEmptyStr, ) // DECLARED FRONTIER (DESIGN 3c). The kernel records dns0 in /proc/net/pnp; it does not write diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index ee3dd1c71ca..40ab3efb58c 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -18,12 +18,21 @@ import gunbc.runner_microvm_network { SlotNetworkReadback, SlotNetworkConfirmed, SlotNetworkRefuted, SlotNetworkUnobservable, SlotNetworkFact, TapUnheldFact, GuestOriginatedConntrackEmptyFact, GuestDestinedConntrackEmptyFact, RulesetMatchesConvergedFact, - SlotNetworkSubject, slot_network_subject, ConvergedRuleset, - TapReading, GuestConntrackReading, RulesetReading, + SlotNetworkSubject, slot_network_subject, runner_slot_host_cidr, + ConvergedSlot, ConvergedSlotNetwork, HelperStanding, + AutoAssignmentAbsentFromKernel, AutoAssignmentSysctlOff, AutoAssignmentOn, + TapIpv6Standing, TapIpv6Disabled, TapIpv6Enabled, + SanitationSubjectResult, SanitationSubjectAdmitted, SanitationSubjectRefused, + ReceiptForDifferentHost, SlotNotInConvergedPopulation, ConvergedSlotDisagreesWithDerivation, + TapIpv6NotEstablishedDisabled, HelpersNotExcluded, slot_network_sanitation_subject, + TapReading, TapIpv6Reading, SlotResidueReading, HostResidueReading, RulesetReading, SlotNetworkReadings, + ResidueObservation, ResidueAbsent, ResiduePresent, ResidueUnreadable, + TapIpv6DisabledFact, GuestNeighbourAbsentFact, ExpectationTableEmptyFact, + runner_slot_teardown_flush_commands, runner_slot_guest_neighbour_show_command, runner_host_expectation_list_command, + runner_slot_ipv6_sysctl_text, runner_slot_ipv6_sysctl_path, runner_slot_ipv6_disabled_proc_path, + runner_slot_guest_network_boot_args, guest_ipv6_disabled_readback_frontier, converged_slot_network_producer_frontier, TapPresentUnheld, TapHeld, TapMissing, TapHoldersUnreadable, - GuestConntrackEmptyAfterFlush, GuestConntrackResidue, GuestConntrackUnreadable, RulesetListed, RulesetUnreadable, slot_network_readback, - runner_slot_conntrack_flush_commands, runner_slot_guest_originated_conntrack_list_command, runner_slot_guest_destined_conntrack_list_command, runner_microvm_nft_list_command, slot_network_readback_consumer_frontier, runner_microvm_guest_resolver, guest_resolver_configuration_frontier, @@ -32,7 +41,8 @@ import gunbc.runner_microvm_network { import gunbc.runner.runner_host_filtered_egress { runner_host_egress_policy } import std.dissolution { UnboundDissolution, BoundDissolution } import std.content_hash { Sha256Digest, Sha256DigestHex } -import gunbc.runner_microvm_network { TapHolderObservation, GuestConntrackObservation } +import gunbc.runner_microvm_network { TapHolderObservation } +import product.placement_supply { HostIdentity } import extdeps.exec.command { argv_words } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -245,143 +255,244 @@ fn subject(i: Nat) -> SlotNetworkSubject { data converged_generation: Int = 7 -fn converged() -> ConvergedRuleset { - ConvergedRuleset { host: operator_host_srv2, generation: converged_generation, digest: converged_digest } +fn converged_slot(i: Nat) -> ConvergedSlot { + let net = runner_slot_network(slot: slot(i: i)) + ConvergedSlot { + subject: subject(i: i), + host_address_cidr: runner_slot_host_cidr(net: net) as NonEmptyStr, + guest_route: join([runner_slot_host_cidr(net: net), " dev ", net.tap_device as String], "") as NonEmptyStr, + tap_ipv6: TapIpv6Disabled, + } } -fn tap(i: Nat, o: TapHolderObservation) -> TapReading { - TapReading { subject: subject(i: i), observation: o } +fn excluded_helpers() -> HelperStanding { + HelperStanding { + auto_assignment: AutoAssignmentAbsentFromKernel { kernel_release: "6.8.0-138-generic" as NonEmptyStr }, + nft_ct_helper_statements: 0, + xtables_ct_helper_targets: 0, + userspace_helpers: 0, + expectations_at_generation: 0, + } } -fn ct(i: Nat, o: GuestConntrackObservation) -> GuestConntrackReading { - GuestConntrackReading { subject: subject(i: i), observation: o } +fn receipt_on(host: HostIdentity, slots: List, helpers: HelperStanding) -> ConvergedSlotNetwork { + ConvergedSlotNetwork { host: host, generation: converged_generation, slots: slots, helpers: helpers, nft_digest: converged_digest } } -fn listed(d: Sha256Digest) -> RulesetReading { - RulesetReading { host: operator_host_srv2, generation: converged_generation, observation: RulesetListed { digest: d } } +fn srv2_receipt() -> ConvergedSlotNetwork { + receipt_on(host: operator_host_srv2, slots: [converged_slot(i: 1), converged_slot(i: 2)], helpers: excluded_helpers()) +} + +fn admission(r: SanitationSubjectResult) -> String { + match r { + SanitationSubjectAdmitted { subject: _ } => "admitted" + SanitationSubjectRefused { refusal: f } => match f { + ReceiptForDifferentHost { slot_host: _, receipt_host: _ } => "other-host" + SlotNotInConvergedPopulation => "not-in-population" + ConvergedSlotDisagreesWithDerivation => "disagrees" + TapIpv6NotEstablishedDisabled => "tap-ipv6" + HelpersNotExcluded => "helpers" + } + } } -fn clean_ct(i: Nat) -> GuestConntrackReading { - ct(i: i, o: GuestConntrackEmptyAfterFlush) +// THE SANITATION SUBJECT JOINS THE SLOT TO ITS OWN HOST'S RECEIPT, OR IS NOT CONSTRUCTED. A slot on +// srv1 against srv2's self-consistent receipt refuses -- the cross-host RED of the ruling on #11675 -- +// as do a slot outside the converged population, a slot whose tap v6 is not established disabled, and +// a host whose helper standing is not excluded. The positive control is the slot's own host. +test fn the_sanitation_subject_is_only_the_slots_own_hosts_established_receipt() -> Bool { + let srv1_slot = RunnerSlotIdentity { host: operator_host_srv1, slot_index: 1 } + admission(r: slot_network_sanitation_subject(slot: slot(i: 1), receipt: srv2_receipt())) == "admitted" + && admission(r: slot_network_sanitation_subject(slot: srv1_slot, receipt: srv2_receipt())) == "other-host" + && admission(r: slot_network_sanitation_subject(slot: slot(i: 3), receipt: srv2_receipt())) == "not-in-population" + && admission(r: slot_network_sanitation_subject( + slot: slot(i: 1), + receipt: receipt_on(host: operator_host_srv2, slots: [ConvergedSlot { + subject: subject(i: 1), host_address_cidr: "172.30.1.1/30" as NonEmptyStr, + guest_route: "172.30.1.0/30 dev gunbc-tap1" as NonEmptyStr, tap_ipv6: TapIpv6Enabled, + }], helpers: excluded_helpers()), + )) == "tap-ipv6" + && admission(r: slot_network_sanitation_subject( + slot: slot(i: 1), + receipt: receipt_on(host: operator_host_srv2, slots: [converged_slot(i: 1)], helpers: HelperStanding { + auto_assignment: AutoAssignmentSysctlOff, + nft_ct_helper_statements: 1, + xtables_ct_helper_targets: 0, + userspace_helpers: 0, + expectations_at_generation: 0, + }), + )) == "helpers" + && admission(r: slot_network_sanitation_subject( + slot: slot(i: 1), + receipt: receipt_on(host: operator_host_srv2, slots: [converged_slot(i: 1)], helpers: HelperStanding { + auto_assignment: AutoAssignmentOn, + nft_ct_helper_statements: 0, + xtables_ct_helper_targets: 0, + userspace_helpers: 0, + expectations_at_generation: 0, + }), + )) == "helpers" +} + +fn clean_readings(i: Nat) -> SlotNetworkReadings { + SlotNetworkReadings { + tap: TapReading { subject: subject(i: i), observation: TapPresentUnheld }, + tap_ipv6: TapIpv6Reading { subject: subject(i: i), observation: TapIpv6Disabled }, + guest_originated_conntrack: SlotResidueReading { subject: subject(i: i), observation: ResidueAbsent }, + guest_destined_conntrack: SlotResidueReading { subject: subject(i: i), observation: ResidueAbsent }, + guest_neighbour: SlotResidueReading { subject: subject(i: i), observation: ResidueAbsent }, + expectations: HostResidueReading { host: operator_host_srv2, generation: converged_generation, observation: ResidueAbsent }, + ruleset: RulesetReading { host: operator_host_srv2, generation: converged_generation, observation: RulesetListed { digest: converged_digest } }, + } } -// Sanitizes slot 1 from the given readings. -fn readback_for_slot_1(t: TapReading, from_guest: GuestConntrackReading, to_guest: GuestConntrackReading, r: RulesetReading) -> SlotNetworkReadback { - slot_network_readback( - expected: subject(i: 1), - converged: converged(), - tap: t, - guest_originated: from_guest, - guest_destined: to_guest, - ruleset: r, - ) +// Sanitizes slot 1 on srv2. A refused subject reads "no-subject", so a claim can never pass by the +// fold being skipped. +fn readback_slot_1(r: SlotNetworkReadings) -> SlotNetworkReadback? { + match slot_network_sanitation_subject(slot: slot(i: 1), receipt: srv2_receipt()) { + SanitationSubjectAdmitted { subject: s } => Present { value: slot_network_readback(s: s, readings: r) } + SanitationSubjectRefused { refusal: _ } => none + } } fn fact_name(f: SlotNetworkFact) -> String { match f { TapUnheldFact => "tap" + TapIpv6DisabledFact => "tap-ipv6" GuestOriginatedConntrackEmptyFact => "conntrack-from" GuestDestinedConntrackEmptyFact => "conntrack-to" + GuestNeighbourAbsentFact => "neighbour" + ExpectationTableEmptyFact => "expectations" RulesetMatchesConvergedFact => "ruleset" } } -fn refuted_fact(r: SlotNetworkReadback) -> String { +fn verdict(r: SlotNetworkReadback?) -> String { match r { - SlotNetworkRefuted { fact: f, detail: _ } => fact_name(f: f) - SlotNetworkConfirmed => "confirmed" - SlotNetworkUnobservable { fact: _, reason: _ } => "unobservable" + Absent => "no-subject" + Present { value: v } => match v { + SlotNetworkConfirmed => "confirmed" + SlotNetworkRefuted { fact: f, detail: _ } => join(["refuted:", fact_name(f: f)], "") + SlotNetworkUnobservable { fact: f, reason: _ } => join(["unobservable:", fact_name(f: f)], "") + } } } -fn unobservable_fact(r: SlotNetworkReadback) -> String { - match r { - SlotNetworkUnobservable { fact: f, reason: _ } => fact_name(f: f) - SlotNetworkConfirmed => "confirmed" - SlotNetworkRefuted { fact: _, detail: _ } => "refuted" - } +fn with_tap(r: SlotNetworkReadings, t: TapReading) -> SlotNetworkReadings { + SlotNetworkReadings { tap: t, tap_ipv6: r.tap_ipv6, guest_originated_conntrack: r.guest_originated_conntrack, guest_destined_conntrack: r.guest_destined_conntrack, guest_neighbour: r.guest_neighbour, expectations: r.expectations, ruleset: r.ruleset } +} + +fn with_tap_ipv6(r: SlotNetworkReadings, t: TapIpv6Reading) -> SlotNetworkReadings { + SlotNetworkReadings { tap: r.tap, tap_ipv6: t, guest_originated_conntrack: r.guest_originated_conntrack, guest_destined_conntrack: r.guest_destined_conntrack, guest_neighbour: r.guest_neighbour, expectations: r.expectations, ruleset: r.ruleset } +} + +fn with_from(r: SlotNetworkReadings, c: SlotResidueReading) -> SlotNetworkReadings { + SlotNetworkReadings { tap: r.tap, tap_ipv6: r.tap_ipv6, guest_originated_conntrack: c, guest_destined_conntrack: r.guest_destined_conntrack, guest_neighbour: r.guest_neighbour, expectations: r.expectations, ruleset: r.ruleset } +} + +fn with_to(r: SlotNetworkReadings, c: SlotResidueReading) -> SlotNetworkReadings { + SlotNetworkReadings { tap: r.tap, tap_ipv6: r.tap_ipv6, guest_originated_conntrack: r.guest_originated_conntrack, guest_destined_conntrack: c, guest_neighbour: r.guest_neighbour, expectations: r.expectations, ruleset: r.ruleset } } -// THE POSITIVE CONTROL: every fact holds for the subject being sanitized, and only then is it confirmed. +fn with_neighbour(r: SlotNetworkReadings, c: SlotResidueReading) -> SlotNetworkReadings { + SlotNetworkReadings { tap: r.tap, tap_ipv6: r.tap_ipv6, guest_originated_conntrack: r.guest_originated_conntrack, guest_destined_conntrack: r.guest_destined_conntrack, guest_neighbour: c, expectations: r.expectations, ruleset: r.ruleset } +} + +fn with_expectations(r: SlotNetworkReadings, e: HostResidueReading) -> SlotNetworkReadings { + SlotNetworkReadings { tap: r.tap, tap_ipv6: r.tap_ipv6, guest_originated_conntrack: r.guest_originated_conntrack, guest_destined_conntrack: r.guest_destined_conntrack, guest_neighbour: r.guest_neighbour, expectations: e, ruleset: r.ruleset } +} + +fn with_ruleset(r: SlotNetworkReadings, l: RulesetReading) -> SlotNetworkReadings { + SlotNetworkReadings { tap: r.tap, tap_ipv6: r.tap_ipv6, guest_originated_conntrack: r.guest_originated_conntrack, guest_destined_conntrack: r.guest_destined_conntrack, guest_neighbour: r.guest_neighbour, expectations: r.expectations, ruleset: l } +} + +fn slot_residue(i: Nat, o: ResidueObservation) -> SlotResidueReading { + SlotResidueReading { subject: subject(i: i), observation: o } +} + +// THE POSITIVE CONTROL: every fact holds for the admitted subject, and only then is it confirmed. test fn a_clean_slot_network_is_confirmed_only_when_every_fact_holds() -> Bool { - refuted_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), - )) == "confirmed" + verdict(r: readback_slot_1(r: clean_readings(i: 1))) == "confirmed" } -// EACH FACT REFUTES ON ITS OWN, AND THE REFUTATION NAMES IT. A VMM fd still on the tap, a flow in -// either original direction that survived the flush, and an edited ruleset are different remedies. +// EACH FACT REFUTES ON ITS OWN AND NAMES ITSELF. A VMM fd on the tap, v6 re-enabled on it, a flow in +// either original direction, a neighbour entry, an expectation, and an edited ruleset are different +// remedies. test fn each_fact_refutes_alone_and_names_itself() -> Bool { - refuted_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapHeld { holders: 1 }), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), - )) == "tap" - && refuted_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapMissing), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), - )) == "tap" - && refuted_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), from_guest: ct(i: 1, o: GuestConntrackResidue { entries: 2 }), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), - )) == "conntrack-from" - && refuted_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: ct(i: 1, o: GuestConntrackResidue { entries: 1 }), r: listed(d: converged_digest), - )) == "conntrack-to" - && refuted_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: drifted_digest), - )) == "ruleset" -} - -// "COULD NOT READ" IS NOT "REFUTED" -- and a refutation elsewhere outranks it. + let c = clean_readings(i: 1) + verdict(r: readback_slot_1(r: with_tap(r: c, t: TapReading { subject: subject(i: 1), observation: TapHeld { holders: 1 } }))) == "refuted:tap" + && verdict(r: readback_slot_1(r: with_tap(r: c, t: TapReading { subject: subject(i: 1), observation: TapMissing }))) == "refuted:tap" + && verdict(r: readback_slot_1(r: with_tap_ipv6(r: c, t: TapIpv6Reading { subject: subject(i: 1), observation: TapIpv6Enabled }))) == "refuted:tap-ipv6" + && verdict(r: readback_slot_1(r: with_from(r: c, c: slot_residue(i: 1, o: ResiduePresent { entries: 2 })))) == "refuted:conntrack-from" + && verdict(r: readback_slot_1(r: with_to(r: c, c: slot_residue(i: 1, o: ResiduePresent { entries: 1 })))) == "refuted:conntrack-to" + && verdict(r: readback_slot_1(r: with_neighbour(r: c, c: slot_residue(i: 1, o: ResiduePresent { entries: 1 })))) == "refuted:neighbour" + && verdict(r: readback_slot_1(r: with_expectations(r: c, e: HostResidueReading { host: operator_host_srv2, generation: converged_generation, observation: ResiduePresent { entries: 1 } }))) == "refuted:expectations" + && verdict(r: readback_slot_1(r: with_ruleset(r: c, l: RulesetReading { host: operator_host_srv2, generation: converged_generation, observation: RulesetListed { digest: drifted_digest } }))) == "refuted:ruleset" +} + +// "COULD NOT READ" IS NOT "REFUTED" -- for the neighbour table as for every fact -- and a refutation +// elsewhere outranks it. test fn an_unreadable_fact_is_unobservable_and_a_refutation_elsewhere_outranks_it() -> Bool { - unobservable_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), - from_guest: ct(i: 1, o: GuestConntrackUnreadable { reason: "conntrack table not readable" as NonEmptyStr }), - to_guest: clean_ct(i: 1), - r: listed(d: converged_digest), - )) == "conntrack-from" - && refuted_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), - from_guest: ct(i: 1, o: GuestConntrackUnreadable { reason: "conntrack table not readable" as NonEmptyStr }), - to_guest: clean_ct(i: 1), - r: listed(d: drifted_digest), - )) == "ruleset" -} - -// SLOT A'S CLEAN READINGS CANNOT SANITIZE SLOT B. Every reading below is clean, but one fact at a time -// is read about slot 2 -- or, for the ruleset, on another host or at another converge generation -- -// while slot 1 is the subject. Each yields Unobservable for exactly that fact, never Confirmed. The red -// is a fold that inspects only the observation (sunny-ant-606 ruling on #11675). + let c = clean_readings(i: 1) + let unreadable_neighbour = with_neighbour(r: c, c: slot_residue(i: 1, o: ResidueUnreadable { reason: "ip neigh show refused" as NonEmptyStr })) + verdict(r: readback_slot_1(r: unreadable_neighbour)) == "unobservable:neighbour" + && verdict(r: readback_slot_1(r: with_from(r: c, c: slot_residue(i: 1, o: ResidueUnreadable { reason: "conntrack table not readable" as NonEmptyStr })))) == "unobservable:conntrack-from" + && verdict(r: readback_slot_1(r: with_ruleset(r: unreadable_neighbour, l: RulesetReading { host: operator_host_srv2, generation: converged_generation, observation: RulesetListed { digest: drifted_digest } }))) == "refuted:ruleset" +} + +// SLOT A'S CLEAN READINGS CANNOT SANITIZE SLOT B. Each slot-scoped fact, read clean about slot 2 while +// slot 1 is the subject, is Unobservable -- the neighbour fact included, the cross-slot RED of the +// ruling. Host-scoped readings taken on another host or at another generation are too. test fn readings_about_another_subject_never_sanitize_this_one() -> Bool { - unobservable_fact(r: readback_for_slot_1( - t: tap(i: 2, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), - )) == "tap" - && unobservable_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 2), to_guest: clean_ct(i: 1), r: listed(d: converged_digest), - )) == "conntrack-from" - && unobservable_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 2), r: listed(d: converged_digest), - )) == "conntrack-to" - && unobservable_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), - r: RulesetReading { host: operator_host_srv1, generation: converged_generation, observation: RulesetListed { digest: converged_digest } }, - )) == "ruleset" - && unobservable_fact(r: readback_for_slot_1( - t: tap(i: 1, o: TapPresentUnheld), from_guest: clean_ct(i: 1), to_guest: clean_ct(i: 1), - r: RulesetReading { host: operator_host_srv2, generation: 6, observation: RulesetListed { digest: converged_digest } }, - )) == "ruleset" -} - -// THE FLUSHES AND LISTS COVER BOTH ORIGINAL DIRECTIONS FOR THIS SLOT'S GUEST. A flow the guest opened -// has it as original source; a flow the host opened toward it has it as original destination. The -// reds: a single-direction flush, or commands naming another slot's guest. -test fn conntrack_commands_cover_both_directions_for_this_slots_guest() -> Bool { + let c = clean_readings(i: 1) + verdict(r: readback_slot_1(r: with_tap(r: c, t: TapReading { subject: subject(i: 2), observation: TapPresentUnheld }))) == "unobservable:tap" + && verdict(r: readback_slot_1(r: with_tap_ipv6(r: c, t: TapIpv6Reading { subject: subject(i: 2), observation: TapIpv6Disabled }))) == "unobservable:tap-ipv6" + && verdict(r: readback_slot_1(r: with_from(r: c, c: slot_residue(i: 2, o: ResidueAbsent)))) == "unobservable:conntrack-from" + && verdict(r: readback_slot_1(r: with_to(r: c, c: slot_residue(i: 2, o: ResidueAbsent)))) == "unobservable:conntrack-to" + && verdict(r: readback_slot_1(r: with_neighbour(r: c, c: slot_residue(i: 2, o: ResidueAbsent)))) == "unobservable:neighbour" + && verdict(r: readback_slot_1(r: with_expectations(r: c, e: HostResidueReading { host: operator_host_srv1, generation: converged_generation, observation: ResidueAbsent }))) == "unobservable:expectations" + && verdict(r: readback_slot_1(r: with_ruleset(r: c, l: RulesetReading { host: operator_host_srv2, generation: 6, observation: RulesetListed { digest: converged_digest } }))) == "unobservable:ruleset" +} + +// TEARDOWN FLUSHES BOTH CONNTRACK DIRECTIONS AND THIS SLOT'S NEIGHBOUR ENTRY, ON THIS SLOT'S TAP; THE +// LISTS READ THE SAME KEYS. The reds: a single-direction flush, a neighbour flush on another device, +// or commands naming another slot's guest. +test fn teardown_commands_cover_every_slot_keyed_state_for_this_slot() -> Bool { let net = runner_slot_network(slot: slot(i: 6)) let guest = render_ipv4_address(addr: net.guest_address) as String - map(runner_slot_conntrack_flush_commands(net: net), c => join(argv_words(command: c), " ")) - == [join(["conntrack -D -s ", guest], ""), join(["conntrack -D -d ", guest], "")] + map(runner_slot_teardown_flush_commands(net: net), c => join(argv_words(command: c), " ")) + == [ + join(["conntrack -D -s ", guest], ""), + join(["conntrack -D -d ", guest], ""), + join(["ip neigh flush to ", guest, " dev gunbc-tap6"], ""), + ] && argv_words(command: runner_slot_guest_originated_conntrack_list_command(net: net)) == ["conntrack", "-L", "-s", guest] && argv_words(command: runner_slot_guest_destined_conntrack_list_command(net: net)) == ["conntrack", "-L", "-d", guest] + && argv_words(command: runner_slot_guest_neighbour_show_command(net: net)) == ["ip", "neigh", "show", "to", guest, "dev", "gunbc-tap6"] + && argv_words(command: runner_host_expectation_list_command()) == ["conntrack", "-L", "expect"] && guest == "172.30.6.2" } +// v6 IS OFF ON BOTH ENDS OF THE LINK. The tap's sysctl.d key names this slot's device, its readback +// path is the kernel's live value for the same device, and the guest boots with ipv6.disable=1. The +// reds: a key naming another device (the setting lands nowhere), or a boot line without the refusal. +test fn ipv6_is_disabled_on_the_tap_and_in_the_guest() -> Bool { + let net = runner_slot_network(slot: slot(i: 4)) + runner_slot_ipv6_sysctl_text(net: net) == "net.ipv6.conf.gunbc-tap4.disable_ipv6 = 1\n" + && runner_slot_ipv6_sysctl_path(net: net) == "/etc/sysctl.d/50-gunbc-tap4-ipv6.conf" + && runner_slot_ipv6_disabled_proc_path(net: net) == "/proc/sys/net/ipv6/conf/gunbc-tap4/disable_ipv6" + && string_contains(s: runner_slot_guest_network_boot_args(net: net), pattern: " ipv6.disable=1") + && string_contains(s: runner_slot_guest_network_boot_args(net: net), pattern: runner_slot_guest_ip_boot_arg(net: net)) + && match guest_ipv6_disabled_readback_frontier { + UnboundDissolution { description: _ } => true + BoundDissolution { trigger: _ } => false + } + && match converged_slot_network_producer_frontier { + UnboundDissolution { description: _ } => true + BoundDissolution { trigger: _ } => false + } +} + // THE READBACK READS THE LISTING OF OUR OWN TABLE, AND ITS CONSUMER IS STILL A FRONTIER. The red for // the first half is a listing of a different table (a digest of dockerd's rules would never match); // for the second, the frontier row bound before the controller executes teardown. From a86f37d173807e6e3141901c00f1cde1927e276e Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 18:42:39 +0000 Subject: [PATCH 12/36] Bind the slot network readback result to its sanitation subject Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_microvm_network.dag | 25 ++++++++++++++++++- .../runner_microvm_network_witness_test.dag | 22 ++++++++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index ed823dad308..3bd4a404864 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -1028,7 +1028,7 @@ fn first_readback_where(rs: List, pick: fn(SlotNetworkReadb ) } -fn slot_network_readback(s: SlotNetworkSanitationSubject, readings: SlotNetworkReadings) -> SlotNetworkReadback { +fn slot_network_verdict(s: SlotNetworkSanitationSubject, readings: SlotNetworkReadings) -> SlotNetworkReadback { let facts = [ tap_readback(s: s, r: readings.tap), tap_ipv6_readback(s: s, r: readings.tap_ipv6), @@ -1047,6 +1047,29 @@ fn slot_network_readback(s: SlotNetworkSanitationSubject, readings: SlotNetworkR } } +// THE RESULT CARRIES THE SUBJECT IT WAS TAKEN FOR. A bare verdict leaves the subject on the input +// side, so a consumer holding a Confirmed would have to re-derive or re-assert which slot it is about +// -- a second authority for that join. The lifecycle controller's teardown readback compares +// bound_slot_network_readback_slot against its CellBoundAttempt's slot instead (tidy-wolf-685 on +// #11670). The record has one constructor, this fold, so its subject and result cannot be paired +// by hand. +type BoundSlotNetworkReadback sole_constructor { + subject: SlotNetworkSanitationSubject + result: SlotNetworkReadback +} + +fn slot_network_readback(s: SlotNetworkSanitationSubject, readings: SlotNetworkReadings) -> BoundSlotNetworkReadback { + BoundSlotNetworkReadback { subject: s, result: slot_network_verdict(s: s, readings: readings) } +} + +fn bound_slot_network_readback_slot(b: BoundSlotNetworkReadback) -> RunnerSlotIdentity { + b.subject.subject.slot +} + +fn bound_slot_network_readback_result(b: BoundSlotNetworkReadback) -> SlotNetworkReadback { + b.result +} + // --- the commands that produce the readback's observations --- // TEARDOWN FLUSHES; THE LIFECYCLE CONTROLLER READS. The flushes are mutations and belong to the diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index 40ab3efb58c..c3b6dbab1ee 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -5,7 +5,7 @@ import std.nat { Nat } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import extdeps.network.ipv4 { render_ipv4_address } import extdeps.network.mac { MacAddressScope, LocallyAdministered, mac_address_scope, render_mac_address } -import gunbc.build_cache_instance { RunnerSlotIdentity } +import gunbc.build_cache_instance { RunnerSlotIdentity, runner_slot_identity_equal } import gunbc.fleet_intent_network { operator_host_srv2, operator_host_srv1 } import gunbc.runner_microvm_network { runner_slot_network, runner_slot_tap_device_name, runner_slot_guest_mac, @@ -25,6 +25,7 @@ import gunbc.runner_microvm_network { SanitationSubjectResult, SanitationSubjectAdmitted, SanitationSubjectRefused, ReceiptForDifferentHost, SlotNotInConvergedPopulation, ConvergedSlotDisagreesWithDerivation, TapIpv6NotEstablishedDisabled, HelpersNotExcluded, slot_network_sanitation_subject, + bound_slot_network_readback_slot, bound_slot_network_readback_result, TapReading, TapIpv6Reading, SlotResidueReading, HostResidueReading, RulesetReading, SlotNetworkReadings, ResidueObservation, ResidueAbsent, ResiduePresent, ResidueUnreadable, TapIpv6DisabledFact, GuestNeighbourAbsentFact, ExpectationTableEmptyFact, @@ -350,7 +351,7 @@ fn clean_readings(i: Nat) -> SlotNetworkReadings { // fold being skipped. fn readback_slot_1(r: SlotNetworkReadings) -> SlotNetworkReadback? { match slot_network_sanitation_subject(slot: slot(i: 1), receipt: srv2_receipt()) { - SanitationSubjectAdmitted { subject: s } => Present { value: slot_network_readback(s: s, readings: r) } + SanitationSubjectAdmitted { subject: s } => Present { value: bound_slot_network_readback_result(b: slot_network_readback(s: s, readings: r)) } SanitationSubjectRefused { refusal: _ } => none } } @@ -534,3 +535,20 @@ test fn a_chain_dropping_host_folds_its_own_grain_and_still_carries_replies() -> && !string_contains(s: text, pattern: "iifname \"gunbc-tap*\" drop\n }") && string_contains(s: text, pattern: "ip daddr 1.1.1.1/32 udp dport 53 accept") } + +// THE RESULT NAMES THE SLOT IT WAS TAKEN FOR, WHICHEVER VERDICT IT CARRIES. The lifecycle controller +// joins on this slot rather than re-deriving it. The red is a result bound to any slot but the +// sanitation subject's -- slot 2's readback reporting slot 1, or the binding lost on a refutation. +test fn the_readback_result_carries_the_slot_it_was_taken_for() -> Bool { + match slot_network_sanitation_subject(slot: slot(i: 2), receipt: srv2_receipt()) { + SanitationSubjectAdmitted { subject: s } => { + let clean = slot_network_readback(s: s, readings: clean_readings(i: 2)) + let refuted = slot_network_readback(s: s, readings: with_neighbour(r: clean_readings(i: 2), c: slot_residue(i: 2, o: ResiduePresent { entries: 1 }))) + runner_slot_identity_equal(a: bound_slot_network_readback_slot(b: clean), b: slot(i: 2)) + && !runner_slot_identity_equal(a: bound_slot_network_readback_slot(b: clean), b: slot(i: 1)) + && runner_slot_identity_equal(a: bound_slot_network_readback_slot(b: refuted), b: slot(i: 2)) + && verdict(r: Present { value: bound_slot_network_readback_result(b: refuted) }) == "refuted:neighbour" + } + SanitationSubjectRefused { refusal: _ } => false + } +} From 25b590e3b40b9f214be96ca2f9aa7301ad8523d4 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 19:41:55 +0000 Subject: [PATCH 13/36] Realize the guest policy in the input hook; one guest boot-argument spelling on every host Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/nftables/ruleset.dag | 2 + .../runner/runner_host_filtered_egress.dag | 20 ++++----- dag/gunbc/runner/runner_microvm_network.dag | 45 ++++++++++++++++++- .../runner_microvm_network_witness_test.dag | 14 +++++- 4 files changed, 66 insertions(+), 15 deletions(-) diff --git a/dag/extdeps/nftables/ruleset.dag b/dag/extdeps/nftables/ruleset.dag index d70b904f843..ff2a8f5142a 100644 --- a/dag/extdeps/nftables/ruleset.dag +++ b/dag/extdeps/nftables/ruleset.dag @@ -49,6 +49,7 @@ type NftHook = NftForward | NftPostrouting | NftPrerouting + | NftInput // PRIORITY IS AN INTEGER WITH NAMED LANDMARKS, and the landmarks are the ones nftables itself // documents: filter is 0, srcnat is 100, raw is -300. raw runs BEFORE connection tracking (-200), so @@ -80,6 +81,7 @@ fn nft_hook_wire(h: NftHook) -> String { NftForward => "forward" NftPostrouting => "postrouting" NftPrerouting => "prerouting" + NftInput => "input" } } diff --git a/dag/gunbc/runner/runner_host_filtered_egress.dag b/dag/gunbc/runner/runner_host_filtered_egress.dag index 6b9afc24471..229983c6feb 100644 --- a/dag/gunbc/runner/runner_host_filtered_egress.dag +++ b/dag/gunbc/runner/runner_host_filtered_egress.dag @@ -3,7 +3,7 @@ module gunbc.runner.runner_host_filtered_egress import std.types { Int, List, NonEmptyStr, String } import extdeps.network.ipv4 { Ipv4Address, render_ipv4_address } import gunbc.runner_microvm_network { - RunnerSlotNetwork, runner_slot_host_cidr, runner_slot_guest_ip_boot_arg, + RunnerSlotNetwork, runner_slot_host_cidr, runner_microvm_network_supernet, GuestEgressPolicy, runner_microvm_guest_egress_policy, HostEgressGrain, HostChainDefaultDrop, IpV6DisabledOnDataPath, @@ -76,7 +76,6 @@ data runner_host_uplink: HostUplinkConfiguration = StaticFromObservedOfferCandid // EVERY CHAIN STARTS CLOSED AND OPENS BY NAME. The grants are the whole of what a guest may reach, // and the denials that matter most are the ones a permissive default would have allowed silently: // the BMC, the rest of the LAN, and any other attempt's tap. -type ChainStance = DefaultDrop | DefaultAccept // THE GUEST-FACING GRANTS AND DENIALS ARE CONSUMED, NOT RESTATED. They are // gunbc.runner_microvm_network runner_microvm_guest_egress_policy, the same rows the shared hosts' @@ -95,15 +94,13 @@ type ChainStance = DefaultDrop | DefaultAccept // and tap to tap is the forward chain's first rule in gunbc.runner_microvm_network; a prose list here // would be a third spelling of both (review 68294). type EgressPolicyShape { - host_input: ChainStance host: HostEgressGrain guest: GuestEgressPolicy nat: NonEmptyStr } data runner_host_egress_policy: EgressPolicyShape = EgressPolicyShape { - host_input: DefaultDrop, - host: HostEgressGrain { forward: HostChainDefaultDrop, ipv6: IpV6DisabledOnDataPath }, + host: HostEgressGrain { forward: HostChainDefaultDrop, input: HostChainDefaultDrop, ipv6: IpV6DisabledOnDataPath }, guest: runner_microvm_guest_egress_policy, nat: "masquerade the admitted guest prefix on the host uplink, and nothing else", } @@ -193,17 +190,16 @@ fn runner_host_admitted_guest_source(net: RunnerSlotNetwork) -> String { join([render_ipv4_address(addr: net.guest_address) as String, "/32"], "") } -// THE HOST SIDE OF THE POINT-TO-POINT LINK, which is the guest's only next hop and therefore the -// only address it may reach on the host at all. +// THE HOST SIDE OF THE POINT-TO-POINT LINK, which is the guest's only next hop. It is a next hop +// and not a destination: the input chain refuses everything a tap sends to the host itself. fn runner_host_tap_cidr(net: RunnerSlotNetwork) -> String { runner_slot_host_cidr(net: net) } -// THE GUEST IS TOLD ITS ADDRESS AT BOOT, WHICH IS WHY NO DHCP CLIENT IS NEEDED ON EITHER SIDE. The -// same rendering the fleet already uses; the raw host does not get a second spelling of it. -fn runner_host_guest_boot_ip_arg(net: RunnerSlotNetwork) -> String { - runner_slot_guest_ip_boot_arg(net: net) -} +// THE GUEST'S BOOT ARGUMENTS ARE gunbc.runner_microvm_network runner_slot_guest_network_boot_args ON +// EVERY HOST. A wrapper here returned only the ip= directive, a second spelling that omitted the v6 +// refusal on the one host with no v6 ruleset behind it (review 68590); it had no caller and is gone. + // TAP-TO-TAP IS DENIED BY THE SUPERNET, not by enumerating peers. Every slot's /30 sits inside the // one supernet, so a single rule denying guest -> supernet forbids reaching EVERY other attempt -- diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 3bd4a404864..48ec65c5946 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -15,7 +15,7 @@ import gunbc.executor_privileged_operation { } import extdeps.nftables.ruleset { NftTable, NftChain, NftRule, NftMatch, NftFamily, NftInet, NftHook, NftForward, NftPostrouting, - NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPriorityRaw, NftPrerouting, NftPolicy, NftPolicyAccept, NftPolicyDrop, + NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPriorityRaw, NftPrerouting, NftInput, NftInputInterface, NftPolicy, NftPolicyAccept, NftPolicyDrop, NftChainType, NftTypeFilter, NftTypeNat, NftVerdict, NftAccept, NftDrop, NftMasquerade, NftInputInterfaceWildcard, NftOutputInterfaceWildcard, NftSourceAddressPrefix, @@ -301,8 +301,14 @@ type GuestEgressGrant { // denials are one row for the fleet. The forward grain and the IPv6 standing are facts about ONE // host -- whether it owns its forward chain, whether v6 is off on its data path -- so each host // supplies its own HostEgressGrain, and no host's row carries another host's values (review 68199). +// THE INPUT HOOK IS PART OF WHAT A GUEST MAY REACH. Traffic a guest sends to the HOST's own addresses +// -- sshd, the dockerd socket, any local listener -- never traverses forward, so a policy projected +// only into forward would deny 172.16.0.0/12 in its type while the host's tap address in that range +// stayed reachable (review 68590). Each host states its input grain the same way it states its +// forward grain, and the fold realizes both. type HostEgressGrain { forward: ForwardDenyGrain + input: ForwardDenyGrain ipv6: IpV6Standing } @@ -326,6 +332,9 @@ data runner_microvm_shared_host_grain: HostEgressGrain = HostEgressGrain { forward: TapScopedFinalDrop { chain_policy_reason: "other tables hook forward on srvN (dockerd on srv2) and every table runs with any drop winning, so a chain-level drop would drop traffic this fleet does not own" as NonEmptyStr, }, + input: TapScopedFinalDrop { + chain_policy_reason: "the shared hosts run their own services and other tables hook input, so a chain-level drop would cut traffic this fleet does not own" as NonEmptyStr, + }, ipv6: IpV6DefaultDenyRuleset, } @@ -435,6 +444,38 @@ fn raw_ipv6_rules(v: IpV6Standing) -> List { } } +// A GUEST REACHES NOTHING ON THE HOST. It resolves through a public resolver and fetches over the +// forward path, so no host-local service is one it needs: from a tap, input accepts only replies to +// flows the host itself opened and drops the rest, on every grain. A host that owns its whole input +// policy (HostChainDefaultDrop) additionally drops at the chain and so admits its own loopback and +// replies explicitly -- the two input rows the Mt. Collins receipt read back. +fn input_chain_rules(g: ForwardDenyGrain) -> List { + concat( + match g { + HostChainDefaultDrop => [ + NftRule { matches: [NftInputInterface { device: "lo" as NonEmptyStr }], verdict: NftAccept }, + NftRule { matches: [NftConnectionTrackingEstablished], verdict: NftAccept }, + ] + TapScopedFinalDrop { chain_policy_reason: _ } => [] + }, + [ + from_guest_tap(m: [NftConnectionTrackingEstablished], verdict: NftAccept), + from_guest_tap(m: [], verdict: NftDrop), + ], + ) +} + +fn runner_microvm_input_chain(host: HostEgressGrain) -> NftChain { + NftChain { + name: "input" as NonEmptyStr, + chain_type: NftTypeFilter, + hook: NftInput, + priority: NftPriorityFilter, + policy: forward_chain_policy(g: host.input), + rules: input_chain_rules(g: host.input), + } +} + fn runner_microvm_raw_prerouting_chain(host: HostEgressGrain) -> NftChain { NftChain { name: "raw_prerouting" as NonEmptyStr, @@ -471,7 +512,7 @@ fn runner_microvm_nft_table(host: HostEgressGrain) -> NftTable { NftTable { family: NftInet, name: runner_microvm_nft_table_name, - chains: [runner_microvm_raw_prerouting_chain(host: host), runner_microvm_forward_chain(host: host), runner_microvm_postrouting_chain()], + chains: [runner_microvm_raw_prerouting_chain(host: host), runner_microvm_input_chain(host: host), runner_microvm_forward_chain(host: host), runner_microvm_postrouting_chain()], } } diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index c3b6dbab1ee..e4cd5a36faa 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -532,7 +532,7 @@ test fn a_chain_dropping_host_folds_its_own_grain_and_still_carries_replies() -> string_contains(s: text, pattern: "hook forward priority 0; policy drop;") && string_contains(s: text, pattern: "oifname \"gunbc-tap*\" ct state established,related accept") && !string_contains(s: text, pattern: "meta nfproto ipv6") - && !string_contains(s: text, pattern: "iifname \"gunbc-tap*\" drop\n }") + && string_contains(s: text, pattern: "tcp dport 443 accept\n }") && string_contains(s: text, pattern: "ip daddr 1.1.1.1/32 udp dport 53 accept") } @@ -552,3 +552,15 @@ test fn the_readback_result_carries_the_slot_it_was_taken_for() -> Bool { SanitationSubjectRefused { refusal: _ } => false } } + +// A GUEST REACHES NOTHING ON THE HOST, ON EITHER GRAIN. Guest-to-host traffic never traverses forward, +// so the policy is realized in input too: from a tap, only replies to host-opened flows are accepted +// and the rest dropped. The shared hosts keep input's policy accept (their own services); the raw +// host drops at the chain and admits loopback and its own replies. The red is the previous table, +// with no input chain at all -- the guest reached every host-local listener (review 68590). +test fn a_guest_reaches_nothing_on_the_host_on_either_grain() -> Bool { + let shared = runner_microvm_shared_host_nft_ruleset_text() + let raw = runner_microvm_nft_ruleset_text_for(host: runner_host_egress_policy.host) + string_contains(s: shared, pattern: "type filter hook input priority 0; policy accept;\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" drop\n }") + && string_contains(s: raw, pattern: "type filter hook input priority 0; policy drop;\n iifname \"lo\" accept\n ct state established,related accept\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" drop\n }") +} From 4791dd46713e86c674b97a8180bd3f17edcd6800 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 20:33:48 +0000 Subject: [PATCH 14/36] Refuse v6 toward a tap in forward and output; annotations name the direction each v6 rule covers Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/nftables/ruleset.dag | 2 + dag/gunbc/runner/runner_microvm_network.dag | 80 +++++++++++++------ .../runner_microvm_network_witness_test.dag | 15 +++- 3 files changed, 72 insertions(+), 25 deletions(-) diff --git a/dag/extdeps/nftables/ruleset.dag b/dag/extdeps/nftables/ruleset.dag index ff2a8f5142a..52ae238f5c5 100644 --- a/dag/extdeps/nftables/ruleset.dag +++ b/dag/extdeps/nftables/ruleset.dag @@ -50,6 +50,7 @@ type NftHook | NftPostrouting | NftPrerouting | NftInput + | NftOutput // PRIORITY IS AN INTEGER WITH NAMED LANDMARKS, and the landmarks are the ones nftables itself // documents: filter is 0, srcnat is 100, raw is -300. raw runs BEFORE connection tracking (-200), so @@ -82,6 +83,7 @@ fn nft_hook_wire(h: NftHook) -> String { NftPostrouting => "postrouting" NftPrerouting => "prerouting" NftInput => "input" + NftOutput => "output" } } diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 48ec65c5946..675399af465 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -15,7 +15,7 @@ import gunbc.executor_privileged_operation { } import extdeps.nftables.ruleset { NftTable, NftChain, NftRule, NftMatch, NftFamily, NftInet, NftHook, NftForward, NftPostrouting, - NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPriorityRaw, NftPrerouting, NftInput, NftInputInterface, NftPolicy, NftPolicyAccept, NftPolicyDrop, + NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPriorityRaw, NftPrerouting, NftInput, NftOutput, NftInputInterface, NftPolicy, NftPolicyAccept, NftPolicyDrop, NftChainType, NftTypeFilter, NftTypeNat, NftVerdict, NftAccept, NftDrop, NftMasquerade, NftInputInterfaceWildcard, NftOutputInterfaceWildcard, NftSourceAddressPrefix, @@ -247,15 +247,18 @@ fn runner_microvm_forward_chain(host: HostEgressGrain) -> NftChain { priority: NftPriorityFilter, policy: forward_chain_policy(g: host.forward), rules: concat( - [ - NftRule { - matches: [ - NftInputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, - NftOutputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, - ], - verdict: NftDrop, - }, - ], + concat( + [ + NftRule { + matches: [ + NftInputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, + NftOutputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, + ], + verdict: NftDrop, + }, + ], + tap_bound_ipv6_rules(v: host.ipv6), + ), guest_egress_rules(p: runner_microvm_guest_egress_policy, host: host), ), } @@ -279,7 +282,9 @@ type ForwardDenyGrain // IPv6 IS AN EXPLICIT STANDING RATHER THAN AN OVERSIGHT. An IPv4-only ruleset on a dual-stack host is // a default-allow wearing a default-deny's name. In the inet family an `ip daddr` match is false on -// a v6 packet, so under DefaultDenyRuleset the v6 drop must precede every v4 grant. +// a v6 packet, so v4 rows never see v6 at all: under DefaultDenyRuleset v6 is refused by its own +// rows, in both directions of the tap (raw_ipv6_rules, tap_bound_ipv6_rules), and the tap's +// disable_ipv6 closes the link underneath them. type IpV6Standing = IpV6DisabledOnDataPath | IpV6DefaultDenyRuleset @@ -431,11 +436,15 @@ fn guest_egress_rules(p: GuestEgressPolicy, host: HostEgressGrain) -> List List { match v { IpV6DefaultDenyRuleset => [from_guest_tap(m: [NftNetworkProtocolIpv6], verdict: NftDrop)] @@ -444,6 +453,32 @@ fn raw_ipv6_rules(v: IpV6Standing) -> List { } } +fn tap_bound_ipv6_rules(v: IpV6Standing) -> List { + match v { + IpV6DefaultDenyRuleset => [ + NftRule { + matches: [NftOutputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }, NftNetworkProtocolIpv6], + verdict: NftDrop, + }, + ] + IpV6DisabledOnDataPath => [] + IpV6Unaddressed { risk: _ } => [] + } +} + +// THE HOST'S OWN TRAFFIC TOWARD A TAP. Output carries only the tap-bound v6 refusal; everything else +// the host sends keeps whatever the host's other tables decide, so the chain's policy is accept. +fn runner_microvm_output_chain(host: HostEgressGrain) -> NftChain { + NftChain { + name: "output" as NonEmptyStr, + chain_type: NftTypeFilter, + hook: NftOutput, + priority: NftPriorityFilter, + policy: NftPolicyAccept, + rules: tap_bound_ipv6_rules(v: host.ipv6), + } +} + // A GUEST REACHES NOTHING ON THE HOST. It resolves through a public resolver and fetches over the // forward path, so no host-local service is one it needs: from a tap, input accepts only replies to // flows the host itself opened and drops the rest, on every grain. A host that owns its whole input @@ -512,7 +547,7 @@ fn runner_microvm_nft_table(host: HostEgressGrain) -> NftTable { NftTable { family: NftInet, name: runner_microvm_nft_table_name, - chains: [runner_microvm_raw_prerouting_chain(host: host), runner_microvm_input_chain(host: host), runner_microvm_forward_chain(host: host), runner_microvm_postrouting_chain()], + chains: [runner_microvm_raw_prerouting_chain(host: host), runner_microvm_input_chain(host: host), runner_microvm_forward_chain(host: host), runner_microvm_output_chain(host: host), runner_microvm_postrouting_chain()], } } @@ -663,12 +698,11 @@ fn runner_slot_network_install_operations( // --- IPv6: disabled structurally on both ends of the link --- // v6 IS OFF ON THE TAP AND IN THE GUEST, SO THERE IS NO v6 PATH TO FILTER. A drop rule covers only the -// paths it names: the raw-prerouting drop below sees v6 ARRIVING from a tap, not v6 the host itself -// sends toward a tap nor v6 routed to a tap from another interface (sunny-ant-606 ruling on #11675). -// With disable_ipv6=1 on the tap the kernel assigns it no v6 address, installs no v6 route through -// it, and discards v6 received on it -- every direction at once, as a property of the link. The -// raw drop stays as the second wall, and the guest boots with ipv6.disable=1 so it never builds a v6 -// stack at all. +// paths it names (sunny-ant-606 ruling on #11675), so the rules name each direction -- from a tap at +// raw prerouting, toward a tap in forward and output (raw_ipv6_rules, tap_bound_ipv6_rules). With +// disable_ipv6=1 on the tap the kernel assigns it no v6 address, installs no v6 route through it, and +// discards v6 received on it -- every direction at once, as a property of the link. The rules are the +// second wall, and the guest boots with ipv6.disable=1 so it never builds a v6 stack at all. // // THE HOST SETTING IS A sysctl.d FILE, not a command. systemd's udev rules re-run systemd-sysctl for // each net device as it appears, so a per-device key applies when networkd creates the tap at boot diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index e4cd5a36faa..45f761c5f40 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -215,7 +215,7 @@ test fn the_nat_chain_is_typed_nat_and_the_filter_chain_stays_filter() -> Bool { // default-deny into default-accept for every port nobody named. test fn guest_egress_is_default_deny_at_tap_grain_in_the_receipted_order() -> Bool { let text = runner_microvm_shared_host_nft_ruleset_text() - string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") + string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" meta nfproto ipv6 drop\n oifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") && string_contains(s: text, pattern: "policy accept") && !string_contains(s: text, pattern: "policy drop") @@ -239,7 +239,7 @@ test fn tap_ipv6_is_refused_at_raw_before_conntrack_and_not_in_forward() -> Bool let text = runner_microvm_shared_host_nft_ruleset_text() string_contains(s: text, pattern: "type filter hook prerouting priority -300; policy accept;\n iifname \"gunbc-tap*\" meta nfproto ipv6 drop\n }") && !string_contains(s: text, pattern: "hook forward priority 0; policy accept;\n iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n iifname \"gunbc-tap*\" meta nfproto ipv6") - && length(split(s: text, delimiter: "meta nfproto ipv6")) == 2 + && length(split(s: text, delimiter: "iifname \"gunbc-tap*\" meta nfproto ipv6")) == 2 } data converged_digest: Sha256Digest = Sha256Digest { @@ -564,3 +564,14 @@ test fn a_guest_reaches_nothing_on_the_host_on_either_grain() -> Bool { string_contains(s: shared, pattern: "type filter hook input priority 0; policy accept;\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" drop\n }") && string_contains(s: raw, pattern: "type filter hook input priority 0; policy drop;\n iifname \"lo\" accept\n ct state established,related accept\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" drop\n }") } + +// v6 TOWARD A TAP IS DROPPED BY A RULE, IN BOTH PATHS THAT REACH ONE. Routed from another interface it +// crosses forward; sent by the host itself it crosses output. The rendered shared-host table drops +// `oifname "gunbc-tap*" meta nfproto ipv6` in each, the forward one directly after the tap-to-tap +// drop and above every accept. The red is the previous table, where v6 was refused only as it arrived +// FROM a tap and a tap-bound v6 packet met no rule (review 68618). +test fn the_rendered_table_drops_ipv6_egress_toward_the_tap() -> Bool { + let text = runner_microvm_shared_host_nft_ruleset_text() + string_contains(s: text, pattern: "hook forward priority 0; policy accept;\n iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" meta nfproto ipv6 drop\n oifname \"gunbc-tap*\" ct state established,related accept\n") + && string_contains(s: text, pattern: "type filter hook output priority 0; policy accept;\n oifname \"gunbc-tap*\" meta nfproto ipv6 drop\n }") +} From 2ecfc0975cd394a185fff67688e092f5e4db63c9 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 21:17:17 +0000 Subject: [PATCH 15/36] Tap-scoped default-deny names the toward-a-tap direction: final oifname drop on the shared hosts Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_microvm_network.dag | 11 ++++++++++- .../runner/runner_microvm_network_witness_test.dag | 12 +++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 675399af465..49b0e1081d5 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -388,9 +388,18 @@ fn grant_rule(g: GuestEgressGrant) -> NftRule { ) } +// THE TAP-SCOPED DEFAULT-DENY COVERS BOTH DIRECTIONS OF THE TAP. From a tap: everything no grant +// admitted. Toward a tap: everything the reply-direction established accept above did not admit -- +// a new flow routed to a guest from the uplink or any other interface would otherwise fall through +// to the chain's accept policy (review 68643). Guest flows still complete, because their replies +// match that established accept before reaching this row. Under HostChainDefaultDrop the chain policy +// is already the drop for both directions. fn final_drop_rules(g: ForwardDenyGrain) -> List { match g { - TapScopedFinalDrop { chain_policy_reason: _ } => [from_guest_tap(m: [], verdict: NftDrop)] + TapScopedFinalDrop { chain_policy_reason: _ } => [ + from_guest_tap(m: [], verdict: NftDrop), + NftRule { matches: [NftOutputInterfaceWildcard { prefix: runner_microvm_tap_device_prefix }], verdict: NftDrop }, + ] HostChainDefaultDrop => [] } } diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index 45f761c5f40..99da65516ad 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -216,7 +216,7 @@ test fn the_nat_chain_is_typed_nat_and_the_filter_chain_stays_filter() -> Bool { test fn guest_egress_is_default_deny_at_tap_grain_in_the_receipted_order() -> Bool { let text = runner_microvm_shared_host_nft_ruleset_text() string_contains(s: text, pattern: "iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" meta nfproto ipv6 drop\n oifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 10.0.0.0/8 drop\n") - && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n }") + && string_contains(s: text, pattern: "iifname \"gunbc-tap*\" ip daddr 224.0.0.0/4 drop\n iifname \"gunbc-tap*\" ct state established,related accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 udp dport 53 accept\n iifname \"gunbc-tap*\" ip daddr 1.1.1.1/32 tcp dport 53 accept\n iifname \"gunbc-tap*\" tcp dport 443 accept\n iifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" drop\n }") && string_contains(s: text, pattern: "policy accept") && !string_contains(s: text, pattern: "policy drop") } @@ -575,3 +575,13 @@ test fn the_rendered_table_drops_ipv6_egress_toward_the_tap() -> Bool { string_contains(s: text, pattern: "hook forward priority 0; policy accept;\n iifname \"gunbc-tap*\" oifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" meta nfproto ipv6 drop\n oifname \"gunbc-tap*\" ct state established,related accept\n") && string_contains(s: text, pattern: "type filter hook output priority 0; policy accept;\n oifname \"gunbc-tap*\" meta nfproto ipv6 drop\n }") } + +// NOTHING NEW REACHES A TAP UNSOLICITED. On the shared hosts the forward chain's policy is accept, so +// the tap-scoped deny must name the toward-a-tap direction too: the chain ends in an oifname drop +// below the reply-direction established accept. The red is the previous chain, which ended at the +// iifname drop and let a new flow routed to 172.30.x.2 from the uplink fall through to accept +// (review 68643). +test fn a_new_flow_toward_a_tap_is_dropped_on_the_shared_hosts() -> Bool { + let text = runner_microvm_shared_host_nft_ruleset_text() + string_contains(s: text, pattern: " iifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" drop\n }\n chain output") +} From 3961ac57140ea5e0ff8c6876e75f218749eeb728 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sat, 19 Sep 2026 22:02:43 +0000 Subject: [PATCH 16/36] The slot install fold carries the v6 sysctl file it declares Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_microvm_network.dag | 14 +++++++++ .../runner_microvm_network_witness_test.dag | 29 ++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 49b0e1081d5..89438acb4af 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -656,6 +656,12 @@ fn runner_slot_host_cidr(net: RunnerSlotNetwork) -> String { ], "") } +// THE SLOT'S NETWORK FILES ARE ENUMERATED HERE AND NOWHERE ELSE. The v6 sysctl file is one of them: +// leaving it out made the fold say two files while the converge frontier's prose said three, so the +// converge would either install no disable_ipv6 -- and then every slot refuses sanitation subject +// construction, since that requires the tap's v6 read back disabled -- or author its own list beside +// this one (review 68660). +// // THE OPERATIONS ARE INSTALLS AND ONE RELOAD, and every one of them converges: installing a file // that is already byte-identical changes nothing, and reload re-reads without tearing a link down. // There is no non-idempotent arm left to guard, which is why the observe-then-act standing the @@ -665,6 +671,7 @@ fn runner_slot_network_install_operations( net: RunnerSlotNetwork, staged_netdev: NonEmptyStr, staged_network: NonEmptyStr, + staged_ipv6_sysctl: NonEmptyStr, ) -> List { [ InstallFile { @@ -681,6 +688,13 @@ fn runner_slot_network_install_operations( group: runner_microvm_networkd_file_owner, mode: runner_microvm_networkd_file_mode, }, + InstallFile { + source: staged_ipv6_sysctl, + destination: runner_slot_ipv6_sysctl_path(net: net) as NonEmptyStr, + owner: runner_microvm_networkd_file_owner, + group: runner_microvm_networkd_file_owner, + mode: runner_microvm_networkd_file_mode, + }, SystemdReloadUnit { unit: runner_microvm_networkd_unit }, ] } diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index 99da65516ad..157d95c69c5 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -30,7 +30,7 @@ import gunbc.runner_microvm_network { ResidueObservation, ResidueAbsent, ResiduePresent, ResidueUnreadable, TapIpv6DisabledFact, GuestNeighbourAbsentFact, ExpectationTableEmptyFact, runner_slot_teardown_flush_commands, runner_slot_guest_neighbour_show_command, runner_host_expectation_list_command, - runner_slot_ipv6_sysctl_text, runner_slot_ipv6_sysctl_path, runner_slot_ipv6_disabled_proc_path, + runner_slot_ipv6_sysctl_text, runner_slot_ipv6_sysctl_path, runner_slot_network_install_operations, runner_slot_ipv6_disabled_proc_path, runner_slot_guest_network_boot_args, guest_ipv6_disabled_readback_frontier, converged_slot_network_producer_frontier, TapPresentUnheld, TapHeld, TapMissing, TapHoldersUnreadable, RulesetListed, RulesetUnreadable, slot_network_readback, @@ -45,6 +45,7 @@ import std.content_hash { Sha256Digest, Sha256DigestHex } import gunbc.runner_microvm_network { TapHolderObservation } import product.placement_supply { HostIdentity } import extdeps.exec.command { argv_words } +import gunbc.executor_privileged_operation { ExecutorPrivilegedOperation, InstallFile, SystemdReloadUnit } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -585,3 +586,29 @@ test fn a_new_flow_toward_a_tap_is_dropped_on_the_shared_hosts() -> Bool { let text = runner_microvm_shared_host_nft_ruleset_text() string_contains(s: text, pattern: " iifname \"gunbc-tap*\" drop\n oifname \"gunbc-tap*\" drop\n }\n chain output") } + +// A SLOT'S NETWORK FILES ARE THE .netdev, THE .network AND THE v6 SYSCTL, AND THE FOLD EMITS ALL +// THREE. The red is the previous fold, which emitted two while the converge frontier's prose named +// three -- the converge would have installed no disable_ipv6, and every slot would then refuse +// sanitation subject construction (review 68660). +test fn a_slots_install_set_carries_every_network_file_it_declares() -> Bool { + let net = runner_slot_network(slot: slot(i: 5)) + map( + runner_slot_network_install_operations( + net: net, + staged_netdev: "/stage/netdev" as NonEmptyStr, + staged_network: "/stage/network" as NonEmptyStr, + staged_ipv6_sysctl: "/stage/ipv6.conf" as NonEmptyStr, + ), + op => match op { + InstallFile { source: _, destination: d, owner: _, group: _, mode: _ } => d as String + SystemdReloadUnit { unit: u } => join(["reload ", u as String], "") + _ => "other" + }, + ) == [ + runner_slot_netdev_path(net: net), + runner_slot_network_file_path(net: net), + runner_slot_ipv6_sysctl_path(net: net), + "reload systemd-networkd", + ] +} From 11fbe4d948bc08522736bbc903e230d0055c84f8 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 04:13:33 +0000 Subject: [PATCH 17/36] microVM network converge: install the slot taps, ruleset and forwarding, then read the host back into a receipt Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/exec/command.dag | 3 + dag/extdeps/iproute2/ip_show.dag | 51 ++ dag/extdeps/linux/nf_conntrack.dag | 31 + dag/extdeps/systemd/systemctl.dag | 9 +- dag/extdeps/tools/uname.dag | 35 + dag/gunbc/ci/ci_spec.dag | 16 + dag/gunbc/executor_privileged_operation.dag | 40 +- dag/gunbc/fleet/fleet_converge_workflow.dag | 48 +- dag/gunbc/fleet/fleet_posix_accounts.dag | 14 + dag/gunbc/host_convergence_census.dag | 9 + dag/gunbc/runner/runner_host_grants.dag | 11 + dag/gunbc/runner/runner_microvm_network.dag | 119 +++- .../runner_microvm_network_converge.dag | 610 ++++++++++++++++++ ..._microvm_network_converge_witness_test.dag | 207 ++++++ .../runner_microvm_network_witness_test.dag | 6 +- 15 files changed, 1196 insertions(+), 13 deletions(-) create mode 100644 dag/extdeps/iproute2/ip_show.dag create mode 100644 dag/extdeps/linux/nf_conntrack.dag create mode 100644 dag/extdeps/tools/uname.dag create mode 100644 dag/gunbc/runner/runner_microvm_network_converge.dag create mode 100644 dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag diff --git a/dag/extdeps/exec/command.dag b/dag/extdeps/exec/command.dag index f46f07a4c51..21e558cdc44 100644 --- a/dag/extdeps/exec/command.dag +++ b/dag/extdeps/exec/command.dag @@ -98,6 +98,9 @@ fn argv_command(program: NonEmptyStr, arguments: List) -> ArgvCommand decl_ref(module_path: "extdeps.tools.conntrack", decl_name: "conntrack_list_expectations_command"), decl_ref(module_path: "extdeps.iproute2.ip_neigh", decl_name: "ip_neigh_flush_command"), decl_ref(module_path: "extdeps.iproute2.ip_neigh", decl_name: "ip_neigh_show_command"), + decl_ref(module_path: "extdeps.iproute2.ip_show", decl_name: "ip_addr_show_command"), + decl_ref(module_path: "extdeps.iproute2.ip_show", decl_name: "ip_route_show_command"), + decl_ref(module_path: "extdeps.tools.uname", decl_name: "uname_release_command"), decl_ref(module_path: "extdeps.tools.make", decl_name: "make_command"), decl_ref(module_path: "extdeps.package_managers.pip", decl_name: "pip_install_pinned_command"), decl_ref(module_path: "extdeps.package_managers.conda_forge", decl_name: "micromamba_create_command"), diff --git a/dag/extdeps/iproute2/ip_show.dag b/dag/extdeps/iproute2/ip_show.dag new file mode 100644 index 00000000000..92dd305cd08 --- /dev/null +++ b/dag/extdeps/iproute2/ip_show.dag @@ -0,0 +1,51 @@ +module extdeps.iproute2.ip_show + +import std.types { String, NonEmptyStr, List } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Uri, Https } +import extdeps.exec.command { ArgvCommand, argv_command } +import extdeps.iproute2.ip_neigh { ip_program } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "man7.org/linux/man-pages/man8/ip-address.8.html" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.iproute2.ip_show", + decl_name: "ip_show_oneline_flag", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +// THE BINARY IS CONSUMED FROM extdeps.iproute2.ip_neigh rather than re-declared: one `ip` on a host, +// one row naming it. +// +// -o PUTS EACH OBJECT ON ONE LINE, which is what makes a reading countable and greppable without a +// parser: without it `ip addr show` wraps one interface across several lines and a line count stops +// meaning anything. Reading is unprivileged, unlike the flush in the sibling module. +data ip_show_oneline_flag: NonEmptyStr = "-o" + +data ip_show_ipv4_family_flag: NonEmptyStr = "-4" + +fn ip_addr_show_command(device: String) -> ArgvCommand { + argv_command( + program: ip_program, + arguments: [ip_show_oneline_flag as String, ip_show_ipv4_family_flag as String, "addr", "show", "dev", device], + ) +} + +fn ip_route_show_command(device: String) -> ArgvCommand { + argv_command( + program: ip_program, + arguments: [ip_show_oneline_flag as String, ip_show_ipv4_family_flag as String, "route", "show", "dev", device], + ) +} diff --git a/dag/extdeps/linux/nf_conntrack.dag b/dag/extdeps/linux/nf_conntrack.dag new file mode 100644 index 00000000000..c57cef6786d --- /dev/null +++ b/dag/extdeps/linux/nf_conntrack.dag @@ -0,0 +1,31 @@ +module extdeps.linux.nf_conntrack + +import std.types { NonEmptyStr } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Uri, Https } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.linux.nf_conntrack", + decl_name: "nf_conntrack_helper_proc_path", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +// AUTOMATIC HELPER ASSIGNMENT, AS A SYSCTL THAT MAY NOT EXIST. nf_conntrack_helper attached a helper +// to a flow by port with no rule saying so. Upstream removed it, so on a current kernel the path is +// ABSENT -- which is a stronger reading than 0, because there is no setting left to turn back on. A +// reader must therefore distinguish absent from 0 from 1, and never treat absence as unreadable. +data nf_conntrack_helper_proc_path: NonEmptyStr = "/proc/sys/net/netfilter/nf_conntrack_helper" diff --git a/dag/extdeps/systemd/systemctl.dag b/dag/extdeps/systemd/systemctl.dag index ba0fd7f171c..ef188cb3a37 100644 --- a/dag/extdeps/systemd/systemctl.dag +++ b/dag/extdeps/systemd/systemctl.dag @@ -124,6 +124,13 @@ data systemctl_start_verb: NonEmptyStr = "start" as NonEmptyStr // verb rather than silently restarting, so this cannot degrade into that. data systemctl_reload_verb: NonEmptyStr = "reload" as NonEmptyStr +// RESTART IS THE ONLY VERB THAT RE-RUNS A ONESHOT UNIT'S ExecStart. `start` on a RemainAfterExit +// oneshot that is already active is a no-op, so a converge that installs new file contents and then +// starts the unit leaves the OLD contents in effect while reporting success -- the fabricated-success +// arm. A oneshot whose ExecStart is idempotent (nft -f replaces its own table) is safe to restart on +// every converge, and that is what makes the installed bytes and the live state the same fact. +data systemctl_restart_verb: NonEmptyStr = "restart" as NonEmptyStr + fn systemctl_enable_argv(unit: NonEmptyStr) -> List { ["systemctl", systemctl_enable_verb as String, unit as String] } @@ -138,7 +145,7 @@ fn systemctl_start_argv(unit: NonEmptyStr) -> List { } fn systemctl_restart_argv(unit: NonEmptyStr) -> List { - ["systemctl", "restart", unit as String] + ["systemctl", systemctl_restart_verb as String, unit as String] } fn systemctl_disable_now_argv(unit: NonEmptyStr) -> List { diff --git a/dag/extdeps/tools/uname.dag b/dag/extdeps/tools/uname.dag new file mode 100644 index 00000000000..69aeb5425c2 --- /dev/null +++ b/dag/extdeps/tools/uname.dag @@ -0,0 +1,35 @@ +module extdeps.tools.uname + +import std.types { String, NonEmptyStr, List } +import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } +import std.decl_ref { DeclarationRef, WholeDeclaration } +import extdeps.uri { Uri, Https } +import extdeps.exec.command { ArgvCommand, argv_command } + +data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { + uri: Uri { + scheme: Https + locator: "pubs.opengroup.org/onlinepubs/9699919799/utilities/uname.html" + } +} + +data extdeps_model_scope: ExternalModelScope = ExternalModelScope { + subject: ExternalSubjectRef { + declaration: DeclarationRef { + module_path: "extdeps.tools.uname", + decl_name: "uname_program", + field: WholeDeclaration + } + }, + first_citation: extdeps_external_authority_anchor, + further_citations: [] +} + +data uname_program: NonEmptyStr = "uname" + +// `-r` PRINTS THE RELEASE, which is the spelling extdeps.linux.kernel LinuxKernelRelease is branded +// on. It is the reading that says WHICH kernel a standing was established against, so a standing +// read on one kernel is not silently carried onto another. +fn uname_release_command() -> ArgvCommand { + argv_command(program: uname_program, arguments: ["-r"]) +} diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 748b01918a7..695c9e5ca17 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -810,6 +810,11 @@ data gunbc_ci_runner_microvm_host_converge_target: GunbcRunStepTarget = GunbcRun function: "runner_microvm_host_converge_wet", } +data gunbc_ci_runner_microvm_network_converge_target: GunbcRunStepTarget = GunbcRunStepTarget { + entry: "dag/gunbc/runner/runner_microvm_network_converge.dag", + function: "runner_microvm_network_converge_wet", +} + data gunbc_ci_runner_host_file_observe_target: GunbcRunStepTarget = GunbcRunStepTarget { entry: "dag/gunbc/runner/runner_host_file_converge.dag", function: "runner_host_file_observe_ci_wet", @@ -1013,6 +1018,7 @@ fn gunbc_run_step_targets() -> List { gunbc_ci_runner_guest_image_observe_target, gunbc_ci_runner_guest_image_converge_target, gunbc_ci_runner_microvm_host_converge_target, + gunbc_ci_runner_microvm_network_converge_target, gunbc_ci_runner_host_file_observe_target, gunbc_ci_runner_host_file_converge_target, gunbc_ci_runner_password_session_tool_converge_target, @@ -1926,6 +1932,16 @@ fn gunbc_ci_runner_microvm_host_converge_invoke() -> String { ) } +fn gunbc_ci_runner_microvm_network_converge_invoke() -> String { + gunbc_run_step_script( + source_roots: witness_layer_roots, + entry: gunbc_ci_runner_microvm_network_converge_target.entry, + function: gunbc_ci_runner_microvm_network_converge_target.function, + claim_run: false, + receipt_rel: "target/microvm-network-converge.txt" + ) +} + fn gunbc_ci_runner_host_file_observe_invoke() -> String { gunbc_run_step_script( source_roots: witness_layer_roots, diff --git a/dag/gunbc/executor_privileged_operation.dag b/dag/gunbc/executor_privileged_operation.dag index a9173c96c87..0d70c40bfa7 100644 --- a/dag/gunbc/executor_privileged_operation.dag +++ b/dag/gunbc/executor_privileged_operation.dag @@ -25,7 +25,7 @@ import extdeps.sudo.elevation { sudo_elevate_argv } import extdeps.posix.shell_command_language { posix_single_quote } import v2.std.algebra { fold_list } import extdeps.github.actions_runner { actions_runner_base_dir, actions_runner_cache_dir } -import extdeps.systemd.systemctl { systemctl_enable_verb, systemctl_start_verb, systemctl_set_property_verb, systemctl_reload_verb } +import extdeps.systemd.systemctl { systemctl_enable_verb, systemctl_start_verb, systemctl_set_property_verb, systemctl_reload_verb, systemctl_restart_verb } import gunbc.fleet_posix_accounts { ci_runner_sudo_binary_path, SudoInstall, @@ -33,8 +33,16 @@ import gunbc.fleet_posix_accounts { SudoUseradd, SudoUsermod, SudoSystemctl, + SudoNft, + SudoConntrack, } +// A GRANTED READ IS AN OPERATION TOO. ReadNftTable mutates nothing, and it is here because the +// sudoers roster is derived from this type: listing a table needs CAP_NET_ADMIN, so a readback that +// is not an operation is a command nobody authorized -- and a converge that cannot READ what it +// installed can only report that the install returned zero, which is the inference DESIGN section 5 +// forbids standing in for evidence. +// // THE OPERATION, PARAMETERIZED. Not a nullary enum of binaries: a binary path alone cannot say // which directory it may create or who ends up owning it, and granting whole /usr/bin/install is // granting root. Every field here narrows the authorization. @@ -60,6 +68,10 @@ type ExecutorPrivilegedOperation mode: NonEmptyStr } | SystemdReloadUnit { unit: NonEmptyStr } + | SystemdRestartUnit { unit: NonEmptyStr } + | ReadNftTable { family: NonEmptyStr, table: NonEmptyStr } + | ReadNftRuleset + | ReadConntrackExpectations // THE ONE PROJECTION. Everything else in this module derives from this function, so there is // exactly one place where an operation becomes words. @@ -111,6 +123,28 @@ fn executor_privileged_operation_argv(op: ExecutorPrivilegedOperation) -> List [ + ci_runner_sudo_binary_path(b: SudoSystemctl), + systemctl_restart_verb, + u + ] + ReadNftRuleset => [ + ci_runner_sudo_binary_path(b: SudoNft), + "list" as NonEmptyStr, + "ruleset" as NonEmptyStr + ] + ReadConntrackExpectations => [ + ci_runner_sudo_binary_path(b: SudoConntrack), + "-L" as NonEmptyStr, + "expect" as NonEmptyStr + ] + ReadNftTable { family: f, table: t } => [ + ci_runner_sudo_binary_path(b: SudoNft), + "list" as NonEmptyStr, + "table" as NonEmptyStr, + f, + t + ] SystemdDaemonReload => [ ci_runner_sudo_binary_path(b: SudoSystemctl), "daemon-reload" as NonEmptyStr @@ -306,6 +340,10 @@ fn executor_privileged_operation_precondition(op: ExecutorPrivilegedOperation) - SystemdSetUnitProperty { unit: _, assignment: _ } => NoPrecondition InstallFile { source: _, destination: _, owner: _, group: _, mode: _ } => NoPrecondition SystemdReloadUnit { unit: _ } => NoPrecondition + SystemdRestartUnit { unit: _ } => NoPrecondition + ReadNftTable { family: _, table: _ } => NoPrecondition + ReadNftRuleset => NoPrecondition + ReadConntrackExpectations => NoPrecondition } } diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index 327a37bd024..431f82ae77f 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -174,6 +174,7 @@ type FleetConvergeWorkflowMode | OrgActionsObserve | AppControlPlaneObserve | MicrovmHostConverge + | MicrovmNetworkConverge | GuestImageObserve | GuestImageConverge | MicrovmBootProbe @@ -200,6 +201,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String OrgActionsObserve => "org_actions_observe" AppControlPlaneObserve => "app_control_plane_observe" MicrovmHostConverge => "microvm_host_converge" + MicrovmNetworkConverge => "microvm_network_converge" GuestImageObserve => "guest_image_observe" GuestImageConverge => "guest_image_converge" MicrovmBootProbe => "microvm_boot_probe" @@ -240,7 +242,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // @@ -271,6 +273,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc OrgActionsObserve => none AppControlPlaneObserve => none MicrovmHostConverge => none + MicrovmNetworkConverge => none GuestImageObserve => none GuestImageConverge => none MicrovmBootProbe => none @@ -669,6 +672,8 @@ data fleet_converge_microvm_boot_probe_step_if: String = fleet_converge_mode_ste data fleet_converge_microvm_host_converge_step_if: String = fleet_converge_mode_step_if(mode: MicrovmHostConverge) +data fleet_converge_microvm_network_converge_step_if: String = fleet_converge_mode_step_if(mode: MicrovmNetworkConverge) + @@ -930,6 +935,45 @@ fn fleet_converge_microvm_host_converge_step() -> Step { } } +// THE NETWORK CONVERGE IS ITS OWN MODE, NOT A STEP OF THE HOST CONVERGE. The host converge installs +// the VMM; this installs the slots' taps, their v6 sysctls, the host ruleset and forwarding, and +// then READS the host back into a ConvergedSlotNetwork receipt. They are separate dispatches because +// they fail for different reasons and an operator routinely wants one without the other: a +// Firecracker release that will not verify says nothing about a tap that will not carry an address. +fn fleet_converge_microvm_network_converge_step() -> Step { + RunStep { + name: Present { value: "Runner micro-VM network: install the slot taps, ruleset and forwarding, then read the host back" }, + id: Present { value: "microvm_network_converge" }, + run: gunbc_ci_runner_microvm_network_converge_invoke(), + shell: none, + env: none, + working_directory: none, + if_condition: Present { value: fleet_converge_microvm_network_converge_step_if }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_runner_guest_image_step_timeout_minutes } + } +} + +// THE RECEIPT UPLOADS ON FAILURE TOO, because a refusal is the reading worth having: it names which +// fact the host did not establish. +fn fleet_converge_microvm_network_receipt_upload_step() -> Step { + UsesStep { + name: Present { value: "Upload runner micro-VM network converge receipt" }, + id: Present { value: "microvm_network_converge_receipt_upload" }, + uses: upload_artifact_action, + with: Present { value: [ + kv(key: "name", value: yaml_string(s: "runner-microvm-network-converge-receipt")), + kv(key: "path", value: yaml_string(s: "target/microvm-network-converge.txt")), + kv(key: "if-no-files-found", value: yaml_string(s: "error")), + kv(key: "retention-days", value: yaml_int(n: 30)), + ] }, + env: none, + if_condition: Present { value: concat("always() && (", concat(fleet_converge_microvm_network_converge_step_if, ")")) }, + continue_on_error: none, + timeout_minutes: Present { value: gunbc_ci_artifact_transfer_step_timeout_minutes } + } +} + fn fleet_converge_microvm_host_standing_upload_step() -> Step { UsesStep { name: Present { value: "Upload runner micro-VM host standing receipt" }, @@ -1592,6 +1636,8 @@ fn fleet_converge_job() -> Job { fleet_converge_guest_image_receipt_upload_step(), fleet_converge_microvm_host_converge_step(), fleet_converge_microvm_host_standing_upload_step(), + fleet_converge_microvm_network_converge_step(), + fleet_converge_microvm_network_receipt_upload_step(), fleet_converge_microvm_boot_probe_step(), fleet_converge_microvm_boot_probe_receipt_upload_step(), fleet_converge_spark_grants_step(), diff --git a/dag/gunbc/fleet/fleet_posix_accounts.dag b/dag/gunbc/fleet/fleet_posix_accounts.dag index 181cb5bce41..a7bd204a380 100644 --- a/dag/gunbc/fleet/fleet_posix_accounts.dag +++ b/dag/gunbc/fleet/fleet_posix_accounts.dag @@ -230,6 +230,14 @@ data fleet_tailscale_binary_path: NonEmptyStr = "/usr/bin/tailscale" data fleet_rm_binary_path: NonEmptyStr = "/usr/bin/rm" +// nft LIVES IN sbin AND IS PRIVILEGED EVEN TO READ: listing a table needs CAP_NET_ADMIN, so a +// readback of the installed ruleset is a granted command like any mutation. +data fleet_nft_binary_path: NonEmptyStr = "/usr/sbin/nft" + +// conntrack(8) READS AND WRITES THE KERNEL'S CONNECTION TRACKING TABLE, both of which need +// CAP_NET_ADMIN -- listing the expectation table is as privileged as flushing a flow. +data fleet_conntrack_binary_path: NonEmptyStr = "/usr/sbin/conntrack" + // A CLOSED VOCABULARY OF WHAT THIS PRINCIPAL MAY SUDO, so that emitting an ungranted sudo command // has no spelling rather than being a mistake something downstream has to notice. // @@ -260,6 +268,8 @@ type CiRunnerSudoBinary | SudoRm | SudoUseradd | SudoUsermod + | SudoNft + | SudoConntrack fn ci_runner_sudo_binary_path(b: CiRunnerSudoBinary) -> NonEmptyStr { match b { @@ -271,6 +281,8 @@ fn ci_runner_sudo_binary_path(b: CiRunnerSudoBinary) -> NonEmptyStr { SudoRm => fleet_rm_binary_path SudoUseradd => fleet_useradd_binary_path SudoUsermod => fleet_usermod_binary_path + SudoNft => fleet_nft_binary_path + SudoConntrack => fleet_conntrack_binary_path } } @@ -284,6 +296,8 @@ fn ci_runner_sudo_binary_probe(b: CiRunnerSudoBinary) -> SudoNopasswdProbeKind { SudoRm => NopasswdExecuteCheck { check_argv: "--version" } SudoUseradd => NopasswdExecuteCheck { check_argv: "-D" } SudoUsermod => NopasswdExecuteCheck { check_argv: "--help" } + SudoNft => NopasswdExecuteCheck { check_argv: "--version" } + SudoConntrack => NopasswdExecuteCheck { check_argv: "--version" } } } diff --git a/dag/gunbc/host_convergence_census.dag b/dag/gunbc/host_convergence_census.dag index d097d8e1d92..08742d4e60f 100644 --- a/dag/gunbc/host_convergence_census.dag +++ b/dag/gunbc/host_convergence_census.dag @@ -692,6 +692,15 @@ data host_convergence_census_rows: List = [ disposition: BindToAuthority { target: fleet_plan() }, first_consumer: cite(module_path: "gunbc.fleet_converge_workflow", decl_name: "fleet_converge_workflow_modes"), ), + census_row( + identity: cite(module_path: "gunbc.runner_microvm_network_converge", decl_name: "runner_microvm_network_converge_wet"), + subject: MicrovmHost, + phases: protocol_phases(false, true, false, false, false, false, true), + current_authority: cite(module_path: "gunbc.runner_microvm_network_converge", decl_name: "runner_microvm_network_converge_wet"), + persistent_host_effect: ReachesPersistentHostMutation, + disposition: BindToAuthority { target: fleet_plan() }, + first_consumer: cite(module_path: "gunbc.fleet_converge_workflow", decl_name: "fleet_converge_workflow_modes"), + ), census_row( identity: cite(module_path: "gunbc.runner_guest_image", decl_name: "runner_guest_image_converge_wet"), subject: GuestImage, diff --git a/dag/gunbc/runner/runner_host_grants.dag b/dag/gunbc/runner/runner_host_grants.dag index 3ecf9f18489..dd699d3f244 100644 --- a/dag/gunbc/runner/runner_host_grants.dag +++ b/dag/gunbc/runner/runner_host_grants.dag @@ -48,6 +48,7 @@ import gunbc.runner_host_deploy { RunnerHostDeploy, RunnerHostSpec, runner_host_ import product.placement_supply { HostIdentity } import gunbc.runner_slot_provision { desired_runner_slot_members, runner_slot_instance_dir } import gunbc.runner_host_deploy { runner_index_seq } +import gunbc.runner_microvm_network_converge { microvm_network_granted_operations } import gunbc.runner_unit { runner_unit_name_of_registration } // THE MANAGED STATE DIRECTORY, DECLARED ONCE. It is currently spelled THREE times in @@ -314,6 +315,15 @@ fn runner_host_wide_refused_operations(spec: RunnerHostSpec) -> List List { + let host = deploy.host_label as HostIdentity + map(runner_index_seq(n: deploy.runner_count), i => RunnerSlotIdentity { host: host, slot_index: i }) +} + fn runner_host_privileged_operations(deploy: RunnerHostDeploy) -> List { let job = deploy.job_user let slot_dirs = map(desired_runner_slot_members(deploy: deploy), m => m.slot_dir) @@ -323,6 +333,7 @@ fn runner_host_privileged_operations(deploy: RunnerHostDeploy) -> List String { + join([runner_microvm_systemd_unit_dir as String, "/", runner_microvm_nft_unit as String], "") +} + +// After= networkd, because the ruleset's rules name tap devices networkd creates; nft accepts rules +// for a device that does not exist yet, but ordering it after removes the window where a tap is up +// and unfiltered. +fn runner_microvm_nft_unit_text() -> String { + join([ + "[Unit]\n", + "Description=gunbc microVM host ruleset\n", + "After=systemd-networkd.service\n", + "Wants=systemd-networkd.service\n", + "\n", + "[Service]\n", + "Type=oneshot\n", + "RemainAfterExit=yes\n", + "ExecStart=", fleet_nft_binary_path as String, " -f ", runner_microvm_nft_ruleset_path as String, "\n", + "\n", + "[Install]\n", + "WantedBy=multi-user.target\n", + ], "") +} + +// FORWARDING IS WHAT MAKES THE TAP A ROUTED LINK RATHER THAN A DEAD END. Without it the guest's +// packets reach the host and stop, and the masquerade never runs. +data runner_microvm_forward_sysctl_path: NonEmptyStr = "/etc/sysctl.d/50-gunbc-microvm-forward.conf" as NonEmptyStr + +data runner_microvm_ip_forward_proc_path: NonEmptyStr = "/proc/sys/net/ipv4/ip_forward" as NonEmptyStr + +fn runner_microvm_forward_sysctl_text() -> String { + "net.ipv4.ip_forward = 1\n" +} + +data runner_microvm_sysctl_unit: NonEmptyStr = "systemd-sysctl.service" as NonEmptyStr + +// THE HOST'S OPERATIONS, IN THE ORDER THEY DEPEND ON EACH OTHER: the files first, then the daemon +// reload that makes systemd see a new unit, then the sysctl re-read, then enable (so it survives a +// reboot) and restart (so the bytes just installed are the live ones). +fn runner_microvm_host_network_install_operations( + staged_ruleset: NonEmptyStr, + staged_unit: NonEmptyStr, + staged_forward_sysctl: NonEmptyStr, +) -> List { + [ + InstallFile { + source: staged_ruleset, + destination: runner_microvm_nft_ruleset_path, + owner: runner_microvm_networkd_file_owner, + group: runner_microvm_networkd_file_owner, + mode: runner_microvm_networkd_file_mode, + }, + InstallFile { + source: staged_unit, + destination: runner_microvm_nft_unit_path() as NonEmptyStr, + owner: runner_microvm_networkd_file_owner, + group: runner_microvm_networkd_file_owner, + mode: runner_microvm_networkd_file_mode, + }, + InstallFile { + source: staged_forward_sysctl, + destination: runner_microvm_forward_sysctl_path, + owner: runner_microvm_networkd_file_owner, + group: runner_microvm_networkd_file_owner, + mode: runner_microvm_networkd_file_mode, + }, + SystemdDaemonReload, + SystemdRestartUnit { unit: runner_microvm_sysctl_unit }, + SystemdEnableUnit { unit: runner_microvm_nft_unit }, + SystemdRestartUnit { unit: runner_microvm_nft_unit }, + ] +} + // --- the slot network readback: what sanitation reads after an attempt --- // THE TAP AND THE RULESET OUTLIVE THE ATTEMPT BY DESIGN, SO SANITATION CANNOT PROVE THEM ABSENT. @@ -838,9 +930,14 @@ fn helpers_excluded(h: HelperStanding) -> Bool { && h.expectations_at_generation == 0 } +// THE GENERATION IS AN IDENTITY, NOT A COUNT. What a consumer needs is "which converge established +// this", and the converge run already has a name; a number would invite arithmetic on something that +// only ever supports equality, and would need a counter with state of its own. +type ConvergeGeneration = NonEmptyStr where brand("ConvergeGeneration") + type ConvergedSlotNetwork { host: HostIdentity - generation: Int + generation: ConvergeGeneration slots: List helpers: HelperStanding nft_digest: Sha256Digest @@ -972,13 +1069,13 @@ type SlotResidueReading { type HostResidueReading { host: HostIdentity - generation: Int + generation: ConvergeGeneration observation: ResidueObservation } type RulesetReading { host: HostIdentity - generation: Int + generation: ConvergeGeneration observation: RulesetObservation } @@ -1068,8 +1165,9 @@ fn slot_residue_readback(s: SlotNetworkSanitationSubject, fact: SlotNetworkFact, } } -fn at_receipt(s: SlotNetworkSanitationSubject, host: HostIdentity, generation: Int) -> Bool { - (s.converged.host as String) == (host as String) && s.converged.generation == generation +fn at_receipt(s: SlotNetworkSanitationSubject, host: HostIdentity, generation: ConvergeGeneration) -> Bool { + (s.converged.host as String) == (host as String) + && (s.converged.generation as String) == (generation as String) } fn host_residue_readback(s: SlotNetworkSanitationSubject, fact: SlotNetworkFact, r: HostResidueReading) -> SlotNetworkReadback { @@ -1202,6 +1300,13 @@ fn runner_microvm_nft_list_command() -> ArgvCommand { nft_list_table_command(family: NftInet, name: runner_microvm_nft_table_name) } +// THE SAME LISTING, AS A GRANTED OPERATION. Listing a table needs CAP_NET_ADMIN, so the converge and +// the lifecycle controller reach it through the operation type that also renders the sudoers line; +// the bare ArgvCommand above is the unelevated spelling for a caller that already has the capability. +fn runner_microvm_nft_read_operation() -> ExecutorPrivilegedOperation { + ReadNftTable { family: nft_family_wire(f: NftInet) as NonEmptyStr, table: runner_microvm_nft_table_name } +} + // DECLARED FRONTIER (DESIGN 3c). The readback and the three commands that feed it have no // executing consumer in this tree yet. NAMED CONSUMER: the host teardown readback of the microVM // slot lifecycle controller (gunbc#11670), whose network fact maps onto product.fabric.sanitation. diff --git a/dag/gunbc/runner/runner_microvm_network_converge.dag b/dag/gunbc/runner/runner_microvm_network_converge.dag new file mode 100644 index 00000000000..c25d1c23de0 --- /dev/null +++ b/dag/gunbc/runner/runner_microvm_network_converge.dag @@ -0,0 +1,610 @@ +module gunbc.runner_microvm_network_converge + +import std.types { String, NonEmptyStr, Bool, Int, List } +import std.nat { Nat } +import std.disposition { Disposition, SingleAuthority } +import std.algebra { trim } +import std.content_hash { Sha256Digest, Sha256DigestHex, sha256_hex_digest } +import extdeps.exec.command { ArgvCommand } +import extdeps.shell.exec { ProcessOutcome, ProcessOutputPresent, ProcessOutputAbsent, ProcessRefused } +import extdeps.filesystem.filesystem_io { + Filesystem, FilesystemReadOutcome, FilesystemReadSucceeded, FilesystemReadRefused, + filesystem_read_outcome, +} +import extdeps.tools.sha256sum { Sha256FileDigest, sha256sum_stdin_digest_via_shell } +import extdeps.crypto.hash { Digest } +import extdeps.iproute2.ip_show { ip_addr_show_command, ip_route_show_command } +import extdeps.linux.nf_conntrack { nf_conntrack_helper_proc_path } +import extdeps.tools.uname { uname_release_command } +import gunbc.command_runner { run_shell_command_capture, LocalExec } +import gunbc.executor_privileged_apply { executor_privileged_operation_command } +import extdeps.network.ipv4 { render_ipv4_address } +import gunbc.build_cache_instance { RunnerSlotIdentity } +import product.placement_supply { HostIdentity } +import gunbc.executor_privileged_operation { ExecutorPrivilegedOperation, ReadNftRuleset, ReadConntrackExpectations } +import gunbc.runner_microvm_network { + RunnerSlotNetwork, runner_slot_network, runner_slot_host_cidr, + runner_slot_netdev_text, runner_slot_network_file_text, runner_slot_ipv6_sysctl_text, + runner_slot_network_install_operations, runner_microvm_host_network_install_operations, + runner_microvm_nft_ruleset_text_for, runner_microvm_nft_unit_text, runner_microvm_forward_sysctl_text, + runner_slot_ipv6_disabled_proc_path, runner_microvm_ip_forward_proc_path, + HostEgressGrain, runner_microvm_shared_host_grain, + ConvergedSlot, ConvergedSlotNetwork, ConvergeGeneration, HelperStanding, HelperAutoAssignment, + AutoAssignmentAbsentFromKernel, AutoAssignmentSysctlOff, AutoAssignmentOn, AutoAssignmentUnread, + TapIpv6Standing, TapIpv6Disabled, TapIpv6Enabled, TapIpv6Unread, + slot_network_subject, runner_microvm_nft_read_operation, +} + +data runner_microvm_network_converge_disposition: Disposition = SingleAuthority + +// THE CONVERGE INSTALLS WHAT THE MODEL DECLARES AND THEN READS THE HOST, NOT ITS OWN INTENT. +// +// Two halves, and the second is the one that earns the receipt. The apply half is ordinary: render +// each file's bytes from gunbc.runner_microvm_network, stage them, and hand the modeled operations +// to the privileged applier. The READ half establishes what the host now IS -- the tap's live +// disable_ipv6 value, the addresses and routes the kernel holds, the helper standing, and the digest +// of the ruleset as the kernel lists it. A converge that reported success because its installs +// exited zero would be asserting its intent as an observation (sunny-ant-606 ruling, 2026-09-20), +// and every consumer of ConvergedSlotNetwork -- the sanitation subject above all -- would inherit +// that inference as if it were a reading. +// +// EVERY FACT THE RECEIPT CARRIES IS READ, OR THE RECEIPT IS NOT PRODUCED. There is no arm that +// substitutes a default for a reading that failed: an unread fact is a typed refusal naming it, +// because a receipt with a fabricated field is worse than no receipt -- the sanitation subject would +// then admit slots on a host nobody observed. + +// --- staging: fixed paths, because a grant names the source it may install from --- + +// THE STAGED PATH IS PART OF THE GRANT. runner_host_grants renders one sudoers line per operation +// with every argument spelled in full, source included, so a staged file at a random temporary path +// would be a command no line authorizes. One deterministic path per file, derived from the same +// names the destinations are derived from. +data runner_microvm_staging_dir: NonEmptyStr = "/var/lib/gunbc/fleet-converge/microvm-network" as NonEmptyStr + +fn staged_path(name: String) -> NonEmptyStr { + join([runner_microvm_staging_dir as String, "/", name], "") as NonEmptyStr +} + +fn staged_slot_netdev(net: RunnerSlotNetwork) -> NonEmptyStr { + staged_path(name: join([net.tap_device as String, ".netdev"], "")) +} + +fn staged_slot_network(net: RunnerSlotNetwork) -> NonEmptyStr { + staged_path(name: join([net.tap_device as String, ".network"], "")) +} + +fn staged_slot_ipv6_sysctl(net: RunnerSlotNetwork) -> NonEmptyStr { + staged_path(name: join([net.tap_device as String, "-ipv6.conf"], "")) +} + +fn staged_host_ruleset() -> NonEmptyStr { + staged_path(name: "microvm.nft") +} + +fn staged_host_unit() -> NonEmptyStr { + staged_path(name: "gunbc-microvm-nft.service") +} + +fn staged_host_forward_sysctl() -> NonEmptyStr { + staged_path(name: "forward.conf") +} + +// --- what is staged: one row per file, so writing and installing cannot disagree --- + +type StagedFile { + path: NonEmptyStr + content: String +} + +fn slot_staged_files(net: RunnerSlotNetwork, tap_owner: NonEmptyStr) -> List { + [ + StagedFile { path: staged_slot_netdev(net: net), content: runner_slot_netdev_text(net: net, owner: tap_owner) }, + StagedFile { path: staged_slot_network(net: net), content: runner_slot_network_file_text(net: net) }, + StagedFile { path: staged_slot_ipv6_sysctl(net: net), content: runner_slot_ipv6_sysctl_text(net: net) }, + ] +} + +fn host_staged_files(host: HostEgressGrain) -> List { + [ + StagedFile { path: staged_host_ruleset(), content: runner_microvm_nft_ruleset_text_for(host: host) }, + StagedFile { path: staged_host_unit(), content: runner_microvm_nft_unit_text() }, + StagedFile { path: staged_host_forward_sysctl(), content: runner_microvm_forward_sysctl_text() }, + ] +} + +fn converge_staged_files(slots: List, host: HostEgressGrain, tap_owner: NonEmptyStr) -> List { + concat( + flat_map(slots, s => slot_staged_files(net: runner_slot_network(slot: s), tap_owner: tap_owner)), + host_staged_files(host: host), + ) +} + +// --- the operations, in dependency order --- + +fn slot_operations(slot: RunnerSlotIdentity) -> List { + let net = runner_slot_network(slot: slot) + runner_slot_network_install_operations( + net: net, + staged_netdev: staged_slot_netdev(net: net), + staged_network: staged_slot_network(net: net), + staged_ipv6_sysctl: staged_slot_ipv6_sysctl(net: net), + ) +} + +// THE SLOTS FIRST AND THE HOST SECOND, because the ruleset's rules name tap devices and the unit is +// ordered after networkd: installing the ruleset before the taps exist would load rules for links +// that arrive later, which nft accepts and which leaves a window where a tap is up and unfiltered. +fn microvm_network_converge_operations(slots: List) -> List { + concat( + flat_map(slots, s => slot_operations(slot: s)), + runner_microvm_host_network_install_operations( + staged_ruleset: staged_host_ruleset(), + staged_unit: staged_host_unit(), + staged_forward_sysctl: staged_host_forward_sysctl(), + ), + ) +} + +// THE READS ARE GRANTED OPERATIONS TOO, so they appear in the host's sudoers roster beside the +// installs. A converge that may install a ruleset and may not list it back could only report its own +// intent, which is the inference this module exists to refuse. +fn microvm_network_readback_operations() -> List { + [ + runner_microvm_nft_read_operation(), + ReadNftRuleset, + ReadConntrackExpectations, + ] +} + +fn microvm_network_granted_operations(slots: List) -> List { + concat(microvm_network_converge_operations(slots: slots), microvm_network_readback_operations()) +} + +// --- reading the host back --- + +// A READING IS A STRING FROM THE HOST AND A JUDGEMENT ABOUT IT, AND THE TWO ARE SEPARATE FUNCTIONS. +// The parse is pure and witnessed; the capture is the wet edge. That is what lets a witness pin what +// "1\n" from a proc file means without a host, and what stops the wet function from deciding. +fn parse_disable_ipv6(readable: Bool, text: String) -> TapIpv6Standing { + if !readable { + TapIpv6Unread { reason: "disable_ipv6 could not be read for this tap" as NonEmptyStr } + } else if trim(s: text) == "1" { + TapIpv6Disabled + } else if trim(s: text) == "0" { + TapIpv6Enabled + } else { + TapIpv6Unread { reason: join(["disable_ipv6 read back as an unexpected value: ", trim(s: text)], "") as NonEmptyStr } + } +} + +fn ip_forward_is_on(readable: Bool, text: String) -> Bool { + readable && trim(s: text) == "1" +} + +// THE HOST'S OWN ADDRESS ON THE TAP, AND THE ROUTE THROUGH IT, ARE READ FROM THE KERNEL rather than +// assumed from the .network file having been installed. `ip -o -4 addr show dev ` prints the +// configured address; a tap whose networkd file landed but whose link never came up prints nothing. +fn addr_reading_carries(stdout: String, cidr: String) -> Bool { + string_contains(s: stdout, pattern: cidr) +} + +fn route_reading_carries(stdout: String, cidr: String) -> Bool { + string_contains(s: stdout, pattern: cidr) +} + +// nf_conntrack_helper WAS REMOVED FROM THE KERNEL, AND ITS ABSENCE IS THE STRONGEST READING. Linux +// dropped automatic helper assignment entirely, so on a kernel that has no such sysctl there is no +// setting to get wrong; on an older one the value must read 0. A file that exists and reads 1 is +// AutoAssignmentOn, and an unreadable probe is neither -- it is unread, and refuses the receipt. +fn parse_auto_assignment(present: Bool, readable: Bool, text: String, kernel_release: String) -> HelperAutoAssignment { + if !present { + AutoAssignmentAbsentFromKernel { kernel_release: kernel_release as NonEmptyStr } + } else if !readable { + AutoAssignmentUnread { reason: "nf_conntrack_helper exists but could not be read" as NonEmptyStr } + } else if trim(s: text) == "0" { + AutoAssignmentSysctlOff + } else if trim(s: text) == "1" { + AutoAssignmentOn + } else { + AutoAssignmentUnread { reason: join(["nf_conntrack_helper read back as an unexpected value: ", trim(s: text)], "") as NonEmptyStr } + } +} + +// EVERY LINE OF THE EFFECTIVE RULESET, NOT OURS. `ct helper` may be attached by any table on the +// host, so the count is taken over the whole listing; ours contains no such statement, which is why +// a nonzero count is someone else's helper and not a miscount of our own rows. +fn ct_helper_statement_count(ruleset_listing: String) -> Int { + length(split(s: ruleset_listing, delimiter: "ct helper")) - 1 +} + +type ConvergeRefusal + = SlotTapIpv6Unread { slot: RunnerSlotIdentity, standing: TapIpv6Standing } + | SlotTapIpv6Enabled { slot: RunnerSlotIdentity } + | SlotAddressNotHeld { slot: RunnerSlotIdentity, expected: NonEmptyStr } + | SlotRouteNotHeld { slot: RunnerSlotIdentity, expected: NonEmptyStr } + | IpForwardingOff + | RulesetUnread { reason: NonEmptyStr } + | HelperStandingUnread { reason: NonEmptyStr } + | HelperAutoAssignmentOn + | HelpersAttached { xtables_targets: Int, userspace: Int, expectations: Int } + +type ConvergeOutcome + = ConvergedNetwork { receipt: ConvergedSlotNetwork } + | ConvergeRefused { refusal: ConvergeRefusal } + +// ONE SLOT'S READINGS, AS THEY CAME OFF THE HOST. +type SlotReading { + slot: RunnerSlotIdentity + tap_ipv6: TapIpv6Standing + addr_stdout: String + route_stdout: String +} + +type HostReading { + host: HostIdentity + generation: ConvergeGeneration + ip_forward_on: Bool + ruleset_listed: Bool + ruleset_digest: Sha256Digest + auto_assignment: HelperAutoAssignment + xtables_ct_helper_targets: Int + userspace_helpers: Int + expectations: Int + helper_probes_read: Bool +} + +fn converged_slot_of_reading(r: SlotReading) -> ConvergedSlot { + let net = runner_slot_network(slot: r.slot) + ConvergedSlot { + subject: slot_network_subject(slot: r.slot), + host_address_cidr: runner_slot_host_cidr(net: net) as NonEmptyStr, + guest_route: trim(s: r.route_stdout) as NonEmptyStr, + tap_ipv6: r.tap_ipv6, + } +} + +// THE FIRST UNREAD OR FAILING FACT REFUSES, AND NOTHING DOWNSTREAM OF IT IS JUDGED. A slot whose tap +// has no address cannot have its route read meaningfully, and a host whose helper probes did not run +// has no helper standing to record. +fn slot_refusal(r: SlotReading) -> List { + let net = runner_slot_network(slot: r.slot) + let cidr = runner_slot_host_cidr(net: net) + match r.tap_ipv6 { + TapIpv6Unread { reason: _ } => [SlotTapIpv6Unread { slot: r.slot, standing: r.tap_ipv6 }] + TapIpv6Enabled => [SlotTapIpv6Enabled { slot: r.slot }] + TapIpv6Disabled => + if !addr_reading_carries(stdout: r.addr_stdout, cidr: cidr) { + [SlotAddressNotHeld { slot: r.slot, expected: cidr as NonEmptyStr }] + } else if !route_reading_carries(stdout: r.route_stdout, cidr: render_slot_subnet(net: net)) { + [SlotRouteNotHeld { slot: r.slot, expected: render_slot_subnet(net: net) as NonEmptyStr }] + } else { + [] + } + } +} + +// THE ROUTE IS OVER THE SUBNET, NOT THE HOST ADDRESS. `ip route show dev tap` prints the connected +// route as the network -- 172.30.N.0/30 -- so matching the host's own /30 address would never hit. +fn render_slot_subnet(net: RunnerSlotNetwork) -> String { + join([ + render_ipv4_address(addr: net.host_address) as String, + "/", + to_string(net.prefix_length), + ], "") +} + +fn first_refusal(rs: List) -> ConvergeRefusal? { + fold(rs, init: none, f: (acc, r) => + match acc { + Present { value: _ } => acc + Absent => Present { value: r } + } + ) +} + +fn host_refusals(h: HostReading) -> List { + concat( + concat( + if h.ip_forward_on { [] } else { [IpForwardingOff] }, + if h.ruleset_listed { [] } else { [RulesetUnread { reason: "nft list table did not return the installed table" as NonEmptyStr }] }, + ), + match h.auto_assignment { + AutoAssignmentUnread { reason: why } => [HelperStandingUnread { reason: why }] + AutoAssignmentOn => [HelperAutoAssignmentOn] + AutoAssignmentAbsentFromKernel { kernel_release: _ } => helper_count_refusals(h: h) + AutoAssignmentSysctlOff => helper_count_refusals(h: h) + }, + ) +} + +fn helper_count_refusals(h: HostReading) -> List { + if !h.helper_probes_read { + [HelperStandingUnread { reason: "a helper probe did not run, so no helper standing was established" as NonEmptyStr }] + } else if h.xtables_ct_helper_targets != 0 || h.userspace_helpers != 0 || h.expectations != 0 { + [HelpersAttached { + xtables_targets: h.xtables_ct_helper_targets, + userspace: h.userspace_helpers, + expectations: h.expectations, + }] + } else { + [] + } +} + +fn helper_standing_of(h: HostReading) -> HelperStanding { + HelperStanding { + auto_assignment: h.auto_assignment, + nft_ct_helper_statements: 0, + xtables_ct_helper_targets: h.xtables_ct_helper_targets, + userspace_helpers: h.userspace_helpers, + expectations_at_generation: h.expectations, + } +} + +// THE RECEIPT IS DERIVED FROM THE READINGS AND FROM NOTHING ELSE. Its slots are the ones that were +// read; its digest is the listing's; its helper standing is the probes'. The only path to a receipt +// runs through every refusal above. +fn converge_outcome(slots: List, h: HostReading) -> ConvergeOutcome { + let refusals = concat(flat_map(slots, r => slot_refusal(r: r)), host_refusals(h: h)) + match first_refusal(rs: refusals) { + Present { value: r } => ConvergeRefused { refusal: r } + Absent => ConvergedNetwork { + receipt: ConvergedSlotNetwork { + host: h.host, + generation: h.generation, + slots: map(slots, r => converged_slot_of_reading(r: r)), + helpers: helper_standing_of(h: h), + nft_digest: h.ruleset_digest, + }, + }, + } +} + +// --- the wet edge: stage, apply, then read the host --- + +// THE ORDER IS STAGE, APPLY, READ, AND THE READ IS NOT OPTIONAL. Nothing between the apply and the +// read may conclude anything: the apply's exit says the operations ran, and only the readings say +// what the host now is. +fn read_proc_value(path: String) -> FilesystemReadOutcome { + let r = Filesystem.Read(path: path) + filesystem_read_outcome(content: r.content, success: r.success, error: r.error) +} + +fn proc_text(o: FilesystemReadOutcome) -> String { + match o { + FilesystemReadSucceeded { content: c } => c + FilesystemReadRefused { error: _ } => "" + } +} + +fn proc_readable(o: FilesystemReadOutcome) -> Bool { + match o { + FilesystemReadSucceeded { content: _ } => true + FilesystemReadRefused { error: _ } => false + } +} + +fn capture_stdout(c: ArgvCommand) -> ProcessOutcome { + run_shell_command_capture(command: c, transport: LocalExec) +} + +// A REFUSED COMMAND HAS NO READING, AND AN EMPTY ONE IS NOT THE SAME AS A MISSING ONE. +// ProcessOutputAbsent is a run that printed nothing -- an empty conntrack expectation table is +// exactly that, and it is a real reading -- while ProcessRefused is a probe that did not answer. +fn outcome_stdout(o: ProcessOutcome) -> String { + match o { + ProcessOutputPresent { text: t } => t + ProcessOutputAbsent => "" + ProcessRefused { exit_code: _, stderr: _ } => "" + } +} + +fn outcome_ran(o: ProcessOutcome) -> Bool { + match o { + ProcessOutputPresent { text: _ } => true + ProcessOutputAbsent => true + ProcessRefused { exit_code: _, stderr: _ } => false + } +} + +// THE DIGEST CROSSES ONE TYPE BOUNDARY AND REFUSES RATHER THAN INVENTING A HEX. sha256sum answers an +// extdeps.crypto Digest; the receipt carries std.content_hash Sha256Digest, whose constructor +// validates the hex. A digest that will not convert is an unlisted ruleset, not a zero digest. +fn listing_digest(d: Sha256FileDigest) -> Sha256Digest? { + match d { + Sha256FileDigestUnavailable { path: _, reason: _ } => none + Sha256FileDigest { digest: dg } => sha256_hex_digest(hex: dg.hex as String) + } +} + +fn read_slot_wet(slot: RunnerSlotIdentity) -> SlotReading { + let net = runner_slot_network(slot: slot) + let v6 = read_proc_value(path: runner_slot_ipv6_disabled_proc_path(net: net)) + SlotReading { + slot: slot, + tap_ipv6: parse_disable_ipv6(readable: proc_readable(o: v6), text: proc_text(o: v6)), + addr_stdout: outcome_stdout(o: capture_stdout(c: ip_addr_show_command(device: net.tap_device as String))), + route_stdout: outcome_stdout(o: capture_stdout(c: ip_route_show_command(device: net.tap_device as String))), + } +} + +// THE LISTING IS DIGESTED BY THE HOST, not by this process reading its own rendering: the receipt's +// digest must be of the bytes `nft list table` produced, because that is what the per-attempt +// readback will compare against. +fn host_ruleset_digest_wet(listing: String) -> Sha256FileDigest { + sha256sum_stdin_digest_via_shell(content: listing) +} + +fn read_host_wet(host: HostIdentity, generation: ConvergeGeneration) -> HostReading { + let forward = read_proc_value(path: runner_microvm_ip_forward_proc_path as String) + let table = capture_stdout(c: executor_privileged_operation_command(op: runner_microvm_nft_read_operation())) + let ruleset = capture_stdout(c: executor_privileged_operation_command(op: ReadNftRuleset)) + let expectations = capture_stdout(c: executor_privileged_operation_command(op: ReadConntrackExpectations)) + let helper_sysctl = read_proc_value(path: nf_conntrack_helper_proc_path as String) + let kernel = capture_stdout(c: uname_release_command()) + let digest = listing_digest(d: host_ruleset_digest_wet(listing: outcome_stdout(o: table))) + HostReading { + host: host, + generation: generation, + ip_forward_on: ip_forward_is_on(readable: proc_readable(o: forward), text: proc_text(o: forward)), + ruleset_listed: outcome_ran(o: table) && digest_present(d: digest), + ruleset_digest: digest_or_zero(d: digest), + auto_assignment: parse_auto_assignment( + present: proc_readable(o: helper_sysctl), + readable: proc_readable(o: helper_sysctl), + text: proc_text(o: helper_sysctl), + kernel_release: trim(s: outcome_stdout(o: kernel)), + ), + xtables_ct_helper_targets: ct_helper_statement_count(ruleset_listing: outcome_stdout(o: ruleset)), + userspace_helpers: 0, + expectations: expectation_line_count(stdout: outcome_stdout(o: expectations)), + helper_probes_read: outcome_ran(o: ruleset) && outcome_ran(o: expectations) && outcome_ran(o: kernel), + } +} + +// AN EXPECTATION PER LINE, AND AN EMPTY TABLE PRINTS NOTHING. conntrack writes its summary to stderr, +// so stdout carries only entries. +// THE ZERO DIGEST IS NEVER READ, BECAUSE ruleset_listed IS FALSE WHENEVER IT IS USED: host_refusals +// refuses on an unlisted ruleset before converge_outcome reaches the receipt, so this placeholder +// cannot travel into a ConvergedSlotNetwork. It exists because the field is not optional -- a reading +// that failed is refused, not carried as an absent digest a consumer might unwrap. +data unread_ruleset_digest: Sha256Digest = Sha256Digest { + hex: "0000000000000000000000000000000000000000000000000000000000000000" as Sha256DigestHex, +} + +fn digest_present(d: Sha256Digest?) -> Bool { + match d { + Present { value: _ } => true + Absent => false + } +} + +fn digest_or_zero(d: Sha256Digest?) -> Sha256Digest { + match d { + Present { value: v } => v + Absent => unread_ruleset_digest + } +} + +fn expectation_line_count(stdout: String) -> Int { + length(filter(xs: split(s: stdout, delimiter: "\n"), predicate: fn(l) { trim(s: l) != "" })) +} + +// --- the converge entry --- + +// THE GENERATION IS THE RUN THAT PRODUCED THE RECEIPT, read from the environment rather than +// counted here: a counter would need state of its own, and the fact a consumer needs is "which +// converge established this", which the run already identifies. A converge with no run identity +// refuses -- a receipt whose generation nobody can name cannot be compared against a later reading, +// which is the whole purpose of the field. +fn converge_generation_reading() -> String { + let env = shell.Env.Get(name: github_run_id_variable_name as NonEmptyStr) + match env.value { + Present { value: raw } => trim(s: raw) + Absent => "" + } +} + +data runner_microvm_network_receipt_path: String = "target/microvm-network-converge.txt" + +data runner_microvm_network_receipt_dir: String = "target" + +fn refusal_text(r: ConvergeRefusal) -> String { + match r { + SlotTapIpv6Unread { slot: s, standing: _ } => join(["slot ", runner_slot_instance_name(slot: s), ": tap disable_ipv6 could not be read"], "") + SlotTapIpv6Enabled { slot: s } => join(["slot ", runner_slot_instance_name(slot: s), ": tap disable_ipv6 reads 0"], "") + SlotAddressNotHeld { slot: s, expected: e } => join(["slot ", runner_slot_instance_name(slot: s), ": the tap does not hold ", e as String], "") + SlotRouteNotHeld { slot: s, expected: e } => join(["slot ", runner_slot_instance_name(slot: s), ": no route over ", e as String], "") + IpForwardingOff => "net.ipv4.ip_forward reads 0" + RulesetUnread { reason: why } => join(["the installed ruleset could not be listed: ", why as String], "") + HelperStandingUnread { reason: why } => join(["helper standing not established: ", why as String], "") + HelperAutoAssignmentOn => "automatic conntrack helper assignment is ON" + HelpersAttached { xtables_targets: x, userspace: u, expectations: e } => + join(["helpers are attached: xtables ", to_string(x), ", userspace ", to_string(u), ", expectations ", to_string(e)], "") + } +} + +fn receipt_text(o: ConvergeOutcome) -> String { + match o { + ConvergeRefused { refusal: r } => join(["microvm-network-converge REFUSED: ", refusal_text(r: r), "\n"], "") + ConvergedNetwork { receipt: c } => + join([ + "microvm-network-converge CONVERGED\n", + "host=", c.host as String, "\n", + "generation=", c.generation as String, "\n", + "slots=", to_string(length(c.slots)), "\n", + "nft_digest=", c.nft_digest.hex as String, "\n", + ], "") + } +} + +// STAGE, APPLY, READ, RECORD -- and refuse at the first step that does not hold. The staging write +// is the executor's own file under its own directory, so it needs no grant; everything that touches +// the host outside that directory is a modeled operation. +fn stage_file_wet(f: StagedFile) -> Bool { + let w = Filesystem.Write(path: f.path as String, content: f.content) + w.success +} + +fn runner_microvm_network_converge_wet() -> ProcessExit { + let host_reading = hostname_short_read(transport: LocalShell) + match host_reading { + HostnameShortCaptureRefused { reason: r } => + exit_failure(reason: join(["microvm-network-converge: the host could not name itself: ", r], "")) + HostnameShortCaptured { value: name } => { + let generation = converge_generation_reading() + if generation == "" { + exit_failure(reason: "microvm-network-converge: no run identity in the environment, so the receipt would carry a generation nobody can compare against") + } else { + match runner_host_deploy_for(host: name as HostIdentity) { + RunnerHostDeployUnmodeled { host: h } => + exit_failure(reason: join(["microvm-network-converge: ", h as String, " is not a modeled runner host"], "")) + RunnerHostDeployRefused { host: h, cause: c } => + exit_failure(reason: join(["microvm-network-converge: ", h as String, " refuses its deploy row: ", c], "")) + RunnerHostDeployFound { deploy: d } => + converge_admitted_host_wet(deploy: d, host: name as HostIdentity, generation: generation as NonEmptyStr) + } + } + } + } +} + +fn converge_admitted_host_wet(deploy: RunnerHostDeploy, host: HostIdentity, generation: NonEmptyStr) -> ProcessExit { + let slots = runner_microvm_network_slot_identities_of(deploy: deploy) + let staged = converge_staged_files( + slots: slots, + host: runner_microvm_shared_host_grain, + tap_owner: deploy.job_user, + ) + let _ = run_shell_commands( + commands: [mkdir_parents_command(path: runner_microvm_staging_dir as String)], + transport: LocalExec, + ) + let staged_ok = fold(staged, init: true, f: (acc, f) => acc && stage_file_wet(f: f)) + if !staged_ok { + exit_failure(reason: "microvm-network-converge: a file could not be staged, so NOTHING was installed") + } else { + match apply_executor_privileged_operations_wet(ops: microvm_network_converge_operations(slots: slots)) { + ExitSuccess => record_converge_readback_wet(slots: slots, host: host, generation: generation) + other => other + } + } +} + +fn record_converge_readback_wet(slots: List, host: HostIdentity, generation: NonEmptyStr) -> ProcessExit { + let outcome = converge_outcome( + slots: map(slots, s => read_slot_wet(slot: s)), + h: read_host_wet(host: host, generation: generation as ConvergeGeneration), + ) + let text = receipt_text(o: outcome) + let _ = run_shell_commands(commands: [mkdir_parents_command(path: runner_microvm_network_receipt_dir)], transport: LocalExec) + let w = Filesystem.Write(path: runner_microvm_network_receipt_path, content: text) + if !w.success { + exit_failure(reason: join(["microvm-network-converge: receipt write refused: ", w.error], "")) + } else { + match outcome { + ConvergedNetwork { receipt: _ } => ExitSuccess + ConvergeRefused { refusal: _ } => exit_failure(reason: join(["microvm-network-converge: ", text], "")) + } + } +} diff --git a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag new file mode 100644 index 00000000000..07d33bd294c --- /dev/null +++ b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag @@ -0,0 +1,207 @@ +module gunbc.test.claim.runner.runner_microvm_network_converge_witness_test + +import std.types { String, NonEmptyStr, Bool, Int, List } +import std.nat { Nat } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import std.content_hash { Sha256Digest, Sha256DigestHex } +import gunbc.build_cache_instance { RunnerSlotIdentity } +import gunbc.fleet_intent_network { operator_host_srv1 } +import gunbc.executor_privileged_operation { + ExecutorPrivilegedOperation, InstallFile, SystemdReloadUnit, SystemdRestartUnit, + SystemdDaemonReload, SystemdEnableUnit, +} +import gunbc.runner_microvm_network { + runner_slot_network, runner_slot_host_cidr, + TapIpv6Disabled, TapIpv6Enabled, TapIpv6Unread, + AutoAssignmentAbsentFromKernel, AutoAssignmentOn, AutoAssignmentUnread, + ConvergedSlotNetwork, ConvergeGeneration, runner_microvm_shared_host_grain, +} +import gunbc.runner_microvm_network_converge { + parse_disable_ipv6, ip_forward_is_on, ct_helper_statement_count, + microvm_network_converge_operations, converge_staged_files, StagedFile, + SlotReading, HostReading, ConvergeOutcome, ConvergedNetwork, ConvergeRefused, + ConvergeRefusal, SlotTapIpv6Unread, SlotTapIpv6Enabled, SlotAddressNotHeld, SlotRouteNotHeld, + IpForwardingOff, RulesetUnread, HelperStandingUnread, HelperAutoAssignmentOn, HelpersAttached, + converge_outcome, render_slot_subnet, +} + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +fn slot(i: Nat) -> RunnerSlotIdentity { + RunnerSlotIdentity { host: operator_host_srv1, slot_index: i } +} + +data converged_digest: Sha256Digest = Sha256Digest { + hex: "3333333333333333333333333333333333333333333333333333333333333333" as Sha256DigestHex, +} + +// A HOST THAT READ BACK CLEAN. Every field is a reading, so a witness that wants a receipt has to +// supply every one -- which is the point of the type. +fn clean_host() -> HostReading { + HostReading { + host: operator_host_srv1, + generation: "run-3" as ConvergeGeneration, + ip_forward_on: true, + ruleset_listed: true, + ruleset_digest: converged_digest, + auto_assignment: AutoAssignmentAbsentFromKernel { kernel_release: "6.8.0-138-generic" as NonEmptyStr }, + xtables_ct_helper_targets: 0, + userspace_helpers: 0, + expectations: 0, + helper_probes_read: true, + } +} + +fn clean_slot(i: Nat) -> SlotReading { + let net = runner_slot_network(slot: slot(i: i)) + SlotReading { + slot: slot(i: i), + tap_ipv6: TapIpv6Disabled, + addr_stdout: join(["7: gunbc-tap", to_string(i), " inet ", runner_slot_host_cidr(net: net), " scope global gunbc-tap", to_string(i)], ""), + route_stdout: join([render_slot_subnet(net: net), " proto kernel scope link src ", runner_slot_host_cidr(net: net)], ""), + } +} + +fn outcome_label(o: ConvergeOutcome) -> String { + match o { + ConvergedNetwork { receipt: _ } => "converged" + ConvergeRefused { refusal: r } => match r { + SlotTapIpv6Unread { slot: _, standing: _ } => "tap-ipv6-unread" + SlotTapIpv6Enabled { slot: _ } => "tap-ipv6-on" + SlotAddressNotHeld { slot: _, expected: _ } => "no-address" + SlotRouteNotHeld { slot: _, expected: _ } => "no-route" + IpForwardingOff => "no-forwarding" + RulesetUnread { reason: _ } => "ruleset-unread" + HelperStandingUnread { reason: _ } => "helpers-unread" + HelperAutoAssignmentOn => "auto-helpers-on" + HelpersAttached { xtables_targets: _, userspace: _, expectations: _ } => "helpers-attached" + } + } +} + +fn receipt_slots(o: ConvergeOutcome) -> Int { + match o { + ConvergedNetwork { receipt: r } => length(r.slots) + ConvergeRefused { refusal: _ } => 0 + } +} + +// THE PROC VALUE IS THE FACT, AND EVERY OTHER READING IS UNREAD RATHER THAN A DEFAULT. "1" is +// disabled, "0" is enabled, an unreadable probe and an unexpected value are both Unread -- never +// silently one of the two real answers. +test fn a_tap_ipv6_reading_is_the_value_or_it_is_unread() -> Bool { + match parse_disable_ipv6(readable: true, text: "1\n") { TapIpv6Disabled => true, _ => false } + && match parse_disable_ipv6(readable: true, text: "0\n") { TapIpv6Enabled => true, _ => false } + && match parse_disable_ipv6(readable: false, text: "") { TapIpv6Unread { reason: _ } => true, _ => false } + && match parse_disable_ipv6(readable: true, text: "") { TapIpv6Unread { reason: _ } => true, _ => false } + && match parse_disable_ipv6(readable: true, text: "yes") { TapIpv6Unread { reason: _ } => true, _ => false } + && ip_forward_is_on(readable: true, text: "1\n") + && !ip_forward_is_on(readable: true, text: "0\n") + && !ip_forward_is_on(readable: false, text: "1\n") +} + +// THE POSITIVE CONTROL: with every fact read and holding, the receipt carries one row per slot read. +test fn a_receipt_is_produced_only_when_every_fact_was_read_and_holds() -> Bool { + let o = converge_outcome(slots: [clean_slot(i: 1), clean_slot(i: 2)], h: clean_host()) + outcome_label(o: o) == "converged" && receipt_slots(o: o) == 2 +} + +// EVERY UNREAD OR FAILING FACT REFUSES THE RECEIPT, AND NAMES ITSELF. The red is any arm that +// defaults a missing reading -- a receipt built on an unread helper probe or an unlisted ruleset +// would let the sanitation subject admit slots on a host nobody observed. +test fn an_unread_or_failing_fact_refuses_the_receipt() -> Bool { + let net1 = runner_slot_network(slot: slot(i: 1)) + let ipv6_on = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Enabled, addr_stdout: "", route_stdout: "" } + let ipv6_unread = SlotReading { + slot: slot(i: 1), + tap_ipv6: TapIpv6Unread { reason: "probe refused" as NonEmptyStr }, + addr_stdout: "", + route_stdout: "", + } + let no_addr = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Disabled, addr_stdout: "", route_stdout: "" } + let no_route = SlotReading { + slot: slot(i: 1), + tap_ipv6: TapIpv6Disabled, + addr_stdout: join(["inet ", runner_slot_host_cidr(net: net1)], ""), + route_stdout: "", + } + outcome_label(o: converge_outcome(slots: [ipv6_on], h: clean_host())) == "tap-ipv6-on" + && outcome_label(o: converge_outcome(slots: [ipv6_unread], h: clean_host())) == "tap-ipv6-unread" + && outcome_label(o: converge_outcome(slots: [no_addr], h: clean_host())) == "no-address" + && outcome_label(o: converge_outcome(slots: [no_route], h: clean_host())) == "no-route" + && receipt_slots(o: converge_outcome(slots: [no_addr], h: clean_host())) == 0 +} + +// THE HOST'S OWN FACTS REFUSE THE SAME WAY, INCLUDING EVERY HELPER ARM. Forwarding off, a ruleset +// that did not list, an unread helper probe, automatic assignment on, and an attached helper are +// five different remedies and five different refusals. +test fn each_host_fact_refuses_under_its_own_name() -> Bool { + let c = clean_host() + let no_fwd = HostReading { host: c.host, generation: c.generation, ip_forward_on: false, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: c.auto_assignment, xtables_ct_helper_targets: c.xtables_ct_helper_targets, userspace_helpers: c.userspace_helpers, expectations: c.expectations, helper_probes_read: c.helper_probes_read } + let unlisted = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: false, ruleset_digest: c.ruleset_digest, auto_assignment: c.auto_assignment, xtables_ct_helper_targets: c.xtables_ct_helper_targets, userspace_helpers: c.userspace_helpers, expectations: c.expectations, helper_probes_read: c.helper_probes_read } + let unread_probe = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: c.auto_assignment, xtables_ct_helper_targets: 0, userspace_helpers: 0, expectations: 0, helper_probes_read: false } + let auto_on = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: AutoAssignmentOn, xtables_ct_helper_targets: 0, userspace_helpers: 0, expectations: 0, helper_probes_read: true } + let unread_auto = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: AutoAssignmentUnread { reason: "sysctl unreadable" as NonEmptyStr }, xtables_ct_helper_targets: 0, userspace_helpers: 0, expectations: 0, helper_probes_read: true } + let attached = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: c.auto_assignment, xtables_ct_helper_targets: 0, userspace_helpers: 0, expectations: 2, helper_probes_read: true } + outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: no_fwd)) == "no-forwarding" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: unlisted)) == "ruleset-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: unread_probe)) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: auto_on)) == "auto-helpers-on" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: unread_auto)) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: attached)) == "helpers-attached" +} + +// A HELPER ATTACHED BY ANY TABLE COUNTS, because an expectation is created by whichever helper is +// attached, not only by one in our table. The red is a count taken over our own rendered ruleset. +test fn ct_helper_statements_are_counted_over_the_whole_listing() -> Bool { + ct_helper_statement_count(ruleset_listing: "table ip docker {\n ct helper ftp-standard\n}\n") == 1 + && ct_helper_statement_count(ruleset_listing: "ct helper a\nct helper b\n") == 2 + && ct_helper_statement_count(ruleset_listing: "table inet gunbc_microvm {\n chain forward {\n }\n}\n") == 0 +} + +// THE SLOTS' FILES LAND BEFORE THE HOST'S, AND THE HOST'S END IN AN ENABLE AND A RESTART. Installing +// the ruleset before the taps exist leaves a window where a tap is up and unfiltered, and `start` on +// an already-active oneshot would leave the OLD ruleset live while reporting success. +test fn the_operations_install_slots_before_the_host_and_restart_the_loader() -> Bool { + let ops = microvm_network_converge_operations(slots: [slot(i: 1)]) + map(ops, op => match op { + InstallFile { source: _, destination: d, owner: _, group: _, mode: _ } => d as String + SystemdReloadUnit { unit: u } => join(["reload ", u as String], "") + SystemdRestartUnit { unit: u } => join(["restart ", u as String], "") + SystemdEnableUnit { unit: u } => join(["enable ", u as String], "") + SystemdDaemonReload => "daemon-reload" + _ => "other" + }) == [ + "/etc/systemd/network/50-gunbc-tap1.netdev", + "/etc/systemd/network/50-gunbc-tap1.network", + "/etc/sysctl.d/50-gunbc-tap1-ipv6.conf", + "reload systemd-networkd", + "/etc/gunbc/microvm.nft", + "/etc/systemd/system/gunbc-microvm-nft.service", + "/etc/sysctl.d/50-gunbc-microvm-forward.conf", + "daemon-reload", + "restart systemd-sysctl.service", + "enable gunbc-microvm-nft.service", + "restart gunbc-microvm-nft.service", + ] +} + +// EVERY FILE THE OPERATIONS INSTALL IS A FILE THE CONVERGE STAGES. The red is a staged set that +// misses one: the install then runs against whatever that path held from a previous converge, or +// fails outright, and either way the bytes on the host are not the bytes the model rendered. +test fn every_installed_file_is_staged_with_its_rendered_bytes() -> Bool { + let staged = converge_staged_files( + slots: [slot(i: 1)], + host: runner_microvm_shared_host_grain, + tap_owner: "ghrunner" as NonEmptyStr, + ) + map(staged, f => f.path as String) == [ + "/var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.netdev", + "/var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.network", + "/var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1-ipv6.conf", + "/var/lib/gunbc/fleet-converge/microvm-network/microvm.nft", + "/var/lib/gunbc/fleet-converge/microvm-network/gunbc-microvm-nft.service", + "/var/lib/gunbc/fleet-converge/microvm-network/forward.conf", + ] + && fold(staged, init: true, f: (acc, x) => acc && x.content != "") +} diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index 157d95c69c5..27ae2d87075 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -19,7 +19,7 @@ import gunbc.runner_microvm_network { SlotNetworkFact, TapUnheldFact, GuestOriginatedConntrackEmptyFact, GuestDestinedConntrackEmptyFact, RulesetMatchesConvergedFact, SlotNetworkSubject, slot_network_subject, runner_slot_host_cidr, - ConvergedSlot, ConvergedSlotNetwork, HelperStanding, + ConvergedSlot, ConvergedSlotNetwork, ConvergeGeneration, HelperStanding, AutoAssignmentAbsentFromKernel, AutoAssignmentSysctlOff, AutoAssignmentOn, TapIpv6Standing, TapIpv6Disabled, TapIpv6Enabled, SanitationSubjectResult, SanitationSubjectAdmitted, SanitationSubjectRefused, @@ -255,7 +255,7 @@ fn subject(i: Nat) -> SlotNetworkSubject { slot_network_subject(slot: slot(i: i)) } -data converged_generation: Int = 7 +data converged_generation: ConvergeGeneration = "run-7" as ConvergeGeneration fn converged_slot(i: Nat) -> ConvergedSlot { let net = runner_slot_network(slot: slot(i: i)) @@ -453,7 +453,7 @@ test fn readings_about_another_subject_never_sanitize_this_one() -> Bool { && verdict(r: readback_slot_1(r: with_to(r: c, c: slot_residue(i: 2, o: ResidueAbsent)))) == "unobservable:conntrack-to" && verdict(r: readback_slot_1(r: with_neighbour(r: c, c: slot_residue(i: 2, o: ResidueAbsent)))) == "unobservable:neighbour" && verdict(r: readback_slot_1(r: with_expectations(r: c, e: HostResidueReading { host: operator_host_srv1, generation: converged_generation, observation: ResidueAbsent }))) == "unobservable:expectations" - && verdict(r: readback_slot_1(r: with_ruleset(r: c, l: RulesetReading { host: operator_host_srv2, generation: 6, observation: RulesetListed { digest: converged_digest } }))) == "unobservable:ruleset" + && verdict(r: readback_slot_1(r: with_ruleset(r: c, l: RulesetReading { host: operator_host_srv2, generation: "run-6" as ConvergeGeneration, observation: RulesetListed { digest: converged_digest } }))) == "unobservable:ruleset" } // TEARDOWN FLUSHES BOTH CONNTRACK DIRECTIONS AND THIS SLOT'S NEIGHBOUR ENTRY, ON THIS SLOT'S TAP; THE From 60212fdb02ef111b586878daca1dbe57f0e013f2 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 06:08:49 +0000 Subject: [PATCH 18/36] Wire the converge's imports and slot roster; the network mode declares its ssh-key demand Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/fleet/fleet_converge_workflow.dag | 1 + dag/gunbc/runner/runner_host_grants.dag | 15 ++++-------- .../runner_microvm_network_converge.dag | 24 +++++++++++++++++-- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index cd487896f57..2011ab041d8 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -330,6 +330,7 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> FleetApply => FleetSshKeyConsumed AppControlPlaneObserve => FleetSshKeyConsumed MicrovmHostConverge => FleetSshKeyConsumed + MicrovmNetworkConverge => FleetSshKeyConsumed GuestImageObserve => FleetSshKeyConsumed GuestImageConverge => FleetSshKeyConsumed MicrovmBootProbe => FleetSshKeyConsumed diff --git a/dag/gunbc/runner/runner_host_grants.dag b/dag/gunbc/runner/runner_host_grants.dag index dd699d3f244..12139ce362e 100644 --- a/dag/gunbc/runner/runner_host_grants.dag +++ b/dag/gunbc/runner/runner_host_grants.dag @@ -48,7 +48,9 @@ import gunbc.runner_host_deploy { RunnerHostDeploy, RunnerHostSpec, runner_host_ import product.placement_supply { HostIdentity } import gunbc.runner_slot_provision { desired_runner_slot_members, runner_slot_instance_dir } import gunbc.runner_host_deploy { runner_index_seq } -import gunbc.runner_microvm_network_converge { microvm_network_granted_operations } +import gunbc.runner_microvm_network_converge { + microvm_network_granted_operations, runner_microvm_network_slot_identities_of, +} import gunbc.runner_unit { runner_unit_name_of_registration } // THE MANAGED STATE DIRECTORY, DECLARED ONCE. It is currently spelled THREE times in @@ -315,15 +317,6 @@ fn runner_host_wide_refused_operations(spec: RunnerHostSpec) -> List List { - let host = deploy.host_label as HostIdentity - map(runner_index_seq(n: deploy.runner_count), i => RunnerSlotIdentity { host: host, slot_index: i }) -} - fn runner_host_privileged_operations(deploy: RunnerHostDeploy) -> List { let job = deploy.job_user let slot_dirs = map(desired_runner_slot_members(deploy: deploy), m => m.slot_dir) @@ -333,7 +326,7 @@ fn runner_host_privileged_operations(deploy: RunnerHostDeploy) -> List ProcessExit { } } +// EVERY SLOT ON THE HOST HAS A microVM NETWORK, because the microVM path replaces the shared slot +// rather than sitting beside it. The roster is derived here rather than imported from +// gunbc.runner_host_grants, which consumes THIS module's operation fold to render its sudoers lines: +// importing it back would be a cycle, and the slot roster is a fact about the deploy row either way. +fn runner_microvm_network_slot_identities_of(deploy: RunnerHostDeploy) -> List { + let host = deploy.host_label as HostIdentity + map(runner_index_seq(n: deploy.runner_count), i => RunnerSlotIdentity { host: host, slot_index: i }) +} + fn converge_admitted_host_wet(deploy: RunnerHostDeploy, host: HostIdentity, generation: NonEmptyStr) -> ProcessExit { let slots = runner_microvm_network_slot_identities_of(deploy: deploy) let staged = converge_staged_files( From 65ca33bf94bb27be520dc808dbd490ee7749597b Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 06:18:21 +0000 Subject: [PATCH 19/36] Regenerate the sudoers rosters and fleet-converge workflow for the microVM network mode Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/fleet-converge.yml | 22 ++++++- provisioning/srv1/gunbc-ghrunner.sudoers | 70 +++++++++++++++++++++ provisioning/srv3/gunbc-ghrunner.sudoers | 78 ++++++++++++++++++++++++ provisioning/srv4/gunbc-ghrunner.sudoers | 78 ++++++++++++++++++++++++ 4 files changed, 246 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 3cdf0ff870a..fd796554f45 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -13,7 +13,7 @@ on: mode: description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -227,7 +227,7 @@ jobs: echo "fleet-key: agent loaded (identity fleet-automation@gunbc; secret versions/1 pinned; fingerprint verified against modeled authority; key file wiped)" env: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} - if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'mtcollins1_boot' + if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_converge' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'mtcollins1_boot' timeout-minutes: 5 - name: Fleet converge plan (membership_reconcile → artifact) id: plan @@ -606,6 +606,24 @@ jobs: retention-days: 30 if: always() && (github.event.inputs.mode == 'microvm_host_converge') timeout-minutes: 10 + - name: "Runner micro-VM network: install the slot taps, ruleset and forwarding, then read the host back" + id: microvm_network_converge + run: | + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_microvm_network_converge.dag --function runner_microvm_network_converge_wet + cat "$ROOT/target/microvm-network-converge.txt" + if: github.event.inputs.mode == 'microvm_network_converge' + timeout-minutes: 30 + - name: Upload runner micro-VM network converge receipt + id: microvm_network_converge_receipt_upload + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: runner-microvm-network-converge-receipt + path: target/microvm-network-converge.txt + if-no-files-found: error + retention-days: 30 + if: always() && (github.event.inputs.mode == 'microvm_network_converge') + timeout-minutes: 10 - name: "Runner micro-VM: boot the guest image and read the serial console" id: microvm_boot_probe run: | diff --git a/provisioning/srv1/gunbc-ghrunner.sudoers b/provisioning/srv1/gunbc-ghrunner.sudoers index 468512ec291..a1ffc5565a8 100644 --- a/provisioning/srv1/gunbc-ghrunner.sudoers +++ b/provisioning/srv1/gunbc-ghrunner.sudoers @@ -113,3 +113,73 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansr ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansrls /opt/actions-runner-cache ghrunner ALL=(root) NOPASSWD: /usr/sbin/usermod -aG kvm ghrunner ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.netdev /etc/systemd/network/50-gunbc-tap1.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.network /etc/systemd/network/50-gunbc-tap1.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1-ipv6.conf /etc/sysctl.d/50-gunbc-tap1-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.netdev /etc/systemd/network/50-gunbc-tap2.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.network /etc/systemd/network/50-gunbc-tap2.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2-ipv6.conf /etc/sysctl.d/50-gunbc-tap2-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.netdev /etc/systemd/network/50-gunbc-tap3.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.network /etc/systemd/network/50-gunbc-tap3.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3-ipv6.conf /etc/sysctl.d/50-gunbc-tap3-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.netdev /etc/systemd/network/50-gunbc-tap4.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.network /etc/systemd/network/50-gunbc-tap4.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4-ipv6.conf /etc/sysctl.d/50-gunbc-tap4-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.netdev /etc/systemd/network/50-gunbc-tap5.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.network /etc/systemd/network/50-gunbc-tap5.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5-ipv6.conf /etc/sysctl.d/50-gunbc-tap5-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.netdev /etc/systemd/network/50-gunbc-tap6.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.network /etc/systemd/network/50-gunbc-tap6.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6-ipv6.conf /etc/sysctl.d/50-gunbc-tap6-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.netdev /etc/systemd/network/50-gunbc-tap7.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.network /etc/systemd/network/50-gunbc-tap7.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7-ipv6.conf /etc/sysctl.d/50-gunbc-tap7-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.netdev /etc/systemd/network/50-gunbc-tap8.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.network /etc/systemd/network/50-gunbc-tap8.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8-ipv6.conf /etc/sysctl.d/50-gunbc-tap8-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.netdev /etc/systemd/network/50-gunbc-tap9.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.network /etc/systemd/network/50-gunbc-tap9.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9-ipv6.conf /etc/sysctl.d/50-gunbc-tap9-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.netdev /etc/systemd/network/50-gunbc-tap10.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.network /etc/systemd/network/50-gunbc-tap10.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10-ipv6.conf /etc/sysctl.d/50-gunbc-tap10-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.netdev /etc/systemd/network/50-gunbc-tap11.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.network /etc/systemd/network/50-gunbc-tap11.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11-ipv6.conf /etc/sysctl.d/50-gunbc-tap11-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.netdev /etc/systemd/network/50-gunbc-tap12.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.network /etc/systemd/network/50-gunbc-tap12.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12-ipv6.conf /etc/sysctl.d/50-gunbc-tap12-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.netdev /etc/systemd/network/50-gunbc-tap13.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.network /etc/systemd/network/50-gunbc-tap13.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13-ipv6.conf /etc/sysctl.d/50-gunbc-tap13-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.netdev /etc/systemd/network/50-gunbc-tap14.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.network /etc/systemd/network/50-gunbc-tap14.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14-ipv6.conf /etc/sysctl.d/50-gunbc-tap14-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.netdev /etc/systemd/network/50-gunbc-tap15.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.network /etc/systemd/network/50-gunbc-tap15.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15-ipv6.conf /etc/sysctl.d/50-gunbc-tap15-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/microvm.nft /etc/gunbc/microvm.nft +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-microvm-nft.service /etc/systemd/system/gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/forward.conf /etc/sysctl.d/50-gunbc-microvm-forward.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart systemd-sysctl.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset +ghrunner ALL=(root) NOPASSWD: /usr/sbin/conntrack -L expect diff --git a/provisioning/srv3/gunbc-ghrunner.sudoers b/provisioning/srv3/gunbc-ghrunner.sudoers index 31a83e0eeae..a4458a1dc24 100644 --- a/provisioning/srv3/gunbc-ghrunner.sudoers +++ b/provisioning/srv3/gunbc-ghrunner.sudoers @@ -128,3 +128,81 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansr ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansrls /opt/actions-runner-cache ghrunner ALL=(root) NOPASSWD: /usr/sbin/usermod -aG kvm ghrunner ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.netdev /etc/systemd/network/50-gunbc-tap1.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.network /etc/systemd/network/50-gunbc-tap1.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1-ipv6.conf /etc/sysctl.d/50-gunbc-tap1-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.netdev /etc/systemd/network/50-gunbc-tap2.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.network /etc/systemd/network/50-gunbc-tap2.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2-ipv6.conf /etc/sysctl.d/50-gunbc-tap2-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.netdev /etc/systemd/network/50-gunbc-tap3.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.network /etc/systemd/network/50-gunbc-tap3.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3-ipv6.conf /etc/sysctl.d/50-gunbc-tap3-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.netdev /etc/systemd/network/50-gunbc-tap4.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.network /etc/systemd/network/50-gunbc-tap4.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4-ipv6.conf /etc/sysctl.d/50-gunbc-tap4-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.netdev /etc/systemd/network/50-gunbc-tap5.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.network /etc/systemd/network/50-gunbc-tap5.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5-ipv6.conf /etc/sysctl.d/50-gunbc-tap5-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.netdev /etc/systemd/network/50-gunbc-tap6.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.network /etc/systemd/network/50-gunbc-tap6.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6-ipv6.conf /etc/sysctl.d/50-gunbc-tap6-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.netdev /etc/systemd/network/50-gunbc-tap7.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.network /etc/systemd/network/50-gunbc-tap7.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7-ipv6.conf /etc/sysctl.d/50-gunbc-tap7-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.netdev /etc/systemd/network/50-gunbc-tap8.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.network /etc/systemd/network/50-gunbc-tap8.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8-ipv6.conf /etc/sysctl.d/50-gunbc-tap8-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.netdev /etc/systemd/network/50-gunbc-tap9.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.network /etc/systemd/network/50-gunbc-tap9.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9-ipv6.conf /etc/sysctl.d/50-gunbc-tap9-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.netdev /etc/systemd/network/50-gunbc-tap10.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.network /etc/systemd/network/50-gunbc-tap10.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10-ipv6.conf /etc/sysctl.d/50-gunbc-tap10-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.netdev /etc/systemd/network/50-gunbc-tap11.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.network /etc/systemd/network/50-gunbc-tap11.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11-ipv6.conf /etc/sysctl.d/50-gunbc-tap11-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.netdev /etc/systemd/network/50-gunbc-tap12.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.network /etc/systemd/network/50-gunbc-tap12.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12-ipv6.conf /etc/sysctl.d/50-gunbc-tap12-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.netdev /etc/systemd/network/50-gunbc-tap13.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.network /etc/systemd/network/50-gunbc-tap13.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13-ipv6.conf /etc/sysctl.d/50-gunbc-tap13-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.netdev /etc/systemd/network/50-gunbc-tap14.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.network /etc/systemd/network/50-gunbc-tap14.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14-ipv6.conf /etc/sysctl.d/50-gunbc-tap14-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.netdev /etc/systemd/network/50-gunbc-tap15.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.network /etc/systemd/network/50-gunbc-tap15.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15-ipv6.conf /etc/sysctl.d/50-gunbc-tap15-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16.netdev /etc/systemd/network/50-gunbc-tap16.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16.network /etc/systemd/network/50-gunbc-tap16.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16-ipv6.conf /etc/sysctl.d/50-gunbc-tap16-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17.netdev /etc/systemd/network/50-gunbc-tap17.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17.network /etc/systemd/network/50-gunbc-tap17.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17-ipv6.conf /etc/sysctl.d/50-gunbc-tap17-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/microvm.nft /etc/gunbc/microvm.nft +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-microvm-nft.service /etc/systemd/system/gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/forward.conf /etc/sysctl.d/50-gunbc-microvm-forward.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart systemd-sysctl.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset +ghrunner ALL=(root) NOPASSWD: /usr/sbin/conntrack -L expect diff --git a/provisioning/srv4/gunbc-ghrunner.sudoers b/provisioning/srv4/gunbc-ghrunner.sudoers index 2ff4d1bcbf6..7d207a42d64 100644 --- a/provisioning/srv4/gunbc-ghrunner.sudoers +++ b/provisioning/srv4/gunbc-ghrunner.sudoers @@ -119,3 +119,81 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansr ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansrls /opt/actions-runner-cache ghrunner ALL=(root) NOPASSWD: /usr/sbin/usermod -aG kvm ghrunner ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.netdev /etc/systemd/network/50-gunbc-tap1.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.network /etc/systemd/network/50-gunbc-tap1.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1-ipv6.conf /etc/sysctl.d/50-gunbc-tap1-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.netdev /etc/systemd/network/50-gunbc-tap2.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.network /etc/systemd/network/50-gunbc-tap2.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2-ipv6.conf /etc/sysctl.d/50-gunbc-tap2-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.netdev /etc/systemd/network/50-gunbc-tap3.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.network /etc/systemd/network/50-gunbc-tap3.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3-ipv6.conf /etc/sysctl.d/50-gunbc-tap3-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.netdev /etc/systemd/network/50-gunbc-tap4.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.network /etc/systemd/network/50-gunbc-tap4.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4-ipv6.conf /etc/sysctl.d/50-gunbc-tap4-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.netdev /etc/systemd/network/50-gunbc-tap5.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.network /etc/systemd/network/50-gunbc-tap5.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5-ipv6.conf /etc/sysctl.d/50-gunbc-tap5-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.netdev /etc/systemd/network/50-gunbc-tap6.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.network /etc/systemd/network/50-gunbc-tap6.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6-ipv6.conf /etc/sysctl.d/50-gunbc-tap6-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.netdev /etc/systemd/network/50-gunbc-tap7.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.network /etc/systemd/network/50-gunbc-tap7.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7-ipv6.conf /etc/sysctl.d/50-gunbc-tap7-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.netdev /etc/systemd/network/50-gunbc-tap8.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.network /etc/systemd/network/50-gunbc-tap8.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8-ipv6.conf /etc/sysctl.d/50-gunbc-tap8-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.netdev /etc/systemd/network/50-gunbc-tap9.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.network /etc/systemd/network/50-gunbc-tap9.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9-ipv6.conf /etc/sysctl.d/50-gunbc-tap9-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.netdev /etc/systemd/network/50-gunbc-tap10.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.network /etc/systemd/network/50-gunbc-tap10.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10-ipv6.conf /etc/sysctl.d/50-gunbc-tap10-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.netdev /etc/systemd/network/50-gunbc-tap11.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.network /etc/systemd/network/50-gunbc-tap11.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11-ipv6.conf /etc/sysctl.d/50-gunbc-tap11-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.netdev /etc/systemd/network/50-gunbc-tap12.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.network /etc/systemd/network/50-gunbc-tap12.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12-ipv6.conf /etc/sysctl.d/50-gunbc-tap12-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.netdev /etc/systemd/network/50-gunbc-tap13.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.network /etc/systemd/network/50-gunbc-tap13.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13-ipv6.conf /etc/sysctl.d/50-gunbc-tap13-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.netdev /etc/systemd/network/50-gunbc-tap14.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.network /etc/systemd/network/50-gunbc-tap14.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14-ipv6.conf /etc/sysctl.d/50-gunbc-tap14-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.netdev /etc/systemd/network/50-gunbc-tap15.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.network /etc/systemd/network/50-gunbc-tap15.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15-ipv6.conf /etc/sysctl.d/50-gunbc-tap15-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16.netdev /etc/systemd/network/50-gunbc-tap16.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16.network /etc/systemd/network/50-gunbc-tap16.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16-ipv6.conf /etc/sysctl.d/50-gunbc-tap16-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17.netdev /etc/systemd/network/50-gunbc-tap17.netdev +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17.network /etc/systemd/network/50-gunbc-tap17.network +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17-ipv6.conf /etc/sysctl.d/50-gunbc-tap17-ipv6.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/microvm.nft /etc/gunbc/microvm.nft +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-microvm-nft.service /etc/systemd/system/gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/forward.conf /etc/sysctl.d/50-gunbc-microvm-forward.conf +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart systemd-sysctl.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset +ghrunner ALL=(root) NOPASSWD: /usr/sbin/conntrack -L expect From f9fc5fc2fe2bd38676b21a2727e896eef1b506e4 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 06:40:03 +0000 Subject: [PATCH 20/36] The two network frontiers name what landed and what a run must still establish Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_microvm_network.dag | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 7809ba66a55..bf8657563f9 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -578,7 +578,7 @@ fn runner_microvm_nft_ruleset_text_for(host: HostEgressGrain) -> String { // slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by a // witness folding a grain, and not by any per-attempt step installing a table. data host_ruleset_installer_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: the microvm_host_converge fleet-converge mode on srvN. TRIGGER: it installs runner_microvm_nft_ruleset_text_for(host: the converged host's HostEgressGrain), reads the listing back with runner_microvm_nft_list_command, and records that digest as the converged digest slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by a witness folding a grain or by a per-attempt step installing a table." as NonEmptyStr, + description: "NAMED CONSUMER: the microvm_network_converge fleet-converge mode (gunbc.runner_microvm_network_converge runner_microvm_network_converge_wet), which HAS LANDED and is wired as a mode, a CI target and a census row. WHAT REMAINS: it has never executed on a host, so no ruleset has been installed by it and no listing has been read back. TRIGGER: that mode runs on srv1 -- its sudoers roster installed, the operations applied, the listing read back with runner_microvm_nft_list_command -- and records the digest as the converged digest slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by the fold existing, by a witness folding a grain, or by the mode being dispatchable." as NonEmptyStr, ) fn runner_microvm_shared_host_nft_ruleset_text() -> String { @@ -951,7 +951,7 @@ type ConvergedSlotNetwork { // deleting this row and host_ruleset_installer_frontier together. NOT satisfied by a witness // building a receipt. data converged_slot_network_producer_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: microvm_host_converge on srvN, producing ConvergedSlotNetwork. TRIGGER: it installs each slot's .netdev, .network and v6 sysctl file and the ruleset for the host's HostEgressGrain, reads back tap, address, route, disable_ipv6, helper standing and the nft listing digest, and records them at a generation. SUFFICIENT FOR deleting this row and host_ruleset_installer_frontier. NOT satisfied by a witness building a receipt." as NonEmptyStr, + description: "NAMED CONSUMER: gunbc.runner_microvm_network_converge runner_microvm_network_converge_wet, which HAS LANDED: it stages each slot's .netdev, .network and v6 sysctl file and the host's ruleset, unit and forwarding sysctl, applies them as modeled operations, and folds the host readings into ConvergedSlotNetwork or a typed refusal. WHAT REMAINS: no run has produced a receipt, so no ConvergedSlotNetwork exists for any host. TRIGGER: that mode executes on srv1 and records a receipt carrying the live tap, address, route, disable_ipv6, helper standing and nft listing digest at a generation. SUFFICIENT FOR deleting this row and host_ruleset_installer_frontier. NOT satisfied by a witness building a receipt, or by the converge compiling." as NonEmptyStr, ) // --- the sanitation subject: sealed, and joined to the receipt --- From 3edbc41ce280cc966ed49690adb4ec8ba05e58e1 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 08:01:43 +0000 Subject: [PATCH 21/36] Read every helper surface, match the route the kernel prints, bind the requested host, and persist the whole receipt Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/linux/nf_conntrack.dag | 15 +- dag/gunbc/executor_privileged_operation.dag | 14 + dag/gunbc/fleet/fleet_posix_accounts.dag | 13 + dag/gunbc/runner/runner_microvm_network.dag | 48 ++- .../runner_microvm_network_converge.dag | 350 +++++++++++++----- ..._microvm_network_converge_witness_test.dag | 134 +++++-- .../runner_microvm_network_witness_test.dag | 47 ++- 7 files changed, 471 insertions(+), 150 deletions(-) diff --git a/dag/extdeps/linux/nf_conntrack.dag b/dag/extdeps/linux/nf_conntrack.dag index c57cef6786d..5843a31c4f5 100644 --- a/dag/extdeps/linux/nf_conntrack.dag +++ b/dag/extdeps/linux/nf_conntrack.dag @@ -1,6 +1,6 @@ module extdeps.linux.nf_conntrack -import std.types { NonEmptyStr } +import std.types { String, NonEmptyStr } import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } import std.decl_ref { DeclarationRef, WholeDeclaration } import extdeps.uri { Uri, Https } @@ -28,4 +28,15 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope { // to a flow by port with no rule saying so. Upstream removed it, so on a current kernel the path is // ABSENT -- which is a stronger reading than 0, because there is no setting left to turn back on. A // reader must therefore distinguish absent from 0 from 1, and never treat absence as unreadable. -data nf_conntrack_helper_proc_path: NonEmptyStr = "/proc/sys/net/netfilter/nf_conntrack_helper" +data nf_conntrack_sysctl_dir: NonEmptyStr = "/proc/sys/net/netfilter" + +data nf_conntrack_helper_sysctl_name: NonEmptyStr = "nf_conntrack_helper" + +// The full path, derived so the directory and the leaf cannot drift apart: a reader establishing the +// sysctl's PRESENCE lists the directory and looks for the name, while a reader of its VALUE opens +// the path, and both must name the same file. +fn nf_conntrack_helper_proc_path_of() -> NonEmptyStr { + join([nf_conntrack_sysctl_dir as String, "/", nf_conntrack_helper_sysctl_name as String], "") as NonEmptyStr +} + +data nf_conntrack_helper_proc_path: NonEmptyStr = nf_conntrack_helper_proc_path_of() diff --git a/dag/gunbc/executor_privileged_operation.dag b/dag/gunbc/executor_privileged_operation.dag index 0d70c40bfa7..f9363b849b9 100644 --- a/dag/gunbc/executor_privileged_operation.dag +++ b/dag/gunbc/executor_privileged_operation.dag @@ -35,6 +35,8 @@ import gunbc.fleet_posix_accounts { SudoSystemctl, SudoNft, SudoConntrack, + SudoIptablesLegacySave, + SudoNfct, } // A GRANTED READ IS AN OPERATION TOO. ReadNftTable mutates nothing, and it is here because the @@ -72,6 +74,8 @@ type ExecutorPrivilegedOperation | ReadNftTable { family: NonEmptyStr, table: NonEmptyStr } | ReadNftRuleset | ReadConntrackExpectations + | ReadLegacyXtablesRules + | ReadUserspaceConntrackHelpers // THE ONE PROJECTION. Everything else in this module derives from this function, so there is // exactly one place where an operation becomes words. @@ -138,6 +142,14 @@ fn executor_privileged_operation_argv(op: ExecutorPrivilegedOperation) -> List [ + ci_runner_sudo_binary_path(b: SudoIptablesLegacySave) + ] + ReadUserspaceConntrackHelpers => [ + ci_runner_sudo_binary_path(b: SudoNfct), + "helper" as NonEmptyStr, + "list" as NonEmptyStr + ] ReadNftTable { family: f, table: t } => [ ci_runner_sudo_binary_path(b: SudoNft), "list" as NonEmptyStr, @@ -344,6 +356,8 @@ fn executor_privileged_operation_precondition(op: ExecutorPrivilegedOperation) - ReadNftTable { family: _, table: _ } => NoPrecondition ReadNftRuleset => NoPrecondition ReadConntrackExpectations => NoPrecondition + ReadLegacyXtablesRules => NoPrecondition + ReadUserspaceConntrackHelpers => NoPrecondition } } diff --git a/dag/gunbc/fleet/fleet_posix_accounts.dag b/dag/gunbc/fleet/fleet_posix_accounts.dag index a7bd204a380..d46faee212c 100644 --- a/dag/gunbc/fleet/fleet_posix_accounts.dag +++ b/dag/gunbc/fleet/fleet_posix_accounts.dag @@ -238,6 +238,13 @@ data fleet_nft_binary_path: NonEmptyStr = "/usr/sbin/nft" // CAP_NET_ADMIN -- listing the expectation table is as privileged as flushing a flow. data fleet_conntrack_binary_path: NonEmptyStr = "/usr/sbin/conntrack" +// THE LEGACY xtables SAVE BINARY AND THE USERSPACE HELPER TOOL, both privileged readers. A ruleset +// created through iptables-legacy is invisible to `nft list ruleset`, and a userspace helper is +// registered with the kernel rather than written in any ruleset, so each needs its own probe. +data fleet_iptables_legacy_save_binary_path: NonEmptyStr = "/usr/sbin/iptables-legacy-save" + +data fleet_nfct_binary_path: NonEmptyStr = "/usr/sbin/nfct" + // A CLOSED VOCABULARY OF WHAT THIS PRINCIPAL MAY SUDO, so that emitting an ungranted sudo command // has no spelling rather than being a mistake something downstream has to notice. // @@ -270,6 +277,8 @@ type CiRunnerSudoBinary | SudoUsermod | SudoNft | SudoConntrack + | SudoIptablesLegacySave + | SudoNfct fn ci_runner_sudo_binary_path(b: CiRunnerSudoBinary) -> NonEmptyStr { match b { @@ -283,6 +292,8 @@ fn ci_runner_sudo_binary_path(b: CiRunnerSudoBinary) -> NonEmptyStr { SudoUsermod => fleet_usermod_binary_path SudoNft => fleet_nft_binary_path SudoConntrack => fleet_conntrack_binary_path + SudoIptablesLegacySave => fleet_iptables_legacy_save_binary_path + SudoNfct => fleet_nfct_binary_path } } @@ -298,6 +309,8 @@ fn ci_runner_sudo_binary_probe(b: CiRunnerSudoBinary) -> SudoNopasswdProbeKind { SudoUsermod => NopasswdExecuteCheck { check_argv: "--help" } SudoNft => NopasswdExecuteCheck { check_argv: "--version" } SudoConntrack => NopasswdExecuteCheck { check_argv: "--version" } + SudoIptablesLegacySave => NopasswdExecuteCheck { check_argv: "--version" } + SudoNfct => NopasswdExecuteCheck { check_argv: "--version" } } } diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index bf8657563f9..e313122d354 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -650,6 +650,22 @@ fn runner_slot_network_file_text(net: RunnerSlotNetwork) -> String { ], "") } +// THE CONNECTED ROUTE IS OVER THE NETWORK ADDRESS, NOT THE HOST'S OWN. `ip -o -4 route show dev tap` +// prints `172.30.N.0/30 proto kernel scope link src 172.30.N.1`: the destination is the NETWORK and +// the host address appears only as the source. A readback matching the host address with a prefix +// would never hit, so a correctly configured tap would read as SlotRouteNotHeld (side-chat hold on +// #11751). The network address is derived here, beside the addresses it shares its octets with, so +// the route matcher and the address authority cannot drift. +fn runner_slot_subnet(net: RunnerSlotNetwork) -> Ipv4Prefix { + ipv4_prefix( + o1: net.host_address.octet1, + o2: net.host_address.octet2, + o3: net.host_address.octet3, + o4: 0, + length: net.prefix_length, + ) +} + fn runner_slot_host_cidr(net: RunnerSlotNetwork) -> String { join([ render_ipv4_address(addr: net.host_address) as String, @@ -908,12 +924,28 @@ type HelperAutoAssignment | AutoAssignmentOn | AutoAssignmentUnread { reason: NonEmptyStr } +// A SURFACE IS COUNTED OR IT IS UNREAD, AND THERE IS NO THIRD ANSWER. An Int field cannot say "I did +// not look", so a converge that could not run a probe had only a zero to write -- which is the +// fabricated reading DESIGN section 5 forbids, and it reads as "no helper is attached" (review +// 68963). The count now carries its own absence, so an unread surface cannot be spelled as an +// excluded one. +type HelperSurfaceReading + = HelperSurfaceCounted { count: Int } + | HelperSurfaceUnread { reason: NonEmptyStr } + type HelperStanding { auto_assignment: HelperAutoAssignment - nft_ct_helper_statements: Int - xtables_ct_helper_targets: Int - userspace_helpers: Int - expectations_at_generation: Int + nft_ct_helper_statements: HelperSurfaceReading + xtables_ct_helper_targets: HelperSurfaceReading + userspace_helpers: HelperSurfaceReading + expectations_at_generation: HelperSurfaceReading +} + +fn helper_surface_is_empty(r: HelperSurfaceReading) -> Bool { + match r { + HelperSurfaceCounted { count: n } => n == 0 + HelperSurfaceUnread { reason: _ } => false + } } fn helpers_excluded(h: HelperStanding) -> Bool { @@ -924,10 +956,10 @@ fn helpers_excluded(h: HelperStanding) -> Bool { AutoAssignmentUnread { reason: _ } => false } auto_off - && h.nft_ct_helper_statements == 0 - && h.xtables_ct_helper_targets == 0 - && h.userspace_helpers == 0 - && h.expectations_at_generation == 0 + && helper_surface_is_empty(r: h.nft_ct_helper_statements) + && helper_surface_is_empty(r: h.xtables_ct_helper_targets) + && helper_surface_is_empty(r: h.userspace_helpers) + && helper_surface_is_empty(r: h.expectations_at_generation) } // THE GENERATION IS AN IDENTITY, NOT A COUNT. What a consumer needs is "which converge established diff --git a/dag/gunbc/runner/runner_microvm_network_converge.dag b/dag/gunbc/runner/runner_microvm_network_converge.dag index b673b94f2e4..5e7857321b5 100644 --- a/dag/gunbc/runner/runner_microvm_network_converge.dag +++ b/dag/gunbc/runner/runner_microvm_network_converge.dag @@ -9,18 +9,23 @@ import extdeps.exec.command { ArgvCommand } import extdeps.shell.exec { ProcessOutcome, ProcessOutputPresent, ProcessOutputAbsent, ProcessRefused } import extdeps.filesystem.filesystem_io { Filesystem, FilesystemReadOutcome, FilesystemReadSucceeded, FilesystemReadRefused, - filesystem_read_outcome, + filesystem_read_outcome, filesystem_listing_observation, filesystem_file_observation, + FilesystemFileObservation, FilesystemFileAbsent, FilesystemFileRead, FilesystemFileIndeterminate, + FilesystemFileObservationsDisagree, FilesystemFileSubjectRefused, } -import extdeps.tools.sha256sum { Sha256FileDigest, sha256sum_stdin_digest_via_shell } +import extdeps.tools.sha256sum { Sha256FileDigest, Sha256FileDigestUnavailable, sha256sum_stdin_digest_via_shell } import extdeps.crypto.hash { Digest } import extdeps.iproute2.ip_show { ip_addr_show_command, ip_route_show_command } -import extdeps.linux.nf_conntrack { nf_conntrack_helper_proc_path } +import extdeps.linux.nf_conntrack { + nf_conntrack_helper_proc_path, nf_conntrack_sysctl_dir, nf_conntrack_helper_sysctl_name, +} import extdeps.tools.uname { uname_release_command } import gunbc.command_runner { run_shell_command_capture, run_shell_commands, LocalExec } import std.process { ProcessExit, ExitSuccess, exit_failure } import extdeps.shell import extdeps.tools.mkdir { mkdir_parents_command } import extdeps.github.actions_environment { github_run_id_variable_name } +import gunbc.actions_run_binding { fleet_converge_expected_host_env_name } import gunbc.hostname_read { hostname_short_read, HostnameShortCaptured, HostnameShortCaptureRefused } import gunbc.host_effect { LocalShell } import gunbc.runner_host_deploy { @@ -29,10 +34,13 @@ import gunbc.runner_host_deploy { } import gunbc.executor_privileged_apply { apply_executor_privileged_operations_wet } import gunbc.executor_privileged_apply { executor_privileged_operation_command } -import extdeps.network.ipv4 { render_ipv4_address } +import extdeps.network.ipv4 { render_ipv4_address, render_ipv4_prefix } import gunbc.build_cache_instance { RunnerSlotIdentity, runner_slot_instance_name } import product.placement_supply { HostIdentity } -import gunbc.executor_privileged_operation { ExecutorPrivilegedOperation, ReadNftRuleset, ReadConntrackExpectations } +import gunbc.executor_privileged_operation { + ExecutorPrivilegedOperation, ReadNftRuleset, ReadConntrackExpectations, + ReadLegacyXtablesRules, ReadUserspaceConntrackHelpers, +} import gunbc.runner_microvm_network { RunnerSlotNetwork, runner_slot_network, runner_slot_host_cidr, runner_slot_netdev_text, runner_slot_network_file_text, runner_slot_ipv6_sysctl_text, @@ -41,9 +49,10 @@ import gunbc.runner_microvm_network { runner_slot_ipv6_disabled_proc_path, runner_microvm_ip_forward_proc_path, HostEgressGrain, runner_microvm_shared_host_grain, ConvergedSlot, ConvergedSlotNetwork, ConvergeGeneration, HelperStanding, HelperAutoAssignment, + HelperSurfaceReading, HelperSurfaceCounted, HelperSurfaceUnread, AutoAssignmentAbsentFromKernel, AutoAssignmentSysctlOff, AutoAssignmentOn, AutoAssignmentUnread, TapIpv6Standing, TapIpv6Disabled, TapIpv6Enabled, TapIpv6Unread, - slot_network_subject, runner_microvm_nft_read_operation, + slot_network_subject, runner_microvm_nft_read_operation, runner_slot_subnet, } data runner_microvm_network_converge_disposition: Disposition = SingleAuthority @@ -163,6 +172,8 @@ fn microvm_network_readback_operations() -> List { [ runner_microvm_nft_read_operation(), ReadNftRuleset, + ReadLegacyXtablesRules, + ReadUserspaceConntrackHelpers, ReadConntrackExpectations, ] } @@ -207,12 +218,22 @@ fn route_reading_carries(stdout: String, cidr: String) -> Bool { // dropped automatic helper assignment entirely, so on a kernel that has no such sysctl there is no // setting to get wrong; on an older one the value must read 0. A file that exists and reads 1 is // AutoAssignmentOn, and an unreadable probe is neither -- it is unread, and refuses the receipt. -fn parse_auto_assignment(present: Bool, readable: Bool, text: String, kernel_release: String) -> HelperAutoAssignment { - if !present { - AutoAssignmentAbsentFromKernel { kernel_release: kernel_release as NonEmptyStr } - } else if !readable { - AutoAssignmentUnread { reason: "nf_conntrack_helper exists but could not be read" as NonEmptyStr } - } else if trim(s: text) == "0" { +fn parse_auto_assignment(observed: FilesystemFileObservation, kernel_release: String) -> HelperAutoAssignment { + match observed { + FilesystemFileAbsent(_) => AutoAssignmentAbsentFromKernel { kernel_release: kernel_release as NonEmptyStr } + FilesystemFileIndeterminate { cause: c } => AutoAssignmentUnread { reason: join(["nf_conntrack_helper could not be observed: ", c], "") as NonEmptyStr } + FilesystemFileObservationsDisagree { path: _, cause: c } => AutoAssignmentUnread { reason: join(["nf_conntrack_helper moved while it was observed: ", c], "") as NonEmptyStr } + FilesystemFileSubjectRefused { directory: _, name: _, cause: c } => AutoAssignmentUnread { reason: join(["nf_conntrack_helper path refused: ", c], "") as NonEmptyStr } + FilesystemFileRead { path: _, content: text } => parse_auto_assignment_value(text: text) + } +} + +// ABSENCE IS ESTABLISHED BY A LISTING, NEVER BY A FAILED READ. A read that fails for permission or +// I/O is not evidence the sysctl is gone, and reporting it as AbsentFromKernel would make an +// unreadable probe say "there is no setting left to turn on" -- the widening arm DESIGN section 5 +// forbids, and the inversion of this module's own nf_conntrack note (review 68963). +fn parse_auto_assignment_value(text: String) -> HelperAutoAssignment { + if trim(s: text) == "0" { AutoAssignmentSysctlOff } else if trim(s: text) == "1" { AutoAssignmentOn @@ -221,6 +242,28 @@ fn parse_auto_assignment(present: Bool, readable: Bool, text: String, kernel_rel } } +// A SURFACE THAT DID NOT ANSWER IS UNREAD, and the count is only minted from a run that answered. +fn surface_reading(o: ProcessOutcome, surface: String, count: Int) -> HelperSurfaceReading { + match o { + ProcessRefused { exit_code: code, stderr: _ } => + HelperSurfaceUnread { reason: join([surface, " probe exited ", to_string(code)], "") as NonEmptyStr } + ProcessOutputPresent { text: _ } => HelperSurfaceCounted { count: count } + ProcessOutputAbsent => HelperSurfaceCounted { count: count } + } +} + +// A LEGACY CT TARGET IS SPELLED `-j CT` IN iptables-save OUTPUT, and `--helper` is the option that +// attaches one. Counting the target lines counts every legacy rule that could attach a helper. +fn xtables_ct_target_count(save_output: String) -> Int { + length(filter(xs: split(s: save_output, delimiter: "\n"), predicate: fn(l) { string_contains(s: l, pattern: "-j CT") })) +} + +// `nfct helper list` PRINTS ONE BLOCK PER REGISTERED USERSPACE HELPER and nothing when none are +// registered, so a non-empty listing is the count of lines naming one. +fn userspace_helper_count(listing: String) -> Int { + length(filter(xs: split(s: listing, delimiter: "\n"), predicate: fn(l) { trim(s: l) != "" })) +} + // EVERY LINE OF THE EFFECTIVE RULESET, NOT OURS. `ct helper` may be attached by any table on the // host, so the count is taken over the whole listing; ours contains no such statement, which is why // a nonzero count is someone else's helper and not a miscount of our own rows. @@ -237,7 +280,7 @@ type ConvergeRefusal | RulesetUnread { reason: NonEmptyStr } | HelperStandingUnread { reason: NonEmptyStr } | HelperAutoAssignmentOn - | HelpersAttached { xtables_targets: Int, userspace: Int, expectations: Int } + | HelpersAttached { surface: NonEmptyStr, count: Int } type ConvergeOutcome = ConvergedNetwork { receipt: ConvergedSlotNetwork } @@ -255,13 +298,12 @@ type HostReading { host: HostIdentity generation: ConvergeGeneration ip_forward_on: Bool - ruleset_listed: Bool - ruleset_digest: Sha256Digest + ruleset_digest: Sha256Digest? auto_assignment: HelperAutoAssignment - xtables_ct_helper_targets: Int - userspace_helpers: Int - expectations: Int - helper_probes_read: Bool + nft_ct_helper_statements: HelperSurfaceReading + xtables_ct_helper_targets: HelperSurfaceReading + userspace_helpers: HelperSurfaceReading + expectations: HelperSurfaceReading } fn converged_slot_of_reading(r: SlotReading) -> ConvergedSlot { @@ -286,23 +328,14 @@ fn slot_refusal(r: SlotReading) -> List { TapIpv6Disabled => if !addr_reading_carries(stdout: r.addr_stdout, cidr: cidr) { [SlotAddressNotHeld { slot: r.slot, expected: cidr as NonEmptyStr }] - } else if !route_reading_carries(stdout: r.route_stdout, cidr: render_slot_subnet(net: net)) { - [SlotRouteNotHeld { slot: r.slot, expected: render_slot_subnet(net: net) as NonEmptyStr }] + } else if !route_reading_carries(stdout: r.route_stdout, cidr: render_ipv4_prefix(p: runner_slot_subnet(net: net)) as String) { + [SlotRouteNotHeld { slot: r.slot, expected: render_ipv4_prefix(p: runner_slot_subnet(net: net)) }] } else { [] } } } -// THE ROUTE IS OVER THE SUBNET, NOT THE HOST ADDRESS. `ip route show dev tap` prints the connected -// route as the network -- 172.30.N.0/30 -- so matching the host's own /30 address would never hit. -fn render_slot_subnet(net: RunnerSlotNetwork) -> String { - join([ - render_ipv4_address(addr: net.host_address) as String, - "/", - to_string(net.prefix_length), - ], "") -} fn first_refusal(rs: List) -> ConvergeRefusal? { fold(rs, init: none, f: (acc, r) => @@ -317,7 +350,10 @@ fn host_refusals(h: HostReading) -> List { concat( concat( if h.ip_forward_on { [] } else { [IpForwardingOff] }, - if h.ruleset_listed { [] } else { [RulesetUnread { reason: "nft list table did not return the installed table" as NonEmptyStr }] }, + match h.ruleset_digest { + Present { value: _ } => [] + Absent => [RulesetUnread { reason: "nft list table did not return a digestible listing of the installed table" as NonEmptyStr }] + }, ), match h.auto_assignment { AutoAssignmentUnread { reason: why } => [HelperStandingUnread { reason: why }] @@ -328,24 +364,35 @@ fn host_refusals(h: HostReading) -> List { ) } -fn helper_count_refusals(h: HostReading) -> List { - if !h.helper_probes_read { - [HelperStandingUnread { reason: "a helper probe did not run, so no helper standing was established" as NonEmptyStr }] - } else if h.xtables_ct_helper_targets != 0 || h.userspace_helpers != 0 || h.expectations != 0 { - [HelpersAttached { - xtables_targets: h.xtables_ct_helper_targets, - userspace: h.userspace_helpers, - expectations: h.expectations, - }] - } else { - [] +// EACH SURFACE REFUSES UNDER ITS OWN NAME. An unread surface is not an empty one: the reason names +// which probe did not answer, so an operator fixes the probe rather than hunting a helper that was +// never observed. +fn surface_refusals(name: String, r: HelperSurfaceReading) -> List { + match r { + HelperSurfaceUnread { reason: why } => + [HelperStandingUnread { reason: join([name, ": ", why as String], "") as NonEmptyStr }] + HelperSurfaceCounted { count: n } => + if n == 0 { [] } else { [HelpersAttached { surface: name as NonEmptyStr, count: n }] } } } +fn helper_count_refusals(h: HostReading) -> List { + concat( + concat( + surface_refusals(name: "nft ct helper statements", r: h.nft_ct_helper_statements), + surface_refusals(name: "legacy xtables CT targets", r: h.xtables_ct_helper_targets), + ), + concat( + surface_refusals(name: "userspace conntrack helpers", r: h.userspace_helpers), + surface_refusals(name: "conntrack expectations", r: h.expectations), + ), + ) +} + fn helper_standing_of(h: HostReading) -> HelperStanding { HelperStanding { auto_assignment: h.auto_assignment, - nft_ct_helper_statements: 0, + nft_ct_helper_statements: h.nft_ct_helper_statements, xtables_ct_helper_targets: h.xtables_ct_helper_targets, userspace_helpers: h.userspace_helpers, expectations_at_generation: h.expectations, @@ -359,13 +406,16 @@ fn converge_outcome(slots: List, h: HostReading) -> ConvergeOutcome let refusals = concat(flat_map(slots, r => slot_refusal(r: r)), host_refusals(h: h)) match first_refusal(rs: refusals) { Present { value: r } => ConvergeRefused { refusal: r } - Absent => ConvergedNetwork { - receipt: ConvergedSlotNetwork { - host: h.host, - generation: h.generation, - slots: map(slots, r => converged_slot_of_reading(r: r)), - helpers: helper_standing_of(h: h), - nft_digest: h.ruleset_digest, + Absent => match h.ruleset_digest { + Absent => ConvergeRefused { refusal: RulesetUnread { reason: "no digestible listing of the installed table" as NonEmptyStr } } + Present { value: d } => ConvergedNetwork { + receipt: ConvergedSlotNetwork { + host: h.host, + generation: h.generation, + slots: map(slots, r => converged_slot_of_reading(r: r)), + helpers: helper_standing_of(h: h), + nft_digest: d, + }, }, }, } @@ -450,53 +500,58 @@ fn read_host_wet(host: HostIdentity, generation: ConvergeGeneration) -> HostRead let forward = read_proc_value(path: runner_microvm_ip_forward_proc_path as String) let table = capture_stdout(c: executor_privileged_operation_command(op: runner_microvm_nft_read_operation())) let ruleset = capture_stdout(c: executor_privileged_operation_command(op: ReadNftRuleset)) + let xtables = capture_stdout(c: executor_privileged_operation_command(op: ReadLegacyXtablesRules)) + let userspace = capture_stdout(c: executor_privileged_operation_command(op: ReadUserspaceConntrackHelpers)) let expectations = capture_stdout(c: executor_privileged_operation_command(op: ReadConntrackExpectations)) - let helper_sysctl = read_proc_value(path: nf_conntrack_helper_proc_path as String) let kernel = capture_stdout(c: uname_release_command()) - let digest = listing_digest(d: host_ruleset_digest_wet(listing: outcome_stdout(o: table))) + let digest = if outcome_ran(o: table) { listing_digest(d: host_ruleset_digest_wet(listing: outcome_stdout(o: table))) } else { none } HostReading { host: host, generation: generation, ip_forward_on: ip_forward_is_on(readable: proc_readable(o: forward), text: proc_text(o: forward)), - ruleset_listed: outcome_ran(o: table) && digest_present(d: digest), - ruleset_digest: digest_or_zero(d: digest), + ruleset_digest: digest, auto_assignment: parse_auto_assignment( - present: proc_readable(o: helper_sysctl), - readable: proc_readable(o: helper_sysctl), - text: proc_text(o: helper_sysctl), + observed: observe_helper_sysctl_wet(), kernel_release: trim(s: outcome_stdout(o: kernel)), ), - xtables_ct_helper_targets: ct_helper_statement_count(ruleset_listing: outcome_stdout(o: ruleset)), - userspace_helpers: 0, - expectations: expectation_line_count(stdout: outcome_stdout(o: expectations)), - helper_probes_read: outcome_ran(o: ruleset) && outcome_ran(o: expectations) && outcome_ran(o: kernel), - } -} - -// AN EXPECTATION PER LINE, AND AN EMPTY TABLE PRINTS NOTHING. conntrack writes its summary to stderr, -// so stdout carries only entries. -// THE ZERO DIGEST IS NEVER READ, BECAUSE ruleset_listed IS FALSE WHENEVER IT IS USED: host_refusals -// refuses on an unlisted ruleset before converge_outcome reaches the receipt, so this placeholder -// cannot travel into a ConvergedSlotNetwork. It exists because the field is not optional -- a reading -// that failed is refused, not carried as an absent digest a consumer might unwrap. -data unread_ruleset_digest: Sha256Digest = Sha256Digest { - hex: "0000000000000000000000000000000000000000000000000000000000000000" as Sha256DigestHex, -} - -fn digest_present(d: Sha256Digest?) -> Bool { - match d { - Present { value: _ } => true - Absent => false + nft_ct_helper_statements: surface_reading( + o: ruleset, + surface: "nft list ruleset", + count: ct_helper_statement_count(ruleset_listing: outcome_stdout(o: ruleset)), + ), + xtables_ct_helper_targets: surface_reading( + o: xtables, + surface: "iptables-legacy-save", + count: xtables_ct_target_count(save_output: outcome_stdout(o: xtables)), + ), + userspace_helpers: surface_reading( + o: userspace, + surface: "nfct helper list", + count: userspace_helper_count(listing: outcome_stdout(o: userspace)), + ), + expectations: surface_reading( + o: expectations, + surface: "conntrack -L expect", + count: expectation_line_count(stdout: outcome_stdout(o: expectations)), + ), } } -fn digest_or_zero(d: Sha256Digest?) -> Sha256Digest { - match d { - Present { value: v } => v - Absent => unread_ruleset_digest - } +// THE SYSCTL'S PRESENCE AND ITS CONTENT ARE TWO OBSERVATIONS, joined by the filesystem carrier that +// keeps "absent" and "could not look" apart. +fn observe_helper_sysctl_wet() -> FilesystemFileObservation { + let dir = nf_conntrack_sysctl_dir as String + let listed = Filesystem.List(path: dir) + filesystem_file_observation( + listing: filesystem_listing_observation(directory: dir, success: listed.success, entries: listed.entries, error: listed.error), + name: nf_conntrack_helper_sysctl_name as String, + path: nf_conntrack_helper_proc_path as String, + read: read_proc_value(path: nf_conntrack_helper_proc_path as String), + ) } +// AN EXPECTATION PER LINE, AND AN EMPTY TABLE PRINTS NOTHING. conntrack writes its summary to stderr, +// so stdout carries only entries. fn expectation_line_count(stdout: String) -> Int { length(filter(xs: split(s: stdout, delimiter: "\n"), predicate: fn(l) { trim(s: l) != "" })) } @@ -530,22 +585,72 @@ fn refusal_text(r: ConvergeRefusal) -> String { RulesetUnread { reason: why } => join(["the installed ruleset could not be listed: ", why as String], "") HelperStandingUnread { reason: why } => join(["helper standing not established: ", why as String], "") HelperAutoAssignmentOn => "automatic conntrack helper assignment is ON" - HelpersAttached { xtables_targets: x, userspace: u, expectations: e } => - join(["helpers are attached: xtables ", to_string(x), ", userspace ", to_string(u), ", expectations ", to_string(e)], "") + HelpersAttached { surface: surface, count: n } => + join([to_string(n), " on the ", surface as String, " surface"], "") } } +// THE PERSISTED RECEIPT IS THE WHOLE RECORD, NOT A SUMMARY OF IT. This file is what an operator and +// the lifecycle controller read to learn what the converge established, so a rendering that dropped +// the per-slot subjects, their addresses, routes and v6 standing, or the helper readings, would make +// the receipt unable to answer the questions ConvergedSlotNetwork exists to answer (side-chat hold +// on #11751). +fn helper_surface_text(name: String, r: HelperSurfaceReading) -> String { + match r { + HelperSurfaceCounted { count: n } => join([" ", name, "=", to_string(n), "\n"], "") + HelperSurfaceUnread { reason: why } => join([" ", name, "=UNREAD ", why as String, "\n"], "") + } +} + +fn auto_assignment_text(a: HelperAutoAssignment) -> String { + match a { + AutoAssignmentAbsentFromKernel { kernel_release: k } => join([" auto_assignment=absent_from_kernel release=", k as String, "\n"], "") + AutoAssignmentSysctlOff => " auto_assignment=sysctl_off\n" + AutoAssignmentOn => " auto_assignment=ON\n" + AutoAssignmentUnread { reason: why } => join([" auto_assignment=UNREAD ", why as String, "\n"], "") + } +} + +fn tap_ipv6_text(t: TapIpv6Standing) -> String { + match t { + TapIpv6Disabled => "disabled" + TapIpv6Enabled => "ENABLED" + TapIpv6Unread { reason: why } => join(["UNREAD ", why as String], "") + } +} + +fn converged_slot_text(c: ConvergedSlot) -> String { + join([ + "slot ", runner_slot_instance_name(slot: c.subject.slot), "\n", + " tap=", c.subject.tap_device as String, "\n", + " guest_address=", render_ipv4_address(addr: c.subject.guest_address) as String, "\n", + " host_address=", c.host_address_cidr as String, "\n", + " route=", c.guest_route as String, "\n", + " tap_ipv6=", tap_ipv6_text(t: c.tap_ipv6), "\n", + ], "") +} + +fn converged_receipt_text(c: ConvergedSlotNetwork) -> String { + join([ + "microvm-network-converge CONVERGED\n", + "host=", c.host as String, "\n", + "generation=", c.generation as String, "\n", + "nft_digest=", c.nft_digest.hex as String, "\n", + "slots=", to_string(length(c.slots)), "\n", + join(map(c.slots, x => converged_slot_text(c: x)), ""), + "helpers\n", + auto_assignment_text(a: c.helpers.auto_assignment), + helper_surface_text(name: "nft_ct_helper_statements", r: c.helpers.nft_ct_helper_statements), + helper_surface_text(name: "xtables_ct_helper_targets", r: c.helpers.xtables_ct_helper_targets), + helper_surface_text(name: "userspace_helpers", r: c.helpers.userspace_helpers), + helper_surface_text(name: "expectations_at_generation", r: c.helpers.expectations_at_generation), + ], "") +} + fn receipt_text(o: ConvergeOutcome) -> String { match o { ConvergeRefused { refusal: r } => join(["microvm-network-converge REFUSED: ", refusal_text(r: r), "\n"], "") - ConvergedNetwork { receipt: c } => - join([ - "microvm-network-converge CONVERGED\n", - "host=", c.host as String, "\n", - "generation=", c.generation as String, "\n", - "slots=", to_string(length(c.slots)), "\n", - "nft_digest=", c.nft_digest.hex as String, "\n", - ], "") + ConvergedNetwork { receipt: c } => converged_receipt_text(c: c) } } @@ -557,26 +662,77 @@ fn stage_file_wet(f: StagedFile) -> Bool { w.success } +// THE DISPATCH NAMES A HOST AND THE MACHINE NAMES ITSELF, AND THE RUN REFUSES UNLESS THEY AGREE. +// This mode mutates host-global state -- the nft table, forwarding, networkd's links -- so applying +// to "whichever modeled host I turn out to be" means a mislabeled runner dispatched for srv1 can +// mutate srv3 (side-chat hold on #11751). The comparison happens BEFORE any staging write and before +// any privileged operation, so a refusal leaves the host untouched. +type HostBinding + = HostBindingHeld { host: HostIdentity } + | HostBindingRefused { expected: String, observed: String } + | HostBindingUnnamed { detail: String } + +fn bind_requested_host(expected: String, observed: String) -> HostBinding { + if trim(s: expected) == "" { + HostBindingUnnamed { detail: "the dispatch named no expected host" } + } else if trim(s: observed) == "" { + HostBindingUnnamed { detail: "the machine did not name itself" } + } else if trim(s: expected) != trim(s: observed) { + HostBindingRefused { expected: trim(s: expected), observed: trim(s: observed) } + } else { + HostBindingHeld { host: trim(s: observed) as HostIdentity } + } +} + +fn expected_host_reading() -> String { + let env = shell.Env.Get(name: fleet_converge_expected_host_env_name as NonEmptyStr) + match env.value { + Present { value: raw } => trim(s: raw) + Absent => "" + } +} + +fn host_binding_refusal_text(b: HostBinding) -> String { + match b { + HostBindingUnnamed { detail: d } => + join(["microvm-network-converge: HostBindingUnnamed -- ", d, "; nothing was staged and no privileged operation ran"], "") + HostBindingRefused { expected: e, observed: o } => + join(["microvm-network-converge: HostBindingRefused -- expected ", e, ", observed ", o, "; nothing was staged and no privileged operation ran"], "") + HostBindingHeld { host: h } => join(["microvm-network-converge: bound to ", h as String], "") + } +} + fn runner_microvm_network_converge_wet() -> ProcessExit { let host_reading = hostname_short_read(transport: LocalShell) match host_reading { HostnameShortCaptureRefused { reason: r } => exit_failure(reason: join(["microvm-network-converge: the host could not name itself: ", r], "")) - HostnameShortCaptured { value: name } => { + HostnameShortCaptured { value: name } => + match bind_requested_host(expected: expected_host_reading(), observed: name) { + HostBindingUnnamed { detail: d } => + exit_failure(reason: host_binding_refusal_text(b: HostBindingUnnamed { detail: d })) + HostBindingRefused { expected: e, observed: o } => + exit_failure(reason: host_binding_refusal_text(b: HostBindingRefused { expected: e, observed: o })) + HostBindingHeld { host: bound } => converge_bound_host_wet(bound: bound) + } + } +} + +fn converge_bound_host_wet(bound: HostIdentity) -> ProcessExit { + { let generation = converge_generation_reading() if generation == "" { exit_failure(reason: "microvm-network-converge: no run identity in the environment, so the receipt would carry a generation nobody can compare against") } else { - match runner_host_deploy_for(host: name as HostIdentity) { + match runner_host_deploy_for(host: bound) { RunnerHostDeployUnmodeled { host: h } => exit_failure(reason: join(["microvm-network-converge: ", h as String, " is not a modeled runner host"], "")) RunnerHostDeployRefused { host: h, cause: c } => exit_failure(reason: join(["microvm-network-converge: ", h as String, " refuses its deploy row: ", c], "")) RunnerHostDeployFound { deploy: d } => - converge_admitted_host_wet(deploy: d, host: name as HostIdentity, generation: generation as NonEmptyStr) + converge_admitted_host_wet(deploy: d, host: bound, generation: generation as NonEmptyStr) } } - } } } diff --git a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag index 07d33bd294c..bd68ba96b6b 100644 --- a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag @@ -13,8 +13,9 @@ import gunbc.executor_privileged_operation { import gunbc.runner_microvm_network { runner_slot_network, runner_slot_host_cidr, TapIpv6Disabled, TapIpv6Enabled, TapIpv6Unread, - AutoAssignmentAbsentFromKernel, AutoAssignmentOn, AutoAssignmentUnread, + AutoAssignmentAbsentFromKernel, AutoAssignmentOn, AutoAssignmentUnread, HelperAutoAssignment, ConvergedSlotNetwork, ConvergeGeneration, runner_microvm_shared_host_grain, + HelperSurfaceReading, HelperSurfaceCounted, HelperSurfaceUnread, } import gunbc.runner_microvm_network_converge { parse_disable_ipv6, ip_forward_is_on, ct_helper_statement_count, @@ -22,7 +23,7 @@ import gunbc.runner_microvm_network_converge { SlotReading, HostReading, ConvergeOutcome, ConvergedNetwork, ConvergeRefused, ConvergeRefusal, SlotTapIpv6Unread, SlotTapIpv6Enabled, SlotAddressNotHeld, SlotRouteNotHeld, IpForwardingOff, RulesetUnread, HelperStandingUnread, HelperAutoAssignmentOn, HelpersAttached, - converge_outcome, render_slot_subnet, + converge_outcome, } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -37,28 +38,44 @@ data converged_digest: Sha256Digest = Sha256Digest { // A HOST THAT READ BACK CLEAN. Every field is a reading, so a witness that wants a receipt has to // supply every one -- which is the point of the type. +fn counted(n: Int) -> HelperSurfaceReading { + HelperSurfaceCounted { count: n } +} + +fn unread(why: String) -> HelperSurfaceReading { + HelperSurfaceUnread { reason: why as NonEmptyStr } +} + fn clean_host() -> HostReading { - HostReading { - host: operator_host_srv1, - generation: "run-3" as ConvergeGeneration, - ip_forward_on: true, - ruleset_listed: true, - ruleset_digest: converged_digest, - auto_assignment: AutoAssignmentAbsentFromKernel { kernel_release: "6.8.0-138-generic" as NonEmptyStr }, - xtables_ct_helper_targets: 0, - userspace_helpers: 0, - expectations: 0, - helper_probes_read: true, - } + host_with(fwd: true, digest: Present { value: converged_digest }, auto: kernel_absent()) +} + +// THE FIXTURE IS SHAPED LIKE `ip -o -4` OUTPUT AND IS AUTHORED, NOT DERIVED. Building it from the +// same renderer the readback matches with would make producer and oracle share any defect -- which +// is exactly how the earlier host-address-instead-of-network bug survived seven green claims +// (side-chat hold on #11751). These strings are what srv1's kernel prints for slot 1. +fn slot_addr_stdout(i: Nat) -> String { + join(["7: gunbc-tap", to_string(i), " inet 172.30.", to_string(i), ".1/30 scope global gunbc-tap", to_string(i)], "") +} + +fn slot_route_stdout(i: Nat) -> String { + join(["172.30.", to_string(i), ".0/30 proto kernel scope link src 172.30.", to_string(i), ".1 "], "") +} + +fn slot_1_addr_stdout() -> String { + slot_addr_stdout(i: 1) +} + +fn slot_1_route_stdout() -> String { + slot_route_stdout(i: 1) } fn clean_slot(i: Nat) -> SlotReading { - let net = runner_slot_network(slot: slot(i: i)) SlotReading { slot: slot(i: i), tap_ipv6: TapIpv6Disabled, - addr_stdout: join(["7: gunbc-tap", to_string(i), " inet ", runner_slot_host_cidr(net: net), " scope global gunbc-tap", to_string(i)], ""), - route_stdout: join([render_slot_subnet(net: net), " proto kernel scope link src ", runner_slot_host_cidr(net: net)], ""), + addr_stdout: slot_addr_stdout(i: i), + route_stdout: slot_route_stdout(i: i), } } @@ -74,7 +91,7 @@ fn outcome_label(o: ConvergeOutcome) -> String { RulesetUnread { reason: _ } => "ruleset-unread" HelperStandingUnread { reason: _ } => "helpers-unread" HelperAutoAssignmentOn => "auto-helpers-on" - HelpersAttached { xtables_targets: _, userspace: _, expectations: _ } => "helpers-attached" + HelpersAttached { surface: _, count: _ } => "helpers-attached" } } } @@ -135,20 +152,60 @@ test fn an_unread_or_failing_fact_refuses_the_receipt() -> Bool { // THE HOST'S OWN FACTS REFUSE THE SAME WAY, INCLUDING EVERY HELPER ARM. Forwarding off, a ruleset // that did not list, an unread helper probe, automatic assignment on, and an attached helper are // five different remedies and five different refusals. +// ONE BUILDER, SIX OVERRIDES. Each claim below supplies the whole HostReading through it, so a field +// added to the reading breaks every fixture at once rather than defaulting silently. +fn host_reading( + fwd: Bool, + digest: Sha256Digest?, + auto: HelperAutoAssignment, + nft: HelperSurfaceReading, + xtables: HelperSurfaceReading, + userspace: HelperSurfaceReading, + expectations: HelperSurfaceReading, +) -> HostReading { + HostReading { + host: operator_host_srv1, + generation: "run-3" as ConvergeGeneration, + ip_forward_on: fwd, + ruleset_digest: digest, + auto_assignment: auto, + nft_ct_helper_statements: nft, + xtables_ct_helper_targets: xtables, + userspace_helpers: userspace, + expectations: expectations, + } +} + +fn kernel_absent() -> HelperAutoAssignment { + AutoAssignmentAbsentFromKernel { kernel_release: "6.8.0-138-generic" as NonEmptyStr } +} + +fn host_with(fwd: Bool, digest: Sha256Digest?, auto: HelperAutoAssignment) -> HostReading { + host_reading(fwd: fwd, digest: digest, auto: auto, nft: counted(n: 0), xtables: counted(n: 0), userspace: counted(n: 0), expectations: counted(n: 0)) +} + +// THE HOST'S OWN FACTS REFUSE THE SAME WAY. Forwarding off, a ruleset that produced no digestible +// listing, automatic helper assignment on, an unreadable sysctl, and an attached helper on any +// surface are five different remedies and five different refusals. test fn each_host_fact_refuses_under_its_own_name() -> Bool { - let c = clean_host() - let no_fwd = HostReading { host: c.host, generation: c.generation, ip_forward_on: false, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: c.auto_assignment, xtables_ct_helper_targets: c.xtables_ct_helper_targets, userspace_helpers: c.userspace_helpers, expectations: c.expectations, helper_probes_read: c.helper_probes_read } - let unlisted = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: false, ruleset_digest: c.ruleset_digest, auto_assignment: c.auto_assignment, xtables_ct_helper_targets: c.xtables_ct_helper_targets, userspace_helpers: c.userspace_helpers, expectations: c.expectations, helper_probes_read: c.helper_probes_read } - let unread_probe = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: c.auto_assignment, xtables_ct_helper_targets: 0, userspace_helpers: 0, expectations: 0, helper_probes_read: false } - let auto_on = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: AutoAssignmentOn, xtables_ct_helper_targets: 0, userspace_helpers: 0, expectations: 0, helper_probes_read: true } - let unread_auto = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: AutoAssignmentUnread { reason: "sysctl unreadable" as NonEmptyStr }, xtables_ct_helper_targets: 0, userspace_helpers: 0, expectations: 0, helper_probes_read: true } - let attached = HostReading { host: c.host, generation: c.generation, ip_forward_on: c.ip_forward_on, ruleset_listed: c.ruleset_listed, ruleset_digest: c.ruleset_digest, auto_assignment: c.auto_assignment, xtables_ct_helper_targets: 0, userspace_helpers: 0, expectations: 2, helper_probes_read: true } - outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: no_fwd)) == "no-forwarding" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: unlisted)) == "ruleset-unread" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: unread_probe)) == "helpers-unread" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: auto_on)) == "auto-helpers-on" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: unread_auto)) == "helpers-unread" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: attached)) == "helpers-attached" + outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: false, digest: Present { value: converged_digest }, auto: kernel_absent()))) == "no-forwarding" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: true, digest: none, auto: kernel_absent()))) == "ruleset-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: true, digest: Present { value: converged_digest }, auto: AutoAssignmentOn))) == "auto-helpers-on" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: true, digest: Present { value: converged_digest }, auto: AutoAssignmentUnread { reason: "sysctl unreadable" as NonEmptyStr }))) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: Present { value: converged_digest }, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 2), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-attached" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: Present { value: converged_digest }, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: counted(n: 1), expectations: counted(n: 0)))) == "helpers-attached" +} + +// A SURFACE NOBODY READ IS NOT AN EMPTY ONE. Each of the three helper surfaces and the expectation +// table, left unread, refuses the receipt. The red is the previous converge, which hard-coded two of +// them to zero and let helpers_excluded return true off those constants (review 68963): a host with +// a legacy CT target or a userspace helper converged green. +test fn an_unread_helper_surface_refuses_rather_than_counting_zero() -> Bool { + let d = Present { value: converged_digest } + outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: d, auto: kernel_absent(), nft: unread(why: "probe exited 1"), xtables: counted(n: 0), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: unread(why: "iptables-legacy-save missing"), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: unread(why: "nfct missing"), expectations: counted(n: 0)))) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: counted(n: 0), expectations: unread(why: "conntrack refused")))) == "helpers-unread" } // A HELPER ATTACHED BY ANY TABLE COUNTS, because an expectation is created by whichever helper is @@ -205,3 +262,18 @@ test fn every_installed_file_is_staged_with_its_rendered_bytes() -> Bool { ] && fold(staged, init: true, f: (acc, x) => acc && x.content != "") } + +// THE ROUTE READBACK MATCHES THE NETWORK THE KERNEL PRINTS, AND REFUSES THE HOST ADDRESS. `ip route +// show dev tap` prints the destination NETWORK with the host address as `src`; a readback matching +// 172.30.1.1/30 never hits, so a perfectly configured tap would refuse. The reds: real kernel output +// judged not-held, and a fixture carrying only the host-address form judged held. +test fn the_route_readback_accepts_the_network_and_refuses_the_host_address() -> Bool { + let host_address_form = SlotReading { + slot: slot(i: 1), + tap_ipv6: TapIpv6Disabled, + addr_stdout: slot_1_addr_stdout(), + route_stdout: "172.30.1.1/30 proto kernel scope link ", + } + outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: clean_host())) == "converged" + && outcome_label(o: converge_outcome(slots: [host_address_form], h: clean_host())) == "no-route" +} diff --git a/dag/test/claim/runner/runner_microvm_network_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_witness_test.dag index 27ae2d87075..3b8ec70fbad 100644 --- a/dag/test/claim/runner/runner_microvm_network_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_witness_test.dag @@ -20,6 +20,7 @@ import gunbc.runner_microvm_network { RulesetMatchesConvergedFact, SlotNetworkSubject, slot_network_subject, runner_slot_host_cidr, ConvergedSlot, ConvergedSlotNetwork, ConvergeGeneration, HelperStanding, + HelperSurfaceReading, HelperSurfaceCounted, HelperSurfaceUnread, AutoAssignmentAbsentFromKernel, AutoAssignmentSysctlOff, AutoAssignmentOn, TapIpv6Standing, TapIpv6Disabled, TapIpv6Enabled, SanitationSubjectResult, SanitationSubjectAdmitted, SanitationSubjectRefused, @@ -270,10 +271,10 @@ fn converged_slot(i: Nat) -> ConvergedSlot { fn excluded_helpers() -> HelperStanding { HelperStanding { auto_assignment: AutoAssignmentAbsentFromKernel { kernel_release: "6.8.0-138-generic" as NonEmptyStr }, - nft_ct_helper_statements: 0, - xtables_ct_helper_targets: 0, - userspace_helpers: 0, - expectations_at_generation: 0, + nft_ct_helper_statements: HelperSurfaceCounted { count: 0 }, + xtables_ct_helper_targets: HelperSurfaceCounted { count: 0 }, + userspace_helpers: HelperSurfaceCounted { count: 0 }, + expectations_at_generation: HelperSurfaceCounted { count: 0 }, } } @@ -318,20 +319,20 @@ test fn the_sanitation_subject_is_only_the_slots_own_hosts_established_receipt() slot: slot(i: 1), receipt: receipt_on(host: operator_host_srv2, slots: [converged_slot(i: 1)], helpers: HelperStanding { auto_assignment: AutoAssignmentSysctlOff, - nft_ct_helper_statements: 1, - xtables_ct_helper_targets: 0, - userspace_helpers: 0, - expectations_at_generation: 0, + nft_ct_helper_statements: HelperSurfaceCounted { count: 1 }, + xtables_ct_helper_targets: HelperSurfaceCounted { count: 0 }, + userspace_helpers: HelperSurfaceCounted { count: 0 }, + expectations_at_generation: HelperSurfaceCounted { count: 0 }, }), )) == "helpers" && admission(r: slot_network_sanitation_subject( slot: slot(i: 1), receipt: receipt_on(host: operator_host_srv2, slots: [converged_slot(i: 1)], helpers: HelperStanding { auto_assignment: AutoAssignmentOn, - nft_ct_helper_statements: 0, - xtables_ct_helper_targets: 0, - userspace_helpers: 0, - expectations_at_generation: 0, + nft_ct_helper_statements: HelperSurfaceCounted { count: 0 }, + xtables_ct_helper_targets: HelperSurfaceCounted { count: 0 }, + userspace_helpers: HelperSurfaceCounted { count: 0 }, + expectations_at_generation: HelperSurfaceCounted { count: 0 }, }), )) == "helpers" } @@ -612,3 +613,25 @@ test fn a_slots_install_set_carries_every_network_file_it_declares() -> Bool { "reload systemd-networkd", ] } + +// AN UNREAD SURFACE IS NOT AN EXCLUDED ONE, SO THE SANITATION SUBJECT REFUSES. The sealed subject +// requires helpers_excluded, and a surface that answered nothing cannot satisfy it -- otherwise a +// host whose xtables or userspace probe never ran would admit slots as if no helper were attached +// (review 68963). +test fn a_subject_refuses_when_a_helper_surface_was_unread() -> Bool { + let unread_xtables = HelperStanding { + auto_assignment: AutoAssignmentAbsentFromKernel { kernel_release: "6.8.0-138-generic" as NonEmptyStr }, + nft_ct_helper_statements: HelperSurfaceCounted { count: 0 }, + xtables_ct_helper_targets: HelperSurfaceUnread { reason: "iptables-legacy-save is not installed" as NonEmptyStr }, + userspace_helpers: HelperSurfaceCounted { count: 0 }, + expectations_at_generation: HelperSurfaceCounted { count: 0 }, + } + admission(r: slot_network_sanitation_subject( + slot: slot(i: 1), + receipt: receipt_on(host: operator_host_srv2, slots: [converged_slot(i: 1)], helpers: unread_xtables), + )) == "helpers" + && admission(r: slot_network_sanitation_subject( + slot: slot(i: 1), + receipt: receipt_on(host: operator_host_srv2, slots: [converged_slot(i: 1)], helpers: excluded_helpers()), + )) == "admitted" +} From 68021c17d4ba97f9c8d5965c952ce500831212c8 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 08:08:33 +0000 Subject: [PATCH 22/36] Regenerate the sudoers rosters for the legacy xtables and userspace helper probes Co-Authored-By: Claude Opus 5 (1M context) --- provisioning/srv1/gunbc-ghrunner.sudoers | 2 ++ provisioning/srv3/gunbc-ghrunner.sudoers | 2 ++ provisioning/srv4/gunbc-ghrunner.sudoers | 2 ++ 3 files changed, 6 insertions(+) diff --git a/provisioning/srv1/gunbc-ghrunner.sudoers b/provisioning/srv1/gunbc-ghrunner.sudoers index a1ffc5565a8..eb2a97db8af 100644 --- a/provisioning/srv1/gunbc-ghrunner.sudoers +++ b/provisioning/srv1/gunbc-ghrunner.sudoers @@ -182,4 +182,6 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.servic ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset +ghrunner ALL=(root) NOPASSWD: /usr/sbin/iptables-legacy-save +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nfct helper list ghrunner ALL=(root) NOPASSWD: /usr/sbin/conntrack -L expect diff --git a/provisioning/srv3/gunbc-ghrunner.sudoers b/provisioning/srv3/gunbc-ghrunner.sudoers index a4458a1dc24..c70480c4df8 100644 --- a/provisioning/srv3/gunbc-ghrunner.sudoers +++ b/provisioning/srv3/gunbc-ghrunner.sudoers @@ -205,4 +205,6 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.servic ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset +ghrunner ALL=(root) NOPASSWD: /usr/sbin/iptables-legacy-save +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nfct helper list ghrunner ALL=(root) NOPASSWD: /usr/sbin/conntrack -L expect diff --git a/provisioning/srv4/gunbc-ghrunner.sudoers b/provisioning/srv4/gunbc-ghrunner.sudoers index 7d207a42d64..813bbf39bda 100644 --- a/provisioning/srv4/gunbc-ghrunner.sudoers +++ b/provisioning/srv4/gunbc-ghrunner.sudoers @@ -196,4 +196,6 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.servic ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset +ghrunner ALL=(root) NOPASSWD: /usr/sbin/iptables-legacy-save +ghrunner ALL=(root) NOPASSWD: /usr/sbin/nfct helper list ghrunner ALL=(root) NOPASSWD: /usr/sbin/conntrack -L expect From d1b7d8ef3a6e2b62ccda589f4e1864efbf299f72 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 08:48:28 +0000 Subject: [PATCH 23/36] An absent helper sysctl names the kernel it is absent from, or it is unread Co-Authored-By: Claude Opus 5 (1M context) --- .../runner_microvm_network_converge.dag | 20 ++++++++++++++++-- ..._microvm_network_converge_witness_test.dag | 21 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/dag/gunbc/runner/runner_microvm_network_converge.dag b/dag/gunbc/runner/runner_microvm_network_converge.dag index 5e7857321b5..cec6129ac74 100644 --- a/dag/gunbc/runner/runner_microvm_network_converge.dag +++ b/dag/gunbc/runner/runner_microvm_network_converge.dag @@ -218,9 +218,24 @@ fn route_reading_carries(stdout: String, cidr: String) -> Bool { // dropped automatic helper assignment entirely, so on a kernel that has no such sysctl there is no // setting to get wrong; on an older one the value must read 0. A file that exists and reads 1 is // AutoAssignmentOn, and an unreadable probe is neither -- it is unread, and refuses the receipt. -fn parse_auto_assignment(observed: FilesystemFileObservation, kernel_release: String) -> HelperAutoAssignment { +// ABSENCE IS ATTRIBUTED TO A KERNEL, SO THE KERNEL MUST HAVE BEEN READ. AbsentFromKernel is the +// strongest reading this type offers -- it says there is no setting left to turn back on -- and it +// names the release that lacks it. A `uname -r` that did not answer leaves that name empty, so the +// receipt would assert the strongest arm against no kernel at all (review 69021). Every sibling +// probe here carries its own unread arm; this one now does too. +fn absent_from_kernel_or_unread(kernel_release: String, kernel_read: Bool) -> HelperAutoAssignment { + if !kernel_read { + AutoAssignmentUnread { reason: "nf_conntrack_helper is absent, but uname -r did not answer, so the absence names no kernel" as NonEmptyStr } + } else if trim(s: kernel_release) == "" { + AutoAssignmentUnread { reason: "nf_conntrack_helper is absent, but uname -r printed no release, so the absence names no kernel" as NonEmptyStr } + } else { + AutoAssignmentAbsentFromKernel { kernel_release: trim(s: kernel_release) as NonEmptyStr } + } +} + +fn parse_auto_assignment(observed: FilesystemFileObservation, kernel_release: String, kernel_read: Bool) -> HelperAutoAssignment { match observed { - FilesystemFileAbsent(_) => AutoAssignmentAbsentFromKernel { kernel_release: kernel_release as NonEmptyStr } + FilesystemFileAbsent(_) => absent_from_kernel_or_unread(kernel_release: kernel_release, kernel_read: kernel_read) FilesystemFileIndeterminate { cause: c } => AutoAssignmentUnread { reason: join(["nf_conntrack_helper could not be observed: ", c], "") as NonEmptyStr } FilesystemFileObservationsDisagree { path: _, cause: c } => AutoAssignmentUnread { reason: join(["nf_conntrack_helper moved while it was observed: ", c], "") as NonEmptyStr } FilesystemFileSubjectRefused { directory: _, name: _, cause: c } => AutoAssignmentUnread { reason: join(["nf_conntrack_helper path refused: ", c], "") as NonEmptyStr } @@ -513,6 +528,7 @@ fn read_host_wet(host: HostIdentity, generation: ConvergeGeneration) -> HostRead auto_assignment: parse_auto_assignment( observed: observe_helper_sysctl_wet(), kernel_release: trim(s: outcome_stdout(o: kernel)), + kernel_read: outcome_ran(o: kernel), ), nft_ct_helper_statements: surface_reading( o: ruleset, diff --git a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag index bd68ba96b6b..f000089f64d 100644 --- a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag @@ -23,7 +23,7 @@ import gunbc.runner_microvm_network_converge { SlotReading, HostReading, ConvergeOutcome, ConvergedNetwork, ConvergeRefused, ConvergeRefusal, SlotTapIpv6Unread, SlotTapIpv6Enabled, SlotAddressNotHeld, SlotRouteNotHeld, IpForwardingOff, RulesetUnread, HelperStandingUnread, HelperAutoAssignmentOn, HelpersAttached, - converge_outcome, + converge_outcome, absent_from_kernel_or_unread, } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -277,3 +277,22 @@ test fn the_route_readback_accepts_the_network_and_refuses_the_host_address() -> outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: clean_host())) == "converged" && outcome_label(o: converge_outcome(slots: [host_address_form], h: clean_host())) == "no-route" } + +// AN ABSENT SYSCTL NAMES THE KERNEL IT IS ABSENT FROM, OR IT IS UNREAD. AbsentFromKernel is the +// strongest helper reading -- there is no setting left to turn back on -- so asserting it with no +// release named would be the strongest claim on the weakest evidence. The reds: a refused `uname -r` +// reported as absent-from-kernel, and an empty release string accepted as one (review 69021). +test fn an_absent_helper_sysctl_names_its_kernel_or_is_unread() -> Bool { + match absent_from_kernel_or_unread(kernel_release: "6.8.0-138-generic", kernel_read: true) { + AutoAssignmentAbsentFromKernel { kernel_release: k } => (k as String) == "6.8.0-138-generic" + _ => false + } + && match absent_from_kernel_or_unread(kernel_release: "", kernel_read: false) { + AutoAssignmentUnread { reason: _ } => true + _ => false + } + && match absent_from_kernel_or_unread(kernel_release: "", kernel_read: true) { + AutoAssignmentUnread { reason: _ } => true + _ => false + } +} From 58a70a4b6cb44f165358dd4c5360171e9ce0fea5 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 09:36:51 +0000 Subject: [PATCH 24/36] Supply the dispatched host to the step, reclassify the local mode, make the receipt a parseable value, and give ip_forward and the slot probes their own unread arms Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/fleet/fleet_converge_workflow.dag | 8 +- dag/gunbc/runner/runner_microvm_network.dag | 323 +++++++++++++++++- .../runner_microvm_network_converge.dag | 98 +++++- ..._microvm_network_converge_witness_test.dag | 136 ++++++-- .../workflow_dispatch_input_witness_test.dag | 51 +++ 5 files changed, 566 insertions(+), 50 deletions(-) diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index 2011ab041d8..140d091556b 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -330,7 +330,7 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> FleetApply => FleetSshKeyConsumed AppControlPlaneObserve => FleetSshKeyConsumed MicrovmHostConverge => FleetSshKeyConsumed - MicrovmNetworkConverge => FleetSshKeyConsumed + MicrovmNetworkConverge => FleetSshKeyNotConsumed GuestImageObserve => FleetSshKeyConsumed GuestImageConverge => FleetSshKeyConsumed MicrovmBootProbe => FleetSshKeyConsumed @@ -1007,7 +1007,11 @@ fn fleet_converge_microvm_network_converge_step() -> Step { id: Present { value: "microvm_network_converge" }, run: gunbc_ci_runner_microvm_network_converge_invoke(), shell: none, - env: none, + env: Present { + value: [ + kv(key: fleet_converge_expected_host_env_name, value: yaml_string(s: "${{ github.event.inputs.host }}")), + ] + }, working_directory: none, if_condition: Present { value: fleet_converge_microvm_network_converge_step_if }, continue_on_error: none, diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index e313122d354..611de9218c9 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -1,13 +1,14 @@ module gunbc.runner_microvm_network import std.types { String, NonEmptyStr, Bool, Int, List, Port } +import std.algebra { trim } import std.nat { Nat } import std.disposition { Disposition, SingleAuthority } import std.dissolution { DissolutionCondition, unbound_dissolution } -import std.content_hash { Sha256Digest, content_hash_eq_cryptographic } +import std.content_hash { Sha256Digest, sha256_hex_digest, content_hash_eq_cryptographic } import extdeps.network.ipv4 { Ipv4Address, ipv4_address, render_ipv4_address, render_ipv4_cidr, Ipv4Prefix, ipv4_prefix, render_ipv4_prefix, ipv4_address_eq } import extdeps.network.mac { Eui48Address, mac_address, render_mac_address, MacAddressScope, LocallyAdministered, mac_address_scope } -import gunbc.build_cache_instance { RunnerSlotIdentity, runner_slot_identity_equal } +import gunbc.build_cache_instance { RunnerSlotIdentity, runner_slot_identity_equal, runner_slot_instance_name } import product.placement_supply { HostIdentity } import gunbc.runner_microvm { RunnerMicroVmNetworking, GuestNetworkTap } import gunbc.executor_privileged_operation { @@ -1359,3 +1360,321 @@ data slot_network_readback_consumer_frontier: DissolutionCondition = unbound_dis data guest_resolver_configuration_frontier: DissolutionCondition = unbound_dissolution( description: "NAMED CONSUMER: gunbc.runner_guest_image resolver configuration. TRIGGER: the guest image makes /etc/resolv.conf follow the booted dns0 (/proc/net/pnp) and a booted guest resolves github.com through runner_microvm_guest_resolver. SUFFICIENT FOR deleting this row. NOT satisfied by the ip= boot argument carrying dns0 alone." as NonEmptyStr, ) + +// --- the receipt as a wire value: what a later process reads back --- + +// A RECEIPT NOBODY CAN PARSE IS A REPORT, NOT A RECEIPT. The converge builds a ConvergedSlotNetwork +// in memory and writes a file; the lifecycle controller runs later, in another process, and needs +// the VALUE -- slot_network_sanitation_subject takes a receipt, not prose (side-chat hold on #11751). +// So the file has a schema line and a total parse back, and the two live beside the type they carry +// rather than in either consumer. +// +// THE DERIVABLE FIELDS ARE VERIFIED, NOT TRUSTED. A slot's tap, guest address and MAC follow from its +// identity, so the parse derives them and REFUSES when the wire disagrees: a receipt describing a +// tap this fleet would never mint is a receipt about something else. +data converged_slot_network_wire_schema: NonEmptyStr = "gunbc.converged_slot_network.v1" as NonEmptyStr + +fn helper_surface_wire(r: HelperSurfaceReading) -> String { + match r { + HelperSurfaceCounted { count: n } => join(["counted:", to_string(n)], "") + HelperSurfaceUnread { reason: why } => join(["unread:", why as String], "") + } +} + +fn auto_assignment_wire(a: HelperAutoAssignment) -> String { + match a { + AutoAssignmentAbsentFromKernel { kernel_release: k } => join(["absent_from_kernel:", k as String], "") + AutoAssignmentSysctlOff => "sysctl_off" + AutoAssignmentOn => "on" + AutoAssignmentUnread { reason: why } => join(["unread:", why as String], "") + } +} + +fn tap_ipv6_wire(t: TapIpv6Standing) -> String { + match t { + TapIpv6Disabled => "disabled" + TapIpv6Enabled => "enabled" + TapIpv6Unread { reason: why } => join(["unread:", why as String], "") + } +} + +fn converged_slot_wire(c: ConvergedSlot) -> String { + join([ + "slot=", runner_slot_instance_name(slot: c.subject.slot), + "|tap=", c.subject.tap_device as String, + "|guest=", render_ipv4_address(addr: c.subject.guest_address) as String, + "|mac=", render_mac_address(addr: c.subject.guest_mac) as String, + "|host_cidr=", c.host_address_cidr as String, + "|route=", c.guest_route as String, + "|ipv6=", tap_ipv6_wire(t: c.tap_ipv6), + ], "") +} + +fn render_converged_slot_network(c: ConvergedSlotNetwork) -> String { + join([ + converged_slot_network_wire_schema as String, "\n", + "host=", c.host as String, "\n", + "generation=", c.generation as String, "\n", + "nft_digest=", c.nft_digest.hex as String, "\n", + "helper_auto=", auto_assignment_wire(a: c.helpers.auto_assignment), "\n", + "helper_nft=", helper_surface_wire(r: c.helpers.nft_ct_helper_statements), "\n", + "helper_xtables=", helper_surface_wire(r: c.helpers.xtables_ct_helper_targets), "\n", + "helper_userspace=", helper_surface_wire(r: c.helpers.userspace_helpers), "\n", + "helper_expectations=", helper_surface_wire(r: c.helpers.expectations_at_generation), "\n", + join(map(c.slots, x => join([converged_slot_wire(c: x), "\n"], "")), ""), + ], "") +} + +type ConvergedSlotNetworkParse + = ConvergedSlotNetworkParsed { receipt: ConvergedSlotNetwork } + | ConvergedSlotNetworkMalformed { cause: NonEmptyStr } + +fn wire_lines(text: String) -> List { + filter(xs: split(s: text, delimiter: "\n"), predicate: fn(l) { trim(s: l) != "" }) +} + +fn wire_value(lines: List, key: String) -> String? { + fold(lines, init: none, f: (acc, l) => + match acc { + Present { value: _ } => acc + Absent => + if trim(s: l).starts_with(prefix: join([key, "="], "")) { + Present { value: drop_prefix(s: trim(s: l), prefix: join([key, "="], "")) } + } else { + acc + } + } + ) +} + +fn drop_prefix(s: String, prefix: String) -> String { + join(skip(xs: split(s: s, delimiter: prefix), n: 1), prefix) +} + +fn parse_helper_surface(raw: String) -> HelperSurfaceReading? { + if raw.starts_with(prefix: "counted:") { + match parse_int(s: drop_prefix(s: raw, prefix: "counted:")) { + Present { value: n } => Present { value: HelperSurfaceCounted { count: n } } + Absent => none + } + } else if raw.starts_with(prefix: "unread:") { + Present { value: HelperSurfaceUnread { reason: drop_prefix(s: raw, prefix: "unread:") as NonEmptyStr } } + } else { + none + } +} + +fn parse_auto_assignment_wire(raw: String) -> HelperAutoAssignment? { + if raw.starts_with(prefix: "absent_from_kernel:") { + Present { value: AutoAssignmentAbsentFromKernel { kernel_release: drop_prefix(s: raw, prefix: "absent_from_kernel:") as NonEmptyStr } } + } else if raw == "sysctl_off" { + Present { value: AutoAssignmentSysctlOff } + } else if raw == "on" { + Present { value: AutoAssignmentOn } + } else if raw.starts_with(prefix: "unread:") { + Present { value: AutoAssignmentUnread { reason: drop_prefix(s: raw, prefix: "unread:") as NonEmptyStr } } + } else { + none + } +} + +fn parse_tap_ipv6_wire(raw: String) -> TapIpv6Standing? { + if raw == "disabled" { + Present { value: TapIpv6Disabled } + } else if raw == "enabled" { + Present { value: TapIpv6Enabled } + } else if raw.starts_with(prefix: "unread:") { + Present { value: TapIpv6Unread { reason: drop_prefix(s: raw, prefix: "unread:") as NonEmptyStr } } + } else { + none + } +} + +fn slot_field(parts: List, key: String) -> String? { + wire_value(lines: parts, key: key) +} + +// THE SLOT IS FOUND IN THE CANDIDATE ROSTER, NOT MINTED FROM THE TEXT. A receipt naming a slot this +// host does not have is a receipt about another fleet, and deriving the identity from a parsed index +// would let the file invent one. +fn candidate_named(candidates: List, name: String) -> RunnerSlotIdentity? { + fold(candidates, init: none, f: (acc, c) => + match acc { + Present { value: _ } => acc + Absent => if runner_slot_instance_name(slot: c) == name { Present { value: c } } else { acc } + } + ) +} + +fn parse_converged_slot_line(line: String, candidates: List) -> ConvergedSlot? { + let parts = split(s: trim(s: line), delimiter: "|") + match slot_field(parts: parts, key: "slot") { + Absent => none + Present { value: name } => + match candidate_named(candidates: candidates, name: name) { + Absent => none + Present { value: identity } => { + let derived = slot_network_subject(slot: identity) + match slot_field(parts: parts, key: "tap") { + Absent => none + Present { value: tap } => + match slot_field(parts: parts, key: "guest") { + Absent => none + Present { value: guest } => + match slot_field(parts: parts, key: "mac") { + Absent => none + Present { value: mac } => + if (tap != (derived.tap_device as String)) + || (guest != (render_ipv4_address(addr: derived.guest_address) as String)) + || (mac != (render_mac_address(addr: derived.guest_mac) as String)) { + none + } else { + parse_converged_slot_tail(parts: parts, derived: derived) + } + } + } + } + } + } + } +} + +fn parse_converged_slot_tail(parts: List, derived: SlotNetworkSubject) -> ConvergedSlot? { + match slot_field(parts: parts, key: "host_cidr") { + Absent => none + Present { value: cidr } => + match slot_field(parts: parts, key: "route") { + Absent => none + Present { value: route } => + match slot_field(parts: parts, key: "ipv6") { + Absent => none + Present { value: v6raw } => + match parse_tap_ipv6_wire(raw: v6raw) { + Absent => none + Present { value: v6 } => + Present { value: ConvergedSlot { + subject: derived, + host_address_cidr: cidr as NonEmptyStr, + guest_route: route as NonEmptyStr, + tap_ipv6: v6, + } } + } + } + } + } +} + +// EVERY FIELD OR NOTHING. A receipt missing its schema line, its host, its generation, its digest, +// any helper reading, or carrying a slot line this host cannot own, is malformed -- the consumer +// gets a typed refusal naming the field rather than a partially populated value. +fn parse_converged_slot_network(text: String, candidates: List) -> ConvergedSlotNetworkParse { + let lines = wire_lines(text: text) + match lines.first() { + Absent => ConvergedSlotNetworkMalformed { cause: "the receipt is empty" as NonEmptyStr } + Present { value: schema } => + if trim(s: schema) != (converged_slot_network_wire_schema as String) { + ConvergedSlotNetworkMalformed { cause: "the first line is not this receipt's schema" as NonEmptyStr } + } else { + parse_converged_body(lines: lines, candidates: candidates) + } + } +} + +fn parse_converged_body(lines: List, candidates: List) -> ConvergedSlotNetworkParse { + match wire_value(lines: lines, key: "host") { + Absent => ConvergedSlotNetworkMalformed { cause: "no host= line" as NonEmptyStr } + Present { value: host } => + match wire_value(lines: lines, key: "generation") { + Absent => ConvergedSlotNetworkMalformed { cause: "no generation= line" as NonEmptyStr } + Present { value: generation } => + match wire_value(lines: lines, key: "nft_digest") { + Absent => ConvergedSlotNetworkMalformed { cause: "no nft_digest= line" as NonEmptyStr } + Present { value: hex } => + match sha256_hex_digest(hex: hex) { + Absent => ConvergedSlotNetworkMalformed { cause: "nft_digest is not a sha256 hex digest" as NonEmptyStr } + Present { value: digest } => + match parse_helper_standing(lines: lines) { + Absent => ConvergedSlotNetworkMalformed { cause: "a helper reading is missing or malformed" as NonEmptyStr } + Present { value: helpers } => + parse_converged_slots(lines: lines, candidates: candidates, host: host, generation: generation, digest: digest, helpers: helpers) + } + } + } + } + } +} + +fn parse_helper_standing(lines: List) -> HelperStanding? { + match wire_value(lines: lines, key: "helper_auto") { + Absent => none + Present { value: autoraw } => + match parse_auto_assignment_wire(raw: autoraw) { + Absent => none + Present { value: auto } => + match parse_helper_surface_at(lines: lines, key: "helper_nft") { + Absent => none + Present { value: nft } => + match parse_helper_surface_at(lines: lines, key: "helper_xtables") { + Absent => none + Present { value: xt } => + match parse_helper_surface_at(lines: lines, key: "helper_userspace") { + Absent => none + Present { value: us } => + match parse_helper_surface_at(lines: lines, key: "helper_expectations") { + Absent => none + Present { value: ex } => + Present { value: HelperStanding { + auto_assignment: auto, + nft_ct_helper_statements: nft, + xtables_ct_helper_targets: xt, + userspace_helpers: us, + expectations_at_generation: ex, + } } + } + } + } + } + } + } +} + +fn parse_helper_surface_at(lines: List, key: String) -> HelperSurfaceReading? { + match wire_value(lines: lines, key: key) { + Absent => none + Present { value: raw } => parse_helper_surface(raw: raw) + } +} + +fn slot_lines(lines: List) -> List { + filter(xs: lines, predicate: fn(l) { trim(s: l).starts_with(prefix: "slot=") }) +} + +fn parse_converged_slots( + lines: List, + candidates: List, + host: String, + generation: String, + digest: Sha256Digest, + helpers: HelperStanding, +) -> ConvergedSlotNetworkParse { + let raw = slot_lines(lines: lines) + let parsed = fold(raw, init: [], f: (acc, l) => + match parse_converged_slot_line(line: l, candidates: candidates) { + Present { value: c } => concat(acc, [c]) + Absent => acc + } + ) + if length(parsed) != length(raw) { + ConvergedSlotNetworkMalformed { cause: "a slot line named a slot this host does not own, or disagreed with the address this fleet derives for it" as NonEmptyStr } + } else { + ConvergedSlotNetworkParsed { + receipt: ConvergedSlotNetwork { + host: host as HostIdentity, + generation: generation as ConvergeGeneration, + slots: parsed, + helpers: helpers, + nft_digest: digest, + }, + } + } +} diff --git a/dag/gunbc/runner/runner_microvm_network_converge.dag b/dag/gunbc/runner/runner_microvm_network_converge.dag index cec6129ac74..e0aee191c3c 100644 --- a/dag/gunbc/runner/runner_microvm_network_converge.dag +++ b/dag/gunbc/runner/runner_microvm_network_converge.dag @@ -53,6 +53,7 @@ import gunbc.runner_microvm_network { AutoAssignmentAbsentFromKernel, AutoAssignmentSysctlOff, AutoAssignmentOn, AutoAssignmentUnread, TapIpv6Standing, TapIpv6Disabled, TapIpv6Enabled, TapIpv6Unread, slot_network_subject, runner_microvm_nft_read_operation, runner_slot_subnet, + render_converged_slot_network, } data runner_microvm_network_converge_disposition: Disposition = SingleAuthority @@ -199,8 +200,38 @@ fn parse_disable_ipv6(readable: Bool, text: String) -> TapIpv6Standing { } } -fn ip_forward_is_on(readable: Bool, text: String) -> Bool { - readable && trim(s: text) == "1" +// FORWARDING IS ON, OFF, OR UNREAD -- A Bool CANNOT SAY "I DID NOT LOOK". A failed read folding into +// false made the refusal say "net.ipv4.ip_forward reads 0", a value nobody read (review 69041). It +// failed closed, but it sent an operator to turn on a setting that may already be on. +type IpForwardStanding + = IpForwardOn + | IpForwardOff + | IpForwardUnread { reason: NonEmptyStr } + +fn ip_forward_standing(readable: Bool, text: String) -> IpForwardStanding { + if !readable { + IpForwardUnread { reason: "net.ipv4.ip_forward could not be read" as NonEmptyStr } + } else if trim(s: text) == "1" { + IpForwardOn + } else if trim(s: text) == "0" { + IpForwardOff + } else { + IpForwardUnread { reason: join(["net.ipv4.ip_forward read back as an unexpected value: ", trim(s: text)], "") as NonEmptyStr } + } +} + +// A SLOT PROBE CARRIES ITS OWN ABSENCE FOR THE SAME REASON. A refused `ip -o -4 addr show` produced +// an empty string, which read as "the tap holds no address" -- so an operator was sent to fix a tap +// when the probe was what failed (review 69041). +type SlotProbeReading + = SlotProbeRead { text: String } + | SlotProbeUnread { reason: NonEmptyStr } + +fn slot_probe_text(r: SlotProbeReading) -> String { + match r { + SlotProbeRead { text: t } => t + SlotProbeUnread { reason: _ } => "" + } } // THE HOST'S OWN ADDRESS ON THE TAP, AND THE ROUTE THROUGH IT, ARE READ FROM THE KERNEL rather than @@ -292,6 +323,9 @@ type ConvergeRefusal | SlotAddressNotHeld { slot: RunnerSlotIdentity, expected: NonEmptyStr } | SlotRouteNotHeld { slot: RunnerSlotIdentity, expected: NonEmptyStr } | IpForwardingOff + | IpForwardingUnread { reason: NonEmptyStr } + | SlotAddressUnread { slot: RunnerSlotIdentity, reason: NonEmptyStr } + | SlotRouteUnread { slot: RunnerSlotIdentity, reason: NonEmptyStr } | RulesetUnread { reason: NonEmptyStr } | HelperStandingUnread { reason: NonEmptyStr } | HelperAutoAssignmentOn @@ -305,14 +339,14 @@ type ConvergeOutcome type SlotReading { slot: RunnerSlotIdentity tap_ipv6: TapIpv6Standing - addr_stdout: String - route_stdout: String + addr: SlotProbeReading + route: SlotProbeReading } type HostReading { host: HostIdentity generation: ConvergeGeneration - ip_forward_on: Bool + ip_forward: IpForwardStanding ruleset_digest: Sha256Digest? auto_assignment: HelperAutoAssignment nft_ct_helper_statements: HelperSurfaceReading @@ -326,7 +360,7 @@ fn converged_slot_of_reading(r: SlotReading) -> ConvergedSlot { ConvergedSlot { subject: slot_network_subject(slot: r.slot), host_address_cidr: runner_slot_host_cidr(net: net) as NonEmptyStr, - guest_route: trim(s: r.route_stdout) as NonEmptyStr, + guest_route: trim(s: slot_probe_text(r: r.route)) as NonEmptyStr, tap_ipv6: r.tap_ipv6, } } @@ -341,12 +375,22 @@ fn slot_refusal(r: SlotReading) -> List { TapIpv6Unread { reason: _ } => [SlotTapIpv6Unread { slot: r.slot, standing: r.tap_ipv6 }] TapIpv6Enabled => [SlotTapIpv6Enabled { slot: r.slot }] TapIpv6Disabled => - if !addr_reading_carries(stdout: r.addr_stdout, cidr: cidr) { - [SlotAddressNotHeld { slot: r.slot, expected: cidr as NonEmptyStr }] - } else if !route_reading_carries(stdout: r.route_stdout, cidr: render_ipv4_prefix(p: runner_slot_subnet(net: net)) as String) { - [SlotRouteNotHeld { slot: r.slot, expected: render_ipv4_prefix(p: runner_slot_subnet(net: net)) }] - } else { - [] + match r.addr { + SlotProbeUnread { reason: why } => [SlotAddressUnread { slot: r.slot, reason: why }] + SlotProbeRead { text: addr_text } => + if !addr_reading_carries(stdout: addr_text, cidr: cidr) { + [SlotAddressNotHeld { slot: r.slot, expected: cidr as NonEmptyStr }] + } else { + match r.route { + SlotProbeUnread { reason: why } => [SlotRouteUnread { slot: r.slot, reason: why }] + SlotProbeRead { text: route_text } => + if !route_reading_carries(stdout: route_text, cidr: render_ipv4_prefix(p: runner_slot_subnet(net: net)) as String) { + [SlotRouteNotHeld { slot: r.slot, expected: render_ipv4_prefix(p: runner_slot_subnet(net: net)) }] + } else { + [] + } + } + } } } } @@ -364,7 +408,11 @@ fn first_refusal(rs: List) -> ConvergeRefusal? { fn host_refusals(h: HostReading) -> List { concat( concat( - if h.ip_forward_on { [] } else { [IpForwardingOff] }, + match h.ip_forward { + IpForwardOn => [] + IpForwardOff => [IpForwardingOff] + IpForwardUnread { reason: why } => [IpForwardingUnread { reason: why }] + }, match h.ruleset_digest { Present { value: _ } => [] Absent => [RulesetUnread { reason: "nft list table did not return a digestible listing of the installed table" as NonEmptyStr }] @@ -493,14 +541,23 @@ fn listing_digest(d: Sha256FileDigest) -> Sha256Digest? { } } +fn slot_probe_reading(o: ProcessOutcome, probe: String) -> SlotProbeReading { + match o { + ProcessOutputPresent { text: t } => SlotProbeRead { text: t } + ProcessOutputAbsent => SlotProbeRead { text: "" } + ProcessRefused { exit_code: code, stderr: _ } => + SlotProbeUnread { reason: join([probe, " exited ", to_string(code)], "") as NonEmptyStr } + } +} + fn read_slot_wet(slot: RunnerSlotIdentity) -> SlotReading { let net = runner_slot_network(slot: slot) let v6 = read_proc_value(path: runner_slot_ipv6_disabled_proc_path(net: net)) SlotReading { slot: slot, tap_ipv6: parse_disable_ipv6(readable: proc_readable(o: v6), text: proc_text(o: v6)), - addr_stdout: outcome_stdout(o: capture_stdout(c: ip_addr_show_command(device: net.tap_device as String))), - route_stdout: outcome_stdout(o: capture_stdout(c: ip_route_show_command(device: net.tap_device as String))), + addr: slot_probe_reading(o: capture_stdout(c: ip_addr_show_command(device: net.tap_device as String)), probe: "ip -o -4 addr show"), + route: slot_probe_reading(o: capture_stdout(c: ip_route_show_command(device: net.tap_device as String)), probe: "ip -o -4 route show"), } } @@ -523,7 +580,7 @@ fn read_host_wet(host: HostIdentity, generation: ConvergeGeneration) -> HostRead HostReading { host: host, generation: generation, - ip_forward_on: ip_forward_is_on(readable: proc_readable(o: forward), text: proc_text(o: forward)), + ip_forward: ip_forward_standing(readable: proc_readable(o: forward), text: proc_text(o: forward)), ruleset_digest: digest, auto_assignment: parse_auto_assignment( observed: observe_helper_sysctl_wet(), @@ -598,6 +655,9 @@ fn refusal_text(r: ConvergeRefusal) -> String { SlotAddressNotHeld { slot: s, expected: e } => join(["slot ", runner_slot_instance_name(slot: s), ": the tap does not hold ", e as String], "") SlotRouteNotHeld { slot: s, expected: e } => join(["slot ", runner_slot_instance_name(slot: s), ": no route over ", e as String], "") IpForwardingOff => "net.ipv4.ip_forward reads 0" + IpForwardingUnread { reason: why } => why as String + SlotAddressUnread { slot: sl, reason: why } => join(["slot ", runner_slot_instance_name(slot: sl), ": ", why as String], "") + SlotRouteUnread { slot: sl, reason: why } => join(["slot ", runner_slot_instance_name(slot: sl), ": ", why as String], "") RulesetUnread { reason: why } => join(["the installed ruleset could not be listed: ", why as String], "") HelperStandingUnread { reason: why } => join(["helper standing not established: ", why as String], "") HelperAutoAssignmentOn => "automatic conntrack helper assignment is ON" @@ -610,7 +670,9 @@ fn refusal_text(r: ConvergeRefusal) -> String { // the lifecycle controller read to learn what the converge established, so a rendering that dropped // the per-slot subjects, their addresses, routes and v6 standing, or the helper readings, would make // the receipt unable to answer the questions ConvergedSlotNetwork exists to answer (side-chat hold -// on #11751). +// on #11751). The operator's reading comes first; the WIRE form follows it in the same file, so one +// artifact serves both readers and the lifecycle controller can reconstruct the sealed value with +// parse_converged_slot_network rather than re-deriving it from prose. fn helper_surface_text(name: String, r: HelperSurfaceReading) -> String { match r { HelperSurfaceCounted { count: n } => join([" ", name, "=", to_string(n), "\n"], "") @@ -666,7 +728,7 @@ fn converged_receipt_text(c: ConvergedSlotNetwork) -> String { fn receipt_text(o: ConvergeOutcome) -> String { match o { ConvergeRefused { refusal: r } => join(["microvm-network-converge REFUSED: ", refusal_text(r: r), "\n"], "") - ConvergedNetwork { receipt: c } => converged_receipt_text(c: c) + ConvergedNetwork { receipt: c } => join([converged_receipt_text(c: c), render_converged_slot_network(c: c)], "") } } diff --git a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag index f000089f64d..f302c20cc9d 100644 --- a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag @@ -15,15 +15,21 @@ import gunbc.runner_microvm_network { TapIpv6Disabled, TapIpv6Enabled, TapIpv6Unread, AutoAssignmentAbsentFromKernel, AutoAssignmentOn, AutoAssignmentUnread, HelperAutoAssignment, ConvergedSlotNetwork, ConvergeGeneration, runner_microvm_shared_host_grain, + render_converged_slot_network, parse_converged_slot_network, + ConvergedSlotNetworkParse, ConvergedSlotNetworkParsed, ConvergedSlotNetworkMalformed, + helpers_excluded, HelperSurfaceReading, HelperSurfaceCounted, HelperSurfaceUnread, } import gunbc.runner_microvm_network_converge { - parse_disable_ipv6, ip_forward_is_on, ct_helper_statement_count, + parse_disable_ipv6, ip_forward_standing, ct_helper_statement_count, microvm_network_converge_operations, converge_staged_files, StagedFile, SlotReading, HostReading, ConvergeOutcome, ConvergedNetwork, ConvergeRefused, ConvergeRefusal, SlotTapIpv6Unread, SlotTapIpv6Enabled, SlotAddressNotHeld, SlotRouteNotHeld, IpForwardingOff, RulesetUnread, HelperStandingUnread, HelperAutoAssignmentOn, HelpersAttached, converge_outcome, absent_from_kernel_or_unread, + IpForwardStanding, IpForwardOn, IpForwardOff, IpForwardUnread, + SlotProbeReading, SlotProbeRead, SlotProbeUnread, + IpForwardingUnread, SlotAddressUnread, SlotRouteUnread, } data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly @@ -47,7 +53,7 @@ fn unread(why: String) -> HelperSurfaceReading { } fn clean_host() -> HostReading { - host_with(fwd: true, digest: Present { value: converged_digest }, auto: kernel_absent()) + host_with(fwd: IpForwardOn, digest: Present { value: converged_digest }, auto: kernel_absent()) } // THE FIXTURE IS SHAPED LIKE `ip -o -4` OUTPUT AND IS AUTHORED, NOT DERIVED. Building it from the @@ -74,8 +80,8 @@ fn clean_slot(i: Nat) -> SlotReading { SlotReading { slot: slot(i: i), tap_ipv6: TapIpv6Disabled, - addr_stdout: slot_addr_stdout(i: i), - route_stdout: slot_route_stdout(i: i), + addr: SlotProbeRead { text: slot_addr_stdout(i: i) }, + route: SlotProbeRead { text: slot_route_stdout(i: i) }, } } @@ -88,6 +94,9 @@ fn outcome_label(o: ConvergeOutcome) -> String { SlotAddressNotHeld { slot: _, expected: _ } => "no-address" SlotRouteNotHeld { slot: _, expected: _ } => "no-route" IpForwardingOff => "no-forwarding" + IpForwardingUnread { reason: _ } => "forwarding-unread" + SlotAddressUnread { slot: _, reason: _ } => "address-unread" + SlotRouteUnread { slot: _, reason: _ } => "route-unread" RulesetUnread { reason: _ } => "ruleset-unread" HelperStandingUnread { reason: _ } => "helpers-unread" HelperAutoAssignmentOn => "auto-helpers-on" @@ -112,9 +121,10 @@ test fn a_tap_ipv6_reading_is_the_value_or_it_is_unread() -> Bool { && match parse_disable_ipv6(readable: false, text: "") { TapIpv6Unread { reason: _ } => true, _ => false } && match parse_disable_ipv6(readable: true, text: "") { TapIpv6Unread { reason: _ } => true, _ => false } && match parse_disable_ipv6(readable: true, text: "yes") { TapIpv6Unread { reason: _ } => true, _ => false } - && ip_forward_is_on(readable: true, text: "1\n") - && !ip_forward_is_on(readable: true, text: "0\n") - && !ip_forward_is_on(readable: false, text: "1\n") + && match ip_forward_standing(readable: true, text: "1\n") { IpForwardOn => true, _ => false } + && match ip_forward_standing(readable: true, text: "0\n") { IpForwardOff => true, _ => false } + && match ip_forward_standing(readable: false, text: "1\n") { IpForwardUnread { reason: _ } => true, _ => false } + && match ip_forward_standing(readable: true, text: "yes") { IpForwardUnread { reason: _ } => true, _ => false } } // THE POSITIVE CONTROL: with every fact read and holding, the receipt carries one row per slot read. @@ -128,19 +138,19 @@ test fn a_receipt_is_produced_only_when_every_fact_was_read_and_holds() -> Bool // would let the sanitation subject admit slots on a host nobody observed. test fn an_unread_or_failing_fact_refuses_the_receipt() -> Bool { let net1 = runner_slot_network(slot: slot(i: 1)) - let ipv6_on = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Enabled, addr_stdout: "", route_stdout: "" } + let ipv6_on = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Enabled, addr: SlotProbeRead { text: "" }, route: SlotProbeRead { text: "" } } let ipv6_unread = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Unread { reason: "probe refused" as NonEmptyStr }, - addr_stdout: "", - route_stdout: "", + addr: SlotProbeRead { text: "" }, + route: SlotProbeRead { text: "" }, } - let no_addr = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Disabled, addr_stdout: "", route_stdout: "" } + let no_addr = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Disabled, addr: SlotProbeRead { text: "" }, route: SlotProbeRead { text: "" } } let no_route = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Disabled, - addr_stdout: join(["inet ", runner_slot_host_cidr(net: net1)], ""), - route_stdout: "", + addr: SlotProbeRead { text: join(["inet ", runner_slot_host_cidr(net: net1)], "") }, + route: SlotProbeRead { text: "" }, } outcome_label(o: converge_outcome(slots: [ipv6_on], h: clean_host())) == "tap-ipv6-on" && outcome_label(o: converge_outcome(slots: [ipv6_unread], h: clean_host())) == "tap-ipv6-unread" @@ -155,7 +165,7 @@ test fn an_unread_or_failing_fact_refuses_the_receipt() -> Bool { // ONE BUILDER, SIX OVERRIDES. Each claim below supplies the whole HostReading through it, so a field // added to the reading breaks every fixture at once rather than defaulting silently. fn host_reading( - fwd: Bool, + fwd: IpForwardStanding, digest: Sha256Digest?, auto: HelperAutoAssignment, nft: HelperSurfaceReading, @@ -166,7 +176,7 @@ fn host_reading( HostReading { host: operator_host_srv1, generation: "run-3" as ConvergeGeneration, - ip_forward_on: fwd, + ip_forward: fwd, ruleset_digest: digest, auto_assignment: auto, nft_ct_helper_statements: nft, @@ -180,7 +190,7 @@ fn kernel_absent() -> HelperAutoAssignment { AutoAssignmentAbsentFromKernel { kernel_release: "6.8.0-138-generic" as NonEmptyStr } } -fn host_with(fwd: Bool, digest: Sha256Digest?, auto: HelperAutoAssignment) -> HostReading { +fn host_with(fwd: IpForwardStanding, digest: Sha256Digest?, auto: HelperAutoAssignment) -> HostReading { host_reading(fwd: fwd, digest: digest, auto: auto, nft: counted(n: 0), xtables: counted(n: 0), userspace: counted(n: 0), expectations: counted(n: 0)) } @@ -188,12 +198,12 @@ fn host_with(fwd: Bool, digest: Sha256Digest?, auto: HelperAutoAssignment) -> Ho // listing, automatic helper assignment on, an unreadable sysctl, and an attached helper on any // surface are five different remedies and five different refusals. test fn each_host_fact_refuses_under_its_own_name() -> Bool { - outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: false, digest: Present { value: converged_digest }, auto: kernel_absent()))) == "no-forwarding" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: true, digest: none, auto: kernel_absent()))) == "ruleset-unread" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: true, digest: Present { value: converged_digest }, auto: AutoAssignmentOn))) == "auto-helpers-on" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: true, digest: Present { value: converged_digest }, auto: AutoAssignmentUnread { reason: "sysctl unreadable" as NonEmptyStr }))) == "helpers-unread" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: Present { value: converged_digest }, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 2), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-attached" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: Present { value: converged_digest }, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: counted(n: 1), expectations: counted(n: 0)))) == "helpers-attached" + outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: IpForwardOff, digest: Present { value: converged_digest }, auto: kernel_absent()))) == "no-forwarding" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: IpForwardOn, digest: none, auto: kernel_absent()))) == "ruleset-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: IpForwardOn, digest: Present { value: converged_digest }, auto: AutoAssignmentOn))) == "auto-helpers-on" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: IpForwardOn, digest: Present { value: converged_digest }, auto: AutoAssignmentUnread { reason: "sysctl unreadable" as NonEmptyStr }))) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: IpForwardOn, digest: Present { value: converged_digest }, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 2), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-attached" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: IpForwardOn, digest: Present { value: converged_digest }, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: counted(n: 1), expectations: counted(n: 0)))) == "helpers-attached" } // A SURFACE NOBODY READ IS NOT AN EMPTY ONE. Each of the three helper surfaces and the expectation @@ -202,10 +212,10 @@ test fn each_host_fact_refuses_under_its_own_name() -> Bool { // a legacy CT target or a userspace helper converged green. test fn an_unread_helper_surface_refuses_rather_than_counting_zero() -> Bool { let d = Present { value: converged_digest } - outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: d, auto: kernel_absent(), nft: unread(why: "probe exited 1"), xtables: counted(n: 0), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-unread" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: unread(why: "iptables-legacy-save missing"), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-unread" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: unread(why: "nfct missing"), expectations: counted(n: 0)))) == "helpers-unread" - && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: true, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: counted(n: 0), expectations: unread(why: "conntrack refused")))) == "helpers-unread" + outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: IpForwardOn, digest: d, auto: kernel_absent(), nft: unread(why: "probe exited 1"), xtables: counted(n: 0), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: IpForwardOn, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: unread(why: "iptables-legacy-save missing"), userspace: counted(n: 0), expectations: counted(n: 0)))) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: IpForwardOn, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: unread(why: "nfct missing"), expectations: counted(n: 0)))) == "helpers-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_reading(fwd: IpForwardOn, digest: d, auto: kernel_absent(), nft: counted(n: 0), xtables: counted(n: 0), userspace: counted(n: 0), expectations: unread(why: "conntrack refused")))) == "helpers-unread" } // A HELPER ATTACHED BY ANY TABLE COUNTS, because an expectation is created by whichever helper is @@ -271,8 +281,8 @@ test fn the_route_readback_accepts_the_network_and_refuses_the_host_address() -> let host_address_form = SlotReading { slot: slot(i: 1), tap_ipv6: TapIpv6Disabled, - addr_stdout: slot_1_addr_stdout(), - route_stdout: "172.30.1.1/30 proto kernel scope link ", + addr: SlotProbeRead { text: slot_1_addr_stdout() }, + route: SlotProbeRead { text: "172.30.1.1/30 proto kernel scope link " }, } outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: clean_host())) == "converged" && outcome_label(o: converge_outcome(slots: [host_address_form], h: clean_host())) == "no-route" @@ -296,3 +306,73 @@ test fn an_absent_helper_sysctl_names_its_kernel_or_is_unread() -> Bool { _ => false } } + +// THE RECEIPT ROUND-TRIPS INTO THE VALUE THE SANITATION SUBJECT TAKES. A converge that wrote prose +// only would leave the lifecycle controller with no modeled route from a completed converge to a +// ConvergedSlotNetwork (side-chat hold on #11751), so the artifact carries a wire form and this +// claim runs it both ways: render, parse, and read the parsed value's fields back. +test fn the_receipt_round_trips_into_a_converged_slot_network() -> Bool { + match converge_outcome(slots: [clean_slot(i: 1), clean_slot(i: 2)], h: clean_host()) { + ConvergeRefused { refusal: _ } => false + ConvergedNetwork { receipt: c } => + match parse_converged_slot_network( + text: render_converged_slot_network(c: c), + candidates: [slot(i: 1), slot(i: 2)], + ) { + ConvergedSlotNetworkMalformed { cause: _ } => false + ConvergedSlotNetworkParsed { receipt: back } => + (back.host as String) == (c.host as String) + && (back.generation as String) == (c.generation as String) + && (back.nft_digest.hex as String) == (c.nft_digest.hex as String) + && length(back.slots) == 2 + && helpers_excluded(h: back.helpers) + } + } +} + +// A RECEIPT ABOUT ANOTHER FLEET, OR A TRUNCATED ONE, IS REFUSED RATHER THAN PARTIALLY READ. The reds: +// a slot line this host does not own accepted, and a missing digest line yielding a receipt. +test fn a_receipt_that_does_not_describe_this_host_is_refused() -> Bool { + match converge_outcome(slots: [clean_slot(i: 1)], h: clean_host()) { + ConvergeRefused { refusal: _ } => false + ConvergedNetwork { receipt: c } => { + let wire = render_converged_slot_network(c: c) + match parse_converged_slot_network(text: wire, candidates: [slot(i: 2)]) { + ConvergedSlotNetworkMalformed { cause: _ } => true + ConvergedSlotNetworkParsed { receipt: _ } => false + } + && match parse_converged_slot_network(text: "gunbc.converged_slot_network.v1\nhost=srv1\n", candidates: [slot(i: 1)]) { + ConvergedSlotNetworkMalformed { cause: _ } => true + ConvergedSlotNetworkParsed { receipt: _ } => false + } + && match parse_converged_slot_network(text: "some other file\n", candidates: [slot(i: 1)]) { + ConvergedSlotNetworkMalformed { cause: _ } => true + ConvergedSlotNetworkParsed { receipt: _ } => false + } + } + } +} + +// A PROBE THAT DID NOT ANSWER IS NOT A HOST THAT FAILED. Forwarding unread, the address probe +// refused and the route probe refused each refuse under their OWN name, so an operator fixes the +// probe rather than being sent to turn on a setting that may already be on or to repair a tap that +// may be perfect (review 69041). The reds are the previous shapes: a failed read folding into +// "ip_forward reads 0", and a refused ip command reading as "the tap does not hold its address". +test fn a_probe_that_did_not_answer_refuses_under_its_own_name() -> Bool { + let unread_addr = SlotReading { + slot: slot(i: 1), + tap_ipv6: TapIpv6Disabled, + addr: SlotProbeUnread { reason: "ip -o -4 addr show exited 1" as NonEmptyStr }, + route: SlotProbeRead { text: slot_route_stdout(i: 1) }, + } + let unread_route = SlotReading { + slot: slot(i: 1), + tap_ipv6: TapIpv6Disabled, + addr: SlotProbeRead { text: slot_addr_stdout(i: 1) }, + route: SlotProbeUnread { reason: "ip -o -4 route show exited 1" as NonEmptyStr }, + } + outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: IpForwardUnread { reason: "ip_forward could not be read" as NonEmptyStr }, digest: Present { value: converged_digest }, auto: kernel_absent()))) == "forwarding-unread" + && outcome_label(o: converge_outcome(slots: [unread_addr], h: clean_host())) == "address-unread" + && outcome_label(o: converge_outcome(slots: [unread_route], h: clean_host())) == "route-unread" + && outcome_label(o: converge_outcome(slots: [clean_slot(i: 1)], h: host_with(fwd: IpForwardOff, digest: Present { value: converged_digest }, auto: kernel_absent()))) == "no-forwarding" +} diff --git a/dag/test/claim/workflow_dispatch_input_witness_test.dag b/dag/test/claim/workflow_dispatch_input_witness_test.dag index 385b093c7bd..16ecc9cb7f3 100644 --- a/dag/test/claim/workflow_dispatch_input_witness_test.dag +++ b/dag/test/claim/workflow_dispatch_input_witness_test.dag @@ -14,6 +14,8 @@ import gunbc.fleet_converge_workflow { RunnerPasswordSessionToolConverge, fleet_converge_spark_bootstrap_step_if, fleet_converge_mode_options, fleet_converge_reset_observer_dispatch_admission_step, fleet_converge_build_job_steps, + fleet_converge_microvm_network_converge_step, fleet_converge_mode_fleet_ssh_key_demand, + MicrovmNetworkConverge, FleetSshKeyNotConsumed, FleetSshKeyConsumed, fleet_converge_reset_subject_mutation_concurrency_group_expression, fleet_converge_reset_subject_map_json, reset_subject_map_json, fleet_converge_workflow, expected_fleet_converge_yml, @@ -415,3 +417,52 @@ test fn dashboard_deploy_selects_live_deploy_scope_while_the_plan_modes_do() -> ) deploy_live && plans_present } + +// THE microVM NETWORK STEP MUST CARRY THE DISPATCHED HOST INTO ITS ENVIRONMENT, OR THE MODE REFUSES +// EVERY TIME IT RUNS. Its wet entry compares FLEET_CONVERGE_EXPECTED_HOST against the machine's own +// hostname before staging anything; a step that does not supply the variable makes that comparison +// answer HostBindingUnnamed on every dispatch, so the mode could never reach its first staging write +// (side-chat hold on #11751). A claim over the pure decision cannot see that: the decision is right +// and nothing joins it to the generated workflow. This reads the STEP and the rendered YAML. +fn yaml_scalar_text(v: YamlValue) -> String { + match v { + YamlString { value: t } => t + YamlNull => "" + YamlBool { value: _ } => "" + YamlInt { lexeme: l } => l + YamlFloat { lexeme: l } => l + YamlSequence { elements: _ } => "" + YamlMapping { entries: _ } => "" + } +} + +fn microvm_network_step_env_text() -> String { + match fleet_converge_microvm_network_converge_step() { + RunStep { name: _, id: _, run: _, shell: _, env: e, working_directory: _, if_condition: _, continue_on_error: _, timeout_minutes: _ } => + match e { + Present { value: pairs } => join(map(pairs, pair => join([pair.key, "=", yaml_scalar_text(v: pair.value)], "")), "\n") + Absent => "" + } + UsesStep { name: _, id: _, uses: _, with: _, env: _, if_condition: _, continue_on_error: _, timeout_minutes: _ } => "" + } +} + +test fn the_microvm_network_step_binds_the_dispatched_host_into_its_environment() -> Bool { + let env = microvm_network_step_env_text() + let yml = expected_fleet_converge_yml_content() + string_contains(s: env, pattern: "FLEET_CONVERGE_EXPECTED_HOST") + && string_contains(s: env, pattern: "github.event.inputs.host") + && string_contains(s: yml, pattern: "id: microvm_network_converge") + && string_contains(s: yml, pattern: "FLEET_CONVERGE_EXPECTED_HOST: $\{\{ github.event.inputs.host \}\}") +} + +// THE MODE IS LOCAL, SO IT CLAIMS NO FLEET KEY. Every effect it has runs on the machine executing it: +// its own hostname, its own staging writes, sudo -n over LocalExec, and procfs, ip, nft, xtables and +// conntrack reads on that host. Materialising the fleet SSH key for it would widen a capability with +// no consumer in the source (side-chat hold on #11751). +test fn the_microvm_network_mode_claims_no_fleet_ssh_key() -> Bool { + match fleet_converge_mode_fleet_ssh_key_demand(mode: MicrovmNetworkConverge) { + FleetSshKeyNotConsumed => true + FleetSshKeyConsumed => false + } +} From cc63588fdd21e4fe3f50a208de33cfddee7831b7 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 09:43:00 +0000 Subject: [PATCH 25/36] Regenerate fleet-converge.yml for the host binding and ssh-key reclassification Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/fleet-converge.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index fd796554f45..97cbf873180 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -227,7 +227,7 @@ jobs: echo "fleet-key: agent loaded (identity fleet-automation@gunbc; secret versions/1 pinned; fingerprint verified against modeled authority; key file wiped)" env: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} - if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'microvm_network_converge' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'mtcollins1_boot' + if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'mtcollins1_boot' timeout-minutes: 5 - name: Fleet converge plan (membership_reconcile → artifact) id: plan @@ -612,6 +612,8 @@ jobs: ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_microvm_network_converge.dag --function runner_microvm_network_converge_wet cat "$ROOT/target/microvm-network-converge.txt" + env: + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} if: github.event.inputs.mode == 'microvm_network_converge' timeout-minutes: 30 - name: Upload runner micro-VM network converge receipt From e80a17b76b3b77d9569ce6e97624d6abb0b44207 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 09:50:01 +0000 Subject: [PATCH 26/36] Regenerate fleet-converge.yml at the merged head Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/fleet-converge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 97cbf873180..5c06518a4ac 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -227,7 +227,7 @@ jobs: echo "fleet-key: agent loaded (identity fleet-automation@gunbc; secret versions/1 pinned; fingerprint verified against modeled authority; key file wiped)" env: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} - if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' || github.event.inputs.mode == 'mtcollins1_boot' + if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' timeout-minutes: 5 - name: Fleet converge plan (membership_reconcile → artifact) id: plan From 74bf5cec47b8bbcb3c19389b0f4a4c77814133fc Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 10:23:53 +0000 Subject: [PATCH 27/36] The persisted receipt leads with its schema and the claim parses the bytes the converge writes Co-Authored-By: Claude Opus 5 (1M context) --- .../runner_microvm_network_converge.dag | 44 +++++++++++-------- ..._microvm_network_converge_witness_test.dag | 15 ++++--- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/dag/gunbc/runner/runner_microvm_network_converge.dag b/dag/gunbc/runner/runner_microvm_network_converge.dag index e0aee191c3c..5564f68afdc 100644 --- a/dag/gunbc/runner/runner_microvm_network_converge.dag +++ b/dag/gunbc/runner/runner_microvm_network_converge.dag @@ -673,19 +673,25 @@ fn refusal_text(r: ConvergeRefusal) -> String { // on #11751). The operator's reading comes first; the WIRE form follows it in the same file, so one // artifact serves both readers and the lifecycle controller can reconstruct the sealed value with // parse_converged_slot_network rather than re-deriving it from prose. +// +// THE WIRE BLOCK COMES FIRST AND THE OPERATOR'S READING IS COMMENTED. The parse refuses anything +// whose first line is not the schema, so a file that opened with the human summary parsed as +// malformed -- every real receipt this converge writes (review 69063). Prefixing the report with +// "# " keeps one artifact for both readers while leaving the wire block the whole parseable content: +// a comment line matches no key and no slot prefix. fn helper_surface_text(name: String, r: HelperSurfaceReading) -> String { match r { - HelperSurfaceCounted { count: n } => join([" ", name, "=", to_string(n), "\n"], "") - HelperSurfaceUnread { reason: why } => join([" ", name, "=UNREAD ", why as String, "\n"], "") + HelperSurfaceCounted { count: n } => join(["# ", name, "=", to_string(n), "\n"], "") + HelperSurfaceUnread { reason: why } => join(["# ", name, "=UNREAD ", why as String, "\n"], "") } } fn auto_assignment_text(a: HelperAutoAssignment) -> String { match a { - AutoAssignmentAbsentFromKernel { kernel_release: k } => join([" auto_assignment=absent_from_kernel release=", k as String, "\n"], "") - AutoAssignmentSysctlOff => " auto_assignment=sysctl_off\n" - AutoAssignmentOn => " auto_assignment=ON\n" - AutoAssignmentUnread { reason: why } => join([" auto_assignment=UNREAD ", why as String, "\n"], "") + AutoAssignmentAbsentFromKernel { kernel_release: k } => join(["# auto_assignment=absent_from_kernel release=", k as String, "\n"], "") + AutoAssignmentSysctlOff => "# auto_assignment=sysctl_off\n" + AutoAssignmentOn => "# auto_assignment=ON\n" + AutoAssignmentUnread { reason: why } => join(["# auto_assignment=UNREAD ", why as String, "\n"], "") } } @@ -699,24 +705,24 @@ fn tap_ipv6_text(t: TapIpv6Standing) -> String { fn converged_slot_text(c: ConvergedSlot) -> String { join([ - "slot ", runner_slot_instance_name(slot: c.subject.slot), "\n", - " tap=", c.subject.tap_device as String, "\n", - " guest_address=", render_ipv4_address(addr: c.subject.guest_address) as String, "\n", - " host_address=", c.host_address_cidr as String, "\n", - " route=", c.guest_route as String, "\n", - " tap_ipv6=", tap_ipv6_text(t: c.tap_ipv6), "\n", + "# slot ", runner_slot_instance_name(slot: c.subject.slot), "\n", + "# tap=", c.subject.tap_device as String, "\n", + "# guest_address=", render_ipv4_address(addr: c.subject.guest_address) as String, "\n", + "# host_address=", c.host_address_cidr as String, "\n", + "# route=", c.guest_route as String, "\n", + "# tap_ipv6=", tap_ipv6_text(t: c.tap_ipv6), "\n", ], "") } fn converged_receipt_text(c: ConvergedSlotNetwork) -> String { join([ - "microvm-network-converge CONVERGED\n", - "host=", c.host as String, "\n", - "generation=", c.generation as String, "\n", - "nft_digest=", c.nft_digest.hex as String, "\n", - "slots=", to_string(length(c.slots)), "\n", + "# microvm-network-converge CONVERGED\n", + "# host=", c.host as String, "\n", + "# generation=", c.generation as String, "\n", + "# nft_digest=", c.nft_digest.hex as String, "\n", + "# slots=", to_string(length(c.slots)), "\n", join(map(c.slots, x => converged_slot_text(c: x)), ""), - "helpers\n", + "# helpers\n", auto_assignment_text(a: c.helpers.auto_assignment), helper_surface_text(name: "nft_ct_helper_statements", r: c.helpers.nft_ct_helper_statements), helper_surface_text(name: "xtables_ct_helper_targets", r: c.helpers.xtables_ct_helper_targets), @@ -728,7 +734,7 @@ fn converged_receipt_text(c: ConvergedSlotNetwork) -> String { fn receipt_text(o: ConvergeOutcome) -> String { match o { ConvergeRefused { refusal: r } => join(["microvm-network-converge REFUSED: ", refusal_text(r: r), "\n"], "") - ConvergedNetwork { receipt: c } => join([converged_receipt_text(c: c), render_converged_slot_network(c: c)], "") + ConvergedNetwork { receipt: c } => join([render_converged_slot_network(c: c), converged_receipt_text(c: c)], "") } } diff --git a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag index f302c20cc9d..0b56007be49 100644 --- a/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag +++ b/dag/test/claim/runner/runner_microvm_network_converge_witness_test.dag @@ -26,7 +26,7 @@ import gunbc.runner_microvm_network_converge { SlotReading, HostReading, ConvergeOutcome, ConvergedNetwork, ConvergeRefused, ConvergeRefusal, SlotTapIpv6Unread, SlotTapIpv6Enabled, SlotAddressNotHeld, SlotRouteNotHeld, IpForwardingOff, RulesetUnread, HelperStandingUnread, HelperAutoAssignmentOn, HelpersAttached, - converge_outcome, absent_from_kernel_or_unread, + converge_outcome, absent_from_kernel_or_unread, receipt_text, IpForwardStanding, IpForwardOn, IpForwardOff, IpForwardUnread, SlotProbeReading, SlotProbeRead, SlotProbeUnread, IpForwardingUnread, SlotAddressUnread, SlotRouteUnread, @@ -307,7 +307,11 @@ test fn an_absent_helper_sysctl_names_its_kernel_or_is_unread() -> Bool { } } -// THE RECEIPT ROUND-TRIPS INTO THE VALUE THE SANITATION SUBJECT TAKES. A converge that wrote prose +// THE CLAIM PARSES THE BYTES THE CONVERGE WRITES, NOT A RENDERING NO CONSUMER SEES. Feeding +// render_converged_slot_network directly was green over a value the artifact never contained: the +// persisted file opened with the operator's summary, so the parser refused every real receipt +// (review 69063). This calls receipt_text -- the function whose output is written to the artifact -- +// and asserts the schema is the first line, which is the property that makes the parse reachable. A converge that wrote prose // only would leave the lifecycle controller with no modeled route from a completed converge to a // ConvergedSlotNetwork (side-chat hold on #11751), so the artifact carries a wire form and this // claim runs it both ways: render, parse, and read the parsed value's fields back. @@ -316,12 +320,13 @@ test fn the_receipt_round_trips_into_a_converged_slot_network() -> Bool { ConvergeRefused { refusal: _ } => false ConvergedNetwork { receipt: c } => match parse_converged_slot_network( - text: render_converged_slot_network(c: c), + text: receipt_text(o: converge_outcome(slots: [clean_slot(i: 1), clean_slot(i: 2)], h: clean_host())), candidates: [slot(i: 1), slot(i: 2)], ) { ConvergedSlotNetworkMalformed { cause: _ } => false ConvergedSlotNetworkParsed { receipt: back } => - (back.host as String) == (c.host as String) + receipt_text(o: converge_outcome(slots: [clean_slot(i: 1), clean_slot(i: 2)], h: clean_host())).starts_with(prefix: "gunbc.converged_slot_network.v1\n") + && (back.host as String) == (c.host as String) && (back.generation as String) == (c.generation as String) && (back.nft_digest.hex as String) == (c.nft_digest.hex as String) && length(back.slots) == 2 @@ -336,7 +341,7 @@ test fn a_receipt_that_does_not_describe_this_host_is_refused() -> Bool { match converge_outcome(slots: [clean_slot(i: 1)], h: clean_host()) { ConvergeRefused { refusal: _ } => false ConvergedNetwork { receipt: c } => { - let wire = render_converged_slot_network(c: c) + let wire = receipt_text(o: converge_outcome(slots: [clean_slot(i: 1)], h: clean_host())) match parse_converged_slot_network(text: wire, candidates: [slot(i: 2)]) { ConvergedSlotNetworkMalformed { cause: _ } => true ConvergedSlotNetworkParsed { receipt: _ } => false From 784f6da3ff3f2349ac31843d37c5bd7e03776277 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 11:11:14 +0000 Subject: [PATCH 28/36] Model the nft loader unit through the systemd unit-file authority; decode the fabric lifetime sample's digests through their optionals Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/unit_file.dag | 7 +++ .../fabric/fabric_required_build_cell.dag | 12 ++++- dag/gunbc/runner/runner_microvm_network.dag | 49 +++++++++++++------ 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/dag/extdeps/systemd/unit_file.dag b/dag/extdeps/systemd/unit_file.dag index e514425fd6c..0bfdda52575 100644 --- a/dag/extdeps/systemd/unit_file.dag +++ b/dag/extdeps/systemd/unit_file.dag @@ -187,8 +187,14 @@ fn systemd_environment_reference(name: NonEmptyStr) -> String { data systemd_exit_code_variable: NonEmptyStr = "EXIT_CODE" as NonEmptyStr data systemd_exit_status_variable: NonEmptyStr = "EXIT_STATUS" as NonEmptyStr +// RemainAfterExit= IS WHAT MAKES A ONESHOT'S WORK OUTLIVE ITS PROCESS. systemd.service(5) (v255): +// a Type=oneshot unit is "active" only while its ExecStart runs unless this is set, and with it the +// unit stays active after the process exits -- which is how a unit whose job is to INSTALL state +// (load a ruleset, apply a sysctl) reports that the state is in place rather than that a program is +// running. The boolean renders as systemd's own yes/no rather than true/false. type SystemdServiceDirective = ServiceType { mode: NonEmptyStr } + | RemainAfterExit { active: Bool } | ExecStartPre { command: NonEmptyStr } | ExecStart { command: NonEmptyStr } | ExecStartPost { command: NonEmptyStr } @@ -621,6 +627,7 @@ fn systemd_unit_directive_line(directive: SystemdUnitDirective) -> String { fn systemd_service_directive_line(directive: SystemdServiceDirective) -> String { match directive { ServiceType { mode: v } => join(["Type=", v as String], "") + RemainAfterExit { active: v } => join(["RemainAfterExit=", if v { "yes" } else { "no" }], "") ExecStartPre { command: v } => join(["ExecStartPre=", v as String], "") ExecStart { command: v } => join(["ExecStart=", v as String], "") ExecStartPost { command: v } => join(["ExecStartPost=", v as String], "") diff --git a/dag/gunbc/fabric/fabric_required_build_cell.dag b/dag/gunbc/fabric/fabric_required_build_cell.dag index 2870a715891..f2a07adfaad 100644 --- a/dag/gunbc/fabric/fabric_required_build_cell.dag +++ b/dag/gunbc/fabric/fabric_required_build_cell.dag @@ -413,10 +413,16 @@ fn decode_required_build_cell_lifetime_sample(wire: String) -> RequiredBuildCell match parse_int(s: fields.skip(n: 1).first()) { Absent => RequiredBuildCellLifetimeSampleUndecodable Present { value: generation } => - match content_hash_from_structural_digest(digest: fields.skip(n: 3).first()) { + match fields.skip(n: 3).first() { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: reservation_digest } => + match content_hash_from_structural_digest(digest: reservation_digest) { Absent => RequiredBuildCellLifetimeSampleUndecodable Present { value: reservation_content } => - match content_hash_from_structural_digest(digest: fields.skip(n: 5).first()) { + match fields.skip(n: 5).first() { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: members_digest } => + match content_hash_from_structural_digest(digest: members_digest) { Absent => RequiredBuildCellLifetimeSampleUndecodable Present { value: members_content } => match fields.first() { @@ -454,6 +460,8 @@ fn decode_required_build_cell_lifetime_sample(wire: String) -> RequiredBuildCell } } } + } + } } fn required_build_cell_lifetime_verdict( diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index a2204355c92..5a9c778b2e9 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -16,6 +16,11 @@ import gunbc.executor_privileged_operation { SystemdDaemonReload, SystemdEnableUnit, ReadNftTable, } import gunbc.fleet_posix_accounts { fleet_nft_binary_path } +import extdeps.systemd.unit_file { + SystemdUnitFile, ServiceUnitFile, SystemdInstallSection, Installable, + Description, After, Wants, ServiceType, RemainAfterExit, ExecStart, WantedBy, + serialize_systemd_unit_file, +} import extdeps.nftables.ruleset { NftTable, NftChain, NftRule, NftMatch, NftFamily, NftInet, NftHook, NftForward, NftPostrouting, NftChainPriority, NftPriorityFilter, NftPrioritySrcnat, NftPriorityRaw, NftPrerouting, NftInput, NftOutput, NftInputInterface, NftPolicy, NftPolicyAccept, NftPolicyDrop, @@ -607,6 +612,11 @@ data runner_microvm_networkd_dir: NonEmptyStr = "/etc/systemd/network" as NonEmp data runner_microvm_networkd_unit: NonEmptyStr = "systemd-networkd" as NonEmptyStr +// THE ORDERING DEPENDENCY NAMES THE UNIT FILE, NOT THE systemctl ARGUMENT. `systemctl reload +// systemd-networkd` resolves the short name, but After=/Wants= in a unit file take the unit's full +// name, so the two spellings are derived from one row rather than authored twice. +data runner_microvm_networkd_unit_name: NonEmptyStr = "systemd-networkd.service" as NonEmptyStr + data runner_microvm_networkd_file_mode: NonEmptyStr = "0644" as NonEmptyStr data runner_microvm_networkd_file_owner: NonEmptyStr = "root" as NonEmptyStr @@ -739,24 +749,35 @@ fn runner_microvm_nft_unit_path() -> String { join([runner_microvm_systemd_unit_dir as String, "/", runner_microvm_nft_unit as String], "") } +// THE UNIT IS A MODELED SystemdUnitFile, NOT CONCATENATED INI. extdeps.systemd.unit_file owns that +// syntax -- its sections, its separators, its terminating newline -- and gunbc.build_cache_unit +// already records what happens when a consumer re-mints it: a second authority for systemd's own +// file format, the DESIGN section 3 nicknaming that module exists to prevent (review 69082). The one +// directive it lacked, RemainAfterExit=, is now a row there rather than a literal here. +// // After= networkd, because the ruleset's rules name tap devices networkd creates; nft accepts rules // for a device that does not exist yet, but ordering it after removes the window where a tap is up // and unfiltered. +fn runner_microvm_nft_unit_file() -> SystemdUnitFile { + ServiceUnitFile { + unit: [ + Description { text: "gunbc microVM host ruleset" as NonEmptyStr }, + After { target: runner_microvm_networkd_unit_name }, + Wants { target: runner_microvm_networkd_unit_name }, + ], + service: [ + ServiceType { mode: "oneshot" as NonEmptyStr }, + RemainAfterExit { active: true }, + ExecStart { + command: join([fleet_nft_binary_path as String, " -f ", runner_microvm_nft_ruleset_path as String], "") as NonEmptyStr, + }, + ], + install: Installable { directives: [WantedBy { target: "multi-user.target" as NonEmptyStr }] }, + } +} + fn runner_microvm_nft_unit_text() -> String { - join([ - "[Unit]\n", - "Description=gunbc microVM host ruleset\n", - "After=systemd-networkd.service\n", - "Wants=systemd-networkd.service\n", - "\n", - "[Service]\n", - "Type=oneshot\n", - "RemainAfterExit=yes\n", - "ExecStart=", fleet_nft_binary_path as String, " -f ", runner_microvm_nft_ruleset_path as String, "\n", - "\n", - "[Install]\n", - "WantedBy=multi-user.target\n", - ], "") + serialize_systemd_unit_file(file: runner_microvm_nft_unit_file()) } // FORWARDING IS WHAT MAKES THE TAP A ROUTED LINK RATHER THAN A DEAD END. Without it the guest's From 875fe8ad7ada3684d3eb8207d621f2e566afe1a2 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 11:43:18 +0000 Subject: [PATCH 29/36] Regenerate fleet-converge.yml at the merged head Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/fleet-converge.yml | 34 ++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index 5c06518a4ac..a13713208dd 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -13,7 +13,7 @@ on: mode: description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -227,7 +227,7 @@ jobs: echo "fleet-key: agent loaded (identity fleet-automation@gunbc; secret versions/1 pinned; fingerprint verified against modeled authority; key file wiped)" env: WIF_ACCESS_TOKEN: ${{ steps.wif_auth.outputs.access_token }} - if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' + if: github.event.inputs.mode == 'plan' || github.event.inputs.mode == 'launch_environment_plan' || github.event.inputs.mode == 'allocation_store_plan' || github.event.inputs.mode == 'apply' || github.event.inputs.mode == 'app_control_plane_observe' || github.event.inputs.mode == 'microvm_host_converge' || github.event.inputs.mode == 'guest_image_observe' || github.event.inputs.mode == 'guest_image_converge' || github.event.inputs.mode == 'microvm_boot_probe' || github.event.inputs.mode == 'spark_grants' || github.event.inputs.mode == 'spark_bootstrap' || github.event.inputs.mode == 'spark_serving_apply' || github.event.inputs.mode == 'spark_runtime_image_probe' || github.event.inputs.mode == 'dashboard_deploy' || github.event.inputs.mode == 'rlm_launch_deployment_receipt' || github.event.inputs.mode == 'host_reset_return' || github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge' || github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge' || github.event.inputs.mode == 'runner_password_session_tool_converge' || github.event.inputs.mode == 'r2_mint_preflight' || github.event.inputs.mode == 'r2_object_write_mint' || github.event.inputs.mode == 'approval_keyring_converge' timeout-minutes: 5 - name: Fleet converge plan (membership_reconcile → artifact) id: plan @@ -810,6 +810,36 @@ jobs: retention-days: 30 if: always() && (github.event.inputs.mode == 'runner_host_file_observe' || github.event.inputs.mode == 'runner_host_file_converge') timeout-minutes: 10 + - name: "Site PXE edge: observe the chainloader digest, dnsmasq, the edge config and unit (no writes)" + id: site_pxe_edge_observe + run: | + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/fleet/site_pxe_edge_converge.dag --function site_pxe_edge_observe_ci_wet + cat "$ROOT/target/site-pxe-edge-converge-receipt.txt" + env: + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} + if: github.event.inputs.mode == 'site_pxe_edge_observe' + timeout-minutes: 15 + - name: "Site PXE edge: if the gate is open, write what differs, restart the unit, read its active state back" + id: site_pxe_edge_converge + run: | + ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/fleet/site_pxe_edge_converge.dag --function site_pxe_edge_converge_ci_wet + cat "$ROOT/target/site-pxe-edge-converge-receipt.txt" + env: + FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} + if: github.event.inputs.mode == 'site_pxe_edge_converge' + timeout-minutes: 15 + - name: Upload site PXE edge converge receipt + id: site_pxe_edge_receipt_upload + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f + with: + name: site-pxe-edge-converge-receipt + path: target/site-pxe-edge-converge-receipt.txt + if-no-files-found: error + retention-days: 30 + if: always() && (github.event.inputs.mode == 'site_pxe_edge_observe' || github.event.inputs.mode == 'site_pxe_edge_converge') + timeout-minutes: 10 - name: Kill the in-run ssh-agent (key never outlives the job) run: | if [ -n "${SSH_AGENT_PID:-}" ]; then ssh-agent -k || true; fi From ecd954a95a4156af9bd487bb2de9cb942ae7cdb7 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 12:13:43 +0000 Subject: [PATCH 30/36] The job user is granted the reads and none of the mutations; the mode observes and records The install+restart pair over a grantee-writable staged path was root-equivalent: the job user stages these files and also executes job steps, so it could write its own ExecStart= and have root run it (review 69104). The roster now carries the readbacks only, the mutating half waits on a principal the job user cannot impersonate, and the mode is named for what it does. Also fixes two latent defects on main that its own newly landed Optional checker refuses: fabric_required_build_cell's lifetime-sample digests and workflow_dispatch_input's trigger index. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/ci/ci_spec.dag | 14 +-- dag/gunbc/fleet/fleet_converge_workflow.dag | 34 +++---- dag/gunbc/host_convergence_census.dag | 4 +- dag/gunbc/runner/runner_microvm_network.dag | 4 +- .../runner_microvm_network_converge.dag | 90 +++++++++++-------- .../workflow_dispatch_input_witness_test.dag | 23 +++-- 6 files changed, 98 insertions(+), 71 deletions(-) diff --git a/dag/gunbc/ci/ci_spec.dag b/dag/gunbc/ci/ci_spec.dag index 1e96f3a3c57..719ceacee9d 100644 --- a/dag/gunbc/ci/ci_spec.dag +++ b/dag/gunbc/ci/ci_spec.dag @@ -822,9 +822,9 @@ data gunbc_ci_runner_microvm_host_converge_target: GunbcRunStepTarget = GunbcRun function: "runner_microvm_host_converge_wet", } -data gunbc_ci_runner_microvm_network_converge_target: GunbcRunStepTarget = GunbcRunStepTarget { +data gunbc_ci_runner_microvm_network_observe_target: GunbcRunStepTarget = GunbcRunStepTarget { entry: "dag/gunbc/runner/runner_microvm_network_converge.dag", - function: "runner_microvm_network_converge_wet", + function: "runner_microvm_network_observe_wet", } data gunbc_ci_runner_host_file_observe_target: GunbcRunStepTarget = GunbcRunStepTarget { @@ -1065,7 +1065,7 @@ fn gunbc_run_step_targets() -> List { gunbc_ci_runner_guest_image_observe_target, gunbc_ci_runner_guest_image_converge_target, gunbc_ci_runner_microvm_host_converge_target, - gunbc_ci_runner_microvm_network_converge_target, + gunbc_ci_runner_microvm_network_observe_target, gunbc_ci_runner_host_file_observe_target, gunbc_ci_runner_host_file_converge_target, gunbc_ci_site_pxe_edge_observe_target, @@ -1983,13 +1983,13 @@ fn gunbc_ci_runner_microvm_host_converge_invoke() -> String { ) } -fn gunbc_ci_runner_microvm_network_converge_invoke() -> String { +fn gunbc_ci_runner_microvm_network_observe_invoke() -> String { gunbc_run_step_script( source_roots: witness_layer_roots, - entry: gunbc_ci_runner_microvm_network_converge_target.entry, - function: gunbc_ci_runner_microvm_network_converge_target.function, + entry: gunbc_ci_runner_microvm_network_observe_target.entry, + function: gunbc_ci_runner_microvm_network_observe_target.function, claim_run: false, - receipt_rel: "target/microvm-network-converge.txt" + receipt_rel: "target/microvm-network-observe.txt" ) } diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index df3e0260669..d1401445897 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -181,7 +181,7 @@ type FleetConvergeWorkflowMode | OrgActionsObserve | AppControlPlaneObserve | MicrovmHostConverge - | MicrovmNetworkConverge + | MicrovmNetworkObserve | GuestImageObserve | GuestImageConverge | MicrovmBootProbe @@ -212,7 +212,7 @@ fn fleet_converge_workflow_mode_wire(mode: FleetConvergeWorkflowMode) -> String OrgActionsObserve => "org_actions_observe" AppControlPlaneObserve => "app_control_plane_observe" MicrovmHostConverge => "microvm_host_converge" - MicrovmNetworkConverge => "microvm_network_converge" + MicrovmNetworkObserve => "microvm_network_observe" GuestImageObserve => "guest_image_observe" GuestImageConverge => "guest_image_converge" MicrovmBootProbe => "microvm_boot_probe" @@ -257,7 +257,7 @@ fn fleet_converge_spark_target_description() -> String { ], "") } -data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkConverge, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, MtCollins1Boot] +data fleet_converge_workflow_modes: List = [FullHostPlan, LaunchEnvironmentPlan, AllocationStorePlan, FleetApply, OrgActionsObserve, AppControlPlaneObserve, MicrovmHostConverge, MicrovmNetworkObserve, GuestImageObserve, GuestImageConverge, MicrovmBootProbe, SparkGrants, SparkBootstrap, SparkServingApply, SparkRuntimeImageProbe, DashboardDeploy, RlmLaunchDeploymentReceipt, HostResetReturn, RunnerHostFileObserve, RunnerHostFileConverge, SitePxeEdgeObserve, SitePxeEdgeConverge, RunnerPasswordSessionToolConverge, R2MintPreflight, R2ObjectWriteMint, OrgRunnerRosterObserve, ApprovalKeyringConverge, MtCollins1Boot] // WHICH SCOPE A MODE SELECTS, WHERE IT SELECTS ONE AT ALL. // @@ -288,7 +288,7 @@ fn fleet_converge_mode_scope(mode: FleetConvergeWorkflowMode) -> FleetConvergeSc OrgActionsObserve => none AppControlPlaneObserve => none MicrovmHostConverge => none - MicrovmNetworkConverge => none + MicrovmNetworkObserve => none GuestImageObserve => none GuestImageConverge => none MicrovmBootProbe => none @@ -342,7 +342,7 @@ fn fleet_converge_mode_fleet_ssh_key_demand(mode: FleetConvergeWorkflowMode) -> FleetApply => FleetSshKeyConsumed AppControlPlaneObserve => FleetSshKeyConsumed MicrovmHostConverge => FleetSshKeyConsumed - MicrovmNetworkConverge => FleetSshKeyNotConsumed + MicrovmNetworkObserve => FleetSshKeyNotConsumed GuestImageObserve => FleetSshKeyConsumed GuestImageConverge => FleetSshKeyConsumed MicrovmBootProbe => FleetSshKeyConsumed @@ -747,7 +747,7 @@ data fleet_converge_microvm_boot_probe_step_if: String = fleet_converge_mode_ste data fleet_converge_microvm_host_converge_step_if: String = fleet_converge_mode_step_if(mode: MicrovmHostConverge) -data fleet_converge_microvm_network_converge_step_if: String = fleet_converge_mode_step_if(mode: MicrovmNetworkConverge) +data fleet_converge_microvm_network_observe_step_if: String = fleet_converge_mode_step_if(mode: MicrovmNetworkObserve) @@ -1021,11 +1021,11 @@ fn fleet_converge_microvm_host_converge_step() -> Step { // then READS the host back into a ConvergedSlotNetwork receipt. They are separate dispatches because // they fail for different reasons and an operator routinely wants one without the other: a // Firecracker release that will not verify says nothing about a tap that will not carry an address. -fn fleet_converge_microvm_network_converge_step() -> Step { +fn fleet_converge_microvm_network_observe_step() -> Step { RunStep { name: Present { value: "Runner micro-VM network: install the slot taps, ruleset and forwarding, then read the host back" }, - id: Present { value: "microvm_network_converge" }, - run: gunbc_ci_runner_microvm_network_converge_invoke(), + id: Present { value: "microvm_network_observe" }, + run: gunbc_ci_runner_microvm_network_observe_invoke(), shell: none, env: Present { value: [ @@ -1033,7 +1033,7 @@ fn fleet_converge_microvm_network_converge_step() -> Step { ] }, working_directory: none, - if_condition: Present { value: fleet_converge_microvm_network_converge_step_if }, + if_condition: Present { value: fleet_converge_microvm_network_observe_step_if }, continue_on_error: none, timeout_minutes: Present { value: gunbc_ci_runner_guest_image_step_timeout_minutes } } @@ -1041,19 +1041,19 @@ fn fleet_converge_microvm_network_converge_step() -> Step { // THE RECEIPT UPLOADS ON FAILURE TOO, because a refusal is the reading worth having: it names which // fact the host did not establish. -fn fleet_converge_microvm_network_receipt_upload_step() -> Step { +fn fleet_converge_microvm_network_observe_receipt_upload_step() -> Step { UsesStep { name: Present { value: "Upload runner micro-VM network converge receipt" }, - id: Present { value: "microvm_network_converge_receipt_upload" }, + id: Present { value: "microvm_network_observe_receipt_upload" }, uses: upload_artifact_action, with: Present { value: [ - kv(key: "name", value: yaml_string(s: "runner-microvm-network-converge-receipt")), - kv(key: "path", value: yaml_string(s: "target/microvm-network-converge.txt")), + kv(key: "name", value: yaml_string(s: "runner-microvm-network-observe-receipt")), + kv(key: "path", value: yaml_string(s: "target/microvm-network-observe.txt")), kv(key: "if-no-files-found", value: yaml_string(s: "error")), kv(key: "retention-days", value: yaml_int(n: 30)), ] }, env: none, - if_condition: Present { value: concat("always() && (", concat(fleet_converge_microvm_network_converge_step_if, ")")) }, + if_condition: Present { value: concat("always() && (", concat(fleet_converge_microvm_network_observe_step_if, ")")) }, continue_on_error: none, timeout_minutes: Present { value: gunbc_ci_artifact_transfer_step_timeout_minutes } } @@ -1820,8 +1820,8 @@ fn fleet_converge_job() -> Job { fleet_converge_guest_image_receipt_upload_step(), fleet_converge_microvm_host_converge_step(), fleet_converge_microvm_host_standing_upload_step(), - fleet_converge_microvm_network_converge_step(), - fleet_converge_microvm_network_receipt_upload_step(), + fleet_converge_microvm_network_observe_step(), + fleet_converge_microvm_network_observe_receipt_upload_step(), fleet_converge_microvm_boot_probe_step(), fleet_converge_microvm_boot_probe_receipt_upload_step(), fleet_converge_spark_grants_step(), diff --git a/dag/gunbc/host_convergence_census.dag b/dag/gunbc/host_convergence_census.dag index 08742d4e60f..2f521e76c4a 100644 --- a/dag/gunbc/host_convergence_census.dag +++ b/dag/gunbc/host_convergence_census.dag @@ -693,10 +693,10 @@ data host_convergence_census_rows: List = [ first_consumer: cite(module_path: "gunbc.fleet_converge_workflow", decl_name: "fleet_converge_workflow_modes"), ), census_row( - identity: cite(module_path: "gunbc.runner_microvm_network_converge", decl_name: "runner_microvm_network_converge_wet"), + identity: cite(module_path: "gunbc.runner_microvm_network_converge", decl_name: "runner_microvm_network_observe_wet"), subject: MicrovmHost, phases: protocol_phases(false, true, false, false, false, false, true), - current_authority: cite(module_path: "gunbc.runner_microvm_network_converge", decl_name: "runner_microvm_network_converge_wet"), + current_authority: cite(module_path: "gunbc.runner_microvm_network_converge", decl_name: "runner_microvm_network_observe_wet"), persistent_host_effect: ReachesPersistentHostMutation, disposition: BindToAuthority { target: fleet_plan() }, first_consumer: cite(module_path: "gunbc.fleet_converge_workflow", decl_name: "fleet_converge_workflow_modes"), diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 5a9c778b2e9..740cda9ffa0 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -584,7 +584,7 @@ fn runner_microvm_nft_ruleset_text_for(host: HostEgressGrain) -> String { // slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by a // witness folding a grain, and not by any per-attempt step installing a table. data host_ruleset_installer_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: the microvm_network_converge fleet-converge mode (gunbc.runner_microvm_network_converge runner_microvm_network_converge_wet), which HAS LANDED and is wired as a mode, a CI target and a census row. WHAT REMAINS: it has never executed on a host, so no ruleset has been installed by it and no listing has been read back. TRIGGER: that mode runs on srv1 -- its sudoers roster installed, the operations applied, the listing read back with runner_microvm_nft_list_command -- and records the digest as the converged digest slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by the fold existing, by a witness folding a grain, or by the mode being dispatchable." as NonEmptyStr, + description: "NAMED CONSUMER: the microvm_network_observe fleet-converge mode (gunbc.runner_microvm_network_converge runner_microvm_network_observe_wet), which HAS LANDED and is wired as a mode, a CI target and a census row. WHAT REMAINS: it has never executed on a host, so no ruleset has been installed by it and no listing has been read back. TRIGGER: that mode runs on srv1 -- its sudoers roster installed, the operations applied, the listing read back with runner_microvm_nft_list_command -- and records the digest as the converged digest slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by the fold existing, by a witness folding a grain, or by the mode being dispatchable." as NonEmptyStr, ) fn runner_microvm_shared_host_nft_ruleset_text() -> String { @@ -1005,7 +1005,7 @@ type ConvergedSlotNetwork { // deleting this row and host_ruleset_installer_frontier together. NOT satisfied by a witness // building a receipt. data converged_slot_network_producer_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: gunbc.runner_microvm_network_converge runner_microvm_network_converge_wet, which HAS LANDED: it stages each slot's .netdev, .network and v6 sysctl file and the host's ruleset, unit and forwarding sysctl, applies them as modeled operations, and folds the host readings into ConvergedSlotNetwork or a typed refusal. WHAT REMAINS: no run has produced a receipt, so no ConvergedSlotNetwork exists for any host. TRIGGER: that mode executes on srv1 and records a receipt carrying the live tap, address, route, disable_ipv6, helper standing and nft listing digest at a generation. SUFFICIENT FOR deleting this row and host_ruleset_installer_frontier. NOT satisfied by a witness building a receipt, or by the converge compiling." as NonEmptyStr, + description: "NAMED CONSUMER: gunbc.runner_microvm_network_converge runner_microvm_network_observe_wet, which HAS LANDED: it stages each slot's .netdev, .network and v6 sysctl file and the host's ruleset, unit and forwarding sysctl, applies them as modeled operations, and folds the host readings into ConvergedSlotNetwork or a typed refusal. WHAT REMAINS: no run has produced a receipt, so no ConvergedSlotNetwork exists for any host. TRIGGER: that mode executes on srv1 and records a receipt carrying the live tap, address, route, disable_ipv6, helper standing and nft listing digest at a generation. SUFFICIENT FOR deleting this row and host_ruleset_installer_frontier. NOT satisfied by a witness building a receipt, or by the converge compiling." as NonEmptyStr, ) // --- the sanitation subject: sealed, and joined to the receipt --- diff --git a/dag/gunbc/runner/runner_microvm_network_converge.dag b/dag/gunbc/runner/runner_microvm_network_converge.dag index 5564f68afdc..30188c284c7 100644 --- a/dag/gunbc/runner/runner_microvm_network_converge.dag +++ b/dag/gunbc/runner/runner_microvm_network_converge.dag @@ -179,10 +179,36 @@ fn microvm_network_readback_operations() -> List { ] } +// THE JOB USER IS GRANTED THE READS AND NONE OF THE MUTATIONS, BECAUSE THE PAIR WOULD BE ROOT. +// +// Every file this converge installs is staged by the converge itself, unprivileged, into a directory +// the job user owns -- and the job user is the SAME principal that executes job steps. Granting +// `install /etc/systemd/system/...` beside `systemctl restart` on that unit +// therefore grants arbitrary root execution to anything running in a job: write your own ExecStart= +// into the staged path, then have root run it. The same pair holds for the nft ruleset loaded by that +// unit and for the sysctl files re-read by systemd-sysctl (review 69104). Spelling the source path in +// full narrows nothing when the source is grantee-writable, which is the guarantee +// gunbc.executor_privileged_operation states this type exists to provide. +// +// So the roster carries the READBACKS ONLY. They are reads: they disclose host configuration to a +// principal that already runs on the host, and they cannot install or start anything. The mutating +// half needs a principal the job user cannot impersonate, which this mode does not have today -- +// microvm_network_host_mutation_principal_frontier names it. fn microvm_network_granted_operations(slots: List) -> List { - concat(microvm_network_converge_operations(slots: slots), microvm_network_readback_operations()) + microvm_network_readback_operations() } +// DECLARED FRONTIER (DESIGN 3c). NAMED CONSUMER: this module's apply half. TRIGGER: the microVM +// network mutations run under a principal the job user cannot impersonate -- the administrator over +// the fleet key, as gunbc.runner_host_file_converge already does for root-owned files -- so the +// staged content cannot be substituted by anything executing in a job. SUFFICIENT FOR deleting this +// row and re-admitting the install operations to the grant roster. NOT satisfied by spelling the +// staged source path in full, by staging into another grantee-owned directory, or by checking the +// staged file's digest from the same principal that can rewrite it. +data microvm_network_host_mutation_principal_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: gunbc.runner_microvm_network_converge's apply half. TRIGGER: the microVM network mutations execute under a principal the job user cannot impersonate (the administrator over the fleet key, as runner_host_file_converge does), so staged content cannot be substituted by anything running in a job. SUFFICIENT FOR deleting this row and re-admitting the install operations to the grant roster. NOT satisfied by spelling the staged source in full, by staging into another grantee-owned directory, or by digesting the staged file from the principal that can rewrite it." as NonEmptyStr, +) + // --- reading the host back --- // A READING IS A STRING FROM THE HOST AND A JUDGEMENT ABOUT IT, AND THE TWO ARE SEPARATE FUNCTIONS. @@ -644,7 +670,7 @@ fn converge_generation_reading() -> String { } } -data runner_microvm_network_receipt_path: String = "target/microvm-network-converge.txt" +data runner_microvm_network_receipt_path: String = "target/microvm-network-observe.txt" data runner_microvm_network_receipt_dir: String = "target" @@ -716,7 +742,7 @@ fn converged_slot_text(c: ConvergedSlot) -> String { fn converged_receipt_text(c: ConvergedSlotNetwork) -> String { join([ - "# microvm-network-converge CONVERGED\n", + "# microvm-network-observe OBSERVED\n", "# host=", c.host as String, "\n", "# generation=", c.generation as String, "\n", "# nft_digest=", c.nft_digest.hex as String, "\n", @@ -733,7 +759,7 @@ fn converged_receipt_text(c: ConvergedSlotNetwork) -> String { fn receipt_text(o: ConvergeOutcome) -> String { match o { - ConvergeRefused { refusal: r } => join(["microvm-network-converge REFUSED: ", refusal_text(r: r), "\n"], "") + ConvergeRefused { refusal: r } => join(["microvm-network-observe REFUSED: ", refusal_text(r: r), "\n"], "") ConvergedNetwork { receipt: c } => join([render_converged_slot_network(c: c), converged_receipt_text(c: c)], "") } } @@ -779,42 +805,42 @@ fn expected_host_reading() -> String { fn host_binding_refusal_text(b: HostBinding) -> String { match b { HostBindingUnnamed { detail: d } => - join(["microvm-network-converge: HostBindingUnnamed -- ", d, "; nothing was staged and no privileged operation ran"], "") + join(["microvm-network-observe: HostBindingUnnamed -- ", d, "; nothing was staged and no privileged operation ran"], "") HostBindingRefused { expected: e, observed: o } => - join(["microvm-network-converge: HostBindingRefused -- expected ", e, ", observed ", o, "; nothing was staged and no privileged operation ran"], "") - HostBindingHeld { host: h } => join(["microvm-network-converge: bound to ", h as String], "") + join(["microvm-network-observe: HostBindingRefused -- expected ", e, ", observed ", o, "; nothing was staged and no privileged operation ran"], "") + HostBindingHeld { host: h } => join(["microvm-network-observe: bound to ", h as String], "") } } -fn runner_microvm_network_converge_wet() -> ProcessExit { +fn runner_microvm_network_observe_wet() -> ProcessExit { let host_reading = hostname_short_read(transport: LocalShell) match host_reading { HostnameShortCaptureRefused { reason: r } => - exit_failure(reason: join(["microvm-network-converge: the host could not name itself: ", r], "")) + exit_failure(reason: join(["microvm-network-observe: the host could not name itself: ", r], "")) HostnameShortCaptured { value: name } => match bind_requested_host(expected: expected_host_reading(), observed: name) { HostBindingUnnamed { detail: d } => exit_failure(reason: host_binding_refusal_text(b: HostBindingUnnamed { detail: d })) HostBindingRefused { expected: e, observed: o } => exit_failure(reason: host_binding_refusal_text(b: HostBindingRefused { expected: e, observed: o })) - HostBindingHeld { host: bound } => converge_bound_host_wet(bound: bound) + HostBindingHeld { host: bound } => observe_bound_host_wet(bound: bound) } } } -fn converge_bound_host_wet(bound: HostIdentity) -> ProcessExit { +fn observe_bound_host_wet(bound: HostIdentity) -> ProcessExit { { let generation = converge_generation_reading() if generation == "" { - exit_failure(reason: "microvm-network-converge: no run identity in the environment, so the receipt would carry a generation nobody can compare against") + exit_failure(reason: "microvm-network-observe: no run identity in the environment, so the receipt would carry a generation nobody can compare against") } else { match runner_host_deploy_for(host: bound) { RunnerHostDeployUnmodeled { host: h } => - exit_failure(reason: join(["microvm-network-converge: ", h as String, " is not a modeled runner host"], "")) + exit_failure(reason: join(["microvm-network-observe: ", h as String, " is not a modeled runner host"], "")) RunnerHostDeployRefused { host: h, cause: c } => - exit_failure(reason: join(["microvm-network-converge: ", h as String, " refuses its deploy row: ", c], "")) + exit_failure(reason: join(["microvm-network-observe: ", h as String, " refuses its deploy row: ", c], "")) RunnerHostDeployFound { deploy: d } => - converge_admitted_host_wet(deploy: d, host: bound, generation: generation as NonEmptyStr) + observe_admitted_host_wet(deploy: d, host: bound, generation: generation as NonEmptyStr) } } } @@ -829,26 +855,18 @@ fn runner_microvm_network_slot_identities_of(deploy: RunnerHostDeploy) -> List RunnerSlotIdentity { host: host, slot_index: i }) } -fn converge_admitted_host_wet(deploy: RunnerHostDeploy, host: HostIdentity, generation: NonEmptyStr) -> ProcessExit { - let slots = runner_microvm_network_slot_identities_of(deploy: deploy) - let staged = converge_staged_files( - slots: slots, - host: runner_microvm_shared_host_grain, - tap_owner: deploy.job_user, - ) - let _ = run_shell_commands( - commands: [mkdir_parents_command(path: runner_microvm_staging_dir as String)], - transport: LocalExec, +// THIS MODE OBSERVES AND RECORDS; IT DOES NOT MUTATE. The readbacks are granted because they are +// reads, and they establish exactly what ConvergedSlotNetwork means: what the host IS at a +// generation. The mutating half is NOT granted to this principal and must not be -- see +// microvm_network_host_mutation_principal_frontier -- so nothing here stages a file or runs an +// install. A mode named for a convergence it cannot perform would be the DESIGN section 3 meaning +// fork, which is why the mode is named observe. +fn observe_admitted_host_wet(deploy: RunnerHostDeploy, host: HostIdentity, generation: NonEmptyStr) -> ProcessExit { + record_converge_readback_wet( + slots: runner_microvm_network_slot_identities_of(deploy: deploy), + host: host, + generation: generation, ) - let staged_ok = fold(staged, init: true, f: (acc, f) => acc && stage_file_wet(f: f)) - if !staged_ok { - exit_failure(reason: "microvm-network-converge: a file could not be staged, so NOTHING was installed") - } else { - match apply_executor_privileged_operations_wet(ops: microvm_network_converge_operations(slots: slots)) { - ExitSuccess => record_converge_readback_wet(slots: slots, host: host, generation: generation) - other => other - } - } } fn record_converge_readback_wet(slots: List, host: HostIdentity, generation: NonEmptyStr) -> ProcessExit { @@ -860,11 +878,11 @@ fn record_converge_readback_wet(slots: List, host: HostIdent let _ = run_shell_commands(commands: [mkdir_parents_command(path: runner_microvm_network_receipt_dir)], transport: LocalExec) let w = Filesystem.Write(path: runner_microvm_network_receipt_path, content: text) if !w.success { - exit_failure(reason: join(["microvm-network-converge: receipt write refused: ", w.error], "")) + exit_failure(reason: join(["microvm-network-observe: receipt write refused: ", w.error], "")) } else { match outcome { ConvergedNetwork { receipt: _ } => ExitSuccess - ConvergeRefused { refusal: _ } => exit_failure(reason: join(["microvm-network-converge: ", text], "")) + ConvergeRefused { refusal: _ } => exit_failure(reason: join(["microvm-network-observe: ", text], "")) } } } diff --git a/dag/test/claim/workflow_dispatch_input_witness_test.dag b/dag/test/claim/workflow_dispatch_input_witness_test.dag index 16ecc9cb7f3..014187d9981 100644 --- a/dag/test/claim/workflow_dispatch_input_witness_test.dag +++ b/dag/test/claim/workflow_dispatch_input_witness_test.dag @@ -14,8 +14,8 @@ import gunbc.fleet_converge_workflow { RunnerPasswordSessionToolConverge, fleet_converge_spark_bootstrap_step_if, fleet_converge_mode_options, fleet_converge_reset_observer_dispatch_admission_step, fleet_converge_build_job_steps, - fleet_converge_microvm_network_converge_step, fleet_converge_mode_fleet_ssh_key_demand, - MicrovmNetworkConverge, FleetSshKeyNotConsumed, FleetSshKeyConsumed, + fleet_converge_microvm_network_observe_step, fleet_converge_mode_fleet_ssh_key_demand, + MicrovmNetworkObserve, FleetSshKeyNotConsumed, FleetSshKeyConsumed, fleet_converge_reset_subject_mutation_concurrency_group_expression, fleet_converge_reset_subject_map_json, reset_subject_map_json, fleet_converge_workflow, expected_fleet_converge_yml, @@ -149,9 +149,18 @@ fn mode_input_has_choice_options(v: YamlValue) -> Bool { } } +// THE TRIGGER IS TAKEN THROUGH ITS OPTIONAL. `on[0]` is an Optional, so passing it +// straight into workflow_trigger_entry was a latent type defect that main's own newly landed +// Optional checker now refuses -- a workflow declaring no trigger is a real (if absurd) shape, and +// this claim has to say what it does with it rather than assume one is there. test fn workflow_dispatch_choice_input_projects_options_list() -> Bool { - let entry = workflow_trigger_entry(trigger: fleet_converge_workflow.on[0]) - entry.key == "workflow_dispatch" && mode_input_has_choice_options(v: entry.value) + match fleet_converge_workflow.on.first() { + Absent => false + Present { value: trigger } => { + let entry = workflow_trigger_entry(trigger: trigger) + entry.key == "workflow_dispatch" && mode_input_has_choice_options(v: entry.value) + } + } } // THE COUNT IS KEPT BESIDE THE NAMES DELIBERATELY, and it moved from 2 to 3 because a @@ -437,7 +446,7 @@ fn yaml_scalar_text(v: YamlValue) -> String { } fn microvm_network_step_env_text() -> String { - match fleet_converge_microvm_network_converge_step() { + match fleet_converge_microvm_network_observe_step() { RunStep { name: _, id: _, run: _, shell: _, env: e, working_directory: _, if_condition: _, continue_on_error: _, timeout_minutes: _ } => match e { Present { value: pairs } => join(map(pairs, pair => join([pair.key, "=", yaml_scalar_text(v: pair.value)], "")), "\n") @@ -452,7 +461,7 @@ test fn the_microvm_network_step_binds_the_dispatched_host_into_its_environment( let yml = expected_fleet_converge_yml_content() string_contains(s: env, pattern: "FLEET_CONVERGE_EXPECTED_HOST") && string_contains(s: env, pattern: "github.event.inputs.host") - && string_contains(s: yml, pattern: "id: microvm_network_converge") + && string_contains(s: yml, pattern: "id: microvm_network_observe") && string_contains(s: yml, pattern: "FLEET_CONVERGE_EXPECTED_HOST: $\{\{ github.event.inputs.host \}\}") } @@ -461,7 +470,7 @@ test fn the_microvm_network_step_binds_the_dispatched_host_into_its_environment( // conntrack reads on that host. Materialising the fleet SSH key for it would widen a capability with // no consumer in the source (side-chat hold on #11751). test fn the_microvm_network_mode_claims_no_fleet_ssh_key() -> Bool { - match fleet_converge_mode_fleet_ssh_key_demand(mode: MicrovmNetworkConverge) { + match fleet_converge_mode_fleet_ssh_key_demand(mode: MicrovmNetworkObserve) { FleetSshKeyNotConsumed => true FleetSshKeyConsumed => false } From bf587c2cb5a2232b10708bf4a093688cdcedc22e Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 12:20:32 +0000 Subject: [PATCH 31/36] Regenerate the rosters and workflow: reads granted, mutations withdrawn Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/fleet-converge.yml | 18 +++--- provisioning/srv1/gunbc-ghrunner.sudoers | 67 --------------------- provisioning/srv3/gunbc-ghrunner.sudoers | 75 ------------------------ provisioning/srv4/gunbc-ghrunner.sudoers | 75 ------------------------ 4 files changed, 9 insertions(+), 226 deletions(-) diff --git a/.github/workflows/fleet-converge.yml b/.github/workflows/fleet-converge.yml index a13713208dd..aaabe9abbd2 100644 --- a/.github/workflows/fleet-converge.yml +++ b/.github/workflows/fleet-converge.yml @@ -13,7 +13,7 @@ on: mode: description: plan shows membership hunks; apply executes a prior plan artifact; org_actions_observe validates the org credential and refuses on runner-group drift; org_runner_roster_observe reads the organization's self-hosted runner registrations to exhaustion under the same token and refuses unless the roster is complete; app_control_plane_observe reads the gunbai-ci App registration and webhook config with a short-lived App JWT and refuses on hook-config drift; microvm_host_converge installs the cited Firecracker release on the selected host and refuses the kvm grant by name; guest_image_observe reports the micro-VM base artifacts and their measured digests on the selected host and builds nothing; guest_image_converge builds the runner guest image there; microvm_boot_probe boots that image and reads its serial console; spark_grants and spark_bootstrap prepare the selected Spark; spark_serving_apply applies the promoted fabric groups' vLLM units over the password session, workers before heads; spark_runtime_image_probe pulls the pinned runtime image on the selected Spark and reads its capabilities from inside its own digest, changing nothing; host_reset_return drives a rostered reset subject through its controller and measures the return FROM A PEER, and takes reset_observer rather than host because the observer must not be the subject; runner_host_file_observe reads the runner teardown drop-in, the needrestart deferral and the loaded teardown on the selected host as its administrator and writes nothing; runner_host_file_converge writes whichever of those two files differs, reloads systemd only if the drop-in changed, and refuses unless the manager then reports the declared teardown required: true - options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_converge, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot] + options: [plan, launch_environment_plan, allocation_store_plan, apply, org_actions_observe, app_control_plane_observe, microvm_host_converge, microvm_network_observe, guest_image_observe, guest_image_converge, microvm_boot_probe, spark_grants, spark_bootstrap, spark_serving_apply, spark_runtime_image_probe, dashboard_deploy, rlm_launch_deployment_receipt, host_reset_return, runner_host_file_observe, runner_host_file_converge, site_pxe_edge_observe, site_pxe_edge_converge, runner_password_session_tool_converge, r2_mint_preflight, r2_object_write_mint, org_runner_roster_observe, approval_keyring_converge, mtcollins1_boot] type: choice target: description: "Spark target host for the spark_* modes, naming the administrator credential the run materializes (any rostered Spark: srv5, srv6, srv7, srv8, srv9, srv10, srv11, srv12); apply reads the subject off the plan artifact" @@ -607,24 +607,24 @@ jobs: if: always() && (github.event.inputs.mode == 'microvm_host_converge') timeout-minutes: 10 - name: "Runner micro-VM network: install the slot taps, ruleset and forwarding, then read the host back" - id: microvm_network_converge + id: microvm_network_observe run: | ROOT=$(git rev-parse --show-toplevel 2>/dev/null || pwd) - "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_microvm_network_converge.dag --function runner_microvm_network_converge_wet - cat "$ROOT/target/microvm-network-converge.txt" + "$ROOT/target/release/gunbc" run --source-root "$ROOT/dag" --source-root "$ROOT/src/v2" --entry dag/gunbc/runner/runner_microvm_network_converge.dag --function runner_microvm_network_observe_wet + cat "$ROOT/target/microvm-network-observe.txt" env: FLEET_CONVERGE_EXPECTED_HOST: ${{ github.event.inputs.host }} - if: github.event.inputs.mode == 'microvm_network_converge' + if: github.event.inputs.mode == 'microvm_network_observe' timeout-minutes: 30 - name: Upload runner micro-VM network converge receipt - id: microvm_network_converge_receipt_upload + id: microvm_network_observe_receipt_upload uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f with: - name: runner-microvm-network-converge-receipt - path: target/microvm-network-converge.txt + name: runner-microvm-network-observe-receipt + path: target/microvm-network-observe.txt if-no-files-found: error retention-days: 30 - if: always() && (github.event.inputs.mode == 'microvm_network_converge') + if: always() && (github.event.inputs.mode == 'microvm_network_observe') timeout-minutes: 10 - name: "Runner micro-VM: boot the guest image and read the serial console" id: microvm_boot_probe diff --git a/provisioning/srv1/gunbc-ghrunner.sudoers b/provisioning/srv1/gunbc-ghrunner.sudoers index eb2a97db8af..fe3fa8fff08 100644 --- a/provisioning/srv1/gunbc-ghrunner.sudoers +++ b/provisioning/srv1/gunbc-ghrunner.sudoers @@ -113,73 +113,6 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansr ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansrls /opt/actions-runner-cache ghrunner ALL=(root) NOPASSWD: /usr/sbin/usermod -aG kvm ghrunner ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.netdev /etc/systemd/network/50-gunbc-tap1.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.network /etc/systemd/network/50-gunbc-tap1.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1-ipv6.conf /etc/sysctl.d/50-gunbc-tap1-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.netdev /etc/systemd/network/50-gunbc-tap2.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.network /etc/systemd/network/50-gunbc-tap2.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2-ipv6.conf /etc/sysctl.d/50-gunbc-tap2-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.netdev /etc/systemd/network/50-gunbc-tap3.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.network /etc/systemd/network/50-gunbc-tap3.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3-ipv6.conf /etc/sysctl.d/50-gunbc-tap3-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.netdev /etc/systemd/network/50-gunbc-tap4.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.network /etc/systemd/network/50-gunbc-tap4.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4-ipv6.conf /etc/sysctl.d/50-gunbc-tap4-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.netdev /etc/systemd/network/50-gunbc-tap5.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.network /etc/systemd/network/50-gunbc-tap5.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5-ipv6.conf /etc/sysctl.d/50-gunbc-tap5-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.netdev /etc/systemd/network/50-gunbc-tap6.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.network /etc/systemd/network/50-gunbc-tap6.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6-ipv6.conf /etc/sysctl.d/50-gunbc-tap6-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.netdev /etc/systemd/network/50-gunbc-tap7.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.network /etc/systemd/network/50-gunbc-tap7.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7-ipv6.conf /etc/sysctl.d/50-gunbc-tap7-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.netdev /etc/systemd/network/50-gunbc-tap8.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.network /etc/systemd/network/50-gunbc-tap8.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8-ipv6.conf /etc/sysctl.d/50-gunbc-tap8-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.netdev /etc/systemd/network/50-gunbc-tap9.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.network /etc/systemd/network/50-gunbc-tap9.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9-ipv6.conf /etc/sysctl.d/50-gunbc-tap9-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.netdev /etc/systemd/network/50-gunbc-tap10.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.network /etc/systemd/network/50-gunbc-tap10.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10-ipv6.conf /etc/sysctl.d/50-gunbc-tap10-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.netdev /etc/systemd/network/50-gunbc-tap11.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.network /etc/systemd/network/50-gunbc-tap11.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11-ipv6.conf /etc/sysctl.d/50-gunbc-tap11-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.netdev /etc/systemd/network/50-gunbc-tap12.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.network /etc/systemd/network/50-gunbc-tap12.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12-ipv6.conf /etc/sysctl.d/50-gunbc-tap12-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.netdev /etc/systemd/network/50-gunbc-tap13.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.network /etc/systemd/network/50-gunbc-tap13.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13-ipv6.conf /etc/sysctl.d/50-gunbc-tap13-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.netdev /etc/systemd/network/50-gunbc-tap14.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.network /etc/systemd/network/50-gunbc-tap14.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14-ipv6.conf /etc/sysctl.d/50-gunbc-tap14-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.netdev /etc/systemd/network/50-gunbc-tap15.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.network /etc/systemd/network/50-gunbc-tap15.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15-ipv6.conf /etc/sysctl.d/50-gunbc-tap15-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/microvm.nft /etc/gunbc/microvm.nft -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-microvm-nft.service /etc/systemd/system/gunbc-microvm-nft.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/forward.conf /etc/sysctl.d/50-gunbc-microvm-forward.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart systemd-sysctl.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset ghrunner ALL=(root) NOPASSWD: /usr/sbin/iptables-legacy-save diff --git a/provisioning/srv3/gunbc-ghrunner.sudoers b/provisioning/srv3/gunbc-ghrunner.sudoers index c70480c4df8..54985d50494 100644 --- a/provisioning/srv3/gunbc-ghrunner.sudoers +++ b/provisioning/srv3/gunbc-ghrunner.sudoers @@ -128,81 +128,6 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansr ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansrls /opt/actions-runner-cache ghrunner ALL=(root) NOPASSWD: /usr/sbin/usermod -aG kvm ghrunner ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.netdev /etc/systemd/network/50-gunbc-tap1.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.network /etc/systemd/network/50-gunbc-tap1.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1-ipv6.conf /etc/sysctl.d/50-gunbc-tap1-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.netdev /etc/systemd/network/50-gunbc-tap2.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.network /etc/systemd/network/50-gunbc-tap2.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2-ipv6.conf /etc/sysctl.d/50-gunbc-tap2-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.netdev /etc/systemd/network/50-gunbc-tap3.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.network /etc/systemd/network/50-gunbc-tap3.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3-ipv6.conf /etc/sysctl.d/50-gunbc-tap3-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.netdev /etc/systemd/network/50-gunbc-tap4.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.network /etc/systemd/network/50-gunbc-tap4.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4-ipv6.conf /etc/sysctl.d/50-gunbc-tap4-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.netdev /etc/systemd/network/50-gunbc-tap5.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.network /etc/systemd/network/50-gunbc-tap5.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5-ipv6.conf /etc/sysctl.d/50-gunbc-tap5-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.netdev /etc/systemd/network/50-gunbc-tap6.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.network /etc/systemd/network/50-gunbc-tap6.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6-ipv6.conf /etc/sysctl.d/50-gunbc-tap6-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.netdev /etc/systemd/network/50-gunbc-tap7.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.network /etc/systemd/network/50-gunbc-tap7.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7-ipv6.conf /etc/sysctl.d/50-gunbc-tap7-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.netdev /etc/systemd/network/50-gunbc-tap8.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.network /etc/systemd/network/50-gunbc-tap8.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8-ipv6.conf /etc/sysctl.d/50-gunbc-tap8-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.netdev /etc/systemd/network/50-gunbc-tap9.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.network /etc/systemd/network/50-gunbc-tap9.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9-ipv6.conf /etc/sysctl.d/50-gunbc-tap9-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.netdev /etc/systemd/network/50-gunbc-tap10.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.network /etc/systemd/network/50-gunbc-tap10.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10-ipv6.conf /etc/sysctl.d/50-gunbc-tap10-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.netdev /etc/systemd/network/50-gunbc-tap11.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.network /etc/systemd/network/50-gunbc-tap11.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11-ipv6.conf /etc/sysctl.d/50-gunbc-tap11-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.netdev /etc/systemd/network/50-gunbc-tap12.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.network /etc/systemd/network/50-gunbc-tap12.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12-ipv6.conf /etc/sysctl.d/50-gunbc-tap12-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.netdev /etc/systemd/network/50-gunbc-tap13.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.network /etc/systemd/network/50-gunbc-tap13.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13-ipv6.conf /etc/sysctl.d/50-gunbc-tap13-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.netdev /etc/systemd/network/50-gunbc-tap14.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.network /etc/systemd/network/50-gunbc-tap14.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14-ipv6.conf /etc/sysctl.d/50-gunbc-tap14-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.netdev /etc/systemd/network/50-gunbc-tap15.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.network /etc/systemd/network/50-gunbc-tap15.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15-ipv6.conf /etc/sysctl.d/50-gunbc-tap15-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16.netdev /etc/systemd/network/50-gunbc-tap16.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16.network /etc/systemd/network/50-gunbc-tap16.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16-ipv6.conf /etc/sysctl.d/50-gunbc-tap16-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17.netdev /etc/systemd/network/50-gunbc-tap17.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17.network /etc/systemd/network/50-gunbc-tap17.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17-ipv6.conf /etc/sysctl.d/50-gunbc-tap17-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/microvm.nft /etc/gunbc/microvm.nft -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-microvm-nft.service /etc/systemd/system/gunbc-microvm-nft.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/forward.conf /etc/sysctl.d/50-gunbc-microvm-forward.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart systemd-sysctl.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset ghrunner ALL=(root) NOPASSWD: /usr/sbin/iptables-legacy-save diff --git a/provisioning/srv4/gunbc-ghrunner.sudoers b/provisioning/srv4/gunbc-ghrunner.sudoers index 813bbf39bda..53ead093363 100644 --- a/provisioning/srv4/gunbc-ghrunner.sudoers +++ b/provisioning/srv4/gunbc-ghrunner.sudoers @@ -119,81 +119,6 @@ ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansr ghrunner ALL=(root) NOPASSWD: /usr/bin/install -d -m 755 -o briansrls -g briansrls /opt/actions-runner-cache ghrunner ALL=(root) NOPASSWD: /usr/sbin/usermod -aG kvm ghrunner ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.netdev /etc/systemd/network/50-gunbc-tap1.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1.network /etc/systemd/network/50-gunbc-tap1.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap1-ipv6.conf /etc/sysctl.d/50-gunbc-tap1-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.netdev /etc/systemd/network/50-gunbc-tap2.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2.network /etc/systemd/network/50-gunbc-tap2.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap2-ipv6.conf /etc/sysctl.d/50-gunbc-tap2-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.netdev /etc/systemd/network/50-gunbc-tap3.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3.network /etc/systemd/network/50-gunbc-tap3.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap3-ipv6.conf /etc/sysctl.d/50-gunbc-tap3-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.netdev /etc/systemd/network/50-gunbc-tap4.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4.network /etc/systemd/network/50-gunbc-tap4.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap4-ipv6.conf /etc/sysctl.d/50-gunbc-tap4-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.netdev /etc/systemd/network/50-gunbc-tap5.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5.network /etc/systemd/network/50-gunbc-tap5.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap5-ipv6.conf /etc/sysctl.d/50-gunbc-tap5-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.netdev /etc/systemd/network/50-gunbc-tap6.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6.network /etc/systemd/network/50-gunbc-tap6.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap6-ipv6.conf /etc/sysctl.d/50-gunbc-tap6-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.netdev /etc/systemd/network/50-gunbc-tap7.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7.network /etc/systemd/network/50-gunbc-tap7.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap7-ipv6.conf /etc/sysctl.d/50-gunbc-tap7-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.netdev /etc/systemd/network/50-gunbc-tap8.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8.network /etc/systemd/network/50-gunbc-tap8.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap8-ipv6.conf /etc/sysctl.d/50-gunbc-tap8-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.netdev /etc/systemd/network/50-gunbc-tap9.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9.network /etc/systemd/network/50-gunbc-tap9.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap9-ipv6.conf /etc/sysctl.d/50-gunbc-tap9-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.netdev /etc/systemd/network/50-gunbc-tap10.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10.network /etc/systemd/network/50-gunbc-tap10.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap10-ipv6.conf /etc/sysctl.d/50-gunbc-tap10-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.netdev /etc/systemd/network/50-gunbc-tap11.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11.network /etc/systemd/network/50-gunbc-tap11.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap11-ipv6.conf /etc/sysctl.d/50-gunbc-tap11-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.netdev /etc/systemd/network/50-gunbc-tap12.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12.network /etc/systemd/network/50-gunbc-tap12.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap12-ipv6.conf /etc/sysctl.d/50-gunbc-tap12-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.netdev /etc/systemd/network/50-gunbc-tap13.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13.network /etc/systemd/network/50-gunbc-tap13.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap13-ipv6.conf /etc/sysctl.d/50-gunbc-tap13-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.netdev /etc/systemd/network/50-gunbc-tap14.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14.network /etc/systemd/network/50-gunbc-tap14.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap14-ipv6.conf /etc/sysctl.d/50-gunbc-tap14-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.netdev /etc/systemd/network/50-gunbc-tap15.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15.network /etc/systemd/network/50-gunbc-tap15.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap15-ipv6.conf /etc/sysctl.d/50-gunbc-tap15-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16.netdev /etc/systemd/network/50-gunbc-tap16.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16.network /etc/systemd/network/50-gunbc-tap16.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap16-ipv6.conf /etc/sysctl.d/50-gunbc-tap16-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17.netdev /etc/systemd/network/50-gunbc-tap17.netdev -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17.network /etc/systemd/network/50-gunbc-tap17.network -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-tap17-ipv6.conf /etc/sysctl.d/50-gunbc-tap17-ipv6.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl reload systemd-networkd -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/microvm.nft /etc/gunbc/microvm.nft -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/gunbc-microvm-nft.service /etc/systemd/system/gunbc-microvm-nft.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/install -m 0644 -o root -g root /var/lib/gunbc/fleet-converge/microvm-network/forward.conf /etc/sysctl.d/50-gunbc-microvm-forward.conf -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl daemon-reload -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart systemd-sysctl.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl enable gunbc-microvm-nft.service -ghrunner ALL=(root) NOPASSWD: /usr/bin/systemctl restart gunbc-microvm-nft.service ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list table inet gunbc_microvm ghrunner ALL=(root) NOPASSWD: /usr/sbin/nft list ruleset ghrunner ALL=(root) NOPASSWD: /usr/sbin/iptables-legacy-save From a9618702a21516a0fa890019a1f59bccccfafb9e Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 13:24:10 +0000 Subject: [PATCH 32/36] The mutation frontier names the folds it covers Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/fleet/fleet_converge_workflow.dag | 14 ++++++++------ dag/gunbc/runner/runner_microvm_network.dag | 4 ++-- .../runner/runner_microvm_network_converge.dag | 10 +--------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/dag/gunbc/fleet/fleet_converge_workflow.dag b/dag/gunbc/fleet/fleet_converge_workflow.dag index d1401445897..dad9643512f 100644 --- a/dag/gunbc/fleet/fleet_converge_workflow.dag +++ b/dag/gunbc/fleet/fleet_converge_workflow.dag @@ -1016,14 +1016,16 @@ fn fleet_converge_microvm_host_converge_step() -> Step { } } -// THE NETWORK CONVERGE IS ITS OWN MODE, NOT A STEP OF THE HOST CONVERGE. The host converge installs -// the VMM; this installs the slots' taps, their v6 sysctls, the host ruleset and forwarding, and -// then READS the host back into a ConvergedSlotNetwork receipt. They are separate dispatches because -// they fail for different reasons and an operator routinely wants one without the other: a -// Firecracker release that will not verify says nothing about a tap that will not carry an address. +// THIS MODE READS; IT INSTALLS NOTHING. The microVM network mutations are not granted to the job +// user and must not be -- that pair would be root (gunbc.runner_microvm_network_converge +// microvm_network_host_mutation_principal_frontier) -- so this mode reads the slot taps, their +// addresses, routes and v6 standing, the host's forwarding, ruleset digest and helper standing, and +// records them as a ConvergedSlotNetwork receipt. It is its own dispatch rather than a step of the +// host converge because the two fail for different reasons and an operator routinely wants one +// without the other: a Firecracker release that will not verify says nothing about a tap's address. fn fleet_converge_microvm_network_observe_step() -> Step { RunStep { - name: Present { value: "Runner micro-VM network: install the slot taps, ruleset and forwarding, then read the host back" }, + name: Present { value: "Runner micro-VM network: read the slot taps, ruleset and forwarding back into a receipt (installs nothing)" }, id: Present { value: "microvm_network_observe" }, run: gunbc_ci_runner_microvm_network_observe_invoke(), shell: none, diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 740cda9ffa0..8f465a10884 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -584,7 +584,7 @@ fn runner_microvm_nft_ruleset_text_for(host: HostEgressGrain) -> String { // slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by a // witness folding a grain, and not by any per-attempt step installing a table. data host_ruleset_installer_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: the microvm_network_observe fleet-converge mode (gunbc.runner_microvm_network_converge runner_microvm_network_observe_wet), which HAS LANDED and is wired as a mode, a CI target and a census row. WHAT REMAINS: it has never executed on a host, so no ruleset has been installed by it and no listing has been read back. TRIGGER: that mode runs on srv1 -- its sudoers roster installed, the operations applied, the listing read back with runner_microvm_nft_list_command -- and records the digest as the converged digest slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by the fold existing, by a witness folding a grain, or by the mode being dispatchable." as NonEmptyStr, + description: "NAMED CONSUMER: whichever mode installs this ruleset once the mutating half has a principal to run under -- microvm_network_observe reads and records but installs NOTHING, because granting the job user install+restart over files it also stages would be granting root (review 69104). WHAT REMAINS: no ruleset has been installed by any modeled route, so no listing has been read back as the converged digest. TRIGGER: the capability named by microvm_network_host_mutation_principal_frontier holds and that route installs runner_microvm_nft_ruleset_text_for(the host's HostEgressGrain), reads the listing back with runner_microvm_nft_list_command, and records that digest as the converged digest slot_network_readback compares against. SUFFICIENT FOR deleting this row. NOT satisfied by the observe mode running, by the fold existing, or by a witness folding a grain." as NonEmptyStr, ) fn runner_microvm_shared_host_nft_ruleset_text() -> String { @@ -1005,7 +1005,7 @@ type ConvergedSlotNetwork { // deleting this row and host_ruleset_installer_frontier together. NOT satisfied by a witness // building a receipt. data converged_slot_network_producer_frontier: DissolutionCondition = unbound_dissolution( - description: "NAMED CONSUMER: gunbc.runner_microvm_network_converge runner_microvm_network_observe_wet, which HAS LANDED: it stages each slot's .netdev, .network and v6 sysctl file and the host's ruleset, unit and forwarding sysctl, applies them as modeled operations, and folds the host readings into ConvergedSlotNetwork or a typed refusal. WHAT REMAINS: no run has produced a receipt, so no ConvergedSlotNetwork exists for any host. TRIGGER: that mode executes on srv1 and records a receipt carrying the live tap, address, route, disable_ipv6, helper standing and nft listing digest at a generation. SUFFICIENT FOR deleting this row and host_ruleset_installer_frontier. NOT satisfied by a witness building a receipt, or by the converge compiling." as NonEmptyStr, + description: "NAMED CONSUMER: gunbc.runner_microvm_network_converge runner_microvm_network_observe_wet, which HAS LANDED: it binds the dispatched host to the machine, reads the taps, addresses, routes, v6 standing, forwarding, ruleset digest and helper standing, and folds them into ConvergedSlotNetwork or a typed refusal. WHAT REMAINS: no run has produced a receipt, so no ConvergedSlotNetwork exists for any host. The staging and install folds beside it are NOT executed by that mode and are covered by microvm_network_host_mutation_principal_frontier, not by this row. TRIGGER: that observe mode executes on srv1 and records a receipt carrying the live readings at a generation. SUFFICIENT FOR deleting this row. NOT satisfied by a witness building a receipt, or by the module compiling." as NonEmptyStr, ) // --- the sanitation subject: sealed, and joined to the receipt --- diff --git a/dag/gunbc/runner/runner_microvm_network_converge.dag b/dag/gunbc/runner/runner_microvm_network_converge.dag index 30188c284c7..522270eaf22 100644 --- a/dag/gunbc/runner/runner_microvm_network_converge.dag +++ b/dag/gunbc/runner/runner_microvm_network_converge.dag @@ -32,7 +32,6 @@ import gunbc.runner_host_deploy { RunnerHostDeploy, runner_host_deploy_for, runner_index_seq, RunnerHostDeployFound, RunnerHostDeployUnmodeled, RunnerHostDeployRefused, } -import gunbc.executor_privileged_apply { apply_executor_privileged_operations_wet } import gunbc.executor_privileged_apply { executor_privileged_operation_command } import extdeps.network.ipv4 { render_ipv4_address, render_ipv4_prefix } import gunbc.build_cache_instance { RunnerSlotIdentity, runner_slot_instance_name } @@ -206,7 +205,7 @@ fn microvm_network_granted_operations(slots: List) -> List String { } } -// STAGE, APPLY, READ, RECORD -- and refuse at the first step that does not hold. The staging write -// is the executor's own file under its own directory, so it needs no grant; everything that touches -// the host outside that directory is a modeled operation. -fn stage_file_wet(f: StagedFile) -> Bool { - let w = Filesystem.Write(path: f.path as String, content: f.content) - w.success -} // THE DISPATCH NAMES A HOST AND THE MACHINE NAMES ITSELF, AND THE RUN REFUSES UNLESS THEY AGREE. // This mode mutates host-global state -- the nft table, forwarding, networkd's links -- so applying From fc721c3a2889fdcb2c1c40bdd9d501b96acb9d7a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 13:46:58 +0000 Subject: [PATCH 33/36] The ssh-key classification is named as flipping with the mutation principal, not a standing fact Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_microvm_network_converge.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag/gunbc/runner/runner_microvm_network_converge.dag b/dag/gunbc/runner/runner_microvm_network_converge.dag index 522270eaf22..6832cbe67e8 100644 --- a/dag/gunbc/runner/runner_microvm_network_converge.dag +++ b/dag/gunbc/runner/runner_microvm_network_converge.dag @@ -205,7 +205,7 @@ fn microvm_network_granted_operations(slots: List) -> List Date: Sun, 20 Sep 2026 14:08:16 +0000 Subject: [PATCH 34/36] The census row says what the observe fold does: it mutates nothing; the grant fold drops its unused slot roster Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/host_convergence_census.dag | 4 ++-- dag/gunbc/runner/runner_host_grants.dag | 6 ++---- dag/gunbc/runner/runner_microvm_network_converge.dag | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dag/gunbc/host_convergence_census.dag b/dag/gunbc/host_convergence_census.dag index 2f521e76c4a..3ccbe4c384e 100644 --- a/dag/gunbc/host_convergence_census.dag +++ b/dag/gunbc/host_convergence_census.dag @@ -695,9 +695,9 @@ data host_convergence_census_rows: List = [ census_row( identity: cite(module_path: "gunbc.runner_microvm_network_converge", decl_name: "runner_microvm_network_observe_wet"), subject: MicrovmHost, - phases: protocol_phases(false, true, false, false, false, false, true), + phases: protocol_phases(false, true, false, false, false, false, false), current_authority: cite(module_path: "gunbc.runner_microvm_network_converge", decl_name: "runner_microvm_network_observe_wet"), - persistent_host_effect: ReachesPersistentHostMutation, + persistent_host_effect: DoesNotMutateHost, disposition: BindToAuthority { target: fleet_plan() }, first_consumer: cite(module_path: "gunbc.fleet_converge_workflow", decl_name: "fleet_converge_workflow_modes"), ), diff --git a/dag/gunbc/runner/runner_host_grants.dag b/dag/gunbc/runner/runner_host_grants.dag index 12139ce362e..4a9a2dcc5f1 100644 --- a/dag/gunbc/runner/runner_host_grants.dag +++ b/dag/gunbc/runner/runner_host_grants.dag @@ -48,9 +48,7 @@ import gunbc.runner_host_deploy { RunnerHostDeploy, RunnerHostSpec, runner_host_ import product.placement_supply { HostIdentity } import gunbc.runner_slot_provision { desired_runner_slot_members, runner_slot_instance_dir } import gunbc.runner_host_deploy { runner_index_seq } -import gunbc.runner_microvm_network_converge { - microvm_network_granted_operations, runner_microvm_network_slot_identities_of, -} +import gunbc.runner_microvm_network_converge { microvm_network_granted_operations } import gunbc.runner_unit { runner_unit_name_of_registration } // THE MANAGED STATE DIRECTORY, DECLARED ONCE. It is currently spelled THREE times in @@ -326,7 +324,7 @@ fn runner_host_privileged_operations(deploy: RunnerHostDeploy) -> List List { // principal that already runs on the host, and they cannot install or start anything. The mutating // half needs a principal the job user cannot impersonate, which this mode does not have today -- // microvm_network_host_mutation_principal_frontier names it. -fn microvm_network_granted_operations(slots: List) -> List { +fn microvm_network_granted_operations() -> List { microvm_network_readback_operations() } From 3cfea8efed268c8d0ee728ddc1b0e58c9b1cf5d3 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 20 Sep 2026 14:58:15 +0000 Subject: [PATCH 35/36] Restore the single SystemdServiceDirective header ahead of its full arm list My merge resolution moved the RemainAfterExit annotation above the type and left a second two-arm header below, orphaning ~25 arms of a widely consumed coproduct (review 69165). Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/unit_file.dag | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dag/extdeps/systemd/unit_file.dag b/dag/extdeps/systemd/unit_file.dag index f5baa015531..3a8d91f279e 100644 --- a/dag/extdeps/systemd/unit_file.dag +++ b/dag/extdeps/systemd/unit_file.dag @@ -275,6 +275,9 @@ fn systemd_readiness_from_active_state(mode: SystemdServiceMode) -> ReadinessFro // unit stays active after the process exits -- which is how a unit whose job is to INSTALL state // (load a ruleset, apply a sysctl) reports that the state is in place rather than that a program is // running. The boolean renders as systemd's own yes/no rather than true/false. +type SystemdServiceDirective + = ServiceType { mode: SystemdServiceMode } + | RemainAfterExit { active: Bool } | ExecStartPre { command: NonEmptyStr } | ExecStart { command: NonEmptyStr } | ExecStartPost { command: NonEmptyStr } @@ -299,9 +302,6 @@ fn systemd_readiness_from_active_state(mode: SystemdServiceMode) -> ReadinessFro | RestartSec { duration: NonEmptyStr } | ServiceTasksMax { count: Int } | TimeoutStopSec { duration: NonEmptyStr } -type SystemdServiceDirective - = ServiceType { mode: SystemdServiceMode } - | RemainAfterExit { active: Bool } // THE THREE RESOURCE-DISCIPLINE DIRECTIVES, AND WHY THE MIDDLE ONE IS A COPRODUCT AND NOT A STRING. From cd2041cc69a46139e985d509e851599f76727b3e Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 02:02:42 +0000 Subject: [PATCH 36/36] The wire-parse family carries a typed frontier row, not an annotation Review 69448 is right: parse_converged_slot_network and the eleven folds around it have no production consumer -- git grep finds only the definition, one prose comment and the witness file -- while the render half beside them is genuinely consumed by receipt_text. Every other unconsumed surface in this module already states that in a typed DissolutionCondition row (host_ruleset_installer_frontier, converged_slot_network_producer_frontier, slot_network_readback_consumer_frontier, guest_resolver_configuration_frontier); the parse half rested on a // annotation, and DESIGN section 4c is explicit that an annotation is never evidence that a machine claim holds, because no Accepted program can read one. So the gap now carries the same row shape the module already uses: it names the consumer and why that consumer cannot avoid the parse (the lifecycle controller runs in a LATER PROCESS than the converge that wrote the receipt, so it cannot hold the in-memory value and must recover it from the artifact), enumerates the twelve folds it covers, and states a trigger in the capability form section 4b(3) requires, with an explicit NOT-satisfied-by clause that rules out the three ways this could be declared discharged without the controller existing. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/runner/runner_microvm_network.dag | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dag/gunbc/runner/runner_microvm_network.dag b/dag/gunbc/runner/runner_microvm_network.dag index 5495b5b25bd..00a27009138 100644 --- a/dag/gunbc/runner/runner_microvm_network.dag +++ b/dag/gunbc/runner/runner_microvm_network.dag @@ -1415,6 +1415,15 @@ data guest_resolver_configuration_frontier: DissolutionCondition = unbound_disso // tap this fleet would never mint is a receipt about something else. data converged_slot_network_wire_schema: NonEmptyStr = "gunbc.converged_slot_network.v1" as NonEmptyStr +// THE PARSE HALF IS UNCONSUMED AND SAYS SO IN A ROW, NOT IN THIS PROSE. The render half has a +// production consumer (receipt_text folds it into the persisted artifact); the parse half does not +// yet, and DESIGN section 4c is explicit that an annotation is never evidence that a machine claim +// holds. So the gap carries the same typed row every other unconsumed surface in this module +// carries, rather than resting on the paragraph above (review 69448). +data converged_slot_network_parse_consumer_frontier: DissolutionCondition = unbound_dissolution( + description: "NAMED CONSUMER: the microVM slot lifecycle controller (gunbc#11670), which runs in a LATER PROCESS than the converge that wrote the receipt and therefore cannot hold the in-memory ConvergedSlotNetwork -- it must read the persisted artifact back and recover the VALUE, because slot_network_sanitation_subject takes a receipt rather than prose. COVERS the whole wire-parse family: parse_converged_slot_network, parse_converged_body, parse_helper_standing, parse_converged_slot_line, parse_converged_slot_tail, parse_helper_surface, parse_auto_assignment_wire, parse_tap_ipv6_wire, candidate_named, wire_value, wire_lines and drop_prefix. WHAT REMAINS: no controller exists to read a receipt, and no receipt has been written by any run, so the only executing callers are witnesses. TRIGGER: that controller reads a receipt this fleet wrote, recovers the ConvergedSlotNetwork through parse_converged_slot_network, and derives the slot's network fact through slot_network_sanitation_subject on the value it parsed rather than on one it built. SUFFICIENT FOR deleting this row. NOT satisfied by the render half having a consumer, by a witness round-tripping render into parse, by the module compiling, or by the annotation above naming the controller." as NonEmptyStr, +) + fn helper_surface_wire(r: HelperSurfaceReading) -> String { match r { HelperSurfaceCounted { count: n } => join(["counted:", to_string(n)], "")