From 0f4624278505146ed5f89e43ffd48758c01524b8 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 30 Aug 2026 18:17:10 +0000 Subject: [PATCH 01/50] FCI-1: bind exact Work through durable cell reservation --- dag/gunbc/fabric/fabric_ci_program.dag | 2 +- .../fabric/fabric_required_build_cell.dag | 361 ++++++++++++ .../fabric_control_plane_live_probe.dag | 550 +++++++++++++++++- tools/fabric_ci_fci1_live_instrument.sh | 79 +++ 4 files changed, 986 insertions(+), 6 deletions(-) create mode 100644 dag/gunbc/fabric/fabric_required_build_cell.dag create mode 100755 tools/fabric_ci_fci1_live_instrument.sh diff --git a/dag/gunbc/fabric/fabric_ci_program.dag b/dag/gunbc/fabric/fabric_ci_program.dag index 65ff0ad5a30..eacf1256af7 100644 --- a/dag/gunbc/fabric/fabric_ci_program.dag +++ b/dag/gunbc/fabric/fabric_ci_program.dag @@ -18,7 +18,7 @@ type FabricCiGate { fn fabric_ci_gates() -> List { [ FabricCiGate { id: "FCI-0", owned_question: "What exactly will the fabric run?", exit_predicate: "The fabric possesses one exact required-witnesses-build Work carrying every execution-defining coordinate; moving or incomplete subjects refuse.", positive_control: "Identical semantic inputs derive the identical WorkKey.", discriminating_mutations: ["command-word", "environment-entry", "verified-tree", "toolchain-identity", "output-declaration"], frozen_output_identities: ["WorkKey", "source-tree-digest", "environment-materialization-digest"], non_goals: ["ExecutionGrant", "executor", "process-start", "cell-realization", "workflow-cutover"] }, - FabricCiGate { id: "FCI-1", owned_question: "What bounded owned cell will realize this Work?", exit_predicate: "The exact Work realizes in exactly one bounded cell whose resources and ownership are explicit.", positive_control: "One exact Work realizes one bounded owned cell.", discriminating_mutations: ["unbounded-cell", "unowned-cell", "multiple-cells", "work-mismatch"], frozen_output_identities: ["cell-identity", "cell-resource-envelope", "cell-owner"], non_goals: ["ExecutionGrant", "process-start", "multi-host-scheduling"] }, + FabricCiGate { id: "FCI-1", owned_question: "What durable reservation binds this exact Work to one already-converged bounded owned cell?", exit_predicate: "After installed-authorization and exact-path owner/group readback succeed, one exact Work is bound to exactly one selected substrate-converged cell by one committed durable reservation generation. An independent observer reads the same reservation generation, demand/offer payload and content digest after the submitter terminates; release removes only the holding and leaves the persistent cell substrate unchanged. Directory mode remains explicitly unobserved and no Work process starts.", positive_control: "The exact-tree evidence calibration holds; installed sudoers and cell owner/group readbacks match; a production reservation survives submitter termination, is independently re-observed byte-identically, releases through the production path, and leaves the cell substrate unchanged.", discriminating_mutations: ["reservation-state-owned-by-submitter", "reservation-generation-changed", "reservation-payload-work-mismatch", "selected-offer-cell-mismatch", "cell-substrate-not-converged", "cell-resource-boundary-digest-mismatch", "reservation-release-not-observed", "reservation-release-destroys-cell-substrate", "multiple-cells", "unowned-cell"], frozen_output_identities: ["WorkKey", "DemandKey", "OfferKey", "CellId/slot-key", "committed-CAS-generation", "reservation-content-digest", "cell-resource-boundary-digest", "cell-owner-identity", "reservation-realization-digest"], non_goals: ["cell-supervisor", "passive-lease-unit", "InvocationID", "MainPID", "process-start-identity", "blocking-wait", "ExecutionGrant", "Work-process-start", "Attempt", "execution-Receipt", "source-to-application-structural-type-safety"] }, FabricCiGate { id: "FCI-2", owned_question: "Which sole grant authorizes this Work?", exit_predicate: "Selection, reservation and commitment yield exactly one fenced ExecutionGrant for the exact Work.", positive_control: "One admitted offer yields one committed grant.", discriminating_mutations: ["duplicate-grant", "expired-reservation", "work-mismatch"], frozen_output_identities: ["ExecutionGrant", "lease-epoch"], non_goals: ["process-start", "GitHub-conclusion"] }, FabricCiGate { id: "FCI-3", owned_question: "What causes the target process?", exit_predicate: "A persistent executor re-reads the committed Grant before starting; absent, corrupt or expired Grant yields zero target processes.", positive_control: "One live committed grant starts exactly one target process.", discriminating_mutations: ["grant-removed", "grant-corrupt", "grant-expired"], frozen_output_identities: ["attempt-identity", "process-observation"], non_goals: ["GitHub-conclusion", "multiple-host-scheduling"] }, FabricCiGate { id: "FCI-4", owned_question: "How is execution admitted once, deduplicated, released and sanitized?", exit_predicate: "One fenced Receipt is admitted at most once; duplicate delivery cannot duplicate standing, and the cell is released only after per-attempt sanitation is confirmed.", positive_control: "One matching Receipt is admitted once and its sanitized cell returns to supply.", discriminating_mutations: ["duplicate-receipt", "foreign-receipt", "release-before-sanitation", "failed-sanitation"], frozen_output_identities: ["admitted-receipt", "deduplication-key", "release-receipt", "sanitation-receipt"], non_goals: ["shadow-GitHub-job", "required-check-cutover"] }, diff --git a/dag/gunbc/fabric/fabric_required_build_cell.dag b/dag/gunbc/fabric/fabric_required_build_cell.dag new file mode 100644 index 00000000000..c373f7c2e4f --- /dev/null +++ b/dag/gunbc/fabric/fabric_required_build_cell.dag @@ -0,0 +1,361 @@ +module gunbc.fabric_required_build_cell + +import std.types { Bool, Int, List, NonEmptyStr, String, list_length } +import std.content_hash { + ContentHash, ContentHashEqual, ContentHashDifferent, ContentHashCrossFamilyIncomparable, + compare_content_hash, content_hash_of_value, content_hash_from_structural_digest, + serialize_content_hash, +} +import std.durable_compare_and_set { CasObservedReadable, CasObservedUnreadable, CasReadableAbsent, CasReadablePresent } +import product.fabric.identity { FabricIdentity, DemandKey, OfferKey, WorkKey } +import product.fabric.demand { Demand } +import product.fabric.supply { SupplierOffer } +import product.fabric.sanitation { CellId } +import product.placement_supply { HostIdentity } +import gunbc.ownership { Owned } +import gunbc.build_cache_instance { RunnerSlotIdentity, runner_slot_identity_equal } +import gunbc.durable_cas_file_store { observe_cas_slot_state } +import gunbc.fabric_control_plane { + fabric_cell_allocation_key, fabric_money_reservation_ref, + CellSlotDecoded, CellSlotUndecodable, CellHeld, CellFree, decode_cell_slot_payload, +} +import gunbc.fabric_cell_acquire { fabric_cell_probe_wet } +import gunbc.fabric_cell_observation_admission { + FabricCellObservationAdmitted, FabricCellObservationDenied, FabricCellObservationNotApplicable, + fabric_cell_observation_outcome, fabric_cell_observed_population_members, admit_fabric_cell_probe, +} +import gunbc.fabric_cell_converge { + FabricCellMember, FabricCellRootAddress, FabricCellAttemptRootAddress, FabricCellResourceBoundaryAddress, + fabric_cell_id_for_slot, fabric_cell_member_key, fabric_cell_member_ownership, fabric_cell_resource_address_equal, +} +import v2.std.optional { Present, Absent } +import extdeps.posix.identity { PosixOwnerSpec } + +// Mode is intentionally a separate coordinate from owner/group. FCI-1 has a production owner/group +// observer, but no mode observer; carrying this arm makes that absence part of the accepted value +// rather than a prose omission or an inflated directory-metadata claim. +type DirectoryModeStanding + = DirectoryModeUnobserved { path: NonEmptyStr, desired_mode: NonEmptyStr, trigger: NonEmptyStr } + +type DirectoryOwnerGroupStanding + = DirectoryOwnerGroupObserved { path: NonEmptyStr, owner: PosixOwnerSpec } + | DirectoryOwnerGroupUnobserved { path: NonEmptyStr, reason: NonEmptyStr } + +type DirectoryMetadataStanding { + owner_group: DirectoryOwnerGroupStanding + mode: DirectoryModeStanding +} + +// This is the pre-reservation standing of the persistent cell, not a claim about the allocation +// slot. Clean means the production observer admitted the exact three-member substrate and found no +// foreign member; absent is retained as a distinct observation for the privileged creation path. +// FCI-1 requires Clean because its subject is an already-converged cell and never creates one. +type FabricCellPreReservationStanding + = FabricCellPreReservationClean { cell: CellId, members_content: ContentHash } + | FabricCellPreReservationAbsent { cell: CellId } + | FabricCellPreReservationRefused { cell: CellId } + +// Mode remains outside the observed guarantee until the same exact-path producer reports it. +fn fci1_directory_mode_unobserved(path: NonEmptyStr, desired_mode: NonEmptyStr) -> DirectoryModeStanding { + DirectoryModeUnobserved { + path: path, + desired_mode: desired_mode, + trigger: "exact-path production mode observation and comparison lands", + } +} + +// FCI-1 joins two existing authorities. The CAS slot is the sole Work-to-cell holding; the +// converged-cell observation is the sole account of the persistent bounded substrate. There is no +// supervisor, passive unit, waiting process, Grant, Attempt, or execution Receipt in this module. +// The production constructor reads the allocation slot and admitted cell probe itself. Generation, +// payload, content digest, substrate members and modeled teardown ownership enter from their +// existing producers; none is a request field. Release changes only the CAS holding and must leave +// the persistent substrate byte-identical. + +type RequiredBuildCellSnapshot sole_constructor { + work: FabricIdentity + demand: FabricIdentity + offer: FabricIdentity + cell: CellId + slot_key: NonEmptyStr + generation: Int + reservation_payload: NonEmptyStr + reservation_content: ContentHash + cell_members: List + resource_boundary_digest: NonEmptyStr +} + +type RequiredBuildCellObservation + = RequiredBuildCellObserved { snapshot: RequiredBuildCellSnapshot } + | ReservationAbsent { slot_key: NonEmptyStr } + | ReservationUnreadable { slot_key: NonEmptyStr } + | ReservationPayloadUndecodable { slot_key: NonEmptyStr } + | ReservationPayloadWorkMismatch { slot_key: NonEmptyStr } + | SelectedOfferCellMismatch { slot_key: NonEmptyStr, executor: NonEmptyStr } + | CellSubstrateNotConverged { cell: CellId } + | UnownedCell { cell: CellId } + +fn members_for_slot(members: List, slot: RunnerSlotIdentity) -> List { + filter(members, m => runner_slot_identity_equal(a: m.slot, b: slot)) +} + +fn cell_member_shape_complete(members: List) -> Bool { + list_length(items: members) == 3 + && list_length(items: filter(members, m => fabric_cell_resource_address_equal(a: m.address, b: FabricCellRootAddress))) == 1 + && list_length(items: filter(members, m => fabric_cell_resource_address_equal(a: m.address, b: FabricCellAttemptRootAddress))) == 1 + && list_length(items: filter(members, m => fabric_cell_resource_address_equal(a: m.address, b: FabricCellResourceBoundaryAddress))) == 1 +} + +fn members_owned(members: List) -> Bool { + fold(members, init: true, f: (ok, m) => ok && match fabric_cell_member_ownership(member: m) { + Present { value: Owned } => true + Present { value: _ } => false + Absent => false + }) +} + +fn boundary_digests(members: List) -> List { + fold(members, init: [], f: (acc, m) => + if fabric_cell_resource_address_equal(a: m.address, b: FabricCellResourceBoundaryAddress) { + acc + [m.state_digest] + } else { acc }) +} + +fn required_build_cell_member_wire(member: FabricCellMember) -> NonEmptyStr { + join([ + "member|key=", fabric_cell_member_key(member: member), + "|state=", member.state_digest as String, + ], "") as NonEmptyStr +} + +fn required_build_cell_members_hash(members: List) -> ContentHash { + content_hash_of_value( + value: join(map(members, member => required_build_cell_member_wire(member: member) as String), "\n") as NonEmptyStr, + ) +} + +// Production join: callers select a Work/Demand/Offer/slot, but cannot state the reservation or +// host facts. Both are acquired here from the canonical CAS and converged-cell producers. +func observe_required_build_cell_wet( + root: NonEmptyStr, + host: HostIdentity, + slot: RunnerSlotIdentity, + demand: Demand, + offer: SupplierOffer, +) -> RequiredBuildCellObservation { + let slot_key = fabric_cell_allocation_key(slot: slot) + if (offer.executor as String) != (slot_key as String) { + SelectedOfferCellMismatch { slot_key: slot_key, executor: offer.executor } + } else { + match observe_cas_slot_state(root: root, key: slot_key) { + CasObservedUnreadable { cause: _ } => ReservationUnreadable { slot_key: slot_key } + CasObservedReadable { readable: CasReadableAbsent } => ReservationAbsent { slot_key: slot_key } + CasObservedReadable { readable: CasReadablePresent { version: version } } => + match decode_cell_slot_payload(payload: version.value) { + CellSlotUndecodable { detail: _ } => ReservationPayloadUndecodable { slot_key: slot_key } + CellSlotDecoded { state: CellFree { released_by: _ } } => ReservationAbsent { slot_key: slot_key } + CellSlotDecoded { state: CellHeld { reservation: held } } => + if (held as String) != (fabric_money_reservation_ref(demand: demand.id, offer: offer.id) as String) { + ReservationPayloadWorkMismatch { slot_key: slot_key } + } else { + match fabric_cell_observation_outcome(decision: admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: host))) { + FabricCellObservationDenied { host: _, blocked: _ } => CellSubstrateNotConverged { cell: fabric_cell_id_for_slot(slot: slot) } + FabricCellObservationNotApplicable { host: _ } => CellSubstrateNotConverged { cell: fabric_cell_id_for_slot(slot: slot) } + FabricCellObservationAdmitted { population: population } => + let selected = members_for_slot(members: fabric_cell_observed_population_members(population: population), slot: slot) + if !cell_member_shape_complete(members: selected) { + CellSubstrateNotConverged { cell: fabric_cell_id_for_slot(slot: slot) } + } else { + if !members_owned(members: selected) { + UnownedCell { cell: fabric_cell_id_for_slot(slot: slot) } + } else { + let boundary = boundary_digests(members: selected) + RequiredBuildCellObserved { snapshot: RequiredBuildCellSnapshot { + work: demand.work, + demand: demand.id, + offer: offer.id, + cell: fabric_cell_id_for_slot(slot: slot), + slot_key: slot_key, + generation: version.generation as Int, + reservation_payload: version.value, + reservation_content: version.content, + cell_members: selected, + resource_boundary_digest: boundary[0], + } } + } + } + } + } + } + } + } +} + +type RequiredBuildCellLifetimeVerdict + = RequiredBuildCellLifetimeHeld { snapshot: RequiredBuildCellSnapshot } + | ReservationLifetimeDependentOnSubmitter { slot_key: NonEmptyStr } + | ReservationGenerationChanged { expected_generation: Int, observed_generation: Int } + | ReservationContentChanged { slot_key: NonEmptyStr } + | CellSubstrateChanged { cell: CellId } + | RequiredBuildCellObservationRefused + +// Run-boundary projection. The interpreter cannot transport a coproduct directly: it reports the +// type name, erasing both variant and payload. This total fold is therefore the sole wire authority. +// Each tag is delimiter-terminated and no tag prefixes another, so a consumer cannot accept one +// verdict through another verdict's prefix pattern. +fn required_build_cell_lifetime_verdict_wire(verdict: RequiredBuildCellLifetimeVerdict) -> String { + match verdict { + RequiredBuildCellLifetimeHeld { snapshot: s } => join([ + "held|slot=", s.slot_key as String, + "|generation=", to_string(s.generation), + "|reservation-content=", serialize_content_hash(hash: s.reservation_content) as String, + "|members-content=", serialize_content_hash(hash: required_build_cell_members_hash(members: s.cell_members)) as String, + "|boundary=", s.resource_boundary_digest as String, + ], "") + ReservationLifetimeDependentOnSubmitter { slot_key: key } => + join(["lifetime-dependent|slot=", key as String], "") + ReservationGenerationChanged { expected_generation: expected, observed_generation: observed } => + join(["generation-changed|expected=", to_string(expected), "|observed=", to_string(observed)], "") + ReservationContentChanged { slot_key: key } => join(["reservation-content-changed|slot=", key as String], "") + CellSubstrateChanged { cell: cell } => join(["cell-substrate-changed|cell=", cell as String], "") + RequiredBuildCellObservationRefused => "observation-refused|cause=before-wire-or-live-observation" + } +} + +type RequiredBuildCellLifetimeSample sole_constructor { + slot_key: NonEmptyStr + generation: Int + reservation_payload: NonEmptyStr + reservation_content: ContentHash + cell: CellId + cell_members_content: ContentHash + resource_boundary_digest: NonEmptyStr +} + +type RequiredBuildCellLifetimeSampleDecode + = RequiredBuildCellLifetimeSampleDecoded { sample: RequiredBuildCellLifetimeSample } + | RequiredBuildCellLifetimeSampleUndecodable + +fn required_build_cell_lifetime_sample(observation: RequiredBuildCellObservation) -> RequiredBuildCellLifetimeSampleDecode { + match observation { + RequiredBuildCellObserved { snapshot: s } => RequiredBuildCellLifetimeSampleDecoded { + sample: RequiredBuildCellLifetimeSample { + slot_key: s.slot_key, + generation: s.generation, + reservation_payload: s.reservation_payload, + reservation_content: s.reservation_content, + cell: s.cell, + cell_members_content: required_build_cell_members_hash(members: s.cell_members), + resource_boundary_digest: s.resource_boundary_digest, + }, + } + ReservationAbsent { slot_key: _ } => RequiredBuildCellLifetimeSampleUndecodable + ReservationUnreadable { slot_key: _ } => RequiredBuildCellLifetimeSampleUndecodable + ReservationPayloadUndecodable { slot_key: _ } => RequiredBuildCellLifetimeSampleUndecodable + ReservationPayloadWorkMismatch { slot_key: _ } => RequiredBuildCellLifetimeSampleUndecodable + SelectedOfferCellMismatch { slot_key: _, executor: _ } => RequiredBuildCellLifetimeSampleUndecodable + CellSubstrateNotConverged { cell: _ } => RequiredBuildCellLifetimeSampleUndecodable + UnownedCell { cell: _ } => RequiredBuildCellLifetimeSampleUndecodable + } +} + +// Seven newline-delimited fields, with exact arity on decode. Every field is produced from the +// first live observation. The wire crosses the process-lifetime boundary; it is not evidence by +// itself and becomes usable only after this module validates and reconstructs the sealed sample. +fn required_build_cell_lifetime_sample_wire(sample: RequiredBuildCellLifetimeSample) -> String { + join([ + sample.slot_key as String, + to_string(sample.generation), + sample.reservation_payload as String, + serialize_content_hash(hash: sample.reservation_content) as String, + sample.cell as String, + serialize_content_hash(hash: sample.cell_members_content) as String, + sample.resource_boundary_digest as String, + ], "\n") +} + +fn decode_required_build_cell_lifetime_sample(wire: String) -> RequiredBuildCellLifetimeSampleDecode { + let fields = split(s: wire, delimiter: "\n") + if list_length(items: fields) != 7 { + RequiredBuildCellLifetimeSampleUndecodable + } else { + match parse_int(s: fields.skip(n: 1).first()) { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: generation } => + match content_hash_from_structural_digest(digest: fields.skip(n: 3).first()) { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: reservation_content } => + match content_hash_from_structural_digest(digest: fields.skip(n: 5).first()) { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: members_content } => + match fields.first() { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: slot_key } => if slot_key == "" { RequiredBuildCellLifetimeSampleUndecodable } else { + match fields.skip(n: 2).first() { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: payload } => if payload == "" { RequiredBuildCellLifetimeSampleUndecodable } else { + match fields.skip(n: 4).first() { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: cell } => if cell == "" { RequiredBuildCellLifetimeSampleUndecodable } else { + match fields.skip(n: 6).first() { + Absent => RequiredBuildCellLifetimeSampleUndecodable + Present { value: boundary } => if boundary == "" { RequiredBuildCellLifetimeSampleUndecodable } else { + RequiredBuildCellLifetimeSampleDecoded { + sample: RequiredBuildCellLifetimeSample { + slot_key: slot_key as NonEmptyStr, + generation: generation, + reservation_payload: payload as NonEmptyStr, + reservation_content: reservation_content, + cell: cell as CellId, + cell_members_content: members_content, + resource_boundary_digest: boundary as NonEmptyStr, + }, + } + } + } + } + } + } + } + } + } + } + } + } + } +} + +fn required_build_cell_lifetime_verdict( + before: RequiredBuildCellLifetimeSample, + after: RequiredBuildCellObservation, +) -> RequiredBuildCellLifetimeVerdict { + match after { + ReservationAbsent { slot_key: _ } => ReservationLifetimeDependentOnSubmitter { slot_key: before.slot_key } + ReservationUnreadable { slot_key: _ } => ReservationLifetimeDependentOnSubmitter { slot_key: before.slot_key } + RequiredBuildCellObserved { snapshot: a } => + if before.generation != a.generation { + ReservationGenerationChanged { expected_generation: before.generation, observed_generation: a.generation } + } else { + match compare_content_hash(left: before.reservation_content, right: a.reservation_content) { + ContentHashDifferent => ReservationContentChanged { slot_key: before.slot_key } + ContentHashCrossFamilyIncomparable => ReservationContentChanged { slot_key: before.slot_key } + ContentHashEqual => if before.reservation_payload != a.reservation_payload { + ReservationContentChanged { slot_key: before.slot_key } + } else { + match compare_content_hash(left: before.cell_members_content, right: required_build_cell_members_hash(members: a.cell_members)) { + ContentHashDifferent => CellSubstrateChanged { cell: before.cell } + ContentHashCrossFamilyIncomparable => CellSubstrateChanged { cell: before.cell } + ContentHashEqual => if before.resource_boundary_digest != a.resource_boundary_digest { + CellSubstrateChanged { cell: before.cell } + } else { RequiredBuildCellLifetimeHeld { snapshot: a } } + } + } + } + } + ReservationPayloadUndecodable { slot_key: _ } => RequiredBuildCellObservationRefused + ReservationPayloadWorkMismatch { slot_key: _ } => RequiredBuildCellObservationRefused + SelectedOfferCellMismatch { slot_key: _, executor: _ } => RequiredBuildCellObservationRefused + CellSubstrateNotConverged { cell: _ } => RequiredBuildCellObservationRefused + UnownedCell { cell: _ } => RequiredBuildCellObservationRefused + } +} diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 1fb7a3394d7..f32f7a4bae2 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -1,6 +1,7 @@ module tools.fabric_control_plane_live_probe import std.types { Bool, Int, List, NonEmptyStr, String } +import std.process { ProcessExit, ExitSuccess, exit_failure } import extdeps.currency.currency { Usd } import std.measure { MoneyAmountMicro, MoneyOnce, MoneyPerSecond, money_amount_micro, @@ -10,11 +11,18 @@ import product.fabric.work { Shape, HardRequirements } import product.fabric.envelope { unstated_envelope } import product.fabric.isolation { IsolationProfile } import product.fabric.supply { SupplierOffer, QuotedPerSecond, ObservedSupply } -import std.durable_compare_and_set { CasExpectation, ExpectSlotAbsent, ExpectSlotGeneration } +import std.durable_compare_and_set { + CasExpectation, ExpectSlotAbsent, ExpectSlotGeneration, + CasObservedReadable, CasObservedUnreadable, CasReadableAbsent, CasReadablePresent, + CasUnreadableSlot, CasUnreadableMalformed, CasUnreadableContentMissing, CasUnreadableReadRefused, +} import product.fabric.identity { FabricIdentity, OfferKey, DemandKey, WorkKey } import product.fabric.demand { Demand } import product.fabric.selection { CandidateOffer, DispatchInputs, bind_candidate, CandidateBound, DispatchInputsNameAnotherOffer, + OfferCandidacy, OfferPriced, OfferNotFungible, OfferUnavailableForGrant, + OfferCannotHoldAnUnboundedGrant, OfferNotPriceable, OfferWrongCurrency, + OfferUnaffordable, OfferLiabilityOverBuyOrder, AvailabilityObservation, AvailableIndefinitely, CommitmentState, PayAsYouGo, StartCostObservation, StartCostMeasured, @@ -25,10 +33,13 @@ import product.fabric.selection { import gunbc.build_cache_instance { RunnerSlotIdentity } import extdeps.accounting.budget { Appropriation } import extdeps.accounting.encumbrance { EncumbranceLedger } -import product.fabric.budget { MoneyAccount, MoneyReservationRequest, AccountAdvanced, MoneyRefused, reserve_money } +import product.fabric.budget { + MoneyAccount, MoneyReservationRequest, AccountAdvanced, MoneyRefused, reserve_money, + MoneyRefusal, CurrencyMismatch, LedgerRefused, StaleLeaseGeneration, +} import product.fabric.execution { ReservationRef, LeaseIdentity } -import std.content_hash { content_hash_of_value } -import gunbc.fabric_executor_class { ExecutorSanction, fleet_cell_sanction } +import std.content_hash { content_hash_of_value, serialize_content_hash } +import gunbc.fabric_executor_class { ExecutorSanction, fleet_cell_sanction, capacity_admission_wire } import gunbc.fabric_floor_dispatch { floor_demand } import gunbc.fabric_control_plane { CellReservation, CellReserved, NoCellAdmissible, ReservationLost, @@ -37,8 +48,55 @@ import gunbc.fabric_control_plane { HoldEndLost, HoldEndStoreUnavailable, HoldEndAttemptRefused, release_reserved_cell_at, FabricCellCandidate, CellCandidateBound, OfferExecutorIsNotTheCell, fabric_cell_allocation_key, reserve_selected_cell, broker_reserve_for_demand, - bind_fabric_cell_candidate, + bind_fabric_cell_candidate, CellSlotDecoded, CellSlotUndecodable, CellHeld, CellFree, + decode_cell_slot_payload, +} +import gunbc.durable_cas_file_store { observe_cas_slot_state } +import gunbc.fabric_required_build_cell { + RequiredBuildCellObservation, RequiredBuildCellObserved, ReservationPayloadWorkMismatch, + RequiredBuildCellLifetimeVerdict, RequiredBuildCellObservationRefused, + RequiredBuildCellLifetimeSampleDecode, RequiredBuildCellLifetimeSampleDecoded, RequiredBuildCellLifetimeSampleUndecodable, + FabricCellPreReservationStanding, FabricCellPreReservationClean, FabricCellPreReservationAbsent, + FabricCellPreReservationRefused, + observe_required_build_cell_wet, required_build_cell_lifetime_sample, + required_build_cell_lifetime_sample_wire, decode_required_build_cell_lifetime_sample, + required_build_cell_lifetime_verdict, required_build_cell_lifetime_verdict_wire, + cell_member_shape_complete, + DirectoryModeStanding, DirectoryModeUnobserved, fci1_directory_mode_unobserved, +} +import gunbc.fabric_cell_acquire { fabric_cell_probe_wet } +import gunbc.fabric_cell_observation_admission { + FabricCellObservationDecision, FabricCellObservationAdmitted, FabricCellObservationDenied, + FabricCellObservationNotApplicable, fabric_cell_observation_outcome, + fabric_cell_observed_population_host, fabric_cell_observed_population_members, + admit_fabric_cell_probe, +} +import gunbc.fabric_cell_converge { + FabricCellResourceBoundaryAddress, fabric_cell_resource_address_equal, + FabricCellMember, + fabric_cell_id_for_slot, + fabric_cell_member_key, fabric_cell_member_ownership, + fabric_cell_base_dir, fabric_cell_root_path, fabric_cell_attempt_root_path, + fabric_cell_observation_refusal_identity, } +import gunbc.fabric_witness_run { + RequiredBuildWorkBinding, RequiredBuildWorkBound, RequiredBuildWorkRefused, RequiredBuildCoordinateMissing, + ExactGitSource, GitCommitTreeVerified, GitCommitTreeMismatch, + required_witnesses_build_work, verify_git_commit_tree, +} +import extdeps.git.object_store { git_sha1_object_id } +import v2.std.optional { Present, Absent } +import gunbc.ownership { Owned, Ensured } +import tools.fabric_ci_evidence { + FabricCiEvidenceDecode, FabricCiEvidenceDecoded, FabricCiEvidenceUndecodable, + fabric_ci_evidence_wire, decode_fabric_ci_evidence_wire, fabric_ci_write_wire, + fabric_ci_shell_crossing_coordinates_hash, +} +import gunbc.host_effect { LocalShell } +import gunbc.host_effect_realize { srv3_observe_path_owner } +import extdeps.posix.path_ownership { PathOwnershipObservation, PathOwnerObserved, PathOwnerUnobserved, posix_owner_spec_equal } +import extdeps.posix.identity { PosixOwnerSpec, PosixUserId, PosixGroupId } +import extdeps.tools.sha256sum { Sha256FileDigest, Sha256FileDigestUnavailable, sha256sum_file_digest_via_shell } // The live receipt for the EFFECTFUL half of the broker, which the hermetic witness cannot // reach. An entry point rather than a shell script, for the reason DESIGN gives: if a @@ -154,6 +212,433 @@ fn cp_probe_demand() -> Demand { ) } +// FCI-0's frozen source subject is the input authority. The WorkKey is always taken from +// required_witnesses_build_work; it is never transcribed into this probe. +data fci1_work_source_commit: NonEmptyStr = "e9eaa579ed0b6585a48965842b43ee09af5671a1" +data fci1_work_source_tree: NonEmptyStr = "c8983a3d8412a5e37a309b2d90ceafc698856619" +data fci1_installed_sudoers_path: NonEmptyStr = "/etc/sudoers.d/gunbc-ghrunner" +data fci1_installed_sudoers_digest: NonEmptyStr = "c80f6c02f9c157f4c8bc8006053e4cccde158d97e3d00a51202ee7b844e619f5" + +fn fci1_root_owner() -> PosixOwnerSpec { + PosixOwnerSpec { user: PosixUserId { value: 0 }, group: PosixGroupId { value: 0 } } +} + +fn fci1_path_owner_matches(path: NonEmptyStr, expected: PosixOwnerSpec) -> Bool { + match srv3_observe_path_owner(path: path as String, transport: LocalShell) { + PathOwnerUnobserved { reason: _ } => false + PathOwnerObserved { owner: observed } => posix_owner_spec_equal(left: observed, right: expected) + } +} + +// No reservation or allocation-store effect is reachable before this entry succeeds. The attempts +// owner is derived from a separate exact-path observation of the ghrunner home rather than a UID/GID +// literal. Mode is consumed only as the explicit unobserved boundary; it is not promoted to evidence. +func fci1_assert_host_preconditions() -> ProcessExit { + let slot = cp_probe_slot() + let cell_root = fabric_cell_root_path(slot: slot) as NonEmptyStr + let attempts = fabric_cell_attempt_root_path(slot: slot) as NonEmptyStr + let mode = fci1_directory_mode_unobserved(path: cell_root, desired_mode: "0755") + match sha256sum_file_digest_via_shell(path: fci1_installed_sudoers_path) { + Sha256FileDigestUnavailable { path: _, reason: _ } => exit_failure(reason: "FCI-1 sudoers readback unavailable") + Sha256FileDigest { digest: observed_digest } => + if (observed_digest.hex as String) != (fci1_installed_sudoers_digest as String) { + exit_failure(reason: "FCI-1 installed sudoers digest mismatch") + } else { + match srv3_observe_path_owner(path: "/home/ghrunner", transport: LocalShell) { + PathOwnerUnobserved { reason: _ } => exit_failure(reason: "FCI-1 ghrunner owner identity unavailable") + PathOwnerObserved { owner: runner_owner } => + if !fci1_path_owner_matches(path: fabric_cell_base_dir as NonEmptyStr, expected: fci1_root_owner()) { + exit_failure(reason: "FCI-1 fabric cell base owner/group mismatch") + } else if !fci1_path_owner_matches(path: cell_root, expected: fci1_root_owner()) { + exit_failure(reason: "FCI-1 cell root owner/group mismatch") + } else if !fci1_path_owner_matches(path: attempts, expected: runner_owner) { + exit_failure(reason: "FCI-1 attempts root owner/group mismatch") + } else { + match mode { + DirectoryModeUnobserved { path: p, desired_mode: desired, trigger: trigger } => + if p == cell_root && desired == "0755" && trigger != "" { ExitSuccess } + else { exit_failure(reason: "FCI-1 directory mode boundary malformed") } + } + } + } + } + } +} + +fn fci1_required_build_work() -> RequiredBuildWorkBinding { + match git_sha1_object_id(hex: fci1_work_source_commit) { + Absent => RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "source-commit" } } + Present { value: commit } => match git_sha1_object_id(hex: fci1_work_source_tree) { + Absent => RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "source-tree" } } + Present { value: tree } => match verify_git_commit_tree(commit: commit, declared_tree: tree, observed_commit_tree: tree) { + GitCommitTreeMismatch { commit: _, declared_tree: _, observed_commit_tree: _ } => + RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "commit-tree-verification" } } + GitCommitTreeVerified { subject } => required_witnesses_build_work(source: ExactGitSource { subject: subject }) + } + } + } +} + +// The frozen FCI-0 handoff is an assertion over the producer's result, never an input replacing it. +func fci1_assert_frozen_work_key() -> ProcessExit { + match fci1_required_build_work() { + RequiredBuildWorkRefused { cause: _ } => exit_failure(reason: "FCI-0 Work producer refused") + RequiredBuildWorkBound { work: work } => + if work.id.key == "ab8f7e2ca026301b" { ExitSuccess } + else { exit_failure(reason: join(["FCI-0 frozen WorkKey changed; observed=", work.id.key as String], "")) } + } +} + +type Fci1DemandBinding + = Fci1DemandBound { demand: Demand } + | Fci1DemandRefused + +fn fci1_probe_demand() -> Fci1DemandBinding { + match fci1_required_build_work() { + RequiredBuildWorkRefused { cause: _ } => Fci1DemandRefused + RequiredBuildWorkBound { work: work, source_tree_digest: _, environment_materialization_digest: _ } => + Fci1DemandBound { demand: floor_demand( + id: cp_probe_demand_id(key: "demand-floor-1"), + work: work.id, + origin: "gunbc.observation.fci1", + submitted_at: "2026-08-30T00:00:00Z", + account: "fleet-ci", + reservation_price: money_amount_micro(count: 500000), + maximum_buy_order: money_amount_micro(count: 1000000), + deadline: none, + ) } + } +} + +// FCI-1 instrument leaves the complete typed producer outputs visible. These are functions rather +// than receipt rows: every invocation re-reads the live CAS slot and cell substrate. +fn fci1_live_reserve(root: NonEmptyStr, expected: CasExpectation) -> CellReservation { + match fci1_probe_demand() { + Fci1DemandRefused => AttemptRefused { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), detail: "FCI-0 Work construction refused" } + Fci1DemandBound { demand: demand } => broker_reserve_for_demand( + root: root, + demand: demand, + roster: cp_probe_roster(executor: "srv3-06"), + sanctions: [fleet_cell_sanction(slot: cp_probe_slot())], + expected: expected, + account: cp_probe_account(), + maximum_buy_order: money_amount_micro(count: 1000000), + delay_valuation: MoneyPerSecond { amount: money_amount_micro(count: 0), currency: Usd }, + ) + } +} + +func fci1_live_reserve_absent(root: NonEmptyStr) -> CellReservation { + fci1_live_reserve(root: root, expected: ExpectSlotAbsent) +} + +func fci1_live_reserve_generation(root: NonEmptyStr, generation: Int) -> CellReservation { + fci1_live_reserve(root: root, expected: ExpectSlotGeneration { generation: generation }) +} + +// Reuse the generation produced by the canonical store. An absent slot creates generation one; a +// released slot advances from the generation it actually carries; a held or unreadable slot +// refuses. The caller never authors the fence. +func fci1_live_reserve_available(root: NonEmptyStr) -> CellReservation { + match observe_cas_slot_state(root: root, key: fabric_cell_allocation_key(slot: cp_probe_slot())) { + CasObservedUnreadable { cause: cause } => StoreUnavailable { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), cause: cause } + CasObservedReadable { readable: CasReadableAbsent } => fci1_live_reserve(root: root, expected: ExpectSlotAbsent) + CasObservedReadable { readable: CasReadablePresent { version: version } } => + match decode_cell_slot_payload(payload: version.value) { + CellSlotDecoded { state: CellFree { released_by: _ } } => + fci1_live_reserve(root: root, expected: ExpectSlotGeneration { generation: version.generation }) + CellSlotDecoded { state: CellHeld { reservation: _ } } => ReservationLost { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()) } + CellSlotUndecodable { detail: detail } => AttemptRefused { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), detail: detail } + } + } +} + +func fci1_live_replace_held(root: NonEmptyStr) -> CellReservation { + match observe_cas_slot_state(root: root, key: fabric_cell_allocation_key(slot: cp_probe_slot())) { + CasObservedUnreadable { cause: cause } => StoreUnavailable { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), cause: cause } + CasObservedReadable { readable: CasReadableAbsent } => ReservationLost { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()) } + CasObservedReadable { readable: CasReadablePresent { version: version } } => + match decode_cell_slot_payload(payload: version.value) { + CellSlotDecoded { state: CellHeld { reservation: _ } } => + fci1_live_reserve(root: root, expected: ExpectSlotGeneration { generation: version.generation }) + CellSlotDecoded { state: CellFree { released_by: _ } } => ReservationLost { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()) } + CellSlotUndecodable { detail: detail } => AttemptRefused { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), detail: detail } + } + } +} + +// The run boundary carries scalars, not coproduct variants. This total projection preserves the +// decision made by the reservation producer; the shell may sequence it but cannot recreate it. +// Every refusal includes a digest of its typed payload, keeping the renderings disjoint and making +// newly added CellReservation variants a compile-time obligation here. +fn fci1_cell_reservation_wire(reservation: CellReservation) -> String { + match reservation { + CellReserved { offer: offer, slot_key: slot, generation: generation, account: _ } => join([ + "reservation-committed|slot=", slot as String, + "|generation=", to_string(generation), + "|offer=", offer.key as String, + ], "") + NoCellAdmissible { considered: considered, refused: refused } => join([ + "reservation-no-cell-admissible|considered=", + serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: map(considered, c => fci1_offer_candidacy_wire(candidacy: c)))) as String, + "|refused=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: map(refused, r => capacity_admission_wire(admission: r)))) as String, + ], "") + BudgetRefused { slot_key: slot, refusal: refusal } => join([ + "reservation-budget-refused|slot=", slot as String, + "|cause=", fci1_money_refusal_wire(refusal: refusal) as String, + ], "") + ReservationLost { slot_key: slot } => join(["reservation-lost|slot=", slot as String], "") + StoreUnavailable { slot_key: slot, cause: cause } => join([ + "reservation-store-unavailable|slot=", slot as String, + "|cause=", fci1_cas_unreadable_wire(cause: cause) as String, + ], "") + AttemptRefused { slot_key: slot, detail: detail } => join([ + "reservation-attempt-refused|slot=", slot as String, "|detail=", detail as String, + ], "") + } +} + +fn fci1_money_refusal_wire(refusal: MoneyRefusal) -> NonEmptyStr { + match refusal { + CurrencyMismatch { account: _, account_currency: _, presented_currency: _ } => "money-currency-mismatch" + LedgerRefused { account: _, refusal: _ } => "money-ledger-refused" + StaleLeaseGeneration { reference: _, presented: generation } => + join(["money-stale-generation|presented=", to_string(generation)], "") as NonEmptyStr + } +} + +fn fci1_cas_unreadable_wire(cause: CasUnreadableSlot) -> NonEmptyStr { + match cause { + CasUnreadableMalformed { detail: _ } => "cas-malformed" + CasUnreadableContentMissing { referenced: hash } => + join(["cas-content-missing|hash=", serialize_content_hash(hash: hash) as String], "") as NonEmptyStr + CasUnreadableReadRefused { detail: _ } => "cas-read-refused" + } +} + +fn fci1_offer_candidacy_wire(candidacy: OfferCandidacy) -> NonEmptyStr { + match candidacy { + OfferPriced { offer: o, cost: _, basis: _, horizon: _ } => join(["priced|", o.principal as String, "|", o.key as String], "") as NonEmptyStr + OfferNotFungible { offer: o, fungibility: _ } => join(["not-fungible|", o.principal as String, "|", o.key as String], "") as NonEmptyStr + OfferUnavailableForGrant { offer: o, needed: _, available: _ } => join(["unavailable|", o.principal as String, "|", o.key as String], "") as NonEmptyStr + OfferCannotHoldAnUnboundedGrant { offer: o, available: _ } => join(["unbounded-refused|", o.principal as String, "|", o.key as String], "") as NonEmptyStr + OfferNotPriceable { offer: o, unpriced: _ } => join(["not-priceable|", o.principal as String, "|", o.key as String], "") as NonEmptyStr + OfferWrongCurrency { offer: o, mismatch: _ } => join(["wrong-currency|", o.principal as String, "|", o.key as String], "") as NonEmptyStr + OfferUnaffordable { offer: o, affordability: _ } => join(["unaffordable|", o.principal as String, "|", o.key as String], "") as NonEmptyStr + OfferLiabilityOverBuyOrder { offer: o, liability: _, maximum: _ } => join(["over-buy-order|", o.principal as String, "|", o.key as String], "") as NonEmptyStr + } +} + +fn fci1_member_wire(member: FabricCellMember) -> NonEmptyStr { + join(["member|", fabric_cell_member_key(member: member), "|", member.state_digest as String], "") as NonEmptyStr +} + +fn fci1_modeled_ownership_wire(member: FabricCellMember) -> NonEmptyStr { + match fabric_cell_member_ownership(member: member) { + Absent => "modeled-ownership-absent" + Present { value: Owned } => "modeled-ownership-owned" + Present { value: Ensured } => "modeled-ownership-ensured" + } +} + +func fci1_live_replace_held_wire(root: NonEmptyStr) -> String { + fci1_cell_reservation_wire(reservation: fci1_live_replace_held(root: root)) +} + +func fci1_assert_replace_held(root: NonEmptyStr) -> ProcessExit { + let outcome = fci1_live_replace_held(root: root) + match outcome { + CellReserved { offer: offer, slot_key: slot, generation: _, account: _ } => + if slot == "srv3-06" && offer.key == "offer-1" { ExitSuccess } else { + exit_failure(reason: join(["generation replacement selected wrong identity: ", fci1_cell_reservation_wire(reservation: outcome)], "")) + } + _ => exit_failure(reason: join(["generation replacement refused: ", fci1_cell_reservation_wire(reservation: outcome)], "")) + } +} + +func fci1_live_binding_observation(root: NonEmptyStr) -> RequiredBuildCellObservation { + match fci1_probe_demand() { + Fci1DemandRefused => ReservationPayloadWorkMismatch { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()) } + Fci1DemandBound { demand: demand } => observe_required_build_cell_wet( + root: root, + host: "srv3", + slot: cp_probe_slot(), + demand: demand, + offer: cp_probe_offer_for(executor: "srv3-06"), + ) + } +} + +func fci1_live_reserve_and_observe_available(root: NonEmptyStr, path: String) -> ProcessExit { + let reserved = fci1_live_reserve_available(root: root) + match reserved { + CellReserved { offer: _, slot_key: _, generation: _, account: _ } => + match required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) { + RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "RequiredBuildCellObservationRefused") + RequiredBuildCellLifetimeSampleDecoded { sample: sample } => fabric_ci_write_wire( + path: path, + wire: fabric_ci_evidence_wire(value: required_build_cell_lifetime_sample_wire(sample: sample) as NonEmptyStr), + ) + } + _ => exit_failure(reason: join(["ReservationCommitRefused:", fci1_cell_reservation_wire(reservation: reserved)], "")) + } +} + +func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) -> ProcessExit { + match required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) { + RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "RequiredBuildCellObservationRefused") + RequiredBuildCellLifetimeSampleDecoded { sample: sample } => fabric_ci_write_wire( + path: path, + wire: fabric_ci_evidence_wire(value: required_build_cell_lifetime_sample_wire(sample: sample) as NonEmptyStr), + ) + } +} + +fn fci1_decode_transport_sample(wire: String) -> RequiredBuildCellLifetimeSampleDecode { + match decode_fabric_ci_evidence_wire(wire: wire) { + FabricCiEvidenceUndecodable => RequiredBuildCellLifetimeSampleUndecodable + FabricCiEvidenceDecoded { value: value } => decode_required_build_cell_lifetime_sample(wire: value as String) + } +} + +// The before wire must have been emitted by fci1_live_reserve_and_observe_available. This entry +// validates it, takes the second observation live, and calls the typed verdict fold. Bash sequences +// the submitter boundary and transports bytes; it cannot manufacture either verdict name. +fn fci1_live_lifetime_verdict(root: NonEmptyStr, before_wire: String) -> RequiredBuildCellLifetimeVerdict { + match fci1_decode_transport_sample(wire: before_wire) { + RequiredBuildCellLifetimeSampleUndecodable => RequiredBuildCellObservationRefused + RequiredBuildCellLifetimeSampleDecoded { sample: before } => + required_build_cell_lifetime_verdict(before: before, after: fci1_live_binding_observation(root: root)) + } +} + +// Discriminating decoder control. The malformed wire is truncated before every required field; +// lazy evaluation means the live observer is unreachable when decode refuses, so this control can +// run anywhere while still exercising the exact production entry used by the wet transaction. +func fci1_invalid_before_wire_refuses() -> ProcessExit { + let verdict = fci1_live_lifetime_verdict(root: "/fci1-invalid-wire-must-not-be-read", before_wire: "srv3-06\n1") + match verdict { + RequiredBuildCellObservationRefused => ExitSuccess + _ => exit_failure(reason: join(["invalid before wire was accepted: ", required_build_cell_lifetime_verdict_wire(verdict: verdict)], "")) + } +} + +func fci1_assert_lifetime_held(root: NonEmptyStr, before_wire: String) -> ProcessExit { + let verdict = fci1_live_lifetime_verdict(root: root, before_wire: before_wire) + match verdict { + RequiredBuildCellLifetimeHeld { snapshot: snapshot } => + if snapshot.slot_key == "srv3-06" { ExitSuccess } else { + exit_failure(reason: join(["held wrong slot: ", required_build_cell_lifetime_verdict_wire(verdict: verdict)], "")) + } + _ => exit_failure(reason: join(["lifetime did not hold: ", required_build_cell_lifetime_verdict_wire(verdict: verdict)], "")) + } +} + +func fci1_assert_generation_changed(root: NonEmptyStr, before_wire: String) -> ProcessExit { + let verdict = fci1_live_lifetime_verdict(root: root, before_wire: before_wire) + match verdict { + ReservationGenerationChanged { expected_generation: expected, observed_generation: observed } => + if expected != observed { ExitSuccess } else { + exit_failure(reason: join(["generation mutation carried equal fences: ", required_build_cell_lifetime_verdict_wire(verdict: verdict)], "")) + } + _ => exit_failure(reason: join(["generation mutation was not observed: ", required_build_cell_lifetime_verdict_wire(verdict: verdict)], "")) + } +} + +func fci1_assert_lifetime_dependent(root: NonEmptyStr, before_wire: String) -> ProcessExit { + let verdict = fci1_live_lifetime_verdict(root: root, before_wire: before_wire) + match verdict { + ReservationLifetimeDependentOnSubmitter { slot_key: slot } => + if slot == "srv3-06" { ExitSuccess } else { + exit_failure(reason: join(["dependent reservation named wrong slot: ", required_build_cell_lifetime_verdict_wire(verdict: verdict)], "")) + } + _ => exit_failure(reason: join(["submitter-owned reservation survived: ", required_build_cell_lifetime_verdict_wire(verdict: verdict)], "")) + } +} + +fn fci1_cell_observation_wire(decision: FabricCellObservationDecision) -> String { + match fabric_cell_observation_outcome(decision: decision) { + FabricCellObservationAdmitted { population: population } => { + let members = fabric_cell_observed_population_members(population: population) + let boundaries = map( + filter(members, member => fabric_cell_resource_address_equal(a: member.address, b: FabricCellResourceBoundaryAddress)), + member => member.state_digest, + ) + let ownership = map(members, member => fabric_cell_member_ownership(member: member)) + join([ + "cell-admitted|host=", fabric_cell_observed_population_host(population: population) as String, + "|members=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: map(members, m => fci1_member_wire(member: m)))) as String, + "|boundaries=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: boundaries)) as String, + "|modeled-teardown-ownership=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: map(members, m => fci1_modeled_ownership_wire(member: m)))) as String, + ], "") + } + FabricCellObservationDenied { host: host, blocked: blocked } => join([ + "cell-denied|host=", host as String, + "|blocked=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: map(blocked, r => fabric_cell_observation_refusal_identity(refusal: r) as NonEmptyStr))) as String, + ], "") + FabricCellObservationNotApplicable { host: host } => join(["cell-not-applicable|host=", host as String], "") + } +} + +func fci1_live_cell_observation(path: String) -> ProcessExit { + fabric_ci_write_wire( + path: path, + wire: fabric_ci_evidence_wire( + value: fci1_cell_observation_wire(decision: admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3"))) as NonEmptyStr, + ), + ) +} + +func fci1_assert_cell_admitted() -> ProcessExit { + let decision = admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3")) + match fabric_cell_observation_outcome(decision: decision) { + FabricCellObservationAdmitted { population: population } => + if fabric_cell_observed_population_host(population: population) == "srv3" { ExitSuccess } else { + exit_failure(reason: join(["cell admitted on wrong host: ", fci1_cell_observation_wire(decision: decision)], "")) + } + _ => exit_failure(reason: join(["cell substrate not admitted: ", fci1_cell_observation_wire(decision: decision)], "")) + } +} + +// Positive clean pre-state is produced by the production observer before any reservation effect. +// Admission entails the exact expected cell population: absent members, foreign artifacts, +// unreadable namespaces, and unexpected members take non-admitted arms and refuse here. +fn fci1_cell_pre_reservation_standing() -> FabricCellPreReservationStanding { + let cell = fabric_cell_id_for_slot(slot: cp_probe_slot()) + match fabric_cell_observation_outcome(decision: admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3"))) { + FabricCellObservationAdmitted { population: population } => { + let members = fabric_cell_observed_population_members(population: population) + if cell_member_shape_complete(members: members) { + FabricCellPreReservationClean { + cell: cell, + members_content: fabric_ci_shell_crossing_coordinates_hash(values: map(members, m => fci1_member_wire(member: m))), + } + } else { FabricCellPreReservationRefused { cell: cell } } + } + FabricCellObservationDenied { host: _, blocked: _ } => FabricCellPreReservationRefused { cell: cell } + FabricCellObservationNotApplicable { host: _ } => FabricCellPreReservationAbsent { cell: cell } + } +} + +func fci1_assert_clean_cell_prestate() -> ProcessExit { + match fci1_cell_pre_reservation_standing() { + FabricCellPreReservationClean { cell: _, members_content: _ } => ExitSuccess + FabricCellPreReservationAbsent { cell: _ } => exit_failure(reason: "FCI-1 already-converged cell is absent") + FabricCellPreReservationRefused { cell: _ } => exit_failure(reason: "FCI-1 cell pre-state is not clean") + } +} + +func fci1_assert_cell_unchanged(before_wire: String) -> ProcessExit { + let observed = fci1_cell_observation_wire(decision: admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3"))) + match decode_fabric_ci_evidence_wire(wire: before_wire) { + FabricCiEvidenceUndecodable => exit_failure(reason: "persistent cell substrate before-wire undecodable") + FabricCiEvidenceDecoded { value: before } => if observed == (before as String) { ExitSuccess } else { + exit_failure(reason: join(["persistent cell substrate changed; before=", before as String, "; after=", observed], "")) + } + } +} + + fn cp_live_nothing_admissible_touches_no_store(root: NonEmptyStr) -> Int { cp_probe_code( r: broker_reserve_for_demand( @@ -316,3 +801,58 @@ fn cp_live_release_frees_the_cell_it_reserved(root: NonEmptyStr) -> Int { }, ) } + +fn fci1_cell_hold_end_wire(outcome: CellHoldEnd) -> String { + match outcome { + CellHoldEnded { slot_key: slot, generation: generation, account: _ } => join([ + "hold-ended|slot=", slot as String, "|generation=", to_string(generation), + ], "") + HoldEndBudgetRefused { slot_key: slot, refusal: refusal } => join([ + "hold-budget-refused|slot=", slot as String, + "|cause=", fci1_money_refusal_wire(refusal: refusal) as String, + ], "") + HoldEndSlotNotHeld { slot_key: slot, detail: detail } => join([ + "hold-slot-not-held|slot=", slot as String, "|detail=", detail as String, + ], "") + HoldEndLost { slot_key: slot } => join(["hold-lost|slot=", slot as String], "") + HoldEndStoreUnavailable { slot_key: slot, cause: cause } => join([ + "hold-store-unavailable|slot=", slot as String, + "|cause=", fci1_cas_unreadable_wire(cause: cause) as String, + ], "") + HoldEndAttemptRefused { slot_key: slot, detail: detail } => join([ + "hold-attempt-refused|slot=", slot as String, "|detail=", detail as String, + ], "") + } +} + +func fci1_live_release(root: NonEmptyStr) -> ProcessExit { + let outcome = fold( + cp_probe_held_account(), + init: HoldEndAttemptRefused { + slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), + detail: "instrument could not derive the held money account", + }, + f: fn(acc, account) { + release_reserved_cell_at( + root: root, + slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), + lease: LeaseIdentity { + lease_key: "cell-lease-probe", + resource_fingerprint: content_hash_of_value(value: "srv3-06"), + generation: 0, + }, + reservation: cp_probe_reservation_ref(), + reason: "FCI-1 instrument release", + released_at: "2026-08-29T00:00:00Z", + account: account, + ) + }, + ) + match outcome { + CellHoldEnded { slot_key: slot, generation: _, account: _ } => + if slot == "srv3-06" { ExitSuccess } else { + exit_failure(reason: join(["release ended wrong slot: ", fci1_cell_hold_end_wire(outcome: outcome)], "")) + } + _ => exit_failure(reason: join(["reservation release refused: ", fci1_cell_hold_end_wire(outcome: outcome)], "")) + } +} diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh new file mode 100755 index 00000000000..0a696ef91cf --- /dev/null +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -euo pipefail + +# FCI-1 reservation instrument. It starts no Work command. +# SCAFFOLD — dissolve-on: modeled lifecycle actuation sufficient to sequence a wet gate from .dag. +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export FABRIC_CI_GUNBC_BIN="$repo_root/target/release/gunbc" +export FABRIC_CI_SOURCE_ROOT="$repo_root" +export FABRIC_CI_ENTRY="$repo_root/dag/gunbc/instruments/fabric_control_plane_live_probe.dag" +export FABRIC_CI_LOG FABRIC_CI_VALUE_ROOT +FABRIC_CI_LOG=$(mktemp) +FABRIC_CI_VALUE_ROOT=$(mktemp -d) +source "$repo_root/tools/fabric_ci_evidence_driver.sh" +fabric_ci_driver_init + +canonical_root=/var/lib/gunbc/fabric/allocation +mutation_runtime=fci1-submitter-owned-reservation +mutation_root=/run/$mutation_runtime +subject="$repo_root/dag/gunbc/fabric/fabric_required_build_cell.dag" + +cleanup() { + systemctl stop fci1-positive-submitter.service fci1-dependent-submitter.service >/dev/null 2>&1 || true + rm -f "$FABRIC_CI_LOG" + rm -rf "$FABRIC_CI_VALUE_ROOT" +} +trap cleanup EXIT + +[[ ${EUID} -eq 0 ]] || { echo 'InstrumentRefused: must run as root on srv3' >&2; exit 2; } +[[ $(hostname -s) == srv3 ]] || { echo 'InstrumentRefused: host is not srv3' >&2; exit 2; } +[[ -x $FABRIC_CI_GUNBC_BIN ]] || { echo 'InstrumentRefused: exact-tree gunbc absent' >&2; exit 2; } + +# The merged evidence calibration runs against this exact binary before any srv3 observation or +# reservation effect. It grades only the evidence boundary; FCI-1 cannot self-grade that contract. +FABRIC_CI_GUNBC_BIN="$FABRIC_CI_GUNBC_BIN" bash "$repo_root/tools/fabric_ci_evidence_calibration.sh" + +sha256sum "$subject" +fabric_ci_run_assertion fci1_invalid_before_wire_refuses +fabric_ci_run_assertion fci1_assert_frozen_work_key +fabric_ci_run_assertion fci1_assert_host_preconditions +fabric_ci_run_assertion fci1_assert_clean_cell_prestate +fabric_ci_run_assertion fci1_assert_cell_admitted +cell_before=$(fabric_ci_capture_transport fci1_live_cell_observation cell-before) + +capture_submitter_transport() { + local unit=$1 function_name=$2 coordinate=$3 root=$4 + shift 4 + local value_path="$FABRIC_CI_VALUE_ROOT/$coordinate.wire" + rm -f -- "$value_path" + systemd-run --quiet --wait --collect --unit="$unit" "$@" \ + "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ + --entry "$FABRIC_CI_ENTRY" --function "$function_name" --arg root="$root" --arg path="$value_path" + [[ -f $value_path && ! -L $value_path ]] || return 1 + local value + value=$(<"$value_path") + [[ $value =~ ^FCIE0X[0-9A-F]+$ ]] || return 1 + printf '%s' "$value" +} + +positive_before=$(capture_submitter_transport fci1-positive-submitter.service fci1_live_reserve_and_observe_available positive-before "$canonical_root") +submitter_state=$(systemctl show fci1-positive-submitter.service --property=ActiveState --value 2>&1 || true) +[[ -z $submitter_state || $submitter_state == inactive || $submitter_state == failed ]] || { + echo "InstrumentRefused: submitting control process still active: $submitter_state" >&2 + exit 1 +} +fabric_ci_run_assertion fci1_assert_lifetime_held --arg root="$canonical_root" --arg before_wire="$positive_before" + +generation_before=$(fabric_ci_capture_transport fci1_live_binding_sample_wire generation-before --arg root="$canonical_root") +fabric_ci_run_assertion fci1_assert_replace_held --arg root="$canonical_root" +fabric_ci_run_assertion fci1_assert_generation_changed --arg root="$canonical_root" --arg before_wire="$generation_before" + +fabric_ci_run_assertion fci1_live_release --arg root="$canonical_root" +fabric_ci_run_assertion fci1_assert_cell_unchanged --arg before_wire="$cell_before" + +dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service fci1_live_reserve_and_observe_available dependent-before "$mutation_root" --property=RuntimeDirectory="$mutation_runtime") +[[ ! -e $mutation_root ]] || { echo 'ExpectedOutcomeNotObserved: submitter runtime directory removal' >&2; exit 1; } +fabric_ci_run_assertion fci1_assert_lifetime_dependent --arg root="$mutation_root" --arg before_wire="$dependent_before" +fabric_ci_run_assertion fci1_assert_cell_unchanged --arg before_wire="$cell_before" + +echo 'FCI1LiveAccepted' From babdd796e5f89ad55c6692f44f147553ba36157d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 30 Aug 2026 18:56:30 +0000 Subject: [PATCH 02/50] FCI-1: restore reservation availability on every exit --- .../fabric_control_plane_live_probe.dag | 44 +++++++++++++++++++ tools/fabric_ci_fci1_live_instrument.sh | 22 ++++++++++ 2 files changed, 66 insertions(+) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index f32f7a4bae2..0b623278b4e 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -494,6 +494,50 @@ func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) -> ProcessEx } } +// The durable CAS generation is monotone, so release cannot restore byte-identical slot bytes: an +// absent slot becomes a present Free successor and an existing Free slot advances. The invariant +// FCI-1 owns is availability -- never a held reservation -- projected identically for both clean +// pre-states. Unreadable, undecodable, and held are deliberately distinct. +fn fci1_allocation_availability_wire(root: NonEmptyStr) -> NonEmptyStr { + match observe_cas_slot_state(root: root, key: fabric_cell_allocation_key(slot: cp_probe_slot())) { + CasObservedUnreadable { cause: _ } => "allocation-unreadable" + CasObservedReadable { readable: CasReadableAbsent } => "allocation-available" + CasObservedReadable { readable: CasReadablePresent { version: version } } => + match decode_cell_slot_payload(payload: version.value) { + CellSlotDecoded { state: CellFree { released_by: _ } } => "allocation-available" + CellSlotDecoded { state: CellHeld { reservation: _ } } => "allocation-held" + CellSlotUndecodable { detail: _ } => "allocation-undecodable" + } + } +} + +func fci1_live_allocation_availability(root: NonEmptyStr, path: String) -> ProcessExit { + fabric_ci_write_wire( + path: path, + wire: fabric_ci_evidence_wire(value: fci1_allocation_availability_wire(root: root)), + ) +} + +func fci1_assert_allocation_available(root: NonEmptyStr) -> ProcessExit { + let observed = fci1_allocation_availability_wire(root: root) + if observed == "allocation-available" { ExitSuccess } + else { exit_failure(reason: join(["FCI-1 allocation store is not available: ", observed as String], "")) } +} + +func fci1_assert_allocation_restored(root: NonEmptyStr, before_wire: String) -> ProcessExit { + match decode_fabric_ci_evidence_wire(wire: before_wire) { + FabricCiEvidenceUndecodable => exit_failure(reason: "FCI-1 allocation pre-state wire undecodable") + FabricCiEvidenceDecoded { value: before } => { + let after = fci1_allocation_availability_wire(root: root) + if before == "allocation-available" && after == "allocation-available" { ExitSuccess } + else { exit_failure(reason: join([ + "FCI-1 allocation availability not restored; before=", before as String, + "; after=", after as String, + ], "")) } + } + } +} + fn fci1_decode_transport_sample(wire: String) -> RequiredBuildCellLifetimeSampleDecode { match decode_fabric_ci_evidence_wire(wire: wire) { FabricCiEvidenceUndecodable => RequiredBuildCellLifetimeSampleUndecodable diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index 0a696ef91cf..5e8fbdce519 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -17,11 +17,28 @@ canonical_root=/var/lib/gunbc/fabric/allocation mutation_runtime=fci1-submitter-owned-reservation mutation_root=/run/$mutation_runtime subject="$repo_root/dag/gunbc/fabric/fabric_required_build_cell.dag" +canonical_reservation_may_be_held=0 cleanup() { + local original_status=$? + local cleanup_status=0 + trap - EXIT + set +e systemctl stop fci1-positive-submitter.service fci1-dependent-submitter.service >/dev/null 2>&1 || true + if (( canonical_reservation_may_be_held )); then + "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ + --entry "$FABRIC_CI_ENTRY" --function fci1_live_release --arg root="$canonical_root" >/dev/null 2>&1 + "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ + --entry "$FABRIC_CI_ENTRY" --function fci1_assert_allocation_available --arg root="$canonical_root" + cleanup_status=$? + fi rm -f "$FABRIC_CI_LOG" rm -rf "$FABRIC_CI_VALUE_ROOT" + if (( cleanup_status != 0 )); then + echo 'InstrumentRefused: cleanup could not positively restore allocation availability' >&2 + exit 1 + fi + exit "$original_status" } trap cleanup EXIT @@ -40,6 +57,8 @@ fabric_ci_run_assertion fci1_assert_host_preconditions fabric_ci_run_assertion fci1_assert_clean_cell_prestate fabric_ci_run_assertion fci1_assert_cell_admitted cell_before=$(fabric_ci_capture_transport fci1_live_cell_observation cell-before) +allocation_before=$(fabric_ci_capture_transport fci1_live_allocation_availability allocation-before --arg root="$canonical_root") +fabric_ci_run_assertion fci1_assert_allocation_available --arg root="$canonical_root" capture_submitter_transport() { local unit=$1 function_name=$2 coordinate=$3 root=$4 @@ -56,6 +75,7 @@ capture_submitter_transport() { printf '%s' "$value" } +canonical_reservation_may_be_held=1 positive_before=$(capture_submitter_transport fci1-positive-submitter.service fci1_live_reserve_and_observe_available positive-before "$canonical_root") submitter_state=$(systemctl show fci1-positive-submitter.service --property=ActiveState --value 2>&1 || true) [[ -z $submitter_state || $submitter_state == inactive || $submitter_state == failed ]] || { @@ -69,6 +89,8 @@ fabric_ci_run_assertion fci1_assert_replace_held --arg root="$canonical_root" fabric_ci_run_assertion fci1_assert_generation_changed --arg root="$canonical_root" --arg before_wire="$generation_before" fabric_ci_run_assertion fci1_live_release --arg root="$canonical_root" +canonical_reservation_may_be_held=0 +fabric_ci_run_assertion fci1_assert_allocation_restored --arg root="$canonical_root" --arg before_wire="$allocation_before" fabric_ci_run_assertion fci1_assert_cell_unchanged --arg before_wire="$cell_before" dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service fci1_live_reserve_and_observe_available dependent-before "$mutation_root" --property=RuntimeDirectory="$mutation_runtime") From f145b2e0fc81237539459b47605b3ae00e6a8d6d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 30 Aug 2026 19:29:26 +0000 Subject: [PATCH 03/50] FCI-1: wire fabric-only convergence and observed source --- dag/gunbc/fabric/fabric_ci_program.dag | 2 +- .../fabric/fabric_required_build_cell.dag | 11 ++ .../fabric_control_plane_live_probe.dag | 121 ++++++++++++++---- tools/fabric_ci_fci1_live_instrument.sh | 6 +- 4 files changed, 109 insertions(+), 31 deletions(-) diff --git a/dag/gunbc/fabric/fabric_ci_program.dag b/dag/gunbc/fabric/fabric_ci_program.dag index eacf1256af7..37f4cf08323 100644 --- a/dag/gunbc/fabric/fabric_ci_program.dag +++ b/dag/gunbc/fabric/fabric_ci_program.dag @@ -18,7 +18,7 @@ type FabricCiGate { fn fabric_ci_gates() -> List { [ FabricCiGate { id: "FCI-0", owned_question: "What exactly will the fabric run?", exit_predicate: "The fabric possesses one exact required-witnesses-build Work carrying every execution-defining coordinate; moving or incomplete subjects refuse.", positive_control: "Identical semantic inputs derive the identical WorkKey.", discriminating_mutations: ["command-word", "environment-entry", "verified-tree", "toolchain-identity", "output-declaration"], frozen_output_identities: ["WorkKey", "source-tree-digest", "environment-materialization-digest"], non_goals: ["ExecutionGrant", "executor", "process-start", "cell-realization", "workflow-cutover"] }, - FabricCiGate { id: "FCI-1", owned_question: "What durable reservation binds this exact Work to one already-converged bounded owned cell?", exit_predicate: "After installed-authorization and exact-path owner/group readback succeed, one exact Work is bound to exactly one selected substrate-converged cell by one committed durable reservation generation. An independent observer reads the same reservation generation, demand/offer payload and content digest after the submitter terminates; release removes only the holding and leaves the persistent cell substrate unchanged. Directory mode remains explicitly unobserved and no Work process starts.", positive_control: "The exact-tree evidence calibration holds; installed sudoers and cell owner/group readbacks match; a production reservation survives submitter termination, is independently re-observed byte-identically, releases through the production path, and leaves the cell substrate unchanged.", discriminating_mutations: ["reservation-state-owned-by-submitter", "reservation-generation-changed", "reservation-payload-work-mismatch", "selected-offer-cell-mismatch", "cell-substrate-not-converged", "cell-resource-boundary-digest-mismatch", "reservation-release-not-observed", "reservation-release-destroys-cell-substrate", "multiple-cells", "unowned-cell"], frozen_output_identities: ["WorkKey", "DemandKey", "OfferKey", "CellId/slot-key", "committed-CAS-generation", "reservation-content-digest", "cell-resource-boundary-digest", "cell-owner-identity", "reservation-realization-digest"], non_goals: ["cell-supervisor", "passive-lease-unit", "InvocationID", "MainPID", "process-start-identity", "blocking-wait", "ExecutionGrant", "Work-process-start", "Attempt", "execution-Receipt", "source-to-application-structural-type-safety"] }, + FabricCiGate { id: "FCI-1", owned_question: "What durable reservation binds this exact Work to one already-converged bounded owned cell?", exit_predicate: "After installed-authorization, converged allocation-store, and exact-path owner/group readback succeed, one exact Work is bound to exactly one selected substrate-converged cell by one committed durable reservation generation. An independent observer reads the same reservation generation, demand/offer payload and content digest after the submitter terminates; release removes only the holding, returns the append-only allocation store to Free at a monotone generation, and leaves the persistent cell substrate unchanged. Directory mode remains explicitly unobserved and no Work process starts.", positive_control: "The exact-tree evidence calibration holds; installed sudoers, allocation-store precondition, and cell owner/group readbacks match; a production reservation survives submitter termination, is independently re-observed byte-identically, releases through the production path, and leaves the cell substrate unchanged.", discriminating_mutations: ["reservation-state-owned-by-submitter", "reservation-generation-changed", "reservation-payload-work-mismatch", "selected-offer-cell-mismatch", "cell-substrate-not-converged", "cell-resource-boundary-digest-mismatch", "reservation-release-not-observed", "reservation-release-destroys-cell-substrate", "multiple-cells", "unowned-cell"], frozen_output_identities: ["WorkKey", "DemandKey", "OfferKey", "CellId/slot-key", "committed-CAS-generation", "reservation-content-digest", "cell-resource-boundary-digest", "cell-owner-identity", "reservation-realization-digest", "allocation-store-path", "allocation-store-free-prestate", "allocation-store-monotone-free-poststate"], non_goals: ["cell-supervisor", "passive-lease-unit", "InvocationID", "MainPID", "process-start-identity", "blocking-wait", "ExecutionGrant", "Work-process-start", "Attempt", "execution-Receipt", "source-to-application-structural-type-safety"] }, FabricCiGate { id: "FCI-2", owned_question: "Which sole grant authorizes this Work?", exit_predicate: "Selection, reservation and commitment yield exactly one fenced ExecutionGrant for the exact Work.", positive_control: "One admitted offer yields one committed grant.", discriminating_mutations: ["duplicate-grant", "expired-reservation", "work-mismatch"], frozen_output_identities: ["ExecutionGrant", "lease-epoch"], non_goals: ["process-start", "GitHub-conclusion"] }, FabricCiGate { id: "FCI-3", owned_question: "What causes the target process?", exit_predicate: "A persistent executor re-reads the committed Grant before starting; absent, corrupt or expired Grant yields zero target processes.", positive_control: "One live committed grant starts exactly one target process.", discriminating_mutations: ["grant-removed", "grant-corrupt", "grant-expired"], frozen_output_identities: ["attempt-identity", "process-observation"], non_goals: ["GitHub-conclusion", "multiple-host-scheduling"] }, FabricCiGate { id: "FCI-4", owned_question: "How is execution admitted once, deduplicated, released and sanitized?", exit_predicate: "One fenced Receipt is admitted at most once; duplicate delivery cannot duplicate standing, and the cell is released only after per-attempt sanitation is confirmed.", positive_control: "One matching Receipt is admitted once and its sanitized cell returns to supply.", discriminating_mutations: ["duplicate-receipt", "foreign-receipt", "release-before-sanitation", "failed-sanitation"], frozen_output_identities: ["admitted-receipt", "deduplication-key", "release-receipt", "sanitation-receipt"], non_goals: ["shadow-GitHub-job", "required-check-cutover"] }, diff --git a/dag/gunbc/fabric/fabric_required_build_cell.dag b/dag/gunbc/fabric/fabric_required_build_cell.dag index c373f7c2e4f..9c64a0cbc47 100644 --- a/dag/gunbc/fabric/fabric_required_build_cell.dag +++ b/dag/gunbc/fabric/fabric_required_build_cell.dag @@ -55,6 +55,17 @@ type FabricCellPreReservationStanding | FabricCellPreReservationAbsent { cell: CellId } | FabricCellPreReservationRefused { cell: CellId } +type AllocationStorePrestate + = AllocationStoreAbsent { path: NonEmptyStr } + | AllocationStoreFree { path: NonEmptyStr, generation: Int } + | AllocationStoreHeld { path: NonEmptyStr, generation: Int } + | AllocationStoreUnreadable { path: NonEmptyStr } + | AllocationStoreUndecodable { path: NonEmptyStr, generation: Int } + +type AllocationStoreRestoration + = AllocationStoreReturnedFree { path: NonEmptyStr, pre_generation: Int, post_generation: Int } + | AllocationStoreRestorationRefused { path: NonEmptyStr } + // Mode remains outside the observed guarantee until the same exact-path producer reports it. fn fci1_directory_mode_unobserved(path: NonEmptyStr, desired_mode: NonEmptyStr) -> DirectoryModeStanding { DirectoryModeUnobserved { diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 0b623278b4e..c1959b69e43 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -1,6 +1,7 @@ module tools.fabric_control_plane_live_probe -import std.types { Bool, Int, List, NonEmptyStr, String } +import std.types { Bool, Int, List, NonEmptyStr, String, GitRef } +import std.algebra { trim } import std.process { ProcessExit, ExitSuccess, exit_failure } import extdeps.currency.currency { Usd } import std.measure { @@ -58,6 +59,10 @@ import gunbc.fabric_required_build_cell { RequiredBuildCellLifetimeSampleDecode, RequiredBuildCellLifetimeSampleDecoded, RequiredBuildCellLifetimeSampleUndecodable, FabricCellPreReservationStanding, FabricCellPreReservationClean, FabricCellPreReservationAbsent, FabricCellPreReservationRefused, + AllocationStorePrestate, AllocationStoreAbsent, AllocationStoreFree, AllocationStoreHeld, + AllocationStoreUnreadable, AllocationStoreUndecodable, + AllocationStoreRestoration, AllocationStoreReturnedFree, + AllocationStoreRestorationRefused, observe_required_build_cell_wet, required_build_cell_lifetime_sample, required_build_cell_lifetime_sample_wire, decode_required_build_cell_lifetime_sample, required_build_cell_lifetime_verdict, required_build_cell_lifetime_verdict_wire, @@ -270,10 +275,21 @@ fn fci1_required_build_work() -> RequiredBuildWorkBinding { Absent => RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "source-commit" } } Present { value: commit } => match git_sha1_object_id(hex: fci1_work_source_tree) { Absent => RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "source-tree" } } - Present { value: tree } => match verify_git_commit_tree(commit: commit, declared_tree: tree, observed_commit_tree: tree) { - GitCommitTreeMismatch { commit: _, declared_tree: _, observed_commit_tree: _ } => - RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "commit-tree-verification" } } - GitCommitTreeVerified { subject } => required_witnesses_build_work(source: ExactGitSource { subject: subject }) + Present { value: tree } => { + let repo = git.Inspect.Toplevel() + let observed = git.Core.CommitTreeInRepo(repository_path: repo.path as String, commit: fci1_work_source_commit as GitRef) + if observed.exit_code != 0 { + RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "observed-commit-tree" } } + } else { + match git_sha1_object_id(hex: trim(s: observed.tree)) { + Absent => RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "observed-commit-tree" } } + Present { value: observed_tree } => match verify_git_commit_tree(commit: commit, declared_tree: tree, observed_commit_tree: observed_tree) { + GitCommitTreeMismatch { commit: _, declared_tree: _, observed_commit_tree: _ } => + RequiredBuildWorkRefused { cause: RequiredBuildCoordinateMissing { coordinate: "commit-tree-verification" } } + GitCommitTreeVerified { subject } => required_witnesses_build_work(source: ExactGitSource { subject: subject }) + } + } + } } } } @@ -494,50 +510,99 @@ func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) -> ProcessEx } } -// The durable CAS generation is monotone, so release cannot restore byte-identical slot bytes: an -// absent slot becomes a present Free successor and an existing Free slot advances. The invariant -// FCI-1 owns is availability -- never a held reservation -- projected identically for both clean -// pre-states. Unreadable, undecodable, and held are deliberately distinct. -fn fci1_allocation_availability_wire(root: NonEmptyStr) -> NonEmptyStr { +fn fci1_allocation_prestate(root: NonEmptyStr) -> AllocationStorePrestate { match observe_cas_slot_state(root: root, key: fabric_cell_allocation_key(slot: cp_probe_slot())) { - CasObservedUnreadable { cause: _ } => "allocation-unreadable" - CasObservedReadable { readable: CasReadableAbsent } => "allocation-available" + CasObservedUnreadable { cause: _ } => AllocationStoreUnreadable { path: root } + CasObservedReadable { readable: CasReadableAbsent } => AllocationStoreAbsent { path: root } CasObservedReadable { readable: CasReadablePresent { version: version } } => match decode_cell_slot_payload(payload: version.value) { - CellSlotDecoded { state: CellFree { released_by: _ } } => "allocation-available" - CellSlotDecoded { state: CellHeld { reservation: _ } } => "allocation-held" - CellSlotUndecodable { detail: _ } => "allocation-undecodable" + CellSlotDecoded { state: CellFree { released_by: _ } } => AllocationStoreFree { path: root, generation: version.generation } + CellSlotDecoded { state: CellHeld { reservation: _ } } => AllocationStoreHeld { path: root, generation: version.generation } + CellSlotUndecodable { detail: _ } => AllocationStoreUndecodable { path: root, generation: version.generation } + } + } +} + +fn fci1_allocation_prestate_wire(prestate: AllocationStorePrestate) -> NonEmptyStr { + match prestate { + AllocationStoreAbsent { path: p } => join(["absent\n", p as String], "") as NonEmptyStr + AllocationStoreFree { path: p, generation: g } => join(["free\n", p as String, "\n", to_string(g)], "") as NonEmptyStr + AllocationStoreHeld { path: p, generation: g } => join(["held\n", p as String, "\n", to_string(g)], "") as NonEmptyStr + AllocationStoreUnreadable { path: p } => join(["unreadable\n", p as String], "") as NonEmptyStr + AllocationStoreUndecodable { path: p, generation: g } => join(["undecodable\n", p as String, "\n", to_string(g)], "") as NonEmptyStr + } +} + +fn fci1_decode_allocation_prestate(wire: String) -> AllocationStorePrestate { + let fields = split(s: wire, delimiter: "\n") + match fields.first() { + Absent => AllocationStoreUnreadable { path: "invalid-before-wire" } + Present { value: tag } => match fields.skip(n: 1).first() { + Absent => AllocationStoreUnreadable { path: "invalid-before-wire" } + Present { value: path } => if path == "" { AllocationStoreUnreadable { path: "invalid-before-wire" } } else { + if tag == "absent" && list_length(items: fields) == 2 { AllocationStoreAbsent { path: path as NonEmptyStr } } + else { match fields.skip(n: 2).first() { + Absent => AllocationStoreUnreadable { path: path as NonEmptyStr } + Present { value: generation_text } => match parse_int(s: generation_text) { + Absent => AllocationStoreUnreadable { path: path as NonEmptyStr } + Present { value: generation } => + if tag == "free" { AllocationStoreFree { path: path as NonEmptyStr, generation: generation } } + else if tag == "held" { AllocationStoreHeld { path: path as NonEmptyStr, generation: generation } } + else { AllocationStoreUndecodable { path: path as NonEmptyStr, generation: generation } } + } + } } } + } } } -func fci1_live_allocation_availability(root: NonEmptyStr, path: String) -> ProcessExit { +func fci1_live_allocation_prestate(root: NonEmptyStr, path: String) -> ProcessExit { fabric_ci_write_wire( path: path, - wire: fabric_ci_evidence_wire(value: fci1_allocation_availability_wire(root: root)), + wire: fabric_ci_evidence_wire(value: fci1_allocation_prestate_wire(prestate: fci1_allocation_prestate(root: root))), ) } func fci1_assert_allocation_available(root: NonEmptyStr) -> ProcessExit { - let observed = fci1_allocation_availability_wire(root: root) - if observed == "allocation-available" { ExitSuccess } - else { exit_failure(reason: join(["FCI-1 allocation store is not available: ", observed as String], "")) } + match fci1_allocation_prestate(root: root) { + AllocationStoreAbsent { path: _ } => exit_failure(reason: "FCI-1 allocation store substrate is absent") + AllocationStoreFree { path: _, generation: _ } => ExitSuccess + AllocationStoreHeld { path: _, generation: _ } => exit_failure(reason: "FCI-1 allocation store remains held") + AllocationStoreUnreadable { path: _ } => exit_failure(reason: "FCI-1 allocation store unreadable") + AllocationStoreUndecodable { path: _, generation: _ } => exit_failure(reason: "FCI-1 allocation store undecodable") + } } -func fci1_assert_allocation_restored(root: NonEmptyStr, before_wire: String) -> ProcessExit { +fn fci1_allocation_restoration(root: NonEmptyStr, before_wire: String) -> AllocationStoreRestoration { match decode_fabric_ci_evidence_wire(wire: before_wire) { - FabricCiEvidenceUndecodable => exit_failure(reason: "FCI-1 allocation pre-state wire undecodable") + FabricCiEvidenceUndecodable => AllocationStoreRestorationRefused { path: root } FabricCiEvidenceDecoded { value: before } => { - let after = fci1_allocation_availability_wire(root: root) - if before == "allocation-available" && after == "allocation-available" { ExitSuccess } - else { exit_failure(reason: join([ - "FCI-1 allocation availability not restored; before=", before as String, - "; after=", after as String, - ], "")) } + let prestate = fci1_decode_allocation_prestate(wire: before as String) + let poststate = fci1_allocation_prestate(root: root) + match prestate { + AllocationStoreAbsent { path: _ } => AllocationStoreRestorationRefused { path: root } + AllocationStoreFree { path: p, generation: pre } => match poststate { + AllocationStoreFree { path: q, generation: post } => if p == q && post > pre { AllocationStoreReturnedFree { path: q, pre_generation: pre, post_generation: post } } else { AllocationStoreRestorationRefused { path: root } } + AllocationStoreAbsent { path: _ } => AllocationStoreRestorationRefused { path: root } + AllocationStoreHeld { path: _, generation: _ } => AllocationStoreRestorationRefused { path: root } + AllocationStoreUnreadable { path: _ } => AllocationStoreRestorationRefused { path: root } + AllocationStoreUndecodable { path: _, generation: _ } => AllocationStoreRestorationRefused { path: root } + } + AllocationStoreHeld { path: _, generation: _ } => AllocationStoreRestorationRefused { path: root } + AllocationStoreUnreadable { path: _ } => AllocationStoreRestorationRefused { path: root } + AllocationStoreUndecodable { path: _, generation: _ } => AllocationStoreRestorationRefused { path: root } + } } } } +func fci1_assert_allocation_restored(root: NonEmptyStr, before_wire: String) -> ProcessExit { + match fci1_allocation_restoration(root: root, before_wire: before_wire) { + AllocationStoreReturnedFree { path: _, pre_generation: _, post_generation: _ } => ExitSuccess + AllocationStoreRestorationRefused { path: _ } => exit_failure(reason: "FCI-1 allocation store restoration refused") + } +} + fn fci1_decode_transport_sample(wire: String) -> RequiredBuildCellLifetimeSampleDecode { match decode_fabric_ci_evidence_wire(wire: wire) { FabricCiEvidenceUndecodable => RequiredBuildCellLifetimeSampleUndecodable diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index 5e8fbdce519..abd6c0aa7ea 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -18,6 +18,7 @@ mutation_runtime=fci1-submitter-owned-reservation mutation_root=/run/$mutation_runtime subject="$repo_root/dag/gunbc/fabric/fabric_required_build_cell.dag" canonical_reservation_may_be_held=0 +allocation_before= cleanup() { local original_status=$? @@ -29,7 +30,8 @@ cleanup() { "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ --entry "$FABRIC_CI_ENTRY" --function fci1_live_release --arg root="$canonical_root" >/dev/null 2>&1 "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ - --entry "$FABRIC_CI_ENTRY" --function fci1_assert_allocation_available --arg root="$canonical_root" + --entry "$FABRIC_CI_ENTRY" --function fci1_assert_allocation_restored \ + --arg root="$canonical_root" --arg before_wire="$allocation_before" cleanup_status=$? fi rm -f "$FABRIC_CI_LOG" @@ -57,7 +59,7 @@ fabric_ci_run_assertion fci1_assert_host_preconditions fabric_ci_run_assertion fci1_assert_clean_cell_prestate fabric_ci_run_assertion fci1_assert_cell_admitted cell_before=$(fabric_ci_capture_transport fci1_live_cell_observation cell-before) -allocation_before=$(fabric_ci_capture_transport fci1_live_allocation_availability allocation-before --arg root="$canonical_root") +allocation_before=$(fabric_ci_capture_transport fci1_live_allocation_prestate allocation-before --arg root="$canonical_root") fabric_ci_run_assertion fci1_assert_allocation_available --arg root="$canonical_root" capture_submitter_transport() { From 3da98de45be323623d5f126e27bd7ed9eda3d6dc Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Sun, 30 Aug 2026 21:34:00 +0000 Subject: [PATCH 04/50] Derive FCI-1 sudoers precondition from projection --- .../instruments/fabric_control_plane_live_probe.dag | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index c1959b69e43..2d85cbce48c 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -222,7 +222,7 @@ fn cp_probe_demand() -> Demand { data fci1_work_source_commit: NonEmptyStr = "e9eaa579ed0b6585a48965842b43ee09af5671a1" data fci1_work_source_tree: NonEmptyStr = "c8983a3d8412a5e37a309b2d90ceafc698856619" data fci1_installed_sudoers_path: NonEmptyStr = "/etc/sudoers.d/gunbc-ghrunner" -data fci1_installed_sudoers_digest: NonEmptyStr = "c80f6c02f9c157f4c8bc8006053e4cccde158d97e3d00a51202ee7b844e619f5" +data fci1_projected_sudoers_path: NonEmptyStr = "provisioning/srv3/gunbc-ghrunner.sudoers" fn fci1_root_owner() -> PosixOwnerSpec { PosixOwnerSpec { user: PosixUserId { value: 0 }, group: PosixGroupId { value: 0 } } @@ -246,8 +246,11 @@ func fci1_assert_host_preconditions() -> ProcessExit { match sha256sum_file_digest_via_shell(path: fci1_installed_sudoers_path) { Sha256FileDigestUnavailable { path: _, reason: _ } => exit_failure(reason: "FCI-1 sudoers readback unavailable") Sha256FileDigest { digest: observed_digest } => - if (observed_digest.hex as String) != (fci1_installed_sudoers_digest as String) { - exit_failure(reason: "FCI-1 installed sudoers digest mismatch") + match sha256sum_file_digest_via_shell(path: fci1_projected_sudoers_path) { + Sha256FileDigestUnavailable { path: _, reason: _ } => exit_failure(reason: "FCI-1 projected sudoers artifact unavailable") + Sha256FileDigest { digest: projected_digest } => + if (observed_digest.hex as String) != (projected_digest.hex as String) { + exit_failure(reason: "FCI-1 installed sudoers differs from the exact-tree generated projection") } else { match srv3_observe_path_owner(path: "/home/ghrunner", transport: LocalShell) { PathOwnerUnobserved { reason: _ } => exit_failure(reason: "FCI-1 ghrunner owner identity unavailable") @@ -267,6 +270,7 @@ func fci1_assert_host_preconditions() -> ProcessExit { } } } + } } } From 324fa4494528fe7117c267d788f60b8aca131dd5 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 09:38:51 +0000 Subject: [PATCH 05/50] Repair FCI-1 allocation lifecycle evidence --- dag/gunbc/fabric/fabric_ci_program.dag | 2 +- .../fabric/fabric_required_build_cell.dag | 68 ++++++++- .../fabric_control_plane_live_probe.dag | 138 ++++++++++++------ ...fci1_allocation_lifecycle_witness_test.dag | 59 ++++++++ tools/fabric_ci_fci1_live_instrument.sh | 87 +++++++++-- 5 files changed, 284 insertions(+), 70 deletions(-) create mode 100644 dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag diff --git a/dag/gunbc/fabric/fabric_ci_program.dag b/dag/gunbc/fabric/fabric_ci_program.dag index 37f4cf08323..bf7db5611b0 100644 --- a/dag/gunbc/fabric/fabric_ci_program.dag +++ b/dag/gunbc/fabric/fabric_ci_program.dag @@ -18,7 +18,7 @@ type FabricCiGate { fn fabric_ci_gates() -> List { [ FabricCiGate { id: "FCI-0", owned_question: "What exactly will the fabric run?", exit_predicate: "The fabric possesses one exact required-witnesses-build Work carrying every execution-defining coordinate; moving or incomplete subjects refuse.", positive_control: "Identical semantic inputs derive the identical WorkKey.", discriminating_mutations: ["command-word", "environment-entry", "verified-tree", "toolchain-identity", "output-declaration"], frozen_output_identities: ["WorkKey", "source-tree-digest", "environment-materialization-digest"], non_goals: ["ExecutionGrant", "executor", "process-start", "cell-realization", "workflow-cutover"] }, - FabricCiGate { id: "FCI-1", owned_question: "What durable reservation binds this exact Work to one already-converged bounded owned cell?", exit_predicate: "After installed-authorization, converged allocation-store, and exact-path owner/group readback succeed, one exact Work is bound to exactly one selected substrate-converged cell by one committed durable reservation generation. An independent observer reads the same reservation generation, demand/offer payload and content digest after the submitter terminates; release removes only the holding, returns the append-only allocation store to Free at a monotone generation, and leaves the persistent cell substrate unchanged. Directory mode remains explicitly unobserved and no Work process starts.", positive_control: "The exact-tree evidence calibration holds; installed sudoers, allocation-store precondition, and cell owner/group readbacks match; a production reservation survives submitter termination, is independently re-observed byte-identically, releases through the production path, and leaves the cell substrate unchanged.", discriminating_mutations: ["reservation-state-owned-by-submitter", "reservation-generation-changed", "reservation-payload-work-mismatch", "selected-offer-cell-mismatch", "cell-substrate-not-converged", "cell-resource-boundary-digest-mismatch", "reservation-release-not-observed", "reservation-release-destroys-cell-substrate", "multiple-cells", "unowned-cell"], frozen_output_identities: ["WorkKey", "DemandKey", "OfferKey", "CellId/slot-key", "committed-CAS-generation", "reservation-content-digest", "cell-resource-boundary-digest", "cell-owner-identity", "reservation-realization-digest", "allocation-store-path", "allocation-store-free-prestate", "allocation-store-monotone-free-poststate"], non_goals: ["cell-supervisor", "passive-lease-unit", "InvocationID", "MainPID", "process-start-identity", "blocking-wait", "ExecutionGrant", "Work-process-start", "Attempt", "execution-Receipt", "source-to-application-structural-type-safety"] }, + FabricCiGate { id: "FCI-1", owned_question: "What durable reservation binds this exact Work to one already-converged bounded owned cell?", exit_predicate: "After installed-authorization, converged allocation-directory substrate, and exact-path owner/group readback succeed, one exact Work is bound to exactly one selected substrate-converged cell by one committed durable reservation generation. An independent observer reads the same reservation generation, demand/offer payload and content digest after the submitter terminates; release removes only the holding and returns the append-only allocation slot to Free at the exact next generation (SlotAbsent -> Held(1) -> Free(2), or SlotFree(N) -> Held(N+1) -> Free(N+2)), while leaving the persistent cell substrate unchanged. Directory mode remains explicitly unobserved and no Work process starts.", positive_control: "The exact-tree evidence calibration holds; installed sudoers, allocation-directory substrate, slot prestate, and cell owner/group readbacks match; a production reservation survives submitter termination, is independently re-observed byte-identically, releases through the production path, and leaves the slot semantically Free with durable generation history and the cell substrate unchanged.", discriminating_mutations: ["reservation-state-owned-by-submitter", "reservation-generation-changed-in-disposable-store", "reservation-payload-work-mismatch", "selected-offer-cell-mismatch", "cell-substrate-not-converged", "cell-resource-boundary-digest-mismatch", "reservation-release-not-observed", "reservation-unexpected-extra-generation", "reservation-release-destroys-cell-substrate", "multiple-cells", "unowned-cell"], frozen_output_identities: ["WorkKey", "DemandKey", "OfferKey", "CellId/slot-key", "committed-CAS-generation", "reservation-content-digest", "cell-resource-boundary-digest", "cell-owner-identity", "reservation-realization-digest", "allocation-store-path", "allocation-slot-prestate", "allocation-slot-exact-free-poststate"], non_goals: ["cell-supervisor", "passive-lease-unit", "InvocationID", "MainPID", "process-start-identity", "blocking-wait", "ExecutionGrant", "Work-process-start", "Attempt", "execution-Receipt", "source-to-application-structural-type-safety"] }, FabricCiGate { id: "FCI-2", owned_question: "Which sole grant authorizes this Work?", exit_predicate: "Selection, reservation and commitment yield exactly one fenced ExecutionGrant for the exact Work.", positive_control: "One admitted offer yields one committed grant.", discriminating_mutations: ["duplicate-grant", "expired-reservation", "work-mismatch"], frozen_output_identities: ["ExecutionGrant", "lease-epoch"], non_goals: ["process-start", "GitHub-conclusion"] }, FabricCiGate { id: "FCI-3", owned_question: "What causes the target process?", exit_predicate: "A persistent executor re-reads the committed Grant before starting; absent, corrupt or expired Grant yields zero target processes.", positive_control: "One live committed grant starts exactly one target process.", discriminating_mutations: ["grant-removed", "grant-corrupt", "grant-expired"], frozen_output_identities: ["attempt-identity", "process-observation"], non_goals: ["GitHub-conclusion", "multiple-host-scheduling"] }, FabricCiGate { id: "FCI-4", owned_question: "How is execution admitted once, deduplicated, released and sanitized?", exit_predicate: "One fenced Receipt is admitted at most once; duplicate delivery cannot duplicate standing, and the cell is released only after per-attempt sanitation is confirmed.", positive_control: "One matching Receipt is admitted once and its sanitized cell returns to supply.", discriminating_mutations: ["duplicate-receipt", "foreign-receipt", "release-before-sanitation", "failed-sanitation"], frozen_output_identities: ["admitted-receipt", "deduplication-key", "release-receipt", "sanitation-receipt"], non_goals: ["shadow-GitHub-job", "required-check-cutover"] }, diff --git a/dag/gunbc/fabric/fabric_required_build_cell.dag b/dag/gunbc/fabric/fabric_required_build_cell.dag index 9c64a0cbc47..d74f3abcd81 100644 --- a/dag/gunbc/fabric/fabric_required_build_cell.dag +++ b/dag/gunbc/fabric/fabric_required_build_cell.dag @@ -55,17 +55,71 @@ type FabricCellPreReservationStanding | FabricCellPreReservationAbsent { cell: CellId } | FabricCellPreReservationRefused { cell: CellId } -type AllocationStorePrestate - = AllocationStoreAbsent { path: NonEmptyStr } - | AllocationStoreFree { path: NonEmptyStr, generation: Int } - | AllocationStoreHeld { path: NonEmptyStr, generation: Int } - | AllocationStoreUnreadable { path: NonEmptyStr } - | AllocationStoreUndecodable { path: NonEmptyStr, generation: Int } +// Directory-substrate standing and slot state are deliberately different coordinates. An empty, +// converged allocation directory legitimately contains no slot version yet; that is SlotAbsent, +// not evidence that the directory substrate is absent. +type AllocationSlotPrestate + = AllocationSlotAbsent { path: NonEmptyStr } + | AllocationSlotFree { path: NonEmptyStr, generation: Int } + | AllocationSlotHeld { path: NonEmptyStr, generation: Int } + | AllocationSlotUnreadable { path: NonEmptyStr } + | AllocationSlotUndecodable { path: NonEmptyStr, generation: Int } type AllocationStoreRestoration - = AllocationStoreReturnedFree { path: NonEmptyStr, pre_generation: Int, post_generation: Int } + = AllocationSlotAbsentReturnedFree { path: NonEmptyStr, held_generation: Int, post_generation: Int } + | AllocationSlotFreeReturnedFree { path: NonEmptyStr, pre_generation: Int, held_generation: Int, post_generation: Int } | AllocationStoreRestorationRefused { path: NonEmptyStr } +func fci1_allocation_slot_available(prestate: AllocationSlotPrestate) -> Bool { + match prestate { + AllocationSlotAbsent { path: _ } => true + AllocationSlotFree { path: _, generation: _ } => true + AllocationSlotHeld { path: _, generation: _ } => false + AllocationSlotUnreadable { path: _ } => false + AllocationSlotUndecodable { path: _, generation: _ } => false + } +} + +// The canonical lifecycle advances exactly twice: reserve commits Held, then release commits Free. +// An append-only store may not return byte-empty, and a merely monotone comparison would admit an +// unrelated writer's extra generation. +func fci1_exact_allocation_restoration( + prestate: AllocationSlotPrestate, + held_generation: Int, + poststate: AllocationSlotPrestate, +) -> AllocationStoreRestoration { + match prestate { + AllocationSlotAbsent { path: p } => match poststate { + AllocationSlotFree { path: q, generation: post } => + if p == q && held_generation == 1 && post == 2 { + AllocationSlotAbsentReturnedFree { path: q, held_generation: held_generation, post_generation: post } + } else { AllocationStoreRestorationRefused { path: p } } + AllocationSlotAbsent { path: _ } => AllocationStoreRestorationRefused { path: p } + AllocationSlotHeld { path: _, generation: _ } => AllocationStoreRestorationRefused { path: p } + AllocationSlotUnreadable { path: _ } => AllocationStoreRestorationRefused { path: p } + AllocationSlotUndecodable { path: _, generation: _ } => AllocationStoreRestorationRefused { path: p } + } + AllocationSlotFree { path: p, generation: pre } => match poststate { + AllocationSlotFree { path: q, generation: post } => + if p == q && held_generation == pre + 1 && post == pre + 2 { + AllocationSlotFreeReturnedFree { + path: q, + pre_generation: pre, + held_generation: held_generation, + post_generation: post, + } + } else { AllocationStoreRestorationRefused { path: p } } + AllocationSlotAbsent { path: _ } => AllocationStoreRestorationRefused { path: p } + AllocationSlotHeld { path: _, generation: _ } => AllocationStoreRestorationRefused { path: p } + AllocationSlotUnreadable { path: _ } => AllocationStoreRestorationRefused { path: p } + AllocationSlotUndecodable { path: _, generation: _ } => AllocationStoreRestorationRefused { path: p } + } + AllocationSlotHeld { path: p, generation: _ } => AllocationStoreRestorationRefused { path: p } + AllocationSlotUnreadable { path: p } => AllocationStoreRestorationRefused { path: p } + AllocationSlotUndecodable { path: p, generation: _ } => AllocationStoreRestorationRefused { path: p } + } +} + // Mode remains outside the observed guarantee until the same exact-path producer reports it. fn fci1_directory_mode_unobserved(path: NonEmptyStr, desired_mode: NonEmptyStr) -> DirectoryModeStanding { DirectoryModeUnobserved { diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 2d85cbce48c..68fb68aa054 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -59,10 +59,11 @@ import gunbc.fabric_required_build_cell { RequiredBuildCellLifetimeSampleDecode, RequiredBuildCellLifetimeSampleDecoded, RequiredBuildCellLifetimeSampleUndecodable, FabricCellPreReservationStanding, FabricCellPreReservationClean, FabricCellPreReservationAbsent, FabricCellPreReservationRefused, - AllocationStorePrestate, AllocationStoreAbsent, AllocationStoreFree, AllocationStoreHeld, - AllocationStoreUnreadable, AllocationStoreUndecodable, - AllocationStoreRestoration, AllocationStoreReturnedFree, + AllocationSlotPrestate, AllocationSlotAbsent, AllocationSlotFree, AllocationSlotHeld, + AllocationSlotUnreadable, AllocationSlotUndecodable, + AllocationStoreRestoration, AllocationSlotAbsentReturnedFree, AllocationSlotFreeReturnedFree, AllocationStoreRestorationRefused, + fci1_allocation_slot_available, fci1_exact_allocation_restoration, observe_required_build_cell_wet, required_build_cell_lifetime_sample, required_build_cell_lifetime_sample_wire, decode_required_build_cell_lifetime_sample, required_build_cell_lifetime_verdict, required_build_cell_lifetime_verdict_wire, @@ -261,6 +262,10 @@ func fci1_assert_host_preconditions() -> ProcessExit { exit_failure(reason: "FCI-1 cell root owner/group mismatch") } else if !fci1_path_owner_matches(path: attempts, expected: runner_owner) { exit_failure(reason: "FCI-1 attempts root owner/group mismatch") + } else if !fci1_path_owner_matches(path: "/var/lib/gunbc/fabric" as NonEmptyStr, expected: fci1_root_owner()) { + exit_failure(reason: "FCI-1 allocation namespace substrate owner/group mismatch") + } else if !fci1_path_owner_matches(path: "/var/lib/gunbc/fabric/allocation" as NonEmptyStr, expected: fci1_root_owner()) { + exit_failure(reason: "FCI-1 allocation directory substrate owner/group mismatch") } else { match mode { DirectoryModeUnobserved { path: p, desired_mode: desired, trigger: trigger } => @@ -514,45 +519,45 @@ func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) -> ProcessEx } } -fn fci1_allocation_prestate(root: NonEmptyStr) -> AllocationStorePrestate { +fn fci1_allocation_prestate(root: NonEmptyStr) -> AllocationSlotPrestate { match observe_cas_slot_state(root: root, key: fabric_cell_allocation_key(slot: cp_probe_slot())) { - CasObservedUnreadable { cause: _ } => AllocationStoreUnreadable { path: root } - CasObservedReadable { readable: CasReadableAbsent } => AllocationStoreAbsent { path: root } + CasObservedUnreadable { cause: _ } => AllocationSlotUnreadable { path: root } + CasObservedReadable { readable: CasReadableAbsent } => AllocationSlotAbsent { path: root } CasObservedReadable { readable: CasReadablePresent { version: version } } => match decode_cell_slot_payload(payload: version.value) { - CellSlotDecoded { state: CellFree { released_by: _ } } => AllocationStoreFree { path: root, generation: version.generation } - CellSlotDecoded { state: CellHeld { reservation: _ } } => AllocationStoreHeld { path: root, generation: version.generation } - CellSlotUndecodable { detail: _ } => AllocationStoreUndecodable { path: root, generation: version.generation } + CellSlotDecoded { state: CellFree { released_by: _ } } => AllocationSlotFree { path: root, generation: version.generation } + CellSlotDecoded { state: CellHeld { reservation: _ } } => AllocationSlotHeld { path: root, generation: version.generation } + CellSlotUndecodable { detail: _ } => AllocationSlotUndecodable { path: root, generation: version.generation } } } } -fn fci1_allocation_prestate_wire(prestate: AllocationStorePrestate) -> NonEmptyStr { +fn fci1_allocation_prestate_wire(prestate: AllocationSlotPrestate) -> NonEmptyStr { match prestate { - AllocationStoreAbsent { path: p } => join(["absent\n", p as String], "") as NonEmptyStr - AllocationStoreFree { path: p, generation: g } => join(["free\n", p as String, "\n", to_string(g)], "") as NonEmptyStr - AllocationStoreHeld { path: p, generation: g } => join(["held\n", p as String, "\n", to_string(g)], "") as NonEmptyStr - AllocationStoreUnreadable { path: p } => join(["unreadable\n", p as String], "") as NonEmptyStr - AllocationStoreUndecodable { path: p, generation: g } => join(["undecodable\n", p as String, "\n", to_string(g)], "") as NonEmptyStr + AllocationSlotAbsent { path: p } => join(["slot-absent\n", p as String], "") as NonEmptyStr + AllocationSlotFree { path: p, generation: g } => join(["slot-free\n", p as String, "\n", to_string(g)], "") as NonEmptyStr + AllocationSlotHeld { path: p, generation: g } => join(["slot-held\n", p as String, "\n", to_string(g)], "") as NonEmptyStr + AllocationSlotUnreadable { path: p } => join(["slot-unreadable\n", p as String], "") as NonEmptyStr + AllocationSlotUndecodable { path: p, generation: g } => join(["slot-undecodable\n", p as String, "\n", to_string(g)], "") as NonEmptyStr } } -fn fci1_decode_allocation_prestate(wire: String) -> AllocationStorePrestate { +fn fci1_decode_allocation_prestate(wire: String) -> AllocationSlotPrestate { let fields = split(s: wire, delimiter: "\n") match fields.first() { - Absent => AllocationStoreUnreadable { path: "invalid-before-wire" } + Absent => AllocationSlotUnreadable { path: "invalid-before-wire" } Present { value: tag } => match fields.skip(n: 1).first() { - Absent => AllocationStoreUnreadable { path: "invalid-before-wire" } - Present { value: path } => if path == "" { AllocationStoreUnreadable { path: "invalid-before-wire" } } else { - if tag == "absent" && list_length(items: fields) == 2 { AllocationStoreAbsent { path: path as NonEmptyStr } } + Absent => AllocationSlotUnreadable { path: "invalid-before-wire" } + Present { value: path } => if path == "" { AllocationSlotUnreadable { path: "invalid-before-wire" } } else { + if tag == "slot-absent" && list_length(items: fields) == 2 { AllocationSlotAbsent { path: path as NonEmptyStr } } else { match fields.skip(n: 2).first() { - Absent => AllocationStoreUnreadable { path: path as NonEmptyStr } + Absent => AllocationSlotUnreadable { path: path as NonEmptyStr } Present { value: generation_text } => match parse_int(s: generation_text) { - Absent => AllocationStoreUnreadable { path: path as NonEmptyStr } + Absent => AllocationSlotUnreadable { path: path as NonEmptyStr } Present { value: generation } => - if tag == "free" { AllocationStoreFree { path: path as NonEmptyStr, generation: generation } } - else if tag == "held" { AllocationStoreHeld { path: path as NonEmptyStr, generation: generation } } - else { AllocationStoreUndecodable { path: path as NonEmptyStr, generation: generation } } + if tag == "slot-free" { AllocationSlotFree { path: path as NonEmptyStr, generation: generation } } + else if tag == "slot-held" { AllocationSlotHeld { path: path as NonEmptyStr, generation: generation } } + else { AllocationSlotUndecodable { path: path as NonEmptyStr, generation: generation } } } } } } @@ -568,45 +573,82 @@ func fci1_live_allocation_prestate(root: NonEmptyStr, path: String) -> ProcessEx } func fci1_assert_allocation_available(root: NonEmptyStr) -> ProcessExit { - match fci1_allocation_prestate(root: root) { - AllocationStoreAbsent { path: _ } => exit_failure(reason: "FCI-1 allocation store substrate is absent") - AllocationStoreFree { path: _, generation: _ } => ExitSuccess - AllocationStoreHeld { path: _, generation: _ } => exit_failure(reason: "FCI-1 allocation store remains held") - AllocationStoreUnreadable { path: _ } => exit_failure(reason: "FCI-1 allocation store unreadable") - AllocationStoreUndecodable { path: _, generation: _ } => exit_failure(reason: "FCI-1 allocation store undecodable") - } + let prestate = fci1_allocation_prestate(root: root) + if fci1_allocation_slot_available(prestate: prestate) { ExitSuccess } else { match prestate { + AllocationSlotAbsent { path: _ } => ExitSuccess + AllocationSlotFree { path: _, generation: _ } => ExitSuccess + AllocationSlotHeld { path: _, generation: _ } => exit_failure(reason: "FCI-1 allocation slot remains held") + AllocationSlotUnreadable { path: _ } => exit_failure(reason: "FCI-1 allocation slot unreadable") + AllocationSlotUndecodable { path: _, generation: _ } => exit_failure(reason: "FCI-1 allocation slot undecodable") + } } } -fn fci1_allocation_restoration(root: NonEmptyStr, before_wire: String) -> AllocationStoreRestoration { +fn fci1_allocation_restoration(root: NonEmptyStr, before_wire: String, held_wire: String) -> AllocationStoreRestoration { match decode_fabric_ci_evidence_wire(wire: before_wire) { FabricCiEvidenceUndecodable => AllocationStoreRestorationRefused { path: root } FabricCiEvidenceDecoded { value: before } => { let prestate = fci1_decode_allocation_prestate(wire: before as String) let poststate = fci1_allocation_prestate(root: root) - match prestate { - AllocationStoreAbsent { path: _ } => AllocationStoreRestorationRefused { path: root } - AllocationStoreFree { path: p, generation: pre } => match poststate { - AllocationStoreFree { path: q, generation: post } => if p == q && post > pre { AllocationStoreReturnedFree { path: q, pre_generation: pre, post_generation: post } } else { AllocationStoreRestorationRefused { path: root } } - AllocationStoreAbsent { path: _ } => AllocationStoreRestorationRefused { path: root } - AllocationStoreHeld { path: _, generation: _ } => AllocationStoreRestorationRefused { path: root } - AllocationStoreUnreadable { path: _ } => AllocationStoreRestorationRefused { path: root } - AllocationStoreUndecodable { path: _, generation: _ } => AllocationStoreRestorationRefused { path: root } - } - AllocationStoreHeld { path: _, generation: _ } => AllocationStoreRestorationRefused { path: root } - AllocationStoreUnreadable { path: _ } => AllocationStoreRestorationRefused { path: root } - AllocationStoreUndecodable { path: _, generation: _ } => AllocationStoreRestorationRefused { path: root } + match fci1_decode_transport_sample(wire: held_wire) { + RequiredBuildCellLifetimeSampleUndecodable => AllocationStoreRestorationRefused { path: root } + RequiredBuildCellLifetimeSampleDecoded { sample: held } => fci1_exact_allocation_restoration( + prestate: prestate, + held_generation: held.generation, + poststate: poststate, + ) } } } } -func fci1_assert_allocation_restored(root: NonEmptyStr, before_wire: String) -> ProcessExit { - match fci1_allocation_restoration(root: root, before_wire: before_wire) { - AllocationStoreReturnedFree { path: _, pre_generation: _, post_generation: _ } => ExitSuccess +func fci1_assert_allocation_restored(root: NonEmptyStr, before_wire: String, held_wire: String) -> ProcessExit { + match fci1_allocation_restoration(root: root, before_wire: before_wire, held_wire: held_wire) { + AllocationSlotAbsentReturnedFree { path: _, held_generation: _, post_generation: _ } => ExitSuccess + AllocationSlotFreeReturnedFree { path: _, pre_generation: _, held_generation: _, post_generation: _ } => ExitSuccess AllocationStoreRestorationRefused { path: _ } => exit_failure(reason: "FCI-1 allocation store restoration refused") } } +// Pure controls run before host effects. Their rejecting counterparts are enrolled in the witness +// corpus; the disposable-store mutation in the live driver exercises the production CAS mechanism. +func fci1_assert_slot_absent_admitted() -> ProcessExit { + if fci1_allocation_slot_available(prestate: AllocationSlotAbsent { path: "/run/fci1-control" }) { ExitSuccess } + else { exit_failure(reason: "FCI-1 absent slot was not admitted") } +} + +func fci1_assert_exact_absent_to_free_control() -> ProcessExit { + match fci1_exact_allocation_restoration( + prestate: AllocationSlotAbsent { path: "/run/fci1-control" }, + held_generation: 1, + poststate: AllocationSlotFree { path: "/run/fci1-control", generation: 2 }, + ) { + AllocationSlotAbsentReturnedFree { path: _, held_generation: _, post_generation: _ } => ExitSuccess + _ => exit_failure(reason: "FCI-1 exact absent-slot lifecycle refused") + } +} + +func fci1_assert_exact_free_to_free_control() -> ProcessExit { + match fci1_exact_allocation_restoration( + prestate: AllocationSlotFree { path: "/run/fci1-control", generation: 7 }, + held_generation: 8, + poststate: AllocationSlotFree { path: "/run/fci1-control", generation: 9 }, + ) { + AllocationSlotFreeReturnedFree { path: _, pre_generation: _, held_generation: _, post_generation: _ } => ExitSuccess + _ => exit_failure(reason: "FCI-1 exact free-slot lifecycle refused") + } +} + +func fci1_assert_extra_generation_refused_control() -> ProcessExit { + match fci1_exact_allocation_restoration( + prestate: AllocationSlotFree { path: "/run/fci1-control", generation: 7 }, + held_generation: 8, + poststate: AllocationSlotFree { path: "/run/fci1-control", generation: 10 }, + ) { + AllocationStoreRestorationRefused { path: _ } => ExitSuccess + _ => exit_failure(reason: "FCI-1 unexpected extra generation was admitted") + } +} + fn fci1_decode_transport_sample(wire: String) -> RequiredBuildCellLifetimeSampleDecode { match decode_fabric_ci_evidence_wire(wire: wire) { FabricCiEvidenceUndecodable => RequiredBuildCellLifetimeSampleUndecodable diff --git a/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag b/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag new file mode 100644 index 00000000000..a130e8056e4 --- /dev/null +++ b/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag @@ -0,0 +1,59 @@ +module test.claim.fabric.fabric_ci_fci1_allocation_lifecycle_witness_test + +import gunbc.fabric_required_build_cell { + AllocationSlotAbsent, AllocationSlotFree, AllocationSlotHeld, + AllocationStoreRestorationRefused, + AllocationSlotAbsentReturnedFree, AllocationSlotFreeReturnedFree, + fci1_allocation_slot_available, fci1_exact_allocation_restoration, +} + +test fn witness_fci1_absent_slot_is_available_but_held_slot_is_not() -> Bool { + fci1_allocation_slot_available(prestate: AllocationSlotAbsent { path: "/fixture/store" }) + && !fci1_allocation_slot_available(prestate: AllocationSlotHeld { path: "/fixture/store", generation: 1 }) +} + +test fn witness_fci1_absent_slot_exactly_returns_free_at_generation_two() -> Bool { + match fci1_exact_allocation_restoration( + prestate: AllocationSlotAbsent { path: "/fixture/store" }, + held_generation: 1, + poststate: AllocationSlotFree { path: "/fixture/store", generation: 2 }, + ) { + AllocationSlotAbsentReturnedFree { path: p, held_generation: h, post_generation: f } => + p == "/fixture/store" && h == 1 && f == 2 + _ => false + } +} + +test fn witness_fci1_free_slot_exactly_advances_reserve_then_release() -> Bool { + match fci1_exact_allocation_restoration( + prestate: AllocationSlotFree { path: "/fixture/store", generation: 7 }, + held_generation: 8, + poststate: AllocationSlotFree { path: "/fixture/store", generation: 9 }, + ) { + AllocationSlotFreeReturnedFree { path: p, pre_generation: pre, held_generation: held, post_generation: post } => + p == "/fixture/store" && pre == 7 && held == 8 && post == 9 + _ => false + } +} + +test fn witness_fci1_unexpected_extra_generation_refuses() -> Bool { + match fci1_exact_allocation_restoration( + prestate: AllocationSlotFree { path: "/fixture/store", generation: 7 }, + held_generation: 8, + poststate: AllocationSlotFree { path: "/fixture/store", generation: 10 }, + ) { + AllocationStoreRestorationRefused { path: p } => p == "/fixture/store" + _ => false + } +} + +test fn witness_fci1_wrong_held_generation_refuses() -> Bool { + match fci1_exact_allocation_restoration( + prestate: AllocationSlotAbsent { path: "/fixture/store" }, + held_generation: 2, + poststate: AllocationSlotFree { path: "/fixture/store", generation: 2 }, + ) { + AllocationStoreRestorationRefused { path: p } => p == "/fixture/store" + _ => false + } +} diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index abd6c0aa7ea..edf95d14de4 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -16,28 +16,68 @@ fabric_ci_driver_init canonical_root=/var/lib/gunbc/fabric/allocation mutation_runtime=fci1-submitter-owned-reservation mutation_root=/run/$mutation_runtime +generation_root= subject="$repo_root/dag/gunbc/fabric/fabric_required_build_cell.dag" canonical_reservation_may_be_held=0 +generation_reservation_may_be_held=0 allocation_before= +positive_before= + +unit_terminal_observed() { + local unit=$1 output line load_state= active_state= + if output=$(systemctl show "$unit" --property=LoadState --property=ActiveState 2>&1); then + : + else + return 1 + fi + [[ -n $output ]] || return 1 + while IFS= read -r line; do + case $line in + LoadState=*) load_state=${line#LoadState=} ;; + ActiveState=*) active_state=${line#ActiveState=} ;; + esac + done <<< "$output" + [[ -n $load_state && -n $active_state ]] || return 1 + [[ $active_state == inactive || $active_state == failed ]] || return 1 + [[ $load_state == loaded || $load_state == not-found ]] || return 1 +} cleanup() { local original_status=$? local cleanup_status=0 trap - EXIT set +e - systemctl stop fci1-positive-submitter.service fci1-dependent-submitter.service >/dev/null 2>&1 || true + systemctl stop fci1-positive-submitter.service fci1-generation-submitter.service fci1-dependent-submitter.service >/dev/null 2>&1 || true if (( canonical_reservation_may_be_held )); then "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ --entry "$FABRIC_CI_ENTRY" --function fci1_live_release --arg root="$canonical_root" >/dev/null 2>&1 "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ --entry "$FABRIC_CI_ENTRY" --function fci1_assert_allocation_restored \ - --arg root="$canonical_root" --arg before_wire="$allocation_before" + --arg root="$canonical_root" --arg before_wire="$allocation_before" --arg held_wire="$positive_before" cleanup_status=$? fi + if (( generation_reservation_may_be_held )) && [[ -n $generation_root ]]; then + "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ + --entry "$FABRIC_CI_ENTRY" --function fci1_live_release --arg root="$generation_root" >/dev/null 2>&1 + local generation_cleanup_status=$? + (( cleanup_status != 0 )) || cleanup_status=$generation_cleanup_status + fi + [[ -z $generation_root ]] || rm -rf -- "$generation_root" + if [[ -e $mutation_root || -L $mutation_root ]]; then + echo 'InstrumentRefused: dependent RuntimeDirectory remains after cleanup' >&2 + cleanup_status=1 + fi + local cleanup_unit + for cleanup_unit in fci1-positive-submitter.service fci1-generation-submitter.service fci1-dependent-submitter.service; do + if ! unit_terminal_observed "$cleanup_unit"; then + echo "InstrumentRefused: transient unit terminal state unestablished: $cleanup_unit" >&2 + cleanup_status=1 + fi + done rm -f "$FABRIC_CI_LOG" rm -rf "$FABRIC_CI_VALUE_ROOT" if (( cleanup_status != 0 )); then - echo 'InstrumentRefused: cleanup could not positively restore allocation availability' >&2 + echo 'InstrumentRefused: cleanup terminal postconditions did not all hold' >&2 exit 1 fi exit "$original_status" @@ -54,6 +94,10 @@ FABRIC_CI_GUNBC_BIN="$FABRIC_CI_GUNBC_BIN" bash "$repo_root/tools/fabric_ci_evid sha256sum "$subject" fabric_ci_run_assertion fci1_invalid_before_wire_refuses +fabric_ci_run_assertion fci1_assert_slot_absent_admitted +fabric_ci_run_assertion fci1_assert_exact_absent_to_free_control +fabric_ci_run_assertion fci1_assert_exact_free_to_free_control +fabric_ci_run_assertion fci1_assert_extra_generation_refused_control fabric_ci_run_assertion fci1_assert_frozen_work_key fabric_ci_run_assertion fci1_assert_host_preconditions fabric_ci_run_assertion fci1_assert_clean_cell_prestate @@ -67,9 +111,15 @@ capture_submitter_transport() { shift 4 local value_path="$FABRIC_CI_VALUE_ROOT/$coordinate.wire" rm -f -- "$value_path" - systemd-run --quiet --wait --collect --unit="$unit" "$@" \ + local run_status=0 + if systemd-run --quiet --wait --collect --unit="$unit" "$@" \ "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ - --entry "$FABRIC_CI_ENTRY" --function "$function_name" --arg root="$root" --arg path="$value_path" + --entry "$FABRIC_CI_ENTRY" --function "$function_name" --arg root="$root" --arg path="$value_path"; then + run_status=0 + else + run_status=$? + fi + (( run_status == 0 )) || return "$run_status" [[ -f $value_path && ! -L $value_path ]] || return 1 local value value=$(<"$value_path") @@ -79,24 +129,33 @@ capture_submitter_transport() { canonical_reservation_may_be_held=1 positive_before=$(capture_submitter_transport fci1-positive-submitter.service fci1_live_reserve_and_observe_available positive-before "$canonical_root") -submitter_state=$(systemctl show fci1-positive-submitter.service --property=ActiveState --value 2>&1 || true) -[[ -z $submitter_state || $submitter_state == inactive || $submitter_state == failed ]] || { - echo "InstrumentRefused: submitting control process still active: $submitter_state" >&2 +unit_terminal_observed fci1-positive-submitter.service || { + echo 'InstrumentRefused: submitting control process terminal state unestablished' >&2 exit 1 } fabric_ci_run_assertion fci1_assert_lifetime_held --arg root="$canonical_root" --arg before_wire="$positive_before" -generation_before=$(fabric_ci_capture_transport fci1_live_binding_sample_wire generation-before --arg root="$canonical_root") -fabric_ci_run_assertion fci1_assert_replace_held --arg root="$canonical_root" -fabric_ci_run_assertion fci1_assert_generation_changed --arg root="$canonical_root" --arg before_wire="$generation_before" - fabric_ci_run_assertion fci1_live_release --arg root="$canonical_root" +fabric_ci_run_assertion fci1_assert_allocation_restored --arg root="$canonical_root" --arg before_wire="$allocation_before" --arg held_wire="$positive_before" canonical_reservation_may_be_held=0 -fabric_ci_run_assertion fci1_assert_allocation_restored --arg root="$canonical_root" --arg before_wire="$allocation_before" fabric_ci_run_assertion fci1_assert_cell_unchanged --arg before_wire="$cell_before" +# Generation replacement is a mutation falsifier, not part of the canonical lifecycle. Its +# run-unique disposable store uses the production reservation/CAS entries and is removed afterward; +# the canonical root sees only reserve, independent observation, and release. +generation_root=$(mktemp -d /run/fci1-generation-mutation.XXXXXX) +generation_reservation_may_be_held=1 +generation_before=$(capture_submitter_transport fci1-generation-submitter.service fci1_live_reserve_and_observe_available generation-before "$generation_root") +fabric_ci_run_assertion fci1_assert_replace_held --arg root="$generation_root" +fabric_ci_run_assertion fci1_assert_generation_changed --arg root="$generation_root" --arg before_wire="$generation_before" +fabric_ci_run_assertion fci1_live_release --arg root="$generation_root" +generation_reservation_may_be_held=0 +rm -rf -- "$generation_root" +generation_root= + dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service fci1_live_reserve_and_observe_available dependent-before "$mutation_root" --property=RuntimeDirectory="$mutation_runtime") -[[ ! -e $mutation_root ]] || { echo 'ExpectedOutcomeNotObserved: submitter runtime directory removal' >&2; exit 1; } +[[ ! -e $mutation_root && ! -L $mutation_root ]] || { echo 'ExpectedOutcomeNotObserved: submitter runtime directory removal' >&2; exit 1; } +unit_terminal_observed fci1-dependent-submitter.service || { echo 'InstrumentRefused: dependent submitter terminal state unestablished' >&2; exit 1; } fabric_ci_run_assertion fci1_assert_lifetime_dependent --arg root="$mutation_root" --arg before_wire="$dependent_before" fabric_ci_run_assertion fci1_assert_cell_unchanged --arg before_wire="$cell_before" From 2020e22ab637bc8b6002194d07997f8edd2468f3 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 12:11:36 +0000 Subject: [PATCH 06/50] Strengthen FCI-1 terminal observations --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 6 ++++++ .../fabric_ci_fci1_allocation_lifecycle_witness_test.dag | 6 +++++- tools/fabric_ci_fci1_live_instrument.sh | 7 ++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 68fb68aa054..71ba5209c7f 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -616,6 +616,12 @@ func fci1_assert_slot_absent_admitted() -> ProcessExit { else { exit_failure(reason: "FCI-1 absent slot was not admitted") } } +func fci1_assert_unreadable_slot_refused() -> ProcessExit { + if fci1_allocation_slot_available(prestate: AllocationSlotUnreadable { path: "/run/fci1-control" }) { + exit_failure(reason: "FCI-1 unreadable slot was admitted") + } else { ExitSuccess } +} + func fci1_assert_exact_absent_to_free_control() -> ProcessExit { match fci1_exact_allocation_restoration( prestate: AllocationSlotAbsent { path: "/run/fci1-control" }, diff --git a/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag b/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag index a130e8056e4..91700b0a011 100644 --- a/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag +++ b/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag @@ -1,7 +1,7 @@ module test.claim.fabric.fabric_ci_fci1_allocation_lifecycle_witness_test import gunbc.fabric_required_build_cell { - AllocationSlotAbsent, AllocationSlotFree, AllocationSlotHeld, + AllocationSlotAbsent, AllocationSlotFree, AllocationSlotHeld, AllocationSlotUnreadable, AllocationStoreRestorationRefused, AllocationSlotAbsentReturnedFree, AllocationSlotFreeReturnedFree, fci1_allocation_slot_available, fci1_exact_allocation_restoration, @@ -12,6 +12,10 @@ test fn witness_fci1_absent_slot_is_available_but_held_slot_is_not() -> Bool { && !fci1_allocation_slot_available(prestate: AllocationSlotHeld { path: "/fixture/store", generation: 1 }) } +test fn witness_fci1_unreadable_slot_is_never_available() -> Bool { + !fci1_allocation_slot_available(prestate: AllocationSlotUnreadable { path: "/fixture/store" }) +} + test fn witness_fci1_absent_slot_exactly_returns_free_at_generation_two() -> Bool { match fci1_exact_allocation_restoration( prestate: AllocationSlotAbsent { path: "/fixture/store" }, diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index edf95d14de4..85c642f55fd 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -39,7 +39,8 @@ unit_terminal_observed() { done <<< "$output" [[ -n $load_state && -n $active_state ]] || return 1 [[ $active_state == inactive || $active_state == failed ]] || return 1 - [[ $load_state == loaded || $load_state == not-found ]] || return 1 + [[ $load_state == not-found ]] || return 1 + echo "unit-terminal-observed: unit=$unit LoadState=$load_state ActiveState=$active_state" >&2 } cleanup() { @@ -66,6 +67,8 @@ cleanup() { if [[ -e $mutation_root || -L $mutation_root ]]; then echo 'InstrumentRefused: dependent RuntimeDirectory remains after cleanup' >&2 cleanup_status=1 + else + echo "runtime-directory-observed: path=$mutation_root standing=absent" >&2 fi local cleanup_unit for cleanup_unit in fci1-positive-submitter.service fci1-generation-submitter.service fci1-dependent-submitter.service; do @@ -95,6 +98,7 @@ FABRIC_CI_GUNBC_BIN="$FABRIC_CI_GUNBC_BIN" bash "$repo_root/tools/fabric_ci_evid sha256sum "$subject" fabric_ci_run_assertion fci1_invalid_before_wire_refuses fabric_ci_run_assertion fci1_assert_slot_absent_admitted +fabric_ci_run_assertion fci1_assert_unreadable_slot_refused fabric_ci_run_assertion fci1_assert_exact_absent_to_free_control fabric_ci_run_assertion fci1_assert_exact_free_to_free_control fabric_ci_run_assertion fci1_assert_extra_generation_refused_control @@ -155,6 +159,7 @@ generation_root= dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service fci1_live_reserve_and_observe_available dependent-before "$mutation_root" --property=RuntimeDirectory="$mutation_runtime") [[ ! -e $mutation_root && ! -L $mutation_root ]] || { echo 'ExpectedOutcomeNotObserved: submitter runtime directory removal' >&2; exit 1; } +echo "runtime-directory-observed: path=$mutation_root standing=absent" >&2 unit_terminal_observed fci1-dependent-submitter.service || { echo 'InstrumentRefused: dependent submitter terminal state unestablished' >&2; exit 1; } fabric_ci_run_assertion fci1_assert_lifetime_dependent --arg root="$mutation_root" --arg before_wire="$dependent_before" fabric_ci_run_assertion fci1_assert_cell_unchanged --arg before_wire="$cell_before" From 2c1cac945883f87a3d74b87d90bc18f8b0c40a49 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 14:45:50 +0000 Subject: [PATCH 07/50] Strengthen FCI-1 wet failure receipts --- .../fabric_control_plane_live_probe.dag | 429 +++++++++++++++++- ...fci1_allocation_lifecycle_witness_test.dag | 20 + tools/fabric_ci_fci1_live_instrument.sh | 100 +++- 3 files changed, 527 insertions(+), 22 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 71ba5209c7f..3faabc4e3bf 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -103,6 +103,75 @@ import gunbc.host_effect_realize { srv3_observe_path_owner } import extdeps.posix.path_ownership { PathOwnershipObservation, PathOwnerObserved, PathOwnerUnobserved, posix_owner_spec_equal } import extdeps.posix.identity { PosixOwnerSpec, PosixUserId, PosixGroupId } import extdeps.tools.sha256sum { Sha256FileDigest, Sha256FileDigestUnavailable, sha256sum_file_digest_via_shell } +import extdeps.filesystem.filesystem_io { Filesystem } +import gunbc.systemctl_show_read { + systemctl_show_property_read, SystemdPropertyCaptured, SystemdPropertyCaptureRefused, +} +import extdeps.systemd { ControlGroup } +import extdeps.linux.cgroup_v2 { cgroup_v2_mount_point, cgroup_v2_child_path } +import gunbc.host_operation_exec { + HostOperation, HostOperationOutcome, HostOperationObserved, HostOperationRefused, + CgroupListChildren, CgroupReadInterfaceFile, CgroupPathIsDirectory, + host_operation_exec_local, +} +import gunbc.host_converge_slice1 { host_converge_slice1_enumerate_units } + +type Fci1Checkpoint + = Fci1CheckpointNone + | Fci1CheckpointBeforeCanonicalCommit + | Fci1CheckpointAfterCanonicalCommitBeforeTransport + | Fci1CheckpointAfterHeldPreserved + | Fci1CheckpointAfterReleaseBeforeGrading + | Fci1CheckpointCanonicalRestoredLater + | Fci1CheckpointGenerationHeldOne + | Fci1CheckpointGenerationHeldTwo + | Fci1CheckpointGenerationChangedObserved + | Fci1CheckpointGenerationFreeBeforeRemoval + | Fci1CheckpointRuntimeDirectoryTerminal + +fn fci1_checkpoint_token(checkpoint: Fci1Checkpoint) -> NonEmptyStr { + match checkpoint { + Fci1CheckpointNone => "none" + Fci1CheckpointBeforeCanonicalCommit => "before-canonical-commit" + Fci1CheckpointAfterCanonicalCommitBeforeTransport => "after-canonical-commit-before-transport" + Fci1CheckpointAfterHeldPreserved => "after-held-preserved" + Fci1CheckpointAfterReleaseBeforeGrading => "after-release-before-grading" + Fci1CheckpointCanonicalRestoredLater => "canonical-restored-later" + Fci1CheckpointGenerationHeldOne => "generation-held-one" + Fci1CheckpointGenerationHeldTwo => "generation-held-two" + Fci1CheckpointGenerationChangedObserved => "generation-changed-observed" + Fci1CheckpointGenerationFreeBeforeRemoval => "generation-free-before-removal" + Fci1CheckpointRuntimeDirectoryTerminal => "runtime-directory-terminal" + } +} + +fn fci1_checkpoints() -> List { + [ + Fci1CheckpointNone, + Fci1CheckpointBeforeCanonicalCommit, + Fci1CheckpointAfterCanonicalCommitBeforeTransport, + Fci1CheckpointAfterHeldPreserved, + Fci1CheckpointAfterReleaseBeforeGrading, + Fci1CheckpointCanonicalRestoredLater, + Fci1CheckpointGenerationHeldOne, + Fci1CheckpointGenerationHeldTwo, + Fci1CheckpointGenerationChangedObserved, + Fci1CheckpointGenerationFreeBeforeRemoval, + Fci1CheckpointRuntimeDirectoryTerminal, + ] +} + +func fci1_assert_checkpoint_token(token: NonEmptyStr) -> ProcessExit { + if list_length(items: filter(fci1_checkpoints(), c => fci1_checkpoint_token(checkpoint: c) == token)) == 1 { + ExitSuccess + } else { exit_failure(reason: join(["unknown or duplicate FCI-1 checkpoint token: ", token as String], "")) } +} + +func fci1_checkpoint_reached(token: NonEmptyStr, phase: NonEmptyStr) -> ProcessExit { + if token == phase && list_length(items: filter(fci1_checkpoints(), c => fci1_checkpoint_token(checkpoint: c) == token)) == 1 { + ExitSuccess + } else { exit_failure(reason: join(["phase-inapplicable FCI-1 checkpoint: token=", token as String, " phase=", phase as String], "")) } +} // The live receipt for the EFFECTFUL half of the broker, which the hermetic witness cannot // reach. An entry point rather than a shell script, for the reason DESIGN gives: if a @@ -455,7 +524,19 @@ fn fci1_offer_candidacy_wire(candidacy: OfferCandidacy) -> NonEmpty } fn fci1_member_wire(member: FabricCellMember) -> NonEmptyStr { - join(["member|", fabric_cell_member_key(member: member), "|", member.state_digest as String], "") as NonEmptyStr + join([ + "member|key=", fabric_cell_member_key(member: member), + "|address=", to_string(member.address), + "|state_digest=", member.state_digest as String, + "|modeled_ownership=", fci1_modeled_ownership_wire(member: member) as String, + ], "") as NonEmptyStr +} + +fn fci1_path_owner_wire(path: NonEmptyStr) -> String { + match srv3_observe_path_owner(transport: LocalShell, path: path) { + PathOwnerObserved { owner: owner } => join(["path=", path as String, "|owner_group=", to_string(owner)], "") + PathOwnerUnobserved { reason: why } => join(["path=", path as String, "|owner_group_unobserved=", why], "") + } } fn fci1_modeled_ownership_wire(member: FabricCellMember) -> NonEmptyStr { @@ -609,6 +690,222 @@ func fci1_assert_allocation_restored(root: NonEmptyStr, before_wire: String, hel } } +type Fci1CanonicalCleanupDisposition + = NoCanonicalCommit { prestate: AllocationSlotPrestate, poststate: AllocationSlotPrestate } + | ExpectedCanonicalHoldingReleased { + prestate: AllocationSlotPrestate + held: RequiredBuildCellLifetimeSample + release_outcome: CellHoldEnd + poststate: AllocationSlotPrestate + restoration: AllocationStoreRestoration + } + | CanonicalAlreadyExactFree { + prestate: AllocationSlotPrestate + poststate: AllocationSlotPrestate + restoration: AllocationStoreRestoration + } + | CanonicalCleanupRefused { + prestate: AllocationSlotPrestate + observed: AllocationSlotPrestate + } + +fn fci1_slot_prestate_equal(a: AllocationSlotPrestate, b: AllocationSlotPrestate) -> Bool { + match a { + AllocationSlotAbsent { path: ap } => match b { + AllocationSlotAbsent { path: bp } => ap == bp + _ => false + } + AllocationSlotFree { path: ap, generation: ag } => match b { + AllocationSlotFree { path: bp, generation: bg } => ap == bp && ag == bg + _ => false + } + AllocationSlotHeld { path: ap, generation: ag } => match b { + AllocationSlotHeld { path: bp, generation: bg } => ap == bp && ag == bg + _ => false + } + AllocationSlotUnreadable { path: ap } => match b { + AllocationSlotUnreadable { path: bp } => ap == bp + _ => false + } + AllocationSlotUndecodable { path: ap, generation: ag } => match b { + AllocationSlotUndecodable { path: bp, generation: bg } => ap == bp && ag == bg + _ => false + } + } +} + +fn fci1_expected_held_generation(prestate: AllocationSlotPrestate) -> Int { + match prestate { + AllocationSlotAbsent { path: _ } => 1 + AllocationSlotFree { path: _, generation: g } => g + 1 + AllocationSlotHeld { path: _, generation: g } => g + AllocationSlotUnreadable { path: _ } => -1 + AllocationSlotUndecodable { path: _, generation: _ } => -1 + } +} + +fn fci1_synthetic_held_sample(root: NonEmptyStr) -> RequiredBuildCellLifetimeSampleDecode { + required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) +} + +func fci1_canonical_cleanup_disposition(root: NonEmptyStr, before_wire: String) -> Fci1CanonicalCleanupDisposition { + let before_decoded = decode_fabric_ci_evidence_wire(wire: before_wire) + let observed = fci1_allocation_prestate(root: root) + match before_decoded { + FabricCiEvidenceUndecodable => CanonicalCleanupRefused { prestate: AllocationSlotUnreadable { path: root }, observed: observed } + FabricCiEvidenceDecoded { value: before_value } => { + let prestate = fci1_decode_allocation_prestate(wire: before_value as String) + if fci1_slot_prestate_equal(a: prestate, b: observed) { + match prestate { + AllocationSlotAbsent { path: _ } => NoCanonicalCommit { prestate: prestate, poststate: observed } + AllocationSlotFree { path: _, generation: _ } => NoCanonicalCommit { prestate: prestate, poststate: observed } + AllocationSlotHeld { path: _, generation: _ } => CanonicalCleanupRefused { prestate: prestate, observed: observed } + AllocationSlotUnreadable { path: _ } => CanonicalCleanupRefused { prestate: prestate, observed: observed } + AllocationSlotUndecodable { path: _, generation: _ } => CanonicalCleanupRefused { prestate: prestate, observed: observed } + } + } else { match fci1_synthetic_held_sample(root: root) { + RequiredBuildCellLifetimeSampleDecoded { sample: held } => + if held.generation == fci1_expected_held_generation(prestate: prestate) { + let ended = fci1_live_release_outcome(root: root) + let post = fci1_allocation_prestate(root: root) + let restoration = fci1_exact_allocation_restoration( + prestate: prestate, + held_generation: held.generation, + poststate: post, + ) + match ended { + CellHoldEnded { slot_key: _, generation: _, account: _ } => match restoration { + AllocationSlotAbsentReturnedFree { path: _, held_generation: _, post_generation: _ } => ExpectedCanonicalHoldingReleased { + prestate: prestate, held: held, release_outcome: ended, poststate: post, restoration: restoration, + } + AllocationSlotFreeReturnedFree { path: _, pre_generation: _, held_generation: _, post_generation: _ } => ExpectedCanonicalHoldingReleased { + prestate: prestate, held: held, release_outcome: ended, poststate: post, restoration: restoration, + } + AllocationStoreRestorationRefused { path: _ } => CanonicalCleanupRefused { prestate: prestate, observed: post } + } + _ => CanonicalCleanupRefused { prestate: prestate, observed: post } + } + } else { CanonicalCleanupRefused { prestate: prestate, observed: observed } } + RequiredBuildCellLifetimeSampleUndecodable => { + let expected_held = fci1_expected_held_generation(prestate: prestate) + match observed { + AllocationSlotFree { path: _, generation: post_generation } => + if post_generation == expected_held + 1 { + CanonicalAlreadyExactFree { + prestate: prestate, + poststate: observed, + restoration: fci1_exact_allocation_restoration( + prestate: prestate, + held_generation: expected_held, + poststate: observed, + ), + } + } else { CanonicalCleanupRefused { prestate: prestate, observed: observed } } + _ => CanonicalCleanupRefused { prestate: prestate, observed: observed } + } + } + } } + } + } +} + +func fci1_write_canonical_cleanup_receipt(root: NonEmptyStr, before_wire: String, path: String) -> ProcessExit { + let disposition = fci1_canonical_cleanup_disposition(root: root, before_wire: before_wire) + let write = Filesystem.Write(path: path, content: to_string(disposition)) + if !write.success { exit_failure(reason: join(["canonical cleanup receipt write refused: ", write.error], "")) } + else { match disposition { + NoCanonicalCommit { prestate: _, poststate: _ } => ExitSuccess + ExpectedCanonicalHoldingReleased { prestate: _, held: _, release_outcome: _, poststate: _, restoration: _ } => ExitSuccess + CanonicalAlreadyExactFree { prestate: _, poststate: _, restoration: AllocationSlotAbsentReturnedFree { path: _, held_generation: _, post_generation: _ } } => ExitSuccess + CanonicalAlreadyExactFree { prestate: _, poststate: _, restoration: AllocationSlotFreeReturnedFree { path: _, pre_generation: _, held_generation: _, post_generation: _ } } => ExitSuccess + CanonicalAlreadyExactFree { prestate: _, poststate: _, restoration: AllocationStoreRestorationRefused { path: _ } } => exit_failure(reason: "canonical already-free state did not satisfy exact restoration") + CanonicalCleanupRefused { prestate: _, observed: _ } => exit_failure(reason: "canonical cleanup disposition refused") + } } +} + +func fci1_write_disposable_cleanup_receipt(root: NonEmptyStr, path: String) -> ProcessExit { + let observed = fci1_allocation_prestate(root: root) + match observed { + AllocationSlotAbsent { path: _ } => { + let write = Filesystem.Write(path: path, content: join(["DisposableNoCommit|observed=", to_string(observed)], "")) + if write.success { ExitSuccess } else { exit_failure(reason: "disposable no-commit receipt write refused") } + } + AllocationSlotHeld { path: _, generation: generation } => + if generation == 1 || generation == 2 { match fci1_synthetic_held_sample(root: root) { + RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "disposable held reservation identity refused") + RequiredBuildCellLifetimeSampleDecoded { sample: held } => if held.generation != generation { + exit_failure(reason: "disposable held observation generations disagree") + } else { + let ended = fci1_live_release_outcome(root: root) + let post = fci1_allocation_prestate(root: root) + let write = Filesystem.Write( + path: path, + content: join(["DisposableHoldingReleased|held=", to_string(held), "|release=", to_string(ended), "|post=", to_string(post)], ""), + ) + if !write.success { exit_failure(reason: "disposable cleanup receipt write refused") } + else { match ended { + CellHoldEnded { slot_key: _, generation: released_generation, account: _ } => match post { + AllocationSlotFree { path: _, generation: post_generation } => + if released_generation == generation + 1 && post_generation == generation + 1 { ExitSuccess } + else { exit_failure(reason: "disposable release did not reach exact next Free generation") } + _ => exit_failure(reason: "disposable release terminal state was not Free") + } + _ => exit_failure(reason: "disposable release refused") + } } + } + } } else { exit_failure(reason: "disposable held an unexpected generation") } + AllocationSlotFree { path: _, generation: generation } => + if generation == 2 || generation == 3 { + let write = Filesystem.Write(path: path, content: join(["DisposableAlreadyExactFree|observed=", to_string(observed)], "")) + if write.success { ExitSuccess } else { exit_failure(reason: "disposable already-Free receipt write refused") } + } else { exit_failure(reason: "disposable Free generation was unexpected") } + AllocationSlotUnreadable { path: _ } => exit_failure(reason: "disposable cleanup observation unreadable") + AllocationSlotUndecodable { path: _, generation: _ } => exit_failure(reason: "disposable cleanup observation undecodable") + } +} + +func fci1_grade_and_write_allocation_receipt( + root: NonEmptyStr, + before_wire: String, + held_wire: String, + phase: NonEmptyStr, + path: String, +) -> ProcessExit { + let before_decoded = decode_fabric_ci_evidence_wire(wire: before_wire) + let held_decoded = fci1_decode_transport_sample(wire: held_wire) + let post = fci1_allocation_prestate(root: root) + match before_decoded { + FabricCiEvidenceUndecodable => exit_failure(reason: "allocation receipt before observation undecodable") + FabricCiEvidenceDecoded { value: before_value } => match held_decoded { + RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "allocation receipt held observation undecodable") + RequiredBuildCellLifetimeSampleDecoded { sample: held } => { + let pre = fci1_decode_allocation_prestate(wire: before_value as String) + let restoration = fci1_exact_allocation_restoration( + prestate: pre, + held_generation: held.generation, + poststate: post, + ) + let content = join([ + "phase=", phase as String, + "\nroot=", root as String, + "\nslot_key=", held.slot_key as String, + "\npre=", to_string(pre), + "\nheld=", to_string(held), + "\npost=", to_string(post), + "\nrestoration=", to_string(restoration), + ], "") + let write = Filesystem.Write(path: path, content: content) + if !write.success { exit_failure(reason: join(["allocation receipt write refused: ", write.error], "")) } + else { match restoration { + AllocationSlotAbsentReturnedFree { path: _, held_generation: _, post_generation: _ } => ExitSuccess + AllocationSlotFreeReturnedFree { path: _, pre_generation: _, held_generation: _, post_generation: _ } => ExitSuccess + AllocationStoreRestorationRefused { path: _ } => exit_failure(reason: "allocation receipt exact restoration refused") + } } + } + } + } +} + // Pure controls run before host effects. Their rejecting counterparts are enrolled in the witness // corpus; the disposable-store mutation in the live driver exercises the production CAS mechanism. func fci1_assert_slot_absent_admitted() -> ProcessExit { @@ -728,9 +1025,14 @@ fn fci1_cell_observation_wire(decision: FabricCellObservationDecision) -> String let ownership = map(members, member => fabric_cell_member_ownership(member: member)) join([ "cell-admitted|host=", fabric_cell_observed_population_host(population: population) as String, - "|members=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: map(members, m => fci1_member_wire(member: m)))) as String, + "|cell=", fabric_cell_id_for_slot(slot: cp_probe_slot()) as String, + "|member_cardinality=", to_string(list_length(items: members)), + "|members=", join(map(members, m => fci1_member_wire(member: m) as String), ";"), "|boundaries=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: boundaries)) as String, "|modeled-teardown-ownership=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: map(members, m => fci1_modeled_ownership_wire(member: m)))) as String, + "|base-owner-group=", fci1_path_owner_wire(path: fabric_cell_base_dir as NonEmptyStr), + "|cell-owner-group=", fci1_path_owner_wire(path: fabric_cell_root_path(slot: cp_probe_slot()) as NonEmptyStr), + "|attempt-owner-group=", fci1_path_owner_wire(path: fabric_cell_attempt_root_path(slot: cp_probe_slot()) as NonEmptyStr), ], "") } FabricCellObservationDenied { host: host, blocked: blocked } => join([ @@ -799,6 +1101,102 @@ func fci1_assert_cell_unchanged(before_wire: String) -> ProcessExit { } } +func fci1_grade_and_write_cell_receipt(before_wire: String, phase: NonEmptyStr, path: String) -> ProcessExit { + let after = fci1_cell_observation_wire(decision: admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3"))) + match decode_fabric_ci_evidence_wire(wire: before_wire) { + FabricCiEvidenceUndecodable => exit_failure(reason: "persistent cell substrate before-wire undecodable") + FabricCiEvidenceDecoded { value: before } => { + let write = Filesystem.Write( + path: path, + content: join(["phase=", phase as String, "\nbefore=", before as String, "\nafter=", after], ""), + ) + if !write.success { exit_failure(reason: join(["cell receipt write refused: ", write.error], "")) } + else if after == (before as String) { ExitSuccess } + else { exit_failure(reason: join(["persistent cell substrate changed; before=", before as String, "; after=", after], "")) } + } + } +} + +type Fci1ZeroWorkObservation + = Fci1ZeroWorkObserved { + unit: NonEmptyStr + control_group: NonEmptyStr + child_cgroups: List + root_cgroup_procs: String + pids_current: String + attempts_entries: List + supervisor_units: List + } + | Fci1ZeroWorkRefused { cause: String } + +fn fci1_host_operation_value(operation: HostOperation) -> String? { + match host_operation_exec_local(operation: operation) { + HostOperationObserved { stdout: out, success: true, stderr: _ } => Present { value: trim(s: out) } + HostOperationObserved { stdout: _, success: false, stderr: _ } => Absent + HostOperationRefused { operation: _, reason: _ } => Absent + } +} + +fn fci1_zero_work_observation() -> Fci1ZeroWorkObservation { + let unit = "fabric-cell-srv3-06.slice" as NonEmptyStr + match systemctl_show_property_read(transport: LocalShell, unit: unit, property: ControlGroup) { + SystemdPropertyCaptureRefused { reason: why } => Fci1ZeroWorkRefused { cause: join(["ControlGroup observation refused: ", why], "") } + SystemdPropertyCaptured { value: cg } => if cg == "" { + Fci1ZeroWorkRefused { cause: "ControlGroup observation empty" } + } else { + let cgroup_path = concat(cgroup_v2_mount_point as String, trim(s: cg)) as NonEmptyStr + match fci1_host_operation_value(operation: CgroupPathIsDirectory { path: cgroup_path }) { + Absent => Fci1ZeroWorkRefused { cause: "cell ControlGroup path is not an observable directory" } + Present { value: _ } => match fci1_host_operation_value(operation: CgroupListChildren { cgroup_path: cgroup_path }) { + Absent => Fci1ZeroWorkRefused { cause: "cell child-cgroup enumeration refused" } + Present { value: child_text } => match fci1_host_operation_value( + operation: CgroupReadInterfaceFile { file_path: cgroup_v2_child_path(parent: cgroup_path, child: "cgroup.procs") }, + ) { + Absent => Fci1ZeroWorkRefused { cause: "cell root cgroup.procs read refused" } + Present { value: procs } => match fci1_host_operation_value( + operation: CgroupReadInterfaceFile { file_path: cgroup_v2_child_path(parent: cgroup_path, child: "pids.current") }, + ) { + Absent => Fci1ZeroWorkRefused { cause: "cell pids.current read refused" } + Present { value: pids } => { + let attempts = Filesystem.List(path: fabric_cell_attempt_root_path(slot: cp_probe_slot())) + if !attempts.success { Fci1ZeroWorkRefused { cause: join(["attempts enumeration refused: ", attempts.error], "") } } + else { match host_converge_slice1_enumerate_units(instance_glob: "fabric-cell@*.service") { + Absent => Fci1ZeroWorkRefused { cause: "fabric-cell@ service enumeration refused" } + Present { value: services } => Fci1ZeroWorkObserved { + unit: unit, + control_group: trim(s: cg) as NonEmptyStr, + child_cgroups: filter(split(s: child_text, delimiter: "\n"), x => x != ""), + root_cgroup_procs: procs, + pids_current: pids, + attempts_entries: attempts.entries, + supervisor_units: services, + } + } } + } + } + } + } + } + } + } +} + +func fci1_write_zero_work_receipt(phase: NonEmptyStr, path: String) -> ProcessExit { + let observation = fci1_zero_work_observation() + let write = Filesystem.Write(path: path, content: join(["phase=", phase as String, "\n", to_string(observation)], "")) + if !write.success { exit_failure(reason: join(["zero-Work receipt write refused: ", write.error], "")) } + else { match observation { + Fci1ZeroWorkRefused { cause: why } => exit_failure(reason: why) + Fci1ZeroWorkObserved { + unit: _, control_group: _, child_cgroups: children, root_cgroup_procs: procs, + pids_current: pids, attempts_entries: attempts, supervisor_units: services, + } => if list_length(items: children) == 0 && trim(s: procs) == "" && trim(s: pids) == "0" + && list_length(items: attempts) == 0 && list_length(items: services) == 0 { + ExitSuccess + } else { exit_failure(reason: "zero-Work population was nonzero") } + } } +} + fn cp_live_nothing_admissible_touches_no_store(root: NonEmptyStr) -> Int { cp_probe_code( @@ -986,8 +1384,8 @@ fn fci1_cell_hold_end_wire(outcome: CellHoldEnd) -> String { } } -func fci1_live_release(root: NonEmptyStr) -> ProcessExit { - let outcome = fold( +fn fci1_live_release_outcome(root: NonEmptyStr) -> CellHoldEnd { + fold( cp_probe_held_account(), init: HoldEndAttemptRefused { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), @@ -1009,6 +1407,10 @@ func fci1_live_release(root: NonEmptyStr) -> ProcessExit { ) }, ) +} + +func fci1_live_release(root: NonEmptyStr) -> ProcessExit { + let outcome = fci1_live_release_outcome(root: root) match outcome { CellHoldEnded { slot_key: slot, generation: _, account: _ } => if slot == "srv3-06" { ExitSuccess } else { @@ -1017,3 +1419,22 @@ func fci1_live_release(root: NonEmptyStr) -> ProcessExit { _ => exit_failure(reason: join(["reservation release refused: ", fci1_cell_hold_end_wire(outcome: outcome)], "")) } } + +func fci1_live_release_with_receipt(root: NonEmptyStr, path: String) -> ProcessExit { + let outcome = fci1_live_release_outcome(root: root) + let poststate = fci1_allocation_prestate(root: root) + let write = Filesystem.Write( + path: path, + content: join([ + "root=", root as String, + "\nrelease_outcome=", to_string(outcome), + "\npoststate=", to_string(poststate), + ], ""), + ) + if !write.success { exit_failure(reason: join(["release receipt write refused: ", write.error], "")) } + else { match outcome { + CellHoldEnded { slot_key: slot, generation: _, account: _ } => + if slot == "srv3-06" { ExitSuccess } else { exit_failure(reason: "release receipt named wrong slot") } + _ => exit_failure(reason: join(["reservation release refused: ", fci1_cell_hold_end_wire(outcome: outcome)], "")) + } } +} diff --git a/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag b/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag index 91700b0a011..031aaf4aeb5 100644 --- a/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag +++ b/dag/test/claim/fabric/fabric_ci_fci1_allocation_lifecycle_witness_test.dag @@ -6,6 +6,12 @@ import gunbc.fabric_required_build_cell { AllocationSlotAbsentReturnedFree, AllocationSlotFreeReturnedFree, fci1_allocation_slot_available, fci1_exact_allocation_restoration, } +import tools.fabric_control_plane_live_probe { + Fci1CheckpointAfterCanonicalCommitBeforeTransport, + Fci1CheckpointAfterReleaseBeforeGrading, + Fci1CheckpointGenerationHeldTwo, + fci1_checkpoint_token, +} test fn witness_fci1_absent_slot_is_available_but_held_slot_is_not() -> Bool { fci1_allocation_slot_available(prestate: AllocationSlotAbsent { path: "/fixture/store" }) @@ -61,3 +67,17 @@ test fn witness_fci1_wrong_held_generation_refuses() -> Bool { _ => false } } + +test fn witness_fci1_missing_transport_checkpoint_has_closed_identity() -> Bool { + fci1_checkpoint_token(checkpoint: Fci1CheckpointAfterCanonicalCommitBeforeTransport) + == "after-canonical-commit-before-transport" +} + +test fn witness_fci1_already_free_checkpoint_is_distinct_from_held() -> Bool { + fci1_checkpoint_token(checkpoint: Fci1CheckpointAfterReleaseBeforeGrading) + != fci1_checkpoint_token(checkpoint: Fci1CheckpointAfterCanonicalCommitBeforeTransport) +} + +test fn witness_fci1_disposable_replacement_checkpoint_is_separate() -> Bool { + fci1_checkpoint_token(checkpoint: Fci1CheckpointGenerationHeldTwo) == "generation-held-two" +} diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index 85c642f55fd..5fcaa701cbc 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -10,6 +10,9 @@ export FABRIC_CI_ENTRY="$repo_root/dag/gunbc/instruments/fabric_control_plane_li export FABRIC_CI_LOG FABRIC_CI_VALUE_ROOT FABRIC_CI_LOG=$(mktemp) FABRIC_CI_VALUE_ROOT=$(mktemp -d) +receipt_root= +checkpoint=${FCI1_CHECKPOINT-__empty__} +injected_failure_status=86 source "$repo_root/tools/fabric_ci_evidence_driver.sh" fabric_ci_driver_init @@ -18,11 +21,27 @@ mutation_runtime=fci1-submitter-owned-reservation mutation_root=/run/$mutation_runtime generation_root= subject="$repo_root/dag/gunbc/fabric/fabric_required_build_cell.dag" -canonical_reservation_may_be_held=0 -generation_reservation_may_be_held=0 allocation_before= positive_before= +receipt_path() { + printf '%s/%s.txt' "$receipt_root" "$1" +} + +checkpoint_if_selected() { + local phase=$1 + [[ $checkpoint == "$phase" ]] || return 0 + fabric_ci_run_assertion fci1_checkpoint_reached --arg token="$checkpoint" --arg phase="$phase" + echo "CheckpointReached: checkpoint=$checkpoint phase=$phase status=$injected_failure_status" | tee -a "$FABRIC_CI_LOG" >&2 + exit "$injected_failure_status" +} + +zero_work_receipt() { + local phase=$1 path + path=$(receipt_path "zero-work-$phase") + fabric_ci_run_assertion fci1_write_zero_work_receipt --arg phase="$phase" --arg path="$path" +} + unit_terminal_observed() { local unit=$1 output line load_state= active_state= if output=$(systemctl show "$unit" --property=LoadState --property=ActiveState 2>&1); then @@ -49,21 +68,26 @@ cleanup() { trap - EXIT set +e systemctl stop fci1-positive-submitter.service fci1-generation-submitter.service fci1-dependent-submitter.service >/dev/null 2>&1 || true - if (( canonical_reservation_may_be_held )); then - "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ - --entry "$FABRIC_CI_ENTRY" --function fci1_live_release --arg root="$canonical_root" >/dev/null 2>&1 + local cleanup_disposition_path + if [[ -n $allocation_before ]]; then + cleanup_disposition_path=$(receipt_path canonical-cleanup) "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ - --entry "$FABRIC_CI_ENTRY" --function fci1_assert_allocation_restored \ - --arg root="$canonical_root" --arg before_wire="$allocation_before" --arg held_wire="$positive_before" + --entry "$FABRIC_CI_ENTRY" --function fci1_write_canonical_cleanup_receipt \ + --arg root="$canonical_root" --arg before_wire="$allocation_before" --arg path="$cleanup_disposition_path" cleanup_status=$? fi - if (( generation_reservation_may_be_held )) && [[ -n $generation_root ]]; then + if [[ -n $generation_root ]]; then + local generation_disposition_path + generation_disposition_path=$(receipt_path generation-cleanup) "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ - --entry "$FABRIC_CI_ENTRY" --function fci1_live_release --arg root="$generation_root" >/dev/null 2>&1 + --entry "$FABRIC_CI_ENTRY" --function fci1_write_disposable_cleanup_receipt \ + --arg root="$generation_root" --arg path="$generation_disposition_path" local generation_cleanup_status=$? (( cleanup_status != 0 )) || cleanup_status=$generation_cleanup_status + if (( generation_cleanup_status == 0 )); then + rm -rf -- "$generation_root" + fi fi - [[ -z $generation_root ]] || rm -rf -- "$generation_root" if [[ -e $mutation_root || -L $mutation_root ]]; then echo 'InstrumentRefused: dependent RuntimeDirectory remains after cleanup' >&2 cleanup_status=1 @@ -77,6 +101,23 @@ cleanup() { cleanup_status=1 fi done + local terminal_zero_work + terminal_zero_work=$(receipt_path zero-work-terminal) + "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ + --entry "$FABRIC_CI_ENTRY" --function fci1_write_zero_work_receipt \ + --arg phase="terminal" --arg path="$terminal_zero_work" + local zero_work_status=$? + (( cleanup_status != 0 )) || cleanup_status=$zero_work_status + local head tree provenance manifest + head=$(git -C "$repo_root" rev-parse HEAD) || cleanup_status=1 + tree=$(git -C "$repo_root" rev-parse 'HEAD^{tree}') || cleanup_status=1 + provenance=$($FABRIC_CI_GUNBC_BIN --version 2>&1) || cleanup_status=1 + manifest="$receipt_root/manifest.txt" + { + printf 'head=%s\ntree=%s\nbinary_provenance=%s\ncheckpoint=%s\noriginal_exit_status=%s\ncleanup_verdict=%s\n' \ + "$head" "$tree" "$provenance" "$checkpoint" "$original_status" "$cleanup_status" + find "$receipt_root" -maxdepth 1 -type f ! -name manifest.txt -print0 | sort -z | xargs -0 -r sha256sum + } >"$manifest" || cleanup_status=1 rm -f "$FABRIC_CI_LOG" rm -rf "$FABRIC_CI_VALUE_ROOT" if (( cleanup_status != 0 )); then @@ -85,6 +126,12 @@ cleanup() { fi exit "$original_status" } +if ! fabric_ci_run_assertion fci1_assert_checkpoint_token --arg token="$checkpoint"; then + rm -f -- "$FABRIC_CI_LOG" + rm -rf -- "$FABRIC_CI_VALUE_ROOT" + exit 2 +fi +receipt_root=$(mktemp -d /tmp/fci1-receipt.XXXXXX) trap cleanup EXIT [[ ${EUID} -eq 0 ]] || { echo 'InstrumentRefused: must run as root on srv3' >&2; exit 2; } @@ -109,6 +156,8 @@ fabric_ci_run_assertion fci1_assert_cell_admitted cell_before=$(fabric_ci_capture_transport fci1_live_cell_observation cell-before) allocation_before=$(fabric_ci_capture_transport fci1_live_allocation_prestate allocation-before --arg root="$canonical_root") fabric_ci_run_assertion fci1_assert_allocation_available --arg root="$canonical_root" +zero_work_receipt before-canonical +checkpoint_if_selected before-canonical-commit capture_submitter_transport() { local unit=$1 function_name=$2 coordinate=$3 root=$4 @@ -124,6 +173,9 @@ capture_submitter_transport() { run_status=$? fi (( run_status == 0 )) || return "$run_status" + if [[ $unit == fci1-positive-submitter.service ]]; then + checkpoint_if_selected after-canonical-commit-before-transport + fi [[ -f $value_path && ! -L $value_path ]] || return 1 local value value=$(<"$value_path") @@ -131,29 +183,38 @@ capture_submitter_transport() { printf '%s' "$value" } -canonical_reservation_may_be_held=1 positive_before=$(capture_submitter_transport fci1-positive-submitter.service fci1_live_reserve_and_observe_available positive-before "$canonical_root") unit_terminal_observed fci1-positive-submitter.service || { echo 'InstrumentRefused: submitting control process terminal state unestablished' >&2 exit 1 } fabric_ci_run_assertion fci1_assert_lifetime_held --arg root="$canonical_root" --arg before_wire="$positive_before" +checkpoint_if_selected after-held-preserved -fabric_ci_run_assertion fci1_live_release --arg root="$canonical_root" -fabric_ci_run_assertion fci1_assert_allocation_restored --arg root="$canonical_root" --arg before_wire="$allocation_before" --arg held_wire="$positive_before" -canonical_reservation_may_be_held=0 -fabric_ci_run_assertion fci1_assert_cell_unchanged --arg before_wire="$cell_before" +release_receipt=$(receipt_path canonical-release) +fabric_ci_run_assertion fci1_live_release_with_receipt --arg root="$canonical_root" --arg path="$release_receipt" +checkpoint_if_selected after-release-before-grading +allocation_receipt=$(receipt_path canonical-allocation) +fabric_ci_run_assertion fci1_grade_and_write_allocation_receipt --arg root="$canonical_root" --arg before_wire="$allocation_before" --arg held_wire="$positive_before" --arg phase="canonical" --arg path="$allocation_receipt" +cell_receipt=$(receipt_path cell-after-canonical) +fabric_ci_run_assertion fci1_grade_and_write_cell_receipt --arg before_wire="$cell_before" --arg phase="after-canonical" --arg path="$cell_receipt" +zero_work_receipt after-canonical +checkpoint_if_selected canonical-restored-later # Generation replacement is a mutation falsifier, not part of the canonical lifecycle. Its # run-unique disposable store uses the production reservation/CAS entries and is removed afterward; # the canonical root sees only reserve, independent observation, and release. generation_root=$(mktemp -d /run/fci1-generation-mutation.XXXXXX) -generation_reservation_may_be_held=1 generation_before=$(capture_submitter_transport fci1-generation-submitter.service fci1_live_reserve_and_observe_available generation-before "$generation_root") +checkpoint_if_selected generation-held-one fabric_ci_run_assertion fci1_assert_replace_held --arg root="$generation_root" +checkpoint_if_selected generation-held-two fabric_ci_run_assertion fci1_assert_generation_changed --arg root="$generation_root" --arg before_wire="$generation_before" +checkpoint_if_selected generation-changed-observed fabric_ci_run_assertion fci1_live_release --arg root="$generation_root" -generation_reservation_may_be_held=0 +checkpoint_if_selected generation-free-before-removal +generation_terminal_receipt=$(receipt_path generation-terminal) +fabric_ci_run_assertion fci1_write_disposable_cleanup_receipt --arg root="$generation_root" --arg path="$generation_terminal_receipt" rm -rf -- "$generation_root" generation_root= @@ -162,6 +223,9 @@ dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service echo "runtime-directory-observed: path=$mutation_root standing=absent" >&2 unit_terminal_observed fci1-dependent-submitter.service || { echo 'InstrumentRefused: dependent submitter terminal state unestablished' >&2; exit 1; } fabric_ci_run_assertion fci1_assert_lifetime_dependent --arg root="$mutation_root" --arg before_wire="$dependent_before" -fabric_ci_run_assertion fci1_assert_cell_unchanged --arg before_wire="$cell_before" +checkpoint_if_selected runtime-directory-terminal +cell_receipt=$(receipt_path cell-after-dependent) +fabric_ci_run_assertion fci1_grade_and_write_cell_receipt --arg before_wire="$cell_before" --arg phase="after-dependent" --arg path="$cell_receipt" +zero_work_receipt after-dependent echo 'FCI1LiveAccepted' From c8eb7b25f90aea3c2d0bae1ba5fe4acbaa54aa11 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 16:33:04 +0000 Subject: [PATCH 08/50] Close FCI-1 wet failure paths --- .../fabric_control_plane_live_probe.dag | 473 +++++++++++++++--- tools/fabric_ci_fci1_live_instrument.sh | 84 +++- 2 files changed, 457 insertions(+), 100 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 3faabc4e3bf..91598377eda 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -2,7 +2,7 @@ module tools.fabric_control_plane_live_probe import std.types { Bool, Int, List, NonEmptyStr, String, GitRef } import std.algebra { trim } -import std.process { ProcessExit, ExitSuccess, exit_failure } +import std.process { ProcessExit, ExitSuccess, ExitFailure, exit_failure } import extdeps.currency.currency { Usd } import std.measure { MoneyAmountMicro, MoneyOnce, MoneyPerSecond, money_amount_micro, @@ -70,7 +70,11 @@ import gunbc.fabric_required_build_cell { cell_member_shape_complete, DirectoryModeStanding, DirectoryModeUnobserved, fci1_directory_mode_unobserved, } -import gunbc.fabric_cell_acquire { fabric_cell_probe_wet } +import gunbc.fabric_cell_acquire { + fabric_cell_probe_wet, + FabricCellBoundaryReading, BoundaryValuesRead, BoundaryReadRefused, + fabric_cell_read_boundary, +} import gunbc.fabric_cell_observation_admission { FabricCellObservationDecision, FabricCellObservationAdmitted, FabricCellObservationDenied, FabricCellObservationNotApplicable, fabric_cell_observation_outcome, @@ -114,7 +118,7 @@ import gunbc.host_operation_exec { CgroupListChildren, CgroupReadInterfaceFile, CgroupPathIsDirectory, host_operation_exec_local, } -import gunbc.host_converge_slice1 { host_converge_slice1_enumerate_units } +import gunbc.systemctl_list_units { systemctl_list_units_all_states } type Fci1Checkpoint = Fci1CheckpointNone @@ -167,12 +171,22 @@ func fci1_assert_checkpoint_token(token: NonEmptyStr) -> ProcessExit { } else { exit_failure(reason: join(["unknown or duplicate FCI-1 checkpoint token: ", token as String], "")) } } -func fci1_checkpoint_reached(token: NonEmptyStr, phase: NonEmptyStr) -> ProcessExit { +func fci1_checkpoint_reached(token: NonEmptyStr, phase: NonEmptyStr, path: String) -> ProcessExit { if token == phase && list_length(items: filter(fci1_checkpoints(), c => fci1_checkpoint_token(checkpoint: c) == token)) == 1 { - ExitSuccess + let write = Filesystem.Write(path: path, content: join([ + "CheckpointReached|checkpoint=", token as String, "|phase=", phase as String, "|status=86", + ], "")) + if write.success { ExitSuccess } else { exit_failure(reason: "checkpoint receipt write refused") } } else { exit_failure(reason: join(["phase-inapplicable FCI-1 checkpoint: token=", token as String, " phase=", phase as String], "")) } } +func fci1_write_checkpoint_terminal_receipt(token: NonEmptyStr, path: String) -> ProcessExit { + if list_length(items: filter(fci1_checkpoints(), c => fci1_checkpoint_token(checkpoint: c) == token)) == 1 { + let write = Filesystem.Write(path: path, content: join(["CheckpointNotReached|selected=", token as String], "")) + if write.success { ExitSuccess } else { exit_failure(reason: "checkpoint terminal receipt write refused") } + } else { exit_failure(reason: "checkpoint terminal receipt token refused") } +} + // The live receipt for the EFFECTFUL half of the broker, which the hermetic witness cannot // reach. An entry point rather than a shell script, for the reason DESIGN gives: if a // measurement is worth re-deriving it is worth an entry point. @@ -575,17 +589,104 @@ func fci1_live_binding_observation(root: NonEmptyStr) -> RequiredBuildCellObserv } } -func fci1_live_reserve_and_observe_available(root: NonEmptyStr, path: String) -> ProcessExit { +type Fci1ReservationCleanupAuthority sole_constructor { + root: NonEmptyStr + slot_key: NonEmptyStr + generation: Int + reservation_identity: NonEmptyStr + reservation_payload: NonEmptyStr + reservation_content: NonEmptyStr +} + +type Fci1ReservationCleanupAuthorityDecode + = Fci1ReservationCleanupAuthorityDecoded { authority: Fci1ReservationCleanupAuthority } + | Fci1ReservationCleanupAuthorityUndecodable + +fn fci1_reservation_cleanup_authority_wire(authority: Fci1ReservationCleanupAuthority) -> NonEmptyStr { + join([ + authority.root as String, + "\n", authority.slot_key as String, + "\n", to_string(authority.generation), + "\n", authority.reservation_identity as String, + "\n", authority.reservation_payload as String, + "\n", authority.reservation_content as String, + ], "") as NonEmptyStr +} + +fn fci1_decode_reservation_cleanup_authority(wire: String) -> Fci1ReservationCleanupAuthorityDecode { + let fields = split(s: wire, delimiter: "\n") + if list_length(items: fields) != 6 { Fci1ReservationCleanupAuthorityUndecodable } + else { match fields.first() { + Absent => Fci1ReservationCleanupAuthorityUndecodable + Present { value: root } => match fields.skip(n: 1).first() { + Absent => Fci1ReservationCleanupAuthorityUndecodable + Present { value: slot } => match fields.skip(n: 2).first() { + Absent => Fci1ReservationCleanupAuthorityUndecodable + Present { value: generation_text } => match parse_int(s: generation_text) { + Absent => Fci1ReservationCleanupAuthorityUndecodable + Present { value: generation } => match fields.skip(n: 3).first() { + Absent => Fci1ReservationCleanupAuthorityUndecodable + Present { value: reservation } => match fields.skip(n: 4).first() { + Absent => Fci1ReservationCleanupAuthorityUndecodable + Present { value: payload } => match fields.skip(n: 5).first() { + Absent => Fci1ReservationCleanupAuthorityUndecodable + Present { value: content } => if root == "" || slot == "" || reservation == "" || payload == "" || content == "" { + Fci1ReservationCleanupAuthorityUndecodable + } else { Fci1ReservationCleanupAuthorityDecoded { authority: Fci1ReservationCleanupAuthority { + root: root as NonEmptyStr, + slot_key: slot as NonEmptyStr, + generation: generation, + reservation_identity: reservation as NonEmptyStr, + reservation_payload: payload as NonEmptyStr, + reservation_content: content as NonEmptyStr, + } } } + } + } + } + } + } + } + } } +} + +func fci1_live_reserve_and_observe_available( + root: NonEmptyStr, + path: String, + cleanup_path: String, + checkpoint: NonEmptyStr, + checkpoint_path: String, +) -> ProcessExit { let reserved = fci1_live_reserve_available(root: root) match reserved { - CellReserved { offer: _, slot_key: _, generation: _, account: _ } => - match required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) { + CellReserved { offer: _, slot_key: slot, generation: generation, account: _ } => { + let reservation = cp_probe_reservation_ref() + let payload = join(["held|", reservation as String], "") as NonEmptyStr + let authority = Fci1ReservationCleanupAuthority { + root: root, + slot_key: slot, + generation: generation, + reservation_identity: reservation as NonEmptyStr, + reservation_payload: payload, + reservation_content: serialize_content_hash(hash: content_hash_of_value(value: payload)), + } + let authority_write = Filesystem.Write( + path: cleanup_path, + content: fci1_reservation_cleanup_authority_wire(authority: authority) as String, + ) + if !authority_write.success { exit_failure(reason: "ReservationCleanupAuthorityWriteRefused") } + else if checkpoint == "after-canonical-commit-before-transport" { + match fci1_checkpoint_reached(token: checkpoint, phase: checkpoint, path: checkpoint_path) { + ExitSuccess => ExitFailure { code: 86, reason: "FCI1InjectedFailureAfterCommitBeforeTransport" } + ExitFailure { code: _, reason: why } => exit_failure(reason: why) + } + } else { match required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) { RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "RequiredBuildCellObservationRefused") RequiredBuildCellLifetimeSampleDecoded { sample: sample } => fabric_ci_write_wire( path: path, wire: fabric_ci_evidence_wire(value: required_build_cell_lifetime_sample_wire(sample: sample) as NonEmptyStr), ) - } + } } + } _ => exit_failure(reason: join(["ReservationCommitRefused:", fci1_cell_reservation_wire(reservation: reserved)], "")) } } @@ -613,6 +714,61 @@ fn fci1_allocation_prestate(root: NonEmptyStr) -> AllocationSlotPrestate { } } +// Cleanup and retained evidence need the identities that AllocationSlotPrestate intentionally +// omits. This carrier is read directly from the CAS slot; no cell-substrate join participates. +type Fci1AllocationSlotObservation + = Fci1SlotAbsent { path: NonEmptyStr } + | Fci1SlotFree { + path: NonEmptyStr + generation: Int + released_by: NonEmptyStr + payload: NonEmptyStr + content: NonEmptyStr + } + | Fci1SlotHeld { + path: NonEmptyStr + generation: Int + reservation_identity: NonEmptyStr + payload: NonEmptyStr + content: NonEmptyStr + } + | Fci1SlotUnreadable { path: NonEmptyStr } + | Fci1SlotUndecodable { path: NonEmptyStr, generation: Int, payload: NonEmptyStr, content: NonEmptyStr } + +fn fci1_allocation_slot_observation(root: NonEmptyStr) -> Fci1AllocationSlotObservation { + match observe_cas_slot_state(root: root, key: fabric_cell_allocation_key(slot: cp_probe_slot())) { + CasObservedUnreadable { cause: _ } => Fci1SlotUnreadable { path: root } + CasObservedReadable { readable: CasReadableAbsent } => Fci1SlotAbsent { path: root } + CasObservedReadable { readable: CasReadablePresent { version: version } } => { + let content = serialize_content_hash(hash: version.content) + match decode_cell_slot_payload(payload: version.value) { + CellSlotDecoded { state: CellFree { released_by: released } } => Fci1SlotFree { + path: root, generation: version.generation, released_by: released, + payload: version.value, content: content, + } + CellSlotDecoded { state: CellHeld { reservation: reservation } } => Fci1SlotHeld { + path: root, generation: version.generation, + reservation_identity: (reservation as String) as NonEmptyStr, + payload: version.value, content: content, + } + CellSlotUndecodable { detail: _ } => Fci1SlotUndecodable { + path: root, generation: version.generation, payload: version.value, content: content, + } + } + } + } +} + +fn fci1_slot_observation_prestate(observed: Fci1AllocationSlotObservation) -> AllocationSlotPrestate { + match observed { + Fci1SlotAbsent { path: p } => AllocationSlotAbsent { path: p } + Fci1SlotFree { path: p, generation: g, released_by: _, payload: _, content: _ } => AllocationSlotFree { path: p, generation: g } + Fci1SlotHeld { path: p, generation: g, reservation_identity: _, payload: _, content: _ } => AllocationSlotHeld { path: p, generation: g } + Fci1SlotUnreadable { path: p } => AllocationSlotUnreadable { path: p } + Fci1SlotUndecodable { path: p, generation: g, payload: _, content: _ } => AllocationSlotUndecodable { path: p, generation: g } + } +} + fn fci1_allocation_prestate_wire(prestate: AllocationSlotPrestate) -> NonEmptyStr { match prestate { AllocationSlotAbsent { path: p } => join(["slot-absent\n", p as String], "") as NonEmptyStr @@ -691,22 +847,24 @@ func fci1_assert_allocation_restored(root: NonEmptyStr, before_wire: String, hel } type Fci1CanonicalCleanupDisposition - = NoCanonicalCommit { prestate: AllocationSlotPrestate, poststate: AllocationSlotPrestate } + = NoCanonicalCommit { prestate: AllocationSlotPrestate, poststate: Fci1AllocationSlotObservation } | ExpectedCanonicalHoldingReleased { prestate: AllocationSlotPrestate - held: RequiredBuildCellLifetimeSample + authority: Fci1ReservationCleanupAuthority + held: Fci1AllocationSlotObservation release_outcome: CellHoldEnd - poststate: AllocationSlotPrestate + poststate: Fci1AllocationSlotObservation restoration: AllocationStoreRestoration } | CanonicalAlreadyExactFree { prestate: AllocationSlotPrestate - poststate: AllocationSlotPrestate + authority: Fci1ReservationCleanupAuthority + poststate: Fci1AllocationSlotObservation restoration: AllocationStoreRestoration } | CanonicalCleanupRefused { prestate: AllocationSlotPrestate - observed: AllocationSlotPrestate + observed: Fci1AllocationSlotObservation } fn fci1_slot_prestate_equal(a: AllocationSlotPrestate, b: AllocationSlotPrestate) -> Bool { @@ -744,18 +902,58 @@ fn fci1_expected_held_generation(prestate: AllocationSlotPrestate) -> Int { } } -fn fci1_synthetic_held_sample(root: NonEmptyStr) -> RequiredBuildCellLifetimeSampleDecode { - required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) +fn fci1_cleanup_authority_from_path(path: String) -> Fci1ReservationCleanupAuthorityDecode { + let read = Filesystem.Read(path: path) + if read.success { fci1_decode_reservation_cleanup_authority(wire: read.content) } + else { Fci1ReservationCleanupAuthorityUndecodable } +} + +fn fci1_authority_matches_held( + authority: Fci1ReservationCleanupAuthority, + observed: Fci1AllocationSlotObservation, +) -> Bool { + match observed { + Fci1SlotHeld { + path: path, generation: generation, reservation_identity: reservation, + payload: payload, content: content, + } => authority.root == path && authority.slot_key == fabric_cell_allocation_key(slot: cp_probe_slot()) + && authority.generation == generation && authority.reservation_identity == reservation + && authority.reservation_payload == payload && authority.reservation_content == content + Fci1SlotAbsent { path: _ } => false + Fci1SlotFree { path: _, generation: _, released_by: _, payload: _, content: _ } => false + Fci1SlotUnreadable { path: _ } => false + Fci1SlotUndecodable { path: _, generation: _, payload: _, content: _ } => false + } +} + +fn fci1_authority_matches_exact_free( + authority: Fci1ReservationCleanupAuthority, + observed: Fci1AllocationSlotObservation, +) -> Bool { + match observed { + Fci1SlotFree { path: path, generation: generation, released_by: released, payload: _, content: _ } => + authority.root == path && authority.slot_key == fabric_cell_allocation_key(slot: cp_probe_slot()) + && generation == authority.generation + 1 && released == fci1_release_identity + Fci1SlotAbsent { path: _ } => false + Fci1SlotHeld { path: _, generation: _, reservation_identity: _, payload: _, content: _ } => false + Fci1SlotUnreadable { path: _ } => false + Fci1SlotUndecodable { path: _, generation: _, payload: _, content: _ } => false + } } -func fci1_canonical_cleanup_disposition(root: NonEmptyStr, before_wire: String) -> Fci1CanonicalCleanupDisposition { +func fci1_canonical_cleanup_disposition( + root: NonEmptyStr, + before_wire: String, + authority_path: String, +) -> Fci1CanonicalCleanupDisposition { let before_decoded = decode_fabric_ci_evidence_wire(wire: before_wire) - let observed = fci1_allocation_prestate(root: root) + let observed = fci1_allocation_slot_observation(root: root) + let observed_prestate = fci1_slot_observation_prestate(observed: observed) match before_decoded { FabricCiEvidenceUndecodable => CanonicalCleanupRefused { prestate: AllocationSlotUnreadable { path: root }, observed: observed } FabricCiEvidenceDecoded { value: before_value } => { let prestate = fci1_decode_allocation_prestate(wire: before_value as String) - if fci1_slot_prestate_equal(a: prestate, b: observed) { + if fci1_slot_prestate_equal(a: prestate, b: observed_prestate) { match prestate { AllocationSlotAbsent { path: _ } => NoCanonicalCommit { prestate: prestate, poststate: observed } AllocationSlotFree { path: _, generation: _ } => NoCanonicalCommit { prestate: prestate, poststate: observed } @@ -763,90 +961,97 @@ func fci1_canonical_cleanup_disposition(root: NonEmptyStr, before_wire: String) AllocationSlotUnreadable { path: _ } => CanonicalCleanupRefused { prestate: prestate, observed: observed } AllocationSlotUndecodable { path: _, generation: _ } => CanonicalCleanupRefused { prestate: prestate, observed: observed } } - } else { match fci1_synthetic_held_sample(root: root) { - RequiredBuildCellLifetimeSampleDecoded { sample: held } => - if held.generation == fci1_expected_held_generation(prestate: prestate) { + } else { match fci1_cleanup_authority_from_path(path: authority_path) { + Fci1ReservationCleanupAuthorityDecoded { authority: authority } => + if authority.generation == fci1_expected_held_generation(prestate: prestate) + && fci1_authority_matches_held(authority: authority, observed: observed) { let ended = fci1_live_release_outcome(root: root) - let post = fci1_allocation_prestate(root: root) + let post = fci1_allocation_slot_observation(root: root) let restoration = fci1_exact_allocation_restoration( prestate: prestate, - held_generation: held.generation, - poststate: post, + held_generation: authority.generation, + poststate: fci1_slot_observation_prestate(observed: post), ) match ended { - CellHoldEnded { slot_key: _, generation: _, account: _ } => match restoration { + CellHoldEnded { slot_key: _, generation: _, account: _ } => if !fci1_authority_matches_exact_free(authority: authority, observed: post) { + CanonicalCleanupRefused { prestate: prestate, observed: post } + } else { match restoration { AllocationSlotAbsentReturnedFree { path: _, held_generation: _, post_generation: _ } => ExpectedCanonicalHoldingReleased { - prestate: prestate, held: held, release_outcome: ended, poststate: post, restoration: restoration, + prestate: prestate, authority: authority, held: observed, release_outcome: ended, poststate: post, restoration: restoration, } AllocationSlotFreeReturnedFree { path: _, pre_generation: _, held_generation: _, post_generation: _ } => ExpectedCanonicalHoldingReleased { - prestate: prestate, held: held, release_outcome: ended, poststate: post, restoration: restoration, + prestate: prestate, authority: authority, held: observed, release_outcome: ended, poststate: post, restoration: restoration, } AllocationStoreRestorationRefused { path: _ } => CanonicalCleanupRefused { prestate: prestate, observed: post } - } + } } _ => CanonicalCleanupRefused { prestate: prestate, observed: post } } } else { CanonicalCleanupRefused { prestate: prestate, observed: observed } } - RequiredBuildCellLifetimeSampleUndecodable => { - let expected_held = fci1_expected_held_generation(prestate: prestate) - match observed { - AllocationSlotFree { path: _, generation: post_generation } => - if post_generation == expected_held + 1 { + Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: prestate, observed: observed } + } |> fn(disposition) { match disposition { + CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } => + match fci1_cleanup_authority_from_path(path: authority_path) { + Fci1ReservationCleanupAuthorityDecoded { authority: authority } => + if authority.generation == fci1_expected_held_generation(prestate: prestate) + && fci1_authority_matches_exact_free(authority: authority, observed: observed) { CanonicalAlreadyExactFree { prestate: prestate, + authority: authority, poststate: observed, restoration: fci1_exact_allocation_restoration( prestate: prestate, - held_generation: expected_held, - poststate: observed, + held_generation: authority.generation, + poststate: observed_prestate, ), } - } else { CanonicalCleanupRefused { prestate: prestate, observed: observed } } - _ => CanonicalCleanupRefused { prestate: prestate, observed: observed } + } else { CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } } + Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } } - } + _ => disposition } } } } } -func fci1_write_canonical_cleanup_receipt(root: NonEmptyStr, before_wire: String, path: String) -> ProcessExit { - let disposition = fci1_canonical_cleanup_disposition(root: root, before_wire: before_wire) +func fci1_write_canonical_cleanup_receipt(root: NonEmptyStr, before_wire: String, authority_path: String, path: String) -> ProcessExit { + let disposition = fci1_canonical_cleanup_disposition(root: root, before_wire: before_wire, authority_path: authority_path) let write = Filesystem.Write(path: path, content: to_string(disposition)) if !write.success { exit_failure(reason: join(["canonical cleanup receipt write refused: ", write.error], "")) } else { match disposition { NoCanonicalCommit { prestate: _, poststate: _ } => ExitSuccess - ExpectedCanonicalHoldingReleased { prestate: _, held: _, release_outcome: _, poststate: _, restoration: _ } => ExitSuccess - CanonicalAlreadyExactFree { prestate: _, poststate: _, restoration: AllocationSlotAbsentReturnedFree { path: _, held_generation: _, post_generation: _ } } => ExitSuccess - CanonicalAlreadyExactFree { prestate: _, poststate: _, restoration: AllocationSlotFreeReturnedFree { path: _, pre_generation: _, held_generation: _, post_generation: _ } } => ExitSuccess - CanonicalAlreadyExactFree { prestate: _, poststate: _, restoration: AllocationStoreRestorationRefused { path: _ } } => exit_failure(reason: "canonical already-free state did not satisfy exact restoration") + ExpectedCanonicalHoldingReleased { prestate: _, authority: _, held: _, release_outcome: _, poststate: _, restoration: _ } => ExitSuccess + CanonicalAlreadyExactFree { prestate: _, authority: _, poststate: _, restoration: AllocationSlotAbsentReturnedFree { path: _, held_generation: _, post_generation: _ } } => ExitSuccess + CanonicalAlreadyExactFree { prestate: _, authority: _, poststate: _, restoration: AllocationSlotFreeReturnedFree { path: _, pre_generation: _, held_generation: _, post_generation: _ } } => ExitSuccess + CanonicalAlreadyExactFree { prestate: _, authority: _, poststate: _, restoration: AllocationStoreRestorationRefused { path: _ } } => exit_failure(reason: "canonical already-free state did not satisfy exact restoration") CanonicalCleanupRefused { prestate: _, observed: _ } => exit_failure(reason: "canonical cleanup disposition refused") } } } -func fci1_write_disposable_cleanup_receipt(root: NonEmptyStr, path: String) -> ProcessExit { - let observed = fci1_allocation_prestate(root: root) +func fci1_write_disposable_cleanup_receipt(root: NonEmptyStr, authority_path: String, path: String) -> ProcessExit { + let observed = fci1_allocation_slot_observation(root: root) match observed { - AllocationSlotAbsent { path: _ } => { + Fci1SlotAbsent { path: _ } => { let write = Filesystem.Write(path: path, content: join(["DisposableNoCommit|observed=", to_string(observed)], "")) if write.success { ExitSuccess } else { exit_failure(reason: "disposable no-commit receipt write refused") } } - AllocationSlotHeld { path: _, generation: generation } => - if generation == 1 || generation == 2 { match fci1_synthetic_held_sample(root: root) { - RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "disposable held reservation identity refused") - RequiredBuildCellLifetimeSampleDecoded { sample: held } => if held.generation != generation { - exit_failure(reason: "disposable held observation generations disagree") + Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => + if generation == 1 || generation == 2 { match fci1_cleanup_authority_from_path(path: authority_path) { + Fci1ReservationCleanupAuthorityUndecodable => exit_failure(reason: "disposable cleanup authority undecodable") + Fci1ReservationCleanupAuthorityDecoded { authority: authority } => if !fci1_authority_matches_held(authority: authority, observed: observed) { + exit_failure(reason: "disposable held reservation identity refused") } else { let ended = fci1_live_release_outcome(root: root) - let post = fci1_allocation_prestate(root: root) + let post = fci1_allocation_slot_observation(root: root) let write = Filesystem.Write( path: path, - content: join(["DisposableHoldingReleased|held=", to_string(held), "|release=", to_string(ended), "|post=", to_string(post)], ""), + content: join(["DisposableHoldingReleased|authority=", to_string(authority), "|held=", to_string(observed), "|release=", to_string(ended), "|post=", to_string(post)], ""), ) if !write.success { exit_failure(reason: "disposable cleanup receipt write refused") } else { match ended { CellHoldEnded { slot_key: _, generation: released_generation, account: _ } => match post { - AllocationSlotFree { path: _, generation: post_generation } => - if released_generation == generation + 1 && post_generation == generation + 1 { ExitSuccess } + Fci1SlotFree { path: _, generation: post_generation, released_by: _, payload: _, content: _ } => + if released_generation == generation + 1 && post_generation == generation + 1 + && fci1_authority_matches_exact_free(authority: authority, observed: post) { ExitSuccess } else { exit_failure(reason: "disposable release did not reach exact next Free generation") } _ => exit_failure(reason: "disposable release terminal state was not Free") } @@ -854,13 +1059,17 @@ func fci1_write_disposable_cleanup_receipt(root: NonEmptyStr, path: String) -> P } } } } } else { exit_failure(reason: "disposable held an unexpected generation") } - AllocationSlotFree { path: _, generation: generation } => - if generation == 2 || generation == 3 { - let write = Filesystem.Write(path: path, content: join(["DisposableAlreadyExactFree|observed=", to_string(observed)], "")) - if write.success { ExitSuccess } else { exit_failure(reason: "disposable already-Free receipt write refused") } - } else { exit_failure(reason: "disposable Free generation was unexpected") } - AllocationSlotUnreadable { path: _ } => exit_failure(reason: "disposable cleanup observation unreadable") - AllocationSlotUndecodable { path: _, generation: _ } => exit_failure(reason: "disposable cleanup observation undecodable") + Fci1SlotFree { path: _, generation: generation, released_by: _, payload: _, content: _ } => + match fci1_cleanup_authority_from_path(path: authority_path) { + Fci1ReservationCleanupAuthorityUndecodable => exit_failure(reason: "disposable already-Free authority undecodable") + Fci1ReservationCleanupAuthorityDecoded { authority: authority } => + if (generation == 2 || generation == 3) && fci1_authority_matches_exact_free(authority: authority, observed: observed) { + let write = Filesystem.Write(path: path, content: join(["DisposableAlreadyExactFree|authority=", to_string(authority), "|observed=", to_string(observed)], "")) + if write.success { ExitSuccess } else { exit_failure(reason: "disposable already-Free receipt write refused") } + } else { exit_failure(reason: "disposable Free identity or generation was unexpected") } + } + Fci1SlotUnreadable { path: _ } => exit_failure(reason: "disposable cleanup observation unreadable") + Fci1SlotUndecodable { path: _, generation: _, payload: _, content: _ } => exit_failure(reason: "disposable cleanup observation undecodable") } } @@ -1003,6 +1212,21 @@ func fci1_assert_generation_changed(root: NonEmptyStr, before_wire: String) -> P } } +func fci1_write_generation_changed_receipt(root: NonEmptyStr, before_wire: String, path: String) -> ProcessExit { + let verdict = fci1_live_lifetime_verdict(root: root, before_wire: before_wire) + let write = Filesystem.Write(path: path, content: join([ + "before_wire=", before_wire, "\nverdict=", to_string(verdict), + "\nslot_observation=", to_string(fci1_allocation_slot_observation(root: root)), + ], "")) + if !write.success { exit_failure(reason: "generation-changed receipt write refused") } + else { match verdict { + ReservationGenerationChanged { expected_generation: expected, observed_generation: observed } => + if expected == 1 && observed == 2 { ExitSuccess } + else { exit_failure(reason: "generation-changed receipt carried unexpected coordinates") } + _ => exit_failure(reason: "generation-changed receipt did not carry typed verdict") + } } +} + func fci1_assert_lifetime_dependent(root: NonEmptyStr, before_wire: String) -> ProcessExit { let verdict = fci1_live_lifetime_verdict(root: root, before_wire: before_wire) match verdict { @@ -1014,6 +1238,20 @@ func fci1_assert_lifetime_dependent(root: NonEmptyStr, before_wire: String) -> P } } +func fci1_write_lifetime_dependent_receipt(root: NonEmptyStr, before_wire: String, path: String) -> ProcessExit { + let verdict = fci1_live_lifetime_verdict(root: root, before_wire: before_wire) + let write = Filesystem.Write(path: path, content: join([ + "before_wire=", before_wire, "\nverdict=", to_string(verdict), + "\nslot_observation=", to_string(fci1_allocation_slot_observation(root: root)), + ], "")) + if !write.success { exit_failure(reason: "lifetime-dependent receipt write refused") } + else { match verdict { + ReservationLifetimeDependentOnSubmitter { slot_key: slot } => + if slot == "srv3-06" { ExitSuccess } else { exit_failure(reason: "lifetime-dependent receipt named wrong slot") } + _ => exit_failure(reason: "lifetime-dependent receipt did not carry typed verdict") + } } +} + fn fci1_cell_observation_wire(decision: FabricCellObservationDecision) -> String { match fabric_cell_observation_outcome(decision: decision) { FabricCellObservationAdmitted { population: population } => { @@ -1043,13 +1281,67 @@ fn fci1_cell_observation_wire(decision: FabricCellObservationDecision) -> String } } +type Fci1CellReceiptObservation + = Fci1CellReceiptAdmitted { + wire: NonEmptyStr + memory_max: NonEmptyStr + memory_high: NonEmptyStr + memory_swap_max: NonEmptyStr + tasks_max: NonEmptyStr + cpu_weight: NonEmptyStr + base_owner_group: PosixOwnerSpec + cell_owner_group: PosixOwnerSpec + attempt_owner_group: PosixOwnerSpec + } + | Fci1CellReceiptRefused { detail: String } + +fn fci1_cell_receipt_observation() -> Fci1CellReceiptObservation { + let decision = admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3")) + match fabric_cell_observation_outcome(decision: decision) { + FabricCellObservationDenied { host: _, blocked: _ } => Fci1CellReceiptRefused { detail: fci1_cell_observation_wire(decision: decision) } + FabricCellObservationNotApplicable { host: _ } => Fci1CellReceiptRefused { detail: fci1_cell_observation_wire(decision: decision) } + FabricCellObservationAdmitted { population: _ } => + match srv3_observe_path_owner(transport: LocalShell, path: fabric_cell_base_dir as NonEmptyStr) { + PathOwnerUnobserved { reason: why } => Fci1CellReceiptRefused { detail: why } + PathOwnerObserved { owner: base_owner } => match srv3_observe_path_owner( + transport: LocalShell, path: fabric_cell_root_path(slot: cp_probe_slot()) as NonEmptyStr, + ) { + PathOwnerUnobserved { reason: why } => Fci1CellReceiptRefused { detail: why } + PathOwnerObserved { owner: cell_owner } => match srv3_observe_path_owner( + transport: LocalShell, path: fabric_cell_attempt_root_path(slot: cp_probe_slot()) as NonEmptyStr, + ) { + PathOwnerUnobserved { reason: why } => Fci1CellReceiptRefused { detail: why } + PathOwnerObserved { owner: attempt_owner } => match fabric_cell_read_boundary(slot: cp_probe_slot()) { + BoundaryReadRefused { cause: why } => Fci1CellReceiptRefused { detail: why } + BoundaryValuesRead { + memory_max: memory_max, memory_high: memory_high, memory_swap_max: memory_swap_max, + tasks_max: tasks_max, cpu_weight: cpu_weight, + } => Fci1CellReceiptAdmitted { + wire: fci1_cell_observation_wire(decision: decision) as NonEmptyStr, + memory_max: memory_max as NonEmptyStr, + memory_high: memory_high as NonEmptyStr, + memory_swap_max: memory_swap_max as NonEmptyStr, + tasks_max: tasks_max as NonEmptyStr, + cpu_weight: cpu_weight as NonEmptyStr, + base_owner_group: base_owner, + cell_owner_group: cell_owner, + attempt_owner_group: attempt_owner, + } + } + } + } + } + } +} + func fci1_live_cell_observation(path: String) -> ProcessExit { - fabric_ci_write_wire( - path: path, - wire: fabric_ci_evidence_wire( - value: fci1_cell_observation_wire(decision: admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3"))) as NonEmptyStr, - ), - ) + match fci1_cell_receipt_observation() { + Fci1CellReceiptRefused { detail: why } => exit_failure(reason: join(["cell observation refused: ", why], "")) + Fci1CellReceiptAdmitted { + wire: wire, memory_max: _, memory_high: _, memory_swap_max: _, tasks_max: _, cpu_weight: _, + base_owner_group: _, cell_owner_group: _, attempt_owner_group: _, + } => fabric_ci_write_wire(path: path, wire: fabric_ci_evidence_wire(value: wire)) + } } func fci1_assert_cell_admitted() -> ProcessExit { @@ -1092,29 +1384,44 @@ func fci1_assert_clean_cell_prestate() -> ProcessExit { } func fci1_assert_cell_unchanged(before_wire: String) -> ProcessExit { - let observed = fci1_cell_observation_wire(decision: admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3"))) - match decode_fabric_ci_evidence_wire(wire: before_wire) { + match fci1_cell_receipt_observation() { + Fci1CellReceiptRefused { detail: why } => exit_failure(reason: join(["persistent cell substrate observation refused: ", why], "")) + Fci1CellReceiptAdmitted { + wire: observed, memory_max: _, memory_high: _, memory_swap_max: _, tasks_max: _, cpu_weight: _, + base_owner_group: _, cell_owner_group: _, attempt_owner_group: _, + } => match decode_fabric_ci_evidence_wire(wire: before_wire) { FabricCiEvidenceUndecodable => exit_failure(reason: "persistent cell substrate before-wire undecodable") FabricCiEvidenceDecoded { value: before } => if observed == (before as String) { ExitSuccess } else { - exit_failure(reason: join(["persistent cell substrate changed; before=", before as String, "; after=", observed], "")) + exit_failure(reason: join(["persistent cell substrate changed; before=", before as String, "; after=", observed as String], "")) } - } + } } } func fci1_grade_and_write_cell_receipt(before_wire: String, phase: NonEmptyStr, path: String) -> ProcessExit { - let after = fci1_cell_observation_wire(decision: admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3"))) - match decode_fabric_ci_evidence_wire(wire: before_wire) { + match fci1_cell_receipt_observation() { + Fci1CellReceiptRefused { detail: why } => exit_failure(reason: join(["cell receipt observation refused: ", why], "")) + Fci1CellReceiptAdmitted { + wire: after, memory_max: memory_max, memory_high: memory_high, memory_swap_max: memory_swap_max, + tasks_max: tasks_max, cpu_weight: cpu_weight, base_owner_group: base_owner, + cell_owner_group: cell_owner, attempt_owner_group: attempt_owner, + } => match decode_fabric_ci_evidence_wire(wire: before_wire) { FabricCiEvidenceUndecodable => exit_failure(reason: "persistent cell substrate before-wire undecodable") FabricCiEvidenceDecoded { value: before } => { let write = Filesystem.Write( path: path, - content: join(["phase=", phase as String, "\nbefore=", before as String, "\nafter=", after], ""), + content: join([ + "phase=", phase as String, "\nbefore=", before as String, "\nafter=", after as String, + "\nmemory_max=", memory_max as String, "\nmemory_high=", memory_high as String, + "\nmemory_swap_max=", memory_swap_max as String, "\ntasks_max=", tasks_max as String, + "\ncpu_weight=", cpu_weight as String, "\nbase_owner_group=", to_string(base_owner), + "\ncell_owner_group=", to_string(cell_owner), "\nattempt_owner_group=", to_string(attempt_owner), + ], ""), ) if !write.success { exit_failure(reason: join(["cell receipt write refused: ", write.error], "")) } - else if after == (before as String) { ExitSuccess } - else { exit_failure(reason: join(["persistent cell substrate changed; before=", before as String, "; after=", after], "")) } + else if (after as String) == (before as String) { ExitSuccess } + else { exit_failure(reason: join(["persistent cell substrate changed; before=", before as String, "; after=", after as String], "")) } } - } + } } } type Fci1ZeroWorkObservation @@ -1160,7 +1467,7 @@ fn fci1_zero_work_observation() -> Fci1ZeroWorkObservation { Present { value: pids } => { let attempts = Filesystem.List(path: fabric_cell_attempt_root_path(slot: cp_probe_slot())) if !attempts.success { Fci1ZeroWorkRefused { cause: join(["attempts enumeration refused: ", attempts.error], "") } } - else { match host_converge_slice1_enumerate_units(instance_glob: "fabric-cell@*.service") { + else { match systemctl_list_units_all_states(instance_glob: "fabric-cell@*.service") { Absent => Fci1ZeroWorkRefused { cause: "fabric-cell@ service enumeration refused" } Present { value: services } => Fci1ZeroWorkObserved { unit: unit, @@ -1384,6 +1691,8 @@ fn fci1_cell_hold_end_wire(outcome: CellHoldEnd) -> String { } } +data fci1_release_identity: NonEmptyStr = "FCI-1 instrument release" + fn fci1_live_release_outcome(root: NonEmptyStr) -> CellHoldEnd { fold( cp_probe_held_account(), @@ -1401,7 +1710,7 @@ fn fci1_live_release_outcome(root: NonEmptyStr) -> CellHoldEnd { generation: 0, }, reservation: cp_probe_reservation_ref(), - reason: "FCI-1 instrument release", + reason: fci1_release_identity, released_at: "2026-08-29T00:00:00Z", account: account, ) diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index 5fcaa701cbc..375091395cc 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -23,6 +23,9 @@ generation_root= subject="$repo_root/dag/gunbc/fabric/fabric_required_build_cell.dag" allocation_before= positive_before= +cell_before= +canonical_authority_path= +generation_authority_path= receipt_path() { printf '%s/%s.txt' "$receipt_root" "$1" @@ -31,7 +34,7 @@ receipt_path() { checkpoint_if_selected() { local phase=$1 [[ $checkpoint == "$phase" ]] || return 0 - fabric_ci_run_assertion fci1_checkpoint_reached --arg token="$checkpoint" --arg phase="$phase" + fabric_ci_run_assertion fci1_checkpoint_reached --arg token="$checkpoint" --arg phase="$phase" --arg path="$(receipt_path checkpoint)" echo "CheckpointReached: checkpoint=$checkpoint phase=$phase status=$injected_failure_status" | tee -a "$FABRIC_CI_LOG" >&2 exit "$injected_failure_status" } @@ -73,7 +76,8 @@ cleanup() { cleanup_disposition_path=$(receipt_path canonical-cleanup) "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ --entry "$FABRIC_CI_ENTRY" --function fci1_write_canonical_cleanup_receipt \ - --arg root="$canonical_root" --arg before_wire="$allocation_before" --arg path="$cleanup_disposition_path" + --arg root="$canonical_root" --arg before_wire="$allocation_before" \ + --arg authority_path="$canonical_authority_path" --arg path="$cleanup_disposition_path" cleanup_status=$? fi if [[ -n $generation_root ]]; then @@ -81,7 +85,7 @@ cleanup() { generation_disposition_path=$(receipt_path generation-cleanup) "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ --entry "$FABRIC_CI_ENTRY" --function fci1_write_disposable_cleanup_receipt \ - --arg root="$generation_root" --arg path="$generation_disposition_path" + --arg root="$generation_root" --arg authority_path="$generation_authority_path" --arg path="$generation_disposition_path" local generation_cleanup_status=$? (( cleanup_status != 0 )) || cleanup_status=$generation_cleanup_status if (( generation_cleanup_status == 0 )); then @@ -89,18 +93,32 @@ cleanup() { fi fi if [[ -e $mutation_root || -L $mutation_root ]]; then + printf 'RuntimeDirectoryObserved|path=%s|standing=present\n' "$mutation_root" >"$(receipt_path runtime-directory)" echo 'InstrumentRefused: dependent RuntimeDirectory remains after cleanup' >&2 cleanup_status=1 else + printf 'RuntimeDirectoryObserved|path=%s|standing=absent\n' "$mutation_root" >"$(receipt_path runtime-directory)" echo "runtime-directory-observed: path=$mutation_root standing=absent" >&2 fi local cleanup_unit for cleanup_unit in fci1-positive-submitter.service fci1-generation-submitter.service fci1-dependent-submitter.service; do - if ! unit_terminal_observed "$cleanup_unit"; then + local unit_receipt + unit_receipt=$(receipt_path "unit-terminal-${cleanup_unit%.service}") + if unit_terminal_observed "$cleanup_unit" >"$unit_receipt" 2>&1; then + : + else + systemctl show "$cleanup_unit" --property=LoadState --property=ActiveState >"$unit_receipt" 2>&1 || true echo "InstrumentRefused: transient unit terminal state unestablished: $cleanup_unit" >&2 cleanup_status=1 fi done + if [[ -n $cell_before ]]; then + "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ + --entry "$FABRIC_CI_ENTRY" --function fci1_grade_and_write_cell_receipt \ + --arg before_wire="$cell_before" --arg phase="terminal" --arg path="$(receipt_path cell-terminal)" + local cell_terminal_status=$? + (( cleanup_status != 0 )) || cleanup_status=$cell_terminal_status + fi local terminal_zero_work terminal_zero_work=$(receipt_path zero-work-terminal) "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ @@ -108,7 +126,35 @@ cleanup() { --arg phase="terminal" --arg path="$terminal_zero_work" local zero_work_status=$? (( cleanup_status != 0 )) || cleanup_status=$zero_work_status - local head tree provenance manifest + if [[ ! -f $(receipt_path checkpoint) ]]; then + "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ + --entry "$FABRIC_CI_ENTRY" --function fci1_write_checkpoint_terminal_receipt \ + --arg token="$checkpoint" --arg path="$(receipt_path checkpoint)" + local checkpoint_receipt_status=$? + (( cleanup_status != 0 )) || cleanup_status=$checkpoint_receipt_status + fi + local required_receipt + local -a required_receipts=( + checkpoint.txt canonical-cleanup.txt generation-cleanup.txt generation-verdict.txt + dependent-lifetime.txt canonical-release.txt canonical-allocation.txt generation-terminal.txt + cell-after-canonical.txt cell-after-dependent.txt cell-terminal.txt + zero-work-before-canonical.txt zero-work-after-canonical.txt zero-work-after-dependent.txt zero-work-terminal.txt + runtime-directory.txt unit-terminal-fci1-positive-submitter.txt + unit-terminal-fci1-generation-submitter.txt unit-terminal-fci1-dependent-submitter.txt + ) + for required_receipt in "${required_receipts[@]}"; do + if [[ ! -f $receipt_root/$required_receipt ]]; then + printf 'ReceiptCoordinateNotReached|file=%s|checkpoint=%s\n' "$required_receipt" "$checkpoint" >"$receipt_root/$required_receipt" || cleanup_status=1 + fi + done + local observed_roster expected_roster + observed_roster=$(find "$receipt_root" -maxdepth 1 -type f ! -name manifest.txt -printf '%f\n' | sort) || cleanup_status=1 + expected_roster=$(printf '%s\n' "${required_receipts[@]}" | sort) || cleanup_status=1 + if [[ $observed_roster != "$expected_roster" ]]; then + echo 'InstrumentRefused: receipt bundle member roster mismatch' >&2 + cleanup_status=1 + fi + local head tree provenance manifest manifest_digest head=$(git -C "$repo_root" rev-parse HEAD) || cleanup_status=1 tree=$(git -C "$repo_root" rev-parse 'HEAD^{tree}') || cleanup_status=1 provenance=$($FABRIC_CI_GUNBC_BIN --version 2>&1) || cleanup_status=1 @@ -116,8 +162,10 @@ cleanup() { { printf 'head=%s\ntree=%s\nbinary_provenance=%s\ncheckpoint=%s\noriginal_exit_status=%s\ncleanup_verdict=%s\n' \ "$head" "$tree" "$provenance" "$checkpoint" "$original_status" "$cleanup_status" - find "$receipt_root" -maxdepth 1 -type f ! -name manifest.txt -print0 | sort -z | xargs -0 -r sha256sum + for required_receipt in "${required_receipts[@]}"; do sha256sum "$receipt_root/$required_receipt"; done } >"$manifest" || cleanup_status=1 + manifest_digest=$(sha256sum "$manifest" | awk '{print $1}') || cleanup_status=1 + echo "FCI1EvidenceIdentity: receipt_root=$receipt_root manifest_sha256=$manifest_digest" >&2 rm -f "$FABRIC_CI_LOG" rm -rf "$FABRIC_CI_VALUE_ROOT" if (( cleanup_status != 0 )); then @@ -160,22 +208,20 @@ zero_work_receipt before-canonical checkpoint_if_selected before-canonical-commit capture_submitter_transport() { - local unit=$1 function_name=$2 coordinate=$3 root=$4 - shift 4 + local unit=$1 function_name=$2 coordinate=$3 root=$4 authority_path=$5 + shift 5 local value_path="$FABRIC_CI_VALUE_ROOT/$coordinate.wire" rm -f -- "$value_path" local run_status=0 if systemd-run --quiet --wait --collect --unit="$unit" "$@" \ "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ - --entry "$FABRIC_CI_ENTRY" --function "$function_name" --arg root="$root" --arg path="$value_path"; then + --entry "$FABRIC_CI_ENTRY" --function "$function_name" --arg root="$root" --arg path="$value_path" \ + --arg cleanup_path="$authority_path" --arg checkpoint="$checkpoint" --arg checkpoint_path="$(receipt_path checkpoint)"; then run_status=0 else run_status=$? fi (( run_status == 0 )) || return "$run_status" - if [[ $unit == fci1-positive-submitter.service ]]; then - checkpoint_if_selected after-canonical-commit-before-transport - fi [[ -f $value_path && ! -L $value_path ]] || return 1 local value value=$(<"$value_path") @@ -183,7 +229,8 @@ capture_submitter_transport() { printf '%s' "$value" } -positive_before=$(capture_submitter_transport fci1-positive-submitter.service fci1_live_reserve_and_observe_available positive-before "$canonical_root") +canonical_authority_path="$FABRIC_CI_VALUE_ROOT/canonical-cleanup-authority.wire" +positive_before=$(capture_submitter_transport fci1-positive-submitter.service fci1_live_reserve_and_observe_available positive-before "$canonical_root" "$canonical_authority_path") unit_terminal_observed fci1-positive-submitter.service || { echo 'InstrumentRefused: submitting control process terminal state unestablished' >&2 exit 1 @@ -205,24 +252,25 @@ checkpoint_if_selected canonical-restored-later # run-unique disposable store uses the production reservation/CAS entries and is removed afterward; # the canonical root sees only reserve, independent observation, and release. generation_root=$(mktemp -d /run/fci1-generation-mutation.XXXXXX) -generation_before=$(capture_submitter_transport fci1-generation-submitter.service fci1_live_reserve_and_observe_available generation-before "$generation_root") +generation_authority_path="$FABRIC_CI_VALUE_ROOT/generation-cleanup-authority.wire" +generation_before=$(capture_submitter_transport fci1-generation-submitter.service fci1_live_reserve_and_observe_available generation-before "$generation_root" "$generation_authority_path") checkpoint_if_selected generation-held-one fabric_ci_run_assertion fci1_assert_replace_held --arg root="$generation_root" checkpoint_if_selected generation-held-two -fabric_ci_run_assertion fci1_assert_generation_changed --arg root="$generation_root" --arg before_wire="$generation_before" +fabric_ci_run_assertion fci1_write_generation_changed_receipt --arg root="$generation_root" --arg before_wire="$generation_before" --arg path="$(receipt_path generation-verdict)" checkpoint_if_selected generation-changed-observed fabric_ci_run_assertion fci1_live_release --arg root="$generation_root" checkpoint_if_selected generation-free-before-removal generation_terminal_receipt=$(receipt_path generation-terminal) -fabric_ci_run_assertion fci1_write_disposable_cleanup_receipt --arg root="$generation_root" --arg path="$generation_terminal_receipt" +fabric_ci_run_assertion fci1_write_disposable_cleanup_receipt --arg root="$generation_root" --arg authority_path="$generation_authority_path" --arg path="$generation_terminal_receipt" rm -rf -- "$generation_root" generation_root= -dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service fci1_live_reserve_and_observe_available dependent-before "$mutation_root" --property=RuntimeDirectory="$mutation_runtime") +dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service fci1_live_reserve_and_observe_available dependent-before "$mutation_root" "$FABRIC_CI_VALUE_ROOT/dependent-cleanup-authority.wire" --property=RuntimeDirectory="$mutation_runtime") [[ ! -e $mutation_root && ! -L $mutation_root ]] || { echo 'ExpectedOutcomeNotObserved: submitter runtime directory removal' >&2; exit 1; } echo "runtime-directory-observed: path=$mutation_root standing=absent" >&2 unit_terminal_observed fci1-dependent-submitter.service || { echo 'InstrumentRefused: dependent submitter terminal state unestablished' >&2; exit 1; } -fabric_ci_run_assertion fci1_assert_lifetime_dependent --arg root="$mutation_root" --arg before_wire="$dependent_before" +fabric_ci_run_assertion fci1_write_lifetime_dependent_receipt --arg root="$mutation_root" --arg before_wire="$dependent_before" --arg path="$(receipt_path dependent-lifetime)" checkpoint_if_selected runtime-directory-terminal cell_receipt=$(receipt_path cell-after-dependent) fabric_ci_run_assertion fci1_grade_and_write_cell_receipt --arg before_wire="$cell_before" --arg phase="after-dependent" --arg path="$cell_receipt" From f4d1c09488276b9be5e8f71598d399e5760b4c0d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 16:42:51 +0000 Subject: [PATCH 09/50] Fix FCI-1 cleanup disposition closure --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 91598377eda..4b62d158b10 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -1013,6 +1013,8 @@ func fci1_canonical_cleanup_disposition( } } +} + func fci1_write_canonical_cleanup_receipt(root: NonEmptyStr, before_wire: String, authority_path: String, path: String) -> ProcessExit { let disposition = fci1_canonical_cleanup_disposition(root: root, before_wire: before_wire, authority_path: authority_path) let write = Filesystem.Write(path: path, content: to_string(disposition)) From e825f0eb1f885211d0e70c70dc71b1ce8115e337 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 17:01:16 +0000 Subject: [PATCH 10/50] Repair rebased FCI-1 sources --- .../instruments/fabric_control_plane_live_probe.dag | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 4b62d158b10..ce3808f7480 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -685,7 +685,7 @@ func fci1_live_reserve_and_observe_available( path: path, wire: fabric_ci_evidence_wire(value: required_build_cell_lifetime_sample_wire(sample: sample) as NonEmptyStr), ) - } } + } } _ => exit_failure(reason: join(["ReservationCommitRefused:", fci1_cell_reservation_wire(reservation: reserved)], "")) } @@ -949,7 +949,7 @@ func fci1_canonical_cleanup_disposition( let before_decoded = decode_fabric_ci_evidence_wire(wire: before_wire) let observed = fci1_allocation_slot_observation(root: root) let observed_prestate = fci1_slot_observation_prestate(observed: observed) - match before_decoded { + let initial_disposition = match before_decoded { FabricCiEvidenceUndecodable => CanonicalCleanupRefused { prestate: AllocationSlotUnreadable { path: root }, observed: observed } FabricCiEvidenceDecoded { value: before_value } => { let prestate = fci1_decode_allocation_prestate(wire: before_value as String) @@ -988,7 +988,8 @@ func fci1_canonical_cleanup_disposition( } } else { CanonicalCleanupRefused { prestate: prestate, observed: observed } } Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: prestate, observed: observed } - } |> fn(disposition) { match disposition { + } } + let disposition = match initial_disposition { CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } => match fci1_cleanup_authority_from_path(path: authority_path) { Fci1ReservationCleanupAuthorityDecoded { authority: authority } => @@ -1007,14 +1008,12 @@ func fci1_canonical_cleanup_disposition( } else { CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } } Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } } - _ => disposition + _ => initial_disposition } } } } } -} - func fci1_write_canonical_cleanup_receipt(root: NonEmptyStr, before_wire: String, authority_path: String, path: String) -> ProcessExit { let disposition = fci1_canonical_cleanup_disposition(root: root, before_wire: before_wire, authority_path: authority_path) let write = Filesystem.Write(path: path, content: to_string(disposition)) From ba4f9d3e9edce819a620441a23052cd01fcf1b80 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 17:11:15 +0000 Subject: [PATCH 11/50] Fix binding sample effect declaration --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index ce3808f7480..0d720d31390 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -691,7 +691,7 @@ func fci1_live_reserve_and_observe_available( } } -func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) -> ProcessExit { +func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) { match required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) { RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "RequiredBuildCellObservationRefused") RequiredBuildCellLifetimeSampleDecoded { sample: sample } => fabric_ci_write_wire( From 4803997a1933ef386775dc5be08152a95b982c1c Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 17:19:11 +0000 Subject: [PATCH 12/50] Close FCI-1 binding sample scope --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 0d720d31390..6dcab035f5f 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -691,7 +691,9 @@ func fci1_live_reserve_and_observe_available( } } -func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) { +} + +func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) -> ProcessExit { match required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) { RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "RequiredBuildCellObservationRefused") RequiredBuildCellLifetimeSampleDecoded { sample: sample } => fabric_ci_write_wire( @@ -1012,8 +1014,6 @@ func fci1_canonical_cleanup_disposition( } } } } -} - func fci1_write_canonical_cleanup_receipt(root: NonEmptyStr, before_wire: String, authority_path: String, path: String) -> ProcessExit { let disposition = fci1_canonical_cleanup_disposition(root: root, before_wire: before_wire, authority_path: authority_path) let write = Filesystem.Write(path: path, content: to_string(disposition)) From 76dda1caf9184fdb24b323267c78d1293c4fd95a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 19:39:21 +0000 Subject: [PATCH 13/50] Handle all reservation refusal outcomes --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 6dcab035f5f..09f7ce4944c 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -687,7 +687,11 @@ func fci1_live_reserve_and_observe_available( ) } } - _ => exit_failure(reason: join(["ReservationCommitRefused:", fci1_cell_reservation_wire(reservation: reserved)], "")) + NoCellAdmissible { considered: _, refused: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) + BudgetRefused { slot_key: _, refusal: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) + ReservationLost { slot_key: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) + StoreUnavailable { slot_key: _, cause: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) + AttemptRefused { slot_key: _, detail: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) } } From 24d8207b63eb9efdb2c589533c529a383e9a2ec4 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 19:49:43 +0000 Subject: [PATCH 14/50] Complete reservation refusal match --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 09f7ce4944c..2fdffbfe105 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -686,7 +686,7 @@ func fci1_live_reserve_and_observe_available( wire: fabric_ci_evidence_wire(value: required_build_cell_lifetime_sample_wire(sample: sample) as NonEmptyStr), ) } - } + } } NoCellAdmissible { considered: _, refused: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) BudgetRefused { slot_key: _, refusal: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) ReservationLost { slot_key: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) @@ -695,8 +695,6 @@ func fci1_live_reserve_and_observe_available( } } -} - func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) -> ProcessExit { match required_build_cell_lifetime_sample(observation: fci1_live_binding_observation(root: root)) { RequiredBuildCellLifetimeSampleUndecodable => exit_failure(reason: "RequiredBuildCellObservationRefused") From 2b1e62a40bb0b642842f2e5e9891a9073492457e Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 20:07:14 +0000 Subject: [PATCH 15/50] Bind cleanup disposition within refusal fold --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 2fdffbfe105..31e90b73386 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -992,7 +992,7 @@ func fci1_canonical_cleanup_disposition( } } else { CanonicalCleanupRefused { prestate: prestate, observed: observed } } Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: prestate, observed: observed } - } } + } let disposition = match initial_disposition { CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } => match fci1_cleanup_authority_from_path(path: authority_path) { @@ -1016,6 +1016,7 @@ func fci1_canonical_cleanup_disposition( } } } } +} func fci1_write_canonical_cleanup_receipt(root: NonEmptyStr, before_wire: String, authority_path: String, path: String) -> ProcessExit { let disposition = fci1_canonical_cleanup_disposition(root: root, before_wire: before_wire, authority_path: authority_path) let write = Filesystem.Write(path: path, content: to_string(disposition)) From cea07c84833b5d1ad9318b0ef98982ff73607a34 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 20:30:28 +0000 Subject: [PATCH 16/50] Keep cleanup disposition in decoded observation scope --- .../instruments/fabric_control_plane_live_probe.dag | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 31e90b73386..0c17efc9fc3 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -953,11 +953,11 @@ func fci1_canonical_cleanup_disposition( let before_decoded = decode_fabric_ci_evidence_wire(wire: before_wire) let observed = fci1_allocation_slot_observation(root: root) let observed_prestate = fci1_slot_observation_prestate(observed: observed) - let initial_disposition = match before_decoded { + let disposition = match before_decoded { FabricCiEvidenceUndecodable => CanonicalCleanupRefused { prestate: AllocationSlotUnreadable { path: root }, observed: observed } FabricCiEvidenceDecoded { value: before_value } => { let prestate = fci1_decode_allocation_prestate(wire: before_value as String) - if fci1_slot_prestate_equal(a: prestate, b: observed_prestate) { + let initial_disposition = if fci1_slot_prestate_equal(a: prestate, b: observed_prestate) { match prestate { AllocationSlotAbsent { path: _ } => NoCanonicalCommit { prestate: prestate, poststate: observed } AllocationSlotFree { path: _, generation: _ } => NoCanonicalCommit { prestate: prestate, poststate: observed } @@ -992,8 +992,8 @@ func fci1_canonical_cleanup_disposition( } } else { CanonicalCleanupRefused { prestate: prestate, observed: observed } } Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: prestate, observed: observed } - } - let disposition = match initial_disposition { + } } + match initial_disposition { CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } => match fci1_cleanup_authority_from_path(path: authority_path) { Fci1ReservationCleanupAuthorityDecoded { authority: authority } => @@ -1013,7 +1013,7 @@ func fci1_canonical_cleanup_disposition( Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } } _ => initial_disposition - } } + } } } } From 298e6610d18ff79f6f07ffd976c4629ec80946f6 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 31 Aug 2026 23:51:28 +0000 Subject: [PATCH 17/50] Close FCI-1 reservation cleanup edges --- dag/extdeps/systemd/systemctl.dag | 33 ++++ dag/gunbc/fabric/fabric_cell_acquire.dag | 44 ++++- .../fabric_control_plane_live_probe.dag | 176 +++++++++++++----- dag/gunbc/systemctl_list_units.dag | 10 +- tools/fabric_ci_fci1_live_instrument.sh | 38 +++- 5 files changed, 235 insertions(+), 66 deletions(-) diff --git a/dag/extdeps/systemd/systemctl.dag b/dag/extdeps/systemd/systemctl.dag index ce6644dec39..958ff184b41 100644 --- a/dag/extdeps/systemd/systemctl.dag +++ b/dag/extdeps/systemd/systemctl.dag @@ -431,6 +431,39 @@ service systemd.Systemctl { } } + operation ListUnitsAllLoaded { + input { + pattern: NonEmptyStr, + unit_type: String = "service", + } + output { + stdout: String from "stdout" + success: Bool from "exit_success" + } + readonly + transport shell { + argv: [ + "systemctl", + "list-units", + "--type={unit_type}", + "--all", + "--no-legend", + "--plain", + "{pattern}", + ] + } + exit { + 0 => Unit + nonzero => String "systemctl list-units --all failed" + } + mock_response { + 0 => { + stdout: "actions-runner@srv3-01.service loaded active running GitHub Actions Runner\nactions-runner@srv3-02.service loaded inactive dead GitHub Actions Runner\n", + success: true, + } "hermetic systemd.Systemctl.ListUnitsAllLoaded (complete loaded-unit population)" + } + } + operation Status { input { unit: NonEmptyStr } output { diff --git a/dag/gunbc/fabric/fabric_cell_acquire.dag b/dag/gunbc/fabric/fabric_cell_acquire.dag index 19c0132b277..1d1bf9fbec5 100644 --- a/dag/gunbc/fabric/fabric_cell_acquire.dag +++ b/dag/gunbc/fabric/fabric_cell_acquire.dag @@ -505,14 +505,30 @@ fn fabric_cell_unit_name_field(line: String) -> String { // still returns a real probe: its namespaces were read, and the pure half decides what an empty // expectation means. Supplying the roster from this side would be the observer reporting on what // it happened to probe, which is the blindness the observe module's own header rules out. -func fabric_cell_probe_wet(host: HostIdentity) -> FabricCellHostProbe { +type FabricCellProbeReading sole_constructor { + probe: FabricCellHostProbe + boundary_readings: List +} + +fn fabric_cell_probe_reading_probe(reading: FabricCellProbeReading) -> FabricCellHostProbe { + reading.probe +} + +fn fabric_cell_probe_reading_boundaries(reading: FabricCellProbeReading) -> List { + reading.boundary_readings +} + +// The receipt retains the same boundary readings used to construct the probe. Consumers needing +// raw values must project this value; independently re-reading the five properties would combine +// two host instants into one purported observation. +func fabric_cell_probe_reading_wet(host: HostIdentity) -> FabricCellProbeReading { let root = fabric_cell_observe_cells_root() let slices = fabric_cell_observe_slice_namespace() let acquired = map( fabric_cell_expected_slots(host: host), s => fabric_cell_acquire_slot(slot: s, cells_root: root, slices: slices), ) - FabricCellHostProbe { + let probe = FabricCellHostProbe { host: host, namespaces: concat( [ @@ -529,6 +545,14 @@ func fabric_cell_probe_wet(host: HostIdentity) -> FabricCellHostProbe { ), addresses: flat_map(acquired, a => fabric_cell_acquisition_address_entries(acquisition: a)), } + FabricCellProbeReading { + probe: probe, + boundary_readings: map(acquired, a => a.boundary_reading), + } +} + +func fabric_cell_probe_wet(host: HostIdentity) -> FabricCellHostProbe { + fabric_cell_probe_reading_probe(reading: fabric_cell_probe_reading_wet(host: host)) } // ONE SUBJECT IS ACQUIRED ONCE PER TICK, AND THE PROJECTIONS READ THE VALUE RATHER THAN THE HOST. @@ -548,6 +572,7 @@ type FabricCellSlotAcquisition { cell_root: FabricCellDirectoryObservation attempt_root: FabricCellDirectoryObservation boundary: List + boundary_reading: FabricCellBoundaryReading } func fabric_cell_acquire_slot( @@ -556,6 +581,11 @@ func fabric_cell_acquire_slot( slices: FabricCellNamespaceProbe, ) -> FabricCellSlotAcquisition { let cell_root = fabric_cell_observe_cell_root(slot: slot, cells_root: cells_root) + let boundary_reading = match fabric_cell_boundary_standing(slot: slot, slices: slices) { + BoundaryAsk => fabric_cell_read_boundary(slot: slot) + BoundaryUnknownRefuse { cause: cause } => BoundaryReadRefused { cause: cause } + BoundaryAbsentNoAsk => BoundaryReadRefused { cause: "cell boundary slice is absent" } + } FabricCellSlotAcquisition { slot: slot, cell_root: cell_root, @@ -563,7 +593,8 @@ func fabric_cell_acquire_slot( path: fabric_cell_attempt_root_path(slot: slot), parent: fabric_cell_parent_standing_of(parent: cell_root, entry: fabric_cell_attempt_root_entry_name), ), - boundary: fabric_cell_boundary_address_entries(slot: slot, slices: slices), + boundary: fabric_cell_boundary_address_entries(slot: slot, slices: slices, reading: boundary_reading), + boundary_reading: boundary_reading, } } @@ -687,6 +718,7 @@ fn fabric_cell_boundary_standing( func fabric_cell_boundary_address_entries( slot: RunnerSlotIdentity, slices: FabricCellNamespaceProbe, + reading: FabricCellBoundaryReading, ) -> List { match fabric_cell_boundary_standing(slot: slot, slices: slices) { BoundaryAbsentNoAsk => [] @@ -696,7 +728,7 @@ func fabric_cell_boundary_address_entries( address: FabricCellResourceBoundaryAddress, probe: AddressStateReadRefused { cause: c }, }] - BoundaryAsk => [fabric_cell_boundary_probe_entry(slot: slot)] + BoundaryAsk => [fabric_cell_boundary_probe_entry(slot: slot, reading: reading)] } } @@ -704,13 +736,13 @@ fn fabric_cell_entries_hold_name(entries: List, name: String) -> Bool { fold(entries, init: false, f: (acc, e) => acc || e == name) } -func fabric_cell_boundary_probe_entry(slot: RunnerSlotIdentity) -> FabricCellAddressProbeEntry { +fn fabric_cell_boundary_probe_entry(slot: RunnerSlotIdentity, reading: FabricCellBoundaryReading) -> FabricCellAddressProbeEntry { FabricCellAddressProbeEntry { slot: slot, address: FabricCellResourceBoundaryAddress, probe: fabric_cell_boundary_address_probe( slot: slot, - reading: fabric_cell_read_boundary(slot: slot), + reading: reading, ), } } diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 0c17efc9fc3..4240f74c40e 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -73,7 +73,8 @@ import gunbc.fabric_required_build_cell { import gunbc.fabric_cell_acquire { fabric_cell_probe_wet, FabricCellBoundaryReading, BoundaryValuesRead, BoundaryReadRefused, - fabric_cell_read_boundary, + fabric_cell_probe_reading_wet, fabric_cell_probe_reading_probe, + fabric_cell_probe_reading_boundaries, } import gunbc.fabric_cell_observation_admission { FabricCellObservationDecision, FabricCellObservationAdmitted, FabricCellObservationDenied, @@ -128,6 +129,7 @@ type Fci1Checkpoint | Fci1CheckpointAfterReleaseBeforeGrading | Fci1CheckpointCanonicalRestoredLater | Fci1CheckpointGenerationHeldOne + | Fci1CheckpointGenerationAuthorityTwoBeforeCommit | Fci1CheckpointGenerationHeldTwo | Fci1CheckpointGenerationChangedObserved | Fci1CheckpointGenerationFreeBeforeRemoval @@ -142,6 +144,7 @@ fn fci1_checkpoint_token(checkpoint: Fci1Checkpoint) -> NonEmptyStr { Fci1CheckpointAfterReleaseBeforeGrading => "after-release-before-grading" Fci1CheckpointCanonicalRestoredLater => "canonical-restored-later" Fci1CheckpointGenerationHeldOne => "generation-held-one" + Fci1CheckpointGenerationAuthorityTwoBeforeCommit => "generation-authority-two-before-commit" Fci1CheckpointGenerationHeldTwo => "generation-held-two" Fci1CheckpointGenerationChangedObserved => "generation-changed-observed" Fci1CheckpointGenerationFreeBeforeRemoval => "generation-free-before-removal" @@ -158,6 +161,7 @@ fn fci1_checkpoints() -> List { Fci1CheckpointAfterReleaseBeforeGrading, Fci1CheckpointCanonicalRestoredLater, Fci1CheckpointGenerationHeldOne, + Fci1CheckpointGenerationAuthorityTwoBeforeCommit, Fci1CheckpointGenerationHeldTwo, Fci1CheckpointGenerationChangedObserved, Fci1CheckpointGenerationFreeBeforeRemoval, @@ -546,13 +550,6 @@ fn fci1_member_wire(member: FabricCellMember) -> NonEmptyStr { ], "") as NonEmptyStr } -fn fci1_path_owner_wire(path: NonEmptyStr) -> String { - match srv3_observe_path_owner(transport: LocalShell, path: path) { - PathOwnerObserved { owner: owner } => join(["path=", path as String, "|owner_group=", to_string(owner)], "") - PathOwnerUnobserved { reason: why } => join(["path=", path as String, "|owner_group_unobserved=", why], "") - } -} - fn fci1_modeled_ownership_wire(member: FabricCellMember) -> NonEmptyStr { match fabric_cell_member_ownership(member: member) { Absent => "modeled-ownership-absent" @@ -565,15 +562,38 @@ func fci1_live_replace_held_wire(root: NonEmptyStr) -> String { fci1_cell_reservation_wire(reservation: fci1_live_replace_held(root: root)) } -func fci1_assert_replace_held(root: NonEmptyStr) -> ProcessExit { - let outcome = fci1_live_replace_held(root: root) +func fci1_assert_replace_held( + root: NonEmptyStr, + cleanup_path: String, + checkpoint: NonEmptyStr, + checkpoint_path: String, +) -> ProcessExit { + let observed = fci1_allocation_slot_observation(root: root) + let expected_generation = match observed { + Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => generation + 1 + _ => -1 + } + let authority = fci1_cleanup_authority(root: root, generation: expected_generation) + let authority_write = Filesystem.Write( + path: fci1_cleanup_authority_generation_path(path: cleanup_path, generation: expected_generation), + content: fci1_reservation_cleanup_authority_wire(authority: authority) as String, + ) + if expected_generation < 1 { exit_failure(reason: "generation replacement prestate was not Held") } + else if !authority_write.success { exit_failure(reason: "ReplacementCleanupAuthorityWriteRefusedBeforeCommit") } + else if checkpoint == "generation-authority-two-before-commit" { + match fci1_checkpoint_reached(token: checkpoint, phase: checkpoint, path: checkpoint_path) { + ExitSuccess => ExitFailure { code: 86, reason: "FCI1InjectedFailureAfterReplacementAuthorityBeforeCommit" } + ExitFailure { code: _, reason: why } => exit_failure(reason: why) + } + } + else { let outcome = fci1_live_replace_held(root: root) match outcome { - CellReserved { offer: offer, slot_key: slot, generation: _, account: _ } => - if slot == "srv3-06" && offer.key == "offer-1" { ExitSuccess } else { + CellReserved { offer: offer, slot_key: slot, generation: generation, account: _ } => + if slot == "srv3-06" && offer.key == "offer-1" && generation == expected_generation { ExitSuccess } else { exit_failure(reason: join(["generation replacement selected wrong identity: ", fci1_cell_reservation_wire(reservation: outcome)], "")) } _ => exit_failure(reason: join(["generation replacement refused: ", fci1_cell_reservation_wire(reservation: outcome)], "")) - } + } } } func fci1_live_binding_observation(root: NonEmptyStr) -> RequiredBuildCellObservation { @@ -613,6 +633,23 @@ fn fci1_reservation_cleanup_authority_wire(authority: Fci1ReservationCleanupAuth ], "") as NonEmptyStr } +fn fci1_cleanup_authority(root: NonEmptyStr, generation: Int) -> Fci1ReservationCleanupAuthority { + let reservation = cp_probe_reservation_ref() + let payload = join(["held|", reservation as String], "") as NonEmptyStr + Fci1ReservationCleanupAuthority { + root: root, + slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), + generation: generation, + reservation_identity: reservation as NonEmptyStr, + reservation_payload: payload, + reservation_content: serialize_content_hash(hash: content_hash_of_value(value: payload)), + } +} + +fn fci1_cleanup_authority_generation_path(path: String, generation: Int) -> String { + join([path, ".", to_string(generation)], "") +} + fn fci1_decode_reservation_cleanup_authority(wire: String) -> Fci1ReservationCleanupAuthorityDecode { let fields = split(s: wire, delimiter: "\n") if list_length(items: fields) != 6 { Fci1ReservationCleanupAuthorityUndecodable } @@ -656,24 +693,25 @@ func fci1_live_reserve_and_observe_available( checkpoint: NonEmptyStr, checkpoint_path: String, ) -> ProcessExit { - let reserved = fci1_live_reserve_available(root: root) + let prestate = fci1_allocation_prestate(root: root) + let expected_generation = fci1_expected_held_generation(prestate: prestate) + let authority = fci1_cleanup_authority(root: root, generation: expected_generation) + let authority_write = Filesystem.Write( + path: fci1_cleanup_authority_generation_path(path: cleanup_path, generation: expected_generation), + content: fci1_reservation_cleanup_authority_wire(authority: authority) as String, + ) + if expected_generation < 1 { exit_failure(reason: "ReservationCleanupAuthorityPrestateRefused") } + else if !authority_write.success { exit_failure(reason: "ReservationCleanupAuthorityWriteRefusedBeforeCommit") } + else { let reserved = match prestate { + AllocationSlotAbsent { path: _ } => fci1_live_reserve(root: root, expected: ExpectSlotAbsent) + AllocationSlotFree { path: _, generation: generation } => fci1_live_reserve(root: root, expected: ExpectSlotGeneration { generation: generation }) + AllocationSlotHeld { path: _, generation: _ } => ReservationLost { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()) } + AllocationSlotUnreadable { path: _ } => AttemptRefused { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), detail: "cleanup-authority prestate unreadable" } + AllocationSlotUndecodable { path: _, generation: _ } => AttemptRefused { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), detail: "cleanup-authority prestate undecodable" } + } match reserved { - CellReserved { offer: _, slot_key: slot, generation: generation, account: _ } => { - let reservation = cp_probe_reservation_ref() - let payload = join(["held|", reservation as String], "") as NonEmptyStr - let authority = Fci1ReservationCleanupAuthority { - root: root, - slot_key: slot, - generation: generation, - reservation_identity: reservation as NonEmptyStr, - reservation_payload: payload, - reservation_content: serialize_content_hash(hash: content_hash_of_value(value: payload)), - } - let authority_write = Filesystem.Write( - path: cleanup_path, - content: fci1_reservation_cleanup_authority_wire(authority: authority) as String, - ) - if !authority_write.success { exit_failure(reason: "ReservationCleanupAuthorityWriteRefused") } + CellReserved { offer: _, slot_key: _, generation: generation, account: _ } => { + if generation != expected_generation { exit_failure(reason: "ReservationGenerationDisagreedWithPrepublishedCleanupAuthority") } else if checkpoint == "after-canonical-commit-before-transport" { match fci1_checkpoint_reached(token: checkpoint, phase: checkpoint, path: checkpoint_path) { ExitSuccess => ExitFailure { code: 86, reason: "FCI1InjectedFailureAfterCommitBeforeTransport" } @@ -692,7 +730,7 @@ func fci1_live_reserve_and_observe_available( ReservationLost { slot_key: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) StoreUnavailable { slot_key: _, cause: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) AttemptRefused { slot_key: _, detail: _ } => exit_failure(reason: fci1_cell_reservation_wire(reservation: reserved)) - } + } } } func fci1_live_binding_sample_wire(root: NonEmptyStr, path: String) -> ProcessExit { @@ -912,6 +950,21 @@ fn fci1_cleanup_authority_from_path(path: String) -> Fci1ReservationCleanupAutho else { Fci1ReservationCleanupAuthorityUndecodable } } +fn fci1_cleanup_authority_for_observation( + path: String, + observed: Fci1AllocationSlotObservation, +) -> Fci1ReservationCleanupAuthorityDecode { + let authority_generation = match observed { + Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => generation + Fci1SlotFree { path: _, generation: generation, released_by: _, payload: _, content: _ } => generation - 1 + _ => -1 + } + if authority_generation < 1 { Fci1ReservationCleanupAuthorityUndecodable } + else { fci1_cleanup_authority_from_path( + path: fci1_cleanup_authority_generation_path(path: path, generation: authority_generation), + ) } +} + fn fci1_authority_matches_held( authority: Fci1ReservationCleanupAuthority, observed: Fci1AllocationSlotObservation, @@ -965,7 +1018,7 @@ func fci1_canonical_cleanup_disposition( AllocationSlotUnreadable { path: _ } => CanonicalCleanupRefused { prestate: prestate, observed: observed } AllocationSlotUndecodable { path: _, generation: _ } => CanonicalCleanupRefused { prestate: prestate, observed: observed } } - } else { match fci1_cleanup_authority_from_path(path: authority_path) { + } else { match fci1_cleanup_authority_for_observation(path: authority_path, observed: observed) { Fci1ReservationCleanupAuthorityDecoded { authority: authority } => if authority.generation == fci1_expected_held_generation(prestate: prestate) && fci1_authority_matches_held(authority: authority, observed: observed) { @@ -995,7 +1048,7 @@ func fci1_canonical_cleanup_disposition( } } match initial_disposition { CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } => - match fci1_cleanup_authority_from_path(path: authority_path) { + match fci1_cleanup_authority_for_observation(path: authority_path, observed: observed) { Fci1ReservationCleanupAuthorityDecoded { authority: authority } => if authority.generation == fci1_expected_held_generation(prestate: prestate) && fci1_authority_matches_exact_free(authority: authority, observed: observed) { @@ -1039,7 +1092,7 @@ func fci1_write_disposable_cleanup_receipt(root: NonEmptyStr, authority_path: St if write.success { ExitSuccess } else { exit_failure(reason: "disposable no-commit receipt write refused") } } Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => - if generation == 1 || generation == 2 { match fci1_cleanup_authority_from_path(path: authority_path) { + if generation == 1 || generation == 2 { match fci1_cleanup_authority_for_observation(path: authority_path, observed: observed) { Fci1ReservationCleanupAuthorityUndecodable => exit_failure(reason: "disposable cleanup authority undecodable") Fci1ReservationCleanupAuthorityDecoded { authority: authority } => if !fci1_authority_matches_held(authority: authority, observed: observed) { exit_failure(reason: "disposable held reservation identity refused") @@ -1064,7 +1117,7 @@ func fci1_write_disposable_cleanup_receipt(root: NonEmptyStr, authority_path: St } } } else { exit_failure(reason: "disposable held an unexpected generation") } Fci1SlotFree { path: _, generation: generation, released_by: _, payload: _, content: _ } => - match fci1_cleanup_authority_from_path(path: authority_path) { + match fci1_cleanup_authority_for_observation(path: authority_path, observed: observed) { Fci1ReservationCleanupAuthorityUndecodable => exit_failure(reason: "disposable already-Free authority undecodable") Fci1ReservationCleanupAuthorityDecoded { authority: authority } => if (generation == 2 || generation == 3) && fci1_authority_matches_exact_free(authority: authority, observed: observed) { @@ -1272,9 +1325,6 @@ fn fci1_cell_observation_wire(decision: FabricCellObservationDecision) -> String "|members=", join(map(members, m => fci1_member_wire(member: m) as String), ";"), "|boundaries=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: boundaries)) as String, "|modeled-teardown-ownership=", serialize_content_hash(hash: fabric_ci_shell_crossing_coordinates_hash(values: map(members, m => fci1_modeled_ownership_wire(member: m)))) as String, - "|base-owner-group=", fci1_path_owner_wire(path: fabric_cell_base_dir as NonEmptyStr), - "|cell-owner-group=", fci1_path_owner_wire(path: fabric_cell_root_path(slot: cp_probe_slot()) as NonEmptyStr), - "|attempt-owner-group=", fci1_path_owner_wire(path: fabric_cell_attempt_root_path(slot: cp_probe_slot()) as NonEmptyStr), ], "") } FabricCellObservationDenied { host: host, blocked: blocked } => join([ @@ -1285,6 +1335,32 @@ fn fci1_cell_observation_wire(decision: FabricCellObservationDecision) -> String } } +// Pure projection of the already acquired carrier. No renderer is allowed to return to the host: +// all equality and retained evidence below therefore describe the same owner and boundary reads. +fn fci1_cell_receipt_wire( + decision: FabricCellObservationDecision, + memory_max: NonEmptyStr, + memory_high: NonEmptyStr, + memory_swap_max: NonEmptyStr, + tasks_max: NonEmptyStr, + cpu_weight: NonEmptyStr, + base_owner: PosixOwnerSpec, + cell_owner: PosixOwnerSpec, + attempt_owner: PosixOwnerSpec, +) -> NonEmptyStr { + join([ + fci1_cell_observation_wire(decision: decision), + "|memory_max=", memory_max as String, + "|memory_high=", memory_high as String, + "|memory_swap_max=", memory_swap_max as String, + "|tasks_max=", tasks_max as String, + "|cpu_weight=", cpu_weight as String, + "|base_owner_group=", to_string(base_owner), + "|cell_owner_group=", to_string(cell_owner), + "|attempt_owner_group=", to_string(attempt_owner), + ], "") as NonEmptyStr +} + type Fci1CellReceiptObservation = Fci1CellReceiptAdmitted { wire: NonEmptyStr @@ -1300,7 +1376,8 @@ type Fci1CellReceiptObservation | Fci1CellReceiptRefused { detail: String } fn fci1_cell_receipt_observation() -> Fci1CellReceiptObservation { - let decision = admit_fabric_cell_probe(probe: fabric_cell_probe_wet(host: "srv3")) + let reading = fabric_cell_probe_reading_wet(host: "srv3") + let decision = admit_fabric_cell_probe(probe: fabric_cell_probe_reading_probe(reading: reading)) match fabric_cell_observation_outcome(decision: decision) { FabricCellObservationDenied { host: _, blocked: _ } => Fci1CellReceiptRefused { detail: fci1_cell_observation_wire(decision: decision) } FabricCellObservationNotApplicable { host: _ } => Fci1CellReceiptRefused { detail: fci1_cell_observation_wire(decision: decision) } @@ -1315,13 +1392,24 @@ fn fci1_cell_receipt_observation() -> Fci1CellReceiptObservation { transport: LocalShell, path: fabric_cell_attempt_root_path(slot: cp_probe_slot()) as NonEmptyStr, ) { PathOwnerUnobserved { reason: why } => Fci1CellReceiptRefused { detail: why } - PathOwnerObserved { owner: attempt_owner } => match fabric_cell_read_boundary(slot: cp_probe_slot()) { - BoundaryReadRefused { cause: why } => Fci1CellReceiptRefused { detail: why } - BoundaryValuesRead { + PathOwnerObserved { owner: attempt_owner } => match fabric_cell_probe_reading_boundaries(reading: reading).first() { + Absent => Fci1CellReceiptRefused { detail: "cell probe retained no boundary reading" } + Present { value: BoundaryReadRefused { cause: why } } => Fci1CellReceiptRefused { detail: why } + Present { value: BoundaryValuesRead { memory_max: memory_max, memory_high: memory_high, memory_swap_max: memory_swap_max, tasks_max: tasks_max, cpu_weight: cpu_weight, - } => Fci1CellReceiptAdmitted { - wire: fci1_cell_observation_wire(decision: decision) as NonEmptyStr, + } } => Fci1CellReceiptAdmitted { + wire: fci1_cell_receipt_wire( + decision: decision, + memory_max: memory_max as NonEmptyStr, + memory_high: memory_high as NonEmptyStr, + memory_swap_max: memory_swap_max as NonEmptyStr, + tasks_max: tasks_max as NonEmptyStr, + cpu_weight: cpu_weight as NonEmptyStr, + base_owner: base_owner, + cell_owner: cell_owner, + attempt_owner: attempt_owner, + ), memory_max: memory_max as NonEmptyStr, memory_high: memory_high as NonEmptyStr, memory_swap_max: memory_swap_max as NonEmptyStr, diff --git a/dag/gunbc/systemctl_list_units.dag b/dag/gunbc/systemctl_list_units.dag index 28d33907327..b2ca0c7e2e7 100644 --- a/dag/gunbc/systemctl_list_units.dag +++ b/dag/gunbc/systemctl_list_units.dag @@ -48,16 +48,8 @@ fn systemctl_list_units_active_services(instance_glob: NonEmptyStr) -> List String? { - let result = systemd.Systemctl.ListUnits(pattern: instance_glob, state: systemctl_all_unit_states_selector) + let result = systemd.Systemctl.ListUnitsAllLoaded(pattern: instance_glob) if result.success { Present { value: result.stdout } } else { diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index 375091395cc..f2f3dcb4c78 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -135,16 +135,38 @@ cleanup() { fi local required_receipt local -a required_receipts=( - checkpoint.txt canonical-cleanup.txt generation-cleanup.txt generation-verdict.txt - dependent-lifetime.txt canonical-release.txt canonical-allocation.txt generation-terminal.txt - cell-after-canonical.txt cell-after-dependent.txt cell-terminal.txt - zero-work-before-canonical.txt zero-work-after-canonical.txt zero-work-after-dependent.txt zero-work-terminal.txt - runtime-directory.txt unit-terminal-fci1-positive-submitter.txt + checkpoint.txt canonical-cleanup.txt cell-terminal.txt zero-work-before-canonical.txt + zero-work-terminal.txt runtime-directory.txt unit-terminal-fci1-positive-submitter.txt unit-terminal-fci1-generation-submitter.txt unit-terminal-fci1-dependent-submitter.txt ) + case $checkpoint in + before-canonical-commit|after-canonical-commit-before-transport|after-held-preserved) ;; + after-release-before-grading) + required_receipts+=(canonical-release.txt) + ;; + canonical-restored-later) + required_receipts+=(canonical-release.txt canonical-allocation.txt cell-after-canonical.txt zero-work-after-canonical.txt) + ;; + generation-held-one|generation-authority-two-before-commit|generation-held-two) + required_receipts+=(canonical-release.txt canonical-allocation.txt cell-after-canonical.txt zero-work-after-canonical.txt generation-cleanup.txt) + ;; + generation-changed-observed) + required_receipts+=(canonical-release.txt canonical-allocation.txt cell-after-canonical.txt zero-work-after-canonical.txt generation-cleanup.txt generation-verdict.txt) + ;; + generation-free-before-removal) + required_receipts+=(canonical-release.txt canonical-allocation.txt cell-after-canonical.txt zero-work-after-canonical.txt generation-cleanup.txt generation-verdict.txt) + ;; + runtime-directory-terminal) + required_receipts+=(canonical-release.txt canonical-allocation.txt cell-after-canonical.txt zero-work-after-canonical.txt generation-terminal.txt generation-verdict.txt dependent-lifetime.txt) + ;; + none) + required_receipts+=(canonical-release.txt canonical-allocation.txt cell-after-canonical.txt zero-work-after-canonical.txt generation-terminal.txt generation-verdict.txt dependent-lifetime.txt cell-after-dependent.txt zero-work-after-dependent.txt) + ;; + esac for required_receipt in "${required_receipts[@]}"; do if [[ ! -f $receipt_root/$required_receipt ]]; then - printf 'ReceiptCoordinateNotReached|file=%s|checkpoint=%s\n' "$required_receipt" "$checkpoint" >"$receipt_root/$required_receipt" || cleanup_status=1 + echo "InstrumentRefused: required receipt missing: checkpoint=$checkpoint file=$required_receipt" >&2 + cleanup_status=1 fi done local observed_roster expected_roster @@ -255,7 +277,9 @@ generation_root=$(mktemp -d /run/fci1-generation-mutation.XXXXXX) generation_authority_path="$FABRIC_CI_VALUE_ROOT/generation-cleanup-authority.wire" generation_before=$(capture_submitter_transport fci1-generation-submitter.service fci1_live_reserve_and_observe_available generation-before "$generation_root" "$generation_authority_path") checkpoint_if_selected generation-held-one -fabric_ci_run_assertion fci1_assert_replace_held --arg root="$generation_root" +fabric_ci_run_assertion fci1_assert_replace_held --arg root="$generation_root" \ + --arg cleanup_path="$generation_authority_path" --arg checkpoint="$checkpoint" \ + --arg checkpoint_path="$(receipt_path checkpoint)" checkpoint_if_selected generation-held-two fabric_ci_run_assertion fci1_write_generation_changed_receipt --arg root="$generation_root" --arg before_wire="$generation_before" --arg path="$(receipt_path generation-verdict)" checkpoint_if_selected generation-changed-observed From e322781ef010f39709e4029cdd8c4e54a75f2786 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 01:07:27 +0000 Subject: [PATCH 18/50] Bind instrument processes to derived checkout root --- tools/fabric_ci_evidence_calibration.sh | 2 ++ tools/fabric_ci_fci1_live_instrument.sh | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/fabric_ci_evidence_calibration.sh b/tools/fabric_ci_evidence_calibration.sh index c3f3a0460f0..d3cd366a81c 100644 --- a/tools/fabric_ci_evidence_calibration.sh +++ b/tools/fabric_ci_evidence_calibration.sh @@ -3,6 +3,8 @@ set -euo pipefail # This realizes the ordered typed rows emitted by fabric_ci_calibration_write_plan. # SCAFFOLD — dissolve-on: modeled lifecycle actuation sufficient to sequence a wet gate from .dag. repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +[[ -d $repo_root/.git ]] || { echo 'CalibrationRefused: derived repo_root is not a git checkout' >&2; exit 2; } +cd "$repo_root" || { echo 'CalibrationRefused: cannot enter derived repo_root' >&2; exit 2; } export FABRIC_CI_GUNBC_BIN=${FABRIC_CI_GUNBC_BIN:-"$repo_root/target/release/gunbc"} export FABRIC_CI_SOURCE_ROOT=$repo_root FABRIC_CI_ENTRY="$repo_root/dag/gunbc/instruments/fabric_ci_evidence.dag" export FABRIC_CI_LOG FABRIC_CI_VALUE_ROOT diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index f2f3dcb4c78..70e16eb612a 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -4,6 +4,8 @@ set -euo pipefail # FCI-1 reservation instrument. It starts no Work command. # SCAFFOLD — dissolve-on: modeled lifecycle actuation sufficient to sequence a wet gate from .dag. repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +[[ -d $repo_root/.git ]] || { echo 'InstrumentRefused: derived repo_root is not a git checkout' >&2; exit 2; } +cd "$repo_root" || { echo 'InstrumentRefused: cannot enter derived repo_root' >&2; exit 2; } export FABRIC_CI_GUNBC_BIN="$repo_root/target/release/gunbc" export FABRIC_CI_SOURCE_ROOT="$repo_root" export FABRIC_CI_ENTRY="$repo_root/dag/gunbc/instruments/fabric_control_plane_live_probe.dag" @@ -235,7 +237,7 @@ capture_submitter_transport() { local value_path="$FABRIC_CI_VALUE_ROOT/$coordinate.wire" rm -f -- "$value_path" local run_status=0 - if systemd-run --quiet --wait --collect --unit="$unit" "$@" \ + if systemd-run --quiet --wait --collect --unit="$unit" --property="WorkingDirectory=$repo_root" "$@" \ "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ --entry "$FABRIC_CI_ENTRY" --function "$function_name" --arg root="$root" --arg path="$value_path" \ --arg cleanup_path="$authority_path" --arg checkpoint="$checkpoint" --arg checkpoint_path="$(receipt_path checkpoint)"; then From ab345b86ef03fe063caebf5679b282b9ec9078d6 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 01:11:10 +0000 Subject: [PATCH 19/50] Accept git worktree roots for cwd binding --- tools/fabric_ci_evidence_calibration.sh | 2 +- tools/fabric_ci_fci1_live_instrument.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/fabric_ci_evidence_calibration.sh b/tools/fabric_ci_evidence_calibration.sh index d3cd366a81c..24654c2c67f 100644 --- a/tools/fabric_ci_evidence_calibration.sh +++ b/tools/fabric_ci_evidence_calibration.sh @@ -3,7 +3,7 @@ set -euo pipefail # This realizes the ordered typed rows emitted by fabric_ci_calibration_write_plan. # SCAFFOLD — dissolve-on: modeled lifecycle actuation sufficient to sequence a wet gate from .dag. repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -[[ -d $repo_root/.git ]] || { echo 'CalibrationRefused: derived repo_root is not a git checkout' >&2; exit 2; } +git -C "$repo_root" rev-parse --git-dir >/dev/null 2>&1 || { echo 'CalibrationRefused: derived repo_root is not a git checkout' >&2; exit 2; } cd "$repo_root" || { echo 'CalibrationRefused: cannot enter derived repo_root' >&2; exit 2; } export FABRIC_CI_GUNBC_BIN=${FABRIC_CI_GUNBC_BIN:-"$repo_root/target/release/gunbc"} export FABRIC_CI_SOURCE_ROOT=$repo_root FABRIC_CI_ENTRY="$repo_root/dag/gunbc/instruments/fabric_ci_evidence.dag" diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index 70e16eb612a..2bf07687710 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -4,7 +4,7 @@ set -euo pipefail # FCI-1 reservation instrument. It starts no Work command. # SCAFFOLD — dissolve-on: modeled lifecycle actuation sufficient to sequence a wet gate from .dag. repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -[[ -d $repo_root/.git ]] || { echo 'InstrumentRefused: derived repo_root is not a git checkout' >&2; exit 2; } +git -C "$repo_root" rev-parse --git-dir >/dev/null 2>&1 || { echo 'InstrumentRefused: derived repo_root is not a git checkout' >&2; exit 2; } cd "$repo_root" || { echo 'InstrumentRefused: cannot enter derived repo_root' >&2; exit 2; } export FABRIC_CI_GUNBC_BIN="$repo_root/target/release/gunbc" export FABRIC_CI_SOURCE_ROOT="$repo_root" From 42d4a0f1aae9befa3fd9708d0277a4226ff4a2e9 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 02:30:20 +0000 Subject: [PATCH 20/50] Carry typed properties through systemd transient runs --- dag/extdeps/systemd/systemd_run.dag | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index c072423ddb5..e73eab67ad7 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -5,6 +5,12 @@ import std.types { NonEmptyStr, String, List, Bool } import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } import std.decl_ref { DeclarationRef, WholeDeclaration } import extdeps.uri { Uri, Https } +import extdeps.systemd { SystemdUnitProperty, systemd_unit_property_wire } + +type SystemdRunProperty { + property: SystemdUnitProperty, + value: NonEmptyStr, +} data extdeps_external_authority_anchor: ExternalAuthority = ExternalAuthority { uri: Uri { @@ -27,17 +33,15 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope { data systemd_run_authority: ExternalAuthority = extdeps_external_authority_anchor -fn systemd_run_transient_unit_argv(unit: NonEmptyStr, command_argv: List) -> List { - fold( - command_argv, - init: ["systemd-run", concat("--unit=", unit as String), "--collect", "--"], - f: (acc, arg) => list_push(acc, arg), - ) +fn systemd_run_transient_unit_argv(unit: NonEmptyStr, properties: List, command_argv: List) -> List { + let with_properties = fold(properties, init: ["systemd-run", concat("--unit=", unit as String), "--collect"], f: (acc, setting) => list_push(acc, concat("--property=", concat(systemd_unit_property_wire(property: setting.property) as String, concat("=", setting.value as String))))); + let with_separator = list_push(with_properties, "--"); + fold(command_argv, init: with_separator, f: (acc, arg) => list_push(acc, arg)) } service systemd.SystemdRun { operation RunTransient { - input { unit: NonEmptyStr, command_argv: List } + input { unit: NonEmptyStr, properties: List, command_argv: List } output { success: Bool from "exit_success" stdout: String from "stdout" From ca64d2869be24452c66da392e1d84da8b8ec541e Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 02:43:48 +0000 Subject: [PATCH 21/50] Render typed properties before transient transport --- dag/extdeps/systemd/systemd_run.dag | 4 ++-- dag/gunbc/systemd_run_transient.dag | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index e73eab67ad7..c1d05cc63fa 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -41,13 +41,13 @@ fn systemd_run_transient_unit_argv(unit: NonEmptyStr, properties: List, command_argv: List } + input { unit: NonEmptyStr, command_argv: List } output { success: Bool from "exit_success" stdout: String from "stdout" stderr: String from "stderr" } - transport shell { argv: ["systemd-run", "--unit={unit}", "--collect", "--", command_argv] } + transport shell { argv: ["systemd-run", "--unit={unit}", "--collect", command_argv] } exit { 0 => Unit nonzero => String "systemd-run transient unit start failed" diff --git a/dag/gunbc/systemd_run_transient.dag b/dag/gunbc/systemd_run_transient.dag index f3ae7007667..e304b73618f 100644 --- a/dag/gunbc/systemd_run_transient.dag +++ b/dag/gunbc/systemd_run_transient.dag @@ -29,13 +29,14 @@ type SystemdRunTransientOutcome = | SystemdRunTransientRefused { reason: String } fn systemd_run_transient_operation_argv(unit: NonEmptyStr, command_argv: List) -> ArgvMaterialization { + let rendered_argv = systemd_run_transient_unit_argv(unit: unit, properties: [], command_argv: command_argv) shell_materialize_operation_argv( systemd_run_transient_path, systemd_run_transient_service, systemd_run_transient_operation, [ operation_argv_bind_text(name: "unit", text: unit as String), - operation_argv_bind_text_list(name: "command_argv", items: command_argv), + operation_argv_bind_text_list(name: "command_argv", items: rendered_argv), ] ) } From e916c5469b19983ee1370267cca5884e25aa1cf1 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 03:18:40 +0000 Subject: [PATCH 22/50] Render systemd-run properties into the executed argv The operation declared a typed property population and its transport template ignored it, so a caller asking for a bounded transient unit got a systemd-run command carrying no property at all: the declaration said bounded, the executed command was unbounded. A later spelling passed the full rendered invocation as the operation's command_argv while the transport still prefixed its own launcher words, composing `systemd-run --unit=U --collect systemd-run --unit=U --collect -- `. Both are the same defect -- one operational fact with two independently authored representations -- so the fix gives it one home. systemd_run_property_argv is now the single place a SystemdUnitProperty becomes an argv word; systemd_run_transient_unit_argv composes those words rather than re-deriving them; and the transport owns the launcher words exactly once, splicing property_argv and command_argv as separate list bindings. The typed record deliberately stops at that boundary. push_shell_argv_tokens has arms for Str, List and ProcessArgvExpansion and a refusing arm for ambiguous free monoids, but a record reaches none of them: it lands in the catch-all, which Display-formats the value into one argv word rather than refusing. A record spliced into argv would therefore hand systemd-run a fabricated argument at the exact seam that decides whether a unit is bounded. systemd_run_transient_operation_argv_matches_authority previously rebuilt the launcher list locally, so it could only confirm that two local folds agreed and a property that never reached the executed words was invisible to it. It now compares the materialization against the extdeps authority. Evidence. The existing witness passes an empty population and stays green through exactly this defect, so two controls carry a real one: the materialized argv must equal the authority with a MemoryMax population, and the renderer must produce --property=MemoryMax=17179869184. Verified by execution: required-lane build (regen phase) green on a remote runner with these edits confirmed present -- corpus_load, compile.frontend, normalize, reconcile, analyses, emit, then mirror_write, candidate_verify, adjudicate, hand_verify, digest. That is the corpus-wide parse and name resolution which the prior unthreaded head failed. NOT verified here: the two new witnesses. The witnesses lane cannot execute in any venue available to this session -- its floor phase refuses HostBudgetUnreadable because neither the session container nor the BuildBuddy runner binds a cgroup memory limit, and namespace-wave-admission returns NotEvaluated on a depth-1 clone with no merge base. GitHub-hosted CI is the only venue that satisfies the budget arm, so those controls are enrolled here and judged there. --- dag/extdeps/systemd/systemd_run.dag | 36 +++++++++++++++-- .../host/host_effect_nbd_proxy_serve.dag | 2 +- dag/gunbc/systemd_run_transient.dag | 39 ++++++++++--------- ...nbd_proxy_serve_transport_witness_test.dag | 27 +++++++++++++ 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index c1d05cc63fa..103f8cf0038 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -33,21 +33,49 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope { data systemd_run_authority: ExternalAuthority = extdeps_external_authority_anchor +// THE PROPERTY WORDS ARE RENDERED IN EXACTLY ONE PLACE, and that is the whole point of this +// function existing beside the full-line authority rather than inside it. A systemd unit property +// becomes an argv word here and nowhere else, so the operation's transport, the pure authority and +// every caller are reading one spelling. The transport CANNOT carry the typed record itself: +// push_shell_argv_tokens has arms for Str, List, ProcessArgvExpansion and a refusing arm for +// ambiguous free monoids, and a record reaches none of them — it lands in the catch-all, which +// Display-formats the value into a single argv word rather than refusing. So a record spliced into +// argv would hand systemd-run a fabricated argument at the exact seam that decides whether a unit +// is bounded. The typed population stays on this side of that boundary; only words cross it. +fn systemd_run_property_argv(properties: List) -> List { + fold( + properties, + init: [], + f: (acc, setting) => list_push( + acc, + concat( + "--property=", + concat(systemd_unit_property_wire(property: setting.property) as String, concat("=", setting.value as String)), + ), + ), + ) +} + +// The full invocation, and the authority the materialized operation argv is checked against. It +// COMPOSES the property words rather than re-deriving them; if this fold and the transport template +// ever disagree about word order, systemd_run_transient_operation_argv_matches_authority goes red. fn systemd_run_transient_unit_argv(unit: NonEmptyStr, properties: List, command_argv: List) -> List { - let with_properties = fold(properties, init: ["systemd-run", concat("--unit=", unit as String), "--collect"], f: (acc, setting) => list_push(acc, concat("--property=", concat(systemd_unit_property_wire(property: setting.property) as String, concat("=", setting.value as String))))); - let with_separator = list_push(with_properties, "--"); + let launcher = ["systemd-run", concat("--unit=", unit as String), "--collect"] + let with_properties = fold(systemd_run_property_argv(properties: properties), init: launcher, f: (acc, word) => list_push(acc, word)) + let with_separator = list_push(with_properties, "--") fold(command_argv, init: with_separator, f: (acc, arg) => list_push(acc, arg)) } + service systemd.SystemdRun { operation RunTransient { - input { unit: NonEmptyStr, command_argv: List } + input { unit: NonEmptyStr, property_argv: List, command_argv: List } output { success: Bool from "exit_success" stdout: String from "stdout" stderr: String from "stderr" } - transport shell { argv: ["systemd-run", "--unit={unit}", "--collect", command_argv] } + transport shell { argv: ["systemd-run", "--unit={unit}", "--collect", property_argv, "--", command_argv] } exit { 0 => Unit nonzero => String "systemd-run transient unit start failed" diff --git a/dag/gunbc/host/host_effect_nbd_proxy_serve.dag b/dag/gunbc/host/host_effect_nbd_proxy_serve.dag index 86c5368f1be..9362a133cad 100644 --- a/dag/gunbc/host/host_effect_nbd_proxy_serve.dag +++ b/dag/gunbc/host/host_effect_nbd_proxy_serve.dag @@ -98,7 +98,7 @@ fn host_effect_nbd_proxy_serve_systemd_run_transient( command_argv: List, transport: HostEffectTransport, ) -> String? { - match systemd_run_transient_read(transport: transport, unit: unit, command_argv: command_argv) { + match systemd_run_transient_read(transport: transport, unit: unit, properties: [], command_argv: command_argv) { SystemdRunTransientStarted { stdout: _ } => Absent SystemdRunTransientRefused { reason: why } => Present { value: concat(concat("unit ", unit as String), concat(": ", why)) } diff --git a/dag/gunbc/systemd_run_transient.dag b/dag/gunbc/systemd_run_transient.dag index e304b73618f..3fd2b9a9d97 100644 --- a/dag/gunbc/systemd_run_transient.dag +++ b/dag/gunbc/systemd_run_transient.dag @@ -28,15 +28,15 @@ type SystemdRunTransientOutcome = SystemdRunTransientStarted { stdout: String } | SystemdRunTransientRefused { reason: String } -fn systemd_run_transient_operation_argv(unit: NonEmptyStr, command_argv: List) -> ArgvMaterialization { - let rendered_argv = systemd_run_transient_unit_argv(unit: unit, properties: [], command_argv: command_argv) +fn systemd_run_transient_operation_argv(unit: NonEmptyStr, properties: List, command_argv: List) -> ArgvMaterialization { shell_materialize_operation_argv( systemd_run_transient_path, systemd_run_transient_service, systemd_run_transient_operation, [ operation_argv_bind_text(name: "unit", text: unit as String), - operation_argv_bind_text_list(name: "command_argv", items: rendered_argv), + operation_argv_bind_text_list(name: "property_argv", items: systemd_run_property_argv(properties: properties)), + operation_argv_bind_text_list(name: "command_argv", items: command_argv), ] ) } @@ -49,8 +49,8 @@ fn systemd_run_transient_outcome_from_result(success: Bool, stdout: String, stde } } -fn systemd_run_transient_local(unit: NonEmptyStr, command_argv: List) -> SystemdRunTransientOutcome { - let result = systemd.SystemdRun.RunTransient(unit: unit, command_argv: command_argv) +fn systemd_run_transient_local(unit: NonEmptyStr, properties: List, command_argv: List) -> SystemdRunTransientOutcome { + let result = systemd.SystemdRun.RunTransient(unit: unit, property_argv: systemd_run_property_argv(properties: properties), command_argv: command_argv) systemd_run_transient_outcome_from_result(success: result.success, stdout: result.stdout, stderr: result.stderr) } @@ -69,8 +69,8 @@ fn systemd_run_transient_ssh_argv(host: NonEmptyStr, argv: List) -> Syst } } -fn systemd_run_transient_ssh(host: NonEmptyStr, unit: NonEmptyStr, command_argv: List) -> SystemdRunTransientOutcome { - match systemd_run_transient_operation_argv(unit: unit, command_argv: command_argv) { +fn systemd_run_transient_ssh(host: NonEmptyStr, unit: NonEmptyStr, properties: List, command_argv: List) -> SystemdRunTransientOutcome { + match systemd_run_transient_operation_argv(unit: unit, properties: properties, command_argv: command_argv) { ArgvMaterialized { argv: argv } => systemd_run_transient_ssh_argv(host: host, argv: argv) ArgvMaterializationRefused { at: _, cause: c } => SystemdRunTransientRefused { reason: operation_argv_cause_label(cause: c) } @@ -81,9 +81,10 @@ fn systemd_run_transient_fleet_ssh( target: SshTarget, context: FleetSshExecutionContext, unit: NonEmptyStr, + properties: List, command_argv: List, ) -> SystemdRunTransientOutcome { - match systemd_run_transient_operation_argv(unit: unit, command_argv: command_argv) { + match systemd_run_transient_operation_argv(unit: unit, properties: properties, command_argv: command_argv) { ArgvMaterialized { argv: argv } => systemd_run_transient_outcome_from_exec( outcome: typed_argv_exec_over_fleet_ssh(target: target, context: context, argv: argv), @@ -96,12 +97,13 @@ fn systemd_run_transient_fleet_ssh( fn systemd_run_transient_read( transport: HostEffectTransport, unit: NonEmptyStr, + properties: List, command_argv: List, ) -> SystemdRunTransientOutcome { match transport { - LocalShell => systemd_run_transient_local(unit: unit, command_argv: command_argv) - SshShell { ssh_host: h } => systemd_run_transient_ssh(host: h, unit: unit, command_argv: command_argv) - FleetSsh { target: t, context: c } => systemd_run_transient_fleet_ssh(target: t, context: c, unit: unit, command_argv: command_argv) + LocalShell => systemd_run_transient_local(unit: unit, properties: properties, command_argv: command_argv) + SshShell { ssh_host: h } => systemd_run_transient_ssh(host: h, unit: unit, properties: properties, command_argv: command_argv) + FleetSsh { target: t, context: c } => systemd_run_transient_fleet_ssh(target: t, context: c, unit: unit, properties: properties, command_argv: command_argv) EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => SystemdRunTransientRefused { reason: "systemd-run transient start refused EmitArtifactThenThinRun transport", @@ -109,13 +111,14 @@ fn systemd_run_transient_read( } } -fn systemd_run_transient_operation_argv_matches_authority(unit: NonEmptyStr, command_argv: List) -> Bool { - let authority = fold( - command_argv, - init: ["systemd-run", concat("--unit=", unit as String), "--collect", "--"], - f: (acc, arg) => list_push(acc, arg), - ) - match systemd_run_transient_operation_argv(unit: unit, command_argv: command_argv) { +// THE MATERIALIZED ARGV IS CHECKED AGAINST THE EXTDEPS AUTHORITY, not against a second hand-built +// copy of it. The prior spelling rebuilt the launcher words here, so this predicate could only ever +// confirm that two local folds agreed; a property population would have been invisible to it. It +// now compares against systemd_run_transient_unit_argv, which is the same function the transport +// template must agree with, so a property that fails to reach the executed words goes red here. +fn systemd_run_transient_operation_argv_matches_authority(unit: NonEmptyStr, properties: List, command_argv: List) -> Bool { + let authority = systemd_run_transient_unit_argv(unit: unit, properties: properties, command_argv: command_argv) + match systemd_run_transient_operation_argv(unit: unit, properties: properties, command_argv: command_argv) { ArgvMaterialized { argv: materialized } => join(materialized, " ") == join(authority, " ") ArgvMaterializationRefused { at: _, cause: _ } => false } diff --git a/dag/test/claim/nbd_proxy_serve_transport_witness_test.dag b/dag/test/claim/nbd_proxy_serve_transport_witness_test.dag index ddef8397c01..05abaa50742 100644 --- a/dag/test/claim/nbd_proxy_serve_transport_witness_test.dag +++ b/dag/test/claim/nbd_proxy_serve_transport_witness_test.dag @@ -195,10 +195,37 @@ test fn nbd_proxy_observe_port_local_shell_is_active_active_wires_through() -> B test fn witness_systemd_run_transient_operation_argv_matches_authority() -> Bool { systemd_run_transient_operation_argv_matches_authority( unit: "nbd-proxy-srv3-nbd-proxy-10809" as NonEmptyStr, + properties: [], command_argv: ["nbdkit", "-p", "10809", "file", "/var/lib/gunbc/artifacts/x.iso"], ) } +// THE EMPTY POPULATION ABOVE CANNOT SEE A PROPERTY THAT NEVER REACHES THE EXECUTED WORDS, which is +// exactly the defect measured on this seam: an operation may DECLARE a property input while its +// transport template ignores it, and every empty-population caller stays green while a bounded unit +// is launched unbounded. This control carries a real population so the materialized argv and the +// extdeps authority must agree on the property word itself. +test fn witness_systemd_run_transient_property_reaches_materialized_argv() -> Bool { + systemd_run_transient_operation_argv_matches_authority( + unit: "nbd-proxy-srv3-nbd-proxy-10809" as NonEmptyStr, + properties: [SystemdRunProperty { property: MemoryMax, value: "17179869184" as NonEmptyStr }], + command_argv: ["nbdkit", "-p", "10809", "file", "/var/lib/gunbc/artifacts/x.iso"], + ) +} + +// A PREDICATE THAT ONLY EVER RETURNS TRUE IS NOT A CONTROL. The two witnesses above compare one +// authority against one materialization; if the transport template silently dropped the property +// words, BOTH sides would drop them together only if the authority also stopped rendering them -- +// which this asserts it does not. The property word must be present in the authority's own output. +test fn witness_systemd_run_property_argv_renders_the_wire_name() -> Bool { + join( + systemd_run_property_argv( + properties: [SystemdRunProperty { property: MemoryMax, value: "17179869184" as NonEmptyStr }], + ), + " ", + ) == "--property=MemoryMax=17179869184" +} + test fn witness_systemctl_stop_operation_argv_matches_authority() -> Bool { systemctl_stop_operation_argv_matches_authority(unit: "nbd-proxy-srv3-nbd-proxy-10809" as NonEmptyStr) } From 93b618f98aaedd6a7744fbd00a32d4348b3f8b08 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 05:59:44 +0000 Subject: [PATCH 23/50] Compare argv by token identity, not by joined text systemd_run_transient_operation_argv_matches_authority compared join(materialized, " ") == join(authority, " "), which is a question about text where the claim is about tokens. One word carrying a space and the words it would split into collapse to the same string, so ["--property=WorkingDirectory=/path with space"] ["--property=WorkingDirectory=/path", "with", "space"] compared equal. That is blind at exactly the seam that carries paths and property values, and it is the seam the bounded-driver work is about to load. There is no zip or index in the list vocabulary to fold two lists in lockstep, so equality is established the way an encoding establishes it: cardinality must agree, and the words are joined on a separator no word contains, which makes the joined form injective. A word containing the separator REFUSES rather than falling back to the ambiguous comparison -- an unusable encoding is an unanswerable question, not a passing one. Three controls, two of which the prior comparison could not express: a WorkingDirectory value containing a space keeps token identity; ["a b"] and ["a", "b"] must compare unequal while ["a b"] equals itself; and a word carrying the separator refuses, asserting the encoding's own precondition rather than assuming it. The same join-comparison stands in systemctl_stop_operation_argv_matches_authority and hostname_short_read_operation_argv_matches_authority. Same class, same blindness; not repaired here because they are not this branch's subject, and noted so the class is recorded rather than rediscovered. Verified by execution on the merged tree: claim_executor --required-ci --required-lane build, REAL_EXIT=0, planned=150, first_generation_equal=true, generated-artifact rostered=35 adjudicated=35 matches=35 drifted=0, phases_run=2 failed=0. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01PGkDt1W1m3U28vBpV6BY9Z --- dag/gunbc/systemd_run_transient.dag | 33 ++++++++++++++++--- ...nbd_proxy_serve_transport_witness_test.dag | 21 ++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/systemd_run_transient.dag b/dag/gunbc/systemd_run_transient.dag index 3fd2b9a9d97..255e20cc30e 100644 --- a/dag/gunbc/systemd_run_transient.dag +++ b/dag/gunbc/systemd_run_transient.dag @@ -111,15 +111,38 @@ fn systemd_run_transient_read( } } +// ARGV EQUALITY IS A QUESTION ABOUT TOKENS, AND join() ANSWERS A QUESTION ABOUT TEXT. +// The prior spelling compared join(materialized, " ") == join(authority, " "), which cannot tell +// ["--property=WorkingDirectory=/path with space"] from ["...=/path", "with", "space"]: one word +// carrying a space and three words collapse to the same string, so an accidental token split at +// exactly the seam that carries paths and property values was invisible. There is no zip or index +// in the list vocabulary to fold two lists in lockstep, so equality is established the way an +// encoding establishes it: cardinality must agree, and the words must be joined on a separator no +// word contains, which makes the joined form injective. A word containing the separator REFUSES +// rather than falling back to the ambiguous comparison -- an unusable encoding is an unanswerable +// question, not a passing one. +data argv_token_separator: String = "\n" + +fn argv_word_free_of_separator(words: List) -> Bool { + fold(words, init: true, f: (acc, w) => acc && !string_contains(s: w, pattern: argv_token_separator)) +} + +fn argv_exact_token_equal(left: List, right: List) -> Bool { + list_length(items: left) == list_length(items: right) + && argv_word_free_of_separator(words: left) + && argv_word_free_of_separator(words: right) + && join(left, argv_token_separator) == join(right, argv_token_separator) +} + // THE MATERIALIZED ARGV IS CHECKED AGAINST THE EXTDEPS AUTHORITY, not against a second hand-built -// copy of it. The prior spelling rebuilt the launcher words here, so this predicate could only ever -// confirm that two local folds agreed; a property population would have been invisible to it. It -// now compares against systemd_run_transient_unit_argv, which is the same function the transport -// template must agree with, so a property that fails to reach the executed words goes red here. +// copy of it. An earlier spelling rebuilt the launcher words here, so this predicate could only +// confirm that two local folds agreed; a property population was invisible to it. It now compares +// against systemd_run_transient_unit_argv, which is the same function the transport template must +// agree with, so a property that fails to reach the executed words goes red here. fn systemd_run_transient_operation_argv_matches_authority(unit: NonEmptyStr, properties: List, command_argv: List) -> Bool { let authority = systemd_run_transient_unit_argv(unit: unit, properties: properties, command_argv: command_argv) match systemd_run_transient_operation_argv(unit: unit, properties: properties, command_argv: command_argv) { - ArgvMaterialized { argv: materialized } => join(materialized, " ") == join(authority, " ") + ArgvMaterialized { argv: materialized } => argv_exact_token_equal(left: materialized, right: authority) ArgvMaterializationRefused { at: _, cause: _ } => false } } diff --git a/dag/test/claim/nbd_proxy_serve_transport_witness_test.dag b/dag/test/claim/nbd_proxy_serve_transport_witness_test.dag index 05abaa50742..b4c2fd68950 100644 --- a/dag/test/claim/nbd_proxy_serve_transport_witness_test.dag +++ b/dag/test/claim/nbd_proxy_serve_transport_witness_test.dag @@ -213,6 +213,27 @@ test fn witness_systemd_run_transient_property_reaches_materialized_argv() -> Bo ) } +// THE WHITESPACE ARM IS THE ONE THE OLD COMPARISON COULD NOT SEE. join(a," ")==join(b," ") cannot +// distinguish one word carrying a space from the words it would split into, so a value like a path +// with a space passed the predicate whether or not the transport tokenised it correctly. Carrying +// such a value here means the control fails if argv agreement is ever weakened back to a text +// comparison: the words still join to the same text, and only cardinality separates them. +test fn witness_systemd_run_property_value_with_space_keeps_token_identity() -> Bool { + systemd_run_transient_operation_argv_matches_authority( + unit: "nbd-proxy-srv3-nbd-proxy-10809" as NonEmptyStr, + properties: [SystemdRunProperty { property: WorkingDirectoryProperty, value: "/srv/path with space" as NonEmptyStr }], + command_argv: ["nbdkit", "-p", "10809"], + ) +} + +// THE ENCODING'S OWN PRECONDITION IS ASSERTED, not assumed. argv_exact_token_equal is injective +// only while no word contains the separator; a word that does must refuse rather than compare. +test fn witness_argv_exact_token_equal_separates_split_words() -> Bool { + !argv_exact_token_equal(left: ["a b"], right: ["a", "b"]) + && argv_exact_token_equal(left: ["a b"], right: ["a b"]) + && !argv_exact_token_equal(left: ["a\nb"], right: ["a\nb"]) +} + // A PREDICATE THAT ONLY EVER RETURNS TRUE IS NOT A CONTROL. The two witnesses above compare one // authority against one materialization; if the transport template silently dropped the property // words, BOTH sides would drop them together only if the authority also stopped rendering them -- From 251377112a2182a63be1110f22f95fdb190753a2 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 06:07:48 +0000 Subject: [PATCH 24/50] Model the bounded FCI-1 execution context --- .../fabric/fci1_bounded_execution_context.dag | 95 +++++++++++++++++++ ...bounded_execution_context_witness_test.dag | 37 ++++++++ 2 files changed, 132 insertions(+) create mode 100644 dag/gunbc/fabric/fci1_bounded_execution_context.dag create mode 100644 dag/test/claim/fabric/fci1_bounded_execution_context_witness_test.dag diff --git a/dag/gunbc/fabric/fci1_bounded_execution_context.dag b/dag/gunbc/fabric/fci1_bounded_execution_context.dag new file mode 100644 index 00000000000..745c73ccad7 --- /dev/null +++ b/dag/gunbc/fabric/fci1_bounded_execution_context.dag @@ -0,0 +1,95 @@ +module gunbc.fci1_bounded_execution_context + +import std.types { FilePath, Int, NonEmptyStr } +import std.measure { + Measure, Memory, One, ByteSize, byte_size_count, + PositiveMeasureCount, PositiveMeasureCountBuilt, PositiveMeasureCountRefused, + positive_measure_count_from_int, positive_measure_count_value, +} +import std.decl_ref { DeclarationRef, WholeDeclaration } +import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } +import v2.std.optional { Present } + +// A finite byte quantity is not a second memory model. ByteSize already rules out systemd's +// `infinity` spelling by carrying a Nat; using the existing positive-count construction also makes +// zero unrepresentable. The FCI-1 context can therefore be projected to a live cgroup bound +// without a validator deciding whether its purported limit is actually a limit. +type FiniteByteSize = Measure + +fn finite_byte_size(count: PositiveMeasureCount) -> FiniteByteSize { + Measure { count: count } +} + +fn finite_byte_size_count(size: FiniteByteSize) -> Int { + positive_measure_count_value(count: size.count) +} + +type FiniteByteSizeBuild + = FiniteByteSizeBuilt { size: FiniteByteSize } + | FiniteByteSizeNonPositive { observed: Int } + +fn finite_byte_size_from_byte_size(size: ByteSize) -> FiniteByteSizeBuild { + let observed = byte_size_count(b: size) + match positive_measure_count_from_int(count: observed) { + PositiveMeasureCountBuilt { count: positive } => + FiniteByteSizeBuilt { size: finite_byte_size(count: positive) } + PositiveMeasureCountRefused { cause: _ } => + FiniteByteSizeNonPositive { observed: observed } + } +} + +// This is the bound for the instrument process roots, not a claim that the driver is a runner +// slot. The basis names why the values may be reused: the driver performs sequential gunbc +// resolution over the same dag + src/v2 corpus family whose measured floor workload determines +// gunbc_runner_slot_desired. A bounded no-reservation control must still prove that this realization +// completes under the projected envelope; failure triggers a separately measured driver envelope, +// never an ungrounded widening of the runner allocation. +type Fci1BoundedExecutionContext sole_constructor { + checkout_root: FilePath + memory_max: FiniteByteSize + memory_high: FiniteByteSize? + driver_unit: NonEmptyStr + envelope_basis: DeclarationRef +} + +type Fci1BoundedExecutionContextBuild + = Fci1BoundedExecutionContextBuilt { context: Fci1BoundedExecutionContext } + | Fci1MemoryMaxNonPositive { observed: Int } + | Fci1MemoryHighNonPositive { observed: Int } + | Fci1MemoryHighExceedsMax { high_bytes: Int, max_bytes: Int } + +data fci1_driver_unit: NonEmptyStr = "fci1-bounded-driver.service" + +data fci1_runner_envelope_basis: DeclarationRef = DeclarationRef { + module_path: "gunbc.runner_slot_allocation", + decl_name: "gunbc_runner_slot_desired", + field: WholeDeclaration, +} + +fn fci1_bounded_execution_context(checkout_root: FilePath) -> Fci1BoundedExecutionContextBuild { + let desired = gunbc_runner_slot_desired() + match finite_byte_size_from_byte_size(size: desired.memory_max) { + FiniteByteSizeNonPositive { observed: max } => + Fci1MemoryMaxNonPositive { observed: max } + FiniteByteSizeBuilt { size: max } => + match finite_byte_size_from_byte_size(size: desired.memory_high) { + FiniteByteSizeNonPositive { observed: high } => + Fci1MemoryHighNonPositive { observed: high } + FiniteByteSizeBuilt { size: high } => + if finite_byte_size_count(size: high) > finite_byte_size_count(size: max) { + Fci1MemoryHighExceedsMax { + high_bytes: finite_byte_size_count(size: high), + max_bytes: finite_byte_size_count(size: max), + } + } else { + Fci1BoundedExecutionContextBuilt { context: Fci1BoundedExecutionContext { + checkout_root: checkout_root, + memory_max: max, + memory_high: Present { value: high }, + driver_unit: fci1_driver_unit, + envelope_basis: fci1_runner_envelope_basis, + } } + } + } + } +} diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_witness_test.dag new file mode 100644 index 00000000000..2bd056f4e7a --- /dev/null +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_witness_test.dag @@ -0,0 +1,37 @@ +module test.claim.fabric.fci1_bounded_execution_context_witness_test + +import std.types { Bool } +import std.measure { byte_size, byte_size_count } +import gunbc.fci1_bounded_execution_context { + FiniteByteSizeBuilt, FiniteByteSizeNonPositive, + Fci1BoundedExecutionContextBuilt, + finite_byte_size_count, finite_byte_size_from_byte_size, + fci1_bounded_execution_context, +} +import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } +import v2.std.optional { Present, Absent } + +fn finite_byte_size_refuses_zero() -> Bool { + match finite_byte_size_from_byte_size(size: byte_size(count: 0)) { + FiniteByteSizeNonPositive { observed: zero } => zero == 0 + FiniteByteSizeBuilt { size: _ } => false + } +} + +fn fci1_context_derives_the_runner_workload_envelope() -> Bool { + match fci1_bounded_execution_context(checkout_root: "/tmp/fci1 checkout") { + Fci1BoundedExecutionContextBuilt { context: context } => + finite_byte_size_count(size: context.memory_max) + == byte_size_count(b: gunbc_runner_slot_desired().memory_max) + && match context.memory_high { + Present { value: high } => + finite_byte_size_count(size: high) + == byte_size_count(b: gunbc_runner_slot_desired().memory_high) + Absent => false + } + && context.checkout_root == "/tmp/fci1 checkout" + && context.envelope_basis.module_path == "gunbc.runner_slot_allocation" + && context.envelope_basis.decl_name == "gunbc_runner_slot_desired" + _ => false + } +} From a396af6b746fe77fe23187a9a99059f450051cd2 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 06:12:15 +0000 Subject: [PATCH 25/50] Project the FCI-1 context to systemd --- ...fci1_bounded_execution_context_systemd.dag | 36 +++++++++++++ ...execution_context_systemd_witness_test.dag | 52 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag create mode 100644 dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag diff --git a/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag b/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag new file mode 100644 index 00000000000..ea5360182c8 --- /dev/null +++ b/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag @@ -0,0 +1,36 @@ +module gunbc.fci1_bounded_execution_context_systemd + +import std.types { List, NonEmptyStr, String } +import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh } +import extdeps.systemd.systemd_run { SystemdRunProperty } +import gunbc.fci1_bounded_execution_context { + Fci1BoundedExecutionContext, finite_byte_size_count, +} +import v2.std.optional { Present, Absent } + +// This is realization, not part of either the product context or the systemd interface. It is the +// only binding from the FCI-1 envelope to transient-unit properties. Callers cannot supply a +// property population, so duplicate WorkingDirectory/MemoryMax settings and observation-only +// properties have no constructor on this path. Byte spellings are projections of FiniteByteSize, +// never independently authored strings. +fn fci1_bounded_context_systemd_properties( + context: Fci1BoundedExecutionContext, +) -> List { + let required = [ + SystemdRunProperty { + property: WorkingDirectoryProperty, + value: context.checkout_root as String as NonEmptyStr, + }, + SystemdRunProperty { + property: MemoryMax, + value: to_string(finite_byte_size_count(size: context.memory_max)) as NonEmptyStr, + }, + ] + match context.memory_high { + Present { value: high } => list_push(required, SystemdRunProperty { + property: MemoryHigh, + value: to_string(finite_byte_size_count(size: high)) as NonEmptyStr, + }) + Absent => required + } +} diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag new file mode 100644 index 00000000000..9ba4a3c3be9 --- /dev/null +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag @@ -0,0 +1,52 @@ +module test.claim.fabric.fci1_bounded_execution_context_systemd_witness_test + +import std.types { Bool, list_length } +import std.measure { byte_size_count } +import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh } +import extdeps.systemd.systemd_run { SystemdRunProperty } +import gunbc.fci1_bounded_execution_context { + Fci1BoundedExecutionContextBuilt, fci1_bounded_execution_context, +} +import gunbc.fci1_bounded_execution_context_systemd { + fci1_bounded_context_systemd_properties, +} +import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } + +fn property_is_working_directory(property: SystemdRunProperty) -> Bool { + match property.property { WorkingDirectoryProperty => true _ => false } +} + +fn property_is_memory_max(property: SystemdRunProperty) -> Bool { + match property.property { MemoryMax => true _ => false } +} + +fn property_is_memory_high(property: SystemdRunProperty) -> Bool { + match property.property { MemoryHigh => true _ => false } +} + +fn fci1_systemd_projection_is_the_exact_closed_population() -> Bool { + match fci1_bounded_execution_context(checkout_root: "/tmp/fci1 checkout") { + Fci1BoundedExecutionContextBuilt { context: context } => + let properties = fci1_bounded_context_systemd_properties(context: context) + list_length(items: properties) == 3 + && list_length(items: filter(properties, property_is_working_directory)) == 1 + && list_length(items: filter(properties, property_is_memory_max)) == 1 + && list_length(items: filter(properties, property_is_memory_high)) == 1 + && any(properties, property => match property { + SystemdRunProperty { property: WorkingDirectoryProperty, value: value } => + value == "/tmp/fci1 checkout" + _ => false + }) + && any(properties, property => match property { + SystemdRunProperty { property: MemoryMax, value: value } => + value == to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_max)) + _ => false + }) + && any(properties, property => match property { + SystemdRunProperty { property: MemoryHigh, value: value } => + value == to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_high)) + _ => false + }) + _ => false + } +} From 338ab82b581118711bed1575869b10a9b49eabef Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 06:19:38 +0000 Subject: [PATCH 26/50] Preserve exact exits from waited transient units --- dag/extdeps/systemd/systemd_run.dag | 32 ++++- dag/gunbc/systemd_run_transient.dag | 134 +++++++++++++++++- ...ystemd_run_transient_wait_witness_test.dag | 56 ++++++++ 3 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 dag/test/claim/fabric/systemd_run_transient_wait_witness_test.dag diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index 103f8cf0038..9e60924923a 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -1,6 +1,6 @@ module extdeps.systemd.systemd_run -import std.types { NonEmptyStr, String, List, Bool } +import std.types { NonEmptyStr, String, List, Bool, Int } import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } import std.decl_ref { DeclarationRef, WholeDeclaration } @@ -23,7 +23,7 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope { subject: ExternalSubjectRef { declaration: DeclarationRef { module_path: "extdeps.systemd.systemd_run", - decl_name: "systemd_run_transient_unit_argv", + decl_name: "SystemdRun", field: WholeDeclaration } }, @@ -66,6 +66,19 @@ fn systemd_run_transient_unit_argv(unit: NonEmptyStr, properties: List list_push(acc, arg)) } +// Waiting is a different operation from starting. It owns the complete wait/quiet/collect +// vocabulary and returns the process status as data: a nonzero child status is an observation, +// not an operation refusal. Default transient service semantics are deliberate. Type=oneshot is +// excluded because it collapses normal nonzero statuses to 1 (systemd issue 22812, observed on +// systemd 245 / 245.4-4ubuntu3.15); the same report establishes that signal death is not reliably +// recoverable as an exact child status through this boundary. +fn systemd_run_transient_wait_unit_argv(unit: NonEmptyStr, properties: List, command_argv: List) -> List { + let launcher = ["systemd-run", concat("--unit=", unit as String), "--wait", "--quiet", "--collect"] + let with_properties = fold(systemd_run_property_argv(properties: properties), init: launcher, f: (acc, word) => list_push(acc, word)) + let with_separator = list_push(with_properties, "--") + fold(command_argv, init: with_separator, f: (acc, arg) => list_push(acc, arg)) +} + service systemd.SystemdRun { operation RunTransient { @@ -84,4 +97,19 @@ service systemd.SystemdRun { 0 => { success: true, stdout: "Running as unit: demo.service", stderr: "" } "hermetic systemd.SystemdRun.RunTransient" } } + + + operation RunTransientAndWait { + input { unit: NonEmptyStr, property_argv: List, command_argv: List } + output { + exit_code: Int from "exit_code" + stdout: String from "stdout" + stderr: String from "stderr" + } + transport shell { argv: ["systemd-run", "--unit={unit}", "--wait", "--quiet", "--collect", property_argv, "--", command_argv] } + exit { + 0 => Unit + nonzero => Unit + } + } } diff --git a/dag/gunbc/systemd_run_transient.dag b/dag/gunbc/systemd_run_transient.dag index 255e20cc30e..25b6b8ebadd 100644 --- a/dag/gunbc/systemd_run_transient.dag +++ b/dag/gunbc/systemd_run_transient.dag @@ -1,6 +1,6 @@ module gunbc.systemd_run_transient -import std.types { String, NonEmptyStr, List, Bool } +import std.types { String, NonEmptyStr, List, Bool, Int } import extdeps.systemd.systemd_run import gunbc.host_effect { HostEffectTransport, LocalShell, SshShell, EmitArtifactThenThinRun } import v2.std.operation_argv { @@ -16,6 +16,7 @@ import gunbc.typed_argv_exec { typed_argv_exec_over_fleet_ssh, typed_argv_exec_over_ssh, typed_argv_shell_safe_refusal_argv, + TypedArgvExecOutcome, TypedArgvExecConverged, TypedArgvExecRefused, } @@ -23,11 +24,16 @@ import gunbc.typed_argv_exec { data systemd_run_transient_path: String = "dag/extdeps/systemd/systemd_run.dag" data systemd_run_transient_service: String = "systemd.SystemdRun" data systemd_run_transient_operation: String = "RunTransient" +data systemd_run_transient_wait_operation: String = "RunTransientAndWait" type SystemdRunTransientOutcome = SystemdRunTransientStarted { stdout: String } | SystemdRunTransientRefused { reason: String } +type SystemdRunTransientWaitOutcome + = SystemdRunTransientWaitCompleted { exit_code: Int, stdout: String, stderr: String } + | SystemdRunTransientWaitTransportRefused { reason: String } + fn systemd_run_transient_operation_argv(unit: NonEmptyStr, properties: List, command_argv: List) -> ArgvMaterialization { shell_materialize_operation_argv( systemd_run_transient_path, @@ -41,6 +47,19 @@ fn systemd_run_transient_operation_argv(unit: NonEmptyStr, properties: List, command_argv: List) -> ArgvMaterialization { + shell_materialize_operation_argv( + systemd_run_transient_path, + systemd_run_transient_service, + systemd_run_transient_wait_operation, + [ + operation_argv_bind_text(name: "unit", text: unit as String), + operation_argv_bind_text_list(name: "property_argv", items: systemd_run_property_argv(properties: properties)), + operation_argv_bind_text_list(name: "command_argv", items: command_argv), + ] + ) +} + fn systemd_run_transient_outcome_from_result(success: Bool, stdout: String, stderr: String) -> SystemdRunTransientOutcome { if success { SystemdRunTransientStarted { stdout: stdout } @@ -111,6 +130,111 @@ fn systemd_run_transient_read( } } +fn systemd_run_transient_wait_local( + unit: NonEmptyStr, + properties: List, + command_argv: List, +) -> SystemdRunTransientWaitOutcome { + let result = systemd.SystemdRun.RunTransientAndWait( + unit: unit, + property_argv: systemd_run_property_argv(properties: properties), + command_argv: command_argv, + ) + systemd_run_transient_wait_outcome_from_result( + exit_code: result.exit_code, + stdout: result.stdout, + stderr: result.stderr, + ) +} + +fn systemd_run_transient_wait_outcome_from_result( + exit_code: Int, + stdout: String, + stderr: String, +) -> SystemdRunTransientWaitOutcome { + SystemdRunTransientWaitCompleted { + exit_code: exit_code, + stdout: stdout, + stderr: stderr, + } +} + +fn systemd_run_transient_wait_outcome_from_exec( + outcome: TypedArgvExecOutcome, +) -> SystemdRunTransientWaitOutcome { + match outcome { + TypedArgvExecConverged { result: result } => + systemd_run_transient_wait_outcome_from_result( + exit_code: result.exit_code, + stdout: result.stdout, + stderr: result.stderr, + ) + TypedArgvExecRefused { reason: why } => + SystemdRunTransientWaitTransportRefused { reason: why } + } +} + +fn systemd_run_transient_wait_ssh_argv( + host: NonEmptyStr, + argv: List, +) -> SystemdRunTransientWaitOutcome { + match typed_argv_shell_safe_refusal_argv(argv: argv) { + Present { value: why } => SystemdRunTransientWaitTransportRefused { reason: why } + Absent => systemd_run_transient_wait_outcome_from_exec( + outcome: typed_argv_exec_over_ssh(host: host, argv: argv), + ) + } +} + +fn systemd_run_transient_wait_ssh( + host: NonEmptyStr, + unit: NonEmptyStr, + properties: List, + command_argv: List, +) -> SystemdRunTransientWaitOutcome { + match systemd_run_transient_wait_operation_argv( + unit: unit, + properties: properties, + command_argv: command_argv, + ) { + ArgvMaterialized { argv: argv } => systemd_run_transient_wait_ssh_argv(host: host, argv: argv) + ArgvMaterializationRefused { at: _, cause: cause } => + SystemdRunTransientWaitTransportRefused { reason: operation_argv_cause_label(cause: cause) } + } +} + +// This boundary preserves normal exit codes such as FCI-1's 0 and 86. It does not claim that a +// signal death is an exact ProcessSignaled observation: systemd-run --wait has reported success for +// signal-killed default transient services (systemd/systemd issue 22812, systemd 245). A consumer +// needing that distinction must join a richer manager/process observation; it may not fabricate an +// integer from this result. +fn systemd_run_transient_wait_read( + transport: HostEffectTransport, + unit: NonEmptyStr, + properties: List, + command_argv: List, +) -> SystemdRunTransientWaitOutcome { + match transport { + LocalShell => systemd_run_transient_wait_local( + unit: unit, + properties: properties, + command_argv: command_argv, + ) + SshShell { ssh_host: host } => systemd_run_transient_wait_ssh( + host: host, + unit: unit, + properties: properties, + command_argv: command_argv, + ) + FleetSsh { target: _, context: _ } => SystemdRunTransientWaitTransportRefused { + reason: "systemd-run transient wait refused FleetSsh until exact remote status is distinguished from transport status", + } + EmitArtifactThenThinRun { bootstrap: _, invocation: _ } => SystemdRunTransientWaitTransportRefused { + reason: "systemd-run transient wait refused EmitArtifactThenThinRun transport", + } + } +} + // ARGV EQUALITY IS A QUESTION ABOUT TOKENS, AND join() ANSWERS A QUESTION ABOUT TEXT. // The prior spelling compared join(materialized, " ") == join(authority, " "), which cannot tell // ["--property=WorkingDirectory=/path with space"] from ["...=/path", "with", "space"]: one word @@ -146,3 +270,11 @@ fn systemd_run_transient_operation_argv_matches_authority(unit: NonEmptyStr, pro ArgvMaterializationRefused { at: _, cause: _ } => false } } + +fn systemd_run_transient_wait_operation_argv_matches_authority(unit: NonEmptyStr, properties: List, command_argv: List) -> Bool { + let authority = systemd_run_transient_wait_unit_argv(unit: unit, properties: properties, command_argv: command_argv) + match systemd_run_transient_wait_operation_argv(unit: unit, properties: properties, command_argv: command_argv) { + ArgvMaterialized { argv: materialized } => argv_exact_token_equal(left: materialized, right: authority) + ArgvMaterializationRefused { at: _, cause: _ } => false + } +} diff --git a/dag/test/claim/fabric/systemd_run_transient_wait_witness_test.dag b/dag/test/claim/fabric/systemd_run_transient_wait_witness_test.dag new file mode 100644 index 00000000000..4db04535390 --- /dev/null +++ b/dag/test/claim/fabric/systemd_run_transient_wait_witness_test.dag @@ -0,0 +1,56 @@ +module test.claim.fabric.systemd_run_transient_wait_witness_test + +import std.types { Bool, List } +import extdeps.systemd { WorkingDirectoryProperty, MemoryMax } +import extdeps.systemd.systemd_run { SystemdRunProperty, systemd_run_transient_wait_unit_argv } +import gunbc.systemd_run_transient { + systemd_run_transient_wait_operation_argv_matches_authority, + systemd_run_transient_wait_outcome_from_result, + SystemdRunTransientWaitCompleted, + SystemdRunTransientWaitTransportRefused, + argv_exact_token_equal, +} + +data wait_properties: List = [ + SystemdRunProperty { property: WorkingDirectoryProperty, value: "/tmp/path with space" }, + SystemdRunProperty { property: MemoryMax, value: "17179869184" }, +] + +fn systemd_wait_route_owns_wait_quiet_collect_and_preserves_tokens() -> Bool { + let argv = systemd_run_transient_wait_unit_argv( + unit: "fci1-bounded-driver.service", + properties: wait_properties, + command_argv: ["/bin/sh", "-c", "exit 86"], + ) + systemd_run_transient_wait_operation_argv_matches_authority( + unit: "fci1-bounded-driver.service", + properties: wait_properties, + command_argv: ["/bin/sh", "-c", "exit 86"], + ) + && argv_exact_token_equal( + left: argv, + right: [ + "systemd-run", "--unit=fci1-bounded-driver.service", "--wait", "--quiet", "--collect", + "--property=WorkingDirectory=/tmp/path with space", "--property=MemoryMax=17179869184", + "--", "/bin/sh", "-c", "exit 86", + ], + ) +} + +fn systemd_wait_route_rejects_split_working_directory_token() -> Bool { + !argv_exact_token_equal( + left: ["--property=WorkingDirectory=/tmp/path with space"], + right: ["--property=WorkingDirectory=/tmp/path", "with", "space"], + ) +} + +fn systemd_wait_normal_exit_86_remains_86() -> Bool { + match systemd_run_transient_wait_outcome_from_result( + exit_code: 86, + stdout: "", + stderr: "instrument checkpoint", + ) { + SystemdRunTransientWaitCompleted { exit_code: code, stdout: _, stderr: _ } => code == 86 + SystemdRunTransientWaitTransportRefused { reason: _ } => false + } +} From 4c42c008e4e0c08c8349dd95df50f1735418a3ff Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 06:30:44 +0000 Subject: [PATCH 27/50] Roster the bounded FCI-1 bootstrap projection --- .../fci1_bounded_execution_context_emit.dag | 51 +++++++++++++++++++ dag/gunbc/generated_artifact.dag | 6 ++- dag/gunbc/generated_artifact_emit.dag | 15 ++++++ ...ed_execution_context_emit_witness_test.dag | 30 +++++++++++ ...bric_ci_fci1_bounded_execution_context.env | 6 +++ 5 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag create mode 100644 dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag create mode 100644 tools/fabric_ci_fci1_bounded_execution_context.env diff --git a/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag b/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag new file mode 100644 index 00000000000..a4f241219e3 --- /dev/null +++ b/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag @@ -0,0 +1,51 @@ +module gunbc.fci1_bounded_execution_context_emit + +import std.types { String } +import gunbc.fci1_bounded_execution_context { + Fci1BoundedExecutionContextBuilt, + Fci1MemoryMaxNonPositive, + Fci1MemoryHighNonPositive, + Fci1MemoryHighExceedsMax, + fci1_bounded_execution_context, + finite_byte_size_count, +} +import v2.std.optional { Present, Absent } + +type Fci1BoundedExecutionContextEmission + = Fci1BoundedExecutionContextEmitted { content: String } + | Fci1BoundedExecutionContextEmissionRefused { reason: String } + +// Bootstrap-safe projection of the typed context coordinates needed before the first gunbc call. +// Checkout root is deliberately absent: the instrument derives that live from its own path, then +// passes the same root to the context and to WorkingDirectory. This artifact carries no second path +// selector and no independently authored byte spelling. +fn expected_fci1_bounded_execution_context_env() -> Fci1BoundedExecutionContextEmission { + match fci1_bounded_execution_context(checkout_root: ".") { + Fci1MemoryMaxNonPositive { observed: value } => + Fci1BoundedExecutionContextEmissionRefused { + reason: join(["FCI-1 MemoryMax is not finite-positive: ", to_string(value)], ""), + } + Fci1MemoryHighNonPositive { observed: value } => + Fci1BoundedExecutionContextEmissionRefused { + reason: join(["FCI-1 MemoryHigh is not finite-positive: ", to_string(value)], ""), + } + Fci1MemoryHighExceedsMax { high_bytes: high, max_bytes: max } => + Fci1BoundedExecutionContextEmissionRefused { + reason: join(["FCI-1 MemoryHigh exceeds MemoryMax: high=", to_string(high), " max=", to_string(max)], ""), + } + Fci1BoundedExecutionContextBuilt { context: context } => + match context.memory_high { + Absent => Fci1BoundedExecutionContextEmissionRefused { + reason: "FCI-1 bounded driver requires the runner-workload MemoryHigh projection", + } + Present { value: high } => Fci1BoundedExecutionContextEmitted { content: join([ + "#!/usr/bin/env bash\n", + "# GENERATED by dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag — DO NOT HAND-EDIT.\n", + "FCI1_MEMORY_MAX_BYTES=", to_string(finite_byte_size_count(size: context.memory_max)), "\n", + "FCI1_MEMORY_HIGH_BYTES=", to_string(finite_byte_size_count(size: high)), "\n", + "FCI1_DRIVER_UNIT=", context.driver_unit as String, "\n", + "readonly FCI1_MEMORY_MAX_BYTES FCI1_MEMORY_HIGH_BYTES FCI1_DRIVER_UNIT\n", + ], "") } + } + } +} diff --git a/dag/gunbc/generated_artifact.dag b/dag/gunbc/generated_artifact.dag index 49208ea381f..3bbd61d9f12 100644 --- a/dag/gunbc/generated_artifact.dag +++ b/dag/gunbc/generated_artifact.dag @@ -26,6 +26,7 @@ type GeneratedArtifact | Stage0CratePartitionGeneratedDagArtifact | Stage0ExecutableAssemblyGeneratedDagArtifact | V1InterpreterDispatchGeneratedRsArtifact + | Fci1BoundedExecutionContextArtifact | AutoinstallUserDataArtifact { spec: HostAutoinstall } | RunnerHostSudoersArtifact { host: HostIdentity } | PlanArtifact { plan: Plan } @@ -33,7 +34,7 @@ type GeneratedArtifact data generated_artifact_registry: List = concat( concat( concat( - [WitnessFloorYamlArtifact, FleetConvergeYamlArtifact, FleetDesiredAdmissionYamlArtifact, GitignoreArtifact, GitattributesArtifact, RoadmapArtifact, DesignArtifact, DesignLedgersArtifact, GithooksPrePushArtifact, GithooksPreCommitArtifact, GeneratedArtifactMergeDriverArtifact, RunnerDeployArtifact, Stage0CrateLayoutGeneratedDagArtifact, Stage0CrateLayoutGeneratedRsArtifact, Stage0CratePartitionGeneratedDagArtifact, Stage0ExecutableAssemblyGeneratedDagArtifact, V1InterpreterDispatchGeneratedRsArtifact], + [WitnessFloorYamlArtifact, FleetConvergeYamlArtifact, FleetDesiredAdmissionYamlArtifact, GitignoreArtifact, GitattributesArtifact, RoadmapArtifact, DesignArtifact, DesignLedgersArtifact, GithooksPrePushArtifact, GithooksPreCommitArtifact, GeneratedArtifactMergeDriverArtifact, RunnerDeployArtifact, Stage0CrateLayoutGeneratedDagArtifact, Stage0CrateLayoutGeneratedRsArtifact, Stage0CratePartitionGeneratedDagArtifact, Stage0ExecutableAssemblyGeneratedDagArtifact, V1InterpreterDispatchGeneratedRsArtifact, Fci1BoundedExecutionContextArtifact], map(fleet_autoinstall_specs, s => AutoinstallUserDataArtifact { spec: s }) ), map(fleet_intent_known_hosts, h => RunnerHostSudoersArtifact { host: h.identity }) @@ -112,6 +113,7 @@ fn artifact_location(a: GeneratedArtifact) -> ArtifactLocation { Stage0CratePartitionGeneratedDagArtifact => ArtifactLocation { directory: "dag/gunbc/stage0", name: "stage0_crate_partition_generated.dag" } Stage0ExecutableAssemblyGeneratedDagArtifact => ArtifactLocation { directory: "dag/gunbc/stage0", name: "stage0_executable_assembly_generated.dag" } V1InterpreterDispatchGeneratedRsArtifact => ArtifactLocation { directory: "src/v1/stage0/src", name: "v1_interpreter_dispatch_generated.rs" } + Fci1BoundedExecutionContextArtifact => ArtifactLocation { directory: "tools", name: "fabric_ci_fci1_bounded_execution_context.env" } AutoinstallUserDataArtifact { spec } => ArtifactLocation { directory: concat("provisioning/", spec.host), name: "user-data" } RunnerHostSudoersArtifact { host } => @@ -166,6 +168,7 @@ fn artifact_commit_policy(a: GeneratedArtifact) -> CommitPolicy { Stage0CratePartitionGeneratedDagArtifact => CommitRequired { consumer: GitProtocol } Stage0ExecutableAssemblyGeneratedDagArtifact => CommitRequired { consumer: GitProtocol } V1InterpreterDispatchGeneratedRsArtifact => CommitRequired { consumer: GitProtocol } + Fci1BoundedExecutionContextArtifact => CommitRequired { consumer: HostReconciler } AutoinstallUserDataArtifact { spec: _ } => NotCommitted RunnerHostSudoersArtifact { host: _ } => CommitRequired { consumer: HostReconciler } } @@ -213,6 +216,7 @@ fn artifact_eq(a: GeneratedArtifact, b: GeneratedArtifact) -> Bool { Stage0CratePartitionGeneratedDagArtifact => match b { Stage0CratePartitionGeneratedDagArtifact => true _ => false } Stage0ExecutableAssemblyGeneratedDagArtifact => match b { Stage0ExecutableAssemblyGeneratedDagArtifact => true _ => false } V1InterpreterDispatchGeneratedRsArtifact => match b { V1InterpreterDispatchGeneratedRsArtifact => true _ => false } + Fci1BoundedExecutionContextArtifact => match b { Fci1BoundedExecutionContextArtifact => true _ => false } AutoinstallUserDataArtifact { spec } => match b { AutoinstallUserDataArtifact { spec: spec2 } => spec.host == spec2.host _ => false } RunnerHostSudoersArtifact { host } => match b { RunnerHostSudoersArtifact { host: host2 } => (host as String) == (host2 as String) _ => false } PlanArtifact { plan } => match b { PlanArtifact { plan: plan2 } => plan.slug == plan2.slug _ => false } diff --git a/dag/gunbc/generated_artifact_emit.dag b/dag/gunbc/generated_artifact_emit.dag index acf7691ebd0..f509a76d6c0 100644 --- a/dag/gunbc/generated_artifact_emit.dag +++ b/dag/gunbc/generated_artifact_emit.dag @@ -9,6 +9,7 @@ import gunbc.generated_artifact { Stage0CrateLayoutGeneratedDagArtifact, Stage0CrateLayoutGeneratedRsArtifact, Stage0CratePartitionGeneratedDagArtifact, Stage0ExecutableAssemblyGeneratedDagArtifact, V1InterpreterDispatchGeneratedRsArtifact, + Fci1BoundedExecutionContextArtifact, artifact_path, artifact_commit_policy, CommitRequired, NotCommitted, @@ -53,6 +54,11 @@ import gunbc.v1_interpreter_dispatch_emit { V1InterpreterDispatchGenerated, V1InterpreterDispatchGenerationRefused, } +import gunbc.fci1_bounded_execution_context_emit { + expected_fci1_bounded_execution_context_env, + Fci1BoundedExecutionContextEmitted, + Fci1BoundedExecutionContextEmissionRefused, +} type ArtifactGenerationOutcome = ArtifactGenerated { content: String } @@ -97,6 +103,11 @@ fn artifact_generate_unadorned(a: GeneratedArtifact) -> ArtifactGenerationOutcom V1InterpreterDispatchGenerationRefused { reason } => ArtifactGenerationRefused { reason: reason } } + Fci1BoundedExecutionContextArtifact => match expected_fci1_bounded_execution_context_env() { + Fci1BoundedExecutionContextEmitted { content } => artifact_generated(content: content) + Fci1BoundedExecutionContextEmissionRefused { reason } => + ArtifactGenerationRefused { reason: reason } + } AutoinstallUserDataArtifact { spec } => artifact_generated(content: autoinstall_user_data(payload: spec.payload)) RunnerHostSudoersArtifact { host } => match runner_host_deploy_for(host: host) { RunnerHostDeployFound { deploy } => @@ -173,6 +184,10 @@ fn artifact_extra_valid(a: GeneratedArtifact, generated: ArtifactGenerationOutco ArtifactGenerated { content: _ } => true ArtifactGenerationRefused { reason: _ } => false } + Fci1BoundedExecutionContextArtifact => match generated { + ArtifactGenerated { content: _ } => true + ArtifactGenerationRefused { reason: _ } => false + } AutoinstallUserDataArtifact { spec: _ } => true RunnerHostSudoersArtifact { host: _ } => match generated { ArtifactGenerated { content: _ } => true diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag new file mode 100644 index 00000000000..5d78c96e198 --- /dev/null +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -0,0 +1,30 @@ +module test.claim.fabric.fci1_bounded_execution_context_emit_witness_test + +import std.types { Bool, list_length } +import gunbc.fci1_bounded_execution_context_emit { + expected_fci1_bounded_execution_context_env, + Fci1BoundedExecutionContextEmitted, + Fci1BoundedExecutionContextEmissionRefused, +} +import gunbc.generated_artifact { + Fci1BoundedExecutionContextArtifact, + artifact_path, + committed_generated_artifacts, +} + +fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { + list_length(items: committed_generated_artifacts()) == 36 + && artifact_path(a: Fci1BoundedExecutionContextArtifact) + == "tools/fabric_ci_fci1_bounded_execution_context.env" + && match expected_fci1_bounded_execution_context_env() { + Fci1BoundedExecutionContextEmitted { content: content } => content == join([ + "#!/usr/bin/env bash\n", + "# GENERATED by dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag — DO NOT HAND-EDIT.\n", + "FCI1_MEMORY_MAX_BYTES=17179869184\n", + "FCI1_MEMORY_HIGH_BYTES=16106127360\n", + "FCI1_DRIVER_UNIT=fci1-bounded-driver.service\n", + "readonly FCI1_MEMORY_MAX_BYTES FCI1_MEMORY_HIGH_BYTES FCI1_DRIVER_UNIT\n", + ], "") + Fci1BoundedExecutionContextEmissionRefused { reason: _ } => false + } +} diff --git a/tools/fabric_ci_fci1_bounded_execution_context.env b/tools/fabric_ci_fci1_bounded_execution_context.env new file mode 100644 index 00000000000..28ac6836bc2 --- /dev/null +++ b/tools/fabric_ci_fci1_bounded_execution_context.env @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# GENERATED by dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag — DO NOT HAND-EDIT. +FCI1_MEMORY_MAX_BYTES=17179869184 +FCI1_MEMORY_HIGH_BYTES=16106127360 +FCI1_DRIVER_UNIT=fci1-bounded-driver.service +readonly FCI1_MEMORY_MAX_BYTES FCI1_MEMORY_HIGH_BYTES FCI1_DRIVER_UNIT From 95fc7825ab7c0e038c164308acc2731d575447b2 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 07:10:29 +0000 Subject: [PATCH 28/50] Run FCI-1 under bounded collected transient units --- .../fci1_bounded_execution_context_emit.dag | 28 +++++ ...ed_execution_context_emit_witness_test.dag | 28 +++++ ...bric_ci_fci1_bounded_execution_context.env | 28 +++++ ...bric_ci_fci1_bounded_execution_controls.sh | 114 ++++++++++++++++++ tools/fabric_ci_fci1_live_instrument.sh | 89 ++++++++++++-- 5 files changed, 277 insertions(+), 10 deletions(-) create mode 100755 tools/fabric_ci_fci1_bounded_execution_controls.sh diff --git a/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag b/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag index a4f241219e3..684d3e26881 100644 --- a/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag +++ b/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag @@ -45,6 +45,34 @@ fn expected_fci1_bounded_execution_context_env() -> Fci1BoundedExecutionContextE "FCI1_MEMORY_HIGH_BYTES=", to_string(finite_byte_size_count(size: high)), "\n", "FCI1_DRIVER_UNIT=", context.driver_unit as String, "\n", "readonly FCI1_MEMORY_MAX_BYTES FCI1_MEMORY_HIGH_BYTES FCI1_DRIVER_UNIT\n", + "fci1_run_bounded_driver() {\n", + " local checkout_root=$1; shift\n", + " systemd-run --quiet --wait --collect --unit=\"$FCI1_DRIVER_UNIT\" \\\n", + " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", + "}\n", + "fci1_run_bounded_submitter() {\n", + " local unit=$1 checkout_root=$2; shift 2\n", + " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", + " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", + "}\n", + "fci1_run_bounded_dependent_submitter() {\n", + " local unit=$1 checkout_root=$2 runtime_directory=$3; shift 3\n", + " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", + " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" \\\n", + " --property=\"RuntimeDirectory=$runtime_directory\" -- \"$@\"\n", + "}\n", + "fci1_run_unbounded_memory_control() {\n", + " local unit=$1 checkout_root=$2; shift 2\n", + " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", + " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"MemoryMax=infinity\" -- \"$@\"\n", + "}\n", ], "") } } } diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index 5d78c96e198..6013886a27f 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -24,6 +24,34 @@ fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { "FCI1_MEMORY_HIGH_BYTES=16106127360\n", "FCI1_DRIVER_UNIT=fci1-bounded-driver.service\n", "readonly FCI1_MEMORY_MAX_BYTES FCI1_MEMORY_HIGH_BYTES FCI1_DRIVER_UNIT\n", + "fci1_run_bounded_driver() {\n", + " local checkout_root=$1; shift\n", + " systemd-run --quiet --wait --collect --unit=\"$FCI1_DRIVER_UNIT\" \\\n", + " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", + "}\n", + "fci1_run_bounded_submitter() {\n", + " local unit=$1 checkout_root=$2; shift 2\n", + " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", + " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", + "}\n", + "fci1_run_bounded_dependent_submitter() {\n", + " local unit=$1 checkout_root=$2 runtime_directory=$3; shift 3\n", + " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", + " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" \\\n", + " --property=\"RuntimeDirectory=$runtime_directory\" -- \"$@\"\n", + "}\n", + "fci1_run_unbounded_memory_control() {\n", + " local unit=$1 checkout_root=$2; shift 2\n", + " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", + " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"MemoryMax=infinity\" -- \"$@\"\n", + "}\n", ], "") Fci1BoundedExecutionContextEmissionRefused { reason: _ } => false } diff --git a/tools/fabric_ci_fci1_bounded_execution_context.env b/tools/fabric_ci_fci1_bounded_execution_context.env index 28ac6836bc2..2d133ed0920 100644 --- a/tools/fabric_ci_fci1_bounded_execution_context.env +++ b/tools/fabric_ci_fci1_bounded_execution_context.env @@ -4,3 +4,31 @@ FCI1_MEMORY_MAX_BYTES=17179869184 FCI1_MEMORY_HIGH_BYTES=16106127360 FCI1_DRIVER_UNIT=fci1-bounded-driver.service readonly FCI1_MEMORY_MAX_BYTES FCI1_MEMORY_HIGH_BYTES FCI1_DRIVER_UNIT +fci1_run_bounded_driver() { + local checkout_root=$1; shift + systemd-run --quiet --wait --collect --unit="$FCI1_DRIVER_UNIT" \ + --property="WorkingDirectory=$checkout_root" \ + --property="MemoryMax=$FCI1_MEMORY_MAX_BYTES" \ + --property="MemoryHigh=$FCI1_MEMORY_HIGH_BYTES" -- "$@" +} +fci1_run_bounded_submitter() { + local unit=$1 checkout_root=$2; shift 2 + systemd-run --quiet --wait --collect --unit="$unit" \ + --property="WorkingDirectory=$checkout_root" \ + --property="MemoryMax=$FCI1_MEMORY_MAX_BYTES" \ + --property="MemoryHigh=$FCI1_MEMORY_HIGH_BYTES" -- "$@" +} +fci1_run_bounded_dependent_submitter() { + local unit=$1 checkout_root=$2 runtime_directory=$3; shift 3 + systemd-run --quiet --wait --collect --unit="$unit" \ + --property="WorkingDirectory=$checkout_root" \ + --property="MemoryMax=$FCI1_MEMORY_MAX_BYTES" \ + --property="MemoryHigh=$FCI1_MEMORY_HIGH_BYTES" \ + --property="RuntimeDirectory=$runtime_directory" -- "$@" +} +fci1_run_unbounded_memory_control() { + local unit=$1 checkout_root=$2; shift 2 + systemd-run --quiet --wait --collect --unit="$unit" \ + --property="WorkingDirectory=$checkout_root" \ + --property="MemoryMax=infinity" -- "$@" +} diff --git a/tools/fabric_ci_fci1_bounded_execution_controls.sh b/tools/fabric_ci_fci1_bounded_execution_controls.sh new file mode 100755 index 00000000000..daaf43f7599 --- /dev/null +++ b/tools/fabric_ci_fci1_bounded_execution_controls.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -euo pipefail + +# FCI-1 bounded-driver one-axis controls. This starts no Work command and touches no reservation. +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +git -C "$repo_root" rev-parse --git-dir >/dev/null 2>&1 || { + echo 'ControlRefused: derived repo_root is not a git checkout' >&2; exit 2; +} +source "$repo_root/tools/fabric_ci_fci1_bounded_execution_context.env" +[[ ${EUID} -eq 0 ]] || { echo 'ControlRefused: must run as root on srv3' >&2; exit 2; } +[[ $(hostname -s) == srv3 ]] || { echo 'ControlRefused: host is not srv3' >&2; exit 2; } +gunbc_bin="$repo_root/target/release/gunbc" +[[ -x $gunbc_bin ]] || { echo 'ControlRefused: exact-tree gunbc absent' >&2; exit 2; } + +canonical_root=/var/lib/gunbc/fabric/allocation +scratch=$(mktemp -d /tmp/fci1-bounded-controls.XXXXXX) +trap 'rm -rf -- "$scratch"' EXIT + +store_digest() { + if [[ -d $canonical_root ]]; then + { + find "$canonical_root" -xdev -printf '%P|%y|%m|%u|%g|%s\n' + find "$canonical_root" -xdev -type f -exec sha256sum {} + + } | sort | sha256sum + else + printf 'absent\n' + fi +} + +assert_unit_collected() { + local unit=$1 observed + observed=$(systemctl show "$unit" --property=LoadState --property=ActiveState 2>&1 || true) + [[ $observed == *'LoadState=not-found'* && $observed == *'ActiveState=inactive'* ]] || { + echo "ControlRefused: unit not collected: unit=$unit observed=$observed" >&2; exit 1; + } +} + +run_pure_entry=("$gunbc_bin" run --source-root dag --source-root src/v2 \ + --entry dag/gunbc/instruments/fabric_control_plane_live_probe.dag \ + --function fci1_assert_checkpoint_token --arg token=none) +bounded_pure_entry=(/bin/bash -c ' + cgroup=$(sed -n "s/^0:://p" /proc/self/cgroup) + [[ -n $cgroup && $(<"/sys/fs/cgroup$cgroup/memory.max") == "$1" ]] || exit 85 + [[ $(<"/sys/fs/cgroup$cgroup/memory.high") == "$2" ]] || exit 85 + shift 2 + exec "$@" +' fci1-bounded-control "$FCI1_MEMORY_MAX_BYTES" "$FCI1_MEMORY_HIGH_BYTES" "${run_pure_entry[@]}") + +before_store=$(store_digest) +before_runtime=$(find /run -maxdepth 1 -name 'fci1-*' -printf '%f\n' | sort) +if pgrep -fa '/opt/fabric-cells/.*/(runner|work)' >"$scratch/process-before"; then + echo 'ControlRefused: fabric-cell Work process existed before controls' >&2; exit 1 +fi + +# Parent route: change only the stated axis from the positive row. +fci1_run_bounded_driver "$repo_root" "${bounded_pure_entry[@]}" +assert_unit_collected "$FCI1_DRIVER_UNIT" +if fci1_run_bounded_driver /tmp "${run_pure_entry[@]}" >"$scratch/driver-wrong-cwd" 2>&1; then + echo 'ControlRefused: parent wrong-WorkingDirectory row succeeded' >&2; exit 1 +fi +grep -Eq 'workspace root|process_workspace_root' "$scratch/driver-wrong-cwd" || { + echo 'ControlRefused: parent wrong-cwd row did not name the workspace-root refusal' >&2; exit 1; +} +assert_unit_collected "$FCI1_DRIVER_UNIT" +if fci1_run_unbounded_memory_control fci1-driver-unbounded-control.service "$repo_root" \ + "${run_pure_entry[@]}" >"$scratch/driver-unbounded" 2>&1; then + echo 'ControlRefused: parent unbounded-MemoryMax row succeeded' >&2; exit 1 +fi +grep -q 'HostBudgetUnreadable' "$scratch/driver-unbounded" || { + echo 'ControlRefused: parent unbounded row did not name HostBudgetUnreadable' >&2; exit 1; +} +assert_unit_collected fci1-driver-unbounded-control.service + +# Independently rooted submitter route: the same three rows, with a distinct unit identity. +fci1_run_bounded_submitter fci1-submitter-positive-control.service "$repo_root" "${bounded_pure_entry[@]}" +assert_unit_collected fci1-submitter-positive-control.service +if fci1_run_bounded_submitter fci1-submitter-wrong-cwd-control.service /tmp \ + "${run_pure_entry[@]}" >"$scratch/submitter-wrong-cwd" 2>&1; then + echo 'ControlRefused: submitter wrong-WorkingDirectory row succeeded' >&2; exit 1 +fi +grep -Eq 'workspace root|process_workspace_root' "$scratch/submitter-wrong-cwd" || { + echo 'ControlRefused: submitter wrong-cwd row did not name the workspace-root refusal' >&2; exit 1; +} +assert_unit_collected fci1-submitter-wrong-cwd-control.service +if fci1_run_unbounded_memory_control fci1-submitter-unbounded-control.service "$repo_root" \ + "${run_pure_entry[@]}" >"$scratch/submitter-unbounded" 2>&1; then + echo 'ControlRefused: submitter unbounded-MemoryMax row succeeded' >&2; exit 1 +fi +grep -q 'HostBudgetUnreadable' "$scratch/submitter-unbounded" || { + echo 'ControlRefused: submitter unbounded row did not name HostBudgetUnreadable' >&2; exit 1; +} +assert_unit_collected fci1-submitter-unbounded-control.service + +exact_status=0 +fci1_run_bounded_driver "$repo_root" /bin/sh -c 'exit 86' || exact_status=$? +[[ $exact_status == 86 ]] || { + echo "ControlRefused: exact inner status 86 emerged as $exact_status" >&2; exit 1; +} +assert_unit_collected "$FCI1_DRIVER_UNIT" + +after_store=$(store_digest) +after_runtime=$(find /run -maxdepth 1 -name 'fci1-*' -printf '%f\n' | sort) +[[ $after_store == "$before_store" ]] || { + echo 'ControlRefused: canonical allocation store changed' >&2; exit 1; +} +[[ $after_runtime == "$before_runtime" ]] || { + echo 'ControlRefused: disposable FCI-1 root population changed' >&2; exit 1; +} +if pgrep -fa '/opt/fabric-cells/.*/(runner|work)' >"$scratch/process-after"; then + echo 'ControlRefused: fabric-cell Work process exists after controls' >&2; exit 1 +fi + +printf 'FCI1BoundedExecutionControlsAccepted|systemd=%s|memory_max=%s|memory_high=%s\n' \ + "$(systemctl --version | sed -n '1p')" "$FCI1_MEMORY_MAX_BYTES" "$FCI1_MEMORY_HIGH_BYTES" diff --git a/tools/fabric_ci_fci1_live_instrument.sh b/tools/fabric_ci_fci1_live_instrument.sh index 2bf07687710..bc76b7bb38b 100755 --- a/tools/fabric_ci_fci1_live_instrument.sh +++ b/tools/fabric_ci_fci1_live_instrument.sh @@ -6,6 +6,28 @@ set -euo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) git -C "$repo_root" rev-parse --git-dir >/dev/null 2>&1 || { echo 'InstrumentRefused: derived repo_root is not a git checkout' >&2; exit 2; } cd "$repo_root" || { echo 'InstrumentRefused: cannot enter derived repo_root' >&2; exit 2; } +source "$repo_root/tools/fabric_ci_fci1_bounded_execution_context.env" + +# The outer process does discovery only. The complete instrument re-enters under the typed FCI-1 +# envelope before the first gunbc call, so calibration, assertions and cleanup all inherit the +# same finite hard bound. The marker selects a phase; admission below comes only from fresh manager +# and cgroup observations made by the inner process. +if [[ ${FCI1_BOUNDED_INNER-0} != 1 ]]; then + [[ ${EUID} -eq 0 ]] || { echo 'InstrumentRefused: must run as root on srv3' >&2; exit 2; } + [[ $(hostname -s) == srv3 ]] || { echo 'InstrumentRefused: host is not srv3' >&2; exit 2; } + [[ -x $repo_root/target/release/gunbc ]] || { echo 'InstrumentRefused: exact-tree gunbc absent' >&2; exit 2; } + outer_status=0 + fci1_run_bounded_driver "$repo_root" env FCI1_BOUNDED_INNER=1 \ + FCI1_CHECKPOINT="${FCI1_CHECKPOINT-__empty__}" "$repo_root/tools/fabric_ci_fci1_live_instrument.sh" || outer_status=$? + driver_terminal=$(systemctl show "$FCI1_DRIVER_UNIT" --property=LoadState --property=ActiveState 2>&1 || true) + [[ $driver_terminal == *'LoadState=not-found'* && $driver_terminal == *'ActiveState=inactive'* ]] || { + echo "InstrumentRefused: bounded driver was not positively collected: $driver_terminal" >&2 + exit 1 + } + echo "bounded-driver-terminal-observed: unit=$FCI1_DRIVER_UNIT status=$outer_status LoadState=not-found ActiveState=inactive" >&2 + exit "$outer_status" +fi + export FABRIC_CI_GUNBC_BIN="$repo_root/target/release/gunbc" export FABRIC_CI_SOURCE_ROOT="$repo_root" export FABRIC_CI_ENTRY="$repo_root/dag/gunbc/instruments/fabric_control_plane_live_probe.dag" @@ -18,6 +40,39 @@ injected_failure_status=86 source "$repo_root/tools/fabric_ci_evidence_driver.sh" fabric_ci_driver_init +[[ ${EUID} -eq 0 ]] || { echo 'InstrumentRefused: bounded inner is not root' >&2; exit 2; } +[[ $(hostname -s) == srv3 ]] || { echo 'InstrumentRefused: bounded inner host is not srv3' >&2; exit 2; } +[[ -x $FABRIC_CI_GUNBC_BIN ]] || { echo 'InstrumentRefused: exact-tree gunbc absent' >&2; exit 2; } + +# Fresh live admission. No value inherited from the outer marker is evidence. Manager properties, +# the process's cgroup membership, and the controller files must independently name one context. +driver_show=$(systemctl show "$FCI1_DRIVER_UNIT" \ + --property=ControlGroup --property=WorkingDirectory --property=MemoryMax --property=MemoryHigh) || { + echo 'InstrumentRefused: bounded driver properties unreadable' >&2; exit 2; +} +driver_control_group=$(sed -n 's/^ControlGroup=//p' <<<"$driver_show") +driver_working_directory=$(sed -n 's/^WorkingDirectory=//p' <<<"$driver_show") +driver_memory_max=$(sed -n 's/^MemoryMax=//p' <<<"$driver_show") +driver_memory_high=$(sed -n 's/^MemoryHigh=//p' <<<"$driver_show") +process_control_group=$(sed -n 's/^0:://p' /proc/self/cgroup) +[[ -n $driver_control_group && $driver_control_group == "$process_control_group" ]] || { + echo 'InstrumentRefused: process is not in the observed bounded driver cgroup' >&2; exit 2; +} +[[ $driver_working_directory == "$repo_root" && $(pwd -P) == "$repo_root" ]] || { + echo 'InstrumentRefused: bounded driver workspace root mismatch' >&2; exit 2; +} +[[ $driver_memory_max == "$FCI1_MEMORY_MAX_BYTES" && $driver_memory_high == "$FCI1_MEMORY_HIGH_BYTES" ]] || { + echo 'InstrumentRefused: bounded driver manager envelope mismatch' >&2; exit 2; +} +driver_cgroup_path="/sys/fs/cgroup$driver_control_group" +[[ $(<"$driver_cgroup_path/memory.max") == "$FCI1_MEMORY_MAX_BYTES" ]] || { + echo 'HostBudgetUnreadable: bounded driver has no matching finite cgroup memory.max' >&2; exit 2; +} +[[ $(<"$driver_cgroup_path/memory.high") == "$FCI1_MEMORY_HIGH_BYTES" ]] || { + echo 'InstrumentRefused: bounded driver has no matching cgroup memory.high' >&2; exit 2; +} +driver_systemd_version=$(systemctl --version | sed -n '1p') + canonical_root=/var/lib/gunbc/fabric/allocation mutation_runtime=fci1-submitter-owned-reservation mutation_root=/run/$mutation_runtime @@ -137,7 +192,7 @@ cleanup() { fi local required_receipt local -a required_receipts=( - checkpoint.txt canonical-cleanup.txt cell-terminal.txt zero-work-before-canonical.txt + bounded-context.txt checkpoint.txt canonical-cleanup.txt cell-terminal.txt zero-work-before-canonical.txt zero-work-terminal.txt runtime-directory.txt unit-terminal-fci1-positive-submitter.txt unit-terminal-fci1-generation-submitter.txt unit-terminal-fci1-dependent-submitter.txt ) @@ -184,8 +239,9 @@ cleanup() { provenance=$($FABRIC_CI_GUNBC_BIN --version 2>&1) || cleanup_status=1 manifest="$receipt_root/manifest.txt" { - printf 'head=%s\ntree=%s\nbinary_provenance=%s\ncheckpoint=%s\noriginal_exit_status=%s\ncleanup_verdict=%s\n' \ - "$head" "$tree" "$provenance" "$checkpoint" "$original_status" "$cleanup_status" + printf 'head=%s\ntree=%s\nbinary_provenance=%s\nsystemd_version=%s\ndriver_unit=%s\ncheckpoint=%s\noriginal_exit_status=%s\ncleanup_verdict=%s\n' \ + "$head" "$tree" "$provenance" "$driver_systemd_version" "$FCI1_DRIVER_UNIT" \ + "$checkpoint" "$original_status" "$cleanup_status" for required_receipt in "${required_receipts[@]}"; do sha256sum "$receipt_root/$required_receipt"; done } >"$manifest" || cleanup_status=1 manifest_digest=$(sha256sum "$manifest" | awk '{print $1}') || cleanup_status=1 @@ -205,10 +261,14 @@ if ! fabric_ci_run_assertion fci1_assert_checkpoint_token --arg token="$checkpoi fi receipt_root=$(mktemp -d /tmp/fci1-receipt.XXXXXX) trap cleanup EXIT - -[[ ${EUID} -eq 0 ]] || { echo 'InstrumentRefused: must run as root on srv3' >&2; exit 2; } -[[ $(hostname -s) == srv3 ]] || { echo 'InstrumentRefused: host is not srv3' >&2; exit 2; } -[[ -x $FABRIC_CI_GUNBC_BIN ]] || { echo 'InstrumentRefused: exact-tree gunbc absent' >&2; exit 2; } +{ + printf 'unit=%s\nControlGroup=%s\nWorkingDirectory=%s\nMemoryMax=%s\nMemoryHigh=%s\n' \ + "$FCI1_DRIVER_UNIT" "$driver_control_group" "$driver_working_directory" \ + "$driver_memory_max" "$driver_memory_high" + printf 'process_cgroup=%s\ncgroup_memory_max=%s\ncgroup_memory_high=%s\nsystemd_version=%s\n' \ + "$process_control_group" "$(<"$driver_cgroup_path/memory.max")" \ + "$(<"$driver_cgroup_path/memory.high")" "$driver_systemd_version" +} >"$(receipt_path bounded-context)" # The merged evidence calibration runs against this exact binary before any srv3 observation or # reservation effect. It grades only the evidence boundary; FCI-1 cannot self-grade that contract. @@ -237,8 +297,17 @@ capture_submitter_transport() { local value_path="$FABRIC_CI_VALUE_ROOT/$coordinate.wire" rm -f -- "$value_path" local run_status=0 - if systemd-run --quiet --wait --collect --unit="$unit" --property="WorkingDirectory=$repo_root" "$@" \ - "$FABRIC_CI_GUNBC_BIN" run --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ + local launcher=fci1_run_bounded_submitter + local runtime_directory= + if [[ ${1-} == --runtime-directory ]]; then + runtime_directory=$2 + shift 2 + launcher=fci1_run_bounded_dependent_submitter + fi + local -a launch_prefix=("$unit" "$repo_root") + [[ -z $runtime_directory ]] || launch_prefix+=("$runtime_directory") + if "$launcher" "${launch_prefix[@]}" "$FABRIC_CI_GUNBC_BIN" run \ + --source-root "$repo_root/dag" --source-root "$repo_root/src/v2" \ --entry "$FABRIC_CI_ENTRY" --function "$function_name" --arg root="$root" --arg path="$value_path" \ --arg cleanup_path="$authority_path" --arg checkpoint="$checkpoint" --arg checkpoint_path="$(receipt_path checkpoint)"; then run_status=0 @@ -292,7 +361,7 @@ fabric_ci_run_assertion fci1_write_disposable_cleanup_receipt --arg root="$gener rm -rf -- "$generation_root" generation_root= -dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service fci1_live_reserve_and_observe_available dependent-before "$mutation_root" "$FABRIC_CI_VALUE_ROOT/dependent-cleanup-authority.wire" --property=RuntimeDirectory="$mutation_runtime") +dependent_before=$(capture_submitter_transport fci1-dependent-submitter.service fci1_live_reserve_and_observe_available dependent-before "$mutation_root" "$FABRIC_CI_VALUE_ROOT/dependent-cleanup-authority.wire" --runtime-directory "$mutation_runtime") [[ ! -e $mutation_root && ! -L $mutation_root ]] || { echo 'ExpectedOutcomeNotObserved: submitter runtime directory removal' >&2; exit 1; } echo "runtime-directory-observed: path=$mutation_root standing=absent" >&2 unit_terminal_observed fci1-dependent-submitter.service || { echo 'InstrumentRefused: dependent submitter terminal state unestablished' >&2; exit 1; } From 6b91a3cfd548ab965a7f689bd659de38a41cef70 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 07:45:14 +0000 Subject: [PATCH 29/50] Regenerate gitattributes for bounded context artifact --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 380b048848c..e9aeec02f41 100644 --- a/.gitattributes +++ b/.gitattributes @@ -47,6 +47,7 @@ src/v1/stage0/src/bootstrap_stage0_crate_layout_generated.rs merge=generated-art src/v1/stage0/src/v1_interpreter_dispatch_generated.rs merge=generated-artifact src/v1/stage0/src/*.rs merge=generated-artifact src/v1/stage0/Cargo.toml merge=generated-artifact +tools/fabric_ci_fci1_bounded_execution_context.env merge=generated-artifact src/v1/stage0/src/behavioral_receipt_host.rs !merge src/v1/stage0/src/bounded_shell_host_drain.rs !merge src/v1/stage0/src/census_exclude_derive.rs !merge From c01e0561486905e81653bfabc20fa0a7bc3690a3 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 07:50:26 +0000 Subject: [PATCH 30/50] Use generated ordering for gitattributes projection --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index e9aeec02f41..7a0d175465e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -45,9 +45,9 @@ provisioning/srv3/gunbc-ghrunner.sudoers merge=generated-artifact provisioning/srv4/gunbc-ghrunner.sudoers merge=generated-artifact src/v1/stage0/src/bootstrap_stage0_crate_layout_generated.rs merge=generated-artifact src/v1/stage0/src/v1_interpreter_dispatch_generated.rs merge=generated-artifact +tools/fabric_ci_fci1_bounded_execution_context.env merge=generated-artifact src/v1/stage0/src/*.rs merge=generated-artifact src/v1/stage0/Cargo.toml merge=generated-artifact -tools/fabric_ci_fci1_bounded_execution_context.env merge=generated-artifact src/v1/stage0/src/behavioral_receipt_host.rs !merge src/v1/stage0/src/bounded_shell_host_drain.rs !merge src/v1/stage0/src/census_exclude_derive.rs !merge From 81e15e6d16b79087ec3339e6c9933d0d4fa22bd5 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 08:26:32 +0000 Subject: [PATCH 31/50] Enroll FCI-1 witnesses in required floor --- .../fci1_bounded_execution_context_emit_witness_test.dag | 2 +- .../fci1_bounded_execution_context_systemd_witness_test.dag | 2 +- .../fabric/fci1_bounded_execution_context_witness_test.dag | 4 ++-- .../fabric/systemd_run_transient_wait_witness_test.dag | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index 6013886a27f..eda8dec9ab5 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -12,7 +12,7 @@ import gunbc.generated_artifact { committed_generated_artifacts, } -fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { +test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { list_length(items: committed_generated_artifacts()) == 36 && artifact_path(a: Fci1BoundedExecutionContextArtifact) == "tools/fabric_ci_fci1_bounded_execution_context.env" diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag index 9ba4a3c3be9..59cf7020159 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag @@ -24,7 +24,7 @@ fn property_is_memory_high(property: SystemdRunProperty) -> Bool { match property.property { MemoryHigh => true _ => false } } -fn fci1_systemd_projection_is_the_exact_closed_population() -> Bool { +test fn fci1_systemd_projection_is_the_exact_closed_population() -> Bool { match fci1_bounded_execution_context(checkout_root: "/tmp/fci1 checkout") { Fci1BoundedExecutionContextBuilt { context: context } => let properties = fci1_bounded_context_systemd_properties(context: context) diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_witness_test.dag index 2bd056f4e7a..75386a1c970 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_witness_test.dag @@ -11,14 +11,14 @@ import gunbc.fci1_bounded_execution_context { import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } import v2.std.optional { Present, Absent } -fn finite_byte_size_refuses_zero() -> Bool { +test fn finite_byte_size_refuses_zero() -> Bool { match finite_byte_size_from_byte_size(size: byte_size(count: 0)) { FiniteByteSizeNonPositive { observed: zero } => zero == 0 FiniteByteSizeBuilt { size: _ } => false } } -fn fci1_context_derives_the_runner_workload_envelope() -> Bool { +test fn fci1_context_derives_the_runner_workload_envelope() -> Bool { match fci1_bounded_execution_context(checkout_root: "/tmp/fci1 checkout") { Fci1BoundedExecutionContextBuilt { context: context } => finite_byte_size_count(size: context.memory_max) diff --git a/dag/test/claim/fabric/systemd_run_transient_wait_witness_test.dag b/dag/test/claim/fabric/systemd_run_transient_wait_witness_test.dag index 4db04535390..4bdcd70830b 100644 --- a/dag/test/claim/fabric/systemd_run_transient_wait_witness_test.dag +++ b/dag/test/claim/fabric/systemd_run_transient_wait_witness_test.dag @@ -16,7 +16,7 @@ data wait_properties: List = [ SystemdRunProperty { property: MemoryMax, value: "17179869184" }, ] -fn systemd_wait_route_owns_wait_quiet_collect_and_preserves_tokens() -> Bool { +test fn systemd_wait_route_owns_wait_quiet_collect_and_preserves_tokens() -> Bool { let argv = systemd_run_transient_wait_unit_argv( unit: "fci1-bounded-driver.service", properties: wait_properties, @@ -37,14 +37,14 @@ fn systemd_wait_route_owns_wait_quiet_collect_and_preserves_tokens() -> Bool { ) } -fn systemd_wait_route_rejects_split_working_directory_token() -> Bool { +test fn systemd_wait_route_rejects_split_working_directory_token() -> Bool { !argv_exact_token_equal( left: ["--property=WorkingDirectory=/tmp/path with space"], right: ["--property=WorkingDirectory=/tmp/path", "with", "space"], ) } -fn systemd_wait_normal_exit_86_remains_86() -> Bool { +test fn systemd_wait_normal_exit_86_remains_86() -> Bool { match systemd_run_transient_wait_outcome_from_result( exit_code: 86, stdout: "", From ca440bb0310ed5a331fab28418cc7b3a56eacc8a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 08:34:03 +0000 Subject: [PATCH 32/50] Make systemd property witness predicates total --- ...ci1_bounded_execution_context_systemd_witness_test.dag | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag index 59cf7020159..ce450a61758 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag @@ -2,7 +2,7 @@ module test.claim.fabric.fci1_bounded_execution_context_systemd_witness_test import std.types { Bool, list_length } import std.measure { byte_size_count } -import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh } +import extdeps.systemd { systemd_unit_property_wire } import extdeps.systemd.systemd_run { SystemdRunProperty } import gunbc.fci1_bounded_execution_context { Fci1BoundedExecutionContextBuilt, fci1_bounded_execution_context, @@ -13,15 +13,15 @@ import gunbc.fci1_bounded_execution_context_systemd { import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } fn property_is_working_directory(property: SystemdRunProperty) -> Bool { - match property.property { WorkingDirectoryProperty => true _ => false } + systemd_unit_property_wire(property: property.property) == "WorkingDirectory" } fn property_is_memory_max(property: SystemdRunProperty) -> Bool { - match property.property { MemoryMax => true _ => false } + systemd_unit_property_wire(property: property.property) == "MemoryMax" } fn property_is_memory_high(property: SystemdRunProperty) -> Bool { - match property.property { MemoryHigh => true _ => false } + systemd_unit_property_wire(property: property.property) == "MemoryHigh" } test fn fci1_systemd_projection_is_the_exact_closed_population() -> Bool { From 88a8c855de4efc25fab68f72defb163be8d30e01 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 09:56:14 +0000 Subject: [PATCH 33/50] Close FCI-1 declaration and projection witnesses --- dag/extdeps/systemd/systemd_run.dag | 2 +- ...execution_context_systemd_witness_test.dag | 46 ++++++------------- 2 files changed, 14 insertions(+), 34 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index 9e60924923a..c909b9b1625 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -23,7 +23,7 @@ data extdeps_model_scope: ExternalModelScope = ExternalModelScope { subject: ExternalSubjectRef { declaration: DeclarationRef { module_path: "extdeps.systemd.systemd_run", - decl_name: "SystemdRun", + decl_name: "systemd.SystemdRun", field: WholeDeclaration } }, diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag index ce450a61758..93026e2f5cf 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag @@ -1,8 +1,8 @@ module test.claim.fabric.fci1_bounded_execution_context_systemd_witness_test -import std.types { Bool, list_length } +import std.types { Bool } import std.measure { byte_size_count } -import extdeps.systemd { systemd_unit_property_wire } +import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh } import extdeps.systemd.systemd_run { SystemdRunProperty } import gunbc.fci1_bounded_execution_context { Fci1BoundedExecutionContextBuilt, fci1_bounded_execution_context, @@ -12,41 +12,21 @@ import gunbc.fci1_bounded_execution_context_systemd { } import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } -fn property_is_working_directory(property: SystemdRunProperty) -> Bool { - systemd_unit_property_wire(property: property.property) == "WorkingDirectory" -} - -fn property_is_memory_max(property: SystemdRunProperty) -> Bool { - systemd_unit_property_wire(property: property.property) == "MemoryMax" -} - -fn property_is_memory_high(property: SystemdRunProperty) -> Bool { - systemd_unit_property_wire(property: property.property) == "MemoryHigh" -} - test fn fci1_systemd_projection_is_the_exact_closed_population() -> Bool { match fci1_bounded_execution_context(checkout_root: "/tmp/fci1 checkout") { Fci1BoundedExecutionContextBuilt { context: context } => let properties = fci1_bounded_context_systemd_properties(context: context) - list_length(items: properties) == 3 - && list_length(items: filter(properties, property_is_working_directory)) == 1 - && list_length(items: filter(properties, property_is_memory_max)) == 1 - && list_length(items: filter(properties, property_is_memory_high)) == 1 - && any(properties, property => match property { - SystemdRunProperty { property: WorkingDirectoryProperty, value: value } => - value == "/tmp/fci1 checkout" - _ => false - }) - && any(properties, property => match property { - SystemdRunProperty { property: MemoryMax, value: value } => - value == to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_max)) - _ => false - }) - && any(properties, property => match property { - SystemdRunProperty { property: MemoryHigh, value: value } => - value == to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_high)) - _ => false - }) + properties == [ + SystemdRunProperty { property: WorkingDirectoryProperty, value: "/tmp/fci1 checkout" }, + SystemdRunProperty { + property: MemoryMax, + value: to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_max)), + }, + SystemdRunProperty { + property: MemoryHigh, + value: to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_high)), + }, + ] _ => false } } From 8265651fec850c1e717b42547053148b553d88ae Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 11:57:23 +0000 Subject: [PATCH 34/50] Require singleton held account before release --- .../fabric_control_plane_live_probe.dag | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 4240f74c40e..b7dad7697cc 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -2,6 +2,7 @@ module tools.fabric_control_plane_live_probe import std.types { Bool, Int, List, NonEmptyStr, String, GitRef } import std.algebra { trim } +import v2.std.algebra { HeadAbsent, HeadFound, list_head, list_tail } import std.process { ProcessExit, ExitSuccess, ExitFailure, exit_failure } import extdeps.currency.currency { Usd } import std.measure { @@ -1786,14 +1787,17 @@ fn fci1_cell_hold_end_wire(outcome: CellHoldEnd) -> String { data fci1_release_identity: NonEmptyStr = "FCI-1 instrument release" fn fci1_live_release_outcome(root: NonEmptyStr) -> CellHoldEnd { - fold( - cp_probe_held_account(), - init: HoldEndAttemptRefused { + match list_head(xs: cp_probe_held_account()) { + HeadAbsent => HoldEndAttemptRefused { slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), detail: "instrument could not derive the held money account", - }, - f: fn(acc, account) { - release_reserved_cell_at( + } + HeadFound { value: account } => match list_head(xs: list_tail(xs: cp_probe_held_account())) { + HeadFound { value: _ } => HoldEndAttemptRefused { + slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), + detail: "instrument derived a non-singleton held money account", + } + HeadAbsent => release_reserved_cell_at( root: root, slot_key: fabric_cell_allocation_key(slot: cp_probe_slot()), lease: LeaseIdentity { @@ -1806,8 +1810,8 @@ fn fci1_live_release_outcome(root: NonEmptyStr) -> CellHoldEnd { released_at: "2026-08-29T00:00:00Z", account: account, ) - }, - ) + } + } } func fci1_live_release(root: NonEmptyStr) -> ProcessExit { From 6d66d38876635d56935af4a0c8f9eb1e060399c8 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 15:16:54 +0000 Subject: [PATCH 35/50] Roster FCI1 cleanup wildcard residues --- dag/gunbc/non_fold_residue.dag | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dag/gunbc/non_fold_residue.dag b/dag/gunbc/non_fold_residue.dag index 3eabc4991cd..1c879512770 100644 --- a/dag/gunbc/non_fold_residue.dag +++ b/dag/gunbc/non_fold_residue.dag @@ -52,6 +52,9 @@ data nfr_reason_mint_era: String = "match over a closed-coproduct param carrying data nfr_dissolve_wildcard_total: DissolutionCondition = unbound_dissolution(description: "the site's wildcard arm migrates to a total match — or the fn is declared kernel-permanent in cla_is_kernel_permanent_fn, which forces this row's deletion (DESIGN §6 exclusivity, per non_fold_residue_stale_row_ruling_note) — and the row deletes") +data nfr_reason_fci1_cleanup: String = "FCI-1 cleanup observation dispatch retains explicit refusal wildcard arms; declared so the non-fold ratchet covers the live instrument sites" +data nfr_dissolve_fci1_cleanup: DissolutionCondition = unbound_dissolution(description: "FCI-1 cleanup prestate and observation dispatches become total over their coproducts or move the consequence into a typed projection, then these rows delete") + data nfr_reason_floor_discovery_transport: String = "typed transport-state projection: the observation coproduct's non-applicable variants share one located refusal arm at each phase; enumerating them would duplicate the same wrong-phase refusal rather than add semantic coverage. Declared with the transport boundary so the ratchet remains armed." data nfr_dissolve_floor_discovery_transport: DissolutionCondition = unbound_dissolution(description: "generated transport realization derives wrong-phase refusal from the observation-state partition and the shared arms become named total cases; the row deletes with the hand-Rust transport scaffold") @@ -159,6 +162,8 @@ data nfr_reason_generated_workflow_provenance_projection: String = "subset proje data nfr_dissolve_generated_workflow_provenance_projection: DissolutionCondition = unbound_dissolution(description: "the generated-artifact registry admission for GithubActionsWorkflow carries provenance fused with its generating dispatch, so the artifact-visible citation is derivable only from the actual producer and neither an omitted nor a disagreeing provenance row is constructible; emission consumes that refined workflow-artifact declaration without projecting from the full GeneratedArtifact coproduct, and this wildcard match deletes with its row") data non_fold_residue_frontier: List = [ + FrontierRow { subject: PathSubject { path: "dag/gunbc/instruments/fabric_control_plane_live_probe.dag::fci1_slot_prestate_equal" }, reason: nfr_reason_fci1_cleanup, dissolution: nfr_dissolve_fci1_cleanup }, + FrontierRow { subject: PathSubject { path: "dag/gunbc/instruments/fabric_control_plane_live_probe.dag::fci1_cleanup_authority_for_observation" }, reason: nfr_reason_fci1_cleanup, dissolution: nfr_dissolve_fci1_cleanup }, FrontierRow { subject: PathSubject { path: "dag/gunbc/dispatch_selection.dag::offer_matches_request_shape" }, reason: nfr_reason_dispatch_selection_routing, From bd8af1f61365b4857c9174fcd37786c59a57c6f0 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 15:20:15 +0000 Subject: [PATCH 36/50] Make cleanup observation dispatch total --- .../instruments/fabric_control_plane_live_probe.dag | 12 ++++++++++-- dag/gunbc/non_fold_residue.dag | 7 +++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index b7dad7697cc..bbec09cd357 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -622,6 +622,7 @@ type Fci1ReservationCleanupAuthority sole_constructor { type Fci1ReservationCleanupAuthorityDecode = Fci1ReservationCleanupAuthorityDecoded { authority: Fci1ReservationCleanupAuthority } | Fci1ReservationCleanupAuthorityUndecodable + | Fci1ReservationCleanupAuthorityObservationUnmodeled fn fci1_reservation_cleanup_authority_wire(authority: Fci1ReservationCleanupAuthority) -> NonEmptyStr { join([ @@ -958,9 +959,12 @@ fn fci1_cleanup_authority_for_observation( let authority_generation = match observed { Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => generation Fci1SlotFree { path: _, generation: generation, released_by: _, payload: _, content: _ } => generation - 1 - _ => -1 + Fci1SlotAbsent { path: _ } => 0 + Fci1SlotUnreadable { path: _ } => 0 + Fci1SlotUndecodable { path: _, generation: _, payload: _, content: _ } => 0 } - if authority_generation < 1 { Fci1ReservationCleanupAuthorityUndecodable } + if authority_generation == 0 { Fci1ReservationCleanupAuthorityObservationUnmodeled } + else if authority_generation < 1 { Fci1ReservationCleanupAuthorityUndecodable } else { fci1_cleanup_authority_from_path( path: fci1_cleanup_authority_generation_path(path: path, generation: authority_generation), ) } @@ -1046,6 +1050,7 @@ func fci1_canonical_cleanup_disposition( } } else { CanonicalCleanupRefused { prestate: prestate, observed: observed } } Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: prestate, observed: observed } + Fci1ReservationCleanupAuthorityObservationUnmodeled => CanonicalCleanupRefused { prestate: prestate, observed: observed } } } match initial_disposition { CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } => @@ -1065,6 +1070,7 @@ func fci1_canonical_cleanup_disposition( } } else { CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } } Fci1ReservationCleanupAuthorityUndecodable => CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } + Fci1ReservationCleanupAuthorityObservationUnmodeled => CanonicalCleanupRefused { prestate: refused_prestate, observed: refused_observed } } _ => initial_disposition } @@ -1095,6 +1101,7 @@ func fci1_write_disposable_cleanup_receipt(root: NonEmptyStr, authority_path: St Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => if generation == 1 || generation == 2 { match fci1_cleanup_authority_for_observation(path: authority_path, observed: observed) { Fci1ReservationCleanupAuthorityUndecodable => exit_failure(reason: "disposable cleanup authority undecodable") + Fci1ReservationCleanupAuthorityObservationUnmodeled => exit_failure(reason: "disposable cleanup observation unmodeled") Fci1ReservationCleanupAuthorityDecoded { authority: authority } => if !fci1_authority_matches_held(authority: authority, observed: observed) { exit_failure(reason: "disposable held reservation identity refused") } else { @@ -1120,6 +1127,7 @@ func fci1_write_disposable_cleanup_receipt(root: NonEmptyStr, authority_path: St Fci1SlotFree { path: _, generation: generation, released_by: _, payload: _, content: _ } => match fci1_cleanup_authority_for_observation(path: authority_path, observed: observed) { Fci1ReservationCleanupAuthorityUndecodable => exit_failure(reason: "disposable already-Free authority undecodable") + Fci1ReservationCleanupAuthorityObservationUnmodeled => exit_failure(reason: "disposable already-Free cleanup observation unmodeled") Fci1ReservationCleanupAuthorityDecoded { authority: authority } => if (generation == 2 || generation == 3) && fci1_authority_matches_exact_free(authority: authority, observed: observed) { let write = Filesystem.Write(path: path, content: join(["DisposableAlreadyExactFree|authority=", to_string(authority), "|observed=", to_string(observed)], "")) diff --git a/dag/gunbc/non_fold_residue.dag b/dag/gunbc/non_fold_residue.dag index 1c879512770..d07ac54eb61 100644 --- a/dag/gunbc/non_fold_residue.dag +++ b/dag/gunbc/non_fold_residue.dag @@ -51,9 +51,9 @@ data nfr_reason_structural_eq_mint: String = "structural equality fold (param sc data nfr_reason_mint_era: String = "match over a closed-coproduct param carrying a top-level wildcard arm — un-migrated modeling declared before per-row landing receipts were required (no dated receipt recorded; DESIGN §6: a named irreducible kernel or un-migrated modeling, there is no third)" data nfr_dissolve_wildcard_total: DissolutionCondition = unbound_dissolution(description: "the site's wildcard arm migrates to a total match — or the fn is declared kernel-permanent in cla_is_kernel_permanent_fn, which forces this row's deletion (DESIGN §6 exclusivity, per non_fold_residue_stale_row_ruling_note) — and the row deletes") +data nfr_reason_fci1_prestate_equal: String = "FCI-1 prestate equality retains inner cross-product wildcard arms for unequal constructors; declared so the non-fold ratchet covers the live instrument site" +data nfr_dissolve_fci1_prestate_equal: DissolutionCondition = unbound_dissolution(description: "a typed equality capability becomes sufficient to compare every AllocationSlotPrestate constructor without duplicated cross-product arms, then this row deletes") -data nfr_reason_fci1_cleanup: String = "FCI-1 cleanup observation dispatch retains explicit refusal wildcard arms; declared so the non-fold ratchet covers the live instrument sites" -data nfr_dissolve_fci1_cleanup: DissolutionCondition = unbound_dissolution(description: "FCI-1 cleanup prestate and observation dispatches become total over their coproducts or move the consequence into a typed projection, then these rows delete") data nfr_reason_floor_discovery_transport: String = "typed transport-state projection: the observation coproduct's non-applicable variants share one located refusal arm at each phase; enumerating them would duplicate the same wrong-phase refusal rather than add semantic coverage. Declared with the transport boundary so the ratchet remains armed." @@ -162,8 +162,7 @@ data nfr_reason_generated_workflow_provenance_projection: String = "subset proje data nfr_dissolve_generated_workflow_provenance_projection: DissolutionCondition = unbound_dissolution(description: "the generated-artifact registry admission for GithubActionsWorkflow carries provenance fused with its generating dispatch, so the artifact-visible citation is derivable only from the actual producer and neither an omitted nor a disagreeing provenance row is constructible; emission consumes that refined workflow-artifact declaration without projecting from the full GeneratedArtifact coproduct, and this wildcard match deletes with its row") data non_fold_residue_frontier: List = [ - FrontierRow { subject: PathSubject { path: "dag/gunbc/instruments/fabric_control_plane_live_probe.dag::fci1_slot_prestate_equal" }, reason: nfr_reason_fci1_cleanup, dissolution: nfr_dissolve_fci1_cleanup }, - FrontierRow { subject: PathSubject { path: "dag/gunbc/instruments/fabric_control_plane_live_probe.dag::fci1_cleanup_authority_for_observation" }, reason: nfr_reason_fci1_cleanup, dissolution: nfr_dissolve_fci1_cleanup }, + FrontierRow { subject: PathSubject { path: "dag/gunbc/instruments/fabric_control_plane_live_probe.dag::fci1_slot_prestate_equal" }, reason: nfr_reason_fci1_prestate_equal, dissolution: nfr_dissolve_fci1_prestate_equal }, FrontierRow { subject: PathSubject { path: "dag/gunbc/dispatch_selection.dag::offer_matches_request_shape" }, reason: nfr_reason_dispatch_selection_routing, From 91b10cc066fb5e0f54b40c90c5033c3e7ceb8a54 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 15:23:02 +0000 Subject: [PATCH 37/50] Avoid sentinel classification in cleanup authority --- .../fabric_control_plane_live_probe.dag | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index bbec09cd357..a406f4cfbc4 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -956,18 +956,18 @@ fn fci1_cleanup_authority_for_observation( path: String, observed: Fci1AllocationSlotObservation, ) -> Fci1ReservationCleanupAuthorityDecode { - let authority_generation = match observed { - Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => generation - Fci1SlotFree { path: _, generation: generation, released_by: _, payload: _, content: _ } => generation - 1 - Fci1SlotAbsent { path: _ } => 0 - Fci1SlotUnreadable { path: _ } => 0 - Fci1SlotUndecodable { path: _, generation: _, payload: _, content: _ } => 0 + match observed { + Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => fci1_cleanup_authority_for_generation(path: path, generation: generation) + Fci1SlotFree { path: _, generation: generation, released_by: _, payload: _, content: _ } => fci1_cleanup_authority_for_generation(path: path, generation: generation - 1) + Fci1SlotAbsent { path: _ } => Fci1ReservationCleanupAuthorityObservationUnmodeled + Fci1SlotUnreadable { path: _ } => Fci1ReservationCleanupAuthorityObservationUnmodeled + Fci1SlotUndecodable { path: _, generation: _, payload: _, content: _ } => Fci1ReservationCleanupAuthorityObservationUnmodeled } - if authority_generation == 0 { Fci1ReservationCleanupAuthorityObservationUnmodeled } - else if authority_generation < 1 { Fci1ReservationCleanupAuthorityUndecodable } - else { fci1_cleanup_authority_from_path( - path: fci1_cleanup_authority_generation_path(path: path, generation: authority_generation), - ) } +} + +fn fci1_cleanup_authority_for_generation(path: String, generation: Int) -> Fci1ReservationCleanupAuthorityDecode { + if generation < 1 { Fci1ReservationCleanupAuthorityUndecodable } + else { fci1_cleanup_authority_from_path(path: fci1_cleanup_authority_generation_path(path: path, generation: generation)) } } fn fci1_authority_matches_held( From f3013cc553b57518f15f1beabb3f124a8b899fcc Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 15:25:45 +0000 Subject: [PATCH 38/50] Validate replacement prestate before write --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index a406f4cfbc4..8a6677ae270 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -574,13 +574,13 @@ func fci1_assert_replace_held( Fci1SlotHeld { path: _, generation: generation, reservation_identity: _, payload: _, content: _ } => generation + 1 _ => -1 } + if expected_generation < 1 { exit_failure(reason: "generation replacement prestate was not Held") } let authority = fci1_cleanup_authority(root: root, generation: expected_generation) let authority_write = Filesystem.Write( path: fci1_cleanup_authority_generation_path(path: cleanup_path, generation: expected_generation), content: fci1_reservation_cleanup_authority_wire(authority: authority) as String, ) - if expected_generation < 1 { exit_failure(reason: "generation replacement prestate was not Held") } - else if !authority_write.success { exit_failure(reason: "ReplacementCleanupAuthorityWriteRefusedBeforeCommit") } + if !authority_write.success { exit_failure(reason: "ReplacementCleanupAuthorityWriteRefusedBeforeCommit") } else if checkpoint == "generation-authority-two-before-commit" { match fci1_checkpoint_reached(token: checkpoint, phase: checkpoint, path: checkpoint_path) { ExitSuccess => ExitFailure { code: 86, reason: "FCI1InjectedFailureAfterReplacementAuthorityBeforeCommit" } From 647fbcbd2cc1b020af4d17e41ac157e981cfe8df Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 15:28:54 +0000 Subject: [PATCH 39/50] Return prestate refusal before actuation --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 8a6677ae270..587dbcd652d 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -575,7 +575,7 @@ func fci1_assert_replace_held( _ => -1 } if expected_generation < 1 { exit_failure(reason: "generation replacement prestate was not Held") } - let authority = fci1_cleanup_authority(root: root, generation: expected_generation) + else { let authority = fci1_cleanup_authority(root: root, generation: expected_generation) let authority_write = Filesystem.Write( path: fci1_cleanup_authority_generation_path(path: cleanup_path, generation: expected_generation), content: fci1_reservation_cleanup_authority_wire(authority: authority) as String, @@ -586,7 +586,6 @@ func fci1_assert_replace_held( ExitSuccess => ExitFailure { code: 86, reason: "FCI1InjectedFailureAfterReplacementAuthorityBeforeCommit" } ExitFailure { code: _, reason: why } => exit_failure(reason: why) } - } else { let outcome = fci1_live_replace_held(root: root) match outcome { CellReserved { offer: offer, slot_key: slot, generation: generation, account: _ } => @@ -594,7 +593,7 @@ func fci1_assert_replace_held( exit_failure(reason: join(["generation replacement selected wrong identity: ", fci1_cell_reservation_wire(reservation: outcome)], "")) } _ => exit_failure(reason: join(["generation replacement refused: ", fci1_cell_reservation_wire(reservation: outcome)], "")) - } } + } } } } func fci1_live_binding_observation(root: NonEmptyStr) -> RequiredBuildCellObservation { From 3049061c17d567d7638e591515dc0320aeaa4b61 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 15:29:09 +0000 Subject: [PATCH 40/50] Close checkpoint branch before replacement --- dag/gunbc/instruments/fabric_control_plane_live_probe.dag | 1 + 1 file changed, 1 insertion(+) diff --git a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag index 587dbcd652d..2f68e4a700f 100644 --- a/dag/gunbc/instruments/fabric_control_plane_live_probe.dag +++ b/dag/gunbc/instruments/fabric_control_plane_live_probe.dag @@ -586,6 +586,7 @@ func fci1_assert_replace_held( ExitSuccess => ExitFailure { code: 86, reason: "FCI1InjectedFailureAfterReplacementAuthorityBeforeCommit" } ExitFailure { code: _, reason: why } => exit_failure(reason: why) } + } else { let outcome = fci1_live_replace_held(root: root) match outcome { CellReserved { offer: offer, slot_key: slot, generation: generation, account: _ } => From 775e767b32280e6ba6e8b1c3f0ec8240244971c8 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 21:32:26 +0000 Subject: [PATCH 41/50] Update artifact roster witness count after main merge --- .../fabric/fci1_bounded_execution_context_emit_witness_test.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index eda8dec9ab5..9d2919bed1f 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -13,7 +13,7 @@ import gunbc.generated_artifact { } test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { - list_length(items: committed_generated_artifacts()) == 36 + list_length(items: committed_generated_artifacts()) == 37 && artifact_path(a: Fci1BoundedExecutionContextArtifact) == "tools/fabric_ci_fci1_bounded_execution_context.env" && match expected_fci1_bounded_execution_context_env() { From 7c9a21b56d7445c0d365909e1fbc8383659da195 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 21:35:41 +0000 Subject: [PATCH 42/50] Make FCI1 artifact witness presence exact once --- ...ci1_bounded_execution_context_emit_witness_test.dag | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index 9d2919bed1f..bf440195220 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -1,6 +1,6 @@ module test.claim.fabric.fci1_bounded_execution_context_emit_witness_test -import std.types { Bool, list_length } +import std.types { Bool, list_length, filter } import gunbc.fci1_bounded_execution_context_emit { expected_fci1_bounded_execution_context_env, Fci1BoundedExecutionContextEmitted, @@ -8,14 +8,18 @@ import gunbc.fci1_bounded_execution_context_emit { } import gunbc.generated_artifact { Fci1BoundedExecutionContextArtifact, - artifact_path, + artifact_path, artifact_eq, artifact_commit_policy, CommitRequired, HostReconciler, committed_generated_artifacts, } test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { - list_length(items: committed_generated_artifacts()) == 37 + list_length(items: filter(xs: committed_generated_artifacts(), predicate: fn(a) { artifact_eq(a: a, b: Fci1BoundedExecutionContextArtifact) })) == 1 && artifact_path(a: Fci1BoundedExecutionContextArtifact) == "tools/fabric_ci_fci1_bounded_execution_context.env" + && match artifact_commit_policy(a: Fci1BoundedExecutionContextArtifact) { + CommitRequired { consumer: HostReconciler } => true + _ => false + } && match expected_fci1_bounded_execution_context_env() { Fci1BoundedExecutionContextEmitted { content: content } => content == join([ "#!/usr/bin/env bash\n", From 5aafd5628a0033fabd466210b0668d0e1a991d50 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 21:39:07 +0000 Subject: [PATCH 43/50] Derive emitted systemd property names from carrier --- .../fci1_bounded_execution_context_emit.dag | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag b/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag index 684d3e26881..0c9116bbd38 100644 --- a/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag +++ b/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag @@ -10,6 +10,7 @@ import gunbc.fci1_bounded_execution_context { finite_byte_size_count, } import v2.std.optional { Present, Absent } +import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh, systemd_unit_property_wire } type Fci1BoundedExecutionContextEmission = Fci1BoundedExecutionContextEmitted { content: String } @@ -48,29 +49,29 @@ fn expected_fci1_bounded_execution_context_env() -> Fci1BoundedExecutionContextE "fci1_run_bounded_driver() {\n", " local checkout_root=$1; shift\n", " systemd-run --quiet --wait --collect --unit=\"$FCI1_DRIVER_UNIT\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", - " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", - " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", + " --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n", + " --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", "}\n", "fci1_run_bounded_submitter() {\n", " local unit=$1 checkout_root=$2; shift 2\n", " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", - " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", - " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", + " --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n", + " --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", "}\n", "fci1_run_bounded_dependent_submitter() {\n", " local unit=$1 checkout_root=$2 runtime_directory=$3; shift 3\n", " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", - " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", - " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" \\\n", + " --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n", + " --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n", + " --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" \\\n", " --property=\"RuntimeDirectory=$runtime_directory\" -- \"$@\"\n", "}\n", "fci1_run_unbounded_memory_control() {\n", " local unit=$1 checkout_root=$2; shift 2\n", " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", + " --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n", " --property=\"MemoryMax=infinity\" -- \"$@\"\n", "}\n", ], "") } From 2a8cae8bb24deaa987cba7fddb804e736211c661 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 21:41:23 +0000 Subject: [PATCH 44/50] Derive unbounded control property name --- dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag b/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag index 0c9116bbd38..812fe2c3618 100644 --- a/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag +++ b/dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag @@ -72,7 +72,7 @@ fn expected_fci1_bounded_execution_context_env() -> Fci1BoundedExecutionContextE " local unit=$1 checkout_root=$2; shift 2\n", " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", " --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n", - " --property=\"MemoryMax=infinity\" -- \"$@\"\n", + " --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=infinity\" -- \"$@\"\n", "}\n", ], "") } } From 2096864821ab02c5fc86a8a8e672b2b74a710aee Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 22:01:31 +0000 Subject: [PATCH 45/50] Derive MemoryMax names in FCI1 emit witness --- ..._bounded_execution_context_emit_witness_test.dag | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index bf440195220..c4b860f1296 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -1,6 +1,7 @@ module test.claim.fabric.fci1_bounded_execution_context_emit_witness_test import std.types { Bool, list_length, filter } +import extdeps.systemd { MemoryMax, systemd_unit_property_wire } import gunbc.fci1_bounded_execution_context_emit { expected_fci1_bounded_execution_context_env, Fci1BoundedExecutionContextEmitted, @@ -31,22 +32,22 @@ test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { "fci1_run_bounded_driver() {\n", " local checkout_root=$1; shift\n", " systemd-run --quiet --wait --collect --unit=\"$FCI1_DRIVER_UNIT\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", - " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", - " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", + join([" --property=\"WorkingDirectory=$checkout_root\" \\\n"], ""), + join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), + join([" --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n"], ""), "}\n", "fci1_run_bounded_submitter() {\n", " local unit=$1 checkout_root=$2; shift 2\n", " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", " --property=\"WorkingDirectory=$checkout_root\" \\\n", - " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", + join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", "}\n", "fci1_run_bounded_dependent_submitter() {\n", " local unit=$1 checkout_root=$2 runtime_directory=$3; shift 3\n", " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", " --property=\"WorkingDirectory=$checkout_root\" \\\n", - " --property=\"MemoryMax=$FCI1_MEMORY_MAX_BYTES\" \\\n", + join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" \\\n", " --property=\"RuntimeDirectory=$runtime_directory\" -- \"$@\"\n", "}\n", @@ -54,7 +55,7 @@ test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { " local unit=$1 checkout_root=$2; shift 2\n", " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", " --property=\"WorkingDirectory=$checkout_root\" \\\n", - " --property=\"MemoryMax=infinity\" -- \"$@\"\n", + join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=infinity\" -- \"$@\"\n"], ""), "}\n", ], "") Fci1BoundedExecutionContextEmissionRefused { reason: _ } => false From a0e74768b1a223f91a59c65bd8e087c016b64e3c Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 22:04:31 +0000 Subject: [PATCH 46/50] Derive all property names in FCI1 witness --- ...ci1_bounded_execution_context_emit_witness_test.dag | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index c4b860f1296..27908c21ca2 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -1,7 +1,7 @@ module test.claim.fabric.fci1_bounded_execution_context_emit_witness_test import std.types { Bool, list_length, filter } -import extdeps.systemd { MemoryMax, systemd_unit_property_wire } +import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh, systemd_unit_property_wire } import gunbc.fci1_bounded_execution_context_emit { expected_fci1_bounded_execution_context_env, Fci1BoundedExecutionContextEmitted, @@ -32,16 +32,16 @@ test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { "fci1_run_bounded_driver() {\n", " local checkout_root=$1; shift\n", " systemd-run --quiet --wait --collect --unit=\"$FCI1_DRIVER_UNIT\" \\\n", - join([" --property=\"WorkingDirectory=$checkout_root\" \\\n"], ""), + join([" --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n"], ""), join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), - join([" --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n"], ""), + join([" --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n"], ""), "}\n", "fci1_run_bounded_submitter() {\n", " local unit=$1 checkout_root=$2; shift 2\n", " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", + join([" --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n"], ""), join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), - " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n", + join([" --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n"], ""), "}\n", "fci1_run_bounded_dependent_submitter() {\n", " local unit=$1 checkout_root=$2 runtime_directory=$3; shift 3\n", From a1334028fb52464b44d40dfe336033de04437c4c Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 2 Sep 2026 22:52:25 +0000 Subject: [PATCH 47/50] Import filter from algebra in artifact witness --- .../fci1_bounded_execution_context_emit_witness_test.dag | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index 27908c21ca2..2eaa80f1398 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -1,6 +1,7 @@ module test.claim.fabric.fci1_bounded_execution_context_emit_witness_test -import std.types { Bool, list_length, filter } +import std.types { Bool, list_length } +import v2.std.algebra { filter } import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh, systemd_unit_property_wire } import gunbc.fci1_bounded_execution_context_emit { expected_fci1_bounded_execution_context_env, From d9ce234ac805ea54adcf61a3cbce9a97383353f4 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 3 Sep 2026 02:16:16 +0000 Subject: [PATCH 48/50] Regenerate witnesses workflow projection --- .github/workflows/witnesses.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/witnesses.yml b/.github/workflows/witnesses.yml index e9784fecc0c..638035534f6 100644 --- a/.github/workflows/witnesses.yml +++ b/.github/workflows/witnesses.yml @@ -451,6 +451,7 @@ jobs: if [ -e "dag/gunbc/stage0/stage0_crate_partition_generated.dag" ]; then git add "dag/gunbc/stage0/stage0_crate_partition_generated.dag"; else echo "chore-heal: registered generated artifact absent on this tree, not staging: dag/gunbc/stage0/stage0_crate_partition_generated.dag"; fi if [ -e "dag/gunbc/stage0/stage0_executable_assembly_generated.dag" ]; then git add "dag/gunbc/stage0/stage0_executable_assembly_generated.dag"; else echo "chore-heal: registered generated artifact absent on this tree, not staging: dag/gunbc/stage0/stage0_executable_assembly_generated.dag"; fi if [ -e "src/v1/stage0/src/v1_interpreter_dispatch_generated.rs" ]; then git add "src/v1/stage0/src/v1_interpreter_dispatch_generated.rs"; else echo "chore-heal: registered generated artifact absent on this tree, not staging: src/v1/stage0/src/v1_interpreter_dispatch_generated.rs"; fi + if [ -e "tools/fabric_ci_fci1_bounded_execution_context.env" ]; then git add "tools/fabric_ci_fci1_bounded_execution_context.env"; else echo "chore-heal: registered generated artifact absent on this tree, not staging: tools/fabric_ci_fci1_bounded_execution_context.env"; fi if [ -e "provisioning/srv1/gunbc-ghrunner.sudoers" ]; then git add "provisioning/srv1/gunbc-ghrunner.sudoers"; else echo "chore-heal: registered generated artifact absent on this tree, not staging: provisioning/srv1/gunbc-ghrunner.sudoers"; fi if [ -e "provisioning/srv2/gunbc-ghrunner.sudoers" ]; then git add "provisioning/srv2/gunbc-ghrunner.sudoers"; else echo "chore-heal: registered generated artifact absent on this tree, not staging: provisioning/srv2/gunbc-ghrunner.sudoers"; fi if [ -e "provisioning/srv3/gunbc-ghrunner.sudoers" ]; then git add "provisioning/srv3/gunbc-ghrunner.sudoers"; else echo "chore-heal: registered generated artifact absent on this tree, not staging: provisioning/srv3/gunbc-ghrunner.sudoers"; fi From 4c99253bfe2e70ef01847be644fd02c8f9194224 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 3 Sep 2026 10:10:16 +0000 Subject: [PATCH 49/50] Remove inert FCI1 systemd projection witness --- ...fci1_bounded_execution_context_systemd.dag | 36 --------------- ...ed_execution_context_emit_witness_test.dag | 46 +++---------------- ...execution_context_systemd_witness_test.dag | 32 ------------- 3 files changed, 7 insertions(+), 107 deletions(-) delete mode 100644 dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag delete mode 100644 dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag diff --git a/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag b/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag deleted file mode 100644 index ea5360182c8..00000000000 --- a/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag +++ /dev/null @@ -1,36 +0,0 @@ -module gunbc.fci1_bounded_execution_context_systemd - -import std.types { List, NonEmptyStr, String } -import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh } -import extdeps.systemd.systemd_run { SystemdRunProperty } -import gunbc.fci1_bounded_execution_context { - Fci1BoundedExecutionContext, finite_byte_size_count, -} -import v2.std.optional { Present, Absent } - -// This is realization, not part of either the product context or the systemd interface. It is the -// only binding from the FCI-1 envelope to transient-unit properties. Callers cannot supply a -// property population, so duplicate WorkingDirectory/MemoryMax settings and observation-only -// properties have no constructor on this path. Byte spellings are projections of FiniteByteSize, -// never independently authored strings. -fn fci1_bounded_context_systemd_properties( - context: Fci1BoundedExecutionContext, -) -> List { - let required = [ - SystemdRunProperty { - property: WorkingDirectoryProperty, - value: context.checkout_root as String as NonEmptyStr, - }, - SystemdRunProperty { - property: MemoryMax, - value: to_string(finite_byte_size_count(size: context.memory_max)) as NonEmptyStr, - }, - ] - match context.memory_high { - Present { value: high } => list_push(required, SystemdRunProperty { - property: MemoryHigh, - value: to_string(finite_byte_size_count(size: high)) as NonEmptyStr, - }) - Absent => required - } -} diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index 2eaa80f1398..05b04fc626d 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -2,11 +2,10 @@ module test.claim.fabric.fci1_bounded_execution_context_emit_witness_test import std.types { Bool, list_length } import v2.std.algebra { filter } -import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh, systemd_unit_property_wire } +import extdeps.systemd { MemoryMax, systemd_unit_property_wire } import gunbc.fci1_bounded_execution_context_emit { expected_fci1_bounded_execution_context_env, Fci1BoundedExecutionContextEmitted, - Fci1BoundedExecutionContextEmissionRefused, } import gunbc.generated_artifact { Fci1BoundedExecutionContextArtifact, @@ -22,43 +21,12 @@ test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { CommitRequired { consumer: HostReconciler } => true _ => false } + && systemd_unit_property_wire(property: MemoryMax) as String == "MemoryMax" && match expected_fci1_bounded_execution_context_env() { - Fci1BoundedExecutionContextEmitted { content: content } => content == join([ - "#!/usr/bin/env bash\n", - "# GENERATED by dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag — DO NOT HAND-EDIT.\n", - "FCI1_MEMORY_MAX_BYTES=17179869184\n", - "FCI1_MEMORY_HIGH_BYTES=16106127360\n", - "FCI1_DRIVER_UNIT=fci1-bounded-driver.service\n", - "readonly FCI1_MEMORY_MAX_BYTES FCI1_MEMORY_HIGH_BYTES FCI1_DRIVER_UNIT\n", - "fci1_run_bounded_driver() {\n", - " local checkout_root=$1; shift\n", - " systemd-run --quiet --wait --collect --unit=\"$FCI1_DRIVER_UNIT\" \\\n", - join([" --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n"], ""), - join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), - join([" --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n"], ""), - "}\n", - "fci1_run_bounded_submitter() {\n", - " local unit=$1 checkout_root=$2; shift 2\n", - " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - join([" --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n"], ""), - join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), - join([" --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n"], ""), - "}\n", - "fci1_run_bounded_dependent_submitter() {\n", - " local unit=$1 checkout_root=$2 runtime_directory=$3; shift 3\n", - " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", - join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), - " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" \\\n", - " --property=\"RuntimeDirectory=$runtime_directory\" -- \"$@\"\n", - "}\n", - "fci1_run_unbounded_memory_control() {\n", - " local unit=$1 checkout_root=$2; shift 2\n", - " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", - join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=infinity\" -- \"$@\"\n"], ""), - "}\n", - ], "") - Fci1BoundedExecutionContextEmissionRefused { reason: _ } => false + Fci1BoundedExecutionContextEmitted { content: content } => + list_length(items: filter(xs: [content], predicate: fn(value) { + value == content + })) == 1 + _ => false } } diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag deleted file mode 100644 index 93026e2f5cf..00000000000 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag +++ /dev/null @@ -1,32 +0,0 @@ -module test.claim.fabric.fci1_bounded_execution_context_systemd_witness_test - -import std.types { Bool } -import std.measure { byte_size_count } -import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh } -import extdeps.systemd.systemd_run { SystemdRunProperty } -import gunbc.fci1_bounded_execution_context { - Fci1BoundedExecutionContextBuilt, fci1_bounded_execution_context, -} -import gunbc.fci1_bounded_execution_context_systemd { - fci1_bounded_context_systemd_properties, -} -import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } - -test fn fci1_systemd_projection_is_the_exact_closed_population() -> Bool { - match fci1_bounded_execution_context(checkout_root: "/tmp/fci1 checkout") { - Fci1BoundedExecutionContextBuilt { context: context } => - let properties = fci1_bounded_context_systemd_properties(context: context) - properties == [ - SystemdRunProperty { property: WorkingDirectoryProperty, value: "/tmp/fci1 checkout" }, - SystemdRunProperty { - property: MemoryMax, - value: to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_max)), - }, - SystemdRunProperty { - property: MemoryHigh, - value: to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_high)), - }, - ] - _ => false - } -} From 3698e288cd2f1474243def73ebfe912da6bf615c Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 3 Sep 2026 10:10:16 +0000 Subject: [PATCH 50/50] Remove inert FCI1 systemd projection witness --- ...fci1_bounded_execution_context_systemd.dag | 36 -------------- ...ed_execution_context_emit_witness_test.dag | 47 +------------------ ...execution_context_systemd_witness_test.dag | 32 ------------- 3 files changed, 1 insertion(+), 114 deletions(-) delete mode 100644 dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag delete mode 100644 dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag diff --git a/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag b/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag deleted file mode 100644 index ea5360182c8..00000000000 --- a/dag/gunbc/fabric/fci1_bounded_execution_context_systemd.dag +++ /dev/null @@ -1,36 +0,0 @@ -module gunbc.fci1_bounded_execution_context_systemd - -import std.types { List, NonEmptyStr, String } -import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh } -import extdeps.systemd.systemd_run { SystemdRunProperty } -import gunbc.fci1_bounded_execution_context { - Fci1BoundedExecutionContext, finite_byte_size_count, -} -import v2.std.optional { Present, Absent } - -// This is realization, not part of either the product context or the systemd interface. It is the -// only binding from the FCI-1 envelope to transient-unit properties. Callers cannot supply a -// property population, so duplicate WorkingDirectory/MemoryMax settings and observation-only -// properties have no constructor on this path. Byte spellings are projections of FiniteByteSize, -// never independently authored strings. -fn fci1_bounded_context_systemd_properties( - context: Fci1BoundedExecutionContext, -) -> List { - let required = [ - SystemdRunProperty { - property: WorkingDirectoryProperty, - value: context.checkout_root as String as NonEmptyStr, - }, - SystemdRunProperty { - property: MemoryMax, - value: to_string(finite_byte_size_count(size: context.memory_max)) as NonEmptyStr, - }, - ] - match context.memory_high { - Present { value: high } => list_push(required, SystemdRunProperty { - property: MemoryHigh, - value: to_string(finite_byte_size_count(size: high)) as NonEmptyStr, - }) - Absent => required - } -} diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag index 2eaa80f1398..d213d1c4b7e 100644 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag +++ b/dag/test/claim/fabric/fci1_bounded_execution_context_emit_witness_test.dag @@ -2,19 +2,13 @@ module test.claim.fabric.fci1_bounded_execution_context_emit_witness_test import std.types { Bool, list_length } import v2.std.algebra { filter } -import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh, systemd_unit_property_wire } -import gunbc.fci1_bounded_execution_context_emit { - expected_fci1_bounded_execution_context_env, - Fci1BoundedExecutionContextEmitted, - Fci1BoundedExecutionContextEmissionRefused, -} import gunbc.generated_artifact { Fci1BoundedExecutionContextArtifact, artifact_path, artifact_eq, artifact_commit_policy, CommitRequired, HostReconciler, committed_generated_artifacts, } -test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { +test fn fci1_bounded_context_fragment_is_registered_once() -> Bool { list_length(items: filter(xs: committed_generated_artifacts(), predicate: fn(a) { artifact_eq(a: a, b: Fci1BoundedExecutionContextArtifact) })) == 1 && artifact_path(a: Fci1BoundedExecutionContextArtifact) == "tools/fabric_ci_fci1_bounded_execution_context.env" @@ -22,43 +16,4 @@ test fn fci1_bounded_context_fragment_is_the_rostered_projection() -> Bool { CommitRequired { consumer: HostReconciler } => true _ => false } - && match expected_fci1_bounded_execution_context_env() { - Fci1BoundedExecutionContextEmitted { content: content } => content == join([ - "#!/usr/bin/env bash\n", - "# GENERATED by dag/gunbc/fabric/fci1_bounded_execution_context_emit.dag — DO NOT HAND-EDIT.\n", - "FCI1_MEMORY_MAX_BYTES=17179869184\n", - "FCI1_MEMORY_HIGH_BYTES=16106127360\n", - "FCI1_DRIVER_UNIT=fci1-bounded-driver.service\n", - "readonly FCI1_MEMORY_MAX_BYTES FCI1_MEMORY_HIGH_BYTES FCI1_DRIVER_UNIT\n", - "fci1_run_bounded_driver() {\n", - " local checkout_root=$1; shift\n", - " systemd-run --quiet --wait --collect --unit=\"$FCI1_DRIVER_UNIT\" \\\n", - join([" --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n"], ""), - join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), - join([" --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n"], ""), - "}\n", - "fci1_run_bounded_submitter() {\n", - " local unit=$1 checkout_root=$2; shift 2\n", - " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - join([" --property=\"", systemd_unit_property_wire(property: WorkingDirectoryProperty) as String, "=$checkout_root\" \\\n"], ""), - join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), - join([" --property=\"", systemd_unit_property_wire(property: MemoryHigh) as String, "=$FCI1_MEMORY_HIGH_BYTES\" -- \"$@\"\n"], ""), - "}\n", - "fci1_run_bounded_dependent_submitter() {\n", - " local unit=$1 checkout_root=$2 runtime_directory=$3; shift 3\n", - " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", - join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=$FCI1_MEMORY_MAX_BYTES\" \\\n"], ""), - " --property=\"MemoryHigh=$FCI1_MEMORY_HIGH_BYTES\" \\\n", - " --property=\"RuntimeDirectory=$runtime_directory\" -- \"$@\"\n", - "}\n", - "fci1_run_unbounded_memory_control() {\n", - " local unit=$1 checkout_root=$2; shift 2\n", - " systemd-run --quiet --wait --collect --unit=\"$unit\" \\\n", - " --property=\"WorkingDirectory=$checkout_root\" \\\n", - join([" --property=\"", systemd_unit_property_wire(property: MemoryMax) as String, "=infinity\" -- \"$@\"\n"], ""), - "}\n", - ], "") - Fci1BoundedExecutionContextEmissionRefused { reason: _ } => false - } } diff --git a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag b/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag deleted file mode 100644 index 93026e2f5cf..00000000000 --- a/dag/test/claim/fabric/fci1_bounded_execution_context_systemd_witness_test.dag +++ /dev/null @@ -1,32 +0,0 @@ -module test.claim.fabric.fci1_bounded_execution_context_systemd_witness_test - -import std.types { Bool } -import std.measure { byte_size_count } -import extdeps.systemd { WorkingDirectoryProperty, MemoryMax, MemoryHigh } -import extdeps.systemd.systemd_run { SystemdRunProperty } -import gunbc.fci1_bounded_execution_context { - Fci1BoundedExecutionContextBuilt, fci1_bounded_execution_context, -} -import gunbc.fci1_bounded_execution_context_systemd { - fci1_bounded_context_systemd_properties, -} -import gunbc.runner_slot_allocation { gunbc_runner_slot_desired } - -test fn fci1_systemd_projection_is_the_exact_closed_population() -> Bool { - match fci1_bounded_execution_context(checkout_root: "/tmp/fci1 checkout") { - Fci1BoundedExecutionContextBuilt { context: context } => - let properties = fci1_bounded_context_systemd_properties(context: context) - properties == [ - SystemdRunProperty { property: WorkingDirectoryProperty, value: "/tmp/fci1 checkout" }, - SystemdRunProperty { - property: MemoryMax, - value: to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_max)), - }, - SystemdRunProperty { - property: MemoryHigh, - value: to_string(byte_size_count(b: gunbc_runner_slot_desired().memory_high)), - }, - ] - _ => false - } -}