From 3e3b72c02fa44afbc76ffa4b4c859ebdcfcf5512 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 14:04:58 +0000 Subject: [PATCH 01/44] Compute admission reserves observed host memory atomically, not a count of leases The local compute provider admitted work by listing its lease directory and comparing the count against a policy budget of two. That is not atomic -- the list and the create are two operations on different names, and O_EXCL excludes a NAME, never a QUANTITY, so two requests with different identities both read one lease in flight and both proceeded -- and a count is not a capacity: two producers on a 125 GiB host and two on a 502 GiB host are the same policy applied to machines that differ fourfold. Admission is now a seat in a memory pool. The homes are the ones DESIGN 3b names for fabric/compute: product.capacity.pool owns the appropriation, encumbrance, term and release law; product.capacity.pool_events owns the transitions; gunbc.fabric_event_log linearizes each append with one compare-and-set against the partition head. gunbc.compute.host_capacity declares which pool, where it is linearized (this host, not the fleet DB -- a host's own memory is not a fact the fleet contends on, and routing it through another machine would refuse local builds on a network fault), and joins it to a real reading of /proc/meminfo. The unit's MemoryMax is the amount the pool granted, so the ledger's arithmetic is the enforcement rather than a parallel description of it. Every ending -- succeeded, failed, mismatch, infrastructure refusal -- releases, and a release that did not land is reported rather than assumed. Disposition: compute_cold_build_cap is DELETED, and HostAtColdBuildCap with it. Its fact is now HostComputeCapacityFull, measured in bytes; the two arms beside it are facts a count could not express (the machine's live availability, and an unreadable ledger). The appropriation is two worker ceilings, exactly the memory the deleted count admitted. Also: fabric_seat_acquire/observe generalize over the measure (memory is not a dimensionless count), and the observe/append/retry loop gunbc.fabric_quota held alone moves to the carrier as fabric_pool_event_append, with quota settlement migrated onto it rather than a second copy growing beside it. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/linux/proc_meminfo.dag | 87 ++++- dag/extdeps/linux/procfs.dag | 25 +- dag/gunbc/ci/ci_layer_roots.dag | 5 + dag/gunbc/compute/host_capacity.dag | 356 ++++++++++++++++++ dag/gunbc/compute/work_provider_local.dag | 150 ++++++-- dag/gunbc/compute/work_request.dag | 22 +- dag/gunbc/fabric/fabric_event_log.dag | 71 +++- dag/gunbc/fabric/fabric_quota.dag | 63 +--- dag/gunbc/harness/harness_guidance.dag | 2 +- dag/std/measure.dag | 10 + .../host_capacity_wet_witness_test.dag | 156 ++++++++ .../compute/work_request_witness_test.dag | 6 +- .../claim/os_proc_meminfo_witness_test.dag | 31 ++ src/v2/workflow/local_repo_wet_terminal.dag | 24 ++ 14 files changed, 913 insertions(+), 95 deletions(-) create mode 100644 dag/gunbc/compute/host_capacity.dag create mode 100644 dag/test/claim/compute/host_capacity_wet_witness_test.dag diff --git a/dag/extdeps/linux/proc_meminfo.dag b/dag/extdeps/linux/proc_meminfo.dag index f52806502d4..010a19d0381 100644 --- a/dag/extdeps/linux/proc_meminfo.dag +++ b/dag/extdeps/linux/proc_meminfo.dag @@ -1,8 +1,9 @@ module extdeps.linux.proc_meminfo -import std.types { NonEmptyStr } +import std.types { NonEmptyStr, String, Bool, Int } import std.types { List } -import std.measure { Kibibyte } +import std.measure { Kibibyte, kibibyte } +import std.algebra { trim } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } @@ -38,3 +39,85 @@ data meminfo_field_names: List = [ "SwapTotal", "SwapFree", ] + +// READING THE FILE'S TEXT BACK INTO THE SHAPE ABOVE. proc.5 describes /proc/meminfo as one metric +// per line, `Name:` followed by a decimal count and, for every field this module names, the unit +// `kB` -- which proc.5 spells in kilobytes and the kernel emits in KIBIBYTES, so the field type is +// Kibibyte and not a kilobyte. The parse lives here because this module owns the file's shape; the +// transport that produces the text is extdeps.linux.procfs, one of N (DESIGN 3). +// +// A MISSING FIELD IS ITS OWN ARM AND NEVER A ZERO. A host whose kernel does not publish +// MemAvailable (pre-3.14) would otherwise read as a host with no memory available, which is the +// state-space conflation an admission decision cannot survive: zero headroom and unknown headroom +// have opposite remedies, and only one of them is a refusal. +type MeminfoRead + = MeminfoParsed { meminfo: ProcMeminfo } + | MeminfoFieldAbsent { field: NonEmptyStr } + +fn meminfo_line_metric(line: String) -> MemoryMetric? { + let tokens = filter(split(s: line, delimiter: " "), t => trim(s: t) != "") + match first(tokens) { + Absent => none + Present { value: head } => + if !ends_with(s: head, suffix: ":") { + none + } else { + let key = substring(s: head, start: 0, end: length(head) - 1) + match first(tokens |> skip(n: 1)) { + Absent => none + Present { value: raw } => + match parse_int(s: trim(s: raw)) { + Absent => none + Present { value: n } => + if key == "" || n < 0 { none } else { Present { value: MemoryMetric { key: key as NonEmptyStr, value: kibibyte(count: n) } } } + } + } + } + } +} + +fn meminfo_metrics(text: String) -> List { + fold(split(s: text, delimiter: "\n"), init: [], f: (acc, line) => + match meminfo_line_metric(line: line) { + Absent => acc + Present { value: m } => concat(acc, [m]) + }) +} + +fn meminfo_metric_named(metrics: List, name: NonEmptyStr) -> Kibibyte? { + match first(filter(metrics, m => (m.key as String) == (name as String))) { + Absent => none + Present { value: m } => Present { value: m.value } + } +} + +// THE SEVEN NAMED FIELDS ARE REQUIRED TOGETHER, in the order meminfo_field_names declares them, so +// the roster above is the authority for what this parse demands rather than a second list here. +fn parse_proc_meminfo(text: String) -> MeminfoRead { + let metrics = meminfo_metrics(text: text) + match first(filter(meminfo_field_names, f => match meminfo_metric_named(metrics: metrics, name: f) { Absent => true Present { value: _ } => false })) { + Present { value: missing } => MeminfoFieldAbsent { field: missing } + Absent => + MeminfoParsed { + meminfo: ProcMeminfo { + mem_total: meminfo_required(metrics: metrics, name: "MemTotal" as NonEmptyStr), + mem_free: meminfo_required(metrics: metrics, name: "MemFree" as NonEmptyStr), + mem_available: meminfo_required(metrics: metrics, name: "MemAvailable" as NonEmptyStr), + buffers: meminfo_required(metrics: metrics, name: "Buffers" as NonEmptyStr), + cached: meminfo_required(metrics: metrics, name: "Cached" as NonEmptyStr), + swap_total: meminfo_required(metrics: metrics, name: "SwapTotal" as NonEmptyStr), + swap_free: meminfo_required(metrics: metrics, name: "SwapFree" as NonEmptyStr), + all_metrics: metrics, + }, + } + } +} + +// Reached only under the absence check above, which is why it can answer at all; the zero is the +// unreachable arm of an optional, not a fabricated reading. +fn meminfo_required(metrics: List, name: NonEmptyStr) -> Kibibyte { + match meminfo_metric_named(metrics: metrics, name: name) { + Present { value: k } => k + Absent => kibibyte(count: 0) + } +} diff --git a/dag/extdeps/linux/procfs.dag b/dag/extdeps/linux/procfs.dag index 7e2721ffc88..7fd793e0995 100644 --- a/dag/extdeps/linux/procfs.dag +++ b/dag/extdeps/linux/procfs.dag @@ -87,9 +87,11 @@ data procfs_paths_read_by_this_repository: List = [ // operation with a shell transport, so gunbc.host_operation_exec can carry it over whichever // transport the caller holds (DESIGN 3: the transport is a realization bound to this shape, never // a fact about procfs). `cat` is the whole realization: procfs files are read-once text and the -// kernel synthesizes them on open, so there is nothing to seek, follow or lock. Three operations -// and not one generic ReadFile, so that the operation's identity says which file it reads and an -// argv over an arbitrary path cannot be materialized from this service. +// kernel synthesizes them on open, so there is nothing to seek, follow or lock. ONE OPERATION PER +// FILE and not one generic ReadFile, so that the operation's identity says which file it reads and +// an argv over an arbitrary path cannot be materialized from this service. ReadMeminfo serves the +// host memory capacity and live availability a compute admission reads before it reserves +// (gunbc.compute.host_capacity). service linux.Procfs { operation ReadStat { input {} @@ -108,6 +110,23 @@ service linux.Procfs { } } + operation ReadMeminfo { + input {} + output { + value: String from "stdout" + success: Bool from "exit_success" + } + readonly + transport shell { argv: ["cat", "/proc/meminfo"] } + exit { + 0 => Unit + nonzero => String "/proc/meminfo read failed" + } + mock_response { + 0 => { value: "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: 8388608 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n", success: true } "hermetic linux.Procfs.ReadMeminfo" + } + } + operation ReadPidStat { input { pid: NonEmptyStr } output { diff --git a/dag/gunbc/ci/ci_layer_roots.dag b/dag/gunbc/ci/ci_layer_roots.dag index 5da6fa4460b..2503885cd4f 100644 --- a/dag/gunbc/ci/ci_layer_roots.dag +++ b/dag/gunbc/ci/ci_layer_roots.dag @@ -1026,6 +1026,11 @@ data witness_exclusion_frontier: List = [ classification: LocalRepoWetLane, reason: excl_local_repo_wet_tempdir_write_reason, dissolution: excl_local_repo_wet_dissolve}, + WitnessExclusionRow { + pattern: "host_capacity_wet_witness_test.dag", + classification: LocalRepoWetLane, + reason: excl_local_repo_wet_tempdir_write_reason, + dissolution: excl_local_repo_wet_dissolve}, WitnessExclusionRow { pattern: "durable_cas_file_store_wet_witness_test.dag", classification: LocalRepoWetLane, diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag new file mode 100644 index 00000000000..51a74c7550d --- /dev/null +++ b/dag/gunbc/compute/host_capacity.dag @@ -0,0 +1,356 @@ +module gunbc.compute.host_capacity + +import std.types { String, Bool, Int, NonEmptyStr, EpochSecs } +import std.nat { Nat } +import std.measure { Measure, Memory, Kibi, Kibibyte, kibibyte, kibibyte_count, kibibyte_from_byte_size_floor, ByteSize, byte_size_count } +import extdeps.linux.procfs +import extdeps.linux.proc_meminfo { MeminfoRead, MeminfoParsed, MeminfoFieldAbsent, parse_proc_meminfo } +import product.capacity.pool { Pool, pool_of, NoReplenishment, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at } +import product.capacity.event_chain { PartitionId } +import product.capacity.pool_events { SeatRequest, PoolReleased } +import product.capacity.lease { LeasePolicy, LeaseGrant, QuiescenceRequired } +import gunbc.fabric_storage_client { FabricStorageBinding, FabricStorageLocalFiles } +import gunbc.fabric_storage_file_store { fabric_storage_file_root, fabric_storage_file_root_ensure, FabricStorageRootStanding, FabricStorageRootReady, FabricStorageRootRefused } +import gunbc.fabric_event_log { + SeatAcquisition, SeatGranted, SeatFull, SeatContended, SeatAcquireRefused, SeatStoreRefused, + fabric_seat_acquire, seat_acquisition_wire, + SeatStandingObservation, SeatRoomObserved, SeatFullObserved, SeatStandingUnobserved, SeatStandingStoreRefused, fabric_seat_observe, + PoolEventAppend, PoolEventAppended, PoolEventContended, PoolEventAppendRefused, fabric_pool_event_append, pool_event_append_wire, +} +import gunbc.fabric_event_log_host { now_epoch_seconds } +import gunbc.roadmap_dashboard_instance { HostDashboardInstance, dashboard_instance_compute_root } + +// THE HOST'S COMPUTE MEMORY IS A POOL, AND A WORK REQUEST TAKES A SEAT IN IT BEFORE IT RUNS. +// +// gunbc.compute.work_provider_local admitted work by COUNTING LEASE FILES against a policy budget +// of two (compute_cold_build_cap): list the lease directory, compare the count, then create a lease +// for THIS identity. Two things were wrong with that and only the second is about arithmetic. +// +// FIRST, IT WAS NOT ATOMIC. The list and the create are two operations on different names, and the +// create excludes only a second producer of the SAME identity. Two requests with DIFFERENT +// identities both read one lease in flight, both compared 1 < 2, and both created their own lease: +// the cap admitted three. O_EXCL cannot close that, because the thing being over-committed is not a +// name -- it is a quantity, and no exclusive create of any path excludes a quantity. +// +// SECOND, A COUNT IS NOT A CAPACITY. Two producers on a 125 GiB host and two on a 502 GiB host are +// the same policy applied to hosts that differ by a factor of four, and a class that needs a +// gibibyte is charged the same as a release build. What the host actually has is memory, so what is +// reserved is memory -- and the ceiling is compared against a READING of the host rather than +// assumed. +// +// WHAT THIS IS NOT A SECOND AUTHORITY FOR. product.capacity.pool owns the appropriation, the +// encumbrance, the term and the release law; product.capacity.pool_events owns the transitions; +// gunbc.fabric_event_log owns the linearization -- one compare-and-set per append against the +// partition's head, which is what makes the reservation atomic across competing requests rather +// than merely checked. This module declares WHICH pool (the host's compute appropriation), WHERE it +// is linearized (this host, not the fleet), and joins it to the observation. It coins no capacity +// vocabulary of its own. + +// ── WHERE THE LEDGER LIVES, AND WHY NOT THE FLEET'S ──────────────────────────────────────────── +// +// gunbc.fabric_storage_placement places the fabric DB on ONE host and every other host reaches it +// over its served endpoint, which is correct for a fact the FLEET contends on -- an upstream's rate +// limit is metered on the credential, so every host must linearize against one partition. +// +// A HOST'S OWN MEMORY IS NOT SUCH A FACT. The contenders for it are all local, and routing their +// reservation through another machine would mean a local build refusing because a remote host was +// unreachable -- a fail-open in the other direction, since the refusal would not be about capacity +// at all. So the store is the SAME realization (gunbc.fabric_storage_file_store, whose exclusive +// create supplies the linearization) rooted under this host's own compute root. The partition names +// the host so a ledger copied between machines does not silently answer for the wrong one. +// THE SUBJECT OF A RESERVATION: which ledger, which partition, how much the pool appropriates and +// how much this request takes. It is separated from the instance deliberately, and not only to be +// testable: everything below is arithmetic and transaction over these four facts, while reading +// them off a HostDashboardInstance is a JOIN with the deployment's authority. Fusing the two would +// have made every reservation carry a whole deployment record in order to add two numbers, and +// would have left no seam at which the transaction can be exercised on a host other than the one +// the deployment describes. +type ComputeCapacitySubject { + root: NonEmptyStr + partition: PartitionId + appropriation: Kibibyte + request: Kibibyte +} + +fn compute_capacity_root(instance: HostDashboardInstance) -> NonEmptyStr { + join([dashboard_instance_compute_root(instance: instance) as String, "/capacity"], "") as NonEmptyStr +} + +fn compute_memory_partition(instance: HostDashboardInstance) -> PartitionId { + join(["compute-memory-", instance.instance_id as String], "") as PartitionId +} + +// THE JOIN WITH THE DEPLOYMENT, and the only function in this module that knows what an instance is. +fn compute_capacity_subject(instance: HostDashboardInstance) -> ComputeCapacitySubject { + ComputeCapacitySubject { + root: compute_capacity_root(instance: instance), + partition: compute_memory_partition(instance: instance), + appropriation: compute_memory_appropriation(instance: instance), + request: compute_request_memory(instance: instance), + } +} + +fn compute_capacity_store(subject: ComputeCapacitySubject) -> FabricStorageBinding { + FabricStorageLocalFiles { root: fabric_storage_file_root(root: subject.root) } +} + +fn compute_capacity_store_ensure(subject: ComputeCapacitySubject) -> FabricStorageRootStanding { + fabric_storage_file_root_ensure(root: fabric_storage_file_root(root: subject.root)) +} + +// ── WHAT THE HOST HAS, READ RATHER THAN SPELLED ──────────────────────────────────────────────── +// +// TWO FIGURES AND THEY ANSWER DIFFERENT QUESTIONS. MemTotal is the machine's installed memory and +// does not move, which is what a ledger needs: an appropriation whose ceiling changed between two +// folds would make the same chain of events add up differently each time it was read. MemAvailable +// is what is free RIGHT NOW including everything outside this pool -- the runner slices, the +// sessions, the page cache the kernel can hand back -- and it is a LIVE FLOOR, not a ceiling. +// +// Both are needed and neither substitutes for the other. The pool alone would admit work while +// another tenant had eaten the machine; MemAvailable alone would admit work whose own ledger says +// the memory is already promised, because memory a reservation holds but has not yet touched is +// still available to read. +type HostMemoryObservation + = HostMemoryObserved { total: Kibibyte, available: Kibibyte } + | HostMemoryUnobserved { detail: String } + +fn observe_host_memory() -> HostMemoryObservation { + let read = linux.Procfs.ReadMeminfo() + if !read.success { + HostMemoryUnobserved { detail: "/proc/meminfo could not be read on this host" } + } else { + match parse_proc_meminfo(text: read.value) { + MeminfoFieldAbsent { field: f } => HostMemoryUnobserved { detail: join(["/proc/meminfo carries no readable ", f as String, " line"], "") } + MeminfoParsed { meminfo: m } => HostMemoryObserved { total: m.mem_total, available: m.mem_available } + } + } +} + +// ── THE APPROPRIATION, AND WHAT IT REPLACES ──────────────────────────────────────────────────── +// +// THE SHARE OF THE HOST THIS PROVIDER MAY COMMIT, denominated in the bytes the old count was +// implicitly worth: compute_cold_build_cap was two producers, and each producer runs under the +// instance's worker memory ceiling, so two ceilings is EXACTLY the memory the deleted row admitted. +// Nothing is widened by this change and a class that needs less than a ceiling now admits more +// requests than a count ever could. +// +// DECLARED FRONTIER (DESIGN 3c), STATED BECAUSE IT IS THE HONEST LIMIT OF THIS ROW: charging this +// appropriation against the host's OTHER commitments -- the runner slice, the sessions slice, the +// fixed overhead -- is gunbc.ci_runner_placement's conservation wall, and this appropriation is +// DECLARED rather than derived from it. What keeps that from being fail-open is the pair of +// observations below: an appropriation larger than the machine refuses outright, and a request is +// refused when the machine's live availability cannot back it whatever the ledger says. +data compute_pool_slots: Nat = 2 + +fn compute_memory_appropriation(instance: HostDashboardInstance) -> Kibibyte { + kibibyte(count: kibibyte_count(k: kibibyte_from_byte_size_floor(b: instance.dispatch_worker_memory_max)) * compute_pool_slots) +} + +// THE UNIT'S CEILING IS THE RESERVED AMOUNT, and that identity is the whole point rather than a +// convenience. A reservation of X that caps the unit at Y is arithmetic about a number nothing +// enforces: if Y > X the pool's total is a fiction, and if Y < X the pool refuses work the host +// could have run. So one figure serves both, and it is the instance's worker memory ceiling -- +// the existing safety limit, unchanged and now also the thing being counted. +// +// DECLARED FRONTIER: a per-work-class amount (a claims run does not need a release build's +// ceiling) needs per-class peak measurements, which the consumption reading this change starts +// recording is the first producer of. Until those exist every class reserves the ceiling, which +// over-reserves and never under-reserves. +fn compute_request_memory(instance: HostDashboardInstance) -> Kibibyte { + kibibyte_from_byte_size_floor(b: instance.dispatch_worker_memory_max) +} + +// A COMPUTE SEAT IS QUIESCENCE-REQUIRED AND THAT IS THE SAFE ARM, NOT A LIMITATION. A running cargo +// cannot be told its lease lapsed -- there is no fence a compute unit honours -- so a term that runs +// out frees nothing, and product.capacity.pool holds the encumbrance until the seat is RELEASED. +// A worker that dies wedges its own share rather than silently handing it to the next arrival, +// which is the only direction that cannot oversubscribe a machine. Recovering a wedged seat is an +// explicit release naming why, which is the same entry completion and cancellation take. +fn compute_lease_policy(term_seconds: Nat) -> LeasePolicy { + LeasePolicy { maximum_duration_seconds: term_seconds, release_law: QuiescenceRequired } +} + +fn compute_memory_root_pool(subject: ComputeCapacitySubject) -> Pool { + pool_of( + identity: subject.partition as String as NonEmptyStr, + ceiling: subject.appropriation, + replenishment: NoReplenishment, + release_law: QuiescenceRequired, + ) +} + +// ── THE RESERVATION ──────────────────────────────────────────────────────────────────────────── +// +// EVERY ARM IS TYPED AND NONE OF THEM WIDENS. Insufficient capacity is ComputeCapacityFull and says +// so; a machine whose live availability cannot back the request is ComputeHostBelowFloor and is a +// DIFFERENT fact with a different remedy (wait, or find out who else is on the host); an +// unobservable host reserves nothing at all rather than reserving against a guess. +type ComputeReservation + = ComputeReserved { grant: LeaseGrant, partition: PartitionId, amount: Kibibyte, store: FabricStorageBinding } + | ComputeCapacityFull { wire: String } + | ComputeHostBelowFloor { requested: Kibibyte, available: Kibibyte } + | ComputeReservationRefused { detail: String } + +data compute_seat_attempts: Nat = 3 +data compute_partition_read_budget: Nat = 4096 + +// THE APPROPRIATION IS CHECKED AGAINST THE MACHINE BEFORE ANY SEAT IS TAKEN. A host too small for +// the declared share is a configuration error that would otherwise present as an OOM kill during a +// build, which is the same information arriving hours later and destructively. +fn compute_reserve(instance: HostDashboardInstance, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { + compute_reserve_on(subject: compute_capacity_subject(instance: instance), reference: reference, term_seconds: term_seconds) +} + +fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { + match observe_host_memory() { + HostMemoryUnobserved { detail: d } => ComputeReservationRefused { detail: join(["the host's memory could not be observed, so nothing was reserved: ", d], "") } + HostMemoryObserved { total: total, available: available } => + if kibibyte_count(k: subject.appropriation) > kibibyte_count(k: total) { + ComputeReservationRefused { + detail: join([ + "the compute appropriation is ", to_string(value: kibibyte_count(k: subject.appropriation)), + " KiB and this host has ", to_string(value: kibibyte_count(k: total)), + " KiB installed, so the share was never backed by the machine", + ], ""), + } + } else if kibibyte_count(k: available) < kibibyte_count(k: subject.request) { + ComputeHostBelowFloor { requested: subject.request, available: available } + } else { + match compute_capacity_store_ensure(subject: subject) { + FabricStorageRootRefused { area: a, detail: d } => ComputeReservationRefused { detail: join(["the capacity ledger could not be prepared at ", a, ": ", d], "") } + FabricStorageRootReady => + match now_epoch_seconds() { + Absent => ComputeReservationRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds) + } + } + } + } +} + +fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: EpochSecs, term_seconds: Nat) -> ComputeReservation { + let store = compute_capacity_store(subject: subject) + match fabric_seat_acquire( + store: store, + partition: subject.partition, + root: compute_memory_root_pool(subject: subject), + actor: reference, + request: SeatRequest { reference: reference, amount: kibibyte_count(k: subject.request), at: now, term_seconds: term_seconds }, + policy: compute_lease_policy(term_seconds: term_seconds), + attempts: compute_seat_attempts, + budget: compute_partition_read_budget, + ) { + SeatGranted { grant: g, generation: _, attempts_used: _ } => ComputeReserved { grant: g, partition: subject.partition, amount: subject.request, store: store } + SeatFull { wire: w, attempts_used: n } => ComputeCapacityFull { wire: join([w, " (attempt ", to_string(value: n), ")"], "") } + other => ComputeReservationRefused { detail: seat_acquisition_wire(a: other) } + } +} + +// ── THE RELEASE, WHICH IS THE ONLY WAY CAPACITY COMES BACK ───────────────────────────────────── +// +// COMPLETION AND CANCELLATION ARE THE SAME TRANSITION WITH DIFFERENT REASONS, which is why the +// reason is carried rather than encoded in two functions: the pool does not care why a holder +// stopped, and a caller that had to choose between two spellings would eventually pick the wrong +// one on the path it exercises least -- which is always the cancellation path. +// +// A RELEASE THAT DID NOT LAND IS NOT A RELEASE. The append is retried against a moving head and +// exhaustion is its own arm; nothing here reports a return of capacity it did not observe. +type ComputeRelease + = ComputeReleased { partition: PartitionId, reference: NonEmptyStr, reason: NonEmptyStr } + | ComputeReleaseRefused { detail: String } + +fn compute_release(reservation: ComputeReservation, reason: NonEmptyStr) -> ComputeRelease { + match reservation { + ComputeCapacityFull { wire: w } => ComputeReleaseRefused { detail: join(["nothing to release: ", w], "") } + ComputeHostBelowFloor { requested: _, available: _ } => ComputeReleaseRefused { detail: "nothing to release: the host was below its live memory floor" } + ComputeReservationRefused { detail: d } => ComputeReleaseRefused { detail: join(["nothing to release: ", d], "") } + ComputeReserved { grant: g, partition: p, amount: _, store: store } => + match now_epoch_seconds() { + Absent => ComputeReleaseRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match fabric_pool_event_append( + store: store, partition: p, actor: g.reference, at: now, + payload: PoolReleased { reference: g.reference, reason: reason }, attempts: compute_seat_attempts, + ) { + PoolEventAppended { id: _ } => ComputeReleased { partition: p, reference: g.reference, reason: reason } + other => ComputeReleaseRefused { detail: pool_event_append_wire(a: other) } + } + } + } +} + +// ── WHAT THE HOST IS CARRYING, WITHOUT TAKING FROM IT ────────────────────────────────────────── +// +// THE AGGREGATE READING A RECEIPT CARRIES: what the ledger says is committed on this host, beside +// what the machine says is free. The two are independent measurements of one quantity and a +// receipt that carried only the first would be a ledger agreeing with itself (DESIGN 5: a +// measurement copied from the same tree is not an oracle). Their DISAGREEMENT is the signal -- +// committed far above the machine's own shortfall means the pool is over-reserving each class, and +// the reverse means work is running outside the pool. +type ComputeCapacityStanding + = ComputeCapacityStandingRead { committed: Kibibyte, ceiling: Kibibyte, headroom: Nat, observed_available: Kibibyte, observed_total: Kibibyte } + | ComputeCapacityStandingUnread { detail: String } + +fn compute_capacity_standing(instance: HostDashboardInstance) -> ComputeCapacityStanding { + compute_capacity_standing_on(subject: compute_capacity_subject(instance: instance)) +} + +fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapacityStanding { + match observe_host_memory() { + HostMemoryUnobserved { detail: d } => ComputeCapacityStandingUnread { detail: d } + HostMemoryObserved { total: total, available: available } => + match now_epoch_seconds() { + Absent => ComputeCapacityStandingUnread { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match fabric_seat_observe( + store: compute_capacity_store(subject: subject), + partition: subject.partition, + root: compute_memory_root_pool(subject: subject), + at: now, + budget: compute_partition_read_budget, + ) { + SeatRoomObserved { headroom: h, generation: _ } => + ComputeCapacityStandingRead { + committed: kibibyte(count: kibibyte_count(k: subject.appropriation) - h), + ceiling: subject.appropriation, + headroom: h, + observed_available: available, + observed_total: total, + } + SeatFullObserved { generation: _ } => + ComputeCapacityStandingRead { + committed: subject.appropriation, + ceiling: subject.appropriation, + headroom: 0, + observed_available: available, + observed_total: total, + } + SeatStandingUnobserved { step: s, reason: why } => ComputeCapacityStandingUnread { detail: join(["the capacity ledger could not be read at ", s, ": ", why], "") } + SeatStandingStoreRefused { cause: _ } => ComputeCapacityStandingUnread { detail: "the capacity ledger refused the read" } + } + } + } +} + +fn compute_reservation_wire(r: ComputeReservation) -> String { + match r { + ComputeReserved { grant: g, partition: p, amount: a, store: _ } => + join(["reserved ", to_string(value: kibibyte_count(k: a)), " KiB on ", p as String, " as ", g.reference as String, " fence=", g.fence.grant as String, " expires_at=", to_string(value: g.expires_at)], "") + ComputeCapacityFull { wire: w } => join(["capacity full: ", w], "") + ComputeHostBelowFloor { requested: rq, available: av } => + join(["host below floor: ", to_string(value: kibibyte_count(k: rq)), " KiB requested, ", to_string(value: kibibyte_count(k: av)), " KiB available"], "") + ComputeReservationRefused { detail: d } => join(["refused: ", d], "") + } +} + +fn compute_capacity_standing_wire(s: ComputeCapacityStanding) -> String { + match s { + ComputeCapacityStandingRead { committed: c, ceiling: cl, headroom: h, observed_available: av, observed_total: tt } => + join([ + "committed ", to_string(value: kibibyte_count(k: c)), " KiB of ", to_string(value: kibibyte_count(k: cl)), + " KiB (headroom ", to_string(value: h), " KiB); host reports ", to_string(value: kibibyte_count(k: av)), + " KiB available of ", to_string(value: kibibyte_count(k: tt)), " KiB installed", + ], "") + ComputeCapacityStandingUnread { detail: d } => join(["unread: ", d], "") + } +} diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index fb1eea1c1eb..5d357f8598b 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -20,30 +20,39 @@ import gunbc.roadmap_dispatch_actuator { dispatch_capability_program } import gunbc.roadmap_execution_contract { ExecutionCapability, GitWorkspaceCapability, CargoCapability, RustcCapability, AttemptStateDirectoryCapability, } +import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size } +import gunbc.compute.host_capacity { + ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, + ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, + ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, +} import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoProfile, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, DeclaredOutput, WorkOutputRoot, CheckoutRoot, TargetRoot, WorkOutput, WorkOutcome, WorkSucceeded, WorkFailed, WorkRefusedByInfrastructure, WorkCancelled, WorkOutputMismatch, - WorkInfrastructureRefusal, ProducerInFlight, HostAtColdBuildCap, CheckoutUnavailable, StoreUnavailable, + WorkInfrastructureRefusal, ProducerInFlight, HostComputeCapacityFull, HostBelowMemoryFloor, ComputeCapacityLedgerUnavailable, + CheckoutUnavailable, StoreUnavailable, ToolchainUnobserved, DependencyNotSucceeded, SubjectUnresolved, work_outcome_wire, work_outcome_ending, StoredOutcomeRead, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_decode, stored_outcome_is_success, } // THE LOCAL PROVIDER: one of N behind the exact work contract, serving the host it runs on. It is a // realization and knows nothing the contract does not: it resolves the subject, observes the -// toolchain, takes a lease, checks the commit out, runs the operation inside a transient user unit -// with a memory bound, returns declared outputs by content into a store, and writes the outcome -// beside them. A second caller with the same identity attaches to the stored outcome and runs +// toolchain, takes a producer lease, RESERVES THE MEMORY THE WORK WILL USE out of the host's +// compute pool, checks the commit out, runs the operation inside a transient user unit capped at +// exactly that reservation, returns declared outputs by content into a store, writes the outcome +// beside them, and releases the reservation on every ending. A second caller with the same identity attaches to the stored outcome and runs // nothing (compute-deduplication-and-admission's first slice: two callers, one build). Placement // across the fabric is the next slice; this provider's boundary is the host. -// HOW MANY COLD BUILDS THIS HOST CARRIES AT ONCE. A policy budget, not a measurement: each producer -// runs under the instance's worker memory ceiling, and two of them fit beside the serve and belt -// units on the smaller build host (srv2, 125 GiB) without the page-cache eviction that hung srv1 in -// August. The dedup row's red control is that a caller refused for capacity is told so and counted; -// it is, as HostAtColdBuildCap. -data compute_cold_build_cap: Int = 2 +// HOW LONG A COMPUTE RESERVATION DECLARES IT WILL HOLD. Two hours covers the cold release build +// this provider's heaviest class runs, and it is a DECLARATION rather than a deadline anything +// enforces: the pool's release law for a compute seat is quiescence-required +// (gunbc.compute.host_capacity), so a term that runs out frees nothing and the seat comes back only +// by an explicit release. Saying so here rather than leaving the reader to infer a reclaim that +// does not happen. +data compute_reservation_term_seconds: Int = 7200 type ComputeLayout { root: String @@ -134,10 +143,6 @@ fn compute_ensure_dir(instance: HostDashboardInstance, path: String) -> Bool { } } -fn compute_count_lines(text: String) -> Int { - count(filter(text.split(sep: "\n"), l => trim(s: l) != "")) -} - // What a provide returns: a fresh run's outcome, or an attachment to a stored one (no run). type ProvideResult = ProvideFresh { outcome: WorkOutcome, document: String } @@ -199,9 +204,11 @@ fn compute_unit_name(identity_hex: String) -> NonEmptyStr { join(["gunbc-compute-", identity_hex], "") as NonEmptyStr } -// Run the operation as a transient user unit and wait for it: the unit carries the memory ceiling, -// the checkout as working directory, the shared per-host target directory and the sccache wrapper. -fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, identity_hex: String, argv: List) -> ComputeExec { +// Run the operation as a transient user unit and wait for it: the unit carries THE MEMORY THE POOL +// GRANTED -- not a ceiling read independently from the instance, which would let the reservation and +// the enforcement drift apart -- the checkout as working directory, the shared per-host target +// directory and the sccache wrapper. +fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, identity_hex: String, argv: List, granted: Kibibyte) -> ComputeExec { let systemd_run = match instance.toolchain.systemd_run { Present { value: p } => p as String Absent => "" } if systemd_run == "" { ComputeExecFailed { exit_code: 127, stdout: "", stderr: "this instance's toolchain declares no systemd-run; the local provider runs only under systemd" } @@ -212,7 +219,7 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden args: systemd_run_user_wait_arguments( unit: compute_unit_name(identity_hex: identity_hex), properties: [ - systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: instance.dispatch_worker_memory_max))), + systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: kibibyte_to_byte_size(k: granted)))), systemd_run_property(name: "WorkingDirectory", value: layout.checkout), ], setenv_bindings: [ @@ -283,29 +290,101 @@ fn compute_provide_leaf( } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { - let leases = Filesystem.List(path: layout.leases_dir) - let in_flight = if leases.success { compute_count_lines(text: leases.entries) } else { 0 } - if in_flight >= compute_cold_build_cap { - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: HostAtColdBuildCap { in_flight: in_flight, cap: compute_cold_build_cap } }, subject: subject, op: op) + // THE PRODUCER LEASE AND THE CAPACITY RESERVATION ARE TWO FACTS AND ARE TAKEN IN THIS ORDER. + // The lease excludes a SECOND PRODUCER OF THIS IDENTITY and is a name, so O_EXCL settles it; the + // reservation is a QUANTITY on the host and no exclusive name can settle that. Taking the lease + // first means a request that then loses on capacity releases a name it holds rather than + // reserving memory it may never use, and it is the cheaper of the two to undo. + let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) + if !lease.success { + compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) } else { - let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) - if !lease.success { - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) - } else { - let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store) - let document = work_outcome_wire(o: outcome, subject: subject, op: op) - let written = Filesystem.Write(path: layout.outcome_path, content: document) - let released = Filesystem.Delete(path: layout.lease_path) - if written.success && released.success { - ProvideFresh { outcome: outcome, document: document } - } else { - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["outcome written: ", if written.success { "yes" } else { written.error }, "; lease released: ", if released.success { "yes" } else { released.error }], "") } }, subject: subject, op: op) + let reservation = compute_reserve(instance: instance, reference: identity as NonEmptyStr, term_seconds: compute_reservation_term_seconds) + match reservation { + ComputeCapacityFull { wire: w } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) + ComputeHostBelowFloor { requested: rq, available: av } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested_kib: kibibyte_count(k: rq), available_kib: kibibyte_count(k: av) }) + ComputeReservationRefused { detail: d } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) + ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => { + // THE WORK RUNS UNDER THE GRANT: the unit's MemoryMax is the amount the pool reserved, so + // the ledger's arithmetic is the enforcement rather than a parallel description of it. + let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) + // AGGREGATE CONSUMPTION IS READ WHILE THE GRANT IS STILL HELD, which is the only instant + // at which the ledger's committed total and the machine's own availability describe the + // same population; after the release the reservation is gone from one side and the + // memory may not yet be gone from the other. + let standing = compute_capacity_standing(instance: instance) + // COMPLETION AND CANCELLATION RELEASE THE SAME WAY. Every ending of compute_run_leased -- + // succeeded, failed, output-mismatch, refused by infrastructure -- reaches here, so there + // is no ending on which the seat stays held. + let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) + let document = work_outcome_wire(o: outcome, subject: subject, op: op) + let written = Filesystem.Write(path: layout.outcome_path, content: join([document, "\n", compute_capacity_standing_wire(s: standing), "\n"], "")) + let released = Filesystem.Delete(path: layout.lease_path) + if written.success && released.success && compute_release_landed(r: returned) { + ProvideFresh { outcome: outcome, document: document } + } else { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { + detail: join([ + "outcome written: ", if written.success { "yes" } else { written.error }, + "; lease released: ", if released.success { "yes" } else { released.error }, + "; reservation released: ", compute_release_wire(r: returned), + "; work ended ", work_outcome_ending(o: outcome), + ], ""), + }, + }, + subject: subject, op: op) + } } } } } } +// A REQUEST REFUSED FOR CAPACITY HOLDS NOTHING. The producer lease is dropped on the way out, so a +// host that is full does not accumulate names that would refuse the next attempt for the wrong +// reason -- a capacity refusal reported as a producer already in flight is the state conflation +// this provider's refusal vocabulary exists to prevent. +fn compute_refused_unreserved( + instance: HostDashboardInstance, + identity: String, + subject: WorkSubject, + op: WorkOperation, + layout: ComputeLayout, + cause: WorkInfrastructureRefusal, +) -> ProvideResult { + let dropped = Filesystem.Delete(path: layout.lease_path) + if dropped.success { + compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) + } else { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { detail: join(["a capacity refusal could not drop its producer lease at ", layout.lease_path, ": ", dropped.error], "") }, + }, + subject: subject, op: op) + } +} + +fn compute_release_landed(r: ComputeRelease) -> Bool { + match r { + ComputeReleased { partition: _, reference: _, reason: _ } => true + ComputeReleaseRefused { detail: _ } => false + } +} + +fn compute_release_wire(r: ComputeRelease) -> String { + match r { + ComputeReleased { partition: _, reference: ref_, reason: why } => join(["released ", ref_ as String, " (", why as String, ")"], "") + ComputeReleaseRefused { detail: d } => join(["NOT released: ", d], "") + } +} + fn compute_fresh(outcome: WorkOutcome, subject: WorkSubject, op: WorkOperation) -> ProvideResult { ProvideFresh { outcome: outcome, document: work_outcome_wire(o: outcome, subject: subject, op: op) } } @@ -317,6 +396,7 @@ fn compute_run_leased( identity: String, op: WorkOperation, dependency_store: String, + granted: Kibibyte, ) -> WorkOutcome { let existing = Filesystem.List(path: layout.checkout) let checkout = if existing.success { @@ -332,7 +412,7 @@ fn compute_run_leased( ComputeExecFailed { exit_code, stdout: _, stderr } => WorkRefusedByInfrastructure { identity: identity, cause: CheckoutUnavailable { detail: join(["git worktree add exited ", to_string(value: exit_code), ": ", stderr], "") } } ComputeExecOk { stdout: _, stderr: _ } => { - let run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store)) + let run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store), granted: granted) let log = Filesystem.Write(path: layout.log_path, content: match run { ComputeExecOk { stdout, stderr } => join([stdout, stderr], "") ComputeExecFailed { exit_code: _, stdout, stderr } => join([stdout, stderr], "") diff --git a/dag/gunbc/compute/work_request.dag b/dag/gunbc/compute/work_request.dag index 53af4e64651..62e1231c219 100644 --- a/dag/gunbc/compute/work_request.dag +++ b/dag/gunbc/compute/work_request.dag @@ -134,12 +134,22 @@ type WorkOutcome | WorkOutputMismatch { identity: String, detail: String } // Infrastructure refusals are typed and counted, never a widen: a producer already holds the lease -// (the caller attaches later, it does not start a second build), the host is at its cold-build cap, -// the checkout or the store could not be prepared, or a dependency of this operation (the build the -// compile consumes) did not succeed. +// (the caller attaches later, it does not start a second build), the host cannot back the memory +// this class reserves, the checkout or the store could not be prepared, or a dependency of this +// operation (the build the compile consumes) did not succeed. +// +// THREE CAPACITY ARMS WHERE THERE USED TO BE ONE COUNT. HostAtColdBuildCap { in_flight, cap } is +// DELETED, and its disposition is that the fact it reported -- this host is already carrying as +// much as it may -- is now HostComputeCapacityFull, measured in the memory the pool actually +// appropriates rather than in producers. The other two are facts the count could not express at +// all and that a caller must not confuse with it: the machine's LIVE availability being below what +// this class needs is somebody else's memory, not this pool's commitments, and an unreadable +// ledger is a store repair. One arm for three remedies is the conflation DESIGN 5 forbids. type WorkInfrastructureRefusal = ProducerInFlight { lease_path: String } - | HostAtColdBuildCap { in_flight: Int, cap: Int } + | HostComputeCapacityFull { wire: String } + | HostBelowMemoryFloor { requested_kib: Int, available_kib: Int } + | ComputeCapacityLedgerUnavailable { detail: String } | CheckoutUnavailable { detail: String } | StoreUnavailable { detail: String } | ToolchainUnobserved { detail: String } @@ -169,7 +179,9 @@ fn work_outcome_ending(o: WorkOutcome) -> String { fn work_refusal_detail(r: WorkInfrastructureRefusal) -> String { match r { ProducerInFlight { lease_path } => join(["a producer already holds the lease at ", lease_path, "; attach later, do not start a second one"], "") - HostAtColdBuildCap { in_flight, cap } => join([to_string(value: in_flight), " producers in flight on this host, cap ", to_string(value: cap)], "") + HostComputeCapacityFull { wire } => join(["this host's compute memory pool has no room for another reservation: ", wire], "") + HostBelowMemoryFloor { requested_kib, available_kib } => join(["this class reserves ", to_string(value: requested_kib), " KiB and the host reports only ", to_string(value: available_kib), " KiB available, so the reservation would not have been backed by the machine"], "") + ComputeCapacityLedgerUnavailable { detail } => join(["the host's compute capacity ledger could not be transacted: ", detail], "") CheckoutUnavailable { detail } => join(["checkout unavailable: ", detail], "") StoreUnavailable { detail } => join(["store unavailable: ", detail], "") ToolchainUnobserved { detail } => join(["toolchain unobserved: ", detail], "") diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index 5225efab78a..ec7ee6ef88f 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -249,7 +249,7 @@ type SeatStandingObservation | SeatStandingUnobserved { step: String, reason: String } | SeatStandingStoreRefused { cause: EventLogRefusal } -fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root: Pool, at: EpochSecs, budget: Nat) -> SeatStandingObservation { +fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root: Pool, at: EpochSecs, budget: Nat) -> SeatStandingObservation { match event_log_read_partition(store: store, partition: partition, budget: budget) { PartitionReadRefused { cause: c } => SeatStandingStoreRefused { cause: c } PartitionReadOk { head: _, walk: walk } => @@ -275,7 +275,7 @@ fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root } } -fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, budget: Nat, attempt: Nat) -> SeatAcquisition? { +fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, budget: Nat, attempt: Nat) -> SeatAcquisition? { match event_log_read_partition(store: store, partition: partition, budget: budget) { PartitionReadRefused { cause: c } => Present { value: SeatStoreRefused { cause: c, attempts_used: attempt } } PartitionReadOk { head: head, walk: walk } => @@ -313,7 +313,7 @@ fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool< // and the drift is invisible because each side is internally consistent. A mismatch is refused before // anything is appended rather than resolved in favour of either side: neither is authoritative over // the other, and picking one silently would be the same fail-open in a different costume. -fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, attempts: Nat, budget: Nat) -> SeatAcquisition { +fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, attempts: Nat, budget: Nat) -> SeatAcquisition { if !release_law_eq(left: root.release_law, right: policy.release_law) { SeatAcquireRefused { step: "release-law", @@ -337,6 +337,71 @@ fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root } } +// APPENDING A POOL EVENT THAT IS NOT AN ACQUISITION -- a settlement or a release -- against the +// partition's current head, re-observing and retrying while the head moves under it. +// +// WHY IT IS HERE AND NOT AT EACH CALLER. Acquisition already lived here because it has to fold the +// pool to decide; settlement and release decide nothing, so each caller wrote its own +// observe/append/retry loop and gunbc.fabric_quota had the only copy. A second consumer +// (gunbc.compute.host_capacity, releasing a compute reservation) would have made it two spellings +// of one transaction, free to disagree about how many times a contended head is retried -- so the +// loop moves to the carrier that owns partition appends and both consumers call it. +// +// A RELEASE THAT LOSES ITS RACE IS RETRIED, NEVER ASSUMED. Nothing here reports a success it did +// not observe: exhausting the attempts is its own arm, and a caller holding it still holds the +// reservation. +type PoolEventAppend + = PoolEventAppended { id: EventId } + | PoolEventContended { attempts: Nat } + | PoolEventAppendRefused { detail: String } + +type PoolEventAttemptState { + done: PoolEventAppend? +} + +fn pool_event_append_attempt(store: FabricStorageBinding, partition: PartitionId, actor: NonEmptyStr, at: EpochSecs, payload: PoolEvent) -> PoolEventAppend? { + match event_log_observe_head(store: store, partition: partition) { + HeadObservationRefused { cause: c } => Present { value: PoolEventAppendRefused { detail: event_log_refusal_wire(cause: c) } } + HeadObserved { head: head } => + match event_log_append( + store: store, + partition: partition, + event: ChainEvent { + partition: partition, + parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, + recorded_at: at, + actor: actor, + payload: payload, + }, + expected: head, + ) { + EventAppended { id: h } => Present { value: PoolEventAppended { id: h } } + EventAppendStale { expected: _, observed: _ } => none + EventAppendRefused { cause: c } => Present { value: PoolEventAppendRefused { detail: event_log_refusal_wire(cause: c) } } + } + } +} + +fn fabric_pool_event_append(store: FabricStorageBinding, partition: PartitionId, actor: NonEmptyStr, at: EpochSecs, payload: PoolEvent, attempts: Nat) -> PoolEventAppend { + let st = fold(nat_range_inclusive(lo: 1, hi: attempts), init: PoolEventAttemptState { done: none }, f: (acc, _i) => + match acc.done { + Present { value: _ } => acc + Absent => PoolEventAttemptState { done: pool_event_append_attempt(store: store, partition: partition, actor: actor, at: at, payload: payload) } + }) + match st.done { + Present { value: d } => d + Absent => PoolEventContended { attempts: attempts } + } +} + +fn pool_event_append_wire(a: PoolEventAppend) -> String { + match a { + PoolEventAppended { id: h } => join(["appended ", h as String], "") + PoolEventContended { attempts: n } => join(["contended after ", to_string(n), " attempts"], "") + PoolEventAppendRefused { detail: d } => join(["refused ", d], "") + } +} + fn seat_acquisition_wire(a: SeatAcquisition) -> String { match a { SeatGranted { grant: g, generation: gen, attempts_used: n } => join(["granted ", g.reference as String, " fence=", g.fence.grant as String, "@", to_string(gen), " expires_at=", to_string(g.expires_at), " attempt=", to_string(n)], "") diff --git a/dag/gunbc/fabric/fabric_quota.dag b/dag/gunbc/fabric/fabric_quota.dag index 5fa0f3429a8..f6c87e0cb75 100644 --- a/dag/gunbc/fabric/fabric_quota.dag +++ b/dag/gunbc/fabric/fabric_quota.dag @@ -1,20 +1,17 @@ module gunbc.fabric_quota import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } -import std.nat { Nat, nat_range_inclusive } -import std.measure { Measure, Count, One } +import std.nat { Nat } import extdeps.api_rate_limit { UpstreamRateLimit, upstream_rate_limit_key } import product.capacity.quota { quota_partition, quota_root_pool } -import product.capacity.event_chain { PartitionId, HeadExpectation, HeadAbsent, HeadAt, ChainEvent } -import product.capacity.pool_events { PoolEvent, PoolSettled, SeatRequest } +import product.capacity.event_chain { PartitionId } +import product.capacity.pool_events { PoolSettled, SeatRequest } import product.capacity.lease { LeasePolicy, LeaseGrant, FencedResource } import gunbc.fabric_event_log_host { HostEventLogStore, HostStoreResolved, HostStoreRefused, event_log_store_for_host, now_epoch_seconds } import gunbc.fabric_storage_client { FabricStorageBinding } import gunbc.fabric_event_log { - event_log_refusal_wire, - HeadObservation, HeadObserved, HeadObservationRefused, event_log_observe_head, - EventAppend, EventAppended, EventAppendStale, EventAppendRefused, event_log_append, SeatAcquisition, SeatGranted, SeatFull, SeatContended, SeatAcquireRefused, fabric_seat_acquire, seat_acquisition_wire, + PoolEventAppend, PoolEventAppended, PoolEventContended, PoolEventAppendRefused, fabric_pool_event_append, pool_event_append_wire, } // A CALLER LEASES UPSTREAM QUOTA BEFORE IT SPENDS IT. The lease holds the calls it is about to @@ -61,46 +58,26 @@ type QuotaSettle = QuotaSettled { partition: PartitionId, actual: Nat } | QuotaSettleRefused { detail: String } -type SettleFold { - done: QuotaSettle? -} - -fn settle_attempt(lease: QuotaLease, actual: Nat, now: EpochSecs) -> QuotaSettle? { +// SETTLEMENT IS ONE PARTITION APPEND, and the observe/append/retry loop it needs is the carrier's +// (gunbc.fabric_event_log fabric_pool_event_append), not a second copy here. It used to be a copy: +// this module held the only one until a compute reservation needed to release, at which point the +// loop would have existed twice and been free to disagree with itself about a contended head. +fn fabric_quota_settle(lease: QuotaLease, actual: Nat) -> QuotaSettle { match lease { - QuotaFull { wire: w } => Present { value: QuotaSettleRefused { detail: join(["nothing to settle: ", w], "") } } - QuotaLeaseRefused { detail: d } => Present { value: QuotaSettleRefused { detail: join(["nothing to settle: ", d], "") } } + QuotaFull { wire: w } => QuotaSettleRefused { detail: join(["nothing to settle: ", w], "") } + QuotaLeaseRefused { detail: d } => QuotaSettleRefused { detail: join(["nothing to settle: ", d], "") } QuotaLeased { grant: g, partition: p, store: store } => - match event_log_observe_head(store: store, partition: p) { - HeadObservationRefused { cause: c } => Present { value: QuotaSettleRefused { detail: event_log_refusal_wire(cause: c) } } - HeadObserved { head: head } => { - let event = ChainEvent { - partition: p, - parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, - recorded_at: now, - actor: g.reference, - payload: PoolSettled { reference: g.reference, actual: actual }, - } - match event_log_append(store: store, partition: p, event: event, expected: head) { - EventAppended { id: _ } => Present { value: QuotaSettled { partition: p, actual: actual } } - EventAppendStale { expected: _, observed: _ } => none - EventAppendRefused { cause: c } => Present { value: QuotaSettleRefused { detail: event_log_refusal_wire(cause: c) } } + match now_epoch_seconds() { + Absent => QuotaSettleRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match fabric_pool_event_append( + store: store, partition: p, actor: g.reference, at: now, + payload: PoolSettled { reference: g.reference, actual: actual }, attempts: 3, + ) { + PoolEventAppended { id: _ } => QuotaSettled { partition: p, actual: actual } + other => QuotaSettleRefused { detail: pool_event_append_wire(a: other) } } - } - } - } -} - -fn fabric_quota_settle(lease: QuotaLease, actual: Nat) -> QuotaSettle { - match now_epoch_seconds() { - Absent => QuotaSettleRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => { - let st = fold(nat_range_inclusive(lo: 1, hi: 3), init: SettleFold { done: none }, f: (acc, _i) => - match acc.done { Present { value: _ } => acc Absent => SettleFold { done: settle_attempt(lease: lease, actual: actual, now: now) } }) - match st.done { - Present { value: r } => r - Absent => QuotaSettleRefused { detail: "settlement contended three times" } } - } } } diff --git a/dag/gunbc/harness/harness_guidance.dag b/dag/gunbc/harness/harness_guidance.dag index 563571b4166..599e907dabb 100644 --- a/dag/gunbc/harness/harness_guidance.dag +++ b/dag/gunbc/harness/harness_guidance.dag @@ -157,7 +157,7 @@ fn harness_worker_guidance_at(worktree: String, test_command: String, test_recei grounded(premise: TestProcessIs { command: test_command, receipt_path: test_receipt_path }, module_path: "gunbc.compute.test_run", decl_name: "test_pattern_cli"), grounded(premise: CapabilityWithheld { capability: BuildOrTestOutsideTheProcess }, - module_path: "gunbc.compute.work_provider_local", decl_name: "compute_cold_build_cap"), + module_path: "gunbc.compute.host_capacity", decl_name: "compute_reserve"), ]) } diff --git a/dag/std/measure.dag b/dag/std/measure.dag index e922d41da36..dad1d7a99f4 100644 --- a/dag/std/measure.dag +++ b/dag/std/measure.dag @@ -328,6 +328,16 @@ fn kibibyte_to_byte_size(k: Kibibyte) -> ByteSize { byte_size(count: kibibyte_count(k) * kibi_factor()) } +// THE INVERSE OF kibibyte_to_byte_size, FLOORING. A byte count that is not a whole number of +// kibibytes has no exact kibibyte reading, and the floor is the only rounding a CAPACITY may take: +// rounding up would report a ceiling the host does not have, and a reservation admitted against it +// would be admitted against memory that is not there. Named _floor for the same reason +// measure_scale_fraction_floor is -- the direction is part of what the caller is asking for, not an +// implementation detail it may discover later. +fn kibibyte_from_byte_size_floor(b: ByteSize) -> Kibibyte { + kibibyte(count: byte_size_count(b: b) / kibi_factor()) +} + fn mebibyte_to_byte_size(m: Mebibyte) -> ByteSize { byte_size(count: mebibyte_count(m) * mebibyte_scale_factor_bytes()) } diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag new file mode 100644 index 00000000000..235a64b7828 --- /dev/null +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -0,0 +1,156 @@ +module test.claim.compute.host_capacity_wet_witness + +import extdeps.shell +import std.types { Bool, Int, NonEmptyStr, String } +import std.measure { Kibibyte, kibibyte, kibibyte_count } +import std.algebra { trim } +import product.capacity.event_chain { PartitionId } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, dashboard_instance_compute_root } +import gunbc.compute.host_capacity { + ComputeCapacitySubject, compute_capacity_subject, compute_capacity_root, compute_memory_partition, + HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, + ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, + compute_reserve_on, compute_release, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, +} + +// WET CONTROLS FOR THE HOST'S COMPUTE MEMORY POOL. Every reservation below is a real append to a +// real event log on a real temporary directory, decided against a real reading of this machine's +// /proc/meminfo. Nothing here is a fixture standing in for the store: the linearization being +// asserted is the one gunbc.compute.work_provider_local runs. +// +// THE SUBJECT IS SUPPLIED AND THE JOIN THAT PRODUCES ONE IS ASSERTED SEPARATELY (DESIGN 3, the +// pairing obligation): a ComputeCapacitySubject is four values, and deriving it here would mean +// standing up a whole deployment record to exercise arithmetic that does not depend on one. The +// inhabitance claim is the_deployment_join_roots_the_ledger_under_the_instance_compute_root below, +// which runs the real producer. + +data one_gibibyte_in_kibibytes: Int = 1048576 + +fn fresh_root() -> String { + shell.Mktemp.DirWithTemplate(template: "/tmp/gunbc_compute_capacity.XXXXXX").path +} + +fn subject_at(root: String, appropriation_kib: Int, request_kib: Int) -> ComputeCapacitySubject { + ComputeCapacitySubject { + root: join([trim(s: root), "/capacity"], "") as NonEmptyStr, + partition: "compute-memory-wet-witness" as PartitionId, + appropriation: kibibyte(count: appropriation_kib), + request: kibibyte(count: request_kib), + } +} + +fn reserved_amount(r: ComputeReservation) -> Int { + match r { + ComputeReserved { grant: _, partition: _, amount: a, store: _ } => kibibyte_count(k: a) + ComputeCapacityFull { wire: _ } => 0 + ComputeHostBelowFloor { requested: _, available: _ } => 0 + ComputeReservationRefused { detail: _ } => 0 + } +} + +fn is_capacity_full(r: ComputeReservation) -> Bool { + match r { + ComputeCapacityFull { wire: w } => string_contains(s: w, pattern: "pool-full") + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeHostBelowFloor { requested: _, available: _ } => false + ComputeReservationRefused { detail: _ } => false + } +} + +fn released(r: ComputeRelease) -> Bool { + match r { + ComputeReleased { partition: _, reference: _, reason: _ } => true + ComputeReleaseRefused { detail: _ } => false + } +} + +fn standing_committed(s: ComputeCapacityStanding) -> Int { + match s { + ComputeCapacityStandingRead { committed: c, ceiling: _, headroom: _, observed_available: _, observed_total: _ } => kibibyte_count(k: c) + ComputeCapacityStandingUnread { detail: _ } => 0 - 1 + } +} + +// THE CONTROL THIS WHOLE CHANGE EXISTS FOR, AND IT IS RED UNDER THE THING IT REPLACED. A pool of two +// gibibytes admits two one-gibibyte reservations and REFUSES THE THIRD; the deleted admission -- +// count the lease directory, compare, then create -- admitted the third whenever the two lease +// files belonged to different identities, because a count of names excludes no quantity. Then the +// release returns exactly what it held and the next request is admitted again, which is the other +// half: capacity that never comes back is a wall, not a pool. +// +// MUTATION CONTROL: raise the appropriation above the sum of the three requests and the third +// reservation is admitted, so this cell goes red for the reason it names rather than for any +// property of the store. +test fn competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity() -> Bool { + let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes * 2, request_kib: one_gibibyte_in_kibibytes) + let first = compute_reserve_on(subject: subject, reference: "wet-first" as NonEmptyStr, term_seconds: 60) + let second = compute_reserve_on(subject: subject, reference: "wet-second" as NonEmptyStr, term_seconds: 60) + let full = compute_reserve_on(subject: subject, reference: "wet-third" as NonEmptyStr, term_seconds: 60) + let at_capacity = compute_capacity_standing_on(subject: subject) + let back = compute_release(reservation: first, reason: "witness completion" as NonEmptyStr) + let after_release = compute_capacity_standing_on(subject: subject) + let readmitted = compute_reserve_on(subject: subject, reference: "wet-fourth" as NonEmptyStr, term_seconds: 60) + reserved_amount(r: first) == one_gibibyte_in_kibibytes + && reserved_amount(r: second) == one_gibibyte_in_kibibytes + && is_capacity_full(r: full) + && standing_committed(s: at_capacity) == one_gibibyte_in_kibibytes * 2 + && released(r: back) + && standing_committed(s: after_release) == one_gibibyte_in_kibibytes + && reserved_amount(r: readmitted) == one_gibibyte_in_kibibytes +} + +// A RESERVATION THE MACHINE CANNOT BACK IS REFUSED BEFORE THE LEDGER IS TOUCHED, and it is a +// DIFFERENT refusal from a full pool: the pool's own commitments are irrelevant here (this ledger is +// empty), and the remedy is to find out what else is on the host rather than to wait for a seat. +// The request is the machine's own installed memory, which is strictly above what it reports +// available on any host that is running anything at all -- including the one running this witness. +test fn a_request_the_machine_cannot_back_refuses_below_the_live_floor() -> Bool { + match observe_host_memory() { + HostMemoryUnobserved { detail: _ } => false + HostMemoryObserved { total: total, available: available } => { + let installed = kibibyte_count(k: total) + let subject = subject_at(root: fresh_root(), appropriation_kib: installed, request_kib: installed) + kibibyte_count(k: available) < installed + && (match compute_reserve_on(subject: subject, reference: "wet-floor" as NonEmptyStr, term_seconds: 60) { + ComputeHostBelowFloor { requested: rq, available: av } => kibibyte_count(k: rq) == installed && kibibyte_count(k: av) > 0 + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeReservationRefused { detail: _ } => false + }) + } + } +} + +// AN APPROPRIATION LARGER THAN THE MACHINE IS A CONFIGURATION ERROR AND SAYS SO, rather than +// presenting hours later as a build the kernel killed. It is refused ahead of the floor check, +// because a share that was never backed is wrong whatever this instant's availability happens to be. +test fn an_appropriation_larger_than_the_machine_refuses_rather_than_admitting() -> Bool { + match observe_host_memory() { + HostMemoryUnobserved { detail: _ } => false + HostMemoryObserved { total: total, available: _ } => { + let subject = subject_at(root: fresh_root(), appropriation_kib: kibibyte_count(k: total) + 1, request_kib: one_gibibyte_in_kibibytes) + match compute_reserve_on(subject: subject, reference: "wet-oversized" as NonEmptyStr, term_seconds: 60) { + ComputeReservationRefused { detail: d } => string_contains(s: d, pattern: "never backed by the machine") + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeHostBelowFloor { requested: _, available: _ } => false + } + } + } +} + +// THE INHABITANCE CLAIM FOR EVERY SUPPLIED SUBJECT ABOVE (DESIGN 3). The real producer is the +// deployment join, and what it must establish is the ROUTE and not only the shape: the ledger a +// live instance reserves against is rooted under THAT INSTANCE'S compute root, and its partition +// names that instance, so two instances on one machine cannot silently transact on one pool. +test fn the_deployment_join_roots_the_ledger_under_the_instance_compute_root() -> Bool { + let instance = srv1_live_dashboard_instance() + let subject = compute_capacity_subject(instance: instance) + let compute_root = dashboard_instance_compute_root(instance: instance) as String + starts_with(s: subject.root as String, prefix: compute_root) + && (subject.root as String) == join([compute_root, "/capacity"], "") + && string_contains(s: subject.partition as String, pattern: instance.instance_id as String) + && kibibyte_count(k: subject.appropriation) == kibibyte_count(k: subject.request) * 2 + && kibibyte_count(k: subject.request) > 0 +} diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 6d57f7a0a71..385963a5879 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -6,7 +6,7 @@ import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, DeclaredOutput, WorkOutputRoot, CheckoutRoot, TargetRoot, - WorkOutput, WorkOutcome, WorkSucceeded, WorkFailed, WorkRefusedByInfrastructure, HostAtColdBuildCap, + WorkOutput, WorkOutcome, WorkSucceeded, WorkFailed, WorkRefusedByInfrastructure, HostComputeCapacityFull, work_outcome_wire, stored_outcome_decode, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_is_success, } @@ -64,12 +64,12 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool let subject = tree(hex: tree_a) let ok = work_outcome_wire(o: WorkSucceeded { identity: "abc", outputs: [WorkOutput { declared: "target/release/gunbc", blob: "deadbeef", store_path: "/s/gunbc" }], log_path: "/l" }, subject: subject, op: work_default_build()) let failed = work_outcome_wire(o: WorkFailed { identity: "abc", exit_code: 101, log_path: "/l" }, subject: subject, op: work_default_build()) - let refused = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: "abc", cause: HostAtColdBuildCap { in_flight: 2, cap: 2 } }, subject: subject, op: work_default_build()) + let refused = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: "abc", cause: HostComputeCapacityFull { wire: "pool-full (attempt 1)" } }, subject: subject, op: work_default_build()) (match stored_outcome_decode(text: ok) { StoredOutcomeFound { ending, identity, document: _ } => ending == "succeeded" && identity == "abc" StoredOutcomeUnreadable { reason: _ } => false }) && stored_outcome_is_success(r: stored_outcome_decode(text: ok)) && !stored_outcome_is_success(r: stored_outcome_decode(text: failed)) && (match stored_outcome_decode(text: refused) { StoredOutcomeFound { ending, identity: _, document: _ } => ending == "refused_by_infrastructure" StoredOutcomeUnreadable { reason: _ } => false }) - && string_contains(s: refused, pattern: "cap 2") + && string_contains(s: refused, pattern: "pool-full") && string_contains(s: ok, pattern: "\"blob\": \"deadbeef\"") && (match stored_outcome_decode(text: "{\"identity\": \"abc\"}") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) && (match stored_outcome_decode(text: "not json") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) diff --git a/dag/test/claim/os_proc_meminfo_witness_test.dag b/dag/test/claim/os_proc_meminfo_witness_test.dag index eb6c7b69ebb..b2dc1a40705 100644 --- a/dag/test/claim/os_proc_meminfo_witness_test.dag +++ b/dag/test/claim/os_proc_meminfo_witness_test.dag @@ -24,3 +24,34 @@ test fn proc_meminfo_witnesses() -> Bool { && measure_count(m: info.mem_available) <= measure_count(m: info.mem_total) && measure_count(m: metric.value) > 0 } + +// THE PARSE, AND THE TWO THINGS IT MUST NOT DO. A field the kernel did not publish may not read as +// a zero (a host with no MemAvailable line would otherwise present as a host with no memory +// available, and an admission would refuse forever for the wrong reason), and a line whose value is +// not a decimal count may not contribute a metric. Both are asserted against the same text with one +// line changed, so the assertion discriminates the parse rather than the fixture. +test fn parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one() -> Bool { + let text = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: 8388608 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nHugePages_Total: 0\n" + let without_available = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" + let unparseable = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: not-a-number kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" + (match parse_proc_meminfo(text: text) { + MeminfoParsed { meminfo: m } => + kibibyte_count(k: m.mem_total) == 16777216 + && kibibyte_count(k: m.mem_available) == 8388608 + && kibibyte_count(k: m.swap_total) == 0 + && count(m.all_metrics) == 8 + MeminfoFieldAbsent { field: _ } => false + }) + && (match parse_proc_meminfo(text: without_available) { + MeminfoFieldAbsent { field: f } => (f as String) == "MemAvailable" + MeminfoParsed { meminfo: _ } => false + }) + && (match parse_proc_meminfo(text: unparseable) { + MeminfoFieldAbsent { field: f } => (f as String) == "MemAvailable" + MeminfoParsed { meminfo: _ } => false + }) + && (match parse_proc_meminfo(text: "") { + MeminfoFieldAbsent { field: f } => (f as String) == "MemTotal" + MeminfoParsed { meminfo: _ } => false + }) +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 8357ffbf07e..60626c07334 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -244,6 +244,30 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // probe failed without output rather than running its payload. fn local_repo_wet_schedule() -> List { [ + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_request_the_machine_cannot_back_refuses_below_the_live_floor" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "a_request_the_machine_cannot_back_refuses_below_the_live_floor", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "an_appropriation_larger_than_the_machine_refuses_rather_than_admitting" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "an_appropriation_larger_than_the_machine_refuses_rather_than_admitting", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "the_deployment_join_roots_the_ledger_under_the_instance_compute_root" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "the_deployment_join_roots_the_ledger_under_the_instance_compute_root", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.effect_plan_bash_materialize_real_execution_witness", function: "effect_plan_bash_two_declared_operations_execute" }, entry: "dag/test/claim/effect_plan_bash_materialize_real_execution_witness_test.dag", From 9bf00d7a3b4c0cd8f8290ead44805f2f0ef75090 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 14:44:23 +0000 Subject: [PATCH 02/44] The meminfo parse resolves each field by the lookup that proves it present review 69561 on #11962. The presence pass walked meminfo_field_names and refused the first missing name; the record was then built from a SECOND spelling of those names, each falling back to kibibyte(count: 0) on an absence the presence pass had supposedly ruled out. Two lists for one concept is DESIGN 3's nicknaming, and the fork was load-bearing: the zero's unreachability was an invariant held AT A DISTANCE between roster and record, so renaming or dropping a roster entry made mem_total and mem_available read 0 KiB -- which compute_reserve_on turns into a permanent below-floor or appropriation-exceeds-total refusal ATTRIBUTED TO THE MACHINE rather than to the parse. A fabricated plausible output behind a guard that looked like a wall (DESIGN 5). The guard is deleted rather than strengthened. Each field is matched once, its absence names itself, and its value is the one that match just bound: meminfo_required and its zero are gone, so the fabricated reading has no constructor left (DESIGN 4b). meminfo_field_names no longer claims to be this fold's authority -- what the parse demands is exactly what ProcMeminfo has fields for, a fact the record carries and cannot drift from itself. The witness gains the control the absence arms cannot supply: every field carries a DISTINCT value and every field is asserted, so a name paired with the wrong field is red; and a field missing from the MIDDLE of the file names itself. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/linux/proc_meminfo.dag | 88 +++++++++++++------ .../claim/os_proc_meminfo_witness_test.dag | 21 ++++- 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/dag/extdeps/linux/proc_meminfo.dag b/dag/extdeps/linux/proc_meminfo.dag index 010a19d0381..abca8818e29 100644 --- a/dag/extdeps/linux/proc_meminfo.dag +++ b/dag/extdeps/linux/proc_meminfo.dag @@ -91,33 +91,71 @@ fn meminfo_metric_named(metrics: List, name: NonEmptyStr) -> Kibib } } -// THE SEVEN NAMED FIELDS ARE REQUIRED TOGETHER, in the order meminfo_field_names declares them, so -// the roster above is the authority for what this parse demands rather than a second list here. +// EVERY NAMED FIELD IS RESOLVED BY THE SAME LOOKUP THAT PROVES IT PRESENT, and that is the whole +// shape of this fold rather than a stylistic preference. +// +// IT WAS WRITTEN THE OTHER WAY FIRST AND THE OTHER WAY WAS WRONG (review 69561 on gunbc#11962). A +// presence pass walked meminfo_field_names and refused the first name it could not find; the record +// was then built from a SECOND spelling of those names, each one falling back to +// `kibibyte(count: 0)` on an absence the presence pass had supposedly ruled out. Two lists for one +// concept is DESIGN 3's nicknaming, and here the fork was load-bearing rather than cosmetic: the +// zero's unreachability was an invariant maintained AT A DISTANCE between the roster and the record, +// so renaming or dropping a roster entry made mem_total and mem_available read 0 KiB -- which a +// compute admission (gunbc.compute.host_capacity compute_reserve_on) turns into a permanent +// below-floor or appropriation-exceeds-total refusal ATTRIBUTED TO THE MACHINE rather than to the +// parse. A fabricated plausible output (DESIGN 5) behind a guard that looked like a wall. +// +// So the guard is deleted rather than strengthened. Each field is matched once, its absence names +// ITSELF, and its value is the one the match just bound: `meminfo_required` and its zero are gone, +// and the fabricated reading has no constructor left to be written in (DESIGN 4b, structurally +// impossible rather than validated). meminfo_field_names is consequently NOT this fold's authority +// and no longer claims to be -- it is the roster of fields this module models, read by consumers +// that want the names; what this parse demands is exactly what ProcMeminfo has fields for, which is +// a fact the record already carries and cannot drift from itself. +// +// The nesting is flat in meaning and deep in shape on purpose: there is no total lookup over seven +// independent optionals that does not either lose which one was missing or introduce a positional +// list, and a positional list would be the same fork one representation over. fn parse_proc_meminfo(text: String) -> MeminfoRead { let metrics = meminfo_metrics(text: text) - match first(filter(meminfo_field_names, f => match meminfo_metric_named(metrics: metrics, name: f) { Absent => true Present { value: _ } => false })) { - Present { value: missing } => MeminfoFieldAbsent { field: missing } - Absent => - MeminfoParsed { - meminfo: ProcMeminfo { - mem_total: meminfo_required(metrics: metrics, name: "MemTotal" as NonEmptyStr), - mem_free: meminfo_required(metrics: metrics, name: "MemFree" as NonEmptyStr), - mem_available: meminfo_required(metrics: metrics, name: "MemAvailable" as NonEmptyStr), - buffers: meminfo_required(metrics: metrics, name: "Buffers" as NonEmptyStr), - cached: meminfo_required(metrics: metrics, name: "Cached" as NonEmptyStr), - swap_total: meminfo_required(metrics: metrics, name: "SwapTotal" as NonEmptyStr), - swap_free: meminfo_required(metrics: metrics, name: "SwapFree" as NonEmptyStr), - all_metrics: metrics, - }, + match meminfo_metric_named(metrics: metrics, name: "MemTotal" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "MemTotal" as NonEmptyStr } + Present { value: mem_total } => + match meminfo_metric_named(metrics: metrics, name: "MemFree" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "MemFree" as NonEmptyStr } + Present { value: mem_free } => + match meminfo_metric_named(metrics: metrics, name: "MemAvailable" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "MemAvailable" as NonEmptyStr } + Present { value: mem_available } => + match meminfo_metric_named(metrics: metrics, name: "Buffers" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "Buffers" as NonEmptyStr } + Present { value: buffers } => + match meminfo_metric_named(metrics: metrics, name: "Cached" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "Cached" as NonEmptyStr } + Present { value: cached } => + match meminfo_metric_named(metrics: metrics, name: "SwapTotal" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "SwapTotal" as NonEmptyStr } + Present { value: swap_total } => + match meminfo_metric_named(metrics: metrics, name: "SwapFree" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "SwapFree" as NonEmptyStr } + Present { value: swap_free } => + MeminfoParsed { + meminfo: ProcMeminfo { + mem_total: mem_total, + mem_free: mem_free, + mem_available: mem_available, + buffers: buffers, + cached: cached, + swap_total: swap_total, + swap_free: swap_free, + all_metrics: metrics, + }, + } + } + } + } + } + } } } } - -// Reached only under the absence check above, which is why it can answer at all; the zero is the -// unreachable arm of an optional, not a fabricated reading. -fn meminfo_required(metrics: List, name: NonEmptyStr) -> Kibibyte { - match meminfo_metric_named(metrics: metrics, name: name) { - Present { value: k } => k - Absent => kibibyte(count: 0) - } -} diff --git a/dag/test/claim/os_proc_meminfo_witness_test.dag b/dag/test/claim/os_proc_meminfo_witness_test.dag index b2dc1a40705..debf04c5405 100644 --- a/dag/test/claim/os_proc_meminfo_witness_test.dag +++ b/dag/test/claim/os_proc_meminfo_witness_test.dag @@ -30,15 +30,25 @@ test fn proc_meminfo_witnesses() -> Bool { // available, and an admission would refuse forever for the wrong reason), and a line whose value is // not a decimal count may not contribute a metric. Both are asserted against the same text with one // line changed, so the assertion discriminates the parse rather than the fixture. +// +// EVERY FIELD CARRIES A DISTINCT VALUE AND EVERY FIELD IS ASSERTED, because with seven lookups +// written by hand a name paired with the WRONG field is the one defect the absence arms cannot +// catch. And a field missing from the MIDDLE of the file names itself: that arm used to be +// unreachable only by an invariant held between two lists, and is now unreachable by construction +// -- there is no fallback left that could answer 0 KiB for it (review 69561 on gunbc#11962). test fn parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one() -> Bool { - let text = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: 8388608 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nHugePages_Total: 0\n" + let text = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: 8388608 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 2097152 kB\nSwapFree: 524288 kB\nHugePages_Total: 0\n" let without_available = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" let unparseable = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: not-a-number kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" (match parse_proc_meminfo(text: text) { MeminfoParsed { meminfo: m } => kibibyte_count(k: m.mem_total) == 16777216 + && kibibyte_count(k: m.mem_free) == 1048576 && kibibyte_count(k: m.mem_available) == 8388608 - && kibibyte_count(k: m.swap_total) == 0 + && kibibyte_count(k: m.buffers) == 65536 + && kibibyte_count(k: m.cached) == 4194304 + && kibibyte_count(k: m.swap_total) == 2097152 + && kibibyte_count(k: m.swap_free) == 524288 && count(m.all_metrics) == 8 MeminfoFieldAbsent { field: _ } => false }) @@ -54,4 +64,11 @@ test fn parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one() -> Bo MeminfoFieldAbsent { field: f } => (f as String) == "MemTotal" MeminfoParsed { meminfo: _ } => false }) + && (match parse_proc_meminfo(text: join([ + "MemTotal: 16777216 kB\n", "MemFree: 1048576 kB\n", "MemAvailable: 8388608 kB\n", + "Buffers: 65536 kB\n", "Cached: 4194304 kB\n", "SwapFree: 524288 kB\n", + ], "")) { + MeminfoFieldAbsent { field: f } => (f as String) == "SwapTotal" + MeminfoParsed { meminfo: _ } => false + }) } From 047bb3eade1f727d61d673959effbc3f972765c0 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 15:15:26 +0000 Subject: [PATCH 03/44] Three defects the first real run on srv1 found, and the controls for them The workload ran under the grant on srv1: a real cargo release build of v1-compiler as gunbc-compute-1bf2556cc4b015f9.service, capped at the 26 GiB the pool granted, binaries returned by content. It also surfaced three defects that no wet cell could see, because none of them touched the provider. 1. THE CONSUMPTION READING BROKE THE ATTACH PATH. It was appended to outcome.json, which the attach path PARSES (stored_outcome_decode), so every later caller of a SUCCEEDED identity read StoredOutcomeUnreadable and rebuilt -- the contract's "two callers, one build" silently dead. Measured: the provider's own dependent compile request refused on a dependency that had in fact succeeded. The reading moves to its own file beside the log; a realization's measurement does not change the shape of the contract's document. 2. THE RESERVATION REFERENCE WAS THE WORK IDENTITY. The identity is stable by construction -- that is its job -- so a second request for the same identity hit the encumbrance ledger's DuplicateReference. The pool's reference names one HOLD, so it is now per attempt; the identity stays in the receipt. 3. A LEDGER REFUSAL WAS REPORTED AS A FULL POOL. The seat carrier renders every pool refusal into SeatFull, so DuplicateReference reached an operator as "capacity full" on a host with 400 GiB free. product.capacity.pool_events names the marker it writes (pool_full_wire) so both sides read one authority, and anything else is carried through as a ledger refusal. The typed-arm fix widens SeatAcquisition and every match on SeatFull, so it is a declared frontier with its trigger rather than a silent widen here. New wet control, red under the conflation: a reused reference on a pool with room for both requests is a ledger refusal and never ComputeCapacityFull. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 14 +++++++-- dag/gunbc/compute/work_provider_local.dag | 26 +++++++++++++++-- dag/gunbc/product/capacity/pool_events.dag | 18 +++++++++++- .../host_capacity_wet_witness_test.dag | 29 +++++++++++++++++++ src/v2/workflow/local_repo_wet_terminal.dag | 6 ++++ 5 files changed, 88 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index 51a74c7550d..f386c1c890d 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -7,7 +7,7 @@ import extdeps.linux.procfs import extdeps.linux.proc_meminfo { MeminfoRead, MeminfoParsed, MeminfoFieldAbsent, parse_proc_meminfo } import product.capacity.pool { Pool, pool_of, NoReplenishment, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at } import product.capacity.event_chain { PartitionId } -import product.capacity.pool_events { SeatRequest, PoolReleased } +import product.capacity.pool_events { SeatRequest, PoolReleased, pool_full_wire } import product.capacity.lease { LeasePolicy, LeaseGrant, QuiescenceRequired } import gunbc.fabric_storage_client { FabricStorageBinding, FabricStorageLocalFiles } import gunbc.fabric_storage_file_store { fabric_storage_file_root, fabric_storage_file_root_ensure, FabricStorageRootStanding, FabricStorageRootReady, FabricStorageRootRefused } @@ -228,6 +228,11 @@ fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, t } } +// A REFUSED ACQUIRE IS NOT AUTOMATICALLY A FULL POOL. The carrier renders every pool refusal into +// SeatFull, so the reason survives only in the wire; a DuplicateReference reported as capacity sent +// an operator looking for a busy host while srv1 had 400 GiB free (2026-09-21). The marker is read +// from the authority that writes it (product.capacity.pool_events pool_full_wire), never re-spelled +// here, and anything else is a ledger refusal with its text carried through. fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: EpochSecs, term_seconds: Nat) -> ComputeReservation { let store = compute_capacity_store(subject: subject) match fabric_seat_acquire( @@ -241,7 +246,12 @@ fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: Ep budget: compute_partition_read_budget, ) { SeatGranted { grant: g, generation: _, attempts_used: _ } => ComputeReserved { grant: g, partition: subject.partition, amount: subject.request, store: store } - SeatFull { wire: w, attempts_used: n } => ComputeCapacityFull { wire: join([w, " (attempt ", to_string(value: n), ")"], "") } + SeatFull { wire: w, attempts_used: n } => + if starts_with(s: w, prefix: pool_full_wire) { + ComputeCapacityFull { wire: join([w, " (attempt ", to_string(value: n), ")"], "") } + } else { + ComputeReservationRefused { detail: join(["the capacity ledger refused the acquire: ", w, " (attempt ", to_string(value: n), ")"], "") } + } other => ComputeReservationRefused { detail: seat_acquisition_wire(a: other) } } } diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 5d357f8598b..433cb1bd7e2 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -65,6 +65,7 @@ type ComputeLayout { target_dir: String log_path: String outcome_path: String + consumption_path: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -81,6 +82,7 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp target_dir: join([root, "/target"], ""), log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), + consumption_path: join([work_dir, "/consumption"], ""), } } @@ -299,7 +301,7 @@ fn compute_provide_leaf( if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) } else { - let reservation = compute_reserve(instance: instance, reference: identity as NonEmptyStr, term_seconds: compute_reservation_term_seconds) + let reservation = compute_reserve(instance: instance, reference: compute_reservation_reference(identity: identity), term_seconds: compute_reservation_term_seconds) match reservation { ComputeCapacityFull { wire: w } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) @@ -321,7 +323,8 @@ fn compute_provide_leaf( // is no ending on which the seat stays held. let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) let document = work_outcome_wire(o: outcome, subject: subject, op: op) - let written = Filesystem.Write(path: layout.outcome_path, content: join([document, "\n", compute_capacity_standing_wire(s: standing), "\n"], "")) + let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) + let written = Filesystem.Write(path: layout.outcome_path, content: document) let released = Filesystem.Delete(path: layout.lease_path) if written.success && released.success && compute_release_landed(r: returned) { ProvideFresh { outcome: outcome, document: document } @@ -334,6 +337,7 @@ fn compute_provide_leaf( "outcome written: ", if written.success { "yes" } else { written.error }, "; lease released: ", if released.success { "yes" } else { released.error }, "; reservation released: ", compute_release_wire(r: returned), + "; consumption recorded: ", if measured.success { "yes" } else { measured.error }, "; work ended ", work_outcome_ending(o: outcome), ], ""), }, @@ -346,6 +350,24 @@ fn compute_provide_leaf( } } +// THE CONSUMPTION READING IS ITS OWN FILE AND MUST NOT JOIN THE OUTCOME DOCUMENT. It was appended +// to outcome.json first, and the first real run on srv1 (2026-09-21) is what said why that is +// wrong: the stored outcome is PARSED by the attach path (stored_outcome_decode), so a line +// appended after the JSON made every later caller of a SUCCEEDED identity read +// StoredOutcomeUnreadable and rebuild -- the contract's "two callers, one build" silently dead, and +// the provider's own dependent request refusing on a dependency that had in fact succeeded. A +// realization's measurement does not get to change the shape of the contract's document. + +// THE RESERVATION REFERENCE NAMES AN ATTEMPT, NOT A WORK IDENTITY, and the first real run on srv1 +// (2026-09-21) is what forced the distinction. The identity is stable by construction -- that is its +// whole job -- so using it as the encumbrance reference meant the SECOND request for the same +// identity hit the ledger's DuplicateReference refusal and was reported as a capacity refusal on a +// host with 400 GiB free. The pool's reference is the name of one HOLD, and two holds of one +// identity at different times are two holds; the identity stays in the receipt, where it belongs. +fn compute_reservation_reference(identity: String) -> NonEmptyStr { + join([identity, "@", clock_now_probed_at_or_unknown() as String], "") as NonEmptyStr +} + // A REQUEST REFUSED FOR CAPACITY HOLDS NOTHING. The producer lease is dropped on the way out, so a // host that is full does not accumulate names that would refuse the next attempt for the wrong // reason -- a capacity refusal reported as a producer already in flight is the state conflation diff --git a/dag/gunbc/product/capacity/pool_events.dag b/dag/gunbc/product/capacity/pool_events.dag index a7c765f7c2c..542f75c540f 100644 --- a/dag/gunbc/product/capacity/pool_events.dag +++ b/dag/gunbc/product/capacity/pool_events.dag @@ -186,11 +186,27 @@ type SeatProposal = SeatProposed { event: ChainEvent, expected_head: HeadExpectation } | SeatRefused { wire: String } +// THE ONE SPELLING OF "THE POOL HAD NO ROOM", named because a consumer has to be able to tell that +// refusal from every other reason an acquire can be refused and the wire is currently the only place +// the distinction survives. SeatRefused carries a rendered string for all of them, so a consumer +// that wants "full" versus "the ledger said no" reads THIS row rather than authoring a second +// literal of its own (DESIGN 3). The reason it matters is measured: gunbc.compute.host_capacity +// mapped every SeatRefused onto a capacity refusal, and a DuplicateReference -- a ledger fact with +// nothing to do with capacity -- was reported to an operator as a full pool on a host with 400 GiB +// free (srv1, 2026-09-21). +// +// THE HONEST FIX IS A TYPED ARM AND IT IS NOT THIS. SeatRefused / SeatFull should carry the +// distinction in their constructors instead of in rendered text; that widens +// gunbc.fabric_event_log.SeatAcquisition and every existing match on SeatFull, so it is its own +// change. DECLARED FRONTIER (DESIGN 3c), trigger: the next consumer that needs a third +// classification, or the first one that must branch on a refusal this row cannot name. +data pool_full_wire: String = "pool-full" + fn seat_refusal_wire(r: PoolRefusal) -> String { match r { LedgerRefusal { refusal: lr } => match lr { - ExceedsAppropriation { requested: _, committed: _, ceiling: _ } => "pool-full" + ExceedsAppropriation { requested: _, committed: _, ceiling: _ } => pool_full_wire _ => pool_refusal_wire(r: r) } _ => pool_refusal_wire(r: r) diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index 235a64b7828..a751587d64f 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -58,6 +58,15 @@ fn is_capacity_full(r: ComputeReservation) -> Bool { } } +fn is_ledger_refusal(r: ComputeReservation) -> Bool { + match r { + ComputeReservationRefused { detail: d } => string_contains(s: d, pattern: "the capacity ledger refused the acquire") + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeHostBelowFloor { requested: _, available: _ } => false + } +} + fn released(r: ComputeRelease) -> Bool { match r { ComputeReleased { partition: _, reference: _, reason: _ } => true @@ -154,3 +163,23 @@ test fn the_deployment_join_roots_the_ledger_under_the_instance_compute_root() - && kibibyte_count(k: subject.appropriation) == kibibyte_count(k: subject.request) * 2 && kibibyte_count(k: subject.request) > 0 } + +// A LEDGER REFUSAL IS NOT A CAPACITY REFUSAL, and this cell exists because the two were conflated in +// production and the conflation was only visible on a real host. The reservation reference names one +// HOLD; re-using it is a DuplicateReference from the encumbrance ledger, which the seat carrier +// renders into the same SeatFull arm a genuinely exhausted pool reaches. Reported as capacity, it +// sent a reader looking for a busy machine while srv1 had 400 GiB free (2026-09-21) -- and the +// provider's own dependent request refused on a dependency that had succeeded. +// +// The pool here has room for BOTH requests, so nothing about capacity can explain the second +// answer: if this cell ever reports ComputeCapacityFull again, the conflation is back. +test fn a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool() -> Bool { + let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes * 4, request_kib: one_gibibyte_in_kibibytes) + let first = compute_reserve_on(subject: subject, reference: "wet-same-reference" as NonEmptyStr, term_seconds: 60) + let again = compute_reserve_on(subject: subject, reference: "wet-same-reference" as NonEmptyStr, term_seconds: 60) + let other = compute_reserve_on(subject: subject, reference: "wet-other-reference" as NonEmptyStr, term_seconds: 60) + reserved_amount(r: first) == one_gibibyte_in_kibibytes + && is_ledger_refusal(r: again) + && !is_capacity_full(r: again) + && reserved_amount(r: other) == one_gibibyte_in_kibibytes +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 60626c07334..2ee01ca1795 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -244,6 +244,12 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // probe failed without output rather than running its payload. fn local_repo_wet_schedule() -> List { [ + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", From dd240f0d171f7e8049c309cc8f40e7e6444282da Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 15:40:36 +0000 Subject: [PATCH 04/44] Body-position annotations refused by gunbc compile; move them to module grain The fourth thing the srv1 run found, and the one that makes the case for running it. DESIGN 4c admits standalone leading blocks attached to MODULE-SCOPE declarations only. The witness route this provider is covered by tolerates a block inside a function body; `gunbc compile` refuses it outright. So work_provider_local carried 14 blocking errors while every witness over it reported green, and the real compile request on srv1 is what said so -- the provider had stopped compiling and nothing in the wet or hermetic evidence could see it. The prose is not deleted: it moves above compute_provide_leaf, which is the declaration it was describing. Verified: `gunbc compile --entry dag/gunbc/compute/work_provider_local.dag` reports 0 blocking errors at this head, against 14 before. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 37 ++++++++++++++--------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 433cb1bd7e2..e06833d05e2 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -272,6 +272,29 @@ fn compute_basename(path: String) -> String { // makes the retry cheap; what must never happen is a caller attaching to a failure it did not ask for); refuse at the cap or behind a lease; take the lease; check out; // run; return outputs; write the outcome; release the lease. A refusal before the lease leaves no // state; a refusal after it leaves the lease released and the outcome written. +// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease +// excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the +// reservation is a QUANTITY on the host, and no exclusive create of any name settles that. The +// lease is taken first because a request that then loses on capacity releases a name it holds +// rather than memory it may never use, and a name is the cheaper of the two to undo. +// +// THE WORK RUNS UNDER THE GRANT: the transient unit's MemoryMax is the amount the pool reserved, so +// the ledger's arithmetic IS the enforcement rather than a parallel description of it. +// +// AGGREGATE CONSUMPTION IS READ WHILE THE GRANT IS STILL HELD, which is the only instant at which +// the ledger's committed total and the machine's own availability describe the same population; +// after the release the reservation is gone from one side and the memory may not yet be gone from +// the other. +// +// COMPLETION AND CANCELLATION RELEASE THE SAME WAY. Every ending of compute_run_leased -- succeeded, +// failed, output-mismatch, refused by infrastructure -- reaches the release, so there is no ending +// on which the seat stays held. +// +// THE COMMENTS ABOVE USED TO SIT INSIDE THIS FUNCTION'S BODY, and the first real compile of this +// module on srv1 (2026-09-21) is what moved them: DESIGN 4c admits standalone leading blocks +// attached to MODULE-SCOPE declarations only, and the interpreter route this provider's witnesses +// run on tolerates a body-position block that `gunbc compile` refuses outright -- 14 blocking +// errors on a module every witness had reported green. fn compute_provide_leaf( instance: HostDashboardInstance, commit: String, @@ -292,11 +315,6 @@ fn compute_provide_leaf( } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { - // THE PRODUCER LEASE AND THE CAPACITY RESERVATION ARE TWO FACTS AND ARE TAKEN IN THIS ORDER. - // The lease excludes a SECOND PRODUCER OF THIS IDENTITY and is a name, so O_EXCL settles it; the - // reservation is a QUANTITY on the host and no exclusive name can settle that. Taking the lease - // first means a request that then loses on capacity releases a name it holds rather than - // reserving memory it may never use, and it is the cheaper of the two to undo. let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) @@ -310,17 +328,8 @@ fn compute_provide_leaf( ComputeReservationRefused { detail: d } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => { - // THE WORK RUNS UNDER THE GRANT: the unit's MemoryMax is the amount the pool reserved, so - // the ledger's arithmetic is the enforcement rather than a parallel description of it. let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) - // AGGREGATE CONSUMPTION IS READ WHILE THE GRANT IS STILL HELD, which is the only instant - // at which the ledger's committed total and the machine's own availability describe the - // same population; after the release the reservation is gone from one side and the - // memory may not yet be gone from the other. let standing = compute_capacity_standing(instance: instance) - // COMPLETION AND CANCELLATION RELEASE THE SAME WAY. Every ending of compute_run_leased -- - // succeeded, failed, output-mismatch, refused by infrastructure -- reaches here, so there - // is no ending on which the seat stays held. let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) let document = work_outcome_wire(o: outcome, subject: subject, op: op) let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) From 725d9c58620b1fe993ee43a52e9081b00e7ea316 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 16:33:22 +0000 Subject: [PATCH 05/44] Four design defects: the double promise, the instance-keyed pool, the record that disagreed with its caller, and the recovery route that did not exist Side-chat review of #11962 at dd240f0d, plus review 69595. All four are real and all four were verified against the code before anything was changed. A. THE DOUBLE PROMISE. Pool capacity and live MemAvailable were checked separately, so with a 52 GiB appropriation, 30 GiB available and two 26 GiB requests, each request saw 26 <= 30 and each fit the appropriation, and BOTH admitted -- neither could see the other's granted-but-not-yet-consumed memory. The compare-and-set was doing its job perfectly on a question that was missing a term. The fix is not a third check. The pool already computes the committed total inside the linearized fold, so the machine's reading belongs in the SAME comparison: the ceiling an attempt is adjudicated against is the smaller of the declared appropriation and what the host says is free. This is worst-case-sound and conservative, and says so: MemAvailable already excludes what outstanding grants have touched, so a resident grant is counted twice. The exact relation needs the resident share, which is observable per unit and is a declared frontier with its trigger. product.capacity.pool already ruled that replay never adjudicates the ceiling, so a moving ceiling cannot poison the fold -- it makes the pool over-committed, which refuses new acquires until it drains. B. THE POOL WAS KEYED BY THE INSTANCE, NOT THE HOST. srv1 declares two instances -- srv1-live and srv1-lab -- with different ids and roots, and the memory they spend is ONE machine's. Both halves are now host-keyed: the partition names the host identity, and the ledger is rooted under the instance that host resolves to through dashboard_instance_for_host, the same host-to-instance authority the fabric event log already uses. A host that authority cannot resolve reserves nothing rather than defaulting to the caller. C. THE RECORD DISAGREED WITH THE CALLER. A run whose work succeeded but whose release failed wrote WorkSucceeded to outcome.json and handed its caller a refusal -- and the next caller ATTACHED to that success, so the leaked seat had no consumer that would ever look for it. The ending and the unresolved obligation are two facts and are recorded as two: the obligation is its own durable file, and the outcome document carries the refusal the caller was given, so a later caller attaches to exactly what this caller was told. D. THE RECOVERY ROUTE DID NOT EXIST (review 69595). The module's prose said a wedged seat is recovered by an explicit release naming why; compute_release took a ComputeReservation whose only constructor lives and dies inside one compute_provide_leaf, so under QuiescenceRequired -- which never lapses -- nothing in the corpus could discharge it, and two such events would consume the whole appropriation and turn every later request into a capacity refusal on a host with its memory free. A claimed route no declaration reaches is the dangling consumption DESIGN 3c refuses. compute_release_reference takes the reference, which is all a recovering operator has, and compute_release_cli is its executing consumer. Both doors go through fabric_seat_release, the checked mirror of fabric_seat_acquire: read, fold, PROPOSE against the folded pool, append only what it admits. That check is load-bearing rather than defensive -- an unheld reference appended anyway would not fail at the append, it would fail at every later READ, when the fold hits UnknownEncumbrance and refuses the whole partition. Controls, all executing wet, seven cells: two requests summing past the observed headroom cannot both admit (RED when the backing ceiling is reverted to the appropriation); two instances on one host resolve to one pool at one root; a held seat is recoverable by reference and an unheld one writes nothing, asserted beside a later successful reservation so the ledger is shown still readable. gunbc compile on the provider closure: 0 blocking errors. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 192 ++++++++++++++--- dag/gunbc/compute/work_provider_local.dag | 193 +++++++++++++++--- dag/gunbc/fabric/fabric_event_log.dag | 98 ++++++++- .../host_capacity_wet_witness_test.dag | 93 +++++++-- src/v2/workflow/local_repo_wet_terminal.dag | 16 +- 5 files changed, 520 insertions(+), 72 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index f386c1c890d..f54b9e10398 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -15,10 +15,14 @@ import gunbc.fabric_event_log { SeatAcquisition, SeatGranted, SeatFull, SeatContended, SeatAcquireRefused, SeatStoreRefused, fabric_seat_acquire, seat_acquisition_wire, SeatStandingObservation, SeatRoomObserved, SeatFullObserved, SeatStandingUnobserved, SeatStandingStoreRefused, fabric_seat_observe, + SeatRelease, SeatReleased, SeatNotHeld, SeatReleaseContended, SeatReleaseRefused, fabric_seat_release, seat_release_wire, PoolEventAppend, PoolEventAppended, PoolEventContended, PoolEventAppendRefused, fabric_pool_event_append, pool_event_append_wire, } import gunbc.fabric_event_log_host { now_epoch_seconds } -import gunbc.roadmap_dashboard_instance { HostDashboardInstance, dashboard_instance_compute_root } +import gunbc.roadmap_dashboard_instance { + HostDashboardInstance, dashboard_instance_compute_root, + DashboardInstanceByHost, DashboardInstanceForHost, DashboardInstanceHostUnknown, dashboard_instance_for_host, +} // THE HOST'S COMPUTE MEMORY IS A POOL, AND A WORK REQUEST TAKES A SEAT IN IT BEFORE IT RUNS. // @@ -72,21 +76,47 @@ type ComputeCapacitySubject { request: Kibibyte } -fn compute_capacity_root(instance: HostDashboardInstance) -> NonEmptyStr { - join([dashboard_instance_compute_root(instance: instance) as String, "/capacity"], "") as NonEmptyStr +// THE POOL IS A HOST'S, AND KEYING IT BY THE INSTANCE WAS A MIS-KEYING (side-chat review of +// gunbc#11962 at dd240f0d, 2026-09-21). srv1 declares TWO dashboard instances -- srv1-live and +// srv1-lab -- with different ids and different instance roots, and the memory they would spend is +// ONE machine's. An instance-keyed partition gives each of them its own appropriation and its own +// ledger, so two pools each promise the same bytes and neither can see the other: the exclusion the +// compare-and-set provides is perfect and is about the wrong subject. +// +// So both halves are keyed by the HOST. The partition names the host identity, and the ledger is +// rooted under the instance THAT HOST resolves to -- gunbc.roadmap_dashboard_instance +// dashboard_instance_for_host, the same host-to-instance authority the fabric event log already +// uses to decide where a host's log lives. Two instances on one host therefore resolve to one +// partition at one root, which is the single host-scoped authority this fact needs; inventing a new +// host-scoped path would have been a second one. +// +// A HOST THAT AUTHORITY CANNOT RESOLVE RESERVES NOTHING. It is not defaulted to the calling +// instance: that is exactly the fork this fix removes, re-entering through a fallback. +type ComputeSubjectResolution + = ComputeSubjectResolved { subject: ComputeCapacitySubject } + | ComputeSubjectHostUnresolved { host: NonEmptyStr } + +fn compute_capacity_root(host_instance: HostDashboardInstance) -> NonEmptyStr { + join([dashboard_instance_compute_root(instance: host_instance) as String, "/capacity"], "") as NonEmptyStr } -fn compute_memory_partition(instance: HostDashboardInstance) -> PartitionId { - join(["compute-memory-", instance.instance_id as String], "") as PartitionId +fn compute_memory_partition(host: NonEmptyStr) -> PartitionId { + join(["compute-memory-", host as String], "") as PartitionId } // THE JOIN WITH THE DEPLOYMENT, and the only function in this module that knows what an instance is. -fn compute_capacity_subject(instance: HostDashboardInstance) -> ComputeCapacitySubject { - ComputeCapacitySubject { - root: compute_capacity_root(instance: instance), - partition: compute_memory_partition(instance: instance), - appropriation: compute_memory_appropriation(instance: instance), - request: compute_request_memory(instance: instance), +fn compute_capacity_subject(instance: HostDashboardInstance) -> ComputeSubjectResolution { + match dashboard_instance_for_host(host: instance.host_identity as String) { + DashboardInstanceHostUnknown { host: h } => ComputeSubjectHostUnresolved { host: h as NonEmptyStr } + DashboardInstanceForHost { instance: host_instance } => + ComputeSubjectResolved { + subject: ComputeCapacitySubject { + root: compute_capacity_root(host_instance: host_instance), + partition: compute_memory_partition(host: instance.host_identity), + appropriation: compute_memory_appropriation(instance: host_instance), + request: compute_request_memory(instance: instance), + }, + } } } @@ -170,10 +200,45 @@ fn compute_lease_policy(term_seconds: Nat) -> LeasePolicy { LeasePolicy { maximum_duration_seconds: term_seconds, release_law: QuiescenceRequired } } -fn compute_memory_root_pool(subject: ComputeCapacitySubject) -> Pool { +// THE BACKING RELATION, AND IT IS THE WHOLE REASON THE TWO OBSERVATIONS ARE NOT INDEPENDENT CHECKS. +// +// The first revision compared the pool's committed total against the appropriation in one place and +// the machine's live availability against ONE request in another. Both checks passed and the pair +// still double-promised (side-chat review of gunbc#11962 at dd240f0d, 2026-09-21): with a 52 GiB +// appropriation, 30 GiB available and two 26 GiB requests, each request sees 26 <= 30 and each fits +// the appropriation, so BOTH admit -- because neither sees the other's granted-but-not-yet-consumed +// memory. The compare-and-set was doing its job perfectly on a question that was missing a term. +// +// The fix is not a third check. The pool already computes the committed total INSIDE the linearized +// fold, so the machine's reading belongs in the SAME comparison: the ceiling this attempt is +// adjudicated against is the smaller of the declared appropriation and what the host says is free. +// Then "everything this pool has promised, plus what I am asking for, must fit in what the machine +// actually has" is one predicate evaluated in one CAS'd step, and the second 26 GiB request is +// refused as a full pool because 26 + 26 > 30. +// +// WORST-CASE-SOUND, AND CONSERVATIVE, AND SAYING SO. MemAvailable already excludes whatever the +// pool's outstanding grants have actually touched, so a grant that is fully resident is counted +// twice -- once in the machine's reading and once in the committed total. That errs toward refusing +// work the host could have run, which is the direction a safety admission must err in when the term +// it needs is unobserved. The exact relation needs the RESIDENT share of outstanding grants, which +// is observable -- each grant runs in a named transient unit with its own cgroup -- and is not +// observed here. DECLARED FRONTIER (DESIGN 3c), trigger: a per-host read of the compute units' +// current usage, whose first consumer is this fold; until it lands the conservative bound stands +// and the cost is stated rather than hidden. +// +// A CEILING THAT MOVES BETWEEN ATTEMPTS IS SAFE BY CONSTRUCTION HERE, which is why this is a ceiling +// and not a pre-check. product.capacity.pool ruled that replay does not adjudicate the ceiling -- +// an event's admission happened when it was written -- so a dip in availability cannot make a +// historical acquire refuse the fold and poison the partition. It makes the pool over-committed, +// which refuses new acquires until it drains. That is the correct consequence of a reduction. +fn compute_backing_ceiling(subject: ComputeCapacitySubject, available: Kibibyte) -> Kibibyte { + if kibibyte_count(k: available) < kibibyte_count(k: subject.appropriation) { available } else { subject.appropriation } +} + +fn compute_memory_root_pool(subject: ComputeCapacitySubject, ceiling: Kibibyte) -> Pool { pool_of( identity: subject.partition as String as NonEmptyStr, - ceiling: subject.appropriation, + ceiling: ceiling, replenishment: NoReplenishment, release_law: QuiescenceRequired, ) @@ -198,7 +263,11 @@ data compute_partition_read_budget: Nat = 4096 // the declared share is a configuration error that would otherwise present as an OOM kill during a // build, which is the same information arriving hours later and destructively. fn compute_reserve(instance: HostDashboardInstance, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { - compute_reserve_on(subject: compute_capacity_subject(instance: instance), reference: reference, term_seconds: term_seconds) + match compute_capacity_subject(instance: instance) { + ComputeSubjectHostUnresolved { host: h } => + ComputeReservationRefused { detail: join(["no dashboard instance is declared for host ", h as String, ", so this host has no compute capacity authority to reserve against"], "") } + ComputeSubjectResolved { subject: subject } => compute_reserve_on(subject: subject, reference: reference, term_seconds: term_seconds) + } } fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { @@ -221,7 +290,7 @@ fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, t FabricStorageRootReady => match now_epoch_seconds() { Absent => ComputeReservationRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds) + Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds, ceiling: compute_backing_ceiling(subject: subject, available: available)) } } } @@ -233,12 +302,12 @@ fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, t // an operator looking for a busy host while srv1 had 400 GiB free (2026-09-21). The marker is read // from the authority that writes it (product.capacity.pool_events pool_full_wire), never re-spelled // here, and anything else is a ledger refusal with its text carried through. -fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: EpochSecs, term_seconds: Nat) -> ComputeReservation { +fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: EpochSecs, term_seconds: Nat, ceiling: Kibibyte) -> ComputeReservation { let store = compute_capacity_store(subject: subject) match fabric_seat_acquire( store: store, partition: subject.partition, - root: compute_memory_root_pool(subject: subject), + root: compute_memory_root_pool(subject: subject, ceiling: ceiling), actor: reference, request: SeatRequest { reference: reference, amount: kibibyte_count(k: subject.request), at: now, term_seconds: term_seconds }, policy: compute_lease_policy(term_seconds: term_seconds), @@ -275,16 +344,78 @@ fn compute_release(reservation: ComputeReservation, reason: NonEmptyStr) -> Comp ComputeHostBelowFloor { requested: _, available: _ } => ComputeReleaseRefused { detail: "nothing to release: the host was below its live memory floor" } ComputeReservationRefused { detail: d } => ComputeReleaseRefused { detail: join(["nothing to release: ", d], "") } ComputeReserved { grant: g, partition: p, amount: _, store: store } => - match now_epoch_seconds() { - Absent => ComputeReleaseRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => - match fabric_pool_event_append( - store: store, partition: p, actor: g.reference, at: now, - payload: PoolReleased { reference: g.reference, reason: reason }, attempts: compute_seat_attempts, - ) { - PoolEventAppended { id: _ } => ComputeReleased { partition: p, reference: g.reference, reason: reason } - other => ComputeReleaseRefused { detail: pool_event_append_wire(a: other) } - } + compute_release_on(store: store, partition: p, appropriation: kibibyte(count: 0), reference: g.reference, reason: reason) + } +} + +// THE RECOVERY ROUTE, AND IT EXISTS BECAUSE THE MODULE CLAIMED IT (review 69595 on gunbc#11962). +// +// The prose above says a wedged seat is recovered by an explicit release naming why. Until this +// function that was FALSE: compute_release took a ComputeReservation, whose only constructor is +// compute_reserve_on, whose result lives and dies inside one compute_provide_leaf invocation. So a +// release that failed to land left an encumbrance that -- under QuiescenceRequired, which by design +// never lapses -- nothing in the corpus could ever discharge. Two of those consume the whole host +// appropriation, and every later request then gets a CAPACITY refusal on a host with its memory +// free: the exact conflation this change added a witness to prevent, arriving through a different +// door. A claimed route that no declaration reaches is the dangling consumption DESIGN 3c refuses. +// +// IT TAKES A REFERENCE, because that is all a recovering operator has: the reservation value is +// gone with the process that held it, and the reference is what the ledger and the obligation file +// both name. It is CHECKED against the folded pool before anything is appended -- a reference the +// pool is not holding writes nothing and refuses by name, because appending a release for an unheld +// reference would make the partition unfoldable for every later reader. +fn compute_release_reference(instance: HostDashboardInstance, reference: NonEmptyStr, reason: NonEmptyStr) -> ComputeRelease { + match compute_capacity_subject(instance: instance) { + ComputeSubjectHostUnresolved { host: h } => + ComputeReleaseRefused { detail: join(["no dashboard instance is declared for host ", h as String], "") } + ComputeSubjectResolved { subject: subject } => + compute_release_on( + store: compute_capacity_store(subject: subject), + partition: subject.partition, + appropriation: subject.appropriation, + reference: reference, + reason: reason, + ) + } +} + +// ONE RELEASE PATH FOR BOTH DOORS. The provider's own completion and the operator's recovery append +// the same event against the same check; a second spelling of "give the seat back" is the fork this +// module would otherwise grow the moment recovery existed. +// +// THE APPROPRIATION PASSED HERE IS NOT LOAD-BEARING AND THE ZERO IS NOT A FABRICATED READING. product.capacity.pool +// ruled that replay never adjudicates the ceiling, so a release is decided by whether the ledger +// HOLDS the reference and by nothing about capacity; a reservation released from its own provider +// carries no subject to read an appropriation from, and inventing one for it would be a number with +// no consequence. Where a subject is at hand it is passed; where none is, zero is passed, and zero +// cannot admit anything -- which is the right direction for a value nothing should be deciding on. +fn compute_release_on( + store: FabricStorageBinding, + partition: PartitionId, + appropriation: Kibibyte, + reference: NonEmptyStr, + reason: NonEmptyStr, +) -> ComputeRelease { + match now_epoch_seconds() { + Absent => ComputeReleaseRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match fabric_seat_release( + store: store, + partition: partition, + root: pool_of( + identity: partition as String as NonEmptyStr, + ceiling: appropriation, + replenishment: NoReplenishment, + release_law: QuiescenceRequired, + ), + reference: reference, + reason: reason, + at: now, + attempts: compute_seat_attempts, + budget: compute_partition_read_budget, + ) { + SeatReleased { reference: r, id: _ } => ComputeReleased { partition: partition, reference: r, reason: reason } + other => ComputeReleaseRefused { detail: seat_release_wire(r: other) } } } } @@ -302,7 +433,10 @@ type ComputeCapacityStanding | ComputeCapacityStandingUnread { detail: String } fn compute_capacity_standing(instance: HostDashboardInstance) -> ComputeCapacityStanding { - compute_capacity_standing_on(subject: compute_capacity_subject(instance: instance)) + match compute_capacity_subject(instance: instance) { + ComputeSubjectHostUnresolved { host: h } => ComputeCapacityStandingUnread { detail: join(["no dashboard instance is declared for host ", h as String], "") } + ComputeSubjectResolved { subject: subject } => compute_capacity_standing_on(subject: subject) + } } fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapacityStanding { @@ -315,7 +449,7 @@ fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapac match fabric_seat_observe( store: compute_capacity_store(subject: subject), partition: subject.partition, - root: compute_memory_root_pool(subject: subject), + root: compute_memory_root_pool(subject: subject, ceiling: subject.appropriation), at: now, budget: compute_partition_read_budget, ) { diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index e06833d05e2..0fb60514c62 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -23,7 +23,7 @@ import gunbc.roadmap_execution_contract { import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size } import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, + ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_release_reference, compute_reservation_wire, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, } import gunbc.compute.work_request { @@ -66,6 +66,7 @@ type ComputeLayout { log_path: String outcome_path: String consumption_path: String + obligation_path: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -83,6 +84,7 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), consumption_path: join([work_dir, "/consumption"], ""), + obligation_path: join([work_dir, "/unreleased-reservation"], ""), } } @@ -272,6 +274,127 @@ fn compute_basename(path: String) -> String { // makes the retry cheap; what must never happen is a caller attaching to a failure it did not ask for); refuse at the cap or behind a lease; take the lease; check out; // run; return outputs; write the outcome; release the lease. A refusal before the lease leaves no // state; a refusal after it leaves the lease released and the outcome written. +// SETTLING A RUN THAT HAS FINISHED, AND THE ONE RULE THAT GOVERNS IT: THE DURABLE RECORD AND THE +// CALLER MUST AGREE ABOUT WHAT HAPPENED. +// +// They did not. The outcome document was written unconditionally, so a run whose work SUCCEEDED but +// whose reservation failed to release wrote WorkSucceeded to outcome.json and handed its caller a +// WorkRefusedByInfrastructure -- and the next caller of that identity ATTACHED to the stored +// success, which is the contract behaving correctly on a record that had quietly become a lie about +// the infrastructure's state. The leaked seat then had no consumer that would ever look for it +// (side-chat review of gunbc#11962 at dd240f0d, 2026-09-21). +// +// THE RESOLUTION IS NOT TO SUPPRESS THE SUCCESS. The work really did succeed and its outputs really +// are in the store; deleting that fact would make every later caller rebuild and would be a second +// lie in the other direction. The run's ending and the infrastructure's unresolved obligation are +// TWO FACTS, so they are recorded as two: the obligation is written as its own durable file, and +// the outcome document carries the refusal the caller is given -- so a later caller attaches to +// EXACTLY what this caller was told, and neither of them attaches to a success whose reservation is +// still outstanding. The run's own ending is never lost: it is named in the obligation and in the +// refusal, and the outputs stay in the store under their content identity. +// +// THE SEAT STAYS HELD, AND THAT IS THE SAFE DIRECTION rather than an omission. The pool's release +// law for a compute seat is quiescence-required precisely because nothing here can prove a unit +// stopped using memory; a release that did not land leaves the capacity encumbered, so the host +// under-admits until the obligation is discharged rather than handing the same bytes to the next +// arrival. Discharging it is an explicit release naming why -- the same entry completion and +// cancellation take. +fn compute_run_and_settle( + instance: HostDashboardInstance, + commit: String, + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + dependency_store: String, + granted: Kibibyte, + reservation: ComputeReservation, +) -> ProvideResult { + let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) + let standing = compute_capacity_standing(instance: instance) + let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) + let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) + let released = Filesystem.Delete(path: layout.lease_path) + let notes = join([ + "; producer lease released: ", if released.success { "yes" } else { released.error }, + "; consumption recorded: ", if measured.success { "yes" } else { measured.error }, + ], "") + if compute_release_landed(r: returned) { + compute_settled_record( + layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, + lease_released: released.success, notes: notes) + } else { + compute_unreleased_record( + layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, + reservation: reservation, returned: returned, notes: notes) + } +} + +// THE ORDINARY ENDING: the reservation came back, so the record is the run's own ending and the +// caller is told the same thing. +fn compute_settled_record( + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + outcome: WorkOutcome, + lease_released: Bool, + notes: String, +) -> ProvideResult { + let document = work_outcome_wire(o: outcome, subject: subject, op: op) + let written = Filesystem.Write(path: layout.outcome_path, content: document) + if written.success && lease_released { + ProvideFresh { outcome: outcome, document: document } + } else { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { + detail: join([ + "the work ended ", work_outcome_ending(o: outcome), + " and its reservation was released, but the record could not be completed -- outcome written: ", + if written.success { "yes" } else { written.error }, notes, + ], ""), + }, + }, + subject: subject, op: op) + } +} + +// THE ENDING WITH AN OUTSTANDING OBLIGATION: one document, written once, given to this caller and +// read by the next one. +fn compute_unreleased_record( + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + outcome: WorkOutcome, + reservation: ComputeReservation, + returned: ComputeRelease, + notes: String, +) -> ProvideResult { + let obligation = join([ + "unreleased reservation: ", compute_reservation_wire(r: reservation), + "; release attempt: ", compute_release_wire(r: returned), + "; the work itself ended ", work_outcome_ending(o: outcome), + " and any declared outputs are in the store; the seat stays held until an explicit release ", + "discharges it, so this host under-admits rather than promising the same memory twice", + ], "") + let recorded = Filesystem.Write(path: layout.obligation_path, content: join([obligation, "\n"], "")) + let cause = ComputeCapacityLedgerUnavailable { + detail: join([ + obligation, "; obligation recorded: ", if recorded.success { layout.obligation_path } else { recorded.error }, notes, + ], ""), + } + let document = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) + let written = Filesystem.Write(path: layout.outcome_path, content: document) + if written.success { + ProvideFresh { outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, document: document } + } else { + compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) + } +} + // THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the // reservation is a QUANTITY on the host, and no exclusive create of any name settles that. The @@ -327,33 +450,10 @@ fn compute_provide_leaf( compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested_kib: kibibyte_count(k: rq), available_kib: kibibyte_count(k: av) }) ComputeReservationRefused { detail: d } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) - ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => { - let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) - let standing = compute_capacity_standing(instance: instance) - let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) - let document = work_outcome_wire(o: outcome, subject: subject, op: op) - let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) - let written = Filesystem.Write(path: layout.outcome_path, content: document) - let released = Filesystem.Delete(path: layout.lease_path) - if written.success && released.success && compute_release_landed(r: returned) { - ProvideFresh { outcome: outcome, document: document } - } else { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: StoreUnavailable { - detail: join([ - "outcome written: ", if written.success { "yes" } else { written.error }, - "; lease released: ", if released.success { "yes" } else { released.error }, - "; reservation released: ", compute_release_wire(r: returned), - "; consumption recorded: ", if measured.success { "yes" } else { measured.error }, - "; work ended ", work_outcome_ending(o: outcome), - ], ""), - }, - }, - subject: subject, op: op) - } - } + ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => + compute_run_and_settle( + instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, + dependency_store: dependency_store, granted: granted, reservation: reservation) } } } @@ -529,6 +629,43 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent } } +// THE OPERATOR'S RECOVERY DOOR, and the executing consumer of compute_release_reference. A +// reservation whose release did not land leaves its reference in +// /unreleased-reservation and in the partition; this is how it is given back. It is a +// verb on the same CLI rather than a script, so the refusal vocabulary is the module's: a reference +// the pool is not holding writes nothing and exits non-zero saying so, which is the answer a +// mistyped reference must get -- appending a release for an unheld reference would make the +// partition unfoldable for every later reader. +fn compute_release_cli(repo_root: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { + match dashboard_instance_for_repo_root(repo_root: repo_root) { + DashboardInstanceRootUnknown { repo_root: r } => ExitFailure { code: 2, reason: join(["compute: no dashboard instance is rooted at ", r], "") } + DashboardInstanceResolved { instance } => + if reference == "" || reason == "" { + ExitFailure { code: 2, reason: "compute: release needs a non-empty reference and a reason naming why the seat is being given back" } + } else { + let released = compute_release_reference(instance: instance, reference: reference as NonEmptyStr, reason: reason as NonEmptyStr) + let wire = compute_release_cli_wire(r: released) + let receipt = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) + if !receipt.success { + ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; release was ", wire], "") } + } else { + match released { + ComputeReleased { partition: _, reference: _, reason: _ } => ExitSuccess + ComputeReleaseRefused { detail: _ } => ExitFailure { code: 1, reason: join(["compute release refused: ", wire], "") } + } + } + } + } +} + +fn compute_release_cli_wire(r: ComputeRelease) -> String { + match r { + ComputeReleased { partition: p, reference: ref_, reason: why } => + join(["released ", ref_ as String, " on ", p as String, " (", why as String, ")"], "") + ComputeReleaseRefused { detail: d } => join(["refused: ", d], "") + } +} + fn compute_parse_operation(operation: String, entry: String, functions: String, hermetic: String) -> WorkOperation? { if operation == "build" { Present { value: work_default_build() } diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index ec7ee6ef88f..d7ae49970ac 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -16,7 +16,7 @@ import std.fabric_storage { import gunbc.fabric_storage_placement { fabric_storage_placement, FabricStoragePlaced, FabricStorageUnplaced } import gunbc.fabric_storage_client { FabricStorageBinding, fabric_storage_binding_for, fabric_storage_head, fabric_storage_put, fabric_storage_advance, fabric_storage_closure } import gunbc.fabric_storage_file_store { fabric_storage_file_root, fabric_storage_file_root_ensure, FabricStorageRootReady, FabricStorageRootRefused } -import product.capacity.pool { Pool, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at } +import product.capacity.pool { Pool, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at, PoolOutcome, PoolAdvanced, PoolRefused, pool_release, pool_refusal_wire } import product.capacity.event_chain { PartitionId, EventId, ChainEvent, ChainEnvelope, HeadExpectation, HeadAbsent, HeadAt, head_expectation_eq, event_parent_expectation, AppendDecision, AppendAdmitted, AppendStale, ChainWalk, ChainWalked, ChainIncomplete, ChainBudgetExhausted, chain_from_head, @@ -25,6 +25,7 @@ import product.capacity.event_chain { import product.capacity.pool_events { PoolEvent, pool_event_wire_text, pool_event_decode, PoolFold, PoolFolded, PoolFoldRefused, pool_fold, SeatRequest, SeatProposal, SeatProposed, SeatRefused, propose_acquire, grant_from_admission, + PoolReleased, } import product.capacity.lease { LeasePolicy, LeaseGrant, release_law_eq, release_law_wire } @@ -402,6 +403,101 @@ fn pool_event_append_wire(a: PoolEventAppend) -> String { } } +// RELEASING A SEAT, AND IT IS CHECKED AGAINST THE FOLDED POOL BEFORE ANYTHING IS APPENDED. +// +// fabric_pool_event_append is unchecked by construction -- it writes the event a caller hands it -- +// which is correct for a settlement whose amount the caller already knows and WRONG for a release, +// because a release names a reference the ledger may not be holding. An unheld reference appended +// anyway does not fail at the append: it fails at the next READ, when pool_fold reaches the event, +// gets UnknownEncumbrance, and answers PoolFoldRefused for the WHOLE PARTITION. One mistyped +// reference in a recovery command would make a host's capacity ledger permanently unreadable, and +// the damage would surface at an unrelated caller's next acquire. +// +// So this is the mirror of fabric_seat_acquire and not of the settlement helper: read the partition, +// fold it, PROPOSE the release against the folded pool, and append only what the pool admits. A +// reference the pool is not holding is a typed refusal that writes nothing. +// +// ITS CONSUMER IS THE RECOVERY ROUTE A HELD SEAT NEEDS (gunbc.compute.host_capacity +// compute_release_reference, reached from the operator CLI compute_release_cli). Under +// QuiescenceRequired a term never frees capacity, so an explicit release is the ONLY way a seat +// comes back, and a module claiming that route owes one that executes (review 69595 on gunbc#11962, +// DESIGN 3c). +type SeatRelease + = SeatReleased { reference: NonEmptyStr, id: EventId } + | SeatNotHeld { reference: NonEmptyStr, wire: String } + | SeatReleaseContended { attempts: Nat } + | SeatReleaseRefused { step: String, reason: String } + +type SeatReleaseState { + done: SeatRelease? +} + +fn seat_release_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, reference: NonEmptyStr, reason: NonEmptyStr, at: EpochSecs, budget: Nat) -> SeatRelease? { + match event_log_read_partition(store: store, partition: partition, budget: budget) { + PartitionReadRefused { cause: c } => Present { value: SeatReleaseRefused { step: "store", reason: event_log_refusal_wire(cause: c) } } + PartitionReadOk { head: head, walk: walk } => + match walk { + ChainIncomplete { missing: m, newest_first_so_far: _ } => Present { value: SeatReleaseRefused { step: "chain", reason: join(["partition chain is missing event ", m as String], "") } } + ChainBudgetExhausted { at: a } => Present { value: SeatReleaseRefused { step: "chain", reason: join(["partition chain exceeded the read budget at ", a as String], "") } } + ChainWalked { oldest_first: xs } => + match pool_fold(root: root, oldest_first: xs) { + PoolFoldRefused { at_event: e, wire: w } => Present { value: SeatReleaseRefused { step: "fold", reason: join(["event ", e as String, " refused: ", w], "") } } + PoolFolded { pool: p, generation: _ } => + match pool_release(pool: p, reference: reference, reason: reason, at: at) { + PoolRefused { refusal: r } => Present { value: SeatNotHeld { reference: reference, wire: pool_refusal_wire(r: r) } } + PoolAdvanced { pool: _ } => + match event_log_append( + store: store, + partition: partition, + event: ChainEvent { + partition: partition, + parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, + recorded_at: at, + actor: reference, + payload: PoolReleased { reference: reference, reason: reason }, + }, + expected: head, + ) { + EventAppended { id: h } => Present { value: SeatReleased { reference: reference, id: h } } + EventAppendStale { expected: _, observed: _ } => none + EventAppendRefused { cause: c } => Present { value: SeatReleaseRefused { step: "store", reason: event_log_refusal_wire(cause: c) } } + } + } + } + } + } +} + +fn fabric_seat_release(store: FabricStorageBinding, partition: PartitionId, root: Pool, reference: NonEmptyStr, reason: NonEmptyStr, at: EpochSecs, attempts: Nat, budget: Nat) -> SeatRelease { + let st = fold(nat_range_inclusive(lo: 1, hi: attempts), init: SeatReleaseState { done: none }, f: (acc, _i) => + match acc.done { + Present { value: _ } => acc + Absent => SeatReleaseState { done: seat_release_attempt(store: store, partition: partition, root: root, reference: reference, reason: reason, at: at, budget: budget) } + }) + match st.done { + Present { value: d } => d + Absent => SeatReleaseContended { attempts: attempts } + } +} + +fn seat_release_wire(r: SeatRelease) -> String { + match r { + SeatReleased { reference: ref_, id: h } => join(["released ", ref_ as String, " at ", h as String], "") + SeatNotHeld { reference: ref_, wire: w } => join(["the pool is not holding ", ref_ as String, ": ", w], "") + SeatReleaseContended { attempts: n } => join(["contended after ", to_string(n), " attempts"], "") + SeatReleaseRefused { step: s, reason: why } => join(["refused at ", s, ": ", why], "") + } +} + +fn seat_release_landed(r: SeatRelease) -> Bool { + match r { + SeatReleased { reference: _, id: _ } => true + SeatNotHeld { reference: _, wire: _ } => false + SeatReleaseContended { attempts: _ } => false + SeatReleaseRefused { step: _, reason: _ } => false + } +} + fn seat_acquisition_wire(a: SeatAcquisition) -> String { match a { SeatGranted { grant: g, generation: gen, attempts_used: n } => join(["granted ", g.reference as String, " fence=", g.fence.grant as String, "@", to_string(gen), " expires_at=", to_string(g.expires_at), " attempt=", to_string(n)], "") diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index a751587d64f..34c3d13f302 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -5,12 +5,14 @@ import std.types { Bool, Int, NonEmptyStr, String } import std.measure { Kibibyte, kibibyte, kibibyte_count } import std.algebra { trim } import product.capacity.event_chain { PartitionId } -import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, dashboard_instance_compute_root } +import gunbc.compute.host_capacity { compute_capacity_store } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, srv1_lab_dashboard_instance, dashboard_instance_compute_root } import gunbc.compute.host_capacity { - ComputeCapacitySubject, compute_capacity_subject, compute_capacity_root, compute_memory_partition, + ComputeCapacitySubject, ComputeSubjectResolution, ComputeSubjectResolved, ComputeSubjectHostUnresolved, + compute_capacity_subject, compute_capacity_root, compute_memory_partition, HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - compute_reserve_on, compute_release, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + compute_reserve_on, compute_release, compute_release_on, ComputeRelease, ComputeReleased, ComputeReleaseRefused, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, } @@ -153,17 +155,30 @@ test fn an_appropriation_larger_than_the_machine_refuses_rather_than_admitting() // deployment join, and what it must establish is the ROUTE and not only the shape: the ledger a // live instance reserves against is rooted under THAT INSTANCE'S compute root, and its partition // names that instance, so two instances on one machine cannot silently transact on one pool. -test fn the_deployment_join_roots_the_ledger_under_the_instance_compute_root() -> Bool { - let instance = srv1_live_dashboard_instance() - let subject = compute_capacity_subject(instance: instance) - let compute_root = dashboard_instance_compute_root(instance: instance) as String - starts_with(s: subject.root as String, prefix: compute_root) - && (subject.root as String) == join([compute_root, "/capacity"], "") - && string_contains(s: subject.partition as String, pattern: instance.instance_id as String) - && kibibyte_count(k: subject.appropriation) == kibibyte_count(k: subject.request) * 2 - && kibibyte_count(k: subject.request) > 0 +test fn two_instances_on_one_host_resolve_to_one_pool_at_one_root() -> Bool { + let live = srv1_live_dashboard_instance() + let lab = srv1_lab_dashboard_instance() + match compute_capacity_subject(instance: live) { + ComputeSubjectHostUnresolved { host: _ } => false + ComputeSubjectResolved { subject: a } => + match compute_capacity_subject(instance: lab) { + ComputeSubjectHostUnresolved { host: _ } => false + ComputeSubjectResolved { subject: b } => + (live.instance_id as String) != (lab.instance_id as String) + && (live.instance_root as String) != (lab.instance_root as String) + && (live.host_identity as String) == (lab.host_identity as String) + && (a.partition as String) == (b.partition as String) + && (a.root as String) == (b.root as String) + && string_contains(s: a.partition as String, pattern: live.host_identity as String) + && !string_contains(s: a.partition as String, pattern: lab.instance_id as String) + && (a.root as String) == join([dashboard_instance_compute_root(instance: live) as String, "/capacity"], "") + && kibibyte_count(k: a.appropriation) == kibibyte_count(k: a.request) * 2 + && kibibyte_count(k: a.request) > 0 + } + } } + // A LEDGER REFUSAL IS NOT A CAPACITY REFUSAL, and this cell exists because the two were conflated in // production and the conflation was only visible on a real host. The reservation reference names one // HOLD; re-using it is a DuplicateReference from the encumbrance ledger, which the seat carrier @@ -183,3 +198,57 @@ test fn a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool && !is_capacity_full(r: again) && reserved_amount(r: other) == one_gibibyte_in_kibibytes } + +// THE DOUBLE PROMISE, AND IT IS THE CELL THE COMPARE-AND-SET COULD NOT SUPPLY. Two requests whose +// SUM exceeds what the machine says is free, against a pool whose declared appropriation is large +// enough for both: the first admits, and the second must be refused because the pool's committed +// total and the host's own reading are adjudicated in ONE step. Before that, each request compared +// itself alone against the reading, both passed, and both fit the appropriation -- so both admitted +// and the host was promised memory it does not have. The linearization was never the defect. +// +// The appropriation here is deliberately FOUR requests wide, so nothing about the declared share +// can explain the refusal; only the machine's reading can. The backing ceiling is derived from the +// live observation, so the fixture states the relation rather than a number: it asks for slightly +// more than half of what the host reports free, twice. +test fn two_requests_summing_past_the_observed_headroom_cannot_both_admit() -> Bool { + match observe_host_memory() { + HostMemoryUnobserved { detail: _ } => false + HostMemoryObserved { total: _, available: available } => { + let each = kibibyte_count(k: available) / 2 + 1024 + let subject = subject_at(root: fresh_root(), appropriation_kib: each * 4, request_kib: each) + let first = compute_reserve_on(subject: subject, reference: "wet-headroom-a" as NonEmptyStr, term_seconds: 60) + let second = compute_reserve_on(subject: subject, reference: "wet-headroom-b" as NonEmptyStr, term_seconds: 60) + reserved_amount(r: first) == each + && is_capacity_full(r: second) + && each * 2 > kibibyte_count(k: available) + && each * 4 > each * 2 + } + } +} + +// A WEDGED SEAT CAN ACTUALLY BE RECOVERED, BY REFERENCE, BY SOMEONE WHO NEVER HELD THE RESERVATION. +// Under QuiescenceRequired a term frees nothing, so this route is the ONLY way capacity returns +// after a release fails to land -- and until review 69595 the module claimed it in prose while no +// declaration reached it. The cell holds the whole appropriation, releases it with nothing but the +// reference (the reservation value is deliberately not used), and shows the capacity is admissible +// again. +// +// AND A REFERENCE THE POOL IS NOT HOLDING WRITES NOTHING. That is the sharper half: an unheld +// reference appended anyway would not fail at the append, it would fail at every later READ, when +// the fold hits UnknownEncumbrance and refuses the WHOLE partition. So the refusal is asserted +// BESIDE a subsequent successful reservation, which is what establishes the ledger is still +// readable after the bad attempt. +test fn a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing() -> Bool { + let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes, request_kib: one_gibibyte_in_kibibytes) + let store = compute_capacity_store(subject: subject) + let held = compute_reserve_on(subject: subject, reference: "wet-wedged" as NonEmptyStr, term_seconds: 60) + let full = compute_reserve_on(subject: subject, reference: "wet-blocked" as NonEmptyStr, term_seconds: 60) + let bogus = compute_release_on(store: store, partition: subject.partition, appropriation: subject.appropriation, reference: "wet-never-held" as NonEmptyStr, reason: "operator recovery" as NonEmptyStr) + let recovered = compute_release_on(store: store, partition: subject.partition, appropriation: subject.appropriation, reference: "wet-wedged" as NonEmptyStr, reason: "operator recovery" as NonEmptyStr) + let readmitted = compute_reserve_on(subject: subject, reference: "wet-after-recovery" as NonEmptyStr, term_seconds: 60) + reserved_amount(r: held) == one_gibibyte_in_kibibytes + && is_capacity_full(r: full) + && !released(r: bogus) + && released(r: recovered) + && reserved_amount(r: readmitted) == one_gibibyte_in_kibibytes +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 2ee01ca1795..6675dbf5577 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -244,6 +244,18 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // probe failed without output rather than running its payload. fn local_repo_wet_schedule() -> List { [ + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "two_requests_summing_past_the_observed_headroom_cannot_both_admit" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "two_requests_summing_past_the_observed_headroom_cannot_both_admit", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", @@ -269,9 +281,9 @@ fn local_repo_wet_schedule() -> List { expectation: ExpectedToHold {} }, WetScheduledClaim { - identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "the_deployment_join_roots_the_ledger_under_the_instance_compute_root" }, + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "two_instances_on_one_host_resolve_to_one_pool_at_one_root" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", - function: "the_deployment_join_roots_the_ledger_under_the_instance_compute_root", + function: "two_instances_on_one_host_resolve_to_one_pool_at_one_root", expectation: ExpectedToHold {} }, WetScheduledClaim { From 9172cf7da57abc1fc6bbced1b028663a77b41b8a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 17:11:20 +0000 Subject: [PATCH 06/44] Quiescence is observed, obligations are per attempt, and the admission cells supply their own inputs Side-chat review of #11962 at 725d9c58. All three blockers verified against the code first; all three were real. 1. QUIESCENCE WAS DECLARED AND NEVER OBSERVED. compute_release ran unconditionally on what compute_run_leased returned, and that is the exit of `systemd-run --wait` -- a LAUNCHER, not the unit's parent. A launcher killed, timed out, or disconnected from the bus yields ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, so the seat's memory would be handed to the next arrival on top of a live cargo. product.capacity.lease already models this exactly (QuiescenceFact) and this provider was not consulting it. Settlement and recovery now both consume a termination decision bound to the actual unit, read through the existing systemd.Systemctl.ShowUserProperty. Only two arms free capacity: the manager has no record of the unit (never started, or ran and was collected -- the same fact for this decision), or it reports it stopped. Running keeps the charge, and so does UNOBSERVED, which is the arm a fail-open would swallow and the one a lost launcher produces. An ActiveState this fold has never seen is unobserved, not permission. The recovery CLI observes it too: an operator releasing "because the worker looked dead" was asserting the one fact nothing checked. A running unit is refused by name with the state that refused it. 2. OBLIGATIONS WERE KEYED BY IDENTITY WHILE RESERVATIONS ARE KEYED BY ATTEMPT, so a retry overwrote the first obligation and the first seat became unrecoverable -- the reference that was the only way to release it was gone. One obligation file per attempt, named by the reference, created exclusively so a second writer cannot replace one. A retry now CONSUMES that lifecycle state: an identity with an outstanding obligation refuses by name and points at the release verb rather than reserving on top of a charged seat and publishing over the record carrying it. And the producer lease is dropped LAST -- it was deleted before the outcome was written, so the exclusion held for the name and not for the publication it exists to protect. 3. THE HEADROOM FIXTURE DERIVED ITSELF FROM THE THING UNDER TEST. appropriation = 4 x (available/2 + 1MiB) exceeds MemTotal on a lightly loaded host, so the cell refused before reaching the comparison it existed to make -- and on a busy host it passed for unrelated reasons. It was flaky in both directions. compute_reserve_against takes the observation as a value and compute_reserve supplies the real one, so the admission cells state the relation exactly: each request fits available, two exceed it, two fit the appropriation, and the appropriation fits MemTotal. Not a skip -- the pairing obligation is discharged by the_real_observation_reads_this_machine, which asserts the real producer reads THIS machine, and by the srv1 receipts. Also, per the accepted (A): the relation is described as admission against an observed MemAvailable ESTIMATE, and explicitly not a no-OOM guarantee. The unit's MemoryMax bounds one workload; this stops the POOL from over-promising, and conflating the two would be the rung inflation DESIGN 4b forbids. Eight wet cells and the termination fold all pass; gunbc compile on the provider closure: 0 blocking errors. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 69 +++-- dag/gunbc/compute/work_provider_local.dag | 258 +++++++++++++++--- .../host_capacity_wet_witness_test.dag | 78 +++--- .../compute/work_request_witness_test.dag | 34 +++ src/v2/workflow/local_repo_wet_terminal.dag | 6 + 5 files changed, 348 insertions(+), 97 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index f54b9e10398..51ac97b922a 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -216,6 +216,14 @@ fn compute_lease_policy(term_seconds: Nat) -> LeasePolicy { // actually has" is one predicate evaluated in one CAS'd step, and the second 26 GiB request is // refused as a full pool because 26 + 26 > 30. // +// WHAT THIS ADMITS AGAINST IS AN ESTIMATE, AND IT IS NOT A NO-OOM GUARANTEE. MemAvailable is the +// kernel's own ESTIMATE of what a new workload could obtain without swapping, it moves between the +// reading and the grant, and nothing here constrains the processes outside this pool. So the claim +// this relation supports is "this pool does not knowingly promise more than the machine reported", +// which is a real property and is strictly weaker than "no admitted workload will be OOM-killed". +// The unit's own MemoryMax is what bounds a single workload; this is what stops the POOL from +// over-promising. Conflating the two would be the rung inflation DESIGN 4b forbids. +// // WORST-CASE-SOUND, AND CONSERVATIVE, AND SAYING SO. MemAvailable already excludes whatever the // pool's outstanding grants have actually touched, so a grant that is fully resident is counted // twice -- once in the machine's reading and once in the committed total. That errs toward refusing @@ -270,30 +278,53 @@ fn compute_reserve(instance: HostDashboardInstance, reference: NonEmptyStr, term } } +// THE DECISION TAKES THE OBSERVATION AS A VALUE, AND THE DOOR ABOVE TAKES THE READING. +// +// It used to read /proc/meminfo itself, which made the admission arithmetic untestable except +// against whatever the host happened to be doing: the headroom control had to DERIVE its fixture +// from the live reading, and a fixture derived from the thing under test is not a fixture. It was +// wrong in both directions -- on a lightly loaded host the derived appropriation exceeded MemTotal +// and the cell refused before reaching the comparison it existed to make, and on a busy one it +// passed for reasons unrelated to the relation (side-chat review of gunbc#11962 at 725d9c58). +// +// So the boundary is here: this fold decides admission from a supplied observation, and +// compute_reserve supplies the real one. DESIGN 3's pairing obligation is discharged by an +// inhabitance claim that observe_host_memory really produces a well-formed reading of this machine, +// and by the srv1 receipts, where the whole path runs for real. fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { match observe_host_memory() { HostMemoryUnobserved { detail: d } => ComputeReservationRefused { detail: join(["the host's memory could not be observed, so nothing was reserved: ", d], "") } HostMemoryObserved { total: total, available: available } => - if kibibyte_count(k: subject.appropriation) > kibibyte_count(k: total) { - ComputeReservationRefused { - detail: join([ - "the compute appropriation is ", to_string(value: kibibyte_count(k: subject.appropriation)), - " KiB and this host has ", to_string(value: kibibyte_count(k: total)), - " KiB installed, so the share was never backed by the machine", - ], ""), - } - } else if kibibyte_count(k: available) < kibibyte_count(k: subject.request) { - ComputeHostBelowFloor { requested: subject.request, available: available } - } else { - match compute_capacity_store_ensure(subject: subject) { - FabricStorageRootRefused { area: a, detail: d } => ComputeReservationRefused { detail: join(["the capacity ledger could not be prepared at ", a, ": ", d], "") } - FabricStorageRootReady => - match now_epoch_seconds() { - Absent => ComputeReservationRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds, ceiling: compute_backing_ceiling(subject: subject, available: available)) - } + compute_reserve_against(subject: subject, reference: reference, term_seconds: term_seconds, total: total, available: available) + } +} + +fn compute_reserve_against( + subject: ComputeCapacitySubject, + reference: NonEmptyStr, + term_seconds: Nat, + total: Kibibyte, + available: Kibibyte, +) -> ComputeReservation { + if kibibyte_count(k: subject.appropriation) > kibibyte_count(k: total) { + ComputeReservationRefused { + detail: join([ + "the compute appropriation is ", to_string(value: kibibyte_count(k: subject.appropriation)), + " KiB and this host has ", to_string(value: kibibyte_count(k: total)), + " KiB installed, so the share was never backed by the machine", + ], ""), + } + } else if kibibyte_count(k: available) < kibibyte_count(k: subject.request) { + ComputeHostBelowFloor { requested: subject.request, available: available } + } else { + match compute_capacity_store_ensure(subject: subject) { + FabricStorageRootRefused { area: a, detail: d } => ComputeReservationRefused { detail: join(["the capacity ledger could not be prepared at ", a, ": ", d], "") } + FabricStorageRootReady => + match now_epoch_seconds() { + Absent => ComputeReservationRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds, ceiling: compute_backing_ceiling(subject: subject, available: available)) } - } + } } } diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 0fb60514c62..6a44e60a36c 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -7,9 +7,11 @@ import std.measure { byte_size_count } import extdeps.filesystem.filesystem_io { Filesystem } import gunbc.output_policy { OutcomeIsData } import extdeps.gunbc +import extdeps.shell import extdeps.git { shape_git_worktree_add_detached_argv } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex, git_object_id_wire_hex } import extdeps.systemd.systemd_run { systemd_run_user_wait_arguments, systemd_run_property } +import extdeps.systemd.systemctl import extdeps.cache.sccache { sccache_installed_binary_path } import gunbc.clock_read { clock_now_probed_at_or_unknown } import gunbc.roadmap_dashboard_instance { @@ -66,7 +68,7 @@ type ComputeLayout { log_path: String outcome_path: String consumption_path: String - obligation_path: String + obligations_dir: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -84,7 +86,7 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), consumption_path: join([work_dir, "/consumption"], ""), - obligation_path: join([work_dir, "/unreleased-reservation"], ""), + obligations_dir: join([root, "/obligations"], ""), } } @@ -236,6 +238,75 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden } } +// ── WHETHER THE UNIT IS STILL USING THE MEMORY ──────────────────────────────────────────────── +// +// THE RELEASE LAW SAID "QUIESCENCE REQUIRED" AND NOTHING OBSERVED QUIESCENCE. That was the gap: +// compute_release ran unconditionally on whatever compute_run_leased returned, and what it returns +// is the exit of `systemd-run --wait`, which is NOT the unit. systemd-run is a launcher, not the +// unit's parent -- the transient service is owned by the manager -- so a launcher that is killed, +// times out, or loses its bus connection yields ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, and +// the seat's memory would be handed to the next arrival on top of a live cargo. product.capacity.lease +// already models this exactly (QuiescenceFact = Quiescent | QuiescenceUnobserved) and the corpus was +// simply not consulting it (side-chat review of gunbc#11962 at 725d9c58). +// +// SO A RELEASE NOW CONSUMES A TERMINATION DECISION BOUND TO THE ACTUAL UNIT, and only two arms may +// free capacity: the manager knows nothing about the unit, or it knows it has stopped. Running and +// UNOBSERVED both keep the commitment -- an unobserved unit is the case a fail-open would swallow, +// and it is the one that matters, because "I could not ask" is exactly what a lost launcher looks +// like. +// +// AN EMPTY ActiveState IS QUIESCENT AND THAT IS NOT A GUESS. `systemctl --user show` prints an empty +// value for a unit the manager has no record of, which for a transient --wait unit means it ran and +// was collected, and for a unit that never started means there is nothing consuming memory. Both are +// the same fact for THIS decision -- no process of ours is holding that memory -- and the reviewer's +// own formulation admits both: never started, or demonstrated quiescent. +type UnitTermination + = UnitQuiescent { state: String } + | UnitStillActive { state: String } + | UnitTerminationUnobserved { detail: String } + +data unit_active_state_property: NonEmptyStr = "ActiveState" as NonEmptyStr + +// THE CLOSED SET IS UPSTREAM'S (systemd.unit(5) ActiveState), and an unrecognised word is UNOBSERVED +// rather than assumed dead. A new systemd state that this fold has never seen must not be read as +// permission to take the memory back. +fn unit_state_termination(state: String) -> UnitTermination { + let s = trim(s: state) + if s == "" || s == "inactive" || s == "failed" { + UnitQuiescent { state: s } + } else if s == "active" || s == "activating" || s == "deactivating" || s == "reloading" || s == "maintenance" { + UnitStillActive { state: s } + } else { + UnitTerminationUnobserved { detail: join(["systemd reported an ActiveState this provider does not model: ", s], "") } + } +} + +fn observe_unit_termination(identity_hex: String) -> UnitTermination { + let unit = compute_unit_name(identity_hex: identity_hex) + let r = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_active_state_property) + if !r.success { + UnitTerminationUnobserved { detail: join(["systemctl --user show ", unit as String, " could not be read"], "") } + } else { + unit_state_termination(state: r.value) + } +} + +fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { + match t { + UnitQuiescent { state: _ } => true + UnitStillActive { state: _ } => false + UnitTerminationUnobserved { detail: _ } => false + } +} + +fn unit_termination_wire(t: UnitTermination) -> String { + match t { + UnitQuiescent { state: s } => join(["quiescent (ActiveState=", if s == "" { "" } else { s }, ")"], "") + UnitStillActive { state: s } => join(["STILL ACTIVE (ActiveState=", s, "), so the seat stays charged"], "") + UnitTerminationUnobserved { detail: d } => join(["termination unobserved, so the seat stays charged: ", d], "") + } +} + // Return declared outputs by content: hash each with git (a blob object id is a content identity // every subject already uses) and install it into the identity's store directory. An absent or // unreadable output is a mismatch, never a success with a hole. @@ -299,6 +370,10 @@ fn compute_basename(path: String) -> String { // under-admits until the obligation is discharged rather than handing the same bytes to the next // arrival. Discharging it is an explicit release naming why -- the same entry completion and // cancellation take. +// THE PRODUCER LEASE IS DROPPED LAST, AND THE ORDER IS THE POINT. It used to be deleted before the +// outcome and the obligation were written, so a second producer could take the lease and start +// against a work directory whose record was still being published -- the exclusion held for the +// name and not for the publication it exists to protect. fn compute_run_and_settle( instance: HostDashboardInstance, commit: String, @@ -308,26 +383,50 @@ fn compute_run_and_settle( op: WorkOperation, dependency_store: String, granted: Kibibyte, + reference: NonEmptyStr, reservation: ComputeReservation, ) -> ProvideResult { let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) let standing = compute_capacity_standing(instance: instance) - let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) + let termination = observe_unit_termination(identity_hex: identity) + let returned = if unit_termination_frees_capacity(t: termination) { + compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome), ", unit ", unit_termination_wire(t: termination)], "") as NonEmptyStr) + } else { + ComputeReleaseRefused { detail: join(["the seat was NOT released: ", unit_termination_wire(t: termination)], "") } + } let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) - let released = Filesystem.Delete(path: layout.lease_path) - let notes = join([ - "; producer lease released: ", if released.success { "yes" } else { released.error }, - "; consumption recorded: ", if measured.success { "yes" } else { measured.error }, - ], "") - if compute_release_landed(r: returned) { - compute_settled_record( - layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, - lease_released: released.success, notes: notes) + let notes = join(["; consumption recorded: ", if measured.success { "yes" } else { measured.error }], "") + let settled = if compute_release_landed(r: returned) { + compute_settled_record(layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, notes: notes) } else { compute_unreleased_record( layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, - reservation: reservation, returned: returned, notes: notes) + reservation: reservation, returned: returned, reference: reference, termination: termination, notes: notes) } + let dropped = Filesystem.Delete(path: layout.lease_path) + if dropped.success { settled } else { compute_lease_undropped(result: settled, layout: layout, identity: identity, subject: subject, op: op, error: dropped.error) } +} + +fn compute_lease_undropped( + result: ProvideResult, + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + error: String, +) -> ProvideResult { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { + detail: join([ + "the record was published as ", provide_result_ending(r: result), + " but the producer lease at ", layout.lease_path, " could not be dropped: ", error, + "; the next request for this identity will refuse as ProducerInFlight until it is removed", + ], ""), + }, + }, + subject: subject, op: op) } // THE ORDINARY ENDING: the reservation came back, so the record is the run's own ending and the @@ -338,12 +437,11 @@ fn compute_settled_record( subject: WorkSubject, op: WorkOperation, outcome: WorkOutcome, - lease_released: Bool, notes: String, ) -> ProvideResult { let document = work_outcome_wire(o: outcome, subject: subject, op: op) let written = Filesystem.Write(path: layout.outcome_path, content: document) - if written.success && lease_released { + if written.success { ProvideFresh { outcome: outcome, document: document } } else { compute_fresh( @@ -353,7 +451,7 @@ fn compute_settled_record( detail: join([ "the work ended ", work_outcome_ending(o: outcome), " and its reservation was released, but the record could not be completed -- outcome written: ", - if written.success { "yes" } else { written.error }, notes, + written.error, notes, ], ""), }, }, @@ -371,19 +469,27 @@ fn compute_unreleased_record( outcome: WorkOutcome, reservation: ComputeReservation, returned: ComputeRelease, + reference: NonEmptyStr, + termination: UnitTermination, notes: String, ) -> ProvideResult { let obligation = join([ - "unreleased reservation: ", compute_reservation_wire(r: reservation), - "; release attempt: ", compute_release_wire(r: returned), - "; the work itself ended ", work_outcome_ending(o: outcome), - " and any declared outputs are in the store; the seat stays held until an explicit release ", - "discharges it, so this host under-admits rather than promising the same memory twice", + "identity=", identity, "\n", + "reference=", reference as String, "\n", + "unit=", unit_termination_wire(t: termination), "\n", + "work_ending=", work_outcome_ending(o: outcome), "\n", + "reservation=", compute_reservation_wire(r: reservation), "\n", + "release_attempt=", compute_release_wire(r: returned), "\n", ], "") - let recorded = Filesystem.Write(path: layout.obligation_path, content: join([obligation, "\n"], "")) + let prepared = compute_ensure_dir_at(path: layout.obligations_dir) + let recorded = Filesystem.WriteCreateNew(path: compute_obligation_path(layout: layout, reference: reference), content: obligation) let cause = ComputeCapacityLedgerUnavailable { detail: join([ - obligation, "; obligation recorded: ", if recorded.success { layout.obligation_path } else { recorded.error }, notes, + "the seat for ", reference as String, " is still charged; the work itself ended ", + work_outcome_ending(o: outcome), " and any declared outputs are in the store. ", + compute_release_wire(r: returned), + ". obligation recorded: ", if prepared && recorded.success { compute_obligation_path(layout: layout, reference: reference) } else { recorded.error }, + notes, ], ""), } let document = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) @@ -395,7 +501,20 @@ fn compute_unreleased_record( } } -// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease +// ONE OBLIGATION FILE PER ATTEMPT, NOT PER IDENTITY. The obligation used to live at a path derived +// from the work identity while the reservation it describes is keyed by the ATTEMPT, so a retry +// overwrote the first obligation and the first seat became unrecoverable -- the reference that was +// the only way to release it was gone. The reference is the name of the hold, so it is the name of +// the obligation, and the create is EXCLUSIVE so a second writer cannot silently replace one. +fn compute_obligation_path(layout: ComputeLayout, reference: NonEmptyStr) -> String { + join([layout.obligations_dir, "/", reference as String], "") +} + +fn compute_ensure_dir_at(path: String) -> Bool { + shell.Mkdir.Parents(path: path as FilePath).success +} + +// THE ORDER THE TWO HOLDS ARE TAKEN IN// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the // reservation is a QUANTITY on the host, and no exclusive create of any name settles that. The // lease is taken first because a request that then loses on capacity releases a name it holds @@ -438,11 +557,28 @@ fn compute_provide_leaf( } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { + let outstanding = compute_outstanding_obligation(layout: layout, identity: identity) + if outstanding != "" { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: ComputeCapacityLedgerUnavailable { + detail: join([ + "this identity has an unreleased reservation from an earlier attempt and will not take a second one: ", + outstanding, + ". Discharge it with the release verb once the unit is demonstrably gone; until then a retry would ", + "reserve on top of a seat that is still charged and could publish an outcome over the obligation.", + ], ""), + }, + }, + subject: subject, op: op) + } else { let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) } else { - let reservation = compute_reserve(instance: instance, reference: compute_reservation_reference(identity: identity), term_seconds: compute_reservation_term_seconds) + let reference = compute_reservation_reference(identity: identity) + let reservation = compute_reserve(instance: instance, reference: reference, term_seconds: compute_reservation_term_seconds) match reservation { ComputeCapacityFull { wire: w } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) @@ -453,9 +589,34 @@ fn compute_provide_leaf( ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => compute_run_and_settle( instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, - dependency_store: dependency_store, granted: granted, reservation: reservation) + dependency_store: dependency_store, granted: granted, reference: reference, reservation: reservation) } } + } + } +} + +// WHAT A RETRY MUST LOOK AT BEFORE IT RESERVES AGAIN, and the reason this is a read rather than a +// convention. An obligation names a seat that is still charged; a retry of the same identity that +// ignored it would take a SECOND reservation while the first is outstanding, and -- because the +// outcome document is keyed by identity -- could publish its own ending over the record that was +// carrying the obligation. So the lifecycle state is consumed, not remembered: the obligations +// directory is listed, and any entry naming this identity refuses the request by name and points at +// the release verb. Empty means nothing outstanding; an unreadable directory means nothing +// outstanding either, because the directory does not exist until the first obligation is written. +fn compute_outstanding_obligation(layout: ComputeLayout, identity: String) -> String { + let listing = Filesystem.List(path: layout.obligations_dir) + if !listing.success { + "" + } else { + fold(split(s: listing.entries, delimiter: "\n"), init: "", f: (acc, entry) => + if acc != "" || trim(s: entry) == "" { + acc + } else if !starts_with(s: trim(s: entry), prefix: identity) { + acc + } else { + join([layout.obligations_dir, "/", trim(s: entry)], "") + }) } } @@ -636,28 +797,45 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent // the pool is not holding writes nothing and exits non-zero saying so, which is the answer a // mistyped reference must get -- appending a release for an unheld reference would make the // partition unfoldable for every later reader. -fn compute_release_cli(repo_root: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { +fn compute_release_cli(repo_root: String, identity: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { match dashboard_instance_for_repo_root(repo_root: repo_root) { DashboardInstanceRootUnknown { repo_root: r } => ExitFailure { code: 2, reason: join(["compute: no dashboard instance is rooted at ", r], "") } DashboardInstanceResolved { instance } => - if reference == "" || reason == "" { - ExitFailure { code: 2, reason: "compute: release needs a non-empty reference and a reason naming why the seat is being given back" } + if identity == "" || reference == "" || reason == "" { + ExitFailure { code: 2, reason: "compute: release needs the identity and the reference the obligation file names, and a reason saying why the seat is being given back" } } else { - let released = compute_release_reference(instance: instance, reference: reference as NonEmptyStr, reason: reason as NonEmptyStr) - let wire = compute_release_cli_wire(r: released) - let receipt = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) - if !receipt.success { - ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; release was ", wire], "") } - } else { - match released { - ComputeReleased { partition: _, reference: _, reason: _ } => ExitSuccess - ComputeReleaseRefused { detail: _ } => ExitFailure { code: 1, reason: join(["compute release refused: ", wire], "") } - } - } + compute_release_observed(instance: instance, identity: identity, reference: reference, reason: reason, receipt_path: receipt_path) } } } +// THE RECOVERY DOOR OBSERVES THE UNIT TOO, and this is the half a reason string cannot supply. An +// operator releasing "because the worker looked dead" is asserting exactly the fact that nothing +// checked -- and releasing a seat whose cargo is still resident hands its memory to the next +// arrival. So recovery consumes the same termination decision settlement does: only a unit the +// manager reports stopped, or has no record of, may be released. A running unit is refused BY NAME +// with the state that refused it, so the operator's next move is to stop the unit rather than to +// argue with the tool. +fn compute_release_observed(instance: HostDashboardInstance, identity: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { + let termination = observe_unit_termination(identity_hex: identity) + let wire = if unit_termination_frees_capacity(t: termination) { + compute_release_cli_wire(r: compute_release_reference( + instance: instance, + reference: reference as NonEmptyStr, + reason: join([reason, " (unit ", unit_termination_wire(t: termination), ")"], "") as NonEmptyStr)) + } else { + join(["refused: ", unit_termination_wire(t: termination), "; stop the unit first, then release"], "") + } + let receipt = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) + if !receipt.success { + ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; release was ", wire], "") } + } else if starts_with(s: wire, prefix: "released ") { + ExitSuccess + } else { + ExitFailure { code: 1, reason: join(["compute release refused: ", wire], "") } + } +} + fn compute_release_cli_wire(r: ComputeRelease) -> String { match r { ComputeReleased { partition: p, reference: ref_, reason: why } => diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index 34c3d13f302..6051e698a4f 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -12,7 +12,7 @@ import gunbc.compute.host_capacity { compute_capacity_subject, compute_capacity_root, compute_memory_partition, HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - compute_reserve_on, compute_release, compute_release_on, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + compute_reserve_on, compute_reserve_against, compute_release, compute_release_on, ComputeRelease, ComputeReleased, ComputeReleaseRefused, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, } @@ -117,19 +117,15 @@ test fn competing_reservations_cannot_over_reserve_and_a_release_returns_the_cap // The request is the machine's own installed memory, which is strictly above what it reports // available on any host that is running anything at all -- including the one running this witness. test fn a_request_the_machine_cannot_back_refuses_below_the_live_floor() -> Bool { - match observe_host_memory() { - HostMemoryUnobserved { detail: _ } => false - HostMemoryObserved { total: total, available: available } => { - let installed = kibibyte_count(k: total) - let subject = subject_at(root: fresh_root(), appropriation_kib: installed, request_kib: installed) - kibibyte_count(k: available) < installed - && (match compute_reserve_on(subject: subject, reference: "wet-floor" as NonEmptyStr, term_seconds: 60) { - ComputeHostBelowFloor { requested: rq, available: av } => kibibyte_count(k: rq) == installed && kibibyte_count(k: av) > 0 - ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false - ComputeCapacityFull { wire: _ } => false - ComputeReservationRefused { detail: _ } => false - }) - } + let total = kibibyte(count: 64 * one_gibibyte_in_kibibytes) + let available = kibibyte(count: 4 * one_gibibyte_in_kibibytes) + let subject = subject_at(root: fresh_root(), appropriation_kib: 52 * one_gibibyte_in_kibibytes, request_kib: 26 * one_gibibyte_in_kibibytes) + match compute_reserve_against(subject: subject, reference: "wet-floor" as NonEmptyStr, term_seconds: 60, total: total, available: available) { + ComputeHostBelowFloor { requested: rq, available: av } => + kibibyte_count(k: rq) == 26 * one_gibibyte_in_kibibytes && kibibyte_count(k: av) == 4 * one_gibibyte_in_kibibytes + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeReservationRefused { detail: _ } => false } } @@ -137,17 +133,13 @@ test fn a_request_the_machine_cannot_back_refuses_below_the_live_floor() -> Bool // presenting hours later as a build the kernel killed. It is refused ahead of the floor check, // because a share that was never backed is wrong whatever this instant's availability happens to be. test fn an_appropriation_larger_than_the_machine_refuses_rather_than_admitting() -> Bool { - match observe_host_memory() { - HostMemoryUnobserved { detail: _ } => false - HostMemoryObserved { total: total, available: _ } => { - let subject = subject_at(root: fresh_root(), appropriation_kib: kibibyte_count(k: total) + 1, request_kib: one_gibibyte_in_kibibytes) - match compute_reserve_on(subject: subject, reference: "wet-oversized" as NonEmptyStr, term_seconds: 60) { - ComputeReservationRefused { detail: d } => string_contains(s: d, pattern: "never backed by the machine") - ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false - ComputeCapacityFull { wire: _ } => false - ComputeHostBelowFloor { requested: _, available: _ } => false - } - } + let total = kibibyte(count: 64 * one_gibibyte_in_kibibytes) + let subject = subject_at(root: fresh_root(), appropriation_kib: 64 * one_gibibyte_in_kibibytes + 1, request_kib: one_gibibyte_in_kibibytes) + match compute_reserve_against(subject: subject, reference: "wet-oversized" as NonEmptyStr, term_seconds: 60, total: total, available: total) { + ComputeReservationRefused { detail: d } => string_contains(s: d, pattern: "never backed by the machine") + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeHostBelowFloor { requested: _, available: _ } => false } } @@ -211,27 +203,37 @@ test fn a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool // live observation, so the fixture states the relation rather than a number: it asks for slightly // more than half of what the host reports free, twice. test fn two_requests_summing_past_the_observed_headroom_cannot_both_admit() -> Bool { + let total = kibibyte(count: 64 * one_gibibyte_in_kibibytes) + let available = kibibyte(count: 30 * one_gibibyte_in_kibibytes) + let subject = subject_at(root: fresh_root(), appropriation_kib: 52 * one_gibibyte_in_kibibytes, request_kib: 26 * one_gibibyte_in_kibibytes) + let first = compute_reserve_against(subject: subject, reference: "wet-headroom-a" as NonEmptyStr, term_seconds: 60, total: total, available: available) + let second = compute_reserve_against(subject: subject, reference: "wet-headroom-b" as NonEmptyStr, term_seconds: 60, total: total, available: available) + kibibyte_count(k: subject.request) <= kibibyte_count(k: available) + && kibibyte_count(k: subject.request) * 2 > kibibyte_count(k: available) + && kibibyte_count(k: subject.appropriation) >= kibibyte_count(k: subject.request) * 2 + && kibibyte_count(k: subject.appropriation) <= kibibyte_count(k: total) + && reserved_amount(r: first) == kibibyte_count(k: subject.request) + && is_capacity_full(r: second) +} + +// THE PAIRING OBLIGATION FOR EVERY SUPPLIED OBSERVATION ABOVE (DESIGN 3): the real producer emits a +// reading of THIS machine, and it is a reading rather than a shape -- installed memory is positive, +// availability does not exceed it, and the two are not the same number on a host that is running +// anything. A fixture cannot establish that and the supplied cells do not claim to. +test fn the_real_observation_reads_this_machine() -> Bool { match observe_host_memory() { HostMemoryUnobserved { detail: _ } => false - HostMemoryObserved { total: _, available: available } => { - let each = kibibyte_count(k: available) / 2 + 1024 - let subject = subject_at(root: fresh_root(), appropriation_kib: each * 4, request_kib: each) - let first = compute_reserve_on(subject: subject, reference: "wet-headroom-a" as NonEmptyStr, term_seconds: 60) - let second = compute_reserve_on(subject: subject, reference: "wet-headroom-b" as NonEmptyStr, term_seconds: 60) - reserved_amount(r: first) == each - && is_capacity_full(r: second) - && each * 2 > kibibyte_count(k: available) - && each * 4 > each * 2 - } + HostMemoryObserved { total: total, available: available } => + kibibyte_count(k: total) > 0 + && kibibyte_count(k: available) > 0 + && kibibyte_count(k: available) < kibibyte_count(k: total) } } // A WEDGED SEAT CAN ACTUALLY BE RECOVERED, BY REFERENCE, BY SOMEONE WHO NEVER HELD THE RESERVATION. // Under QuiescenceRequired a term frees nothing, so this route is the ONLY way capacity returns // after a release fails to land -- and until review 69595 the module claimed it in prose while no -// declaration reached it. The cell holds the whole appropriation, releases it with nothing but the -// reference (the reservation value is deliberately not used), and shows the capacity is admissible -// again. +// declaration reached it. // // AND A REFERENCE THE POOL IS NOT HOLDING WRITES NOTHING. That is the sharper half: an unheld // reference appended anyway would not fail at the append, it would fail at every later READ, when diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 385963a5879..21b15320449 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -2,6 +2,10 @@ module test.claim.compute.work_request_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } +import gunbc.compute.work_provider_local { + UnitTermination, UnitQuiescent, UnitStillActive, UnitTerminationUnobserved, + unit_state_termination, unit_termination_frees_capacity, +} import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, @@ -74,3 +78,33 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool && (match stored_outcome_decode(text: "{\"identity\": \"abc\"}") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) && (match stored_outcome_decode(text: "not json") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) } + +// ONLY A UNIT THAT IS DEMONSTRABLY GONE MAY FREE ITS SEAT, and the arm that matters is the one in +// the middle. `systemd-run --wait` is a LAUNCHER, not the unit's parent, so a launcher that dies or +// times out returns failure while the transient service keeps running; releasing on that signal +// would hand a live cargo's memory to the next arrival. Empty and inactive and failed are all +// "nothing of ours is holding that memory" -- never started and already collected are the same fact +// for this decision. Everything systemd calls live keeps the charge, and a word this fold has never +// seen keeps it too: a future ActiveState must not read as permission to take memory back. +test fn only_a_gone_unit_frees_its_seat_and_an_unknown_state_does_not() -> Bool { + all(["", "inactive", "failed", " inactive "], s => + match unit_state_termination(state: s) { + UnitQuiescent { state: _ } => true + UnitStillActive { state: _ } => false + UnitTerminationUnobserved { detail: _ } => false + }) + && all(["active", "activating", "deactivating", "reloading", "maintenance"], s => + match unit_state_termination(state: s) { + UnitStillActive { state: _ } => true + UnitQuiescent { state: _ } => false + UnitTerminationUnobserved { detail: _ } => false + }) + && all(["quantum-superposed", "ACTIVE", "1"], s => + match unit_state_termination(state: s) { + UnitTerminationUnobserved { detail: _ } => true + UnitQuiescent { state: _ } => false + UnitStillActive { state: _ } => false + }) + && all(["", "inactive", "failed"], s => unit_termination_frees_capacity(t: unit_state_termination(state: s))) + && all(["active", "activating", "deactivating", "quantum-superposed"], s => !unit_termination_frees_capacity(t: unit_state_termination(state: s))) +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 6675dbf5577..6e8d09c7ff7 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -244,6 +244,12 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // probe failed without output rather than running its payload. fn local_repo_wet_schedule() -> List { [ + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "the_real_observation_reads_this_machine" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "the_real_observation_reads_this_machine", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", From 5ee0cceea8922117e09a0e86670527e20864c9e2 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 18:04:19 +0000 Subject: [PATCH 07/44] PR 1 rescoped to admission+pool: conservative release, declared discharge gap, and the memory carriers the refusals were stripping Split approved by the manager. All seven of the side chat's blockers at 9172cf7 are lifecycle/recovery; none touches admission or the pool. The argument for splitting is safety, not workload: WITHOUT a recovery route every one of those defects degrades to "the seat stays charged" -- the host under-admits and nothing is over-promised. WITH a half-built one they FREE SEATS THAT SHOULD NOT BE FREED. Shipping recovery half-built is worse than not shipping it. OUT OF THIS PR, and named rather than quietly dropped: compute_release_cli, compute_release_reference, the obligation files and the retry-blocking read. BLOCKER (1) IS NOT RECOVERY-ONLY AND IS FIXED HERE, per the manager's correction: automatic settlement releases too. ActiveState alone is not the evidence -- `failed` can coexist with live processes while a unit's stop reaches its SIGKILL timeout, and an empty ActiveState is the manager declining to answer, not authoritative absence. The decision is now four-valued over evidence the repo already models: authoritative absence from LoadState=not-found (the wire value this corpus reads positively), observed termination from the unit's own cgroup reporting populated 0, populated, and unavailable. Only the first two free capacity; unavailable keeps the charge, and that is the arm that matters, because a lost launcher, an unreadable manager and a missing cgroup all look like it. BLOCKER (7) STILL REACHED PR 1 AND IS FIXED: the lease-undropped arm handed the caller a refusal while the outcome document already said the run SUCCEEDED, then claimed the next request would refuse as ProducerInFlight -- three claims that cannot all be true, since the stored record is what the next caller attaches to. One contract now: the stored record is the result, the caller is told what the next caller will read, and the stale lease is recorded as an operational fact rather than converted into a verdict about the work. Blockers (2)-(6) are no longer reached: each was about the recovery door or the obligation files, and both are out. THE GAP IS CLASSIFIED AS THE MANAGER ASKED. No discharge route existed on main -- admission was a count of lease files and there were no reservations at all -- so this is not a rung that fell and is not a 4b(3) drop. It is a new class: gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge, rostered, with a trigger naming PR 2's CAPABILITY (termination from absence or an empty cgroup; attempt-to-unit binding; coordination with a pending producer; a settled state a retry consumes) rather than an artifact. Its remedy sentence reaches the caller through compute_stuck_reservation_remedy. review 69624, unit modeling: HostBelowMemoryFloor carried Int fields NAMED _kib while its producer already had Kibibyte and the provider called kibibyte_count to strip the carrier on the way in -- the unit modeled twice and checked nowhere. It carries Kibibyte now. And SeatRoomObserved.headroom was the boundary THIS PR moved: a bare Nat was right while the pool was fixed at seats, and generalizing fabric_seat_observe to Pool made the same field mean KiB for one caller and seats for another. SeatStandingObservation is now generic and headroom carries Measure. Seven wet cells plus the termination fold pass; gunbc compile on the provider closure: 0 blocking errors. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 54 +-- dag/gunbc/compute/work_provider_local.dag | 330 +++++++----------- dag/gunbc/compute/work_request.dag | 13 +- dag/gunbc/fabric/fabric_event_log.dag | 17 +- dag/gunbc/harness/harness_seat.dag | 4 +- ...ompute_seat_has_no_in_corpus_discharge.dag | 34 ++ .../host_capacity_wet_witness_test.dag | 27 +- .../compute/work_request_witness_test.dag | 50 ++- ...serving_d0_real_execution_witness_test.dag | 4 +- src/v2/workflow/local_repo_wet_terminal.dag | 6 - 10 files changed, 228 insertions(+), 311 deletions(-) create mode 100644 dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index 51ac97b922a..3248b01262d 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -379,40 +379,20 @@ fn compute_release(reservation: ComputeReservation, reason: NonEmptyStr) -> Comp } } -// THE RECOVERY ROUTE, AND IT EXISTS BECAUSE THE MODULE CLAIMED IT (review 69595 on gunbc#11962). +// THE ONE RELEASE PATH, CHECKED. It reads, folds, PROPOSES the release against the folded pool and +// appends only what the pool admits -- the mirror of the acquire. That check is load-bearing rather +// than defensive: a reference the pool is not holding, appended anyway, would not fail at the +// append; it would fail at every later READ, when the fold hits UnknownEncumbrance and refuses the +// WHOLE partition. // -// The prose above says a wedged seat is recovered by an explicit release naming why. Until this -// function that was FALSE: compute_release took a ComputeReservation, whose only constructor is -// compute_reserve_on, whose result lives and dies inside one compute_provide_leaf invocation. So a -// release that failed to land left an encumbrance that -- under QuiescenceRequired, which by design -// never lapses -- nothing in the corpus could ever discharge. Two of those consume the whole host -// appropriation, and every later request then gets a CAPACITY refusal on a host with its memory -// free: the exact conflation this change added a witness to prevent, arriving through a different -// door. A claimed route that no declaration reaches is the dangling consumption DESIGN 3c refuses. -// -// IT TAKES A REFERENCE, because that is all a recovering operator has: the reservation value is -// gone with the process that held it, and the reference is what the ledger and the obligation file -// both name. It is CHECKED against the folded pool before anything is appended -- a reference the -// pool is not holding writes nothing and refuses by name, because appending a release for an unheld -// reference would make the partition unfoldable for every later reader. -fn compute_release_reference(instance: HostDashboardInstance, reference: NonEmptyStr, reason: NonEmptyStr) -> ComputeRelease { - match compute_capacity_subject(instance: instance) { - ComputeSubjectHostUnresolved { host: h } => - ComputeReleaseRefused { detail: join(["no dashboard instance is declared for host ", h as String], "") } - ComputeSubjectResolved { subject: subject } => - compute_release_on( - store: compute_capacity_store(subject: subject), - partition: subject.partition, - appropriation: subject.appropriation, - reference: reference, - reason: reason, - ) - } -} - -// ONE RELEASE PATH FOR BOTH DOORS. The provider's own completion and the operator's recovery append -// the same event against the same check; a second spelling of "give the seat back" is the fork this -// module would otherwise grow the moment recovery existed. +// THE OPERATOR'S RECOVERY DOOR IS NOT HERE, and its absence is declared rather than implied. The +// discharge of a seat whose release did not land needs a termination decision bound to the unit, an +// attempt-to-unit binding that cannot release another attempt's live reservation, and a settled +// state a retry consumes -- see gunbc.recurring_failure_mode +// a_held_compute_seat_has_no_in_corpus_discharge, whose trigger names that capability. Shipping a +// recovery route with any of those missing would FREE SEATS THAT SHOULD NOT BE FREED, which is +// worse than not having one: without it every failure here keeps the charge and the host merely +// under-admits. // // THE APPROPRIATION PASSED HERE IS NOT LOAD-BEARING AND THE ZERO IS NOT A FABRICATED READING. product.capacity.pool // ruled that replay never adjudicates the ceiling, so a release is decided by whether the ledger @@ -460,7 +440,7 @@ fn compute_release_on( // committed far above the machine's own shortfall means the pool is over-reserving each class, and // the reverse means work is running outside the pool. type ComputeCapacityStanding - = ComputeCapacityStandingRead { committed: Kibibyte, ceiling: Kibibyte, headroom: Nat, observed_available: Kibibyte, observed_total: Kibibyte } + = ComputeCapacityStandingRead { committed: Kibibyte, ceiling: Kibibyte, headroom: Kibibyte, observed_available: Kibibyte, observed_total: Kibibyte } | ComputeCapacityStandingUnread { detail: String } fn compute_capacity_standing(instance: HostDashboardInstance) -> ComputeCapacityStanding { @@ -486,7 +466,7 @@ fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapac ) { SeatRoomObserved { headroom: h, generation: _ } => ComputeCapacityStandingRead { - committed: kibibyte(count: kibibyte_count(k: subject.appropriation) - h), + committed: kibibyte(count: kibibyte_count(k: subject.appropriation) - kibibyte_count(k: h)), ceiling: subject.appropriation, headroom: h, observed_available: available, @@ -496,7 +476,7 @@ fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapac ComputeCapacityStandingRead { committed: subject.appropriation, ceiling: subject.appropriation, - headroom: 0, + headroom: kibibyte(count: 0), observed_available: available, observed_total: total, } @@ -523,7 +503,7 @@ fn compute_capacity_standing_wire(s: ComputeCapacityStanding) -> String { ComputeCapacityStandingRead { committed: c, ceiling: cl, headroom: h, observed_available: av, observed_total: tt } => join([ "committed ", to_string(value: kibibyte_count(k: c)), " KiB of ", to_string(value: kibibyte_count(k: cl)), - " KiB (headroom ", to_string(value: h), " KiB); host reports ", to_string(value: kibibyte_count(k: av)), + " KiB (headroom ", to_string(value: kibibyte_count(k: h)), " KiB); host reports ", to_string(value: kibibyte_count(k: av)), " KiB available of ", to_string(value: kibibyte_count(k: tt)), " KiB installed", ], "") ComputeCapacityStandingUnread { detail: d } => join(["unread: ", d], "") diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 6a44e60a36c..3e19b143212 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -12,6 +12,10 @@ import extdeps.git { shape_git_worktree_add_detached_argv } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex, git_object_id_wire_hex } import extdeps.systemd.systemd_run { systemd_run_user_wait_arguments, systemd_run_property } import extdeps.systemd.systemctl +import extdeps.linux.cgroup_v2 { + cgroup_v2_mount_point, cgroup_v2_events_path, + CgroupEventsDecoding, CgroupEventsDecoded, CgroupEventsUnparseable, decode_cgroup_events_populated, +} import extdeps.cache.sccache { sccache_installed_binary_path } import gunbc.clock_read { clock_now_probed_at_or_unknown } import gunbc.roadmap_dashboard_instance { @@ -25,7 +29,7 @@ import gunbc.roadmap_execution_contract { import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size } import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_release_reference, compute_reservation_wire, + ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, } import gunbc.compute.work_request { @@ -68,7 +72,6 @@ type ComputeLayout { log_path: String outcome_path: String consumption_path: String - obligations_dir: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -86,7 +89,6 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), consumption_path: join([work_dir, "/consumption"], ""), - obligations_dir: join([root, "/obligations"], ""), } } @@ -240,70 +242,105 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden // ── WHETHER THE UNIT IS STILL USING THE MEMORY ──────────────────────────────────────────────── // -// THE RELEASE LAW SAID "QUIESCENCE REQUIRED" AND NOTHING OBSERVED QUIESCENCE. That was the gap: -// compute_release ran unconditionally on whatever compute_run_leased returned, and what it returns -// is the exit of `systemd-run --wait`, which is NOT the unit. systemd-run is a launcher, not the -// unit's parent -- the transient service is owned by the manager -- so a launcher that is killed, -// times out, or loses its bus connection yields ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, and -// the seat's memory would be handed to the next arrival on top of a live cargo. product.capacity.lease -// already models this exactly (QuiescenceFact = Quiescent | QuiescenceUnobserved) and the corpus was -// simply not consulting it (side-chat review of gunbc#11962 at 725d9c58). +// THE RELEASE LAW SAID "QUIESCENCE REQUIRED" AND NOTHING OBSERVED QUIESCENCE. compute_release ran +// on whatever compute_run_leased returned, and what it returns is the exit of `systemd-run --wait`, +// which is NOT the unit. systemd-run is a launcher, not the unit's parent -- the transient service +// is owned by the manager -- so a launcher killed, timed out, or disconnected from the bus yields +// ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, and the seat's memory would be handed to the next +// arrival on top of a live cargo. product.capacity.lease already models this (QuiescenceFact) and +// this provider was not consulting it. +// +// ActiveState ALONE IS NOT THE EVIDENCE, AND THE FIRST FIX OF THIS THOUGHT IT WAS. Two arms it +// treated as free are not (side-chat review of gunbc#11962 at 9172cf7): // -// SO A RELEASE NOW CONSUMES A TERMINATION DECISION BOUND TO THE ACTUAL UNIT, and only two arms may -// free capacity: the manager knows nothing about the unit, or it knows it has stopped. Running and -// UNOBSERVED both keep the commitment -- an unobserved unit is the case a fail-open would swallow, -// and it is the one that matters, because "I could not ask" is exactly what a lost launcher looks -// like. +// `failed` CAN COEXIST WITH LIVE PROCESSES. A unit whose stop reaches the SIGKILL timeout is +// reported failed while its processes are still being killed, and a task wedged in +// uninterruptible sleep outlives the state word entirely. // -// AN EMPTY ActiveState IS QUIESCENT AND THAT IS NOT A GUESS. `systemctl --user show` prints an empty -// value for a unit the manager has no record of, which for a transient --wait unit means it ran and -// was collected, and for a unit that never started means there is nothing consuming memory. Both are -// the same fact for THIS decision -- no process of ours is holding that memory -- and the reviewer's -// own formulation admits both: never started, or demonstrated quiescent. +// AN EMPTY ActiveState IS NOT AUTHORITATIVE ABSENCE. It is the manager declining to answer in a +// form this provider cannot distinguish from a unit it has never heard of. The repo's own +// systemctl authority already names the observation that proves absence, and it is LoadState: +// `systemctl show` exits 0 for a unit the manager has never heard of and prints not-found, so a +// nonzero exit means the MANAGER was unreadable, which is a different refusal from an absent unit +// and must not be folded into it. +// +// SO THE DECISION IS FOUR-VALUED AND ONLY THE FIRST TWO FREE CAPACITY. Authoritative absence, from +// the one wire value that proves it. Observed termination, from the cgroup this unit owns reporting +// no processes -- the kernel's own answer to "is anything still running here", which is the question +// the memory is actually about. Populated keeps the charge. And UNAVAILABLE keeps it too, which is +// the arm that matters most: "I could not ask" is what a lost launcher, an unreadable manager and a +// missing cgroup all look like, and reading any of them as permission to take the memory back is the +// absorbing fallback DESIGN 5 forbids. type UnitTermination - = UnitQuiescent { state: String } - | UnitStillActive { state: String } - | UnitTerminationUnobserved { detail: String } - -data unit_active_state_property: NonEmptyStr = "ActiveState" as NonEmptyStr - -// THE CLOSED SET IS UPSTREAM'S (systemd.unit(5) ActiveState), and an unrecognised word is UNOBSERVED -// rather than assumed dead. A new systemd state that this fold has never seen must not be read as -// permission to take the memory back. -fn unit_state_termination(state: String) -> UnitTermination { - let s = trim(s: state) - if s == "" || s == "inactive" || s == "failed" { - UnitQuiescent { state: s } - } else if s == "active" || s == "activating" || s == "deactivating" || s == "reloading" || s == "maintenance" { - UnitStillActive { state: s } + = UnitAbsentAuthoritatively { detail: String } + | UnitTerminatedObserved { detail: String } + | UnitPopulated { detail: String } + | UnitTerminationUnavailable { detail: String } + +data unit_load_state_property: NonEmptyStr = "LoadState" as NonEmptyStr +data unit_control_group_property: NonEmptyStr = "ControlGroup" as NonEmptyStr +data unit_load_state_absent_wire: String = "not-found" + +// THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is +// system-scope; these are transient --user units, so asking the system manager would answer +// not-found for EVERY one of them -- a fabricated authoritative absence, which is the single worst +// answer this fold can produce. The property goes through the user-scope show operation for that +// reason and no other. +fn observe_unit_termination(identity_hex: String) -> UnitTermination { + let unit = compute_unit_name(identity_hex: identity_hex) + let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) + if !load.success { + UnitTerminationUnavailable { detail: join(["the user manager could not be read for ", unit as String], "") } + } else if trim(s: load.value) == unit_load_state_absent_wire { + UnitAbsentAuthoritatively { detail: join([unit as String, " LoadState=", unit_load_state_absent_wire], "") } } else { - UnitTerminationUnobserved { detail: join(["systemd reported an ActiveState this provider does not model: ", s], "") } + observe_unit_cgroup(unit: unit, load_state: trim(s: load.value)) } } -fn observe_unit_termination(identity_hex: String) -> UnitTermination { - let unit = compute_unit_name(identity_hex: identity_hex) - let r = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_active_state_property) - if !r.success { - UnitTerminationUnobserved { detail: join(["systemctl --user show ", unit as String, " could not be read"], "") } +// THE UNIT EXISTS, SO THE QUESTION IS WHETHER ANYTHING IS STILL RUNNING IN IT, and the kernel +// answers that directly. ControlGroup is empty for a unit that is loaded and holds no processes -- +// systemd removes the cgroup when the last one exits -- so an empty value here is observed +// termination rather than a failed read, and it is the ordinary answer after a clean run. +fn observe_unit_cgroup(unit: NonEmptyStr, load_state: String) -> UnitTermination { + let cg = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_control_group_property) + if !cg.success { + UnitTerminationUnavailable { detail: join(["the control group of ", unit as String, " could not be read"], "") } + } else if trim(s: cg.value) == "" { + UnitTerminatedObserved { detail: join([unit as String, " LoadState=", load_state, " holds no control group, so no process of this unit survives"], "") } } else { - unit_state_termination(state: r.value) + let events = linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) + if !events.success { + UnitTerminationUnavailable { detail: join(["the cgroup events of ", unit as String, " could not be read at ", trim(s: cg.value)], "") } + } else { + match decode_cgroup_events_populated(body: events.value) { + CgroupEventsUnparseable { cause: c } => UnitTerminationUnavailable { detail: join(["the cgroup events of ", unit as String, " could not be decoded: ", c], "") } + CgroupEventsDecoded { populated: p } => + if p { + UnitPopulated { detail: join([unit as String, " still has processes in ", trim(s: cg.value)], "") } + } else { + UnitTerminatedObserved { detail: join([unit as String, " cgroup ", trim(s: cg.value), " reports populated 0"], "") } + } + } + } } } fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { match t { - UnitQuiescent { state: _ } => true - UnitStillActive { state: _ } => false - UnitTerminationUnobserved { detail: _ } => false + UnitAbsentAuthoritatively { detail: _ } => true + UnitTerminatedObserved { detail: _ } => true + UnitPopulated { detail: _ } => false + UnitTerminationUnavailable { detail: _ } => false } } fn unit_termination_wire(t: UnitTermination) -> String { match t { - UnitQuiescent { state: s } => join(["quiescent (ActiveState=", if s == "" { "" } else { s }, ")"], "") - UnitStillActive { state: s } => join(["STILL ACTIVE (ActiveState=", s, "), so the seat stays charged"], "") - UnitTerminationUnobserved { detail: d } => join(["termination unobserved, so the seat stays charged: ", d], "") + UnitAbsentAuthoritatively { detail: d } => join(["absent (", d, ")"], "") + UnitTerminatedObserved { detail: d } => join(["terminated (", d, ")"], "") + UnitPopulated { detail: d } => join(["STILL POPULATED (", d, "), so the seat stays charged"], "") + UnitTerminationUnavailable { detail: d } => join(["termination UNAVAILABLE (", d, "), so the seat stays charged"], "") } } @@ -394,39 +431,38 @@ fn compute_run_and_settle( } else { ComputeReleaseRefused { detail: join(["the seat was NOT released: ", unit_termination_wire(t: termination)], "") } } - let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) - let notes = join(["; consumption recorded: ", if measured.success { "yes" } else { measured.error }], "") let settled = if compute_release_landed(r: returned) { - compute_settled_record(layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, notes: notes) + compute_settled_record(layout: layout, identity: identity, subject: subject, op: op, outcome: outcome) } else { compute_unreleased_record( layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, - reservation: reservation, returned: returned, reference: reference, termination: termination, notes: notes) + reservation: reservation, returned: returned, reference: reference, termination: termination) } let dropped = Filesystem.Delete(path: layout.lease_path) - if dropped.success { settled } else { compute_lease_undropped(result: settled, layout: layout, identity: identity, subject: subject, op: op, error: dropped.error) } -} - -fn compute_lease_undropped( - result: ProvideResult, - layout: ComputeLayout, - identity: String, - subject: WorkSubject, - op: WorkOperation, - error: String, -) -> ProvideResult { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: StoreUnavailable { - detail: join([ - "the record was published as ", provide_result_ending(r: result), - " but the producer lease at ", layout.lease_path, " could not be dropped: ", error, - "; the next request for this identity will refuse as ProducerInFlight until it is removed", - ], ""), - }, - }, - subject: subject, op: op) + let measured = Filesystem.Write( + path: layout.consumption_path, + content: join([ + compute_capacity_standing_wire(s: standing), "\n", + "unit: ", unit_termination_wire(t: termination), "\n", + "reservation: ", compute_release_wire(r: returned), "\n", + if dropped.success { "" } else { compute_stale_lease_note(layout: layout, error: dropped.error) }, + ], "")) + if measured.success { settled } else { settled } +} + +// BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the +// caller a refusal while the outcome document already said the run had SUCCEEDED, and then state +// that the next request would refuse as ProducerInFlight -- three claims that cannot all be true, +// because the stored record is exactly what the next caller attaches to. So the stored record +// decides: a published success IS the result, the caller is told the same thing the next caller +// will read, and the stale lease is recorded as the operational fact it is rather than converted +// into a verdict about the work. An operator removes the name; nothing about the run changed. +fn compute_stale_lease_note(layout: ComputeLayout, error: String) -> String { + join([ + "STALE PRODUCER LEASE: ", layout.lease_path, " could not be dropped: ", error, + "; the record for this identity is published and is authoritative, and the next request will ", + "refuse as ProducerInFlight until an operator removes the name\n", + ], "") } // THE ORDINARY ENDING: the reservation came back, so the record is the run's own ending and the @@ -437,7 +473,6 @@ fn compute_settled_record( subject: WorkSubject, op: WorkOperation, outcome: WorkOutcome, - notes: String, ) -> ProvideResult { let document = work_outcome_wire(o: outcome, subject: subject, op: op) let written = Filesystem.Write(path: layout.outcome_path, content: document) @@ -451,7 +486,7 @@ fn compute_settled_record( detail: join([ "the work ended ", work_outcome_ending(o: outcome), " and its reservation was released, but the record could not be completed -- outcome written: ", - written.error, notes, + written.error, ], ""), }, }, @@ -471,25 +506,13 @@ fn compute_unreleased_record( returned: ComputeRelease, reference: NonEmptyStr, termination: UnitTermination, - notes: String, ) -> ProvideResult { - let obligation = join([ - "identity=", identity, "\n", - "reference=", reference as String, "\n", - "unit=", unit_termination_wire(t: termination), "\n", - "work_ending=", work_outcome_ending(o: outcome), "\n", - "reservation=", compute_reservation_wire(r: reservation), "\n", - "release_attempt=", compute_release_wire(r: returned), "\n", - ], "") - let prepared = compute_ensure_dir_at(path: layout.obligations_dir) - let recorded = Filesystem.WriteCreateNew(path: compute_obligation_path(layout: layout, reference: reference), content: obligation) let cause = ComputeCapacityLedgerUnavailable { detail: join([ - "the seat for ", reference as String, " is still charged; the work itself ended ", - work_outcome_ending(o: outcome), " and any declared outputs are in the store. ", - compute_release_wire(r: returned), - ". obligation recorded: ", if prepared && recorded.success { compute_obligation_path(layout: layout, reference: reference) } else { recorded.error }, - notes, + "the seat for ", reference as String, " is STILL CHARGED and this provider has no way to discharge it: ", + unit_termination_wire(t: termination), ". ", compute_release_wire(r: returned), + ". The work itself ended ", work_outcome_ending(o: outcome), + " and any declared outputs are in the store. ", compute_stuck_reservation_remedy, ], ""), } let document = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) @@ -501,18 +524,17 @@ fn compute_unreleased_record( } } -// ONE OBLIGATION FILE PER ATTEMPT, NOT PER IDENTITY. The obligation used to live at a path derived -// from the work identity while the reservation it describes is keyed by the ATTEMPT, so a retry -// overwrote the first obligation and the first seat became unrecoverable -- the reference that was -// the only way to release it was gone. The reference is the name of the hold, so it is the name of -// the obligation, and the create is EXCLUSIVE so a second writer cannot silently replace one. -fn compute_obligation_path(layout: ComputeLayout, reference: NonEmptyStr) -> String { - join([layout.obligations_dir, "/", reference as String], "") -} - -fn compute_ensure_dir_at(path: String) -> Bool { - shell.Mkdir.Parents(path: path as FilePath).success -} +// THE REMEDY SENTENCE FOR A SEAT THIS PROVIDER CANNOT GIVE BACK, and it is a declared absence +// rather than a route claimed in prose. No discharge existed on main -- there were no reservations +// at all, only a count of lease files -- so this is not a rung that dropped; it is a failure mode +// this change makes reachable for the first time, rostered as +// gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge, whose trigger names +// the capability that closes it. +// +// WHAT KEEPS IT FAIL-CLOSED MEANWHILE: every arm that cannot prove the unit is gone keeps the +// charge, so the host UNDER-admits. Nothing is over-promised; the pool simply shrinks until an +// operator clears the partition by hand. That is the direction a capacity ledger must fail in. +data compute_stuck_reservation_remedy: String = "Until the compute lifecycle capability lands (gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge), the only discharge is an operator clearing this host's compute capacity partition by hand; this host will under-admit until then, which is the fail-closed direction." // THE ORDER THE TWO HOLDS ARE TAKEN IN// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the @@ -557,22 +579,6 @@ fn compute_provide_leaf( } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { - let outstanding = compute_outstanding_obligation(layout: layout, identity: identity) - if outstanding != "" { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: ComputeCapacityLedgerUnavailable { - detail: join([ - "this identity has an unreleased reservation from an earlier attempt and will not take a second one: ", - outstanding, - ". Discharge it with the release verb once the unit is demonstrably gone; until then a retry would ", - "reserve on top of a seat that is still charged and could publish an outcome over the obligation.", - ], ""), - }, - }, - subject: subject, op: op) - } else { let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) @@ -583,7 +589,7 @@ fn compute_provide_leaf( ComputeCapacityFull { wire: w } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) ComputeHostBelowFloor { requested: rq, available: av } => - compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested_kib: kibibyte_count(k: rq), available_kib: kibibyte_count(k: av) }) + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested: rq, available: av }) ComputeReservationRefused { detail: d } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => @@ -592,33 +598,9 @@ fn compute_provide_leaf( dependency_store: dependency_store, granted: granted, reference: reference, reservation: reservation) } } - } } } -// WHAT A RETRY MUST LOOK AT BEFORE IT RESERVES AGAIN, and the reason this is a read rather than a -// convention. An obligation names a seat that is still charged; a retry of the same identity that -// ignored it would take a SECOND reservation while the first is outstanding, and -- because the -// outcome document is keyed by identity -- could publish its own ending over the record that was -// carrying the obligation. So the lifecycle state is consumed, not remembered: the obligations -// directory is listed, and any entry naming this identity refuses the request by name and points at -// the release verb. Empty means nothing outstanding; an unreadable directory means nothing -// outstanding either, because the directory does not exist until the first obligation is written. -fn compute_outstanding_obligation(layout: ComputeLayout, identity: String) -> String { - let listing = Filesystem.List(path: layout.obligations_dir) - if !listing.success { - "" - } else { - fold(split(s: listing.entries, delimiter: "\n"), init: "", f: (acc, entry) => - if acc != "" || trim(s: entry) == "" { - acc - } else if !starts_with(s: trim(s: entry), prefix: identity) { - acc - } else { - join([layout.obligations_dir, "/", trim(s: entry)], "") - }) - } -} // THE CONSUMPTION READING IS ITS OWN FILE AND MUST NOT JOIN THE OUTCOME DOCUMENT. It was appended // to outcome.json first, and the first real run on srv1 (2026-09-21) is what said why that is @@ -790,60 +772,6 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent } } -// THE OPERATOR'S RECOVERY DOOR, and the executing consumer of compute_release_reference. A -// reservation whose release did not land leaves its reference in -// /unreleased-reservation and in the partition; this is how it is given back. It is a -// verb on the same CLI rather than a script, so the refusal vocabulary is the module's: a reference -// the pool is not holding writes nothing and exits non-zero saying so, which is the answer a -// mistyped reference must get -- appending a release for an unheld reference would make the -// partition unfoldable for every later reader. -fn compute_release_cli(repo_root: String, identity: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { - match dashboard_instance_for_repo_root(repo_root: repo_root) { - DashboardInstanceRootUnknown { repo_root: r } => ExitFailure { code: 2, reason: join(["compute: no dashboard instance is rooted at ", r], "") } - DashboardInstanceResolved { instance } => - if identity == "" || reference == "" || reason == "" { - ExitFailure { code: 2, reason: "compute: release needs the identity and the reference the obligation file names, and a reason saying why the seat is being given back" } - } else { - compute_release_observed(instance: instance, identity: identity, reference: reference, reason: reason, receipt_path: receipt_path) - } - } -} - -// THE RECOVERY DOOR OBSERVES THE UNIT TOO, and this is the half a reason string cannot supply. An -// operator releasing "because the worker looked dead" is asserting exactly the fact that nothing -// checked -- and releasing a seat whose cargo is still resident hands its memory to the next -// arrival. So recovery consumes the same termination decision settlement does: only a unit the -// manager reports stopped, or has no record of, may be released. A running unit is refused BY NAME -// with the state that refused it, so the operator's next move is to stop the unit rather than to -// argue with the tool. -fn compute_release_observed(instance: HostDashboardInstance, identity: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { - let termination = observe_unit_termination(identity_hex: identity) - let wire = if unit_termination_frees_capacity(t: termination) { - compute_release_cli_wire(r: compute_release_reference( - instance: instance, - reference: reference as NonEmptyStr, - reason: join([reason, " (unit ", unit_termination_wire(t: termination), ")"], "") as NonEmptyStr)) - } else { - join(["refused: ", unit_termination_wire(t: termination), "; stop the unit first, then release"], "") - } - let receipt = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) - if !receipt.success { - ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; release was ", wire], "") } - } else if starts_with(s: wire, prefix: "released ") { - ExitSuccess - } else { - ExitFailure { code: 1, reason: join(["compute release refused: ", wire], "") } - } -} - -fn compute_release_cli_wire(r: ComputeRelease) -> String { - match r { - ComputeReleased { partition: p, reference: ref_, reason: why } => - join(["released ", ref_ as String, " on ", p as String, " (", why as String, ")"], "") - ComputeReleaseRefused { detail: d } => join(["refused: ", d], "") - } -} - fn compute_parse_operation(operation: String, entry: String, functions: String, hermetic: String) -> WorkOperation? { if operation == "build" { Present { value: work_default_build() } diff --git a/dag/gunbc/compute/work_request.dag b/dag/gunbc/compute/work_request.dag index 62e1231c219..16883a0830b 100644 --- a/dag/gunbc/compute/work_request.dag +++ b/dag/gunbc/compute/work_request.dag @@ -1,6 +1,7 @@ module gunbc.compute.work_request import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } +import std.measure { Kibibyte, kibibyte_count } import std.content_hash { Fnv1a64Structural, content_hash_atom, content_hash_tagged_structural } import extdeps.git.object_store { GitObjectId, git_object_id_wire_hex } import extdeps.languages.json.emit { @@ -138,6 +139,14 @@ type WorkOutcome // this class reserves, the checkout or the store could not be prepared, or a dependency of this // operation (the build the compile consumes) did not succeed. // +// THE MEMORY ARMS CARRY Kibibyte, NOT Int FIELDS NAMED _kib. A unit in a field NAME on a bare Int +// is the unit modeled twice -- once in the name and once in the " KiB" this module then renders -- +// and modeled nowhere the compiler can see. The producer already had it right +// (gunbc.compute.host_capacity ComputeHostBelowFloor carries Kibibyte) and the provider was calling +// kibibyte_count to STRIP the carrier on the way in, which is the tell (review 69624). The deleted +// arm this replaced, HostAtColdBuildCap { in_flight, cap }, was a dimensionless COUNT, so Int was +// right there and is not right here. +// // THREE CAPACITY ARMS WHERE THERE USED TO BE ONE COUNT. HostAtColdBuildCap { in_flight, cap } is // DELETED, and its disposition is that the fact it reported -- this host is already carrying as // much as it may -- is now HostComputeCapacityFull, measured in the memory the pool actually @@ -148,7 +157,7 @@ type WorkOutcome type WorkInfrastructureRefusal = ProducerInFlight { lease_path: String } | HostComputeCapacityFull { wire: String } - | HostBelowMemoryFloor { requested_kib: Int, available_kib: Int } + | HostBelowMemoryFloor { requested: Kibibyte, available: Kibibyte } | ComputeCapacityLedgerUnavailable { detail: String } | CheckoutUnavailable { detail: String } | StoreUnavailable { detail: String } @@ -180,7 +189,7 @@ fn work_refusal_detail(r: WorkInfrastructureRefusal) -> String { match r { ProducerInFlight { lease_path } => join(["a producer already holds the lease at ", lease_path, "; attach later, do not start a second one"], "") HostComputeCapacityFull { wire } => join(["this host's compute memory pool has no room for another reservation: ", wire], "") - HostBelowMemoryFloor { requested_kib, available_kib } => join(["this class reserves ", to_string(value: requested_kib), " KiB and the host reports only ", to_string(value: available_kib), " KiB available, so the reservation would not have been backed by the machine"], "") + HostBelowMemoryFloor { requested, available } => join(["this class reserves ", to_string(value: kibibyte_count(k: requested)), " KiB and the host reports only ", to_string(value: kibibyte_count(k: available)), " KiB available, so the reservation would not have been backed by the machine"], "") ComputeCapacityLedgerUnavailable { detail } => join(["the host's compute capacity ledger could not be transacted: ", detail], "") CheckoutUnavailable { detail } => join(["checkout unavailable: ", detail], "") StoreUnavailable { detail } => join(["store unavailable: ", detail], "") diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index d7ae49970ac..ee33b3251b6 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -2,7 +2,7 @@ module gunbc.fabric_event_log import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat, nat_range_inclusive } -import std.measure { Count, One } +import std.measure { Count, One, Measure } import product.placement_supply { HostIdentity } import std.process { ProcessExit, ExitSuccess, exit_failure } import std.fabric_storage { @@ -244,13 +244,20 @@ type SeatAttemptState { // and take a reading. Every step is the one seat_attempt already runs, which is what keeps this a // second QUESTION rather than a second authority -- there is no separate notion here of what a seat // is or when a pool is full. -type SeatStandingObservation - = SeatRoomObserved { headroom: Nat, generation: Nat } +// THE HEADROOM CARRIES ITS DIMENSION, AND THIS PR IS WHY IT HAS TO. A bare Nat was legitimate while +// this fold was fixed at Pool: seats are dimensionless and the field meant seats. The +// generalization to Pool above made the same field mean KiB for a memory pool and seats for a +// seat pool, with nothing in the type to say which -- so the unit moved into the consumer's head and +// into rendered strings, which is the unit modeled twice and checked nowhere (review 69624). The +// earliest unjustified boundary is this declaration, not its consumers, and it is one this change +// moved itself (DESIGN 6b). +type SeatStandingObservation + = SeatRoomObserved { headroom: Measure, generation: Nat } | SeatFullObserved { generation: Nat } | SeatStandingUnobserved { step: String, reason: String } | SeatStandingStoreRefused { cause: EventLogRefusal } -fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root: Pool, at: EpochSecs, budget: Nat) -> SeatStandingObservation { +fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root: Pool, at: EpochSecs, budget: Nat) -> SeatStandingObservation { match event_log_read_partition(store: store, partition: partition, budget: budget) { PartitionReadRefused { cause: c } => SeatStandingStoreRefused { cause: c } PartitionReadOk { head: _, walk: walk } => @@ -268,7 +275,7 @@ fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId PoolReadingRefused { at: t, anchor: a } => SeatStandingUnobserved { step: "reading", reason: join(["instant ", to_string(t), " precedes the pool anchor ", to_string(a)], "") } PoolRead { committed: _, ceiling: _, headroom: h, over_committed: _ } => - if h.count > 0 { SeatRoomObserved { headroom: h.count, generation: g } } + if h.count > 0 { SeatRoomObserved { headroom: h, generation: g } } else { SeatFullObserved { generation: g } } } } diff --git a/dag/gunbc/harness/harness_seat.dag b/dag/gunbc/harness/harness_seat.dag index 4b6de567bae..7fb2d6751a5 100644 --- a/dag/gunbc/harness/harness_seat.dag +++ b/dag/gunbc/harness/harness_seat.dag @@ -3,7 +3,7 @@ module gunbc.harness.harness_seat import product.placement_supply { HostIdentity } import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat, nat_range_inclusive, nat_min } -import std.measure { Measure, Count, One, money_amount_micro, second } +import std.measure { Measure, Count, One, money_amount_micro, second, measure_count } import gunbc.spark.fabric_switch_observed { FabricGroup, FabricGroupA, FabricGroupB, fabric_group_wire, fabric_group_hosts } import gunbc.serving.serving_enrollment { ServingRoute, ServingRouteResolved, ServingRouteUnresolved, ServingRouteEndpoint, @@ -334,7 +334,7 @@ fn harness_class_held(route: ServingRoute, class: ServingCoTenancyClass, store: } SeatFullObserved { generation: _ } => ClassSeatsHeld { held: ceiling } SeatRoomObserved { headroom: h, generation: _ } => - ClassSeatsHeld { held: ceiling - nat_min(a: h, b: ceiling) } + ClassSeatsHeld { held: ceiling - nat_min(a: measure_count(h), b: ceiling) } } } } diff --git a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag new file mode 100644 index 00000000000..2857c2a58ff --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag @@ -0,0 +1,34 @@ +module gunbc.recurring_failure_mode.a_held_compute_seat_has_no_in_corpus_discharge + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, decl_ref } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = RecurringFailureMode { + identity: "a_held_compute_seat_has_no_in_corpus_discharge" as NonEmptyStr, + + receipts: [ + "INVALID STATE: a quantitative reservation is taken under a release law that never lapses, and the only path that returns it is the settlement of the run that took it. When that settlement cannot prove the unit is gone -- or cannot append at all -- the encumbrance stays Held and NO DECLARATION IN THE CORPUS CAN DISCHARGE IT. HARM: the host's compute pool shrinks by one grant per occurrence and never grows back; at gunbc.compute.host_capacity compute_pool_slots = 2, two occurrences consume the whole appropriation, after which every request is refused for capacity on a host with its memory free.", + + "THIS IS A NEW CLASS RATHER THAN A DECLARED RUNG DROP, and the distinction is the one DESIGN 4b(3) turns on: a drop is a rung that FELL. Before gunbc#11962 this host admitted work by counting lease files (gunbc.compute.work_provider_local compute_cold_build_cap, deleted), so there were no reservations at all and nothing to discharge. The state becomes reachable for the first time with the pool, so it is filed here and not in gunbc.rung_drop.", + + "WHY THE RESERVATION IS NOT SIMPLY FREED ON DOUBT, which is the repair that suggests itself and is the wrong one: the pool's release law for a compute seat is QuiescenceRequired precisely because nothing can prove a systemd transient unit stopped using memory from outside it. Freeing a seat whose cargo is still resident hands its memory to the next arrival on top of a live build. Every arm that cannot prove termination therefore keeps the charge, so this class's harm is UNDER-admission -- the fail-closed direction -- and its opposite would be over-admission on a shared host.", + + "SPECIMEN, EXECUTED ON srv1 (2026-09-21, gunbc#11962): a real release build was started under a grant of 27262976 KiB and the gunbc driver was SIGKILLed mid-run. The unit stayed active, the partition carried the acquire and no release, and the producer lease stayed held -- the commitment correctly stayed charged, and nothing in the corpus could return it.", + + "RUNG FOUND AT: mitigatable. The state is reachable, its harm is bounded to one host's compute appropriation, it is visible in the outcome document and in the partition, and it never over-promises.", + + "CEILING: mechanically preventable. A discharge route exists to be built -- the evidence needed is a termination decision bound to the unit plus a settled lifecycle state that a retry consumes -- so this is not a class that must stay a ratchet.", + + "NEXT-RUNG TRIGGER, NAMING THE CAPABILITY AND NOT AN ARTIFACT: the compute reservation LIFECYCLE capability -- a discharge that (a) decides termination from authoritative absence or an observed-empty cgroup rather than from a state word, (b) binds the attempt and its unit from the authoritative attempt record so releasing one cannot release another's live reservation, (c) coordinates with a pending producer so a release cannot race a launch that has not happened yet, and (d) produces a settled state a retry consumes, with history kept. Until a discharge with all four properties is consumed by a real caller, this row stands.", + + "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration, and gunbc.compute.work_provider_local compute_stuck_reservation_remedy carries the remedy sentence into the refusal a caller actually receives.", + ], + + evidence: [ + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "compute_stuck_reservation_remedy"), + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "unit_termination_frees_capacity"), + decl_ref(module_path: "gunbc.compute.host_capacity", decl_name: "compute_release"), + decl_ref(module_path: "test.claim.compute.work_request_witness", decl_name: "only_proven_absence_or_an_empty_cgroup_frees_a_seat"), + ], +} diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index 6051e698a4f..490ef641cd8 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -5,14 +5,13 @@ import std.types { Bool, Int, NonEmptyStr, String } import std.measure { Kibibyte, kibibyte, kibibyte_count } import std.algebra { trim } import product.capacity.event_chain { PartitionId } -import gunbc.compute.host_capacity { compute_capacity_store } import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, srv1_lab_dashboard_instance, dashboard_instance_compute_root } import gunbc.compute.host_capacity { ComputeCapacitySubject, ComputeSubjectResolution, ComputeSubjectResolved, ComputeSubjectHostUnresolved, compute_capacity_subject, compute_capacity_root, compute_memory_partition, HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - compute_reserve_on, compute_reserve_against, compute_release, compute_release_on, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + compute_reserve_on, compute_reserve_against, compute_release, ComputeRelease, ComputeReleased, ComputeReleaseRefused, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, } @@ -230,27 +229,3 @@ test fn the_real_observation_reads_this_machine() -> Bool { } } -// A WEDGED SEAT CAN ACTUALLY BE RECOVERED, BY REFERENCE, BY SOMEONE WHO NEVER HELD THE RESERVATION. -// Under QuiescenceRequired a term frees nothing, so this route is the ONLY way capacity returns -// after a release fails to land -- and until review 69595 the module claimed it in prose while no -// declaration reached it. -// -// AND A REFERENCE THE POOL IS NOT HOLDING WRITES NOTHING. That is the sharper half: an unheld -// reference appended anyway would not fail at the append, it would fail at every later READ, when -// the fold hits UnknownEncumbrance and refuses the WHOLE partition. So the refusal is asserted -// BESIDE a subsequent successful reservation, which is what establishes the ledger is still -// readable after the bad attempt. -test fn a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing() -> Bool { - let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes, request_kib: one_gibibyte_in_kibibytes) - let store = compute_capacity_store(subject: subject) - let held = compute_reserve_on(subject: subject, reference: "wet-wedged" as NonEmptyStr, term_seconds: 60) - let full = compute_reserve_on(subject: subject, reference: "wet-blocked" as NonEmptyStr, term_seconds: 60) - let bogus = compute_release_on(store: store, partition: subject.partition, appropriation: subject.appropriation, reference: "wet-never-held" as NonEmptyStr, reason: "operator recovery" as NonEmptyStr) - let recovered = compute_release_on(store: store, partition: subject.partition, appropriation: subject.appropriation, reference: "wet-wedged" as NonEmptyStr, reason: "operator recovery" as NonEmptyStr) - let readmitted = compute_reserve_on(subject: subject, reference: "wet-after-recovery" as NonEmptyStr, term_seconds: 60) - reserved_amount(r: held) == one_gibibyte_in_kibibytes - && is_capacity_full(r: full) - && !released(r: bogus) - && released(r: recovered) - && reserved_amount(r: readmitted) == one_gibibyte_in_kibibytes -} diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 21b15320449..ac8dbabde15 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -3,8 +3,8 @@ module test.claim.compute.work_request_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } import gunbc.compute.work_provider_local { - UnitTermination, UnitQuiescent, UnitStillActive, UnitTerminationUnobserved, - unit_state_termination, unit_termination_frees_capacity, + UnitTermination, UnitAbsentAuthoritatively, UnitTerminatedObserved, UnitPopulated, UnitTerminationUnavailable, + unit_termination_frees_capacity, unit_termination_wire, } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, @@ -79,32 +79,22 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool && (match stored_outcome_decode(text: "not json") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) } -// ONLY A UNIT THAT IS DEMONSTRABLY GONE MAY FREE ITS SEAT, and the arm that matters is the one in -// the middle. `systemd-run --wait` is a LAUNCHER, not the unit's parent, so a launcher that dies or -// times out returns failure while the transient service keeps running; releasing on that signal -// would hand a live cargo's memory to the next arrival. Empty and inactive and failed are all -// "nothing of ours is holding that memory" -- never started and already collected are the same fact -// for this decision. Everything systemd calls live keeps the charge, and a word this fold has never -// seen keeps it too: a future ActiveState must not read as permission to take memory back. -test fn only_a_gone_unit_frees_its_seat_and_an_unknown_state_does_not() -> Bool { - all(["", "inactive", "failed", " inactive "], s => - match unit_state_termination(state: s) { - UnitQuiescent { state: _ } => true - UnitStillActive { state: _ } => false - UnitTerminationUnobserved { detail: _ } => false - }) - && all(["active", "activating", "deactivating", "reloading", "maintenance"], s => - match unit_state_termination(state: s) { - UnitStillActive { state: _ } => true - UnitQuiescent { state: _ } => false - UnitTerminationUnobserved { detail: _ } => false - }) - && all(["quantum-superposed", "ACTIVE", "1"], s => - match unit_state_termination(state: s) { - UnitTerminationUnobserved { detail: _ } => true - UnitQuiescent { state: _ } => false - UnitStillActive { state: _ } => false - }) - && all(["", "inactive", "failed"], s => unit_termination_frees_capacity(t: unit_state_termination(state: s))) - && all(["active", "activating", "deactivating", "quantum-superposed"], s => !unit_termination_frees_capacity(t: unit_state_termination(state: s))) +// ONLY PROVEN ABSENCE OR AN OBSERVED-EMPTY CGROUP MAY FREE A SEAT, and the two arms this cell +// exists to pin RED are the ones an earlier revision got wrong: `failed` can coexist with live +// processes while a unit's stop reaches its SIGKILL timeout, and an empty ActiveState is the +// manager declining to answer rather than authoritative absence. Neither may free capacity. The +// unavailable arm is the one that matters most -- it is what a lost launcher, an unreadable manager +// and a missing cgroup all look like, and reading any of them as permission is the absorbing +// fallback DESIGN 5 forbids. +test fn only_proven_absence_or_an_empty_cgroup_frees_a_seat() -> Bool { + let absent = UnitAbsentAuthoritatively { detail: "LoadState=not-found" } + let terminated = UnitTerminatedObserved { detail: "populated 0" } + let populated = UnitPopulated { detail: "populated 1" } + let unavailable = UnitTerminationUnavailable { detail: "the manager could not be read" } + unit_termination_frees_capacity(t: absent) + && unit_termination_frees_capacity(t: terminated) + && !unit_termination_frees_capacity(t: populated) + && !unit_termination_frees_capacity(t: unavailable) + && string_contains(s: unit_termination_wire(t: populated), pattern: "STILL POPULATED") + && string_contains(s: unit_termination_wire(t: unavailable), pattern: "UNAVAILABLE") } diff --git a/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag b/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag index d8b4c0c4ebf..5ffe31d285e 100644 --- a/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag +++ b/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag @@ -3,7 +3,7 @@ module test.claim.spark.pair_serving_d0_real_execution import std.logic { Bool } import std.types { String, NonEmptyStr, FilePath, List, Int } import v2.std.optional { Present, Absent } -import std.measure { second } +import std.measure { second, measure_count } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import extdeps.shell import std.content_hash { ContentHash, Sha256Hash, sha256_hex_digest, compare_content_hash, ContentHashEqual, ContentHashDifferent, ContentHashCrossFamilyIncomparable, serialize_content_hash, content_hash_equal } @@ -390,7 +390,7 @@ test fn a_seat_granted_after_the_suspension_landed_is_released_by_real_execution _ => false }) && (match fabric_seat_observe(store: store, partition: partition, root: root, at: 1002, budget: 64) { - SeatRoomObserved { headroom: room, generation: _ } => room == ceiling + SeatRoomObserved { headroom: room, generation: _ } => measure_count(room) == ceiling _ => false }) _ => false diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 6e8d09c7ff7..2115d42e0af 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -250,12 +250,6 @@ fn local_repo_wet_schedule() -> List { function: "the_real_observation_reads_this_machine", expectation: ExpectedToHold {} }, - WetScheduledClaim { - identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing" }, - entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", - function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing", - expectation: ExpectedToHold {} - }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "two_requests_summing_past_the_observed_headroom_cannot_both_admit" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", From 773825cf91dd9eea6888a6845e5363c1ec18830f Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 18:49:51 +0000 Subject: [PATCH 08/44] The empty-ControlGroup release arm, the discarded host record, and two remedy texts that told a reader the wrong thing Side-chat review at 5ee0cce plus review 69645. All four verified against the code first. THE THIRD RELEASE ARM. observe_unit_cgroup read an empty ControlGroup as UnitTerminatedObserved without reading any cgroup at all, and an empty or unrecognized successful LoadState could reach it. An empty ControlGroup does not mean finished: systemd realizes the cgroup AT SPAWN, so it equally describes a start that is still PENDING -- which makes a whole unbacked launch reachable. Reserve; the start is pending; the LAUNCHER is lost while this driver lives on; settlement sees the empty property, releases; the job then starts against memory nobody holds. The arm is deleted and an empty required property is never promoted to a conclusion -- it is UNAVAILABLE like every other absence of evidence. Capacity is now freed on two facts that establish the attempt ENDED, plus one that requires a realized cgroup: THE ATTEMPT COMPLETED -- `systemd-run --wait` returned success, which it does only after the unit ran to completion. A lost launcher does not return success and a pending job has not returned at all. This is a SEPARATE fact from the work's ending, and conflating them is what let a lost launcher free a live unit's memory: a WorkFailed says the unit exited nonzero OR that the launcher never got an answer. THE MANAGER HAS NO RECORD -- LoadState=not-found. A REALIZED CGROUP REPORTING populated 0 -- which cannot be a pending start, because the cgroup exists only from spawn onward. THE DECISION IS NOW A FOLD OVER SUPPLIED READINGS (UnitObservations), so the controls sit at the OBSERVATION-TO-DECISION boundary rather than at the variant-to-release mapping, which cannot see any of the defects that actually occurred: a failed query whose text reads like an answer, an empty property, a pending start. A failed query is unavailable whatever it printed -- the exit status decides first and the text is never consulted. review 69645, THE DISCARDED HOST RECORD: `if measured.success { settled } else { settled }` had identical arms, so a failed write vanished by construction. The compound case was the sharp one -- the producer lease failing to drop AND that write failing left an identity whose lease is permanently held with NO RECORD ANYWHERE that it is stale, while the caller got an ordinary success. It now refuses, naming what was lost. This does not reopen blocker (7): that was a refusal DENYING a published success; this one affirms the run's ending, says the stored record is authoritative, and reports a host fault that happened after publication. review 69645, THE DEAD CITATION: fabric_seat_release's annotation named compute_release_reference and compute_release_cli, which the split removed. It now names its real consumer and states that the operator door is the capability PR 2 lands. TWO REMEDY TEXTS THAT MISLED. compute_stale_lease_note said the next request would refuse as ProducerInFlight, contradicting the contract the same block establishes: a caller reaching a published SUCCESS attaches, and the attach is decided before the lease is consulted -- the stale name only blocks a request that must PRODUCE again. And compute_stuck_reservation_remedy told an operator to clear the partition by hand, which under live admission removes the record the admission arithmetic folds and trades a host that under-admits for one that over-promises; it now refuses that and states the ordering any manual intervention requires. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 257 +++++++++++++----- dag/gunbc/fabric/fabric_event_log.dag | 17 +- ...ompute_seat_has_no_in_corpus_discharge.dag | 4 +- .../compute/work_request_witness_test.dag | 64 +++-- 4 files changed, 250 insertions(+), 92 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 3e19b143212..3fcd3b6b23f 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -240,39 +240,49 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden } } -// ── WHETHER THE UNIT IS STILL USING THE MEMORY ──────────────────────────────────────────────── +// ── WHETHER THE ATTEMPT HAS ENDED AND ITS UNIT IS GONE ──────────────────────────────────────── // // THE RELEASE LAW SAID "QUIESCENCE REQUIRED" AND NOTHING OBSERVED QUIESCENCE. compute_release ran -// on whatever compute_run_leased returned, and what it returns is the exit of `systemd-run --wait`, -// which is NOT the unit. systemd-run is a launcher, not the unit's parent -- the transient service -// is owned by the manager -- so a launcher killed, timed out, or disconnected from the bus yields -// ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, and the seat's memory would be handed to the next -// arrival on top of a live cargo. product.capacity.lease already models this (QuiescenceFact) and -// this provider was not consulting it. +// on whatever compute_run_leased returned, and that is the exit of `systemd-run --wait`, which is +// NOT the unit: systemd-run is a launcher, the transient service is owned by the manager, and a +// launcher killed or disconnected yields a failure WHILE THE UNIT KEEPS RUNNING. // -// ActiveState ALONE IS NOT THE EVIDENCE, AND THE FIRST FIX OF THIS THOUGHT IT WAS. Two arms it -// treated as free are not (side-chat review of gunbc#11962 at 9172cf7): +// TWO EARLIER SHAPES OF THIS FOLD WERE WRONG, AND BOTH WERE WRONG IN THE SAME DIRECTION -- they +// found a way to say "gone" from evidence that does not establish it. // -// `failed` CAN COEXIST WITH LIVE PROCESSES. A unit whose stop reaches the SIGKILL timeout is -// reported failed while its processes are still being killed, and a task wedged in -// uninterruptible sleep outlives the state word entirely. +// ActiveState ALONE (first shape): `failed` coexists with live processes while a unit's stop +// reaches its SIGKILL timeout, and an empty ActiveState is the manager declining to answer. // -// AN EMPTY ActiveState IS NOT AUTHORITATIVE ABSENCE. It is the manager declining to answer in a -// form this provider cannot distinguish from a unit it has never heard of. The repo's own -// systemctl authority already names the observation that proves absence, and it is LoadState: -// `systemctl show` exits 0 for a unit the manager has never heard of and prints not-found, so a -// nonzero exit means the MANAGER was unreadable, which is a different refusal from an absent unit -// and must not be folded into it. +// AN EMPTY ControlGroup (second shape, side-chat review at 5ee0cce): it was read as observed +// termination without reading any cgroup at all, and it is not. systemd realizes the cgroup AT +// SPAWN, so an empty ControlGroup equally describes a unit whose start is still PENDING -- which +// makes a whole unbacked launch reachable: reserve, the start is pending, the launcher is lost +// while this driver lives on, settlement sees the empty property, releases, and THEN the job +// starts against memory nobody holds. An empty required property is now never promoted to a +// conclusion; it is UNAVAILABLE, like every other absence of evidence. // -// SO THE DECISION IS FOUR-VALUED AND ONLY THE FIRST TWO FREE CAPACITY. Authoritative absence, from -// the one wire value that proves it. Observed termination, from the cgroup this unit owns reporting -// no processes -- the kernel's own answer to "is anything still running here", which is the question -// the memory is actually about. Populated keeps the charge. And UNAVAILABLE keeps it too, which is -// the arm that matters most: "I could not ask" is what a lost launcher, an unreadable manager and a -// missing cgroup all look like, and reading any of them as permission to take the memory back is the -// absorbing fallback DESIGN 5 forbids. +// SO CAPACITY IS FREED ON TWO FACTS AND NO OTHERS, both of which establish that the attempt ENDED +// rather than that it looks quiet: +// +// THE ATTEMPT COMPLETED. `systemd-run --wait` returned success, which it does only after the +// unit it started ran to completion. A lost launcher does not return success, and a pending job +// has not returned at all. +// +// THE MANAGER HAS NO RECORD OF THE UNIT. LoadState=not-found is the wire value this corpus reads +// positively, and a transient --wait unit reaches it once it has run and been collected. +// +// A REALIZED CGROUP REPORTING populated 0 IS THE THIRD, AND IT IS EVIDENCE RATHER THAN A GUESS, +// which is why it survives where the empty property did not: the cgroup exists only from spawn +// onward, so a NON-EMPTY ControlGroup whose cgroup reports no processes cannot be a pending start. +// It spawned, and everything in it exited. +// +// EVERYTHING ELSE KEEPS THE CHARGE: populated, an unreadable manager, an unreadable cgroup, an +// undecodable body, a property that came back empty. "I could not establish it" is what a lost +// launcher, a pending job and a broken bus all look like, and reading any of them as permission to +// take the memory back is the absorbing fallback DESIGN 5 forbids. type UnitTermination - = UnitAbsentAuthoritatively { detail: String } + = UnitAttemptCompleted { detail: String } + | UnitAbsentAuthoritatively { detail: String } | UnitTerminatedObserved { detail: String } | UnitPopulated { detail: String } | UnitTerminationUnavailable { detail: String } @@ -281,53 +291,90 @@ data unit_load_state_property: NonEmptyStr = "LoadState" as NonEmptyStr data unit_control_group_property: NonEmptyStr = "ControlGroup" as NonEmptyStr data unit_load_state_absent_wire: String = "not-found" -// THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is -// system-scope; these are transient --user units, so asking the system manager would answer -// not-found for EVERY one of them -- a fabricated authoritative absence, which is the single worst -// answer this fold can produce. The property goes through the user-scope show operation for that -// reason and no other. -fn observe_unit_termination(identity_hex: String) -> UnitTermination { - let unit = compute_unit_name(identity_hex: identity_hex) - let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) - if !load.success { - UnitTerminationUnavailable { detail: join(["the user manager could not be read for ", unit as String], "") } - } else if trim(s: load.value) == unit_load_state_absent_wire { - UnitAbsentAuthoritatively { detail: join([unit as String, " LoadState=", unit_load_state_absent_wire], "") } +// ONE PROPERTY READ, AS READ. success is the query's, value is the manager's answer; the two are +// separate because a successful query that answers nothing is a different fact from a query that +// failed, and the fold below must be able to tell them apart without re-deriving either. +type UnitPropertyReading { + queried: Bool + value: String +} + +// THE RAW OBSERVATIONS THE DECISION IS MADE FROM, carried as a value so the decision is a fold over +// supplied readings rather than a fold over live service calls. That is what lets the controls +// drive the OBSERVATION-TO-DECISION boundary -- a failed query whose text looks like an answer, a +// property that came back empty, a pending start -- instead of only the variant-to-release mapping, +// which cannot see any of those (side-chat review at 5ee0cce). +type UnitObservations { + attempt_completed: Bool + load_state: UnitPropertyReading + control_group: UnitPropertyReading + events: UnitPropertyReading +} + +// A FAILED QUERY IS UNAVAILABLE WHATEVER IT PRINTED. systemctl writes diagnostics that read like +// answers -- "Failed to connect to bus", "not-found" inside a sentence -- so the exit status +// decides first and the text is never consulted on a failed query. +fn unit_termination_decide(o: UnitObservations) -> UnitTermination { + if o.attempt_completed { + UnitAttemptCompleted { detail: "systemd-run --wait returned success, so the unit it started ran to completion" } + } else if !o.load_state.queried { + UnitTerminationUnavailable { detail: "the user manager could not be queried for this unit, so nothing about it is established" } + } else if trim(s: o.load_state.value) == unit_load_state_absent_wire { + UnitAbsentAuthoritatively { detail: join(["LoadState=", unit_load_state_absent_wire], "") } + } else if trim(s: o.load_state.value) == "" { + UnitTerminationUnavailable { detail: "the manager answered the LoadState query with nothing, which establishes neither presence nor absence" } + } else if !o.control_group.queried { + UnitTerminationUnavailable { detail: join(["LoadState=", trim(s: o.load_state.value), " but the control group could not be queried"], "") } + } else if trim(s: o.control_group.value) == "" { + UnitTerminationUnavailable { + detail: join([ + "LoadState=", trim(s: o.load_state.value), + " and the unit holds no control group, which is EITHER a unit that has finished OR a start that is still pending -- the cgroup is realized at spawn -- so it establishes neither", + ], ""), + } + } else if !o.events.queried { + UnitTerminationUnavailable { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be read"], "") } } else { - observe_unit_cgroup(unit: unit, load_state: trim(s: load.value)) + match decode_cgroup_events_populated(body: o.events.value) { + CgroupEventsUnparseable { cause: c } => UnitTerminationUnavailable { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be decoded: ", c], "") } + CgroupEventsDecoded { populated: p } => + if p { + UnitPopulated { detail: join(["processes remain in ", trim(s: o.control_group.value)], "") } + } else { + UnitTerminatedObserved { detail: join(["the realized cgroup ", trim(s: o.control_group.value), " reports populated 0, so it spawned and everything in it exited"], "") } + } + } } } -// THE UNIT EXISTS, SO THE QUESTION IS WHETHER ANYTHING IS STILL RUNNING IN IT, and the kernel -// answers that directly. ControlGroup is empty for a unit that is loaded and holds no processes -- -// systemd removes the cgroup when the last one exits -- so an empty value here is observed -// termination rather than a failed read, and it is the ordinary answer after a clean run. -fn observe_unit_cgroup(unit: NonEmptyStr, load_state: String) -> UnitTermination { +// THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is +// system-scope; these are transient --user units, so asking the system manager would answer +// not-found for EVERY one of them -- a fabricated authoritative absence, the worst answer this fold +// can produce. The property goes through the user-scope show operation for that reason and no other. +// +// THE READS ARE TAKEN UNCONDITIONALLY AND THE FOLD DECIDES. Short-circuiting them here would put +// half the decision in this function and half in the fold, which is the split that let an empty +// property become a conclusion in the first place. +fn observe_unit_termination(identity_hex: String, attempt_completed: Bool) -> UnitTermination { + let unit = compute_unit_name(identity_hex: identity_hex) + let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) let cg = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_control_group_property) - if !cg.success { - UnitTerminationUnavailable { detail: join(["the control group of ", unit as String, " could not be read"], "") } - } else if trim(s: cg.value) == "" { - UnitTerminatedObserved { detail: join([unit as String, " LoadState=", load_state, " holds no control group, so no process of this unit survives"], "") } + let events = if cg.success && trim(s: cg.value) != "" { + linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) } else { - let events = linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) - if !events.success { - UnitTerminationUnavailable { detail: join(["the cgroup events of ", unit as String, " could not be read at ", trim(s: cg.value)], "") } - } else { - match decode_cgroup_events_populated(body: events.value) { - CgroupEventsUnparseable { cause: c } => UnitTerminationUnavailable { detail: join(["the cgroup events of ", unit as String, " could not be decoded: ", c], "") } - CgroupEventsDecoded { populated: p } => - if p { - UnitPopulated { detail: join([unit as String, " still has processes in ", trim(s: cg.value)], "") } - } else { - UnitTerminatedObserved { detail: join([unit as String, " cgroup ", trim(s: cg.value), " reports populated 0"], "") } - } - } - } + linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: cgroup_v2_mount_point)) } + unit_termination_decide(o: UnitObservations { + attempt_completed: attempt_completed, + load_state: UnitPropertyReading { queried: load.success, value: load.value }, + control_group: UnitPropertyReading { queried: cg.success, value: cg.value }, + events: UnitPropertyReading { queried: events.success, value: events.value }, + }) } fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { match t { + UnitAttemptCompleted { detail: _ } => true UnitAbsentAuthoritatively { detail: _ } => true UnitTerminatedObserved { detail: _ } => true UnitPopulated { detail: _ } => false @@ -337,10 +384,11 @@ fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { fn unit_termination_wire(t: UnitTermination) -> String { match t { + UnitAttemptCompleted { detail: d } => join(["attempt completed (", d, ")"], "") UnitAbsentAuthoritatively { detail: d } => join(["absent (", d, ")"], "") UnitTerminatedObserved { detail: d } => join(["terminated (", d, ")"], "") UnitPopulated { detail: d } => join(["STILL POPULATED (", d, "), so the seat stays charged"], "") - UnitTerminationUnavailable { detail: d } => join(["termination UNAVAILABLE (", d, "), so the seat stays charged"], "") + UnitTerminationUnavailable { detail: d } => join(["termination UNESTABLISHED (", d, "), so the seat stays charged"], "") } } @@ -411,6 +459,21 @@ fn compute_basename(path: String) -> String { // outcome and the obligation were written, so a second producer could take the lease and start // against a work directory whose record was still being published -- the exclusion held for the // name and not for the publication it exists to protect. +// THE ATTEMPT'S OWN COMPLETION IS A SEPARATE FACT FROM THE WORK'S ENDING, and only the first one +// licenses a release. A WorkFailed says the unit exited nonzero OR that the launcher never got an +// answer; a successful `systemd-run --wait` says the unit ran to completion, which is the fact the +// termination decision needs. So the ending is what the caller is told and this is what the seat is +// decided on; they are not the same question and were conflated before. +fn compute_attempt_completed(outcome: WorkOutcome) -> Bool { + match outcome { + WorkSucceeded { identity: _, outputs: _, log_path: _ } => true + WorkOutputMismatch { identity: _, detail: _ } => true + WorkFailed { identity: _, exit_code: _, log_path: _ } => false + WorkRefusedByInfrastructure { identity: _, cause: _ } => false + WorkCancelled { identity: _ } => false + } +} + fn compute_run_and_settle( instance: HostDashboardInstance, commit: String, @@ -425,7 +488,7 @@ fn compute_run_and_settle( ) -> ProvideResult { let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) let standing = compute_capacity_standing(instance: instance) - let termination = observe_unit_termination(identity_hex: identity) + let termination = observe_unit_termination(identity_hex: identity, attempt_completed: compute_attempt_completed(outcome: outcome)) let returned = if unit_termination_frees_capacity(t: termination) { compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome), ", unit ", unit_termination_wire(t: termination)], "") as NonEmptyStr) } else { @@ -447,7 +510,58 @@ fn compute_run_and_settle( "reservation: ", compute_release_wire(r: returned), "\n", if dropped.success { "" } else { compute_stale_lease_note(layout: layout, error: dropped.error) }, ], "")) - if measured.success { settled } else { settled } + if measured.success { + settled + } else { + compute_host_record_lost( + layout: layout, identity: identity, subject: subject, op: op, settled: settled, + dropped_ok: dropped.success, dropped_error: dropped.error, error: measured.error) + } +} + +// THE HOST RECORD IS THE ONLY DURABLE CARRIER OF THREE FACTS, so losing it stops the line rather +// than being discarded. The arm used to read `if measured.success { settled } else { settled }` -- +// both arms identical, so a failed write vanished by construction (review 69645), and the +// COMPOUND case was the sharp one: the producer lease failing to drop AND this write failing left +// an identity whose lease is permanently held with NO RECORD ANYWHERE that it is stale, while the +// caller was handed an ordinary success. The module's own prose claimed that stale lease "is +// recorded as the operational fact it is"; the arm let that recording disappear. +// +// WHY THIS DOES NOT REOPEN BLOCKER (7). That blocker was a refusal that DENIED a published +// success -- the caller was told the run failed while the stored record said it succeeded and the +// next caller would attach to that success. This refusal denies nothing: it names the run's real +// ending, says the stored record is published and authoritative, and reports a HOST fault that +// happened after publication. The two callers still read the same thing about the work; what this +// one additionally learns is that the host has state no file describes. +fn compute_host_record_lost( + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + settled: ProvideResult, + dropped_ok: Bool, + dropped_error: String, + error: String, +) -> ProvideResult { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { + detail: join([ + "the work's own record is published at ", layout.outcome_path, " and is authoritative -- it ended ", + provide_result_ending(r: settled), + " and the next caller reads exactly that -- but this host's record at ", layout.consumption_path, + " could not be written: ", error, + "; so the consumption reading is lost, and ", + if dropped_ok { + "the producer lease was dropped, so nothing else is outstanding" + } else { + join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be requested again"], "") + }, + ], ""), + }, + }, + subject: subject, op: op) } // BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the @@ -457,11 +571,18 @@ fn compute_run_and_settle( // decides: a published success IS the result, the caller is told the same thing the next caller // will read, and the stale lease is recorded as the operational fact it is rather than converted // into a verdict about the work. An operator removes the name; nothing about the run changed. +// +// AND THE NOTE SAYS WHAT THE NAME ACTUALLY BLOCKS. It used to say the next request would refuse as +// ProducerInFlight, which contradicts the very contract this block establishes: a caller reaching a +// published SUCCESS attaches to it, and the attach is decided before the lease is consulted. The +// stale name only blocks a request that must PRODUCE again. fn compute_stale_lease_note(layout: ComputeLayout, error: String) -> String { join([ "STALE PRODUCER LEASE: ", layout.lease_path, " could not be dropped: ", error, - "; the record for this identity is published and is authoritative, and the next request will ", - "refuse as ProducerInFlight until an operator removes the name\n", + "; the record for this identity is published at ", layout.outcome_path, " and is authoritative, ", + "and a caller reaching that record ATTACHES to it -- the attach is decided before this name is ", + "consulted. The stale name matters only to a request that must PRODUCE again, which it will ", + "refuse as ProducerInFlight until an operator removes it\n", ], "") } @@ -534,7 +655,7 @@ fn compute_unreleased_record( // WHAT KEEPS IT FAIL-CLOSED MEANWHILE: every arm that cannot prove the unit is gone keeps the // charge, so the host UNDER-admits. Nothing is over-promised; the pool simply shrinks until an // operator clears the partition by hand. That is the direction a capacity ledger must fail in. -data compute_stuck_reservation_remedy: String = "Until the compute lifecycle capability lands (gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge), the only discharge is an operator clearing this host's compute capacity partition by hand; this host will under-admit until then, which is the fail-closed direction." +data compute_stuck_reservation_remedy: String = "There is NO safe discharge for this seat while this host is admitting work: clearing the capacity partition by hand under live admission removes the record the admission arithmetic is computed from, so the next requests would be admitted against memory this one still holds -- trading a host that under-admits for one that over-promises. The discharge is the compute lifecycle capability (gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge). Any manual intervention before then requires compute admission STOPPED on this host and every gunbc-compute unit PROVEN gone, in that order. Until then this host under-admits, which is the fail-closed direction and is not an incident." // THE ORDER THE TWO HOLDS ARE TAKEN IN// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index ee33b3251b6..884a065cb3f 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -424,11 +424,18 @@ fn pool_event_append_wire(a: PoolEventAppend) -> String { // fold it, PROPOSE the release against the folded pool, and append only what the pool admits. A // reference the pool is not holding is a typed refusal that writes nothing. // -// ITS CONSUMER IS THE RECOVERY ROUTE A HELD SEAT NEEDS (gunbc.compute.host_capacity -// compute_release_reference, reached from the operator CLI compute_release_cli). Under -// QuiescenceRequired a term never frees capacity, so an explicit release is the ONLY way a seat -// comes back, and a module claiming that route owes one that executes (review 69595 on gunbc#11962, -// DESIGN 3c). +// ITS CONSUMER IS gunbc.compute.host_capacity compute_release_on, which every compute settlement +// reaches when a run ends and its unit is proven gone. Under QuiescenceRequired a term never frees +// capacity, so an explicit release is the ONLY way a seat comes back. +// +// AN OPERATOR-FACING RECOVERY DOOR IS NOT AMONG ITS CONSUMERS, and this annotation said it was +// until review 69645 caught the citation naming two symbols that do not exist -- they were removed +// when the compute lifecycle was split out, and the comment was not. A citation that does not +// resolve is a citation defect (DESIGN 3), and worse here because host_capacity's own annotation +// states the opposite. Discharging a seat whose release did not land is the capability named by +// gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge; when it lands it +// will reach this same function, which is why the check below is built to serve a caller that did +// not take the reservation. type SeatRelease = SeatReleased { reference: NonEmptyStr, id: EventId } | SeatNotHeld { reference: NonEmptyStr, wire: String } diff --git a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag index 2857c2a58ff..b137018ba21 100644 --- a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag +++ b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag @@ -16,6 +16,8 @@ data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = Recu "SPECIMEN, EXECUTED ON srv1 (2026-09-21, gunbc#11962): a real release build was started under a grant of 27262976 KiB and the gunbc driver was SIGKILLed mid-run. The unit stayed active, the partition carried the acquire and no release, and the producer lease stayed held -- the commitment correctly stayed charged, and nothing in the corpus could return it.", + "THERE IS NO SAFE MANUAL DISCHARGE UNDER LIVE ADMISSION, and saying so is part of the row rather than a caveat on it. Clearing the capacity partition by hand while this host is admitting work removes the very record the admission arithmetic folds, so the next requests are admitted against memory the stuck grant still holds -- trading a host that under-admits for one that over-promises, which is the direction this whole change exists to prevent. Any manual intervention requires compute admission STOPPED on the host and every gunbc-compute unit PROVEN gone, in that order.", + "RUNG FOUND AT: mitigatable. The state is reachable, its harm is bounded to one host's compute appropriation, it is visible in the outcome document and in the partition, and it never over-promises.", "CEILING: mechanically preventable. A discharge route exists to be built -- the evidence needed is a termination decision bound to the unit plus a settled lifecycle state that a retry consumes -- so this is not a class that must stay a ratchet.", @@ -29,6 +31,6 @@ data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = Recu decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "compute_stuck_reservation_remedy"), decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "unit_termination_frees_capacity"), decl_ref(module_path: "gunbc.compute.host_capacity", decl_name: "compute_release"), - decl_ref(module_path: "test.claim.compute.work_request_witness", decl_name: "only_proven_absence_or_an_empty_cgroup_frees_a_seat"), + decl_ref(module_path: "test.claim.compute.work_request_witness", decl_name: "only_proven_absence_or_an_ended_cgroup_frees_a_seat"), ], } diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index ac8dbabde15..93878ef609b 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -3,8 +3,9 @@ module test.claim.compute.work_request_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } import gunbc.compute.work_provider_local { - UnitTermination, UnitAbsentAuthoritatively, UnitTerminatedObserved, UnitPopulated, UnitTerminationUnavailable, - unit_termination_frees_capacity, unit_termination_wire, + UnitTermination, UnitAttemptCompleted, UnitAbsentAuthoritatively, UnitTerminatedObserved, UnitPopulated, UnitTerminationUnavailable, + UnitPropertyReading, UnitObservations, unit_termination_decide, + unit_termination_frees_capacity, unit_termination_wire, compute_attempt_completed, } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, @@ -79,22 +80,49 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool && (match stored_outcome_decode(text: "not json") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) } -// ONLY PROVEN ABSENCE OR AN OBSERVED-EMPTY CGROUP MAY FREE A SEAT, and the two arms this cell -// exists to pin RED are the ones an earlier revision got wrong: `failed` can coexist with live -// processes while a unit's stop reaches its SIGKILL timeout, and an empty ActiveState is the -// manager declining to answer rather than authoritative absence. Neither may free capacity. The -// unavailable arm is the one that matters most -- it is what a lost launcher, an unreadable manager -// and a missing cgroup all look like, and reading any of them as permission is the absorbing -// fallback DESIGN 5 forbids. -test fn only_proven_absence_or_an_empty_cgroup_frees_a_seat() -> Bool { - let absent = UnitAbsentAuthoritatively { detail: "LoadState=not-found" } - let terminated = UnitTerminatedObserved { detail: "populated 0" } - let populated = UnitPopulated { detail: "populated 1" } - let unavailable = UnitTerminationUnavailable { detail: "the manager could not be read" } - unit_termination_frees_capacity(t: absent) - && unit_termination_frees_capacity(t: terminated) +// THE CONTROLS SIT AT THE OBSERVATION-TO-DECISION BOUNDARY, not at the variant-to-release mapping, +// because the mapping cannot see the defects that actually occurred: a failed query whose TEXT +// looks like an answer, a property that came back EMPTY, and a start that is still PENDING all +// arrive as readings and are wrong only in how they are read (side-chat review at 5ee0cce). +// +// THE PENDING-START ROW IS THE ONE THIS CELL EXISTS FOR. An empty ControlGroup was read as observed +// termination, and it equally describes a unit whose start has not spawned yet -- systemd realizes +// the cgroup AT SPAWN. That made a whole unbacked launch reachable: reserve, start pending, the +// LAUNCHER lost while this driver lives on, settlement sees the empty property, releases, and the +// job then starts against memory nobody holds. It must establish nothing. +test fn only_proven_absence_or_an_ended_cgroup_frees_a_seat() -> Bool { + let no_events = UnitPropertyReading { queried: false, value: "" } + let completed = unit_termination_decide(o: UnitObservations { attempt_completed: true, load_state: UnitPropertyReading { queried: false, value: "" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) + let absent = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "not-found" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) + let misleading = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: false, value: "Failed to connect to bus: not-found" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) + let empty_load = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 0\n" } }) + let pending = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "" }, events: no_events }) + let populated = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 1\n" } }) + let failed_unit = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "failed" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 1\n" } }) + let ended = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 0\n" } }) + let undecodable = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "garbage\n" } }) + unit_termination_frees_capacity(t: completed) + && unit_termination_frees_capacity(t: absent) + && unit_termination_frees_capacity(t: ended) + && !unit_termination_frees_capacity(t: misleading) + && !unit_termination_frees_capacity(t: empty_load) + && !unit_termination_frees_capacity(t: pending) && !unit_termination_frees_capacity(t: populated) - && !unit_termination_frees_capacity(t: unavailable) + && !unit_termination_frees_capacity(t: failed_unit) + && !unit_termination_frees_capacity(t: undecodable) + && string_contains(s: unit_termination_wire(t: pending), pattern: "pending") + && string_contains(s: unit_termination_wire(t: misleading), pattern: "UNESTABLISHED") && string_contains(s: unit_termination_wire(t: populated), pattern: "STILL POPULATED") - && string_contains(s: unit_termination_wire(t: unavailable), pattern: "UNAVAILABLE") +} + +// A COMPLETED ATTEMPT RELEASES, AND AN ATTEMPT THAT DID NOT COMPLETE DOES NOT RELEASE ON ITS OWN. +// The work's ENDING and the attempt's COMPLETION are different questions: a WorkFailed says the +// unit exited nonzero OR that the launcher never got an answer, and only the second is a lost +// launcher. The ending is what the caller is told; completion is what the seat is decided on. +test fn the_attempt_completion_fact_is_not_the_works_ending() -> Bool { + compute_attempt_completed(outcome: WorkSucceeded { identity: "a", outputs: [], log_path: "/l" }) + && compute_attempt_completed(outcome: WorkOutputMismatch { identity: "a", detail: "d" }) + && !compute_attempt_completed(outcome: WorkFailed { identity: "a", exit_code: 1, log_path: "/l" }) + && !compute_attempt_completed(outcome: WorkCancelled { identity: "a" }) + && !compute_attempt_completed(outcome: WorkRefusedByInfrastructure { identity: "a", cause: HostComputeCapacityFull { wire: "pool-full" } }) } From 73c570f9afa41f6b179367e5e1fac3b0dad4487d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 19:39:45 +0000 Subject: [PATCH 09/44] A completed wait never overrides a live cgroup; an uncertain completion keeps the charge; the record and the caller stop disagreeing Side-chat review at 773825cf, plus review 69663. All four verified first. (1) THE ORDER OF THE TESTS WAS THE DEFECT. attempt_completed was checked FIRST and returned immediately, so {completed, populated 1} RELEASED -- a launcher's success overriding the kernel saying processes are still there. `--wait` returns when the MAIN process exits, and under KillMode=process or none a forked child outlives it in the unit's cgroup. Both halves of the reviewer's either/or are taken, because each is independently right. KillMode=control-group is now BOUND on the invocation rather than inherited -- a default is not a declaration, and a drop-in could change it while this provider went on treating a completed wait as an empty cgroup. AND contrary evidence wins regardless: a positive populated=1 refuses the release before anything else is considered and is never discarded. (2) A MOMENTARILY EMPTY CGROUP IS NOT TERMINAL. populated=0 with completion NOT established used to release, and a realized cgroup can be empty between execs. In this PR an uncertain completion keeps the charge; the population reading now only CONFIRMS a completion or REFUSES one. UnitTerminatedObserved is gone as a releasing arm. Establishing termination without a completed attempt is the lifecycle capability's job. (3) THE RECORD AND THE CALLER DISAGREED AGAIN, one layer along: the lost-record repair published the real outcome and synthesized a WorkRefusedByInfrastructure for the caller, so the caller and the next attacher read different contracts -- the same fusion as before, in the opposite direction. The two facts are now returned as two. ProvideResult carries the stored outcome UNCHANGED plus a HostRecordStatus beside it, so re-requesting an identity yields exactly the ending the first caller was told. The CLI turns a lost record into a nonzero exit naming both halves, which is how the recording failure stays visible without becoming a verdict about work that really ran. review 69663: seat_release_landed had no consumer anywhere in the tree -- compute_release_on matches SeatRelease structurally and the provider uses its own compute_release_landed. Deleted rather than given a consumer, because the carrier already answers the question by match (DESIGN 3c). Controls, all at the observation-to-decision boundary: a completed wait with populated=1 keeps the charge; populated=0 without completion keeps the charge; a pending start establishes nothing; a failed query whose text reads like an answer is unavailable; and a lost host record changes no contract the next caller reads. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 227 ++++++++++++------ dag/gunbc/fabric/fabric_event_log.dag | 9 - .../compute/work_request_witness_test.dag | 79 ++++-- 3 files changed, 218 insertions(+), 97 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 3fcd3b6b23f..61bc72840c5 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -152,34 +152,72 @@ fn compute_ensure_dir(instance: HostDashboardInstance, path: String) -> Bool { } // What a provide returns: a fresh run's outcome, or an attachment to a stored one (no run). +// THE WORK'S RESULT AND THIS HOST'S RECORD ARE TWO FACTS AND ARE RETURNED AS TWO. +// +// They were fused twice, in opposite directions, and both were wrong. First a successful run whose +// release failed handed the caller a REFUSAL while outcome.json said it succeeded -- so the caller +// and the next attacher read different contracts. Then the repair for a lost host record did the +// same thing again: it published the real outcome and synthesized a WorkRefusedByInfrastructure for +// the caller (side-chat review at 773825cf). The stored document is what a later caller attaches +// to, so a returned outcome that differs from it is a disagreement no wording can fix. +// +// So the outcome carried here is ALWAYS the outcome that was stored, and whether this host managed +// to record its own side facts rides beside it. Both consumers get the same contract about the +// work; only this caller learns that the host lost its record, which is the only consumer that +// could act on it anyway. +type HostRecordStatus + = HostRecordWritten + | HostRecordLost { detail: String } + type ProvideResult - = ProvideFresh { outcome: WorkOutcome, document: String } + = ProvideFresh { outcome: WorkOutcome, document: String, host_record: HostRecordStatus } | ProvideAttached { identity: String, ending: String, document: String } +fn provide_result_host_record(r: ProvideResult) -> HostRecordStatus { + match r { + ProvideFresh { outcome: _, document: _, host_record: h } => h + ProvideAttached { identity: _, ending: _, document: _ } => HostRecordWritten + } +} + +fn host_record_wire(h: HostRecordStatus) -> String { + match h { + HostRecordWritten => "" + HostRecordLost { detail: d } => join(["; HOST RECORD LOST: ", d], "") + } +} + +fn host_record_was_written(h: HostRecordStatus) -> Bool { + match h { + HostRecordWritten => true + HostRecordLost { detail: _ } => false + } +} + fn provide_result_succeeded(r: ProvideResult) -> Bool { match r { - ProvideFresh { outcome, document: _ } => work_outcome_ending(o: outcome) == "succeeded" + ProvideFresh { outcome, document: _, host_record: _ } => work_outcome_ending(o: outcome) == "succeeded" ProvideAttached { identity: _, ending, document: _ } => ending == "succeeded" } } fn provide_result_identity(r: ProvideResult) -> String { match r { - ProvideFresh { outcome, document: _ } => gunbc.compute.work_request.work_outcome_identity(o: outcome) + ProvideFresh { outcome, document: _, host_record: _ } => gunbc.compute.work_request.work_outcome_identity(o: outcome) ProvideAttached { identity, ending: _, document: _ } => identity } } fn provide_result_ending(r: ProvideResult) -> String { match r { - ProvideFresh { outcome, document: _ } => work_outcome_ending(o: outcome) + ProvideFresh { outcome, document: _, host_record: _ } => work_outcome_ending(o: outcome) ProvideAttached { identity: _, ending, document: _ } => ending } } fn provide_result_document(r: ProvideResult) -> String { match r { - ProvideFresh { outcome: _, document } => document + ProvideFresh { outcome: _, document, host_record: _ } => document ProvideAttached { identity: _, ending: _, document } => document } } @@ -212,6 +250,20 @@ fn compute_unit_name(identity_hex: String) -> NonEmptyStr { join(["gunbc-compute-", identity_hex], "") as NonEmptyStr } +// THE TERMINATION POLICY IS BOUND, NOT INHERITED, and that is what makes a successful wait mean +// anything about the cgroup. `systemd-run --wait` returns when the MAIN process exits; under +// KillMode=process or none a forked child survives in the unit's cgroup and goes on using the +// memory this seat is accounting for. control-group is systemd's default, but a default is not a +// declaration -- a drop-in or a future default could change it, and this provider would keep +// treating a completed wait as an empty cgroup (side-chat review at 773825cf). Binding it makes the +// policy a fact of the invocation. +// +// IT IS BOUND *AND* THE EVIDENCE IS STILL REQUIRED. A positive populated=1 is never discarded in +// favour of a completed wait, because a bound policy says what systemd will do and the cgroup says +// what is true. +data compute_kill_mode_property: String = "KillMode" +data compute_kill_mode_value: String = "control-group" + // Run the operation as a transient user unit and wait for it: the unit carries THE MEMORY THE POOL // GRANTED -- not a ceiling read independently from the instance, which would let the reservation and // the enforcement drift apart -- the checkout as working directory, the shared per-host target @@ -228,6 +280,7 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden unit: compute_unit_name(identity_hex: identity_hex), properties: [ systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: kibibyte_to_byte_size(k: granted)))), + systemd_run_property(name: compute_kill_mode_property, value: compute_kill_mode_value), systemd_run_property(name: "WorkingDirectory", value: layout.checkout), ], setenv_bindings: [ @@ -283,7 +336,6 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden type UnitTermination = UnitAttemptCompleted { detail: String } | UnitAbsentAuthoritatively { detail: String } - | UnitTerminatedObserved { detail: String } | UnitPopulated { detail: String } | UnitTerminationUnavailable { detail: String } @@ -314,39 +366,71 @@ type UnitObservations { // A FAILED QUERY IS UNAVAILABLE WHATEVER IT PRINTED. systemctl writes diagnostics that read like // answers -- "Failed to connect to bus", "not-found" inside a sentence -- so the exit status // decides first and the text is never consulted on a failed query. -fn unit_termination_decide(o: UnitObservations) -> UnitTermination { - if o.attempt_completed { - UnitAttemptCompleted { detail: "systemd-run --wait returned success, so the unit it started ran to completion" } - } else if !o.load_state.queried { - UnitTerminationUnavailable { detail: "the user manager could not be queried for this unit, so nothing about it is established" } - } else if trim(s: o.load_state.value) == unit_load_state_absent_wire { - UnitAbsentAuthoritatively { detail: join(["LoadState=", unit_load_state_absent_wire], "") } - } else if trim(s: o.load_state.value) == "" { - UnitTerminationUnavailable { detail: "the manager answered the LoadState query with nothing, which establishes neither presence nor absence" } - } else if !o.control_group.queried { - UnitTerminationUnavailable { detail: join(["LoadState=", trim(s: o.load_state.value), " but the control group could not be queried"], "") } +// WHAT THE POPULATION READING ESTABLISHES, AS ITS OWN THREE-VALUED FACT. It has to be separable +// from the decision because the decision now consults it TWICE -- once to refuse a release outright, +// and once to confirm one -- and "not positively populated" is not the same as "positively empty". +type UnitPopulation + = PopulationOccupied { detail: String } + | PopulationEmpty { detail: String } + | PopulationUnknown { detail: String } + +fn unit_population(o: UnitObservations) -> UnitPopulation { + if !o.control_group.queried { + PopulationUnknown { detail: "the control group could not be queried" } } else if trim(s: o.control_group.value) == "" { - UnitTerminationUnavailable { - detail: join([ - "LoadState=", trim(s: o.load_state.value), - " and the unit holds no control group, which is EITHER a unit that has finished OR a start that is still pending -- the cgroup is realized at spawn -- so it establishes neither", - ], ""), - } + PopulationUnknown { detail: "the unit holds no control group, which is EITHER a unit that has finished OR a start that is still pending -- the cgroup is realized at spawn" } } else if !o.events.queried { - UnitTerminationUnavailable { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be read"], "") } + PopulationUnknown { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be read"], "") } } else { match decode_cgroup_events_populated(body: o.events.value) { - CgroupEventsUnparseable { cause: c } => UnitTerminationUnavailable { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be decoded: ", c], "") } + CgroupEventsUnparseable { cause: c } => PopulationUnknown { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be decoded: ", c], "") } CgroupEventsDecoded { populated: p } => if p { - UnitPopulated { detail: join(["processes remain in ", trim(s: o.control_group.value)], "") } + PopulationOccupied { detail: join(["processes remain in ", trim(s: o.control_group.value)], "") } } else { - UnitTerminatedObserved { detail: join(["the realized cgroup ", trim(s: o.control_group.value), " reports populated 0, so it spawned and everything in it exited"], "") } + PopulationEmpty { detail: join([trim(s: o.control_group.value), " reports populated 0"], "") } } } } } +// THE ORDER OF THESE TESTS IS THE SAFETY PROPERTY, and the previous order was the defect. Completion +// was checked FIRST and returned immediately, so {completed, populated 1} RELEASED -- a completed +// wait overriding the kernel telling us processes are still there. Contrary evidence now wins: +// a positive population refuses the release before anything else is considered, and is never +// discarded (side-chat review at 773825cf). +// +// AND A MOMENTARILY EMPTY CGROUP IS NOT TERMINAL ON ITS OWN. populated=0 with completion NOT +// established used to release; a realized cgroup can be empty between execs, and emptiness does not +// cover a pending job. So in this PR an uncertain completion KEEPS THE CHARGE, and the population +// reading serves only to CONFIRM a completion or to REFUSE one. Establishing termination without a +// completed attempt is the lifecycle capability's job, not this one's. +fn unit_termination_decide(o: UnitObservations) -> UnitTermination { + let population = unit_population(o: o) + match population { + PopulationOccupied { detail: d } => UnitPopulated { detail: d } + _ => + if o.load_state.queried && trim(s: o.load_state.value) == unit_load_state_absent_wire { + UnitAbsentAuthoritatively { detail: join(["LoadState=", unit_load_state_absent_wire, ", so the manager has no record of this unit"], "") } + } else if !o.attempt_completed { + UnitTerminationUnavailable { + detail: join([ + "the attempt did not complete, so its end is not established -- ", + match population { PopulationEmpty { detail: pd } => join([pd, ", but a realized cgroup can be empty between execs and emptiness does not cover a pending job"], "") PopulationUnknown { detail: pd } => pd PopulationOccupied { detail: pd } => pd }, + ], ""), + } + } else { + UnitAttemptCompleted { + detail: join([ + "systemd-run --wait returned success under ", compute_kill_mode_property, "=", compute_kill_mode_value, + ", so the unit ran to completion and its cgroup was torn down; ", + match population { PopulationEmpty { detail: pd } => pd PopulationUnknown { detail: pd } => join(["population unread (", pd, ") and not positively occupied"], "") PopulationOccupied { detail: pd } => pd }, + ], ""), + } + } + } +} + // THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is // system-scope; these are transient --user units, so asking the system manager would answer // not-found for EVERY one of them -- a fabricated authoritative absence, the worst answer this fold @@ -376,7 +460,6 @@ fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { match t { UnitAttemptCompleted { detail: _ } => true UnitAbsentAuthoritatively { detail: _ } => true - UnitTerminatedObserved { detail: _ } => true UnitPopulated { detail: _ } => false UnitTerminationUnavailable { detail: _ } => false } @@ -386,7 +469,6 @@ fn unit_termination_wire(t: UnitTermination) -> String { match t { UnitAttemptCompleted { detail: d } => join(["attempt completed (", d, ")"], "") UnitAbsentAuthoritatively { detail: d } => join(["absent (", d, ")"], "") - UnitTerminatedObserved { detail: d } => join(["terminated (", d, ")"], "") UnitPopulated { detail: d } => join(["STILL POPULATED (", d, "), so the seat stays charged"], "") UnitTerminationUnavailable { detail: d } => join(["termination UNESTABLISHED (", d, "), so the seat stays charged"], "") } @@ -513,57 +595,48 @@ fn compute_run_and_settle( if measured.success { settled } else { - compute_host_record_lost( - layout: layout, identity: identity, subject: subject, op: op, settled: settled, + compute_with_lost_host_record( + settled: settled, layout: layout, dropped_ok: dropped.success, dropped_error: dropped.error, error: measured.error) } } -// THE HOST RECORD IS THE ONLY DURABLE CARRIER OF THREE FACTS, so losing it stops the line rather -// than being discarded. The arm used to read `if measured.success { settled } else { settled }` -- -// both arms identical, so a failed write vanished by construction (review 69645), and the -// COMPOUND case was the sharp one: the producer lease failing to drop AND this write failing left -// an identity whose lease is permanently held with NO RECORD ANYWHERE that it is stale, while the -// caller was handed an ordinary success. The module's own prose claimed that stale lease "is -// recorded as the operational fact it is"; the arm let that recording disappear. +// THE HOST RECORD IS THE ONLY DURABLE CARRIER OF THREE FACTS, so losing it is reported rather than +// discarded -- and reported BESIDE the work's result, not instead of it. // -// WHY THIS DOES NOT REOPEN BLOCKER (7). That blocker was a refusal that DENIED a published -// success -- the caller was told the run failed while the stored record said it succeeded and the -// next caller would attach to that success. This refusal denies nothing: it names the run's real -// ending, says the stored record is published and authoritative, and reports a HOST fault that -// happened after publication. The two callers still read the same thing about the work; what this -// one additionally learns is that the host has state no file describes. -fn compute_host_record_lost( - layout: ComputeLayout, - identity: String, - subject: WorkSubject, - op: WorkOperation, +// TWO DEFECTS MET HERE. The arm read `if measured.success { settled } else { settled }`, so a failed +// write vanished by construction (review 69645); and the compound case was the sharp one, because +// the stale-lease note lives only in this file -- a lease that failed to drop AND a write that +// failed left an identity whose lease is permanently held with NO RECORD ANYWHERE. The repair then +// over-corrected: it synthesized a WorkRefusedByInfrastructure for the caller while outcome.json +// said the run had succeeded, which is the stored-record disagreement again, one layer along +// (side-chat review at 773825cf). +// +// So the outcome is untouched -- the caller reads exactly what the next attacher will read -- and +// the lost record rides as its own fact. The CLI turns that fact into a nonzero exit with both +// halves named, which is how a recording failure stays visible without becoming a verdict about +// work that really did run. +fn compute_with_lost_host_record( settled: ProvideResult, + layout: ComputeLayout, dropped_ok: Bool, dropped_error: String, error: String, ) -> ProvideResult { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: StoreUnavailable { - detail: join([ - "the work's own record is published at ", layout.outcome_path, " and is authoritative -- it ended ", - provide_result_ending(r: settled), - " and the next caller reads exactly that -- but this host's record at ", layout.consumption_path, - " could not be written: ", error, - "; so the consumption reading is lost, and ", - if dropped_ok { - "the producer lease was dropped, so nothing else is outstanding" - } else { - join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be requested again"], "") - }, - ], ""), - }, + let detail = join([ + "this host's record at ", layout.consumption_path, " could not be written: ", error, + "; the consumption reading is lost, and ", + if dropped_ok { + "the producer lease was dropped, so nothing else is outstanding" + } else { + join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be produced again"], "") }, - subject: subject, op: op) + ], "") + match settled { + ProvideFresh { outcome: o, document: d, host_record: _ } => ProvideFresh { outcome: o, document: d, host_record: HostRecordLost { detail: detail } } + ProvideAttached { identity: i, ending: e, document: d } => ProvideAttached { identity: i, ending: e, document: d } + } } - // BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the // caller a refusal while the outcome document already said the run had SUCCEEDED, and then state // that the next request would refuse as ProducerInFlight -- three claims that cannot all be true, @@ -598,7 +671,7 @@ fn compute_settled_record( let document = work_outcome_wire(o: outcome, subject: subject, op: op) let written = Filesystem.Write(path: layout.outcome_path, content: document) if written.success { - ProvideFresh { outcome: outcome, document: document } + ProvideFresh { outcome: outcome, document: document, host_record: HostRecordWritten } } else { compute_fresh( outcome: WorkRefusedByInfrastructure { @@ -639,7 +712,7 @@ fn compute_unreleased_record( let document = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) let written = Filesystem.Write(path: layout.outcome_path, content: document) if written.success { - ProvideFresh { outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, document: document } + ProvideFresh { outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, document: document, host_record: HostRecordWritten } } else { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) } @@ -781,7 +854,7 @@ fn compute_release_wire(r: ComputeRelease) -> String { } fn compute_fresh(outcome: WorkOutcome, subject: WorkSubject, op: WorkOperation) -> ProvideResult { - ProvideFresh { outcome: outcome, document: work_outcome_wire(o: outcome, subject: subject, op: op) } + ProvideFresh { outcome: outcome, document: work_outcome_wire(o: outcome, subject: subject, op: op), host_record: HostRecordWritten } } fn compute_run_leased( @@ -880,9 +953,19 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent Absent => ExitFailure { code: 2, reason: join(["compute: operation must be build, compile or claims (compile and claims need entry; claims needs functions and hermetic=true|false); got ", operation], "") } Present { value: op } => { let result = compute_provide(instance: instance, commit: commit, op: op) - let receipt = Filesystem.Write(path: receipt_path, content: provide_result_document(r: result)) + let record = provide_result_host_record(r: result) + let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), host_record_wire(h: record), "\n"], "")) if !receipt.success { ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; outcome was ", provide_result_document(r: result)], "") } + } else if !host_record_was_written(h: record) { + ExitFailure { + code: 1, + reason: join([ + "compute: the work ended ", provide_result_ending(r: result), + " and its record at the store is authoritative -- a later request for this identity reads exactly that -- but ", + host_record_wire(h: record), + ], ""), + } } else if provide_result_succeeded(r: result) { ExitSuccess } else { diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index 884a065cb3f..5b109c419f3 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -503,15 +503,6 @@ fn seat_release_wire(r: SeatRelease) -> String { } } -fn seat_release_landed(r: SeatRelease) -> Bool { - match r { - SeatReleased { reference: _, id: _ } => true - SeatNotHeld { reference: _, wire: _ } => false - SeatReleaseContended { attempts: _ } => false - SeatReleaseRefused { step: _, reason: _ } => false - } -} - fn seat_acquisition_wire(a: SeatAcquisition) -> String { match a { SeatGranted { grant: g, generation: gen, attempts_used: n } => join(["granted ", g.reference as String, " fence=", g.fence.grant as String, "@", to_string(gen), " expires_at=", to_string(g.expires_at), " attempt=", to_string(n)], "") diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 93878ef609b..8392c547723 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -3,10 +3,14 @@ module test.claim.compute.work_request_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } import gunbc.compute.work_provider_local { - UnitTermination, UnitAttemptCompleted, UnitAbsentAuthoritatively, UnitTerminatedObserved, UnitPopulated, UnitTerminationUnavailable, + UnitTermination, UnitAttemptCompleted, UnitAbsentAuthoritatively, UnitPopulated, UnitTerminationUnavailable, UnitPropertyReading, UnitObservations, unit_termination_decide, unit_termination_frees_capacity, unit_termination_wire, compute_attempt_completed, + ProvideResult, ProvideFresh, ProvideAttached, HostRecordStatus, HostRecordWritten, HostRecordLost, + provide_result_host_record, provide_result_ending, provide_result_document, host_record_was_written, host_record_wire, + compute_with_lost_host_record, ComputeLayout, compute_layout, } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, @@ -85,34 +89,56 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool // looks like an answer, a property that came back EMPTY, and a start that is still PENDING all // arrive as readings and are wrong only in how they are read (side-chat review at 5ee0cce). // +// A COMPLETED WAIT NEVER OVERRIDES A POSITIVE POPULATION, which is the row the previous revision +// got backwards: completion was tested FIRST and returned immediately, so {completed, populated 1} +// RELEASED -- a launcher's success overriding the kernel saying processes are still there. Under +// KillMode=process or none a forked child outlives the main process exactly like that. Contrary +// evidence wins now, and a positive populated=1 is never discarded. +// +// AND A MOMENTARILY EMPTY CGROUP IS NOT TERMINAL ON ITS OWN: populated=0 WITHOUT an established +// completion used to release, and a realized cgroup can be empty between execs. Uncertain +// completion keeps the charge. +// // THE PENDING-START ROW IS THE ONE THIS CELL EXISTS FOR. An empty ControlGroup was read as observed // termination, and it equally describes a unit whose start has not spawned yet -- systemd realizes // the cgroup AT SPAWN. That made a whole unbacked launch reachable: reserve, start pending, the // LAUNCHER lost while this driver lives on, settlement sees the empty property, releases, and the // job then starts against memory nobody holds. It must establish nothing. +fn reading(queried: Bool, value: String) -> UnitPropertyReading { + UnitPropertyReading { queried: queried, value: value } +} + +fn decided(done: Bool, load: UnitPropertyReading, cg: UnitPropertyReading, ev: UnitPropertyReading) -> UnitTermination { + unit_termination_decide(o: UnitObservations { attempt_completed: done, load_state: load, control_group: cg, events: ev }) +} + test fn only_proven_absence_or_an_ended_cgroup_frees_a_seat() -> Bool { - let no_events = UnitPropertyReading { queried: false, value: "" } - let completed = unit_termination_decide(o: UnitObservations { attempt_completed: true, load_state: UnitPropertyReading { queried: false, value: "" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) - let absent = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "not-found" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) - let misleading = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: false, value: "Failed to connect to bus: not-found" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) - let empty_load = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 0\n" } }) - let pending = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "" }, events: no_events }) - let populated = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 1\n" } }) - let failed_unit = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "failed" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 1\n" } }) - let ended = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 0\n" } }) - let undecodable = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "garbage\n" } }) - unit_termination_frees_capacity(t: completed) + let no_events = reading(queried: false, value: "") + let unread = reading(queried: false, value: "") + let completed_empty = decided(done: true, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 0\n")) + let completed_populated = decided(done: true, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 1\n")) + let completed_unknown = decided(done: true, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: ""), ev: no_events) + let absent = decided(done: false, load: reading(queried: true, value: "not-found"), cg: unread, ev: no_events) + let misleading = decided(done: false, load: reading(queried: false, value: "Failed to connect to bus: not-found"), cg: unread, ev: no_events) + let empty_load = decided(done: false, load: reading(queried: true, value: ""), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 0\n")) + let pending = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: ""), ev: no_events) + let empty_no_completion = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 0\n")) + let populated_no_completion = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 1\n")) + let undecodable = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "garbage\n")) + unit_termination_frees_capacity(t: completed_empty) + && unit_termination_frees_capacity(t: completed_unknown) && unit_termination_frees_capacity(t: absent) - && unit_termination_frees_capacity(t: ended) + && !unit_termination_frees_capacity(t: completed_populated) + && !unit_termination_frees_capacity(t: empty_no_completion) + && !unit_termination_frees_capacity(t: populated_no_completion) && !unit_termination_frees_capacity(t: misleading) && !unit_termination_frees_capacity(t: empty_load) && !unit_termination_frees_capacity(t: pending) - && !unit_termination_frees_capacity(t: populated) - && !unit_termination_frees_capacity(t: failed_unit) && !unit_termination_frees_capacity(t: undecodable) + && string_contains(s: unit_termination_wire(t: completed_populated), pattern: "STILL POPULATED") + && string_contains(s: unit_termination_wire(t: empty_no_completion), pattern: "did not complete") && string_contains(s: unit_termination_wire(t: pending), pattern: "pending") && string_contains(s: unit_termination_wire(t: misleading), pattern: "UNESTABLISHED") - && string_contains(s: unit_termination_wire(t: populated), pattern: "STILL POPULATED") } // A COMPLETED ATTEMPT RELEASES, AND AN ATTEMPT THAT DID NOT COMPLETE DOES NOT RELEASE ON ITS OWN. @@ -126,3 +152,24 @@ test fn the_attempt_completion_fact_is_not_the_works_ending() -> Bool { && !compute_attempt_completed(outcome: WorkCancelled { identity: "a" }) && !compute_attempt_completed(outcome: WorkRefusedByInfrastructure { identity: "a", cause: HostComputeCapacityFull { wire: "pool-full" } }) } + +// THE STORED RECORD AND THE RETURNED OUTCOME AGREE, WHATEVER HAPPENED TO THIS HOST'S OWN RECORD. +// The fusion was made twice in opposite directions -- a refusal handed to the caller over a stored +// success, then a synthesized refusal over a published outcome -- and both times the next attacher +// read something different from the caller. Losing the host record now changes the host_record +// fact and NOTHING about the work, so re-requesting the identity yields the same ending the first +// caller was told; what the first caller additionally gets is the recording failure, which is the +// only consumer that could act on it. +test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { + let succeeded = WorkSucceeded { identity: "abc", outputs: [], log_path: "/l" } + let document = work_outcome_wire(o: succeeded, subject: tree(hex: tree_a), op: work_default_build()) + let settled = ProvideFresh { outcome: succeeded, document: document, host_record: HostRecordWritten } + let layout = compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc") + let lost = compute_with_lost_host_record(settled: settled, layout: layout, dropped_ok: false, dropped_error: "EBUSY", error: "ENOSPC") + provide_result_ending(r: lost) == provide_result_ending(r: settled) + && provide_result_document(r: lost) == provide_result_document(r: settled) + && provide_result_ending(r: lost) == "succeeded" + && host_record_was_written(h: provide_result_host_record(r: settled)) + && !host_record_was_written(h: provide_result_host_record(r: lost)) + && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost)), pattern: "STILL HELD") +} From 1248df9502e3d37d1896aaba87df1b8522b8f32b Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 19:56:54 +0000 Subject: [PATCH 10/44] The bound termination policy is a fold a control can read, not a sentence A completed wait may confirm a release only because KillMode=control-group means systemd tears the cgroup down with the unit. Spelled inline at the call, that was an assertion nothing checked; the properties are a fold now, and the_unit_binds_the_termination_policy_the_release_decision_relies_on is what makes "bound" a fact. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 18 ++++++++++++----- .../compute/work_request_witness_test.dag | 20 ++++++++++++++++++- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 61bc72840c5..20abb8159b2 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -264,6 +264,18 @@ fn compute_unit_name(identity_hex: String) -> NonEmptyStr { data compute_kill_mode_property: String = "KillMode" data compute_kill_mode_value: String = "control-group" +// THE PROPERTIES ARE A FOLD SO THE BOUND POLICY IS CHECKABLE. A termination policy that is only +// asserted in prose is exactly the thing the release decision must not trust, and the whole point +// of binding it is that the decision leans on it -- so the invocation's properties are produced +// where a control can read them rather than spelled inline at the call. +fn compute_unit_properties(layout: ComputeLayout, granted: Kibibyte) -> List { + [ + systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: kibibyte_to_byte_size(k: granted)))), + systemd_run_property(name: compute_kill_mode_property, value: compute_kill_mode_value), + systemd_run_property(name: "WorkingDirectory", value: layout.checkout), + ] +} + // Run the operation as a transient user unit and wait for it: the unit carries THE MEMORY THE POOL // GRANTED -- not a ceiling read independently from the instance, which would let the reservation and // the enforcement drift apart -- the checkout as working directory, the shared per-host target @@ -278,11 +290,7 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden workdir: layout.checkout, args: systemd_run_user_wait_arguments( unit: compute_unit_name(identity_hex: identity_hex), - properties: [ - systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: kibibyte_to_byte_size(k: granted)))), - systemd_run_property(name: compute_kill_mode_property, value: compute_kill_mode_value), - systemd_run_property(name: "WorkingDirectory", value: layout.checkout), - ], + properties: compute_unit_properties(layout: layout, granted: granted), setenv_bindings: [ join(["CARGO_TARGET_DIR=", layout.target_dir], ""), join(["RUSTC_WRAPPER=", sccache_installed_binary_path], ""), diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 8392c547723..4704bc5ac96 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -8,8 +8,10 @@ import gunbc.compute.work_provider_local { unit_termination_frees_capacity, unit_termination_wire, compute_attempt_completed, ProvideResult, ProvideFresh, ProvideAttached, HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, provide_result_ending, provide_result_document, host_record_was_written, host_record_wire, - compute_with_lost_host_record, ComputeLayout, compute_layout, + compute_with_lost_host_record, ComputeLayout, compute_layout, compute_unit_properties, + compute_kill_mode_property, compute_kill_mode_value, } +import std.measure { kibibyte } import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, @@ -173,3 +175,19 @@ test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { && !host_record_was_written(h: provide_result_host_record(r: lost)) && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost)), pattern: "STILL HELD") } + +// THE TERMINATION POLICY THE RELEASE DECISION LEANS ON IS ACTUALLY BOUND ON THE INVOCATION. A +// completed wait is allowed to confirm a release only because KillMode=control-group means systemd +// tears the cgroup down with the unit; under process or none a forked child outlives the main +// process and goes on using the memory this seat accounts for. systemd's DEFAULT is +// control-group -- but a default is not a declaration, and a drop-in or a future default could +// change it while this provider kept reasoning from the old one. So it is bound, and this cell is +// what makes "bound" a fact rather than a sentence. +test fn the_unit_binds_the_termination_policy_the_release_decision_relies_on() -> Bool { + let props = compute_unit_properties( + layout: compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc"), + granted: kibibyte(count: 1048576)) + contains(props, join([compute_kill_mode_property, "=", compute_kill_mode_value], "")) + && compute_kill_mode_value == "control-group" + && count(filter(props, p => starts_with(s: p, prefix: "MemoryMax="))) == 1 +} From c901fc008c3dd97f27c8128b23bd9a1787cb7424 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 20:25:38 +0000 Subject: [PATCH 11/44] The release predicate is exactly four cases; a lost record reaches a terminal consumer; the KillMode control claims only what it checks Side-chat verdict at f5344b28. All three verified first. (1) THE PREDICATE READ "not occupied AND (absent OR completed)", so completed + UNKNOWN released -- and the witness ASSERTED that, which is the worse half: a control that pins the defect is not a control. It is now exactly: occupied keeps; authoritative absence releases; completed AND POSITIVELY EMPTY releases; everything else keeps. A bound KillMode says what systemd WILL do and is not a reading of what is true, so an unreadable cgroup establishes nothing beside a completed wait. The witness is flipped. The KillMode control now claims only what it checks: it reads the argv this provider CONSTRUCTS -- invocation wiring -- and does NOT establish what systemd applied. A drop-in, a rejected property, or a unit started by some other path are outside what any fold over our own argv can see, which is exactly why the release predicate does not lean on the binding. (2) HostRecordLost WAS ERASED BY THREE CONSUMERS and is carried through all of them now. compute_provide_dependent returned only the child's result, so a BUILD that lost its record followed by a successful dependent reported clean -- the status is joined onto the dependent's result, accumulating rather than letting the later arm win, because the two details name different identities. ProvideAttached carries the status too, so an attach cannot drop a carried loss. compile_entry_cli exits nonzero naming the loss, and run_selected_module carries it onto ModuleRan, into the emitted JSON, and into the module's verdict. None of this changes the stored work outcome: the ending and the document reported are exactly what was stored, and only whether the run is called clean moves. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/test_run.dag | 22 +++-- dag/gunbc/compute/work_provider_local.dag | 83 ++++++++++++++----- .../compute/work_request_witness_test.dag | 55 ++++++++++-- 3 files changed, 127 insertions(+), 33 deletions(-) diff --git a/dag/gunbc/compute/test_run.dag b/dag/gunbc/compute/test_run.dag index 3711f1b1f6e..0a338b0a4ce 100644 --- a/dag/gunbc/compute/test_run.dag +++ b/dag/gunbc/compute/test_run.dag @@ -19,6 +19,7 @@ import gunbc.roadmap_execution_contract { GitWorkspaceCapability } import gunbc.compute.work_request { WorkOperation, RunClaims, CompileEntry } import gunbc.compute.work_provider_local { ProvideResult, ProvideFresh, ProvideAttached, compute_provide, provide_result_succeeded, provide_result_ending, provide_result_identity, provide_result_document, + HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, host_record_wire, host_record_was_written, } import gunbc.compute.test_selection { SelectedFile, select_witness_files, classify_witness_module, planned_functions_joined, @@ -86,7 +87,7 @@ fn show_file(instance: HostDashboardInstance, worktree: String, commit: String, // One selected module's result in the report. type ModuleRun - = ModuleRan { label: String, module_path: String, planned: List, declined: List, ending: String, identity: String, document: String } + = ModuleRan { label: String, module_path: String, planned: List, declined: List, ending: String, identity: String, document: String, host_record: String } | ModuleAllDeclined { label: String, module_path: String, declined: List } | ModuleUnreadable { label: String, path: String, reason: String } | ModuleNotAWitness { label: String, path: String, reason: String } @@ -105,15 +106,22 @@ fn run_selected_module(instance: HostDashboardInstance, worktree: String, commit ModuleRan { label: render_label(l: label), module_path: module_path, planned: planned, declined: declined, ending: provide_result_ending(r: result), identity: provide_result_identity(r: result), document: provide_result_document(r: result), + host_record: host_record_wire(h: provide_result_host_record(r: result)), } } } } } +// A HOST-RECORD LOSS COUNTS AGAINST THE MODULE'S VERDICT, and that is the point of carrying it this +// far. The worker CLIs are where a compute result stops being a value and becomes an exit code, so a +// loss that reaches here and is not counted is a loss nobody ever hears about: the dependent request +// dropped it at one seam (compute_provide_dependent) and these two consumers dropped it at the next +// (side-chat review at f5344b28). The work's own ending is still reported verbatim in the receipt -- +// what changes is only whether the run is called clean. fn module_run_succeeded(r: ModuleRun) -> Bool { match r { - ModuleRan { label: _, module_path: _, planned: _, declined: _, ending, identity: _, document: _ } => ending == "succeeded" + ModuleRan { label: _, module_path: _, planned: _, declined: _, ending, identity: _, document: _, host_record: record } => ending == "succeeded" && record == "" ModuleAllDeclined { label: _, module_path: _, declined: _ } => true ModuleUnreadable { label: _, path: _, reason: _ } => false ModuleNotAWitness { label: _, path: _, reason: _ } => true @@ -129,7 +137,7 @@ fn standing_json(s: WitnessStanding) -> JsonValue { fn module_run_json(r: ModuleRun) -> JsonValue { match r { - ModuleRan { label, module_path, planned, declined, ending, identity, document: _ } => + ModuleRan { label, module_path, planned, declined, ending, identity, document: _, host_record: record } => json_object(members: [ json_kv(key: "label", value: json_string(s: label)), json_kv(key: "module", value: json_string(s: module_path)), @@ -137,6 +145,7 @@ fn module_run_json(r: ModuleRun) -> JsonValue { json_kv(key: "declined", value: json_array(elements: map(declined, standing_json))), json_kv(key: "ending", value: json_string(s: ending)), json_kv(key: "identity", value: json_string(s: identity)), + json_kv(key: "host_record", value: json_string(s: record)), ]) ModuleAllDeclined { label, module_path, declined } => json_object(members: [ @@ -164,7 +173,7 @@ fn test_run_report_json(r: TestRunReport) -> JsonValue { json_kv(key: "pattern", value: json_string(s: pattern)), json_kv(key: "snapshot_commit", value: json_string(s: commit)), json_kv(key: "selected", value: json_int(n: count(modules))), - json_kv(key: "succeeded", value: json_int(n: count(filter(modules, m => match m { ModuleRan { label: _, module_path: _, planned: _, declined: _, ending, identity: _, document: _ } => ending == "succeeded" _ => false })))), + json_kv(key: "succeeded", value: json_int(n: count(filter(modules, m => match m { ModuleRan { label: _, module_path: _, planned: _, declined: _, ending, identity: _, document: _, host_record: record } => ending == "succeeded" && record == "" _ => false })))), json_kv(key: "not_succeeded", value: json_int(n: count(filter(modules, m => !module_run_succeeded(r: m))))), json_kv(key: "modules", value: json_array(elements: map(modules, module_run_json))), ]) @@ -264,9 +273,12 @@ fn compile_entry_cli(repo_root: String, worktree: String, entry: String, receipt SnapshotRefused { step, detail } => ExitFailure { code: 2, reason: join(["gunbc compile: snapshot refused at ", step, ": ", detail], "") } SnapshotResolved { commit } => { let result = compute_provide(instance: instance, commit: commit, op: CompileEntry { entry: entry as FilePath }) - let receipt = Filesystem.Write(path: receipt_path, content: provide_result_document(r: result)) + let record = provide_result_host_record(r: result) + let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), host_record_wire(h: record)], "")) if !receipt.success { ExitFailure { code: 2, reason: join(["gunbc compile: receipt could not be written to ", receipt_path, ": ", receipt.error, "; outcome was ", provide_result_document(r: result)], "") } + } else if !host_record_was_written(h: record) { + ExitFailure { code: 1, reason: join(["gunbc compile: the work ended ", provide_result_ending(r: result), " and its stored record is authoritative, but ", host_record_wire(h: record)], "") } } else if provide_result_succeeded(r: result) { ExitSuccess } else { diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 20abb8159b2..f6a4070179d 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -171,12 +171,12 @@ type HostRecordStatus type ProvideResult = ProvideFresh { outcome: WorkOutcome, document: String, host_record: HostRecordStatus } - | ProvideAttached { identity: String, ending: String, document: String } + | ProvideAttached { identity: String, ending: String, document: String, host_record: HostRecordStatus } fn provide_result_host_record(r: ProvideResult) -> HostRecordStatus { match r { ProvideFresh { outcome: _, document: _, host_record: h } => h - ProvideAttached { identity: _, ending: _, document: _ } => HostRecordWritten + ProvideAttached { identity: _, ending: _, document: _, host_record: h } => h } } @@ -187,6 +187,29 @@ fn host_record_wire(h: HostRecordStatus) -> String { } } +// JOINING TWO HOST-RECORD FACTS: a loss anywhere in the chain is a loss the caller must hear about, +// so the arms accumulate rather than the later one winning. Both details are kept because they name +// different identities and an operator needs each one. +fn host_record_join(a: HostRecordStatus, b: HostRecordStatus) -> HostRecordStatus { + match a { + HostRecordWritten => b + HostRecordLost { detail: da } => + match b { + HostRecordWritten => HostRecordLost { detail: da } + HostRecordLost { detail: db } => HostRecordLost { detail: join([da, "; and ", db], "") } + } + } +} + +fn provide_result_with_host_record(r: ProvideResult, carried: HostRecordStatus) -> ProvideResult { + match r { + ProvideFresh { outcome: o, document: d, host_record: h } => + ProvideFresh { outcome: o, document: d, host_record: host_record_join(a: carried, b: h) } + ProvideAttached { identity: i, ending: e, document: d, host_record: h } => + ProvideAttached { identity: i, ending: e, document: d, host_record: host_record_join(a: carried, b: h) } + } +} + fn host_record_was_written(h: HostRecordStatus) -> Bool { match h { HostRecordWritten => true @@ -197,28 +220,28 @@ fn host_record_was_written(h: HostRecordStatus) -> Bool { fn provide_result_succeeded(r: ProvideResult) -> Bool { match r { ProvideFresh { outcome, document: _, host_record: _ } => work_outcome_ending(o: outcome) == "succeeded" - ProvideAttached { identity: _, ending, document: _ } => ending == "succeeded" + ProvideAttached { identity: _, ending, document: _, host_record: _ } => ending == "succeeded" } } fn provide_result_identity(r: ProvideResult) -> String { match r { ProvideFresh { outcome, document: _, host_record: _ } => gunbc.compute.work_request.work_outcome_identity(o: outcome) - ProvideAttached { identity, ending: _, document: _ } => identity + ProvideAttached { identity, ending: _, document: _, host_record: _ } => identity } } fn provide_result_ending(r: ProvideResult) -> String { match r { ProvideFresh { outcome, document: _, host_record: _ } => work_outcome_ending(o: outcome) - ProvideAttached { identity: _, ending, document: _ } => ending + ProvideAttached { identity: _, ending, document: _, host_record: _ } => ending } } fn provide_result_document(r: ProvideResult) -> String { match r { ProvideFresh { outcome: _, document, host_record: _ } => document - ProvideAttached { identity: _, ending: _, document } => document + ProvideAttached { identity: _, ending: _, document, host_record: _ } => document } } @@ -258,9 +281,12 @@ fn compute_unit_name(identity_hex: String) -> NonEmptyStr { // treating a completed wait as an empty cgroup (side-chat review at 773825cf). Binding it makes the // policy a fact of the invocation. // -// IT IS BOUND *AND* THE EVIDENCE IS STILL REQUIRED. A positive populated=1 is never discarded in -// favour of a completed wait, because a bound policy says what systemd will do and the cgroup says -// what is true. +// IT IS BOUND *AND* THE EVIDENCE IS STILL REQUIRED, IN BOTH DIRECTIONS. A positive populated=1 is +// never discarded in favour of a completed wait; and an UNREADABLE population is not licensed by one +// either. A bound policy says what systemd WILL do, the cgroup says what IS true, and "I could not +// look" is not the second (side-chat review at f5344b28). Completion confirms a release only beside +// a positively empty cgroup -- or, short-circuiting all of this, a manager that has no record of +// the unit at all. data compute_kill_mode_property: String = "KillMode" data compute_kill_mode_value: String = "control-group" @@ -428,12 +454,22 @@ fn unit_termination_decide(o: UnitObservations) -> UnitTermination { ], ""), } } else { - UnitAttemptCompleted { - detail: join([ - "systemd-run --wait returned success under ", compute_kill_mode_property, "=", compute_kill_mode_value, - ", so the unit ran to completion and its cgroup was torn down; ", - match population { PopulationEmpty { detail: pd } => pd PopulationUnknown { detail: pd } => join(["population unread (", pd, ") and not positively occupied"], "") PopulationOccupied { detail: pd } => pd }, - ], ""), + match population { + PopulationEmpty { detail: pd } => + UnitAttemptCompleted { + detail: join([ + "systemd-run --wait returned success under ", compute_kill_mode_property, "=", compute_kill_mode_value, + " and ", pd, + ], ""), + } + _ => + UnitTerminationUnavailable { + detail: join([ + "the attempt completed, but the population could not be read (", + match population { PopulationUnknown { detail: pd } => pd PopulationEmpty { detail: pd } => pd PopulationOccupied { detail: pd } => pd }, + ") -- a bound KillMode says what systemd WILL do, not what is true, so an unreadable cgroup establishes nothing even beside a completed wait", + ], ""), + } } } } @@ -642,7 +678,7 @@ fn compute_with_lost_host_record( ], "") match settled { ProvideFresh { outcome: o, document: d, host_record: _ } => ProvideFresh { outcome: o, document: d, host_record: HostRecordLost { detail: detail } } - ProvideAttached { identity: i, ending: e, document: d } => ProvideAttached { identity: i, ending: e, document: d } + ProvideAttached { identity: i, ending: e, document: d, host_record: h } => ProvideAttached { identity: i, ending: e, document: d, host_record: h } } } // BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the @@ -775,7 +811,7 @@ fn compute_provide_leaf( let attached = if stored.success { stored_outcome_decode(text: stored.content) } else { StoredOutcomeUnreadable { reason: "no stored outcome" } } if stored_outcome_is_success(r: attached) { match attached { - StoredOutcomeFound { ending, identity: stored_identity, document } => ProvideAttached { identity: stored_identity, ending: ending, document: document } + StoredOutcomeFound { ending, identity: stored_identity, document } => ProvideAttached { identity: stored_identity, ending: ending, document: document, host_record: HostRecordWritten } StoredOutcomeUnreadable { reason } => compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: reason } }, subject: subject, op: op) } } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { @@ -926,14 +962,23 @@ fn compute_provide(instance: HostDashboardInstance, commit: String, op: WorkOper } } +// THE DEPENDENCY'S LOST RECORD TRAVELS WITH THE RESULT, because the caller of a dependent request +// never sees the dependency's own result. Returning the second leaf's result alone silently dropped +// the fact that the BUILD failed to record its side facts -- the stuck lease or the lost consumption +// reading vanished at exactly the seam where nobody was left to report it (side-chat review at +// f5344b28). The work contract is untouched; only the host-record fact is joined. fn compute_provide_dependent(instance: HostDashboardInstance, commit: String, subject: WorkSubject, op: WorkOperation, toolchain: WorkToolchainIdentity) -> ProvideResult { let build = compute_provide_leaf(instance: instance, commit: commit, subject: subject, op: work_default_build(), toolchain: toolchain, dependency_store: "") if provide_result_succeeded(r: build) { let build_identity = provide_result_identity(r: build) - compute_provide_leaf(instance: instance, commit: commit, subject: subject, op: op, toolchain: toolchain, dependency_store: compute_layout(instance: instance, identity_hex: build_identity).store_dir) + provide_result_with_host_record( + r: compute_provide_leaf(instance: instance, commit: commit, subject: subject, op: op, toolchain: toolchain, dependency_store: compute_layout(instance: instance, identity_hex: build_identity).store_dir), + carried: provide_result_host_record(r: build)) } else { let identity = work_identity_hex(id: work_identity(subject: subject, op: op, toolchain: toolchain)) - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: DependencyNotSucceeded { dependency_identity: provide_result_identity(r: build), ending: provide_result_ending(r: build) } }, subject: subject, op: op) + provide_result_with_host_record( + r: compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: DependencyNotSucceeded { dependency_identity: provide_result_identity(r: build), ending: provide_result_ending(r: build) } }, subject: subject, op: op), + carried: provide_result_host_record(r: build)) } } diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 4704bc5ac96..4f396da6242 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -8,7 +8,8 @@ import gunbc.compute.work_provider_local { unit_termination_frees_capacity, unit_termination_wire, compute_attempt_completed, ProvideResult, ProvideFresh, ProvideAttached, HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, provide_result_ending, provide_result_document, host_record_was_written, host_record_wire, - compute_with_lost_host_record, ComputeLayout, compute_layout, compute_unit_properties, + compute_with_lost_host_record, provide_result_with_host_record, host_record_join, + ComputeLayout, compute_layout, compute_unit_properties, compute_kill_mode_property, compute_kill_mode_value, } import std.measure { kibibyte } @@ -91,6 +92,14 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool // looks like an answer, a property that came back EMPTY, and a start that is still PENDING all // arrive as readings and are wrong only in how they are read (side-chat review at 5ee0cce). // +// THE PREDICATE IS EXACTLY FOUR WORDS LONG AND EVERY OTHER READING KEEPS THE CHARGE: occupied +// keeps; authoritative absence releases; completed AND POSITIVELY EMPTY releases; everything else +// keeps. The revision before this one read "not occupied AND (absent OR completed)", so +// completed + UNKNOWN released -- and this cell ASSERTED that, which is the worse half: a control +// that pins the defect is not a control (side-chat verdict at f5344b28). A bound KillMode says what +// systemd WILL do; it is not a reading of what is true, and an unreadable cgroup establishes +// nothing beside a completed wait. +// // A COMPLETED WAIT NEVER OVERRIDES A POSITIVE POPULATION, which is the row the previous revision // got backwards: completion was tested FIRST and returned immediately, so {completed, populated 1} // RELEASED -- a launcher's success overriding the kernel saying processes are still there. Under @@ -128,8 +137,8 @@ test fn only_proven_absence_or_an_ended_cgroup_frees_a_seat() -> Bool { let populated_no_completion = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 1\n")) let undecodable = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "garbage\n")) unit_termination_frees_capacity(t: completed_empty) - && unit_termination_frees_capacity(t: completed_unknown) && unit_termination_frees_capacity(t: absent) + && !unit_termination_frees_capacity(t: completed_unknown) && !unit_termination_frees_capacity(t: completed_populated) && !unit_termination_frees_capacity(t: empty_no_completion) && !unit_termination_frees_capacity(t: populated_no_completion) @@ -138,6 +147,7 @@ test fn only_proven_absence_or_an_ended_cgroup_frees_a_seat() -> Bool { && !unit_termination_frees_capacity(t: pending) && !unit_termination_frees_capacity(t: undecodable) && string_contains(s: unit_termination_wire(t: completed_populated), pattern: "STILL POPULATED") + && string_contains(s: unit_termination_wire(t: completed_unknown), pattern: "establishes nothing even beside a completed wait") && string_contains(s: unit_termination_wire(t: empty_no_completion), pattern: "did not complete") && string_contains(s: unit_termination_wire(t: pending), pattern: "pending") && string_contains(s: unit_termination_wire(t: misleading), pattern: "UNESTABLISHED") @@ -176,13 +186,13 @@ test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost)), pattern: "STILL HELD") } -// THE TERMINATION POLICY THE RELEASE DECISION LEANS ON IS ACTUALLY BOUND ON THE INVOCATION. A -// completed wait is allowed to confirm a release only because KillMode=control-group means systemd -// tears the cgroup down with the unit; under process or none a forked child outlives the main -// process and goes on using the memory this seat accounts for. systemd's DEFAULT is -// control-group -- but a default is not a declaration, and a drop-in or a future default could -// change it while this provider kept reasoning from the old one. So it is bound, and this cell is -// what makes "bound" a fact rather than a sentence. +// THIS CHECKS INVOCATION WIRING, NOT A RUNTIME POLICY READBACK, and the distinction is the whole +// honesty of the cell. It reads the argv this provider CONSTRUCTS and establishes that the +// KillMode word is in it. It does NOT establish what systemd applied: a drop-in, a manager that +// rejected the property, or a unit started by some other path are all outside what any fold over +// our own argv can see. That is precisely why the release predicate does not lean on the binding -- +// completion must still be confirmed by a positively empty cgroup, and this row only ensures we are +// not ALSO shipping an invocation that opts into the dangerous mode. test fn the_unit_binds_the_termination_policy_the_release_decision_relies_on() -> Bool { let props = compute_unit_properties( layout: compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc"), @@ -191,3 +201,30 @@ test fn the_unit_binds_the_termination_policy_the_release_decision_relies_on() - && compute_kill_mode_value == "control-group" && count(filter(props, p => starts_with(s: p, prefix: "MemoryMax="))) == 1 } + +// A LOSS ANYWHERE IN THE CHAIN REACHES A TERMINAL CONSUMER, which is the property three separate +// seams were erasing: compute_provide_dependent returned only the child's result, so a BUILD that +// lost its record followed by a successful dependent reported clean; and compile_entry_cli and +// run_selected_module both read only the work's ending (side-chat verdict at f5344b28). +// +// The join is what carries it, and it accumulates rather than letting the later arm win -- both +// details are kept because they name different identities and an operator needs each. The work +// contract is untouched by all of this: the ending and the document are exactly what was stored. +test fn a_lost_record_on_a_dependency_survives_a_successful_dependent() -> Bool { + let succeeded = WorkSucceeded { identity: "child", outputs: [], log_path: "/l" } + let document = work_outcome_wire(o: succeeded, subject: tree(hex: tree_a), op: work_default_build()) + let clean_child = ProvideFresh { outcome: succeeded, document: document, host_record: HostRecordWritten } + let attached_child = ProvideAttached { identity: "child", ending: "succeeded", document: document, host_record: HostRecordWritten } + let build_lost = HostRecordLost { detail: "the build's lease is still held" } + let carried_fresh = provide_result_with_host_record(r: clean_child, carried: build_lost) + let carried_attached = provide_result_with_host_record(r: attached_child, carried: build_lost) + let both_lost = host_record_join(a: build_lost, b: HostRecordLost { detail: "and the dependent lost its reading" }) + !host_record_was_written(h: provide_result_host_record(r: carried_fresh)) + && !host_record_was_written(h: provide_result_host_record(r: carried_attached)) + && provide_result_ending(r: carried_fresh) == "succeeded" + && provide_result_document(r: carried_fresh) == document + && provide_result_ending(r: carried_attached) == "succeeded" + && host_record_was_written(h: provide_result_host_record(r: clean_child)) + && string_contains(s: host_record_wire(h: both_lost), pattern: "still held") + && string_contains(s: host_record_wire(h: both_lost), pattern: "lost its reading") +} From 1c670b8d4ea992e5b4b4456380799736a9848e6a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 20:54:26 +0000 Subject: [PATCH 12/44] The receipt stays parseable: the host-record fact gets its own file beside it Found by the srv1 fault-injection run for the re-attach control, and it is the same defect as the consumption line once appended to outcome.json -- committed a second time, one file along. Appending the host-record sentence to the receipt stopped it being parseable JSON, so a consumer reading it back got "Extra data" instead of an outcome. A document a consumer parses carries exactly what its contract says. The receipt is the document again; the loss is written to .host-record and the exit reason names where it went. Both CLIs. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/test_run.dag | 7 ++++--- dag/gunbc/compute/work_provider_local.dag | 25 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/compute/test_run.dag b/dag/gunbc/compute/test_run.dag index 0a338b0a4ce..4ad563fe459 100644 --- a/dag/gunbc/compute/test_run.dag +++ b/dag/gunbc/compute/test_run.dag @@ -19,7 +19,7 @@ import gunbc.roadmap_execution_contract { GitWorkspaceCapability } import gunbc.compute.work_request { WorkOperation, RunClaims, CompileEntry } import gunbc.compute.work_provider_local { ProvideResult, ProvideFresh, ProvideAttached, compute_provide, provide_result_succeeded, provide_result_ending, provide_result_identity, provide_result_document, - HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, host_record_wire, host_record_was_written, + HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, host_record_wire, host_record_was_written, compute_write_host_record_note, } import gunbc.compute.test_selection { SelectedFile, select_witness_files, classify_witness_module, planned_functions_joined, @@ -274,11 +274,12 @@ fn compile_entry_cli(repo_root: String, worktree: String, entry: String, receipt SnapshotResolved { commit } => { let result = compute_provide(instance: instance, commit: commit, op: CompileEntry { entry: entry as FilePath }) let record = provide_result_host_record(r: result) - let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), host_record_wire(h: record)], "")) + let receipt = Filesystem.Write(path: receipt_path, content: provide_result_document(r: result)) + let record_note = compute_write_host_record_note(receipt_path: receipt_path, record: record) if !receipt.success { ExitFailure { code: 2, reason: join(["gunbc compile: receipt could not be written to ", receipt_path, ": ", receipt.error, "; outcome was ", provide_result_document(r: result)], "") } } else if !host_record_was_written(h: record) { - ExitFailure { code: 1, reason: join(["gunbc compile: the work ended ", provide_result_ending(r: result), " and its stored record is authoritative, but ", host_record_wire(h: record)], "") } + ExitFailure { code: 1, reason: join(["gunbc compile: the work ended ", provide_result_ending(r: result), " and its stored record is authoritative, but ", host_record_wire(h: record), "; ", record_note], "") } } else if provide_result_succeeded(r: result) { ExitSuccess } else { diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index f6a4070179d..59e1eada9d1 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -1007,7 +1007,8 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent Present { value: op } => { let result = compute_provide(instance: instance, commit: commit, op: op) let record = provide_result_host_record(r: result) - let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), host_record_wire(h: record), "\n"], "")) + let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), "\n"], "")) + let record_note = compute_write_host_record_note(receipt_path: receipt_path, record: record) if !receipt.success { ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; outcome was ", provide_result_document(r: result)], "") } } else if !host_record_was_written(h: record) { @@ -1016,7 +1017,7 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent reason: join([ "compute: the work ended ", provide_result_ending(r: result), " and its record at the store is authoritative -- a later request for this identity reads exactly that -- but ", - host_record_wire(h: record), + host_record_wire(h: record), "; ", record_note, ], ""), } } else if provide_result_succeeded(r: result) { @@ -1029,6 +1030,26 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent } } +// THE RECEIPT STAYS THE DOCUMENT, AND THE HOST-RECORD FACT GETS ITS OWN FILE BESIDE IT. +// +// The first version of this appended the host-record sentence to the receipt, and the srv1 +// fault-injection run is what caught it: the receipt stopped being parseable JSON, so a consumer +// reading it back got "Extra data" instead of an outcome. That is the SAME defect as the +// consumption line once appended to outcome.json, committed a second time one file along -- a +// realization's side fact changing the shape of a document some consumer parses. The rule has to be +// the same in both places: a document a consumer parses carries exactly what its contract says, and +// everything else lives beside it. +fn compute_write_host_record_note(receipt_path: String, record: HostRecordStatus) -> String { + match record { + HostRecordWritten => "no host-record note was needed" + HostRecordLost { detail: d } => { + let path = join([receipt_path, ".host-record"], "") + let written = Filesystem.Write(path: path, content: join([d, "\n"], "")) + if written.success { join(["the loss is recorded at ", path], "") } else { join(["and the note could not be written to ", path, " either: ", written.error], "") } + } + } +} + fn compute_parse_operation(operation: String, entry: String, functions: String, hermetic: String) -> WorkOperation? { if operation == "build" { Present { value: work_default_build() } From bf58075f152847c1497f86066f72e02998f4b107 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 21:38:42 +0000 Subject: [PATCH 13/44] SeatRequest carries its dimension; the strip moves to where the pool proves it review 69724, and it is this change's own diagnosis unapplied one boundary over. gunbc.compute.host_capacity was calling kibibyte_count to STRIP its Kibibyte on the way into SeatRequest -- the exact tell work_request.dag names for HostBelowMemoryFloor and fabric_event_log.dag names for SeatRoomObserved. A bare Nat was right while every pool reachable through this fold was Pool: seats are dimensionless and the field meant seats. This PR generalized the carrier to Pool, which made the same field mean KiB for a memory pool and seats for a seat pool with nothing in the type to say which. So SeatRequest is generic and amount carries Measure. WHERE THE STRIP LEGITIMATELY HAPPENS IS ONE LAYER DOWN, and the difference is not cosmetic: propose_acquire holds the POOL, so the pool's own type parameters prove the request's dimension matches the appropriation it is adjudicated against. A consumer stripping the carrier ASSERTS that match; this fold CHECKS it. THE PERSISTED EVENT KEEPS A BARE Nat AND THAT IS NOT THE SAME DEFECT, stated rather than left to be re-found. PoolAcquired is a wire record: its dimension is the partition's, and it is restored at both ends from the pool's own parameters -- minted from a Measure the pool typed, and re-minted at replay by pool_apply_event into the Pool being folded. A JSON integer carrying a magnitude whose unit the surrounding type fixes is a serialization, not a second model of the unit. Call sites: harness_seat and fabric_quota wrap their counts, the capacity and spark witnesses likewise, and host_capacity passes its Kibibyte straight through. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 2 +- dag/gunbc/fabric/fabric_event_log.dag | 4 +-- dag/gunbc/fabric/fabric_quota.dag | 3 +- dag/gunbc/harness/harness_seat.dag | 2 +- dag/gunbc/product/capacity/pool_events.dag | 32 +++++++++++++++---- .../capacity_lease_chain_witness_test.dag | 12 +++---- ...serving_d0_real_execution_witness_test.dag | 4 +-- 7 files changed, 39 insertions(+), 20 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index 3248b01262d..a4dbfb0a8a7 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -340,7 +340,7 @@ fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: Ep partition: subject.partition, root: compute_memory_root_pool(subject: subject, ceiling: ceiling), actor: reference, - request: SeatRequest { reference: reference, amount: kibibyte_count(k: subject.request), at: now, term_seconds: term_seconds }, + request: SeatRequest { reference: reference, amount: subject.request, at: now, term_seconds: term_seconds }, policy: compute_lease_policy(term_seconds: term_seconds), attempts: compute_seat_attempts, budget: compute_partition_read_budget, diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index 5b109c419f3..877a4368837 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -283,7 +283,7 @@ fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId } } -fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, budget: Nat, attempt: Nat) -> SeatAcquisition? { +fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, budget: Nat, attempt: Nat) -> SeatAcquisition? { match event_log_read_partition(store: store, partition: partition, budget: budget) { PartitionReadRefused { cause: c } => Present { value: SeatStoreRefused { cause: c, attempts_used: attempt } } PartitionReadOk { head: head, walk: walk } => @@ -321,7 +321,7 @@ fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: // and the drift is invisible because each side is internally consistent. A mismatch is refused before // anything is appended rather than resolved in favour of either side: neither is authoritative over // the other, and picking one silently would be the same fail-open in a different costume. -fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, attempts: Nat, budget: Nat) -> SeatAcquisition { +fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, attempts: Nat, budget: Nat) -> SeatAcquisition { if !release_law_eq(left: root.release_law, right: policy.release_law) { SeatAcquireRefused { step: "release-law", diff --git a/dag/gunbc/fabric/fabric_quota.dag b/dag/gunbc/fabric/fabric_quota.dag index f6c87e0cb75..870172d2f17 100644 --- a/dag/gunbc/fabric/fabric_quota.dag +++ b/dag/gunbc/fabric/fabric_quota.dag @@ -2,6 +2,7 @@ module gunbc.fabric_quota import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat } +import std.measure { Measure } import extdeps.api_rate_limit { UpstreamRateLimit, upstream_rate_limit_key } import product.capacity.quota { quota_partition, quota_root_pool } import product.capacity.event_chain { PartitionId } @@ -42,7 +43,7 @@ fn fabric_quota_lease(short_hostname: String, limit: UpstreamRateLimit, amount: let partition = quota_partition(limit: limit) match fabric_seat_acquire( store: store, partition: partition, root: quota_root_pool(limit: limit), actor: actor, - request: SeatRequest { reference: join([actor as String, "@", to_string(now)], "") as NonEmptyStr, amount: amount, at: now, term_seconds: term_seconds }, + request: SeatRequest { reference: join([actor as String, "@", to_string(now)], "") as NonEmptyStr, amount: Measure { count: amount }, at: now, term_seconds: term_seconds }, policy: quota_policy(term_seconds: term_seconds), attempts: 3, budget: 4096, ) { SeatGranted { grant: g, generation: _, attempts_used: _ } => QuotaLeased { grant: g, partition: partition, store: store } diff --git a/dag/gunbc/harness/harness_seat.dag b/dag/gunbc/harness/harness_seat.dag index 7fb2d6751a5..dd4c79fc6cf 100644 --- a/dag/gunbc/harness/harness_seat.dag +++ b/dag/gunbc/harness/harness_seat.dag @@ -1225,7 +1225,7 @@ fn harness_acquire_under_authority(candidate: HarnessCandidate, class: ServingCo HeadObserved { head: h } => { let request = SeatRequest { reference: harness_seat_reference(attempt: attempt, offer_key: candidate.key, now: now, round: round, head: harness_head_word(head: h)), - amount: 1, + amount: Measure { count: 1 }, at: now, term_seconds: policy.maximum_duration_seconds, } diff --git a/dag/gunbc/product/capacity/pool_events.dag b/dag/gunbc/product/capacity/pool_events.dag index 542f75c540f..13ca830929a 100644 --- a/dag/gunbc/product/capacity/pool_events.dag +++ b/dag/gunbc/product/capacity/pool_events.dag @@ -2,7 +2,7 @@ module product.capacity.pool_events import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat } -import std.measure { Measure } +import std.measure { Measure, measure_count } import extdeps.languages.json.emit { JsonValue, JsonKeyValue, json_kv, json_string, json_int } import product.capacity.event_json { chain_event_wire_text, chain_event_decode, member_nonempty, member_nat } import product.capacity.pool { @@ -175,9 +175,27 @@ fn pool_fold(root: Pool, oldest_first: List // event with the head it was decided against. The carrier performs the compare-and-set; a stale // answer sends the caller back through this function with the new head. A full pool is a typed // refusal, never a queue. -type SeatRequest { +// THE REQUESTED AMOUNT CARRIES ITS DIMENSION, and this is the same repair SeatStandingObservation +// took for the same reason. A bare Nat was right while every pool reachable through this fold was +// Pool: seats are dimensionless and the field meant seats. Generalizing the carrier to +// Pool made the same field mean KiB for a memory pool and seats for a seat pool, with nothing +// in the type to say which -- so gunbc.compute.host_capacity had to call kibibyte_count to STRIP its +// Kibibyte on the way in, which is precisely the tell this change diagnoses one boundary over +// (review 69724 on gunbc#11962). +// +// WHERE THE STRIP LEGITIMATELY HAPPENS IS HERE, one layer down, and the difference is not cosmetic: +// propose_acquire holds the POOL, so the pool's own type parameters prove the request's dimension +// matches the appropriation it is being adjudicated against. A consumer stripping the carrier is +// asserting that match; this fold checks it. +// +// THE PERSISTED EVENT KEEPS A BARE Nat AND THAT IS NOT THE SAME DEFECT. PoolAcquired is a WIRE +// RECORD: its dimension is the partition's, and it is restored at both ends from the pool's own +// parameters -- minted here from a Measure the pool typed, and re-minted at replay by +// pool_apply_event into the Pool being folded. A JSON integer carrying a magnitude whose unit +// the surrounding type fixes is a serialization, not a second model of the unit. +type SeatRequest { reference: NonEmptyStr - amount: Nat + amount: Measure at: EpochSecs term_seconds: Nat } @@ -213,8 +231,8 @@ fn seat_refusal_wire(r: PoolRefusal) -> String { } } -fn propose_acquire(pool: Pool, generation: Nat, head: HeadExpectation, partition: PartitionId, actor: NonEmptyStr, request: SeatRequest) -> SeatProposal { - match pool_acquire(pool: pool, reference: request.reference, amount: Measure { count: request.amount }, at: request.at, term_seconds: request.term_seconds) { +fn propose_acquire(pool: Pool, generation: Nat, head: HeadExpectation, partition: PartitionId, actor: NonEmptyStr, request: SeatRequest) -> SeatProposal { + match pool_acquire(pool: pool, reference: request.reference, amount: request.amount, at: request.at, term_seconds: request.term_seconds) { PoolRefused { refusal: r } => SeatRefused { wire: seat_refusal_wire(r: r) } PoolAdvanced { pool: _ } => SeatProposed { event: ChainEvent { @@ -222,7 +240,7 @@ fn propose_acquire(pool: Pool, generation: Nat, head: HeadExpectatio parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, recorded_at: request.at, actor: actor, - payload: PoolAcquired { reference: request.reference, amount: request.amount, term_seconds: request.term_seconds }, + payload: PoolAcquired { reference: request.reference, amount: measure_count(request.amount), term_seconds: request.term_seconds }, }, expected_head: head, } @@ -231,7 +249,7 @@ fn propose_acquire(pool: Pool, generation: Nat, head: HeadExpectatio // THE GRANT IS MINTED FROM THE ADMISSION, never from the proposal: a proposal that lost the // compare-and-set mints nothing. -fn grant_from_admission(request: SeatRequest, admission: AppendDecision, policy: LeasePolicy) -> LeaseGrant? { +fn grant_from_admission(request: SeatRequest, admission: AppendDecision, policy: LeasePolicy) -> LeaseGrant? { match admission { AppendStale { expected: _, observed: _ } => none AppendAdmitted { new_head: h, generation: g } => diff --git a/dag/test/claim/capacity_lease_chain_witness_test.dag b/dag/test/claim/capacity_lease_chain_witness_test.dag index 825874a4632..50b263bff49 100644 --- a/dag/test/claim/capacity_lease_chain_witness_test.dag +++ b/dag/test/claim/capacity_lease_chain_witness_test.dag @@ -90,9 +90,9 @@ test fn two_writers_from_one_head_produce_one_admission_one_stale_and_then_a_ful PoolFoldRefused { at_event: _, wire: _ } => false PoolFolded { pool: p, generation: g } => { let a = propose_acquire(pool: p, generation: g, head: head, partition: w_partition(), actor: "srv1-harness" as NonEmptyStr, - request: SeatRequest { reference: "srv1-turn-7" as NonEmptyStr, amount: 1, at: 1100, term_seconds: 300 }) + request: SeatRequest { reference: "srv1-turn-7" as NonEmptyStr, amount: Measure { count: 1 }, at: 1100, term_seconds: 300 }) let b = propose_acquire(pool: p, generation: g, head: head, partition: w_partition(), actor: "srv2-harness" as NonEmptyStr, - request: SeatRequest { reference: "srv2-turn-3" as NonEmptyStr, amount: 1, at: 1100, term_seconds: 300 }) + request: SeatRequest { reference: "srv2-turn-3" as NonEmptyStr, amount: Measure { count: 1 }, at: 1100, term_seconds: 300 }) match a { SeatRefused { wire: _ } => false SeatProposed { event: ea, expected_head: xa } => @@ -109,7 +109,7 @@ test fn two_writers_from_one_head_produce_one_admission_one_stale_and_then_a_ful PoolFolded { pool: p2, generation: g2 } => g2 == 2 && w_headroom(p: p2, at: 1100) == 0 && (match propose_acquire(pool: p2, generation: g2, head: HeadAt { id: w_id(s: "e2") }, partition: w_partition(), actor: "srv2-harness" as NonEmptyStr, - request: SeatRequest { reference: "srv2-turn-3" as NonEmptyStr, amount: 1, at: 1101, term_seconds: 300 }) { + request: SeatRequest { reference: "srv2-turn-3" as NonEmptyStr, amount: Measure { count: 1 }, at: 1101, term_seconds: 300 }) { SeatRefused { wire: w } => w == "pool-full" SeatProposed { event: _, expected_head: _ } => false }) @@ -147,7 +147,7 @@ test fn the_fence_rejects_obsolete_and_foreign_grants_and_expiry_frees_only_unde // THE GRANT IS MINTED FROM THE ADMISSION, never from the proposal: its fence is the admitted event // and generation, and its deadline is the declared policy applied at the grant instant. test fn a_grant_carries_the_admitted_event_as_its_fence_and_the_declared_deadline() -> Bool { - let req = SeatRequest { reference: "srv1-turn-7" as NonEmptyStr, amount: 1, at: 1100, term_seconds: 300 } + let req = SeatRequest { reference: "srv1-turn-7" as NonEmptyStr, amount: Measure { count: 1 }, at: 1100, term_seconds: 300 } match grant_from_admission(request: req, admission: AppendAdmitted { new_head: w_id(s: "e2"), generation: 2 }, policy: w_policy()) { Absent => false Present { value: g } => @@ -212,7 +212,7 @@ test fn an_over_committed_pool_still_refuses_a_new_acquire() -> Bool { PoolFolded { pool: p, generation: g } => match propose_acquire(pool: p, generation: g, head: HeadAt { id: w_id(s: "o4") }, partition: w_partition(), actor: "srv1-harness" as NonEmptyStr, - request: SeatRequest { reference: "seat-5" as NonEmptyStr, amount: 1, at: 1100, term_seconds: 300 }) { + request: SeatRequest { reference: "seat-5" as NonEmptyStr, amount: Measure { count: 1 }, at: 1100, term_seconds: 300 }) { SeatProposed { event: _, expected_head: _ } => false SeatRefused { wire: w } => w == "pool-full" } @@ -311,7 +311,7 @@ test fn a_lapsed_quiescence_required_partition_still_refuses_the_next_seat() -> PoolFolded { pool: p, generation: g } => match propose_acquire(pool: p, generation: g, head: HeadAt { id: w_id(s: "l2") }, partition: w_partition(), actor: "srv1-harness" as NonEmptyStr, - request: SeatRequest { reference: "seat-arriving" as NonEmptyStr, amount: 1, at: 2001, term_seconds: 300 }) { + request: SeatRequest { reference: "seat-arriving" as NonEmptyStr, amount: Measure { count: 1 }, at: 2001, term_seconds: 300 }) { SeatProposed { event: _, expected_head: _ } => false SeatRefused { wire: w } => w == "pool-full" } diff --git a/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag b/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag index 1cce5d07974..00aea13fd0c 100644 --- a/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag +++ b/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag @@ -3,7 +3,7 @@ module test.claim.spark.pair_serving_d0_real_execution import std.logic { Bool } import std.types { String, NonEmptyStr, FilePath, List, Int } import v2.std.optional { Present, Absent } -import std.measure { second, measure_count } +import std.measure { second, measure_count, Measure } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import extdeps.shell import std.content_hash { ContentHash, Sha256Hash, sha256_hex_digest, compare_content_hash, ContentHashEqual, ContentHashDifferent, ContentHashCrossFamilyIncomparable, serialize_content_hash, content_hash_equal } @@ -383,7 +383,7 @@ test fn a_seat_granted_after_the_suspension_landed_is_released_by_real_execution AuthorityTransitionAppended { id: _, generation: g1, next: _ } => g1 == 1 _ => false }) - && (match fabric_seat_acquire(store: store, partition: partition, root: root, actor: "witness" as NonEmptyStr, request: SeatRequest { reference: "witness-seat-1" as NonEmptyStr, amount: 1, at: 1001, term_seconds: 300 }, policy: policy, attempts: 3, budget: 64) { + && (match fabric_seat_acquire(store: store, partition: partition, root: root, actor: "witness" as NonEmptyStr, request: SeatRequest { reference: "witness-seat-1" as NonEmptyStr, amount: Measure { count: 1 }, at: 1001, term_seconds: 300 }, policy: policy, attempts: 3, budget: 64) { SeatGranted { grant: g, generation: _, attempts_used: _ } => (match harness_seat_after_grant_authority(store: store, group: FabricGroupA, partition: partition, ceiling: ceiling, grant: g, now: 1001) { SeatReleasedAuthorityMoved { cause: _ } => true From afb46eb704e57c53a8fa241738286780ccd80143 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 21:54:10 +0000 Subject: [PATCH 14/44] The attached arm carried the loss it had just built into a discard review 69724 finding 2. compute_with_lost_host_record matched the arms itself, and the ProvideAttached arm returned the status it was handed while THROWING AWAY the detail it had just built -- the exact vanish this function exists to prevent, one arm along, in the function whose annotation is about a fact vanishing. It was latent only because the single call site passes a fresh result, which is the kind of "safe today" that stops being true at the next call site. Two spellings of one join is also the fork DESIGN 3 forbids, so there is now one: provide_result_with_host_record, which already handled both arms and accumulates rather than letting either side win. The control drives BOTH arms. Mutation, run and reverted: restore the arm-matching discard and a_lost_host_record_changes_no_contract_the_next_caller_reads goes FAIL. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 14 ++++++++++---- .../claim/compute/work_request_witness_test.dag | 11 +++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 59e1eada9d1..13ca10a3a44 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -660,6 +660,15 @@ fn compute_run_and_settle( // the lost record rides as its own fact. The CLI turns that fact into a nonzero exit with both // halves named, which is how a recording failure stays visible without becoming a verdict about // work that really did run. +// +// AND IT GOES THROUGH THE ONE JOIN RATHER THAN MATCHING THE ARMS ITSELF. It matched them, and the +// ProvideAttached arm returned the status it was handed while DISCARDING the detail it had just +// built -- the exact vanish this function exists to prevent, one arm along, in the function whose +// annotation describes the vanish (review 69724). It was latent only because the single call site +// passes a fresh result, which is the kind of "safe today" that stops being true at the next call +// site. Two spellings of one join is the fork DESIGN 3 forbids, so there is now one: +// provide_result_with_host_record, which already handled both arms and accumulates rather than +// letting either side win. fn compute_with_lost_host_record( settled: ProvideResult, layout: ComputeLayout, @@ -676,10 +685,7 @@ fn compute_with_lost_host_record( join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be produced again"], "") }, ], "") - match settled { - ProvideFresh { outcome: o, document: d, host_record: _ } => ProvideFresh { outcome: o, document: d, host_record: HostRecordLost { detail: detail } } - ProvideAttached { identity: i, ending: e, document: d, host_record: h } => ProvideAttached { identity: i, ending: e, document: d, host_record: h } - } + provide_result_with_host_record(r: settled, carried: HostRecordLost { detail: detail }) } // BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the // caller a refusal while the outcome document already said the run had SUCCEEDED, and then state diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 4f396da6242..c641d411fed 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -165,6 +165,11 @@ test fn the_attempt_completion_fact_is_not_the_works_ending() -> Bool { && !compute_attempt_completed(outcome: WorkRefusedByInfrastructure { identity: "a", cause: HostComputeCapacityFull { wire: "pool-full" } }) } +// AN ATTACHED RESULT CARRIES THE LOSS TOO, which is the arm that used to throw it away. The only +// call site passes a FRESH result, so the discard was latent -- and "latent" is the kind of safe +// that stops being true at the next call site, in the very function whose annotation is about a +// fact vanishing (review 69724). Both arms are driven here. +// // THE STORED RECORD AND THE RETURNED OUTCOME AGREE, WHATEVER HAPPENED TO THIS HOST'S OWN RECORD. // The fusion was made twice in opposite directions -- a refusal handed to the caller over a stored // success, then a synthesized refusal over a published outcome -- and both times the next attacher @@ -176,14 +181,20 @@ test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { let succeeded = WorkSucceeded { identity: "abc", outputs: [], log_path: "/l" } let document = work_outcome_wire(o: succeeded, subject: tree(hex: tree_a), op: work_default_build()) let settled = ProvideFresh { outcome: succeeded, document: document, host_record: HostRecordWritten } + let attached = ProvideAttached { identity: "abc", ending: "succeeded", document: document, host_record: HostRecordWritten } let layout = compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc") let lost = compute_with_lost_host_record(settled: settled, layout: layout, dropped_ok: false, dropped_error: "EBUSY", error: "ENOSPC") + let lost_attached = compute_with_lost_host_record(settled: attached, layout: layout, dropped_ok: false, dropped_error: "EBUSY", error: "ENOSPC") provide_result_ending(r: lost) == provide_result_ending(r: settled) && provide_result_document(r: lost) == provide_result_document(r: settled) && provide_result_ending(r: lost) == "succeeded" && host_record_was_written(h: provide_result_host_record(r: settled)) && !host_record_was_written(h: provide_result_host_record(r: lost)) && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost)), pattern: "STILL HELD") + && provide_result_ending(r: lost_attached) == "succeeded" + && provide_result_document(r: lost_attached) == document + && !host_record_was_written(h: provide_result_host_record(r: lost_attached)) + && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost_attached)), pattern: "STILL HELD") } // THIS CHECKS INVOCATION WIRING, NOT A RUNTIME POLICY READBACK, and the distinction is the whole From 2700b015ac9774392941f5889c8c9c184ec80387 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 22:04:31 +0000 Subject: [PATCH 15/44] File the partition-identity class the PoolAcquired argument would have buried Ruled 2026-09-21 (loyal-swift-608): do NOT carry Measure through PoolAcquired and the codec -- the decoder would still mint the caller-requested Q,S because the wire has nothing to disagree with, so the RED would not be authorable anywhere a check could run, which DESIGN 4b names as worse than absent. But the error class the argument identified is real and nothing in the corpus holds it, so it does not leave with the argument. gunbc.recurring_failure_mode a_pool_partition_does_not_declare_its_quantity: a persisted capacity partition carries magnitudes and does not declare the QUANTITY they are denominated in. pool_apply_event mints every replayed amount into the Q and S OF THE POOL BEING FOLDED, so a seat partition replayed by a memory pool reads 1 seat as 1 KiB and a memory partition replayed by a seat pool reads 27262976 KiB as 27262976 seats -- arithmetically consistent, semantically unrelated, nothing refuses. THE CLASS BECAME REACHABLE IN THIS PR, which is why it is filed here: generalizing fabric_seat_acquire and fabric_seat_observe from Pool to Pool is what made two quantities share one linearization. Before it, every partition reachable through that fold was seats and the question could not be asked. What is NOT claimed: that it has occurred. The two partition namers are disjoint by construction today -- `quota-` over a rate-limit key, `compute-memory-` over a host -- so the row is filed because the discipline preventing it is a naming convention held by two authors, not a fact the substrate checks. Trigger names the CAPABILITY -- partition identity binds the pool quantity, so a mismatched fold refuses -- deliberately not "carry Measure through the codec", which could land in full while the class stayed exactly as dead. Rostered, and stated in the PR body beside the other declared gaps so a reader does not have to find it in a failure-mode file. Co-Authored-By: Claude Opus 5 (1M context) --- ...artition_does_not_declare_its_quantity.dag | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 dag/gunbc/recurring_failure_mode/a_pool_partition_does_not_declare_its_quantity.dag diff --git a/dag/gunbc/recurring_failure_mode/a_pool_partition_does_not_declare_its_quantity.dag b/dag/gunbc/recurring_failure_mode/a_pool_partition_does_not_declare_its_quantity.dag new file mode 100644 index 00000000000..138a4b7c99c --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/a_pool_partition_does_not_declare_its_quantity.dag @@ -0,0 +1,34 @@ +module gunbc.recurring_failure_mode.a_pool_partition_does_not_declare_its_quantity + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, decl_ref } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data a_pool_partition_does_not_declare_its_quantity: RecurringFailureMode = RecurringFailureMode { + identity: "a_pool_partition_does_not_declare_its_quantity" as NonEmptyStr, + + receipts: [ + "INVALID STATE: a persisted capacity partition carries magnitudes and does not declare the QUANTITY they are denominated in. product.capacity.pool_events pool_apply_event mints every replayed amount into the Q and S OF THE POOL BEING FOLDED, and the wire has nothing to disagree with -- so a partition written by a seat pool and replayed by a memory pool reinterprets 1 seat as 1 KiB, and a memory partition replayed by a seat pool reinterprets 27262976 KiB as 27262976 seats. HARM: an admission decision computed against a committed total that means something else; the numbers are arithmetically consistent and semantically unrelated, so nothing refuses and nothing looks wrong.", + + "THE CLASS BECAME REACHABLE WHEN THE POOL BECAME POLYMORPHIC. gunbc#11962 generalized gunbc.fabric_event_log fabric_seat_acquire and fabric_seat_observe from Pool to Pool so a host memory pool could use the same linearization as the seat pools. Before that every partition reachable through that fold was denominated in seats, the question could not be asked, and the answer could not be wrong. The generalization is what made two quantities share one carrier.", + + "WHY TYPING THE WIRE FIELD IS NOT THE REPAIR, ruled 2026-09-21 and recorded here because the wrong repair is the attractive one. Making PoolAcquired.amount a Measure looks like the fix and buys nothing: the DECODER would still mint the caller-requested Q and S, because there is nothing on the wire to disagree with. The RED would not be authorable anywhere a check could run, which DESIGN 4b names as worse than absent -- a decoration that gets cited as coverage. The admission boundary, where the hazard WAS reachable, is closed separately and by construction: SeatRequest must match Pool at propose_acquire, so a seat count can no longer be charged against a memory pool at the door.", + + "WHAT IS NOT CLAIMED: that this has occurred. The two partition namers in the corpus are disjoint by construction today -- product.capacity.quota quota_partition prefixes `quota-` over an upstream rate-limit key and gunbc.compute.host_capacity compute_memory_partition prefixes `compute-memory-` over a host identity -- so no name collides and no fold has been observed reading the wrong one. The class is filed because the DISCIPLINE preventing it is a naming convention held by two authors, not a fact the substrate checks.", + + "RUNG FOUND AT: mitigatable, and only by that convention. Nothing refuses a cross-quantity fold; what prevents it is that the two producers happen to choose non-overlapping prefixes and that each consumer happens to pass the pool it built the partition name from.", + + "CEILING: structurally guaranteed. A partition identity that BINDS the quantity it is denominated in, checked where the fold is assembled, makes the mismatched pairing refuse rather than reinterpret -- the root pool and the partition are already supplied together at every call site, so the join exists and is simply not made.", + + "NEXT-RUNG TRIGGER, NAMING THE CAPABILITY: partition identity binds the pool quantity, so assembling a fold from a partition and a root pool whose quantities disagree is a typed refusal rather than a silent reinterpretation. It is deliberately NOT `carry Measure through the codec` -- that artifact could land in full while this class stayed exactly as dead, which is the grain mismatch DESIGN 4b(3) warns about between a loss sentence and its trigger.", + + "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration.", + ], + + evidence: [ + decl_ref(module_path: "product.capacity.pool_events", decl_name: "pool_apply_event"), + decl_ref(module_path: "product.capacity.pool_events", decl_name: "pool_event_decode"), + decl_ref(module_path: "product.capacity.quota", decl_name: "quota_partition"), + decl_ref(module_path: "gunbc.compute.host_capacity", decl_name: "compute_memory_partition"), + ], +} From 319edef76504422945da5ae265d23e004ac56a33 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 22:25:51 +0000 Subject: [PATCH 16/44] PR 2 spine: the attempt slot, the CAS-gated launch, the five-arm union and the recovery trigger Not the finished package -- this is the spine that criteria 1, 2, 3, 4 and 6 rest on, with its controls. The provider is not wired to it yet (5 and 7 are next), so nothing in PR 1's behaviour changes. THE ATTEMPT SLOT. One durable CAS slot per attempt, keyed by the reservation reference, on gunbc.durable_cas_file_store. Its value is the attempt record: identity, reference, UNIT NAME, granted amount, launch deadline, state. (3) THE LAUNCH IS CAS-GATED AND THAT IS ONE STEP. The producer advances Reserved -> Launching expecting the generation it read; recovery advances Reserved -> Terminal expecting the SAME generation. The store admits exactly one, and the losing PRODUCER has not spawned yet -- the refusal arrives before the launch, which is the difference between a gate and a regret. Both interleavings are driven on a real store: terminate-then-launch (producer refused, slot stays terminal) and launch-then-terminate (recovery refused, slot stays launching). (2) The unit name is READ from the record the attempt wrote, never derived from the identity or a path spelling -- a derived name is a second authority for the binding, and a wrong derivation stops somebody else's unit. (1) FIVE ARMS, NOT FOUR, and the count is a consequence rather than a target (ruling, loyal-swift-608). Folding authoritative absence into a failure arm would have spent a distinction PR 1 landed: "the manager has no record of this unit" and "the attempt failed" are different facts with different evidence, and only the first is safe to free a seat on WITHOUT an attempt record to consult. So TerminatedCauseUnknown carries it under a name that says what it is. Each arm's refusal is asserted individually. (4) Unreadable is never absent, at the slot and at the record: an undecodable record is AttemptSlotUnreadable, and a state word this fold does not know is unreadable rather than defaulted -- a future state must not read as Reserved, which is the one state a launch may proceed from. (6) Settling carries the verdict AND why, and the generation-suffixed store keeps what it settled: the reserved and launching generations are still readable after the terminal one lands. THE GAP THE GATE DOES NOT CLOSE, AND THE TRIGGER THAT DOES. A compare-and-set decides who WINS a race; it says nothing about whether recovery should have entered one. A live producer that has not yet reached its CAS sits in Reserved -- exactly what an impatient recovery would terminate -- and the CAS then makes that outcome look CORRECT: the producer refuses, releases, and the system appears to work while a legitimate attempt was killed by a bystander. So the trigger splits on the state, structurally rather than by policy. In Reserved there is NO UNIT TO OBSERVE, so observation cannot distinguish a pre-launch producer from a dead one -- both present the same absence -- and the trigger is TEMPORAL, bounded by the deadline the producer itself declared when it opened. In Launching the unit exists, so the trigger is the five-arm observation with criterion 4 in full: only the three ended arms may terminate. Eight wet controls pass against a real generation-file store. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/attempt_lifecycle.dag | 397 ++++++++++++++++++ .../attempt_lifecycle_wet_witness_test.dag | 231 ++++++++++ 2 files changed, 628 insertions(+) create mode 100644 dag/gunbc/compute/attempt_lifecycle.dag create mode 100644 dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag diff --git a/dag/gunbc/compute/attempt_lifecycle.dag b/dag/gunbc/compute/attempt_lifecycle.dag new file mode 100644 index 00000000000..71ac56e4d20 --- /dev/null +++ b/dag/gunbc/compute/attempt_lifecycle.dag @@ -0,0 +1,397 @@ +module gunbc.compute.attempt_lifecycle + +import std.types { String, Bool, Int, NonEmptyStr } +import std.content_hash { content_hash_of_value } +import std.durable_compare_and_set { + CasAttempt, cas_attempt, CasExpectation, ExpectSlotAbsent, ExpectSlotGeneration, + CasOutcome, CasCommitted, CasPreconditionFailed, CasStoreRefused, + CasSlotObservation, CasObservedReadable, CasObservedUnreadable, + CasReadableSlot, CasReadableAbsent, CasReadablePresent, + CasGeneration, cas_generation_count, cas_unreadable_slot_detail, + CasStoreFailure, CasSlotObservationRefused, CasGenerationPublicationRefused, +} +import gunbc.durable_cas_file_store { + CasAttemptAdmission, CasAttemptAdmitted, CasAttemptDigestMismatch, CasAttemptDigestIncomparable, + CasAttemptKeyNotSlotAddressable, admit_cas_attempt, file_compare_and_set, DefaultAccessCreateOnly, + observe_cas_slot_state, +} +import extdeps.languages.json.emit { JsonValue, json_object, json_kv, json_string, json_int, serialize_json } +import extdeps.languages.json.parse { + parse_json_document, JsonDocumentParsed, JsonDocumentUnreadable, json_document_gap_text, + json_object_unique_member, JsonMemberFound, JsonMemberAbsent, JsonMemberDuplicated, JsonMemberNotAnObject, + JsonString, JsonNumber, +} + +// THE ATTEMPT'S OWN RECORD, AND THE ONE PLACE ITS LIFECYCLE IS DECIDED. +// +// gunbc#11962 (PR 1) admits a reservation and releases it when the run ends. What it deliberately +// does NOT have is a way to give a seat back when that release does not land, and it declares that +// absence as gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge rather +// than shipping a recovery whose failure modes would FREE SEATS THAT SHOULD STAY HELD. This module +// is that capability, and every shape in it exists because one of those failure modes is real. +// +// WHY A CAS SLOT AND NOT A FILE, A FLAG OR A RE-LISTING. Recovery and the producer are two +// principals racing over one attempt, and the question "may this attempt still launch" has to be +// answered in the SAME step that answers "is this attempt terminal". A file the producer checks and +// then acts on is two steps with a window between them, and the window is exactly where an unbacked +// launch lives: reserve, recovery sees no unit and releases, producer spawns against memory nobody +// holds. So the state lives in a compare-and-set slot and every transition names the generation it +// expects -- the store decides the winner, and the loser is refused by precondition rather than by +// noticing afterwards. +// +// THE SLOT IS KEYED BY THE RESERVATION REFERENCE, which names ONE HOLD (identity plus the instant +// it was taken). Keying by work identity would make a retry collide with the attempt it is +// retrying -- the defect PR 1 already paid for once with its obligation files. + +// THE UNIT LIFECYCLE, AND IT IS A UNION OF TWO AXES RATHER THAN EITHER ONE OF THEM. +// +// Two lists met here and neither was the answer. PR 1's -- attempt-completed, authoritative-absence, +// populated, unavailable -- is closest to an OBSERVATION axis: what we were able to learn. +// The lifecycle brief's -- terminated-success, terminated-failure, still-running, unknown -- is an +// OUTCOME axis: what happened to the attempt. Making either the carrier and deriving the other +// loses something real (ruling, loyal-swift-608, 2026-09-21): +// +// FOLDING AUTHORITATIVE ABSENCE INTO A FAILURE ARM IS THE LOSS THAT MATTERS. "The manager has no +// record of this unit" and "the attempt failed" are different facts with different evidence, and +// only the FIRST is safe to free a seat on WITHOUT an attempt record to consult -- absence needs +// no record to be conclusive, while a failure verdict is a claim about a run whose record may not +// exist. PR 1 landed that distinction and a derivation must not quietly spend it. +// +// So the union keeps every distinction either list carries, and the count is a consequence rather +// than a target. TerminatedCauseUnknown is PR 1's authoritative absence under the name that says +// what it is: ended, and nothing here knows why. +type UnitLifecycle + = UnitTerminatedSuccess { detail: String } + | UnitTerminatedFailure { detail: String } + | UnitTerminatedCauseUnknown { detail: String } + | UnitStillRunning { detail: String } + | UnitLifecycleUnknown { detail: String } + +// THE THREE ENDED ARMS FREE CAPACITY AND THE TWO UNENDED ONES DO NOT, which is the same predicate +// PR 1 enforced -- every arm it refused on still refuses, and that is asserted ARM BY ARM rather +// than in aggregate, because a derived predicate that is merely green overall is how a refusal +// disappears in a refactor. +fn unit_lifecycle_ended(u: UnitLifecycle) -> Bool { + match u { + UnitTerminatedSuccess { detail: _ } => true + UnitTerminatedFailure { detail: _ } => true + UnitTerminatedCauseUnknown { detail: _ } => true + UnitStillRunning { detail: _ } => false + UnitLifecycleUnknown { detail: _ } => false + } +} + +fn unit_lifecycle_wire(u: UnitLifecycle) -> String { + match u { + UnitTerminatedSuccess { detail: d } => join(["terminated-success (", d, ")"], "") + UnitTerminatedFailure { detail: d } => join(["terminated-failure (", d, ")"], "") + UnitTerminatedCauseUnknown { detail: d } => join(["terminated, cause unknown (", d, ")"], "") + UnitStillRunning { detail: d } => join(["STILL RUNNING (", d, "), so the seat stays charged"], "") + UnitLifecycleUnknown { detail: d } => join(["UNKNOWN (", d, "), so the seat stays charged"], "") + } +} + +// THE LIFECYCLE STATE THE SLOT CARRIES. Reserved is the only state a launch may proceed from and +// the only state recovery may terminate from without observing the unit -- because in Reserved no +// unit exists yet, which is the whole reason the two transitions must exclude each other. +type AttemptState + = AttemptReserved + | AttemptLaunching + | AttemptTerminal { verdict: String, detail: String } + +fn attempt_state_wire(s: AttemptState) -> String { + match s { + AttemptReserved => "reserved" + AttemptLaunching => "launching" + AttemptTerminal { verdict: v, detail: _ } => join(["terminal:", v], "") + } +} + +// THE RECORD RECOVERY READS INSTEAD OF RECONSTRUCTING. The unit name is written here by the attempt +// that owns it; recovery never derives it from the identity, a path spelling or a directory listing. +// That is not tidiness: a derived name is a second authority for the binding, and a recovery acting +// on a name it invented can stop a unit that belongs to a different attempt. +type AttemptRecord { + identity: String + reference: String + unit: String + granted_kib: Int + launch_deadline: Int + state: AttemptState +} + +fn attempt_record_json(r: AttemptRecord) -> JsonValue { + json_object(members: [ + json_kv(key: "identity", value: json_string(s: r.identity)), + json_kv(key: "reference", value: json_string(s: r.reference)), + json_kv(key: "unit", value: json_string(s: r.unit)), + json_kv(key: "granted_kib", value: json_int(n: r.granted_kib)), + json_kv(key: "launch_deadline", value: json_int(n: r.launch_deadline)), + json_kv(key: "state", value: json_string(s: attempt_state_wire(s: r.state))), + json_kv(key: "detail", value: json_string(s: match r.state { AttemptTerminal { verdict: _, detail: d } => d AttemptReserved => "" AttemptLaunching => "" })), + ]) +} + +fn attempt_record_wire(r: AttemptRecord) -> String { + serialize_json(v: attempt_record_json(r: r)) +} + +fn attempt_record_with_state(r: AttemptRecord, state: AttemptState) -> AttemptRecord { + AttemptRecord { identity: r.identity, reference: r.reference, unit: r.unit, granted_kib: r.granted_kib, launch_deadline: r.launch_deadline, state: state } +} + +// ── READING THE SLOT, AND UNREADABLE IS NEVER ABSENT ────────────────────────────────────────── +// +// This is the absorbing fallback in its purest form and DESIGN 5 makes it a hard reject: a slot +// that could not be READ and a slot that holds NOTHING are opposite facts with opposite remedies, +// and collapsing them means a recovery creates state on top of one it merely failed to observe, or +// a producer launches because it could not tell it had been terminated. Every arm below is typed +// and none of them widens. +type AttemptSlotReading + = AttemptSlotAbsent + | AttemptSlotHeld { record: AttemptRecord, generation: CasGeneration } + | AttemptSlotUnreadable { detail: String } + +fn attempt_member(doc: JsonValue, key: String) -> String? { + match json_object_unique_member(v: doc, key: key) { + JsonMemberFound { value } => match value { JsonString { value: s } => Present { value: s } _ => none } + JsonMemberAbsent => none + JsonMemberDuplicated => none + JsonMemberNotAnObject => none + } +} + +fn attempt_member_int(doc: JsonValue, key: String) -> Int? { + match json_object_unique_member(v: doc, key: key) { + JsonMemberFound { value } => match value { JsonNumber { lexeme: n } => parse_int(s: n as String) _ => none } + JsonMemberAbsent => none + JsonMemberDuplicated => none + JsonMemberNotAnObject => none + } +} + +// A RECORD THAT DOES NOT DECODE IS UNREADABLE, NOT EMPTY, for the same reason the slot is. The +// state word is reconstructed from the same spelling attempt_state_wire emits, and a word this fold +// does not know is unreadable rather than defaulted -- a future state must not read as Reserved, +// because Reserved is the one state a launch may proceed from. +fn attempt_record_decode(text: String) -> AttemptRecord? { + match parse_json_document(s: text) { + JsonDocumentUnreadable { gap: _ } => none + JsonDocumentParsed { value: doc } => + match attempt_member(doc: doc, key: "identity") { + Absent => none + Present { value: identity } => + match attempt_member(doc: doc, key: "reference") { + Absent => none + Present { value: reference } => + match attempt_member(doc: doc, key: "unit") { + Absent => none + Present { value: unit } => + match attempt_member_int(doc: doc, key: "granted_kib") { + Absent => none + Present { value: granted } => + match attempt_member_int(doc: doc, key: "launch_deadline") { + Absent => none + Present { value: deadline } => + match attempt_member(doc: doc, key: "state") { + Absent => none + Present { value: word } => + match attempt_state_decode(word: word, detail: match attempt_member(doc: doc, key: "detail") { Present { value: d } => d Absent => "" }) { + Absent => none + Present { value: state } => + Present { value: AttemptRecord { identity: identity, reference: reference, unit: unit, granted_kib: granted, launch_deadline: deadline, state: state } } + } + } + } + } + } + } + } + } +} + +fn attempt_state_decode(word: String, detail: String) -> AttemptState? { + if word == "reserved" { + Present { value: AttemptReserved } + } else if word == "launching" { + Present { value: AttemptLaunching } + } else if starts_with(s: word, prefix: "terminal:") { + Present { value: AttemptTerminal { verdict: substring(s: word, start: length("terminal:"), end: length(word)), detail: detail } } + } else { + none + } +} + +fn attempt_slot_read(root: NonEmptyStr, reference: NonEmptyStr) -> AttemptSlotReading { + match observe_cas_slot_state(root: root, key: reference) { + CasObservedUnreadable { cause: c } => AttemptSlotUnreadable { detail: join(["the attempt slot could not be read: ", cas_unreadable_slot_detail(cause: c)], "") } + CasObservedReadable { readable: r } => + match r { + CasReadableAbsent => AttemptSlotAbsent + CasReadablePresent { version: v } => + match attempt_record_decode(text: v.value as String) { + Absent => AttemptSlotUnreadable { detail: join(["the attempt slot at generation ", to_string(value: cas_generation_count(g: v.generation)), " holds a record this fold cannot decode"], "") } + Present { value: rec } => AttemptSlotHeld { record: rec, generation: v.generation } + } + } + } +} + +// ── THE TRANSITIONS, EACH NAMING THE GENERATION IT EXPECTS ──────────────────────────────────── +// +// THIS IS WHERE CRITERION 3 LIVES AND IT IS ONE STEP, NOT TWO. A producer that is about to spawn +// advances Reserved -> Launching expecting the generation it read; recovery that wants to end a +// reserved-but-unlaunched attempt advances Reserved -> Terminal expecting the SAME generation. +// The store admits exactly one of them. The loser gets CasPreconditionFailed and, crucially, the +// losing PRODUCER has not spawned anything yet -- the refusal arrives BEFORE the launch, not after +// it, which is the difference between a gate and a regret. +type AttemptTransition + = AttemptAdvanced { state: AttemptState, generation: CasGeneration } + | AttemptLostRace { expected: String, observed: String } + | AttemptTransitionRefused { detail: String } + +fn attempt_commit(root: NonEmptyStr, record: AttemptRecord, expected: CasExpectation) -> AttemptTransition { + let payload = attempt_record_wire(r: record) as NonEmptyStr + match admit_cas_attempt(attempt: cas_attempt( + key: record.reference as NonEmptyStr, + expected: expected, + proposed: payload, + proposed_content: content_hash_of_value(value: payload), + )) { + CasAttemptKeyNotSlotAddressable { key: k } => AttemptTransitionRefused { detail: join(["the reservation reference is not slot-addressable: ", k as String], "") } + CasAttemptDigestMismatch { claimed: _, actual: _ } => AttemptTransitionRefused { detail: "the attempt record's digest did not match its bytes" } + CasAttemptDigestIncomparable { claimed: _, actual: _ } => AttemptTransitionRefused { detail: "the attempt record's digest was incomparable" } + CasAttemptAdmitted { verified: v } => + match file_compare_and_set(root: root, publication: DefaultAccessCreateOnly, verified: v) { + CasCommitted { committed: c } => AttemptAdvanced { state: record.state, generation: c.generation } + CasPreconditionFailed { expected: e, observed: o } => + AttemptLostRace { + expected: match e { ExpectSlotGeneration { generation: g } => join(["generation ", to_string(value: cas_generation_count(g: g))], "") ExpectSlotAbsent => "an absent slot" }, + observed: match o { CasReadablePresent { version: v2 } => join(["generation ", to_string(value: cas_generation_count(g: v2.generation))], "") CasReadableAbsent => "an absent slot" }, + } + CasStoreRefused { cause: c } => + AttemptTransitionRefused { + detail: match c { + CasSlotObservationRefused { cause: u } => join(["the attempt store refused the read: ", cas_unreadable_slot_detail(cause: u)], "") + CasGenerationPublicationRefused { detail: d } => join(["the attempt store refused the write: ", d as String], "") + }, + } + } + } +} + +// OPENING: the attempt declares itself Reserved, and it must be the FIRST writer of this reference. +// ExpectSlotAbsent is what makes a duplicated reference a refusal rather than a silent overwrite of +// somebody else's attempt. +fn attempt_open(root: NonEmptyStr, record: AttemptRecord) -> AttemptTransition { + attempt_commit(root: root, record: attempt_record_with_state(r: record, state: AttemptReserved), expected: ExpectSlotAbsent) +} + +// THE LAUNCH GATE. Called BEFORE the unit is spawned, never after. +fn attempt_begin_launch(root: NonEmptyStr, record: AttemptRecord, expected: CasGeneration) -> AttemptTransition { + attempt_commit(root: root, record: attempt_record_with_state(r: record, state: AttemptLaunching), expected: ExpectSlotGeneration { generation: expected }) +} + +// THE TERMINAL TRANSITION. Carries the verdict AND why, so the generation that records the end also +// records the reason -- history the next reader can act on rather than a bare flag. +fn attempt_terminate(root: NonEmptyStr, record: AttemptRecord, expected: CasGeneration, verdict: String, detail: String) -> AttemptTransition { + attempt_commit( + root: root, + record: attempt_record_with_state(r: record, state: AttemptTerminal { verdict: verdict, detail: detail }), + expected: ExpectSlotGeneration { generation: expected }) +} + +fn attempt_transition_wire(t: AttemptTransition) -> String { + match t { + AttemptAdvanced { state: s, generation: g } => join(["advanced to ", attempt_state_wire(s: s), " at generation ", to_string(value: cas_generation_count(g: g))], "") + AttemptLostRace { expected: e, observed: o } => join(["lost the race: expected ", e, " and the slot holds ", o], "") + AttemptTransitionRefused { detail: d } => join(["refused: ", d], "") + } +} + +fn attempt_transition_advanced(t: AttemptTransition) -> Bool { + match t { + AttemptAdvanced { state: _, generation: _ } => true + AttemptLostRace { expected: _, observed: _ } => false + AttemptTransitionRefused { detail: _ } => false + } +} + +// ── WHAT MAKES RECOVERY DECIDE AN ATTEMPT IS DEAD ───────────────────────────────────────────── +// +// THE GATE ALONE IS NOT ENOUGH AND THIS IS THE GAP IT DOES NOT CLOSE. A compare-and-set decides who +// WINS a race; it says nothing about whether recovery should have entered the race at all. A +// producer that is alive and simply has not reached its launch CAS yet sits in Reserved -- exactly +// the state an impatient recovery would terminate -- and the CAS then makes that outcome look +// CORRECT rather than wrong: the producer's own CAS refuses, it releases, and the system appears to +// be working while a legitimate attempt was killed by a bystander (ruling, loyal-swift-608). +// +// THE TRIGGER THEREFORE SPLITS ON THE STATE, and the reason is structural rather than a policy +// choice: +// +// IN Reserved THERE IS NO UNIT TO OBSERVE, so observation is INCAPABLE of distinguishing a +// pre-launch producer from a dead one -- both present exactly the same absence. An observational +// trigger here would read "nothing there" as "dead", which is criterion 4's absorbing fallback +// with the stakes reversed. So the trigger is TEMPORAL, and the window is bounded BY THE +// ATTEMPT'S OWN DECLARATION: the producer writes launch_deadline when it opens, and recovery may +// not terminate a reserved attempt before it. The producer declares what it needs; recovery +// respects what was declared; neither guesses. +// +// IN Launching THE UNIT EXISTS, so the trigger is the five-arm observation and criterion 4 +// applies in full: only the three ENDED arms may terminate, and still-running or unknown keep the +// charge. An unreadable observation is never read as dead. +// +// WHAT BOUNDS THE PRE-LAUNCH WINDOW, stated because "bounded by construction" is worth nothing if +// nobody says by what: between opening and launching, the provider resolves a checkout and builds +// an argv. That is the whole window, it is the producer's own to declare, and a producer that +// exceeds its declared budget does not get to launch anyway -- its gate CAS expects the generation +// recovery has by then moved, so it is refused before it spawns. The deadline does not decide the +// winner; it decides who may enter. +type RecoveryTrigger + = RecoveryMayTerminate { reason: String } + | RecoveryTooEarly { deadline: Int, now: Int } + | RecoveryUnitNotEnded { lifecycle: UnitLifecycle } + | RecoveryStateNotRecoverable { state: String } + +fn recovery_trigger(record: AttemptRecord, lifecycle: UnitLifecycle, now: Int) -> RecoveryTrigger { + match record.state { + AttemptReserved => + if now < record.launch_deadline { + RecoveryTooEarly { deadline: record.launch_deadline, now: now } + } else { + RecoveryMayTerminate { + reason: join([ + "the attempt declared a launch deadline of ", to_string(value: record.launch_deadline), + " and it is now ", to_string(value: now), + ", so the window the producer asked for has passed without a launch", + ], ""), + } + } + AttemptLaunching => + if unit_lifecycle_ended(u: lifecycle) { + RecoveryMayTerminate { reason: join(["the unit is ", unit_lifecycle_wire(u: lifecycle)], "") } + } else { + RecoveryUnitNotEnded { lifecycle: lifecycle } + } + AttemptTerminal { verdict: v, detail: _ } => RecoveryStateNotRecoverable { state: join(["terminal:", v], "") } + } +} + +fn recovery_trigger_fires(t: RecoveryTrigger) -> Bool { + match t { + RecoveryMayTerminate { reason: _ } => true + RecoveryTooEarly { deadline: _, now: _ } => false + RecoveryUnitNotEnded { lifecycle: _ } => false + RecoveryStateNotRecoverable { state: _ } => false + } +} + +fn recovery_trigger_wire(t: RecoveryTrigger) -> String { + match t { + RecoveryMayTerminate { reason: r } => join(["may terminate: ", r], "") + RecoveryTooEarly { deadline: d, now: n } => + join(["TOO EARLY: the attempt declared a launch deadline of ", to_string(value: d), " and it is only ", to_string(value: n), "; a producer inside its own declared window is not dead"], "") + RecoveryUnitNotEnded { lifecycle: u } => join(["the unit has not ended: ", unit_lifecycle_wire(u: u)], "") + RecoveryStateNotRecoverable { state: s } => join(["the attempt is already ", s, ", so there is nothing to recover"], "") + } +} diff --git a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag new file mode 100644 index 00000000000..270bb319394 --- /dev/null +++ b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag @@ -0,0 +1,231 @@ +module test.claim.compute.attempt_lifecycle_wet_witness + +import extdeps.shell +import extdeps.filesystem.filesystem_io { Filesystem } +import std.types { Bool, Int, NonEmptyStr, String } +import std.algebra { trim } +import std.durable_compare_and_set { CasGeneration, cas_generation_count } +import gunbc.compute.attempt_lifecycle { + AttemptRecord, AttemptState, AttemptReserved, AttemptLaunching, AttemptTerminal, + AttemptSlotReading, AttemptSlotAbsent, AttemptSlotHeld, AttemptSlotUnreadable, attempt_slot_read, + AttemptTransition, AttemptAdvanced, AttemptLostRace, AttemptTransitionRefused, + attempt_open, attempt_begin_launch, attempt_terminate, attempt_transition_advanced, attempt_transition_wire, + attempt_state_wire, attempt_record_wire, attempt_record_decode, + UnitLifecycle, UnitTerminatedSuccess, UnitTerminatedFailure, UnitTerminatedCauseUnknown, + UnitStillRunning, UnitLifecycleUnknown, unit_lifecycle_ended, unit_lifecycle_wire, + RecoveryTrigger, RecoveryMayTerminate, RecoveryTooEarly, RecoveryUnitNotEnded, RecoveryStateNotRecoverable, + recovery_trigger, recovery_trigger_fires, recovery_trigger_wire, +} + +// WET CONTROLS FOR THE ATTEMPT LIFECYCLE. Every transition below is a real compare-and-set against +// a real generation-file store in a real temporary directory. The exclusion being asserted is the +// one the operating system performs on the write, not one this witness arranges. + +fn fresh_root() -> String { + shell.Mktemp.DirWithTemplate(template: "/tmp/gunbc_attempt.XXXXXX").path +} + +fn record_for(reference: String) -> AttemptRecord { + AttemptRecord { + identity: "abcdef0123456789", + reference: reference, + unit: join(["gunbc-compute-abcdef0123456789-", reference], ""), + granted_kib: 27262976, + launch_deadline: 2000, + state: AttemptReserved, + } +} + +fn generation_of(r: AttemptSlotReading) -> Int { + match r { + AttemptSlotHeld { record: _, generation: g } => cas_generation_count(g: g) + AttemptSlotAbsent => 0 + AttemptSlotUnreadable { detail: _ } => 0 - 1 + } +} + +fn state_word_of(r: AttemptSlotReading) -> String { + match r { + AttemptSlotHeld { record: rec, generation: _ } => attempt_state_wire(s: rec.state) + AttemptSlotAbsent => "" + AttemptSlotUnreadable { detail: _ } => "" + } +} + +fn held_record(r: AttemptSlotReading) -> AttemptRecord { + match r { + AttemptSlotHeld { record: rec, generation: _ } => rec + AttemptSlotAbsent => record_for(reference: "") + AttemptSlotUnreadable { detail: _ } => record_for(reference: "") + } +} + +fn lost_race(t: AttemptTransition) -> Bool { + match t { + AttemptLostRace { expected: _, observed: _ } => true + AttemptAdvanced { state: _, generation: _ } => false + AttemptTransitionRefused { detail: _ } => false + } +} + +// CRITERION 3, ORDER ONE: THE TERMINAL TRANSITION WINS AND THE LAUNCH IS REFUSED BEFORE IT SPAWNS. +// +// This is the interleaving that produces an unbacked launch if the two steps are not one: the +// attempt reserves, recovery decides it is dead and terminates it, and THEN the producer -- which +// has been paused somewhere between reserving and spawning -- goes to launch. With a gate, the +// producer's compare-and-set is refused against a generation that has moved, and it learns this +// BEFORE it spawns rather than after. The refusal is the store's, not a check this witness wrote. +test fn a_terminal_transition_refuses_a_launch_that_had_not_spawned_yet() -> Bool { + let root = fresh_root() as NonEmptyStr + let record = record_for(reference: "attempt-terminate-first") + let opened = attempt_open(root: root, record: record) + let after_open = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + let terminated = attempt_terminate(root: root, record: record, expected: generation_of(r: after_open), verdict: "recovered", detail: "the producer never launched") + let after_terminal = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + let late_launch = attempt_begin_launch(root: root, record: record, expected: generation_of(r: after_open)) + let final_read = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + attempt_transition_advanced(t: opened) + && generation_of(r: after_open) == 1 + && state_word_of(r: after_open) == "reserved" + && attempt_transition_advanced(t: terminated) + && state_word_of(r: after_terminal) == "terminal:recovered" + && lost_race(t: late_launch) + && state_word_of(r: final_read) == "terminal:recovered" + && generation_of(r: final_read) == 2 +} + +// CRITERION 3, ORDER TWO: THE LAUNCH WINS AND RECOVERY IS REFUSED. +// +// The mirror, and it is the half that keeps the gate from being a one-way wall: a producer that HAS +// advanced to launching owns the attempt, and a recovery arriving afterwards must not be able to +// declare it terminal behind its back -- that is how a live unit's memory gets handed away. Same +// mechanism, opposite winner, and the loser is again refused by precondition rather than by +// noticing. +test fn a_launch_that_won_refuses_a_recovery_arriving_behind_it() -> Bool { + let root = fresh_root() as NonEmptyStr + let record = record_for(reference: "attempt-launch-first") + let opened = attempt_open(root: root, record: record) + let after_open = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + let launched = attempt_begin_launch(root: root, record: record, expected: generation_of(r: after_open)) + let after_launch = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + let late_recovery = attempt_terminate(root: root, record: record, expected: generation_of(r: after_open), verdict: "recovered", detail: "arrived after the launch") + let final_read = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + attempt_transition_advanced(t: opened) + && attempt_transition_advanced(t: launched) + && state_word_of(r: after_launch) == "launching" + && lost_race(t: late_recovery) + && state_word_of(r: final_read) == "launching" + && generation_of(r: final_read) == 2 +} + +// CRITERION 2: THE BINDING IS READ, NOT RECONSTRUCTED. The unit name recovery would act on comes +// out of the record the attempt wrote, and it survives the round trip through the store bytes. A +// recovery deriving the name from the identity would be a second authority for the binding, and a +// wrong derivation stops somebody else's unit. +test fn the_unit_binding_is_read_back_from_the_record_the_attempt_wrote() -> Bool { + let root = fresh_root() as NonEmptyStr + let record = record_for(reference: "attempt-binding") + let opened = attempt_open(root: root, record: record) + let read_back = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + let recovered = held_record(r: read_back) + attempt_transition_advanced(t: opened) + && recovered.unit == record.unit + && recovered.identity == record.identity + && recovered.reference == record.reference + && recovered.granted_kib == 27262976 + && recovered.unit != record.identity +} + +// CRITERION 6: THE TERMINAL GENERATION CARRIES WHY, AND THE EARLIER GENERATIONS STILL EXIST. The +// store is generation-suffixed and append-only, so settling does not erase what it settled -- a +// reader can still see the attempt was reserved and launched before it ended. Recovery that +// overwrote its own evidence would be indistinguishable from an attempt that had never run. +test fn settling_records_why_and_does_not_erase_what_it_settled() -> Bool { + let root = fresh_root() as NonEmptyStr + let record = record_for(reference: "attempt-history") + let opened = attempt_open(root: root, record: record) + let g1 = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + let launched = attempt_begin_launch(root: root, record: record, expected: generation_of(r: g1)) + let g2 = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + let settled = attempt_terminate(root: root, record: record, expected: generation_of(r: g2), verdict: "released", detail: "unit terminated-failure, exit 101") + let g3 = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) + let terminal = held_record(r: g3) + attempt_transition_advanced(t: opened) && attempt_transition_advanced(t: launched) && attempt_transition_advanced(t: settled) + && generation_of(r: g1) == 1 && generation_of(r: g2) == 2 && generation_of(r: g3) == 3 + && (match terminal.state { AttemptTerminal { verdict: v, detail: d } => v == "released" && string_contains(s: d, pattern: "exit 101") AttemptReserved => false AttemptLaunching => false }) + && Filesystem.Read(path: join([root as String, "/", record.reference, ".1"], "")).success + && Filesystem.Read(path: join([root as String, "/", record.reference, ".2"], "")).success + && string_contains(s: Filesystem.Read(path: join([root as String, "/", record.reference, ".1"], "")).content, pattern: "reserved") + && string_contains(s: Filesystem.Read(path: join([root as String, "/", record.reference, ".2"], "")).content, pattern: "launching") +} + +// CRITERION 1: FOUR STATES, AND THE TWO THAT MUST NEVER FREE CAPACITY ARE SEPARATE ARMS CARRYING +// WHY. Still-running and unknown are not one "not ended" bucket: one waits, the other is +// investigated, and a reader told only "could not establish" has nothing to act on. +test fn the_union_keeps_every_arm_either_list_carried_and_each_refuses_as_before() -> Bool { + unit_lifecycle_ended(u: UnitTerminatedSuccess { detail: "exit 0" }) + && unit_lifecycle_ended(u: UnitTerminatedFailure { detail: "exit 101" }) + && unit_lifecycle_ended(u: UnitTerminatedCauseUnknown { detail: "LoadState=not-found" }) + && !unit_lifecycle_ended(u: UnitStillRunning { detail: "populated 1" }) + && !unit_lifecycle_ended(u: UnitLifecycleUnknown { detail: "the manager could not be read" }) + && string_contains(s: unit_lifecycle_wire(u: UnitTerminatedCauseUnknown { detail: "LoadState=not-found" }), pattern: "cause unknown") + && string_contains(s: unit_lifecycle_wire(u: UnitStillRunning { detail: "populated 1" }), pattern: "STILL RUNNING") + && string_contains(s: unit_lifecycle_wire(u: UnitLifecycleUnknown { detail: "the manager could not be read" }), pattern: "the manager could not be read") +} + +// THE RECOVERY TRIGGER, AND THE ROW THAT MATTERS IS THE FIRST ONE. A producer that is alive and has +// simply not reached its launch CAS yet sits in Reserved -- indistinguishable by OBSERVATION from a +// dead one, because in Reserved there is no unit to observe. So recovery must not be able to enter +// the race at all inside the window the producer declared; the gate decides who wins a race, and +// this decides who may start one. Without it an impatient recovery kills a live attempt and the CAS +// makes that outcome look correct. +test fn recovery_cannot_fire_inside_the_window_the_producer_declared() -> Bool { + let record = record_for(reference: "attempt-trigger") + let unobservable = UnitLifecycleUnknown { detail: "no unit exists yet, which is what pre-launch looks like" } + let early = recovery_trigger(record: record, lifecycle: unobservable, now: 1999) + let at_deadline = recovery_trigger(record: record, lifecycle: unobservable, now: 2000) + let late = recovery_trigger(record: record, lifecycle: unobservable, now: 2001) + !recovery_trigger_fires(t: early) + && recovery_trigger_fires(t: at_deadline) + && recovery_trigger_fires(t: late) + && (match early { RecoveryTooEarly { deadline: d, now: n } => d == 2000 && n == 1999 _ => false }) + && string_contains(s: recovery_trigger_wire(t: early), pattern: "not dead") +} + +// AND ONCE LAUNCHED THE TRIGGER IS THE OBSERVATION, WITH CRITERION 4 IN FULL. Only the three ended +// arms may terminate a launched attempt; still-running and unknown keep the charge, and unknown is +// the one that matters because it is what a lost launcher, an unreadable manager and a missing +// cgroup all look like. +test fn a_launched_attempt_is_recovered_only_on_an_ended_unit() -> Bool { + let launched = AttemptRecord { + identity: "abcdef0123456789", reference: "attempt-launched", + unit: "gunbc-compute-abcdef0123456789-attempt-launched", granted_kib: 27262976, + launch_deadline: 2000, state: AttemptLaunching, + } + let terminal = AttemptRecord { + identity: "abcdef0123456789", reference: "attempt-done", + unit: "gunbc-compute-abcdef0123456789-attempt-done", granted_kib: 27262976, + launch_deadline: 2000, state: AttemptTerminal { verdict: "released", detail: "done" }, + } + recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitTerminatedSuccess { detail: "exit 0" }, now: 9999)) + && recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitTerminatedFailure { detail: "exit 101" }, now: 9999)) + && recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitTerminatedCauseUnknown { detail: "LoadState=not-found" }, now: 9999)) + && !recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitStillRunning { detail: "populated 1" }, now: 9999)) + && !recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitLifecycleUnknown { detail: "the manager could not be read" }, now: 9999)) + && !recovery_trigger_fires(t: recovery_trigger(record: terminal, lifecycle: UnitTerminatedSuccess { detail: "exit 0" }, now: 9999)) + && string_contains(s: recovery_trigger_wire(t: recovery_trigger(record: terminal, lifecycle: UnitTerminatedSuccess { detail: "exit 0" }, now: 9999)), pattern: "nothing to recover") +} + +// CRITERION 4: A SLOT THAT CANNOT BE READ IS NOT AN EMPTY SLOT. The distinction has opposite +// remedies and opposite hazards -- an unreadable slot read as absent lets a producer open a second +// attempt over a live one, and lets recovery believe there is nothing to terminate. +test fn an_undecodable_slot_is_unreadable_and_never_absent() -> Bool { + let root = fresh_root() as NonEmptyStr + let reference = "attempt-corrupt" + let planted = Filesystem.Write(path: join([root as String, "/", reference, ".1"], ""), content: "this is not a json document at all") + let reading = attempt_slot_read(root: root, reference: reference as NonEmptyStr) + let absent_reading = attempt_slot_read(root: root, reference: "attempt-never-written" as NonEmptyStr) + planted.success + && (match reading { AttemptSlotUnreadable { detail: d } => string_contains(s: d, pattern: "cannot decode") AttemptSlotHeld { record: _, generation: _ } => false AttemptSlotAbsent => false }) + && (match absent_reading { AttemptSlotAbsent => true AttemptSlotHeld { record: _, generation: _ } => false AttemptSlotUnreadable { detail: _ } => false }) +} From a67ccbe69e600550fd79cbcd146bd8c9eaeddbe9 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 22:31:34 +0000 Subject: [PATCH 17/44] The self-declared window gets a ceiling that refuses, and the clock is named Both gaps closed before the provider is wired to the trigger rather than after, because both are about the thing that is about to become load-bearing. THE DEADLINE IS DECLARED BY THE PARTY IT PROTECTS. A producer says what it needs for checkout and argv assembly, and that same declaration is what stops anyone reclaiming its seat -- so nothing stopped it declaring an hour. The cost does not land on the producer: it lands on every other contender on the host as a seat nobody may reclaim, which is DESIGN 5's externalization exactly -- an accepted risk re-exported to another principal while the contract keeps its name. Ceiling: 300 seconds, and it is an AUTHORED POLICY BUDGET, not a measurement. What the window covers is one `git worktree add` of this repository plus argv assembly; the srv1 receipts show whole attempts including checkout at about four minutes, so five minutes for the checkout alone is generous by a wide margin and still bounded. When a measured checkout distribution exists this row is re-derived from it rather than defended. IT REFUSES, IT DOES NOT CLAMP. Clamping would admit the attempt under a bound it did not ask for and then reclaim its seat mid-checkout -- a widen wearing a safety limit's clothes. The refusal names both numbers so the author learns the ceiling instead of meeting it as a mysterious reclaim. A deadline that precedes the opening instant refuses too: a window already closed is reclaimable the moment it is written. WHICH CLOCK, AND WHY IT IS SAFE HERE. The host's own epoch clock, and for two reasons rather than convenience: the compute pool is host-local by construction -- PR 1 placed its ledger on the host precisely because a host's memory is not a fleet fact -- so both principals read one clock; and the pool's lease terms are already denominated in these same epoch seconds, so a second source here would be a second time authority for one quantity. The store offers no ordering to prefer: its generations order TRANSITIONS, not elapsed time. BOTH STEP DIRECTIONS NAMED, AND THE DANGEROUS-SOUNDING ONE IS BOUNDED BY THE GATE RATHER THAN THE CLOCK. Forward makes recovery fire late: the seat stays charged, the fail-closed direction, costing throughput. Backward makes it fire early against a live pre-launch producer -- and that does NOT produce an unbacked launch, because the producer's gate CAS then expects a generation recovery has moved and it is refused before it spawns. The loss is the attempt, not the invariant: a retry, not a seat handed to two holders. The deadline decides who may ENTER the race; the compare-and-set decides who wins it, and only the second is load-bearing for safety. Nine wet controls pass, including the ceiling refusing at one second over and admitting at exactly the ceiling. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/attempt_lifecycle.dag | 77 ++++++++++++++++++- .../attempt_lifecycle_wet_witness_test.dag | 45 ++++++++++- 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/compute/attempt_lifecycle.dag b/dag/gunbc/compute/attempt_lifecycle.dag index 71ac56e4d20..a92b97a2351 100644 --- a/dag/gunbc/compute/attempt_lifecycle.dag +++ b/dag/gunbc/compute/attempt_lifecycle.dag @@ -116,6 +116,7 @@ type AttemptRecord { reference: String unit: String granted_kib: Int + opened_at: Int launch_deadline: Int state: AttemptState } @@ -126,6 +127,7 @@ fn attempt_record_json(r: AttemptRecord) -> JsonValue { json_kv(key: "reference", value: json_string(s: r.reference)), json_kv(key: "unit", value: json_string(s: r.unit)), json_kv(key: "granted_kib", value: json_int(n: r.granted_kib)), + json_kv(key: "opened_at", value: json_int(n: r.opened_at)), json_kv(key: "launch_deadline", value: json_int(n: r.launch_deadline)), json_kv(key: "state", value: json_string(s: attempt_state_wire(s: r.state))), json_kv(key: "detail", value: json_string(s: match r.state { AttemptTerminal { verdict: _, detail: d } => d AttemptReserved => "" AttemptLaunching => "" })), @@ -137,7 +139,7 @@ fn attempt_record_wire(r: AttemptRecord) -> String { } fn attempt_record_with_state(r: AttemptRecord, state: AttemptState) -> AttemptRecord { - AttemptRecord { identity: r.identity, reference: r.reference, unit: r.unit, granted_kib: r.granted_kib, launch_deadline: r.launch_deadline, state: state } + AttemptRecord { identity: r.identity, reference: r.reference, unit: r.unit, granted_kib: r.granted_kib, opened_at: r.opened_at, launch_deadline: r.launch_deadline, state: state } } // ── READING THE SLOT, AND UNREADABLE IS NEVER ABSENT ────────────────────────────────────────── @@ -190,6 +192,9 @@ fn attempt_record_decode(text: String) -> AttemptRecord? { match attempt_member_int(doc: doc, key: "granted_kib") { Absent => none Present { value: granted } => + match attempt_member_int(doc: doc, key: "opened_at") { + Absent => none + Present { value: opened } => match attempt_member_int(doc: doc, key: "launch_deadline") { Absent => none Present { value: deadline } => @@ -199,10 +204,11 @@ fn attempt_record_decode(text: String) -> AttemptRecord? { match attempt_state_decode(word: word, detail: match attempt_member(doc: doc, key: "detail") { Present { value: d } => d Absent => "" }) { Absent => none Present { value: state } => - Present { value: AttemptRecord { identity: identity, reference: reference, unit: unit, granted_kib: granted, launch_deadline: deadline, state: state } } + Present { value: AttemptRecord { identity: identity, reference: reference, unit: unit, granted_kib: granted, opened_at: opened, launch_deadline: deadline, state: state } } } } } + } } } } @@ -280,11 +286,54 @@ fn attempt_commit(root: NonEmptyStr, record: AttemptRecord, expected: CasExpecta } } +// THE CEILING ON A SELF-DECLARED BOUND, AND WHY A DECLARATION IS NOT ENOUGH ON ITS OWN. +// +// The launch deadline is declared by the party it protects. A producer says what it needs for +// checkout and argv assembly, and that same declaration is what stops anyone reclaiming its seat -- +// so nothing in the mechanism as described stops a producer declaring an hour. The cost of an +// over-long declaration does not land on the producer: it lands on every other contender on that +// host, as a seat nobody may reclaim. That is the EXTERNALIZATION DESIGN 5 names -- an accepted +// risk quietly re-exported to another principal while the contract keeps its name -- and it is the +// reason a self-declared bound needs a bound of its own. +// +// FIVE MINUTES, AND IT IS A POLICY BUDGET RATHER THAN A MEASUREMENT, said plainly because the +// number is authored. What the window has to cover is one `git worktree add` of this repository and +// the assembly of an argv; the srv1 receipts on gunbc#11962 show whole build attempts, checkout +// included, completing in about four minutes, so a ceiling of five minutes for the checkout ALONE +// is generous by a wide margin and still bounded. It is not derived from a measured checkout +// distribution, and when one exists this row should be re-derived from it rather than defended. +// +// A DECLARATION ABOVE THE CEILING REFUSES AND IS NOT CLAMPED. Clamping would admit the attempt +// under a bound it did not ask for and did not agree to, then let it be reclaimed mid-checkout -- +// a widen wearing a safety limit's clothes. The refusal is typed and names both numbers, so the +// author of an over-long declaration learns what the ceiling is rather than discovering it as a +// mysterious reclaim. +data compute_pre_launch_ceiling_seconds: Int = 300 + // OPENING: the attempt declares itself Reserved, and it must be the FIRST writer of this reference. // ExpectSlotAbsent is what makes a duplicated reference a refusal rather than a silent overwrite of // somebody else's attempt. fn attempt_open(root: NonEmptyStr, record: AttemptRecord) -> AttemptTransition { - attempt_commit(root: root, record: attempt_record_with_state(r: record, state: AttemptReserved), expected: ExpectSlotAbsent) + let declared = record.launch_deadline - record.opened_at + if declared > compute_pre_launch_ceiling_seconds { + AttemptTransitionRefused { + detail: join([ + "the attempt declared a pre-launch window of ", to_string(value: declared), + " seconds and the ceiling is ", to_string(value: compute_pre_launch_ceiling_seconds), + "; a window nobody may reclaim a seat inside is a cost borne by every other contender on this host, so an over-long declaration is refused rather than clamped", + ], ""), + } + } else if declared < 0 { + AttemptTransitionRefused { + detail: join([ + "the attempt declared a launch deadline of ", to_string(value: record.launch_deadline), + " which precedes the instant it opened, ", to_string(value: record.opened_at), + "; a window that has already closed would be reclaimable the moment it is written", + ], ""), + } + } else { + attempt_commit(root: root, record: attempt_record_with_state(r: record, state: AttemptReserved), expected: ExpectSlotAbsent) + } } // THE LAUNCH GATE. Called BEFORE the unit is spawned, never after. @@ -329,6 +378,28 @@ fn attempt_transition_advanced(t: AttemptTransition) -> Bool { // THE TRIGGER THEREFORE SPLITS ON THE STATE, and the reason is structural rather than a policy // choice: // +// WHICH CLOCK, AND WHY THIS ONE IS SAFE HERE. The comparison is made by a process that did not +// write the deadline, so the clock is a shared authority between two principals. It is the host's +// own epoch clock (gunbc.fabric_event_log_host now_epoch_seconds), and it is the RIGHT one for two +// reasons rather than for convenience: the compute pool is host-local by construction -- PR 1 +// placed its ledger on the host precisely because a host's own memory is not a fleet fact -- so +// both principals read one clock; and the pool's own lease terms are already denominated in these +// same epoch seconds, so reaching for a second time source here would be a second time authority +// for one quantity. +// +// THE STORE OFFERS NO ORDERING TO PREFER OVER IT. The slot's generations order TRANSITIONS, not +// elapsed time, so there is nothing in the store that answers "has the window passed". +// +// BOTH STEP DIRECTIONS, NAMED, AND THE ONE THAT MATTERS IS BOUNDED BY THE GATE RATHER THAN BY THE +// CLOCK. A clock that steps FORWARD makes recovery fire late: the seat stays charged longer, which +// is the fail-closed direction and costs throughput. A clock that steps BACKWARD makes recovery +// fire early against a live pre-launch producer -- and that is the dangerous-sounding one, so it is +// worth being exact about what it actually costs. It does NOT produce an unbacked launch: the +// producer's gate CAS then expects a generation recovery has moved, so it is refused and never +// spawns. The loss is the attempt, not the invariant -- a retry, not a seat handed to two holders. +// The deadline decides who may ENTER the race; the compare-and-set decides who wins it, and only +// the second is load-bearing for safety. +// // IN Reserved THERE IS NO UNIT TO OBSERVE, so observation is INCAPABLE of distinguishing a // pre-launch producer from a dead one -- both present exactly the same absence. An observational // trigger here would read "nothing there" as "dead", which is criterion 4's absorbing fallback diff --git a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag index 270bb319394..721c5524ff4 100644 --- a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag +++ b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag @@ -10,7 +10,7 @@ import gunbc.compute.attempt_lifecycle { AttemptSlotReading, AttemptSlotAbsent, AttemptSlotHeld, AttemptSlotUnreadable, attempt_slot_read, AttemptTransition, AttemptAdvanced, AttemptLostRace, AttemptTransitionRefused, attempt_open, attempt_begin_launch, attempt_terminate, attempt_transition_advanced, attempt_transition_wire, - attempt_state_wire, attempt_record_wire, attempt_record_decode, + attempt_state_wire, attempt_record_wire, attempt_record_decode, compute_pre_launch_ceiling_seconds, UnitLifecycle, UnitTerminatedSuccess, UnitTerminatedFailure, UnitTerminatedCauseUnknown, UnitStillRunning, UnitLifecycleUnknown, unit_lifecycle_ended, unit_lifecycle_wire, RecoveryTrigger, RecoveryMayTerminate, RecoveryTooEarly, RecoveryUnitNotEnded, RecoveryStateNotRecoverable, @@ -31,6 +31,7 @@ fn record_for(reference: String) -> AttemptRecord { reference: reference, unit: join(["gunbc-compute-abcdef0123456789-", reference], ""), granted_kib: 27262976, + opened_at: 1800, launch_deadline: 2000, state: AttemptReserved, } @@ -200,12 +201,12 @@ test fn a_launched_attempt_is_recovered_only_on_an_ended_unit() -> Bool { let launched = AttemptRecord { identity: "abcdef0123456789", reference: "attempt-launched", unit: "gunbc-compute-abcdef0123456789-attempt-launched", granted_kib: 27262976, - launch_deadline: 2000, state: AttemptLaunching, + opened_at: 1800, launch_deadline: 2000, state: AttemptLaunching, } let terminal = AttemptRecord { identity: "abcdef0123456789", reference: "attempt-done", unit: "gunbc-compute-abcdef0123456789-attempt-done", granted_kib: 27262976, - launch_deadline: 2000, state: AttemptTerminal { verdict: "released", detail: "done" }, + opened_at: 1800, launch_deadline: 2000, state: AttemptTerminal { verdict: "released", detail: "done" }, } recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitTerminatedSuccess { detail: "exit 0" }, now: 9999)) && recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitTerminatedFailure { detail: "exit 101" }, now: 9999)) @@ -229,3 +230,41 @@ test fn an_undecodable_slot_is_unreadable_and_never_absent() -> Bool { && (match reading { AttemptSlotUnreadable { detail: d } => string_contains(s: d, pattern: "cannot decode") AttemptSlotHeld { record: _, generation: _ } => false AttemptSlotAbsent => false }) && (match absent_reading { AttemptSlotAbsent => true AttemptSlotHeld { record: _, generation: _ } => false AttemptSlotUnreadable { detail: _ } => false }) } + +// THE SELF-DECLARED BOUND HAS A BOUND, AND AN OVER-LONG DECLARATION REFUSES RATHER THAN BEING +// CLAMPED. The deadline is declared by the party it protects, so nothing in the gate alone stops a +// producer declaring an hour -- and the cost of that does not land on the producer, it lands on +// every other contender as a seat nobody may reclaim. That is DESIGN 5's externalization: an +// accepted risk re-exported to another principal while the contract keeps its name. +// +// CLAMPING WOULD BE THE WIDEN. Admitting the attempt under a bound it did not ask for, and then +// reclaiming its seat mid-checkout, is a safety limit's clothes on a silent narrowing of somebody +// else's window. It refuses, and the refusal names both numbers so the author learns the ceiling +// instead of meeting it as a mysterious reclaim. +test fn an_over_long_pre_launch_declaration_refuses_at_open_and_is_not_clamped() -> Bool { + let root = fresh_root() as NonEmptyStr + let base = record_for(reference: "attempt-greedy") + let greedy = AttemptRecord { + identity: base.identity, reference: base.reference, unit: base.unit, granted_kib: base.granted_kib, + opened_at: 1800, launch_deadline: 1800 + compute_pre_launch_ceiling_seconds + 1, state: AttemptReserved, + } + let at_ceiling = AttemptRecord { + identity: base.identity, reference: "attempt-at-ceiling", unit: base.unit, granted_kib: base.granted_kib, + opened_at: 1800, launch_deadline: 1800 + compute_pre_launch_ceiling_seconds, state: AttemptReserved, + } + let backwards = AttemptRecord { + identity: base.identity, reference: "attempt-backwards", unit: base.unit, granted_kib: base.granted_kib, + opened_at: 1800, launch_deadline: 1799, state: AttemptReserved, + } + let refused = attempt_open(root: root, record: greedy) + let admitted = attempt_open(root: root, record: at_ceiling) + let closed_window = attempt_open(root: root, record: backwards) + let greedy_slot = attempt_slot_read(root: root, reference: greedy.reference as NonEmptyStr) + !attempt_transition_advanced(t: refused) + && attempt_transition_advanced(t: admitted) + && !attempt_transition_advanced(t: closed_window) + && string_contains(s: attempt_transition_wire(t: refused), pattern: "every other contender") + && string_contains(s: attempt_transition_wire(t: refused), pattern: to_string(value: compute_pre_launch_ceiling_seconds)) + && string_contains(s: attempt_transition_wire(t: closed_window), pattern: "already closed") + && (match greedy_slot { AttemptSlotAbsent => true AttemptSlotHeld { record: _, generation: _ } => false AttemptSlotUnreadable { detail: _ } => false }) +} From 18b329d847c90cec871ab219b71e4cfd506e3653 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 22:43:28 +0000 Subject: [PATCH 18/44] Two unmigrated SeatRequest callers outside the gate, and the truncated grep that missed them review 69769. Generalizing SeatRequest to SeatRequest left gunbc.instruments.fabric_seat_probe (3 sites) and gunbc.instruments.fabric_event_log_probe (3 sites) passing a bare `amount: 1`. Both are operator-invoked entry points, so nothing in the required gate compiles them and the breakage would have stayed silent until somebody ran the probe -- which is exactly the case DESIGN 3 legislates for: "for a root outside the gate, enumerate the consumers by name before you delete". THE CAUSE IS THE ENUMERATION, NOT THE EDIT. I did enumerate the consumers, and then piped the enumeration through `head -12` and acted on what survived the truncation. A capped list is not a population; it is a list that has already agreed with whatever I was about to do. The re-run is unfiltered and uncapped: six construction sites, all six now carrying Measure { count: 1 }, and zero bare ones left anywhere in dag or src/v2. The PoolAcquired wire record in fabric_event_log_probe deliberately keeps its bare Nat, for the reason already recorded on that field: its dimension is the partition's and is restored at both ends from the pool's parameters. Verified: both instrument closures compile with 0 blocking errors, which is the check that did not exist for them before -- nothing routinely compiles these, so the compile is the enumeration's receipt rather than a formality. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/instruments/fabric_event_log_probe.dag | 6 +++--- dag/gunbc/instruments/fabric_seat_probe.dag | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/instruments/fabric_event_log_probe.dag b/dag/gunbc/instruments/fabric_event_log_probe.dag index a6fb6382359..b8cb28f25a6 100644 --- a/dag/gunbc/instruments/fabric_event_log_probe.dag +++ b/dag/gunbc/instruments/fabric_event_log_probe.dag @@ -91,14 +91,14 @@ fn fabric_event_log_probe_step(store: FabricStorageBinding, partition: Partition ChainBudgetExhausted { at: _ } => 6 ChainWalked { oldest_first: xs } => if !(length(xs) == 2 && (match first(xs) { Present { value: x } => (x.id as String) == (first_id as String) Absent => false })) { 6 } else { - match fabric_seat_acquire(store: store, partition: partition, root: probe_root_pool(), actor: probe_actor(), request: SeatRequest { reference: "turn-a" as NonEmptyStr, amount: 1, at: 2000, term_seconds: 300 }, policy: probe_policy(), attempts: 3, budget: 16) { + match fabric_seat_acquire(store: store, partition: partition, root: probe_root_pool(), actor: probe_actor(), request: SeatRequest { reference: "turn-a" as NonEmptyStr, amount: Measure { count: 1 }, at: 2000, term_seconds: 300 }, policy: probe_policy(), attempts: 3, budget: 16) { SeatFull { wire: _, attempts_used: _ } => 7 SeatContended { attempts: _ } => 7 SeatAcquireRefused { step: _, reason: _, attempts_used: _ } => 7 SeatStoreRefused { cause: _, attempts_used: _ } => 7 SeatGranted { grant: ga, generation: gen_a, attempts_used: _ } => if !(gen_a == 3 && ga.expires_at == 2300) { 7 } else { - match fabric_seat_acquire(store: store, partition: partition, root: probe_root_pool(), actor: probe_actor(), request: SeatRequest { reference: "turn-b" as NonEmptyStr, amount: 1, at: 2001, term_seconds: 300 }, policy: probe_policy(), attempts: 3, budget: 16) { + match fabric_seat_acquire(store: store, partition: partition, root: probe_root_pool(), actor: probe_actor(), request: SeatRequest { reference: "turn-b" as NonEmptyStr, amount: Measure { count: 1 }, at: 2001, term_seconds: 300 }, policy: probe_policy(), attempts: 3, budget: 16) { SeatGranted { grant: _, generation: _, attempts_used: _ } => 8 SeatContended { attempts: _ } => 8 SeatAcquireRefused { step: _, reason: _, attempts_used: _ } => 8 @@ -109,7 +109,7 @@ fn fabric_event_log_probe_step(store: FabricStorageBinding, partition: Partition EventAppendRefused { cause: _ } => 9 EventAppendStale { expected: _, observed: _ } => 9 EventAppended { id: _ } => - match fabric_seat_acquire(store: store, partition: partition, root: probe_root_pool(), actor: probe_actor(), request: SeatRequest { reference: "turn-c" as NonEmptyStr, amount: 1, at: 2003, term_seconds: 300 }, policy: probe_policy(), attempts: 3, budget: 16) { + match fabric_seat_acquire(store: store, partition: partition, root: probe_root_pool(), actor: probe_actor(), request: SeatRequest { reference: "turn-c" as NonEmptyStr, amount: Measure { count: 1 }, at: 2003, term_seconds: 300 }, policy: probe_policy(), attempts: 3, budget: 16) { SeatGranted { grant: _, generation: gen_c, attempts_used: _ } => if gen_c == 5 { 0 } else { 9 } SeatFull { wire: _, attempts_used: _ } => 9 SeatContended { attempts: _ } => 9 diff --git a/dag/gunbc/instruments/fabric_seat_probe.dag b/dag/gunbc/instruments/fabric_seat_probe.dag index 041fd8bbfc1..e010e35be4c 100644 --- a/dag/gunbc/instruments/fabric_seat_probe.dag +++ b/dag/gunbc/instruments/fabric_seat_probe.dag @@ -93,7 +93,7 @@ fn fabric_seat_collision_probe(endpoint: NonEmptyStr, partition: NonEmptyStr, re partition: (partition as String) as PartitionId, root: pool, actor: join(["collision-probe:", short], "") as NonEmptyStr, - request: SeatRequest { reference: join([short, "@", to_string(now)], "") as NonEmptyStr, amount: 1, at: now, term_seconds: 120 }, + request: SeatRequest { reference: join([short, "@", to_string(now)], "") as NonEmptyStr, amount: Measure { count: 1 }, at: now, term_seconds: 120 }, policy: LeasePolicy { maximum_duration_seconds: 120, release_law: QuiescenceRequired }, attempts: 3, budget: 256, @@ -154,7 +154,7 @@ fn fabric_seat_collision_probe_from(endpoint: NonEmptyStr, partition: NonEmptySt match harness_now_epoch() { Absent => exit_failure(reason: "collision-at: clock unreadable") Present { value: now } => - match propose_acquire(pool: folded, generation: g, head: head, partition: pid, actor: actor, request: SeatRequest { reference: join([short, "@", to_string(now)], "") as NonEmptyStr, amount: 1, at: now, term_seconds: 120 }) { + match propose_acquire(pool: folded, generation: g, head: head, partition: pid, actor: actor, request: SeatRequest { reference: join([short, "@", to_string(now)], "") as NonEmptyStr, amount: Measure { count: 1 }, at: now, term_seconds: 120 }) { SeatRefused { wire: w } => { let w1 = Filesystem.Write(path: receipt as String, content: join(["host=", short, " proposal refused before the race: ", w, "\n"], "")) ExitSuccess @@ -169,7 +169,7 @@ fn fabric_seat_collision_probe_from(endpoint: NonEmptyStr, partition: NonEmptySt } EventAppendStale { expected: _, observed: _ } => { let after = fabric_seat_acquire(store: store, partition: pid, root: pool, actor: actor, - request: SeatRequest { reference: join([short, "@", to_string(now), "-retry"], "") as NonEmptyStr, amount: 1, at: now, term_seconds: 120 }, + request: SeatRequest { reference: join([short, "@", to_string(now), "-retry"], "") as NonEmptyStr, amount: Measure { count: 1 }, at: now, term_seconds: 120 }, policy: LeasePolicy { maximum_duration_seconds: 120, release_law: QuiescenceRequired }, attempts: 3, budget: 256) let w3 = Filesystem.Write(path: receipt as String, content: join(["host=", short, " STALE on first push, then ", seat_acquisition_wire(a: after), "\n"], "")) ExitSuccess From ac364a13665d4a1f266a89f1b49254c9784c4d88 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 22:50:13 +0000 Subject: [PATCH 19/44] File the truncated-enumeration class, for the truncation and not for the probes Asked for by the owning manager after review 69769, and filed for the SHAPE rather than the instance: the invalid state is a census whose denominator was silently reduced by the reading tool and then used to justify a deletion or a re-signature. THE SHARP EDGE IS THAT IT DOES NOT FAIL RANDOMLY. A cap fails toward the consumers nothing else would catch: a module with few mentions sorts to the bottom and is cut first, and having few mentions is exactly what makes it unlikely to be caught by any other route. The arms the cap hides are POSITIVELY CORRELATED with the arms that matter. WHY IT SURVIVES REVIEW: incompleteness is a property of the READING, not of the diff. Every migrated site is correct, the typechecker is silent because the missed consumers are outside the gate's closure, and the enumeration itself is not an artifact anybody reviews -- it happened in a shell and left no trace. Re-running it uncapped is the only thing that finds it. DISCRIMINATOR, phrased as a question about the enumeration rather than the diff: was the enumeration that justified this change itself UNBOUNDED? Not "did you check the consumers" -- the author of the specimen did check them. WHY IT IS NOT truncated_local_diagnostics_indistinguishable_from_the_complete_set, stated because the two look alike and the manager asked me to grow a row rather than split a class: there an ORACLE DIED mid-measure and nobody chose the truncation, so the discriminators are out of band -- an abort line, a missing receipt, a stack limit. Here the truncation is CHOSEN, by the reader, as a display convenience, on a population the reader is about to act on; there is no abort to notice and no receipt to be missing. Same silence, different cause, different discriminator, different remedy. TRIGGER NAMES THE CAPABILITY: a change that alters or removes a declaration is checked against the declaration's full consumer set derived from the namespace tree, consumers outside the required gate included by construction. Deliberately not "do not use head" -- a habit is not a wall, and the class would stay exactly as alive under a different paging tool. Filed on the lifecycle branch rather than PR 1, to keep PR 1's head still. Co-Authored-By: Claude Opus 5 (1M context) --- ...enumeration_capped_by_its_reading_tool.dag | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 dag/gunbc/recurring_failure_mode/an_enumeration_capped_by_its_reading_tool.dag diff --git a/dag/gunbc/recurring_failure_mode/an_enumeration_capped_by_its_reading_tool.dag b/dag/gunbc/recurring_failure_mode/an_enumeration_capped_by_its_reading_tool.dag new file mode 100644 index 00000000000..fde713dabef --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/an_enumeration_capped_by_its_reading_tool.dag @@ -0,0 +1,39 @@ +module gunbc.recurring_failure_mode.an_enumeration_capped_by_its_reading_tool + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, decl_ref } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data an_enumeration_capped_by_its_reading_tool: RecurringFailureMode = RecurringFailureMode { + identity: "an_enumeration_capped_by_its_reading_tool" as NonEmptyStr, + + receipts: [ + "INVALID STATE: a census whose denominator was silently reduced by the READING TOOL, and then used as the justification for a deletion or a re-signature. The author enumerates the consumers -- correctly -- and pipes the enumeration through a display cap (`head -n`, a `take`, a paged view, a truncating summary). Every line that survives is true. Nothing in the output says a line did not survive. The edit is then made against the survivors and is complete with respect to a population that was never the population.", + + "HARM, AND THE SHARP EDGE IS THAT IT DOES NOT FAIL RANDOMLY. The cap fails TOWARD THE CONSUMERS NOTHING ELSE WOULD CATCH. A module with few mentions of the symbol sorts to the bottom of the enumeration and is cut first -- and having few mentions is exactly what makes it unlikely to be caught by any other route. So the arms the cap hides are CORRELATED with the arms that matter, and the correlation is positive: the less a consumer is referenced, the more likely both that the cap hides it and that no gate compiles it.", + + "WHY IT SURVIVES REVIEW, WHICH IS WHAT MAKES IT A CLASS RATHER THAN A SLIP: incompleteness is a property of the READING, not of the diff. The diff is internally consistent, every migrated site is correct, the typechecker is silent because the missed consumers are outside whatever closure the gate compiles, and the enumeration itself is not an artifact anybody reviews -- it happened in a shell and left no trace. A reviewer re-running the enumeration UNCAPPED is the only thing that finds it.", + + "SPECIMEN (gunbc#11962, review 69769, 2026-09-21). product.capacity.pool_events SeatRequest was re-signatured to SeatRequest. The consumers WERE enumerated before the change -- and the enumeration was piped through `head -12`. Six construction sites exist; the cap showed the first four files and hid gunbc.instruments.fabric_seat_probe and gunbc.instruments.fabric_event_log_probe, which are operator-invoked entry points that NO required lane compiles. The breakage would have surfaced the next time an operator ran a probe, with a type error in a module the author had never opened. The re-run, unfiltered and uncapped, found all six.", + + "THE PLACE IT LANDED IS THE PLACE DESIGN 3 SAYS ENUMERATION IS LOAD-BEARING: 'Outside the required gate the substrate still refuses and nothing asks it to ... so for a root outside the gate, enumerate the consumers by name before you delete.' The rule exists precisely because the compiler is not the backstop there. A capped enumeration removes the only backstop that remained.", + + "THIS IS NOT truncated_local_diagnostics_indistinguishable_from_the_complete_set, and the boundary is worth stating because the two look alike. There, an ORACLE DIED mid-measure and its partial output read as a verdict -- nobody chose the truncation, and the discriminators are out of band (the abort line, the missing receipt, the stack limit). Here the truncation is CHOSEN, by the reader, as a display convenience, on a population the reader is about to act on; there is no abort to notice and no receipt to be missing. Same silence, different cause, different discriminator, different remedy.", + + "DISCRIMINATOR, and it is a question about the ENUMERATION rather than about the diff: was the enumeration that justified this deletion or re-signature itself UNBOUNDED? Not 'did you check the consumers' -- the author of this specimen did check them. Whether the check could have shown all of them.", + + "RUNG FOUND AT: mitigatable, by the discipline of not capping a census and by a reviewer re-running it. Nothing refuses a capped enumeration because nothing sees one.", + + "CEILING: mechanically preventable. A re-signature or deletion of a declaration can have its consumer set computed from the namespace tree -- the same Node-tree read DESIGN 3 already relies on for citation checking -- and compared against what the change touched, so an unmigrated consumer outside the gate is a refusal rather than a surprise. DESIGN 3c names the same climb for dangling declarations: the consumer set is a fact the tree carries, not one a shell has to be trusted to report.", + + "NEXT-RUNG TRIGGER, NAMING THE CAPABILITY: a change that alters a declaration's signature or removes it is checked against the declaration's full consumer set as derived from the namespace tree, with consumers outside the required gate included by construction. Not 'do not use head' -- a habit is not a wall, and the class would stay exactly as alive under a different paging tool.", + + "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration.", + ], + + evidence: [ + decl_ref(module_path: "product.capacity.pool_events", decl_name: "SeatRequest"), + decl_ref(module_path: "gunbc.instruments.fabric_seat_probe", decl_name: "fabric_seat_probe"), + decl_ref(module_path: "gunbc.instruments.fabric_event_log_probe", decl_name: "probe_root_pool"), + ], +} From 09b62f9c40d19e0cff2186b140451c2107b7eb96 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 23:34:26 +0000 Subject: [PATCH 20/44] /proc/meminfo is not uniformly kB, and the witness asserted that it was review 69784. meminfo_line_metric took token 0 as the key and token 1 as the count and DISCARDED token 2 -- the `kB` the module's own annotation leans on. /proc/meminfo is not uniformly kB: four lines on every Linux host (HugePages_Total, HugePages_Free, HugePages_Rsvd, HugePages_Surp) carry NO unit because they are counts of PAGES, verified on this machine -- 54 lines, 4 without a unit, exactly the four named. Minting those as Kibibyte is wrong by the huge page size, 2048x at 2 MiB, and nothing in the type says so: a MemoryMetric carrying 0 pages and one carrying 0 KiB are indistinguishable. The seven named fields are all genuinely kB, so the admission arithmetic was never affected. What was wrong is the surface all_metrics and meminfo_metric_named expose to the next consumer. AND THE WITNESS LOCKED IT IN: count(all_metrics) == 8 against a fixture whose eighth line is `HugePages_Total: 0` -- it asserted the non-kB line WAS admitted. That is the second control in this change to pin the behaviour it existed to refuse, after the one that asserted completed+unknown frees a seat. Both were mine and both were written the same way: state what the code does, then assert it. THE UNIT IS NOW REQUIRED AND COMPARED, not discarded, and all_metrics is named for what it is -- the kB-denominated metrics, not every line. A page count is a different quantity and would need its own carrier; that is a row to add when a consumer exists, not a field to fake now. TWO FIXTURES BECAUSE TWO DIFFERENT PARTS OF THE FOLD EXCLUDE THEM, and finding the second is the reason the mutation was worth running. A UNITLESS line is excluded by the arity requirement. A line with the WRONG unit is excluded only by the comparison -- and nothing in /proc/meminfo authors that shape, so without a fixture the comparison was a permanently green decoration. Mutating the comparison away left the cell GREEN; with `SomeFutureField: 64 MB` added it goes FAIL. Restored and green again. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/linux/proc_meminfo.dag | 66 ++++++++++++++----- .../claim/os_proc_meminfo_witness_test.dag | 40 ++++++++++- 2 files changed, 89 insertions(+), 17 deletions(-) diff --git a/dag/extdeps/linux/proc_meminfo.dag b/dag/extdeps/linux/proc_meminfo.dag index abca8818e29..f513ff3e962 100644 --- a/dag/extdeps/linux/proc_meminfo.dag +++ b/dag/extdeps/linux/proc_meminfo.dag @@ -54,25 +54,59 @@ type MeminfoRead = MeminfoParsed { meminfo: ProcMeminfo } | MeminfoFieldAbsent { field: NonEmptyStr } +// THE UNIT IS THE DISCRIMINATOR AND IT IS PRESENT IN THE TEXT, so throwing it away and minting a +// Kibibyte for every line is the one thing this fold must not do (review 69784). +// +// /proc/meminfo IS NOT UNIFORMLY kB. Most lines are `Name: N kB`, and four on every Linux host -- +// HugePages_Total, HugePages_Free, HugePages_Rsvd, HugePages_Surp -- are `Name: N` with NO unit, +// because they are counts of PAGES. Reading a page count as a kibibyte value is wrong by the huge +// page size, 2048x at the usual 2 MiB, and nothing in the resulting type says so: a MemoryMetric +// carrying 0 pages and a MemoryMetric carrying 0 KiB are indistinguishable, and one carrying 3 +// pages claims 3 KiB. That is DESIGN 5's fabricated plausible output in the small -- syntactically +// fine, semantically not the quantity it claims -- and DESIGN 3's instruction for extdeps is to +// model what the API actually returns rather than what would be convenient. +// +// SO A LINE BECOMES A MemoryMetric ONLY IF IT SAYS kB. The third token is required and compared, +// not discarded. A line without it is not a memory measure and has no business in a +// List; excluding it is the type being honest rather than data being lost. +// +// WHAT THIS MEANS FOR all_metrics, said because the name overpromises on its own: it is the +// kB-DENOMINATED metrics, not every line of the file. The unitless lines are unmodelled here and +// would need their own carrier -- a page count is a different quantity, not a memory one -- which +// is a row to add when a consumer exists (DESIGN 3c) rather than a field to fake now. +data meminfo_kibibyte_unit: String = "kB" + fn meminfo_line_metric(line: String) -> MemoryMetric? { let tokens = filter(split(s: line, delimiter: " "), t => trim(s: t) != "") - match first(tokens) { - Absent => none - Present { value: head } => - if !ends_with(s: head, suffix: ":") { - none - } else { - let key = substring(s: head, start: 0, end: length(head) - 1) - match first(tokens |> skip(n: 1)) { - Absent => none - Present { value: raw } => - match parse_int(s: trim(s: raw)) { - Absent => none - Present { value: n } => - if key == "" || n < 0 { none } else { Present { value: MemoryMetric { key: key as NonEmptyStr, value: kibibyte(count: n) } } } - } + if count(tokens) != 3 { + none + } else { + match first(tokens) { + Absent => none + Present { value: head } => + if !ends_with(s: head, suffix: ":") { + none + } else { + let key = substring(s: head, start: 0, end: length(head) - 1) + match first(tokens |> skip(n: 1)) { + Absent => none + Present { value: raw } => + match first(tokens |> skip(n: 2)) { + Absent => none + Present { value: unit } => + if trim(s: unit) != meminfo_kibibyte_unit { + none + } else { + match parse_int(s: trim(s: raw)) { + Absent => none + Present { value: n } => + if key == "" || n < 0 { none } else { Present { value: MemoryMetric { key: key as NonEmptyStr, value: kibibyte(count: n) } } } + } + } + } + } } - } + } } } diff --git a/dag/test/claim/os_proc_meminfo_witness_test.dag b/dag/test/claim/os_proc_meminfo_witness_test.dag index debf04c5405..cee634e8adc 100644 --- a/dag/test/claim/os_proc_meminfo_witness_test.dag +++ b/dag/test/claim/os_proc_meminfo_witness_test.dag @@ -31,6 +31,21 @@ test fn proc_meminfo_witnesses() -> Bool { // not a decimal count may not contribute a metric. Both are asserted against the same text with one // line changed, so the assertion discriminates the parse rather than the fixture. // +// TWO DIFFERENT NON-kB SHAPES, BECAUSE THEY ARE CAUGHT BY TWO DIFFERENT PARTS OF THE FOLD AND ONE +// OF THEM HAS NO OTHER WITNESS. A UNITLESS line (`HugePages_Total: 0`) is excluded by the arity +// requirement; a line with the WRONG unit (`SomeFutureField: 64 MB`) is excluded only by the unit +// COMPARISON, and nothing in /proc/meminfo authors that shape today -- so without the fixture the +// comparison would be a permanently green decoration (DESIGN 4b: ask whether the RED is authorable +// before writing the check, and a fixture is where it is authorable). Found by mutating the +// comparison away and watching this cell stay GREEN. +// +// THE UNITLESS LINE IS EXCLUDED, AND THE PREVIOUS REVISION OF THIS CELL ASSERTED THE OPPOSITE. It +// required count(all_metrics) == 8 against a fixture whose eighth line is `HugePages_Total: 0` -- +// so it pinned the defect it should have caught, which is the second time in this change a control +// of mine locked in the behaviour it existed to refuse (review 69784). A MemoryMetric carrying a +// page count is wrong by the huge page size and indistinguishable in the type from a real reading; +// the seven kB-denominated lines are admitted and the unitless one is not. +// // EVERY FIELD CARRIES A DISTINCT VALUE AND EVERY FIELD IS ASSERTED, because with seven lookups // written by hand a name paired with the WRONG field is the one defect the absence arms cannot // catch. And a field missing from the MIDDLE of the file names itself: that arm used to be @@ -49,7 +64,7 @@ test fn parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one() -> Bo && kibibyte_count(k: m.cached) == 4194304 && kibibyte_count(k: m.swap_total) == 2097152 && kibibyte_count(k: m.swap_free) == 524288 - && count(m.all_metrics) == 8 + && count(m.all_metrics) == 7 MeminfoFieldAbsent { field: _ } => false }) && (match parse_proc_meminfo(text: without_available) { @@ -64,6 +79,29 @@ test fn parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one() -> Bo MeminfoFieldAbsent { field: f } => (f as String) == "MemTotal" MeminfoParsed { meminfo: _ } => false }) + && (match parse_proc_meminfo(text: text) { + MeminfoParsed { meminfo: m } => + count(filter(m.all_metrics, x => (x.key as String) == "HugePages_Total")) == 0 + && count(filter(m.all_metrics, x => (x.key as String) == "SwapFree")) == 1 + MeminfoFieldAbsent { field: _ } => false + }) + && (match parse_proc_meminfo(text: join([ + "MemTotal: 16777216 kB\n", "MemFree: 1048576 kB\n", "MemAvailable: 8388608 kB\n", + "Buffers: 65536 kB\n", "Cached: 4194304 kB\n", "SwapTotal: 2097152 kB\n", + "SwapFree: 524288 kB\n", "SomeFutureField: 64 MB\n", + ], "")) { + MeminfoParsed { meminfo: m } => + count(m.all_metrics) == 7 && count(filter(m.all_metrics, x => (x.key as String) == "SomeFutureField")) == 0 + MeminfoFieldAbsent { field: _ } => false + }) + && (match parse_proc_meminfo(text: join([ + "MemTotal: 16777216 kB\n", "MemFree: 1048576 kB\n", "MemAvailable: 8388608 kB\n", + "Buffers: 65536 kB\n", "Cached: 4194304 kB\n", "SwapTotal: 2097152 kB\n", + "SwapFree: 524288\n", + ], "")) { + MeminfoFieldAbsent { field: f } => (f as String) == "SwapFree" + MeminfoParsed { meminfo: _ } => false + }) && (match parse_proc_meminfo(text: join([ "MemTotal: 16777216 kB\n", "MemFree: 1048576 kB\n", "MemAvailable: 8388608 kB\n", "Buffers: 65536 kB\n", "Cached: 4194304 kB\n", "SwapFree: 524288 kB\n", From ce54fa4836de6a4e578347b56c9b8128fcbd9f6b Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 23:45:17 +0000 Subject: [PATCH 21/44] The cited witness module is named with its _test suffix, unlike its siblings Side-chat verdict on #11962: one cited-module-absent, reported by the floor's declaration phase. a_held_compute_seat_has_no_in_corpus_discharge cited "test.claim.compute.work_request_witness"; the module declares itself "test.claim.compute.work_request_witness_test". The declaration it names is real and is in that module, so this is the string and nothing else. THE TRAP IS A NAMING INCONSISTENCY AMONG SIBLINGS, worth recording because the next citation will meet it too: host_capacity_wet_witness_test.dag declares `module test.claim.compute.host_capacity_wet_witness` and os_proc_meminfo_witness_test.dag declares `module test.claim.os_proc_meminfo_witness` -- both DROP the _test -- while work_request_witness_test.dag KEEPS it. A citation written from the pattern rather than from the file is wrong for exactly one file in the directory. So I audited rather than fixed the one: every module_path and decl_name in both rows this change adds, resolved against the real `module` lines and declarations. All resolve. It predates the approved baseline bf58075f -- no delta commit introduced it -- but it is a live declaration refusal inside this package, so it is fixed here rather than deferred. The census-image wet refusal is disposed nonblocking by the owning manager and is not touched. Co-Authored-By: Claude Opus 5 (1M context) --- .../a_held_compute_seat_has_no_in_corpus_discharge.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag index b137018ba21..7abfa38c7ee 100644 --- a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag +++ b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag @@ -31,6 +31,6 @@ data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = Recu decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "compute_stuck_reservation_remedy"), decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "unit_termination_frees_capacity"), decl_ref(module_path: "gunbc.compute.host_capacity", decl_name: "compute_release"), - decl_ref(module_path: "test.claim.compute.work_request_witness", decl_name: "only_proven_absence_or_an_ended_cgroup_frees_a_seat"), + decl_ref(module_path: "test.claim.compute.work_request_witness_test", decl_name: "only_proven_absence_or_an_ended_cgroup_frees_a_seat"), ], } From d3ab45b7d539ae15c422410d13424acce22b321e Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 23:52:45 +0000 Subject: [PATCH 22/44] File the shared-derivation class, and sharpen the enumeration rule Two findings arrived at independently by two sessions within an hour, which is what makes the class worth a row rather than two apologies. a_control_that_shares_its_derivation_with_its_subject: a control whose content is DERIVED FROM the thing it is meant to discriminate cannot disagree with it. Two routes, and neither author recognised the other's until they were compared -- DESCRIBING (read what the code does, assert that) and COPYING (inline the production body with one input swapped). Copying is just the most literal way of writing a control from the implementation. Specimen one is mine, twice in one change: a meminfo witness that asserted the non-kB line WAS admitted, and a termination control that asserted completed-plus-unknown frees a seat. Specimen two is loyal-swift-608's on gunbc#12017: a RED carrying the production door's body inlined, so it evaluated its own copy of the test it existed to guard, with an annotation above it asserting the opposite in as many words. THE SMELL THAT NEEDS NO RUN, which is their contribution and is cheaper than the mutation: ASK WHAT THE CONTROL SHARES WITH ITS SUBJECT. If it re-derives, re-implements or re-describes the thing under test it is suspect before any mutation is attempted. The mutation is the proof; the shared-derivation question is what says to go looking. AND THE REPAIR HAS ITS OWN FAILURE MODE, which is the more transferable half: the first fix of specimen one added the missing unit comparison, and mutating that comparison away left the control STILL GREEN, because a unitless line is excluded by the ARITY requirement and /proc/meminfo publishes no wrong-unit line at all. THE FIXTURE POPULATION HAS TO CONTAIN THE CASE THE CHECK EXISTS FOR -- and a real-world corpus is exactly where that case tends not to occur, which is why the fixture is where it must be authored. Boundary against predicate_vacuously_true_on_an_empty_domain stated in the row: there the domain is empty, here it is populated and healthy-looking and merely lacks the one case. Also into an_enumeration_capped_by_its_reading_tool, a statement of the rule sharper than either the manager's or mine (reviewer on gunbc#11962): PRESERVE THE COMPLETE CONSUMER SET AS THE MIGRATION INPUT, AND CHECK EVERY AFFECTED OUT-OF-GATE ROOT FROM THAT SET; TRUNCATE ONLY ITS DISPLAY. That splits the legitimate cap from the illegitimate one, which neither "never truncate" nor "do not use head" manages: a cap on what you LOOK AT is how anyone reads a long list; a cap on what you ACT FROM is the defect. Co-Authored-By: Claude Opus 5 (1M context) --- ...shares_its_derivation_with_its_subject.dag | 39 +++++++++++++++++++ ...enumeration_capped_by_its_reading_tool.dag | 2 + 2 files changed, 41 insertions(+) create mode 100644 dag/gunbc/recurring_failure_mode/a_control_that_shares_its_derivation_with_its_subject.dag diff --git a/dag/gunbc/recurring_failure_mode/a_control_that_shares_its_derivation_with_its_subject.dag b/dag/gunbc/recurring_failure_mode/a_control_that_shares_its_derivation_with_its_subject.dag new file mode 100644 index 00000000000..ca61c314f95 --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/a_control_that_shares_its_derivation_with_its_subject.dag @@ -0,0 +1,39 @@ +module gunbc.recurring_failure_mode.a_control_that_shares_its_derivation_with_its_subject + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, decl_ref } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data a_control_that_shares_its_derivation_with_its_subject: RecurringFailureMode = RecurringFailureMode { + identity: "a_control_that_shares_its_derivation_with_its_subject" as NonEmptyStr, + + receipts: [ + "INVALID STATE: a control whose content is DERIVED FROM the thing it is meant to discriminate, so it cannot disagree with it. The general form is one sentence -- A CONTROL WRITTEN FROM THE IMPLEMENTATION CAN ONLY AGREE WITH IT -- and the two routes to it look different enough that neither author recognised the other's until they were put side by side. DESCRIBING: read what the code does, write the assertion that says so. COPYING: inline the production body into the control with one input swapped. Copying is simply the most literal way of writing a control from the implementation.", + + "HARM: the suite reports coverage where it has none, and reports it most confidently exactly where the subject is most load-bearing, because a heavily-relied-on fold is the one an author is most likely to describe accurately rather than interrogate. The mutation that removes the wall leaves the control green, so the wall's absence is invisible to every later reader -- and the control is then CITED as the reason the wall is safe to lean on.", + + "SPECIMEN ONE, DESCRIBING (gunbc#11962, review 69784, 2026-09-21). A witness over the /proc/meminfo parse asserted count(all_metrics) == 8 against a fixture whose eighth line is `HugePages_Total: 0` -- a unitless page count the parse was minting as a Kibibyte. The control did not catch the defect; it RECORDED it, because it was written by reading the fold and stating the result. The same change carried a second instance: a termination control that asserted completed-plus-unknown frees a seat, which was the defect the four-state predicate existed to remove.", + + "SPECIMEN TWO, COPYING (gunbc#12017, review 69791, 2026-09-21, session loyal-swift-608). a_refused_identity_refuses_the_store -- a RED written to satisfy a previous review -- carried the production door's body INLINED with only the verdict source swapped, so it evaluated its own copy of the green test it existed to guard. Deleting that test from the real door would have left the control green. The annotation above it asserted the opposite in as many words.", + + "THE SMELL THAT NEEDS NO RUN, and it is what makes this cheaper to catch than to prove: ASK WHAT THE CONTROL SHARES WITH ITS SUBJECT. If it re-derives, re-implements or re-describes the thing under test, it is suspect before any mutation is attempted. Specimen two shared a copied body; specimen one shared a description. The mutation is the proof; the shared-derivation question is the smell that says to go looking.", + + "AND THE REPAIR HAS ITS OWN FAILURE MODE, WHICH IS THE MORE TRANSFERABLE HALF. The first fix of specimen one added the missing unit comparison -- and mutating that comparison away left the control STILL GREEN. A unitless line is excluded by the ARITY requirement rather than by the comparison, and /proc/meminfo publishes no wrong-unit line at all, so the comparison had no instance in the fixture population to discriminate and was a permanently green decoration (DESIGN 4b: worse than absent, because it is cited as coverage). THE FIXTURE POPULATION HAS TO CONTAIN THE CASE THE CHECK EXISTS FOR -- and a REAL-WORLD CORPUS IS EXACTLY WHERE THAT CASE TENDS NOT TO OCCUR, which is why the fixture is where it must be authored. Adding `SomeFutureField: 64 MB` is what made the mutation fail.", + + "NOT predicate_vacuously_true_on_an_empty_domain, and the boundary matters because the remedy differs. There the domain is EMPTY and the check answers true about nothing. Here the domain is POPULATED and healthy-looking -- seven real metrics -- and merely lacks the one case the check discriminates. An emptiness guard does not catch it; only asking what case this check exists for, and whether the fixture contains it, does.", + + "RUNG FOUND AT: mitigatable, by the two habits above -- run the mutation before reporting a control, and ask what the control shares with its subject. Nothing refuses a control that agrees with its subject, because agreement is what a passing control looks like.", + + "CEILING: mechanically preventable. A mutation pass that rebinds each control's subject to a wrong arm and requires the control to red is decidable over the corpus, and a control that survives every mutation of its own subject is reportable as establishing nothing.", + + "NEXT-RUNG TRIGGER, NAMING THE CAPABILITY: every control is executed against a mutated subject and must red, with a control that survives all mutations of its subject reported as non-discriminating. Deliberately not 'review controls for copied bodies' -- specimen one had no copied body and would pass that review unchanged.", + + "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration.", + ], + + evidence: [ + decl_ref(module_path: "extdeps.linux.proc_meminfo", decl_name: "meminfo_line_metric"), + decl_ref(module_path: "test.claim.os_proc_meminfo_witness", decl_name: "parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one"), + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "unit_termination_decide"), + ], +} diff --git a/dag/gunbc/recurring_failure_mode/an_enumeration_capped_by_its_reading_tool.dag b/dag/gunbc/recurring_failure_mode/an_enumeration_capped_by_its_reading_tool.dag index fde713dabef..971c6e0b9ee 100644 --- a/dag/gunbc/recurring_failure_mode/an_enumeration_capped_by_its_reading_tool.dag +++ b/dag/gunbc/recurring_failure_mode/an_enumeration_capped_by_its_reading_tool.dag @@ -20,6 +20,8 @@ data an_enumeration_capped_by_its_reading_tool: RecurringFailureMode = Recurring "THIS IS NOT truncated_local_diagnostics_indistinguishable_from_the_complete_set, and the boundary is worth stating because the two look alike. There, an ORACLE DIED mid-measure and its partial output read as a verdict -- nobody chose the truncation, and the discriminators are out of band (the abort line, the missing receipt, the stack limit). Here the truncation is CHOSEN, by the reader, as a display convenience, on a population the reader is about to act on; there is no abort to notice and no receipt to be missing. Same silence, different cause, different discriminator, different remedy.", + "THE RULE, SHARPER THAN EITHER STATEMENT THAT PRODUCED IT (reviewer on gunbc#11962, endorsed by the owning manager): PRESERVE THE COMPLETE CONSUMER SET AS THE MIGRATION INPUT, AND CHECK EVERY AFFECTED OUT-OF-GATE ROOT FROM THAT SET; TRUNCATE ONLY ITS DISPLAY. That splits the legitimate use of a cap from the illegitimate one cleanly, which neither `never truncate` nor `do not use head` manages: a cap on what you LOOK AT is fine and is how anyone reads a long list, and a cap on what you ACT FROM is the defect. The input and the view are two things, and the failure is using one as the other.", + "DISCRIMINATOR, and it is a question about the ENUMERATION rather than about the diff: was the enumeration that justified this deletion or re-signature itself UNBOUNDED? Not 'did you check the consumers' -- the author of this specimen did check them. Whether the check could have shown all of them.", "RUNG FOUND AT: mitigatable, by the discipline of not capping a census and by a reviewer re-running it. Nothing refuses a capped enumeration because nothing sees one.", From c3df024bef93f8e22744cb6145d33aff744cd16a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 00:35:57 +0000 Subject: [PATCH 23/44] Criteria 5 and 7: the attach decision moves inside the lease, and three lease-drop contracts become one (5) RE-CHECK UNDER THE LEASE, and the instance is concrete rather than ceremonial. The stored-outcome attach was read BEFORE the producer lease existed, which makes it evidence about a past world: between reading "no stored outcome" and winning the lease, another producer can finish and publish one. The first caller then re-produces work already done AND takes a second reservation for an identity already satisfied -- the dedup the contract promises, lost to a window nobody held anything across. The observation is now re-taken while holding the lease and THAT one is the decision. The early read is kept as an optimisation, so the common case -- an identity already built -- still answers without touching the lease at all, and the annotation says which of the two decides. (7) ONE LEASE-UNDROPPED CONTRACT, because there were THREE and they already disagreed. compute_refused_unreserved turned a failed drop into a verdict about the work (StoreUnavailable); compute_run_and_settle folded it into the consumption note and did not; and the attach path added by (5) would have needed a third. Three places deciding one fact is the DESIGN 3 fork, and the thing to delete rather than to keep consistent by hand. The contract, stated once and applied at every site: a lease that could not be dropped NEVER changes what the work did. It is a host-operational fact, it rides as a HostRecordLost so the caller hears it without the stored record being contradicted, and the remedy is an operator removing the name. compute_drop_producer_lease is now the only caller of Filesystem.Delete on the lease path. Also carried here, on the owning manager's instruction not to push it to PR 1: the duplicated annotation fragment review 69814 noted at work_provider_local line 783. Pushing a comment typo to PR 1 would restart CI and invalidate an APPROVE that already sits on its head -- the whole tally for one character class of edit. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 98 +++++++++++++++++------ 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 13ca10a3a44..b8ebaddc520 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -627,21 +627,19 @@ fn compute_run_and_settle( layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, reservation: reservation, returned: returned, reference: reference, termination: termination) } - let dropped = Filesystem.Delete(path: layout.lease_path) + let dropped = compute_drop_producer_lease(layout: layout) let measured = Filesystem.Write( path: layout.consumption_path, content: join([ compute_capacity_standing_wire(s: standing), "\n", "unit: ", unit_termination_wire(t: termination), "\n", "reservation: ", compute_release_wire(r: returned), "\n", - if dropped.success { "" } else { compute_stale_lease_note(layout: layout, error: dropped.error) }, + match dropped { LeaseDropped => "" LeaseNotDropped { detail: d } => compute_stale_lease_note(layout: layout, error: d) }, ], "")) if measured.success { settled } else { - compute_with_lost_host_record( - settled: settled, layout: layout, - dropped_ok: dropped.success, dropped_error: dropped.error, error: measured.error) + compute_with_lost_host_record(settled: settled, layout: layout, dropped: dropped, error: measured.error) } } @@ -672,17 +670,15 @@ fn compute_run_and_settle( fn compute_with_lost_host_record( settled: ProvideResult, layout: ComputeLayout, - dropped_ok: Bool, - dropped_error: String, + dropped: LeaseDrop, error: String, ) -> ProvideResult { let detail = join([ "this host's record at ", layout.consumption_path, " could not be written: ", error, "; the consumption reading is lost, and ", - if dropped_ok { - "the producer lease was dropped, so nothing else is outstanding" - } else { - join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be produced again"], "") + match dropped { + LeaseDropped => "the producer lease was dropped, so nothing else is outstanding" + LeaseNotDropped { detail: d } => join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", d, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be produced again"], "") }, ], "") provide_result_with_host_record(r: settled, carried: HostRecordLost { detail: detail }) @@ -780,7 +776,7 @@ fn compute_unreleased_record( // operator clears the partition by hand. That is the direction a capacity ledger must fail in. data compute_stuck_reservation_remedy: String = "There is NO safe discharge for this seat while this host is admitting work: clearing the capacity partition by hand under live admission removes the record the admission arithmetic is computed from, so the next requests would be admitted against memory this one still holds -- trading a host that under-admits for one that over-promises. The discharge is the compute lifecycle capability (gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge). Any manual intervention before then requires compute admission STOPPED on this host and every gunbc-compute unit PROVEN gone, in that order. Until then this host under-admits, which is the fail-closed direction and is not an incident." -// THE ORDER THE TWO HOLDS ARE TAKEN IN// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease +// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the // reservation is a QUANTITY on the host, and no exclusive create of any name settles that. The // lease is taken first because a request that then loses on capacity releases a name it holds @@ -803,6 +799,63 @@ data compute_stuck_reservation_remedy: String = "There is NO safe discharge for // attached to MODULE-SCOPE declarations only, and the interpreter route this provider's witnesses // run on tolerates a body-position block that `gunbc compile` refuses outright -- 14 blocking // errors on a module every witness had reported green. +// THE ATTACH CHECK IS MADE TWICE, AND THE SECOND ONE IS THE LOAD-BEARING ONE. +// +// The first read happens BEFORE the producer lease exists, which makes it evidence about a past +// world: between reading "no stored outcome" and winning the lease, another producer can finish and +// publish one. The first caller then re-produces work that is already done AND takes a second +// reservation for an identity that is already satisfied -- the dedup the contract promises, lost to +// a window nobody was holding anything across. +// +// So whatever was observed before the lease is re-observed while holding it. The early read is kept +// because it lets the overwhelmingly common case -- an identity that is already built -- answer +// without touching the lease at all; it is an optimisation, and the one inside the lease is the +// decision. +fn compute_read_stored_outcome(layout: ComputeLayout) -> StoredOutcomeRead { + let stored = Filesystem.Read(path: layout.outcome_path) + if stored.success { stored_outcome_decode(text: stored.content) } else { StoredOutcomeUnreadable { reason: "no stored outcome" } } +} + +// ATTACHING FROM INSIDE THE LEASE MEANS GIVING THE LEASE BACK, and a failure to do so is reported +// through the one lease-drop authority rather than a second spelling of it. +fn compute_attach_under_lease(layout: ComputeLayout, identity: String, subject: WorkSubject, op: WorkOperation) -> ProvideResult { + let attached = compute_read_stored_outcome(layout: layout) + let dropped = compute_drop_producer_lease(layout: layout) + match attached { + StoredOutcomeFound { ending, identity: stored_identity, document } => + ProvideAttached { identity: stored_identity, ending: ending, document: document, host_record: compute_lease_drop_record(dropped: dropped, layout: layout) } + StoredOutcomeUnreadable { reason } => + compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: reason } }, subject: subject, op: op) + } +} + +// ONE AUTHORITY FOR "THE PRODUCER LEASE WAS NOT DROPPED", because there were three. +// +// compute_refused_unreserved mapped a failed drop to StoreUnavailable; compute_run_and_settle +// folded it into the consumption note; and the attach path above would have needed a third. Three +// places deciding one fact is the DESIGN 3 fork, and the thing to delete rather than to keep +// consistent by hand -- they were already inconsistent, since one turned the failure into a verdict +// about the work and another did not. +// +// THE CONTRACT, stated once and applied everywhere: a lease that could not be dropped NEVER changes +// what the work did. It is a host-operational fact, it rides as a HostRecordLost so the caller hears +// it without the stored record being contradicted, and the remedy is an operator removing the name. +type LeaseDrop + = LeaseDropped + | LeaseNotDropped { detail: String } + +fn compute_drop_producer_lease(layout: ComputeLayout) -> LeaseDrop { + let dropped = Filesystem.Delete(path: layout.lease_path) + if dropped.success { LeaseDropped } else { LeaseNotDropped { detail: dropped.error } } +} + +fn compute_lease_drop_record(dropped: LeaseDrop, layout: ComputeLayout) -> HostRecordStatus { + match dropped { + LeaseDropped => HostRecordWritten + LeaseNotDropped { detail: d } => HostRecordLost { detail: compute_stale_lease_note(layout: layout, error: d) } + } +} + fn compute_provide_leaf( instance: HostDashboardInstance, commit: String, @@ -813,8 +866,7 @@ fn compute_provide_leaf( ) -> ProvideResult { let identity = work_identity_hex(id: work_identity(subject: subject, op: op, toolchain: toolchain)) let layout = compute_layout(instance: instance, identity_hex: identity) - let stored = Filesystem.Read(path: layout.outcome_path) - let attached = if stored.success { stored_outcome_decode(text: stored.content) } else { StoredOutcomeUnreadable { reason: "no stored outcome" } } + let attached = compute_read_stored_outcome(layout: layout) if stored_outcome_is_success(r: attached) { match attached { StoredOutcomeFound { ending, identity: stored_identity, document } => ProvideAttached { identity: stored_identity, ending: ending, document: document, host_record: HostRecordWritten } @@ -826,6 +878,8 @@ fn compute_provide_leaf( let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) + } else if stored_outcome_is_success(r: compute_read_stored_outcome(layout: layout)) { + compute_attach_under_lease(layout: layout, identity: identity, subject: subject, op: op) } else { let reference = compute_reservation_reference(identity: identity) let reservation = compute_reserve(instance: instance, reference: reference, term_seconds: compute_reservation_term_seconds) @@ -876,16 +930,12 @@ fn compute_refused_unreserved( layout: ComputeLayout, cause: WorkInfrastructureRefusal, ) -> ProvideResult { - let dropped = Filesystem.Delete(path: layout.lease_path) - if dropped.success { - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) - } else { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: StoreUnavailable { detail: join(["a capacity refusal could not drop its producer lease at ", layout.lease_path, ": ", dropped.error], "") }, - }, - subject: subject, op: op) + let dropped = compute_drop_producer_lease(layout: layout) + match compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) { + ProvideFresh { outcome: o, document: d, host_record: _ } => + ProvideFresh { outcome: o, document: d, host_record: compute_lease_drop_record(dropped: dropped, layout: layout) } + ProvideAttached { identity: i, ending: e, document: d, host_record: h } => + ProvideAttached { identity: i, ending: e, document: d, host_record: h } } } From db3e6a54fa338cfaebbc923f0ca29aa92e9048c2 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 02:36:48 +0000 Subject: [PATCH 24/44] Wire the attempt slot into the provider route, and give the seat a door out The gate is now ON the real route: reserve, open the attempt, check out, ASK, spawn. The ask sits last because the window the launch deadline protects IS the checkout -- with it immediately after the open, the two compare-and-sets are microseconds apart and the interleaving the gate exists to refuse cannot occur on the real route, so the wall would have been permanently green by construction. compute_recover_cli is the discharge the seat had no route to. It reads the attempt slot, observes the unit through the manager and its cgroup, evaluates the recovery trigger, and commits the terminal generation EXPECTING the generation it read -- so a producer that advanced meanwhile wins and the recovery is refused by the store rather than by noticing afterwards. unit_population is narrowed to the two readings it uses. Recovery previously had to construct the producer's whole observation record, which carries attempt_completed -- a first-hand fact recovery cannot have. Recovery now has no carrier for first-hand completion anywhere on its route, so the distinction is unwritable rather than unwritten. RecoverySettledButHeld separates the terminal transition landing from the seat coming back. A recovery whose release the ledger refuses closes the attempt and leaves the memory charged, and says so with a nonzero exit. a_held_compute_seat_has_no_in_corpus_discharge records its trigger as fired and its rung as climbed rather than being deleted, and the refusal text that pointed at an unbuilt capability now names the door. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 21 +- dag/gunbc/compute/work_provider_local.dag | 513 ++++++++++++++++-- ...ompute_seat_has_no_in_corpus_discharge.dag | 13 +- .../compute/work_request_witness_test.dag | 76 +++ 4 files changed, 579 insertions(+), 44 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index a4dbfb0a8a7..0441eb44451 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -385,14 +385,19 @@ fn compute_release(reservation: ComputeReservation, reason: NonEmptyStr) -> Comp // append; it would fail at every later READ, when the fold hits UnknownEncumbrance and refuses the // WHOLE partition. // -// THE OPERATOR'S RECOVERY DOOR IS NOT HERE, and its absence is declared rather than implied. The -// discharge of a seat whose release did not land needs a termination decision bound to the unit, an -// attempt-to-unit binding that cannot release another attempt's live reservation, and a settled -// state a retry consumes -- see gunbc.recurring_failure_mode -// a_held_compute_seat_has_no_in_corpus_discharge, whose trigger names that capability. Shipping a -// recovery route with any of those missing would FREE SEATS THAT SHOULD NOT BE FREED, which is -// worse than not having one: without it every failure here keeps the charge and the host merely -// under-admits. +// THE OPERATOR'S RECOVERY DOOR IS NOT HERE, AND IT IS NO LONGER ABSENT: it is one layer up, at +// gunbc.compute.work_provider_local compute_recover_cli, and it comes back through THIS function +// rather than growing a release path of its own. What had to exist before one could ship was named +// in this annotation and by gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge +// -- a termination decision bound to the unit, an attempt-to-unit binding that cannot release +// another attempt's live reservation, and a settled state a retry consumes. All three are now +// present, and the reason the door is up there rather than here is the same reason this fold takes +// a store and a partition rather than an instance: the pool knows about encumbrances, and deciding +// that a particular attempt is dead is not a fact about a pool. +// +// WHAT HAS NOT CHANGED is why a recovery may not simply free on doubt. Shipping one with any of +// those three missing would FREE SEATS THAT SHOULD NOT BE FREED, which is worse than having none: +// every failure here keeps the charge and the host merely under-admits. // // THE APPROPRIATION PASSED HERE IS NOT LOAD-BEARING AND THE ZERO IS NOT A FABRICATED READING. product.capacity.pool // ruled that replay never adjudicates the ceiling, so a release is decided by whether the ledger diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index b8ebaddc520..26fc8c6f509 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -18,6 +18,19 @@ import extdeps.linux.cgroup_v2 { } import extdeps.cache.sccache { sccache_installed_binary_path } import gunbc.clock_read { clock_now_probed_at_or_unknown } +import extdeps.clock { Clock } +import std.durable_compare_and_set { CasGeneration, cas_generation_count } +import gunbc.compute.attempt_lifecycle { + AttemptRecord, AttemptState, AttemptReserved, AttemptLaunching, AttemptTerminal, + UnitLifecycle, UnitTerminatedSuccess, UnitTerminatedFailure, UnitTerminatedCauseUnknown, + UnitStillRunning, UnitLifecycleUnknown, unit_lifecycle_ended, unit_lifecycle_wire, + RecoveryTrigger, RecoveryMayTerminate, RecoveryTooEarly, RecoveryUnitNotEnded, RecoveryStateNotRecoverable, + recovery_trigger, recovery_trigger_fires, recovery_trigger_wire, + AttemptTransition, AttemptAdvanced, AttemptLostRace, AttemptTransitionRefused, + AttemptSlotReading, AttemptSlotAbsent, AttemptSlotHeld, AttemptSlotUnreadable, + attempt_open, attempt_begin_launch, attempt_terminate, attempt_transition_wire, attempt_slot_read, + compute_pre_launch_ceiling_seconds, +} import gunbc.roadmap_dashboard_instance { HostDashboardInstance, dashboard_instance_for_repo_root, DashboardInstanceResolved, DashboardInstanceRootUnknown, dashboard_instance_compute_root, @@ -31,7 +44,10 @@ import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, + ComputeCapacitySubject, ComputeSubjectResolution, ComputeSubjectResolved, ComputeSubjectHostUnresolved, + compute_capacity_subject, compute_capacity_store, compute_release_on, } +import std.measure { kibibyte } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoProfile, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, @@ -72,6 +88,7 @@ type ComputeLayout { log_path: String outcome_path: String consumption_path: String + attempts_dir: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -89,6 +106,7 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), consumption_path: join([work_dir, "/consumption"], ""), + attempts_dir: join([root, "/attempts"], ""), } } @@ -408,21 +426,31 @@ type UnitPopulation | PopulationEmpty { detail: String } | PopulationUnknown { detail: String } -fn unit_population(o: UnitObservations) -> UnitPopulation { - if !o.control_group.queried { +// IT TAKES THE TWO READINGS IT USES AND NOT THE PRODUCER'S WHOLE OBSERVATION RECORD, and that is +// a structural fact rather than tidiness. UnitObservations carries `attempt_completed` -- the +// first-hand fact that only the process which ran `systemd-run --wait` can have. If this fold took +// that record, RECOVERY WOULD HAVE TO CONSTRUCT ONE, and a caller that must supply a field it +// cannot observe supplies a fabricated value; the honest-looking `false` is still a value recovery +// invented. Narrowing the parameters means recovery has no carrier for first-hand completion +// anywhere in its route -- the distinction is unwritable rather than merely unwritten +// (ruling, loyal-swift-608, 2026-09-22: make the difference structural or the two folds can come +// to disagree about the same facts, which is the moment they become two authorities for one +// question). The kernel-outranks-systemd rule still has exactly ONE authority, and it is this one. +fn unit_population(control_group: UnitPropertyReading, events: UnitPropertyReading) -> UnitPopulation { + if !control_group.queried { PopulationUnknown { detail: "the control group could not be queried" } - } else if trim(s: o.control_group.value) == "" { + } else if trim(s: control_group.value) == "" { PopulationUnknown { detail: "the unit holds no control group, which is EITHER a unit that has finished OR a start that is still pending -- the cgroup is realized at spawn" } - } else if !o.events.queried { - PopulationUnknown { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be read"], "") } + } else if !events.queried { + PopulationUnknown { detail: join(["the cgroup events of ", trim(s: control_group.value), " could not be read"], "") } } else { - match decode_cgroup_events_populated(body: o.events.value) { - CgroupEventsUnparseable { cause: c } => PopulationUnknown { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be decoded: ", c], "") } + match decode_cgroup_events_populated(body: events.value) { + CgroupEventsUnparseable { cause: c } => PopulationUnknown { detail: join(["the cgroup events of ", trim(s: control_group.value), " could not be decoded: ", c], "") } CgroupEventsDecoded { populated: p } => if p { - PopulationOccupied { detail: join(["processes remain in ", trim(s: o.control_group.value)], "") } + PopulationOccupied { detail: join(["processes remain in ", trim(s: control_group.value)], "") } } else { - PopulationEmpty { detail: join([trim(s: o.control_group.value), " reports populated 0"], "") } + PopulationEmpty { detail: join([trim(s: control_group.value), " reports populated 0"], "") } } } } @@ -440,7 +468,7 @@ fn unit_population(o: UnitObservations) -> UnitPopulation { // reading serves only to CONFIRM a completion or to REFUSE one. Establishing termination without a // completed attempt is the lifecycle capability's job, not this one's. fn unit_termination_decide(o: UnitObservations) -> UnitTermination { - let population = unit_population(o: o) + let population = unit_population(control_group: o.control_group, events: o.events) match population { PopulationOccupied { detail: d } => UnitPopulated { detail: d } _ => @@ -475,6 +503,93 @@ fn unit_termination_decide(o: UnitObservations) -> UnitTermination { } } +// ── THE LIFECYCLE VERDICT, WHICH IS WHAT RECOVERY ASKS AND SETTLEMENT DOES NOT ──────────────── +// +// THE TWO FOLDS ARE NOT A FORK AND THE DIFFERENCE IS WHAT EACH ONE IS ENTITLED TO ASSUME. +// unit_termination_decide answers for a producer that JUST RAN `systemd-run --wait` and therefore +// holds a fact nobody else has -- whether its own attempt completed -- and it is deliberately +// conservative because its wrong answer frees a live seat. This fold answers for RECOVERY, which +// holds no such fact and must decide from the manager alone. Merging them would mean giving one of +// the two callers evidence it does not have: either recovery inherits a completion it never +// observed, or the producer throws away the one reading that makes its release safe. +// +// WHAT THEY DO SHARE IS THE POPULATION READING, and it is shared rather than restated -- +// unit_population is called by both, so the kernel-outranks-systemd rule has ONE authority. +type UnitLifecycleObservations { + load_state: UnitPropertyReading + active_state: UnitPropertyReading + result: UnitPropertyReading + population: UnitPopulation +} + +data unit_active_state_property: NonEmptyStr = "ActiveState" as NonEmptyStr +data unit_result_property: NonEmptyStr = "Result" as NonEmptyStr +data unit_active_state_failed_wire: String = "failed" +data unit_active_state_inactive_wire: String = "inactive" +data unit_result_success_wire: String = "success" + +// FOUR STATES AND UNKNOWN IS AN ARM RATHER THAN A DEFAULT (criterion 1). Every path that cannot +// establish an end answers UnitLifecycleUnknown CARRYING WHY, and the seat stays charged on it -- +// "I could not tell" is a reading recovery must act on by NOT acting, never a gap a caller fills. +// +// THE ORDER IS THE SAFETY PROPERTY AND IT IS PR 1'S ORDER. A positively occupied cgroup refuses +// first and no manager property overrides it: the kernel saying processes are there outranks +// systemd's bookkeeping about what it thinks it ran. +// +// LoadState=not-found IS AN ENDING AND ITS CAUSE IS GONE WITH IT. These are transient --collect +// units, so one that ran and finished is REMOVED and the manager truthfully has no record. Calling +// that success would be fabrication and calling it failure would be pessimism, so it is the third +// ended arm -- which is exactly PR 1's authoritative absence under the name that says what it is. +fn unit_lifecycle_decide(o: UnitLifecycleObservations) -> UnitLifecycle { + match o.population { + PopulationOccupied { detail: d } => UnitStillRunning { detail: d } + _ => + if !o.load_state.queried { + UnitLifecycleUnknown { detail: "the user manager could not be queried for this unit's load state" } + } else if trim(s: o.load_state.value) == unit_load_state_absent_wire { + UnitTerminatedCauseUnknown { detail: join(["LoadState=", unit_load_state_absent_wire, ", so the manager has no record of this unit: a transient --collect unit that ran has been removed, and its cause went with it"], "") } + } else if !o.active_state.queried { + UnitLifecycleUnknown { detail: "the unit is known to the manager but its active state could not be read" } + } else if trim(s: o.active_state.value) == unit_active_state_failed_wire { + UnitTerminatedFailure { detail: join(["ActiveState=failed, Result=", if o.result.queried { trim(s: o.result.value) } else { "" }], "") } + } else if trim(s: o.active_state.value) != unit_active_state_inactive_wire { + UnitStillRunning { detail: join(["ActiveState=", trim(s: o.active_state.value)], "") } + } else if !o.result.queried { + UnitLifecycleUnknown { detail: "the unit is inactive but its Result could not be read, so whether it ended well is not established" } + } else if trim(s: o.result.value) == "" { + UnitLifecycleUnknown { detail: "the unit is inactive and the manager answered its Result with nothing, which is not a verdict" } + } else if trim(s: o.result.value) == unit_result_success_wire { + UnitTerminatedSuccess { detail: join(["ActiveState=", unit_active_state_inactive_wire, ", Result=", unit_result_success_wire], "") } + } else { + UnitTerminatedFailure { detail: join(["ActiveState=", unit_active_state_inactive_wire, ", Result=", trim(s: o.result.value)], "") } + } + } +} + +// THE UNIT NAME COMES FROM THE ATTEMPT RECORD (criterion 2). This fold takes the name it is given +// and never rebuilds it from an identity, a path spelling or a listing: a recovery acting on a name +// it invented can stop a unit belonging to a different attempt, and the record is the only +// authority for that binding. +fn observe_unit_lifecycle(unit: NonEmptyStr) -> UnitLifecycle { + let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) + let active = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_active_state_property) + let result = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_result_property) + let cg = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_control_group_property) + let events = if cg.success && trim(s: cg.value) != "" { + linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) + } else { + linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: cgroup_v2_mount_point)) + } + unit_lifecycle_decide(o: UnitLifecycleObservations { + load_state: UnitPropertyReading { queried: load.success, value: load.value }, + active_state: UnitPropertyReading { queried: active.success, value: active.value }, + result: UnitPropertyReading { queried: result.success, value: result.value }, + population: unit_population( + control_group: UnitPropertyReading { queried: cg.success, value: cg.value }, + events: UnitPropertyReading { queried: events.success, value: events.value }), + }) +} + // THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is // system-scope; these are transient --user units, so asking the system manager would answer // not-found for EVERY one of them -- a fabricated authoritative absence, the worst answer this fold @@ -600,6 +715,140 @@ fn compute_attempt_completed(outcome: WorkOutcome) -> Bool { } } +// ── THE ATTEMPT GATE ON THE REAL PROVIDER ROUTE ────────────────────────────────────────────── +// +// THIS IS WHERE CRITERION 3 STOPS BEING A CARRIER PROPERTY AND BECOMES A PROPERTY OF THIS +// PROVIDER. gunbc.compute.attempt_lifecycle can refuse a launch that lost its generation, but a +// refusal nothing calls stops nothing: until the producer asks the gate in the real order -- +// reserve, open, ASK, spawn -- the module is a wall beside the door rather than in it. So the +// spawn is downstream of the gate's answer by construction, and every failure arm below returns +// WITHOUT reaching compute_run_unit. +// +// AND THE ASK SITS AFTER THE CHECKOUT, NOT BEFORE IT, because the window the launch deadline +// protects IS the checkout. Placing the ask before it made the two steps adjacent -- the gate +// would have been asked microseconds after the open, so the interleaving it exists to refuse could +// not occur on the real route and the wall would have been a decoration in the DESIGN 4b sense +// (ask whether the RED is authorable). Reserve, open, check out, ASK, spawn: the ask is the last +// thing that happens before a unit exists. +// +// THE CLOCK IS AN INPUT AND AN UNREADABLE ONE REFUSES. Both bounds the attempt declares -- when it +// opened and when its pre-launch window closes -- are instants, and a fabricated instant is worse +// than no attempt: a zero opened_at makes the window look long expired, so recovery would reclaim +// the seat out from under a producer that is running normally. There is no honest default, so the +// arm refuses and releases (DESIGN 5: a failure arm refuses, never widens). +type LeasedRun { + outcome: WorkOutcome + gate: AttemptGate +} + +type AttemptGate + = AttemptGateReady { record: AttemptRecord, generation: CasGeneration } + | AttemptGateRefused { detail: String } + +// THE WINDOW THE PRODUCER DECLARES, AND IT IS AT THE CEILING RATHER THAN UNDER IT BY ACCIDENT. +// What it has to cover is one `git worktree add` of this repository plus argv assembly, and the +// ceiling in attempt_lifecycle is the authority on what is tolerable to the other contenders on +// this host. Naming the ceiling's own row rather than restating 300 keeps one number: a change +// there moves this declaration with it instead of silently leaving the producer at a stale bound. +data compute_launch_window_seconds: Int = compute_pre_launch_ceiling_seconds + +fn compute_now_seconds() -> Int? { + match parse_int(s: trim(s: Clock.UnixSecs().unix_secs)) { + Present { value: n } => if n < 0 { none } else { Present { value: n } } + Absent => none + } +} + +// THE UNIT NAME IS WRITTEN INTO THE RECORD BY THE ATTEMPT THAT OWNS IT, from the same fold the +// invocation uses. That is the binding criterion 2 is about: recovery reads this field rather than +// rebuilding the name from the identity, so a recovery can never act on a unit it named itself. +fn compute_attempt_record(identity: String, reference: NonEmptyStr, granted: Kibibyte, now: Int) -> AttemptRecord { + AttemptRecord { + identity: identity, + reference: reference as String, + unit: compute_unit_name(identity_hex: identity) as String, + granted_kib: kibibyte_count(k: granted) as Int, + opened_at: now, + launch_deadline: now + compute_launch_window_seconds, + state: AttemptReserved, + } +} + +fn compute_gate_from_transition(t: AttemptTransition, record: AttemptRecord, what: String) -> AttemptGate { + match t { + AttemptAdvanced { state: _, generation: g } => AttemptGateReady { record: record, generation: g } + AttemptLostRace { expected: e, observed: o } => + AttemptGateRefused { detail: join([what, " did not happen: the attempt slot expected ", e, " and holds ", o, ", so this attempt is no longer the one entitled to the seat"], "") } + AttemptTransitionRefused { detail: d } => AttemptGateRefused { detail: join([what, " was refused: ", d], "") } + } +} + +fn compute_open_attempt(layout: ComputeLayout, identity: String, reference: NonEmptyStr, granted: Kibibyte) -> AttemptGate { + match compute_now_seconds() { + Absent => AttemptGateRefused { detail: "the host clock could not be read as epoch seconds, and an attempt's opened_at and launch deadline are both instants -- a fabricated one would make this attempt look expired to recovery the moment it is written" } + Present { value: now } => { + let record = compute_attempt_record(identity: identity, reference: reference, granted: granted, now: now) + compute_gate_from_transition(t: attempt_open(root: layout.attempts_dir as NonEmptyStr, record: record), record: record, what: "opening the attempt") + } + } +} + +// THE ASK, IMMEDIATELY BEFORE THE SPAWN. Reserved -> Launching against the generation the open +// returned. A terminal transition that got there first moved the generation, so this is refused by +// the STORE -- one linearized step -- and the producer learns it here rather than discovering it +// after a unit is already consuming memory nobody holds. +fn compute_begin_launch(layout: ComputeLayout, gate: AttemptGate) -> AttemptGate { + match gate { + AttemptGateRefused { detail: d } => AttemptGateRefused { detail: d } + AttemptGateReady { record: rec, generation: g } => + compute_gate_from_transition(t: attempt_begin_launch(root: layout.attempts_dir as NonEmptyStr, record: rec, expected: g), record: rec, what: "the launch") + } +} + +// SETTLEMENT CLOSES THE SLOT, AND A FAILURE TO CLOSE IT IS A HOST-RECORD LOSS RATHER THAN SILENCE. +// The terminal generation carries the verdict and why (criterion 6: the obligation is settled and +// the history is kept), and if the transition does not land the attempt is still open in the store +// -- which is a fact an operator needs, so it rides out on the same carrier as every other +// recording failure this provider can suffer. +fn compute_settle_attempt(layout: ComputeLayout, gate: AttemptGate, verdict: String, detail: String) -> HostRecordStatus { + match gate { + AttemptGateRefused { detail: _ } => HostRecordWritten + AttemptGateReady { record: rec, generation: g } => + match attempt_terminate(root: layout.attempts_dir as NonEmptyStr, record: rec, expected: g, verdict: verdict, detail: detail) { + AttemptAdvanced { state: _, generation: _ } => HostRecordWritten + AttemptLostRace { expected: e, observed: o } => + HostRecordLost { detail: join(["the attempt slot for ", rec.reference, " could not be closed: expected ", e, " and it holds ", o, "; the attempt stays open in the store and the seat's settlement is recorded only here"], "") } + AttemptTransitionRefused { detail: d } => + HostRecordLost { detail: join(["the attempt slot for ", rec.reference, " could not be closed: ", d, "; the attempt stays open in the store"], "") } + } + } +} + +// A GATE REFUSAL RELEASES THE SEAT IT NEVER USED. Nothing was spawned, so the reservation is +// returned unconditionally rather than through the unit-termination fold -- there is no unit to +// observe, and routing an unspawned attempt through a termination decision would make the release +// depend on evidence that cannot exist. +fn compute_refused_before_launch( + instance: HostDashboardInstance, + identity: String, + subject: WorkSubject, + op: WorkOperation, + layout: ComputeLayout, + reservation: ComputeReservation, + detail: String, +) -> ProvideResult { + let returned = compute_release(reservation: reservation, reason: join(["the attempt never launched: ", detail], "") as NonEmptyStr) + let dropped = compute_drop_producer_lease(layout: layout) + let fresh = compute_fresh( + outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join([detail, "; ", compute_release_wire(r: returned)], "") } }, + subject: subject, op: op) + provide_result_with_host_record( + r: fresh, + carried: host_record_join( + a: compute_lease_drop_record(dropped: dropped, layout: layout), + b: if compute_release_landed(r: returned) { HostRecordWritten } else { HostRecordLost { detail: join(["the seat for this unlaunched attempt was NOT returned: ", compute_release_wire(r: returned)], "") } })) +} + fn compute_run_and_settle( instance: HostDashboardInstance, commit: String, @@ -612,7 +861,33 @@ fn compute_run_and_settle( reference: NonEmptyStr, reservation: ComputeReservation, ) -> ProvideResult { - let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) + match compute_open_attempt(layout: layout, identity: identity, reference: reference, granted: granted) { + AttemptGateRefused { detail: d } => + compute_refused_before_launch(instance: instance, identity: identity, subject: subject, op: op, layout: layout, reservation: reservation, detail: d) + AttemptGateReady { record: rec, generation: g } => + compute_run_under_grant( + instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, + dependency_store: dependency_store, granted: granted, reference: reference, reservation: reservation, + opened: AttemptGateReady { record: rec, generation: g }) + } +} + +fn compute_run_under_grant( + instance: HostDashboardInstance, + commit: String, + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + dependency_store: String, + granted: Kibibyte, + reference: NonEmptyStr, + reservation: ComputeReservation, + opened: AttemptGate, +) -> ProvideResult { + let run = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted, opened: opened) + let outcome = run.outcome + let launched = run.gate let standing = compute_capacity_standing(instance: instance) let termination = observe_unit_termination(identity_hex: identity, attempt_completed: compute_attempt_completed(outcome: outcome)) let returned = if unit_termination_frees_capacity(t: termination) { @@ -627,6 +902,10 @@ fn compute_run_and_settle( layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, reservation: reservation, returned: returned, reference: reference, termination: termination) } + let closed = compute_settle_attempt( + layout: layout, gate: launched, + verdict: work_outcome_ending(o: outcome), + detail: join([unit_termination_wire(t: termination), "; ", compute_release_wire(r: returned)], "")) let dropped = compute_drop_producer_lease(layout: layout) let measured = Filesystem.Write( path: layout.consumption_path, @@ -634,13 +913,16 @@ fn compute_run_and_settle( compute_capacity_standing_wire(s: standing), "\n", "unit: ", unit_termination_wire(t: termination), "\n", "reservation: ", compute_release_wire(r: returned), "\n", + "attempt: ", host_record_wire(h: closed), "\n", match dropped { LeaseDropped => "" LeaseNotDropped { detail: d } => compute_stale_lease_note(layout: layout, error: d) }, ], "")) - if measured.success { - settled - } else { - compute_with_lost_host_record(settled: settled, layout: layout, dropped: dropped, error: measured.error) - } + provide_result_with_host_record( + r: if measured.success { + settled + } else { + compute_with_lost_host_record(settled: settled, layout: layout, dropped: dropped, error: measured.error) + }, + carried: closed) } // THE HOST RECORD IS THE ONLY DURABLE CARRIER OF THREE FACTS, so losing it is reported rather than @@ -774,7 +1056,7 @@ fn compute_unreleased_record( // WHAT KEEPS IT FAIL-CLOSED MEANWHILE: every arm that cannot prove the unit is gone keeps the // charge, so the host UNDER-admits. Nothing is over-promised; the pool simply shrinks until an // operator clears the partition by hand. That is the direction a capacity ledger must fail in. -data compute_stuck_reservation_remedy: String = "There is NO safe discharge for this seat while this host is admitting work: clearing the capacity partition by hand under live admission removes the record the admission arithmetic is computed from, so the next requests would be admitted against memory this one still holds -- trading a host that under-admits for one that over-promises. The discharge is the compute lifecycle capability (gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge). Any manual intervention before then requires compute admission STOPPED on this host and every gunbc-compute unit PROVEN gone, in that order. Until then this host under-admits, which is the fail-closed direction and is not an incident." +data compute_stuck_reservation_remedy: String = "THE REMEDY IS compute_recover_cli, run with this reservation reference: it reads the attempt slot, observes the unit through the manager and its cgroup, and releases the seat ONLY if the unit has ended -- refusing, and saying why, if it has not. What is still NOT safe is clearing the capacity partition by hand under live admission: that removes the record the admission arithmetic is computed from, so the next requests would be admitted against memory this one still holds, trading a host that under-admits for one that over-promises. If the door declines, the seat is meant to stay charged; a manual intervention over its refusal requires compute admission STOPPED on this host and every gunbc-compute unit PROVEN gone, in that order. Until the door is run this host under-admits, which is the fail-closed direction and is not an incident." // THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the @@ -872,7 +1154,7 @@ fn compute_provide_leaf( StoredOutcomeFound { ending, identity: stored_identity, document } => ProvideAttached { identity: stored_identity, ending: ending, document: document, host_record: HostRecordWritten } StoredOutcomeUnreadable { reason } => compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: reason } }, subject: subject, op: op) } - } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { + } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir) && compute_ensure_dir(instance: instance, path: layout.attempts_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) @@ -965,7 +1247,8 @@ fn compute_run_leased( op: WorkOperation, dependency_store: String, granted: Kibibyte, -) -> WorkOutcome { + opened: AttemptGate, +) -> LeasedRun { let existing = Filesystem.List(path: layout.checkout) let checkout = if existing.success { ComputeExecOk { stdout: "", stderr: "" } @@ -978,22 +1261,42 @@ fn compute_run_leased( } match checkout { ComputeExecFailed { exit_code, stdout: _, stderr } => - WorkRefusedByInfrastructure { identity: identity, cause: CheckoutUnavailable { detail: join(["git worktree add exited ", to_string(value: exit_code), ": ", stderr], "") } } - ComputeExecOk { stdout: _, stderr: _ } => { - let run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store), granted: granted) - let log = Filesystem.Write(path: layout.log_path, content: match run { - ComputeExecOk { stdout, stderr } => join([stdout, stderr], "") - ComputeExecFailed { exit_code: _, stdout, stderr } => join([stdout, stderr], "") - }) - match run { - ComputeExecFailed { exit_code, stdout: _, stderr: _ } => WorkFailed { identity: identity, exit_code: exit_code, log_path: layout.log_path } - ComputeExecOk { stdout: _, stderr: _ } => - match compute_return_outputs(instance: instance, layout: layout, declared: work_declared_outputs(op: op)) { - OutputsMismatch { detail } => WorkOutputMismatch { identity: identity, detail: detail } - OutputsReturned { outputs } => WorkSucceeded { identity: identity, outputs: outputs, log_path: if log.success { layout.log_path } else { "" } } - } + LeasedRun { outcome: WorkRefusedByInfrastructure { identity: identity, cause: CheckoutUnavailable { detail: join(["git worktree add exited ", to_string(value: exit_code), ": ", stderr], "") } }, gate: opened } + ComputeExecOk { stdout: _, stderr: _ } => + match compute_begin_launch(layout: layout, gate: opened) { + AttemptGateRefused { detail: d } => LeasedRun { outcome: WorkCancelled { identity: identity }, gate: AttemptGateRefused { detail: d } } + AttemptGateReady { record: rec2, generation: g2 } => + compute_spawn_and_collect( + instance: instance, layout: layout, identity: identity, op: op, dependency_store: dependency_store, + granted: granted, launched: AttemptGateReady { record: rec2, generation: g2 }) } - } + } +} + +fn compute_spawn_and_collect( + instance: HostDashboardInstance, + layout: ComputeLayout, + identity: String, + op: WorkOperation, + dependency_store: String, + granted: Kibibyte, + launched: AttemptGate, +) -> LeasedRun { + let run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store), granted: granted) + let log = Filesystem.Write(path: layout.log_path, content: match run { + ComputeExecOk { stdout, stderr } => join([stdout, stderr], "") + ComputeExecFailed { exit_code: _, stdout, stderr } => join([stdout, stderr], "") + }) + LeasedRun { + gate: launched, + outcome: match run { + ComputeExecFailed { exit_code, stdout: _, stderr: _ } => WorkFailed { identity: identity, exit_code: exit_code, log_path: layout.log_path } + ComputeExecOk { stdout: _, stderr: _ } => + match compute_return_outputs(instance: instance, layout: layout, declared: work_declared_outputs(op: op)) { + OutputsMismatch { detail } => WorkOutputMismatch { identity: identity, detail: detail } + OutputsReturned { outputs } => WorkSucceeded { identity: identity, outputs: outputs, log_path: if log.success { layout.log_path } else { "" } } + } + }, } } @@ -1086,6 +1389,148 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent } } +// ── THE RECOVERY DOOR (criteria 4, 5, 6) ───────────────────────────────────────────────────── +// +// THIS IS THE CAPABILITY gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge +// NAMED AS ITS TRIGGER, and PR 1's compute_release_on says in its own annotation what had to exist +// before a discharge could ship: a termination decision bound to the unit, an attempt-to-unit +// binding that cannot release another attempt's live reservation, and a settled state a retry +// consumes. All three are here -- observe_unit_lifecycle decides from the manager, the unit name +// comes from the attempt record rather than from an identity or a listing, and the terminal +// generation IS the settled state. +// +// EVERY REFUSAL ARM BELOW EXISTS BECAUSE ITS OPPOSITE FREES A SEAT THAT SHOULD STAY HELD, so none +// of them widens toward "release anyway". An unreadable slot is NOT an absent one (criterion 4): a +// recovery that read "absent" from a failed read would release a seat whose attempt it never saw. +// THE ATTEMPT SLOTS ARE PER-HOST, NOT PER-IDENTITY, which is why this is its own fold rather than +// a field read off a layout: a layout is built around one work identity and recovery arrives +// holding a reference, not an identity. Deriving the root through a layout with an empty identity +// would work and would also be a second spelling of the same path (DESIGN 3). +fn compute_attempts_root(instance: HostDashboardInstance) -> NonEmptyStr { + join([dashboard_instance_compute_root(instance: instance) as String, "/attempts"], "") as NonEmptyStr +} + +// THE TERMINAL TRANSITION AND THE RELEASE ARE TWO FACTS AND THE MIDDLE ARM IS THE ONE THAT +// MATTERS. The attempt can settle -- history written, generation advanced -- and the seat can still +// fail to come back, and reporting that as a success would be exactly the recording-failure-as- +// verdict conflation the settlement path already had to be repaired for. It gets its own arm so +// the operator learns the attempt is closed AND the memory is still charged. +type ComputeRecovery + = RecoveryReleased { reference: String, verdict: String, seat: String } + | RecoverySettledButHeld { reference: String, verdict: String, seat: String } + | RecoveryDeclined { reference: String, why: String } + | RecoveryRefused { detail: String } + +fn compute_recovery_wire(r: ComputeRecovery) -> String { + match r { + RecoveryReleased { reference: ref_, verdict: v, seat: rel } => join(["recovered ", ref_, ": ", v, "; ", rel], "") + RecoverySettledButHeld { reference: ref_, verdict: v, seat: rel } => + join(["SETTLED BUT STILL CHARGED ", ref_, ": the attempt is closed (", v, ") and the seat did NOT come back -- ", rel, "; this host under-admits by that grant until the ledger accepts a release for this reference"], "") + RecoveryDeclined { reference: ref_, why: w } => join(["declined ", ref_, ": ", w], "") + RecoveryRefused { detail: d } => join(["recovery refused: ", d], "") + } +} + +// THE RE-CHECK UNDER THE LEASE (criterion 5), AND IT IS THE SAME GENERATION OR IT IS NOTHING. +// The slot is read, the unit is observed, the trigger is evaluated -- and then the terminal +// transition is committed EXPECTING THE GENERATION THAT WAS READ. If the producer advanced in the +// meantime, the observation recovery decided on is stale and the store refuses it. The re-check is +// not a second read that could also go stale; it is the compare-and-set itself, which is the only +// form of re-check that cannot be overtaken between the checking and the acting. +fn compute_recover_attempt(instance: HostDashboardInstance, reference: NonEmptyStr) -> ComputeRecovery { + let attempts_root = compute_attempts_root(instance: instance) + match attempt_slot_read(root: attempts_root, reference: reference) { + AttemptSlotUnreadable { detail: d } => + RecoveryRefused { detail: join(["the attempt slot for ", reference as String, " could not be read (", d, "), and an unreadable slot is not an absent one -- releasing on a read that failed would return a seat whose attempt was never observed"], "") } + AttemptSlotAbsent => + RecoveryRefused { detail: join(["no attempt slot exists for ", reference as String, "; there is no record binding that reference to a unit, and this door will not release a seat it cannot name the attempt of"], "") } + AttemptSlotHeld { record: rec, generation: g } => { + let lifecycle = observe_unit_lifecycle(unit: rec.unit as NonEmptyStr) + match compute_now_seconds() { + Absent => RecoveryRefused { detail: "the host clock could not be read as epoch seconds, and the reserved-state trigger is a comparison against a deadline" } + Present { value: now } => + match recovery_trigger(record: rec, lifecycle: lifecycle, now: now) { + RecoveryTooEarly { deadline: d, now: n } => RecoveryDeclined { reference: rec.reference, why: recovery_trigger_wire(t: RecoveryTooEarly { deadline: d, now: n }) } + RecoveryUnitNotEnded { lifecycle: u } => RecoveryDeclined { reference: rec.reference, why: recovery_trigger_wire(t: RecoveryUnitNotEnded { lifecycle: u }) } + RecoveryStateNotRecoverable { state: st } => RecoveryDeclined { reference: rec.reference, why: recovery_trigger_wire(t: RecoveryStateNotRecoverable { state: st }) } + RecoveryMayTerminate { reason: why } => + match attempt_terminate(root: attempts_root, record: rec, expected: g, verdict: "recovered", detail: join([why, "; unit ", unit_lifecycle_wire(u: lifecycle)], "")) { + AttemptLostRace { expected: e, observed: o } => + RecoveryRefused { detail: join(["the attempt moved while recovery was deciding: expected ", e, " and the slot holds ", o, "; the observation this recovery would act on is stale, so nothing is released"], "") } + AttemptTransitionRefused { detail: d } => RecoveryRefused { detail: join(["the attempt slot would not take the terminal generation: ", d], "") } + AttemptAdvanced { state: _, generation: _ } => + compute_recovery_after_settlement( + reference: rec.reference, + verdict: join([why, "; unit ", unit_lifecycle_wire(u: lifecycle)], ""), + returned: compute_recover_release(instance: instance, reference: reference, reason: why)) + } + } + } + } + } +} + +fn compute_recovery_after_settlement(reference: String, verdict: String, returned: ComputeRelease) -> ComputeRecovery { + if compute_release_landed(r: returned) { + RecoveryReleased { reference: reference, verdict: verdict, seat: compute_release_wire(r: returned) } + } else { + RecoverySettledButHeld { reference: reference, verdict: verdict, seat: compute_release_wire(r: returned) } + } +} + +// THE SEAT COMES BACK THROUGH PR 1'S OWN RELEASE PATH, never a second one. compute_release_on +// reads, folds and PROPOSES against the folded pool, so a reference the ledger is not holding is +// refused at the proposal rather than appended -- which is what keeps a recovery from poisoning the +// whole partition with an UnknownEncumbrance every later read would refuse on. +fn compute_recover_release(instance: HostDashboardInstance, reference: NonEmptyStr, reason: String) -> ComputeRelease { + match compute_capacity_subject(instance: instance) { + ComputeSubjectHostUnresolved { host: h } => ComputeReleaseRefused { detail: join(["no capacity subject resolves for host ", h as String], "") } + ComputeSubjectResolved { subject: subj } => + compute_release_on( + store: compute_capacity_store(subject: subj), + partition: subj.partition, + appropriation: kibibyte(count: 0), + reference: reference, + reason: join(["recovered: ", reason], "") as NonEmptyStr) + } +} + +// THE DOOR A WORKER OR AN OPERATOR ACTUALLY OPENS. Three exits and they are the three the rest of +// this provider uses: 0 the seat came back, 1 recovery looked and declined (which is the SAFE +// answer and must not read as a tool failure), 2 recovery could not establish enough to answer. +fn compute_recover_cli(repo_root: String, reference: String, receipt_path: String) -> ProcessExit { + match dashboard_instance_for_repo_root(repo_root: repo_root) { + DashboardInstanceRootUnknown { repo_root: r } => ExitFailure { code: 2, reason: join(["compute-recover: no dashboard instance is rooted at ", r], "") } + DashboardInstanceResolved { instance } => + if trim(s: reference) == "" { + ExitFailure { code: 2, reason: "compute-recover: a reservation reference is required (it is the attempt slot's key: @)" } + } else { + compute_recover_exit( + outcome: compute_recover_attempt(instance: instance, reference: trim(s: reference) as NonEmptyStr), + receipt_path: receipt_path) + } + } +} + +// THE RECOVERED CASE WRITES ITS RECEIPT TOO, and that is not symmetry for its own sake: exit 0 +// carries no text, so a discharge that reported only a status would leave the operator who ran it +// unable to say WHICH seat came back or on what evidence. The durable history is the attempt +// slot's terminal generation; this is the copy the caller can read without opening the store. +fn compute_recover_exit(outcome: ComputeRecovery, receipt_path: String) -> ProcessExit { + let wire = compute_recovery_wire(r: outcome) + let written = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) + if !written.success { + ExitFailure { code: 2, reason: join(["compute-recover: the receipt could not be written to ", receipt_path, ": ", written.error, "; the outcome was ", wire], "") } + } else { + match outcome { + RecoveryReleased { reference: _, verdict: _, seat: _ } => ExitSuccess + RecoverySettledButHeld { reference: _, verdict: _, seat: _ } => ExitFailure { code: 2, reason: wire } + RecoveryDeclined { reference: _, why: _ } => ExitFailure { code: 1, reason: wire } + RecoveryRefused { detail: _ } => ExitFailure { code: 2, reason: wire } + } + } +} + // THE RECEIPT STAYS THE DOCUMENT, AND THE HOST-RECORD FACT GETS ITS OWN FILE BESIDE IT. // // The first version of this appended the host-record sentence to the receipt, and the srv1 diff --git a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag index 7abfa38c7ee..f927e65ebf4 100644 --- a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag +++ b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag @@ -18,11 +18,17 @@ data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = Recu "THERE IS NO SAFE MANUAL DISCHARGE UNDER LIVE ADMISSION, and saying so is part of the row rather than a caveat on it. Clearing the capacity partition by hand while this host is admitting work removes the very record the admission arithmetic folds, so the next requests are admitted against memory the stuck grant still holds -- trading a host that under-admits for one that over-promises, which is the direction this whole change exists to prevent. Any manual intervention requires compute admission STOPPED on the host and every gunbc-compute unit PROVEN gone, in that order.", + "THE TRIGGER FIRED, AND THIS ROW IS WHERE IT IS RECORDED RATHER THAN DELETED. The lifecycle capability the trigger names is gunbc.compute.work_provider_local compute_recover_cli, and it has all four properties the trigger asked for: (a) it decides termination from the manager and the cgroup through unit_lifecycle_decide, never from a state word; (b) it binds the unit from the attempt record the producer wrote, so it cannot stop a unit belonging to a different attempt; (c) it commits the terminal generation EXPECTING the generation it read, so a producer that advanced in the meantime wins and the recovery is refused by the store; (d) the terminal generation IS the settled state, carrying the verdict and why. The class does not stop existing -- a seat can still be held while the discharge has not been run -- so what changes is the RUNG, not the membership.", + + "AND THE DISCHARGE DOES NOT ANNOUNCE ITSELF ON A HALF-SUCCESS, which is the arm that would have quietly re-created this class under a name that reads as fixed. The terminal transition landing and the seat coming back are two facts: gunbc.compute.work_provider_local RecoverySettledButHeld is the arm where the attempt closed and the ledger refused the release, and it exits 2 saying the host still under-admits by that grant. A recovery that reported success on the transition alone would leave exactly the state this row describes while claiming the row was retired.", + "RUNG FOUND AT: mitigatable. The state is reachable, its harm is bounded to one host's compute appropriation, it is visible in the outcome document and in the partition, and it never over-promises.", - "CEILING: mechanically preventable. A discharge route exists to be built -- the evidence needed is a termination decision bound to the unit plus a settled lifecycle state that a retry consumes -- so this is not a class that must stay a ratchet.", + "CURRENT RUNG: mechanically preventable, which is the CEILING for this class and is not structural impossibility. The invalid state is still writable -- a held seat whose release did not land is a state the store can be in -- and safety now depends on the discharge being RUN. What the discharge cannot do is free a seat it has not established is free, and that is the part held by construction: every arm that fails to establish an end keeps the charge.", + + "CEILING: mechanically preventable, and the reason it goes no higher is worth stating rather than leaving as an unexplained stop. Structural impossibility would mean no reachable state in which a seat is held with nothing running, and that would require proving a systemd transient unit has stopped using memory from outside it -- an external fact at the boundary DESIGN 4b keeps OFF the ladder. So this class is at its ceiling, and the honest residual is that a seat stays charged until someone runs the door.", - "NEXT-RUNG TRIGGER, NAMING THE CAPABILITY AND NOT AN ARTIFACT: the compute reservation LIFECYCLE capability -- a discharge that (a) decides termination from authoritative absence or an observed-empty cgroup rather than from a state word, (b) binds the attempt and its unit from the authoritative attempt record so releasing one cannot release another's live reservation, (c) coordinates with a pending producer so a release cannot race a launch that has not happened yet, and (d) produces a settled state a retry consumes, with history kept. Until a discharge with all four properties is consumed by a real caller, this row stands.", + "NEXT-RUNG TRIGGER: NONE, because the class is at its ceiling. The trigger this row carried -- the compute reservation LIFECYCLE capability, with all four properties, consumed by a real caller -- is discharged as described above. A row at its ceiling with no trigger is not a stalled row: DESIGN 4b(2) separates cannot-climb-further from can-climb-after-one-grounding, and this is the first.", "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration, and gunbc.compute.work_provider_local compute_stuck_reservation_remedy carries the remedy sentence into the refusal a caller actually receives.", ], @@ -32,5 +38,8 @@ data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = Recu decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "unit_termination_frees_capacity"), decl_ref(module_path: "gunbc.compute.host_capacity", decl_name: "compute_release"), decl_ref(module_path: "test.claim.compute.work_request_witness_test", decl_name: "only_proven_absence_or_an_ended_cgroup_frees_a_seat"), + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "compute_recover_cli"), + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "unit_lifecycle_decide"), + decl_ref(module_path: "test.claim.compute.work_request_witness_test", decl_name: "the_lifecycle_verdict_has_four_states_and_unknown_carries_why"), ], } diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index c641d411fed..dcfb74b8ab4 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -10,8 +10,14 @@ import gunbc.compute.work_provider_local { provide_result_host_record, provide_result_ending, provide_result_document, host_record_was_written, host_record_wire, compute_with_lost_host_record, provide_result_with_host_record, host_record_join, ComputeLayout, compute_layout, compute_unit_properties, + UnitLifecycleObservations, unit_lifecycle_decide, unit_population, + UnitPopulation, PopulationOccupied, PopulationEmpty, PopulationUnknown, compute_kill_mode_property, compute_kill_mode_value, } +import gunbc.compute.attempt_lifecycle { + UnitLifecycle, UnitTerminatedSuccess, UnitTerminatedFailure, UnitTerminatedCauseUnknown, + UnitStillRunning, UnitLifecycleUnknown, unit_lifecycle_ended, unit_lifecycle_wire, +} import std.measure { kibibyte } import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } import gunbc.compute.work_request { @@ -239,3 +245,73 @@ test fn a_lost_record_on_a_dependency_survives_a_successful_dependent() -> Bool && string_contains(s: host_record_wire(h: both_lost), pattern: "still held") && string_contains(s: host_record_wire(h: both_lost), pattern: "lost its reading") } + + +fn lifecycle_word(u: UnitLifecycle) -> String { + match u { + UnitTerminatedSuccess { detail: _ } => "terminated-success" + UnitTerminatedFailure { detail: _ } => "terminated-failure" + UnitTerminatedCauseUnknown { detail: _ } => "terminated-cause-unknown" + UnitStillRunning { detail: _ } => "still-running" + UnitLifecycleUnknown { detail: _ } => "unknown" + } +} + +data unqueried: UnitPropertyReading = UnitPropertyReading { queried: false, value: "" } + +fn lifecycle_of(load: UnitPropertyReading, active: UnitPropertyReading, result: UnitPropertyReading, pop: UnitPopulation) -> UnitLifecycle { + unit_lifecycle_decide(o: UnitLifecycleObservations { load_state: load, active_state: active, result: result, population: pop }) +} + +// CRITERION 1: FOUR STATES FROM THE MANAGER'S OWN VOCABULARY, AND UNKNOWN IS AN ARM. +// +// DISCRIMINATING IN FIVE DIRECTIONS AT ONCE, and the last two are the ones that fail open if the +// fold is wrong. An occupied cgroup must outrank LoadState=not-found -- the manager forgetting a +// unit while the kernel still holds processes in its cgroup is precisely the state where freeing +// the seat hands away live memory. And every reading that fails to ESTABLISH an end must answer +// unknown rather than picking the plausible neighbour: an inactive unit whose Result could not be +// read is not a success, and a unit whose Result came back empty is not a verdict. +// +// THE UNKNOWN ARMS ASSERT THEIR REASON RATHER THAN THEIR ARM. A negative control satisfied by any +// unknown would be satisfied by a fold that answered unknown to everything, so each one names the +// fact it could not establish (a refusal witness must assert the reason). +test fn the_lifecycle_verdict_has_four_states_and_unknown_carries_why() -> Bool { + let empty = PopulationEmpty { detail: "cg reports populated 0" } + let occupied = PopulationOccupied { detail: "processes remain in /user.slice/x" } + let unknown_pop = PopulationUnknown { detail: "the cgroup events could not be read" } + + let succeeded = lifecycle_of(load: reading(queried: true, value: "loaded"), active: reading(queried: true, value: "inactive"), result: reading(queried: true, value: "success"), pop: empty) + let failed = lifecycle_of(load: reading(queried: true, value: "loaded"), active: reading(queried: true, value: "failed"), result: reading(queried: true, value: "exit-code"), pop: empty) + let oom = lifecycle_of(load: reading(queried: true, value: "loaded"), active: reading(queried: true, value: "inactive"), result: reading(queried: true, value: "oom-kill"), pop: empty) + let running = lifecycle_of(load: reading(queried: true, value: "loaded"), active: reading(queried: true, value: "active"), result: reading(queried: true, value: "success"), pop: unknown_pop) + let collected = lifecycle_of(load: reading(queried: true, value: "not-found"), active: unqueried, result: unqueried, pop: unknown_pop) + + let occupied_but_forgotten = lifecycle_of(load: reading(queried: true, value: "not-found"), active: unqueried, result: unqueried, pop: occupied) + let no_manager = lifecycle_of(load: unqueried, active: unqueried, result: unqueried, pop: unknown_pop) + let result_unreadable = lifecycle_of(load: reading(queried: true, value: "loaded"), active: reading(queried: true, value: "inactive"), result: unqueried, pop: empty) + let result_empty = lifecycle_of(load: reading(queried: true, value: "loaded"), active: reading(queried: true, value: "inactive"), result: reading(queried: true, value: ""), pop: empty) + + lifecycle_word(u: succeeded) == "terminated-success" + && lifecycle_word(u: failed) == "terminated-failure" + && lifecycle_word(u: oom) == "terminated-failure" + && string_contains(s: unit_lifecycle_wire(u: oom), pattern: "oom-kill") + && lifecycle_word(u: running) == "still-running" + && lifecycle_word(u: collected) == "terminated-cause-unknown" + + && lifecycle_word(u: occupied_but_forgotten) == "still-running" + && lifecycle_word(u: no_manager) == "unknown" + && string_contains(s: unit_lifecycle_wire(u: no_manager), pattern: "load state") + && lifecycle_word(u: result_unreadable) == "unknown" + && string_contains(s: unit_lifecycle_wire(u: result_unreadable), pattern: "Result could not be read") + && lifecycle_word(u: result_empty) == "unknown" + && string_contains(s: unit_lifecycle_wire(u: result_empty), pattern: "not a verdict") + + && unit_lifecycle_ended(u: succeeded) + && unit_lifecycle_ended(u: failed) + && unit_lifecycle_ended(u: collected) + && !unit_lifecycle_ended(u: running) + && !unit_lifecycle_ended(u: occupied_but_forgotten) + && !unit_lifecycle_ended(u: no_manager) + && !unit_lifecycle_ended(u: result_unreadable) + && !unit_lifecycle_ended(u: result_empty) +} From 546c7cca06e161cad6c3c86c4b664b60957deeac Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 02:51:33 +0000 Subject: [PATCH 25/44] The settlement line says something on the good path, and the lease-undropped contract gets a control The consumption receipt's new "attempt:" line was rendered with host_record_wire, which returns the empty string on success because it is built to be appended to a sentence. The first real srv1 run at the wired head printed "attempt: " with nothing after it. A line that says nothing on the good path cannot be read as evidence the good path happened, which is why the line exists. work_request_witness_test was still calling compute_with_lost_host_record with the pre-refactor dropped_ok/dropped_error pair, so the file did not typecheck on this branch at all. It now takes the LeaseDrop carrier, and criterion 7 gets the control it was missing: an undropped producer lease is recorded, never becomes the work's verdict, and its note does not tell the operator something the contract contradicts. The AttemptState annotation now names what ENFORCES the entailment the temporal trigger rests on: compute_spawn_and_collect is reachable from exactly one place, so a slot reading "reserved" means nothing was ever spawned. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/attempt_lifecycle.dag | 9 ++++ dag/gunbc/compute/work_provider_local.dag | 17 +++++-- .../compute/work_request_witness_test.dag | 51 ++++++++++++++++++- 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/compute/attempt_lifecycle.dag b/dag/gunbc/compute/attempt_lifecycle.dag index a92b97a2351..3463d54d30a 100644 --- a/dag/gunbc/compute/attempt_lifecycle.dag +++ b/dag/gunbc/compute/attempt_lifecycle.dag @@ -94,6 +94,15 @@ fn unit_lifecycle_wire(u: UnitLifecycle) -> String { // THE LIFECYCLE STATE THE SLOT CARRIES. Reserved is the only state a launch may proceed from and // the only state recovery may terminate from without observing the unit -- because in Reserved no // unit exists yet, which is the whole reason the two transitions must exclude each other. +// +// THAT ENTAILMENT IS ENFORCED BY THE ROUTE AND NOT BY AGREEMENT, which is what makes the temporal +// trigger safe to act on. gunbc.compute.work_provider_local reaches compute_spawn_and_collect from +// exactly one place: the AttemptGateReady arm of compute_begin_launch. So a unit exists only if +// Reserved -> Launching was ADMITTED by the store first, and a slot still reading Reserved means +// no unit was ever spawned for that attempt. Recovery terminating a stale Reserved attempt +// therefore cannot be stopping anything, and it does not need to look to know that. If some later +// caller spawned without passing the gate, this entailment would break silently -- which is why +// the gate is one call site rather than a rule. type AttemptState = AttemptReserved | AttemptLaunching diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 26fc8c6f509..dd7d3a78e97 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -824,12 +824,23 @@ fn compute_settle_attempt(layout: ComputeLayout, gate: AttemptGate, verdict: Str } } +// THE SETTLEMENT LINE HAS ITS OWN WORDS BECAUSE host_record_wire RENDERS SUCCESS AS THE EMPTY +// STRING -- it is built to be APPENDED to a sentence, so reusing it here produced a receipt line +// reading "attempt: " with nothing after it. A line that says nothing on the good path cannot be +// read as evidence that the good path happened, which is the whole reason the line is in the +// receipt. Found in the first srv1 run at this head. +fn compute_attempt_settlement_wire(closed: HostRecordStatus) -> String { + match closed { + HostRecordWritten => "settled in the attempt slot" + HostRecordLost { detail: d } => join(["NOT SETTLED -- ", d], "") + } +} + // A GATE REFUSAL RELEASES THE SEAT IT NEVER USED. Nothing was spawned, so the reservation is // returned unconditionally rather than through the unit-termination fold -- there is no unit to // observe, and routing an unspawned attempt through a termination decision would make the release // depend on evidence that cannot exist. fn compute_refused_before_launch( - instance: HostDashboardInstance, identity: String, subject: WorkSubject, op: WorkOperation, @@ -863,7 +874,7 @@ fn compute_run_and_settle( ) -> ProvideResult { match compute_open_attempt(layout: layout, identity: identity, reference: reference, granted: granted) { AttemptGateRefused { detail: d } => - compute_refused_before_launch(instance: instance, identity: identity, subject: subject, op: op, layout: layout, reservation: reservation, detail: d) + compute_refused_before_launch(identity: identity, subject: subject, op: op, layout: layout, reservation: reservation, detail: d) AttemptGateReady { record: rec, generation: g } => compute_run_under_grant( instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, @@ -913,7 +924,7 @@ fn compute_run_under_grant( compute_capacity_standing_wire(s: standing), "\n", "unit: ", unit_termination_wire(t: termination), "\n", "reservation: ", compute_release_wire(r: returned), "\n", - "attempt: ", host_record_wire(h: closed), "\n", + "attempt: ", compute_attempt_settlement_wire(closed: closed), "\n", match dropped { LeaseDropped => "" LeaseNotDropped { detail: d } => compute_stale_lease_note(layout: layout, error: d) }, ], "")) provide_result_with_host_record( diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index dcfb74b8ab4..685d67a9684 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -10,6 +10,8 @@ import gunbc.compute.work_provider_local { provide_result_host_record, provide_result_ending, provide_result_document, host_record_was_written, host_record_wire, compute_with_lost_host_record, provide_result_with_host_record, host_record_join, ComputeLayout, compute_layout, compute_unit_properties, + LeaseDrop, LeaseDropped, LeaseNotDropped, compute_lease_drop_record, compute_stale_lease_note, + compute_stuck_reservation_remedy, UnitLifecycleObservations, unit_lifecycle_decide, unit_population, UnitPopulation, PopulationOccupied, PopulationEmpty, PopulationUnknown, compute_kill_mode_property, compute_kill_mode_value, @@ -189,8 +191,8 @@ test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { let settled = ProvideFresh { outcome: succeeded, document: document, host_record: HostRecordWritten } let attached = ProvideAttached { identity: "abc", ending: "succeeded", document: document, host_record: HostRecordWritten } let layout = compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc") - let lost = compute_with_lost_host_record(settled: settled, layout: layout, dropped_ok: false, dropped_error: "EBUSY", error: "ENOSPC") - let lost_attached = compute_with_lost_host_record(settled: attached, layout: layout, dropped_ok: false, dropped_error: "EBUSY", error: "ENOSPC") + let lost = compute_with_lost_host_record(settled: settled, layout: layout, dropped: LeaseNotDropped { detail: "EBUSY" }, error: "ENOSPC") + let lost_attached = compute_with_lost_host_record(settled: attached, layout: layout, dropped: LeaseNotDropped { detail: "EBUSY" }, error: "ENOSPC") provide_result_ending(r: lost) == provide_result_ending(r: settled) && provide_result_document(r: lost) == provide_result_document(r: settled) && provide_result_ending(r: lost) == "succeeded" @@ -315,3 +317,48 @@ test fn the_lifecycle_verdict_has_four_states_and_unknown_carries_why() -> Bool && !unit_lifecycle_ended(u: result_unreadable) && !unit_lifecycle_ended(u: result_empty) } + + +// CRITERION 7: ONE CONTRACT FOR AN UNDROPPED LEASE, AND IT IS THE STORED RECORD THAT DECIDES. +// +// The state is: the work ran and its outcome was published, and the producer lease could not be +// removed. Three claims used to be made about it and they cannot all be true -- the caller was +// handed a refusal, the stored document said the run SUCCEEDED, and the note said the next request +// would refuse as ProducerInFlight. The stored record is exactly what the next caller attaches to, +// so it decides: the caller is told what the next caller will read, and the stale name is carried +// as the operational fact it is. +// +// DISCRIMINATING ON THE THREE WAYS THIS GOES WRONG, each of which has been written at least once +// in this change's history. (1) The undropped lease VANISHES -- LeaseNotDropped mapping to +// HostRecordWritten. (2) The undropped lease becomes a VERDICT -- the outcome or the document +// changing because a file could not be deleted. (3) The note tells the operator something the +// contract contradicts: it used to say the next request refuses as ProducerInFlight, which is +// false for a published success, because the attach is decided BEFORE the lease is consulted. +// +// AND THE REMEDY IS ASSERTED TO NAME A ROUTE THAT EXISTS. A refusal whose remedy names a +// capability nobody built decays into advice; this one names compute_recover_cli, and the +// assertion is here so that deleting the door makes this cell red rather than leaving the text +// pointing at nothing. +test fn an_undropped_lease_is_recorded_and_never_becomes_the_works_verdict() -> Bool { + let layout = compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc") + let succeeded = WorkSucceeded { identity: "abc", outputs: [], log_path: "/l" } + let document = work_outcome_wire(o: succeeded, subject: tree(hex: tree_a), op: work_default_build()) + let settled = ProvideFresh { outcome: succeeded, document: document, host_record: HostRecordWritten } + + let clean = compute_lease_drop_record(dropped: LeaseDropped, layout: layout) + let stuck = compute_lease_drop_record(dropped: LeaseNotDropped { detail: "EBUSY" }, layout: layout) + let carried = provide_result_with_host_record(r: settled, carried: stuck) + + host_record_was_written(h: clean) + && !host_record_was_written(h: stuck) + && string_contains(s: host_record_wire(h: stuck), pattern: "STALE PRODUCER LEASE") + && string_contains(s: host_record_wire(h: stuck), pattern: layout.lease_path) + && string_contains(s: host_record_wire(h: stuck), pattern: "EBUSY") + + && provide_result_ending(r: carried) == "succeeded" + && provide_result_document(r: carried) == document + && !host_record_was_written(h: provide_result_host_record(r: carried)) + + && !string_contains(s: compute_stale_lease_note(layout: layout, error: "EBUSY"), pattern: "ProducerInFlight") + && string_contains(s: compute_stuck_reservation_remedy, pattern: "compute_recover_cli") +} From 951df74d9f32162ec5cec7fa62a0df438104725c Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 02:57:02 +0000 Subject: [PATCH 26/44] A cancellation carries why WorkCancelled was a bare identity. Its only producer is a launch refused by the attempt gate -- somebody else ended this attempt before it spawned -- so a caller reading an ending of "cancelled" with no cause could not tell that apart from any other cancellation, and neither could the next reader of the stored document. The detail now rides in the durable outcome like every other arm's. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 4 ++-- dag/gunbc/compute/work_request.dag | 16 ++++++++++++---- .../claim/compute/work_request_witness_test.dag | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index dd7d3a78e97..fc53dde591c 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -711,7 +711,7 @@ fn compute_attempt_completed(outcome: WorkOutcome) -> Bool { WorkOutputMismatch { identity: _, detail: _ } => true WorkFailed { identity: _, exit_code: _, log_path: _ } => false WorkRefusedByInfrastructure { identity: _, cause: _ } => false - WorkCancelled { identity: _ } => false + WorkCancelled { identity: _, detail: _ } => false } } @@ -1275,7 +1275,7 @@ fn compute_run_leased( LeasedRun { outcome: WorkRefusedByInfrastructure { identity: identity, cause: CheckoutUnavailable { detail: join(["git worktree add exited ", to_string(value: exit_code), ": ", stderr], "") } }, gate: opened } ComputeExecOk { stdout: _, stderr: _ } => match compute_begin_launch(layout: layout, gate: opened) { - AttemptGateRefused { detail: d } => LeasedRun { outcome: WorkCancelled { identity: identity }, gate: AttemptGateRefused { detail: d } } + AttemptGateRefused { detail: d } => LeasedRun { outcome: WorkCancelled { identity: identity, detail: d }, gate: AttemptGateRefused { detail: d } } AttemptGateReady { record: rec2, generation: g2 } => compute_spawn_and_collect( instance: instance, layout: layout, identity: identity, op: op, dependency_store: dependency_store, diff --git a/dag/gunbc/compute/work_request.dag b/dag/gunbc/compute/work_request.dag index 16883a0830b..befcf3a2f43 100644 --- a/dag/gunbc/compute/work_request.dag +++ b/dag/gunbc/compute/work_request.dag @@ -127,11 +127,17 @@ type WorkOutput { // THE FIVE ENDINGS, and only five (compute-exact-work-contract's red control: a machine that killed // the work cannot render as the work having failed). Every arm carries the identity so a caller can // never attach a verdict to a different request. +// A CANCELLATION CARRIES WHY, and the arm was a bare identity until a real run made the gap +// visible. The only producer of this arm is a launch refused by the attempt gate -- somebody else +// ended this attempt before it spawned -- and an ending of "cancelled" with no cause cannot be +// told apart from any other cancellation by the caller who reads the document. The detail is in +// the durable outcome for the same reason every other arm's is: the next reader of this identity +// gets the same account the first caller got. type WorkOutcome = WorkSucceeded { identity: String, outputs: List, log_path: String } | WorkFailed { identity: String, exit_code: Int, log_path: String } | WorkRefusedByInfrastructure { identity: String, cause: WorkInfrastructureRefusal } - | WorkCancelled { identity: String } + | WorkCancelled { identity: String, detail: String } | WorkOutputMismatch { identity: String, detail: String } // Infrastructure refusals are typed and counted, never a widen: a producer already holds the lease @@ -170,7 +176,7 @@ fn work_outcome_identity(o: WorkOutcome) -> String { WorkSucceeded { identity, outputs: _, log_path: _ } => identity WorkFailed { identity, exit_code: _, log_path: _ } => identity WorkRefusedByInfrastructure { identity, cause: _ } => identity - WorkCancelled { identity } => identity + WorkCancelled { identity, detail: _ } => identity WorkOutputMismatch { identity, detail: _ } => identity } } @@ -180,7 +186,7 @@ fn work_outcome_ending(o: WorkOutcome) -> String { WorkSucceeded { identity: _, outputs: _, log_path: _ } => "succeeded" WorkFailed { identity: _, exit_code: _, log_path: _ } => "failed" WorkRefusedByInfrastructure { identity: _, cause: _ } => "refused_by_infrastructure" - WorkCancelled { identity: _ } => "cancelled" + WorkCancelled { identity: _, detail: _ } => "cancelled" WorkOutputMismatch { identity: _, detail: _ } => "output_mismatch" } } @@ -228,7 +234,9 @@ fn work_outcome_json(o: WorkOutcome, subject: WorkSubject, op: WorkOperation) -> WorkRefusedByInfrastructure { identity: _, cause } => [ json_kv(key: "cause", value: json_string(s: work_refusal_detail(r: cause))), ] - WorkCancelled { identity: _ } => [] + WorkCancelled { identity: _, detail } => [ + json_kv(key: "detail", value: json_string(s: detail)), + ] WorkOutputMismatch { identity: _, detail } => [ json_kv(key: "detail", value: json_string(s: detail)), ] diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 685d67a9684..34a1bf439f2 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -169,7 +169,7 @@ test fn the_attempt_completion_fact_is_not_the_works_ending() -> Bool { compute_attempt_completed(outcome: WorkSucceeded { identity: "a", outputs: [], log_path: "/l" }) && compute_attempt_completed(outcome: WorkOutputMismatch { identity: "a", detail: "d" }) && !compute_attempt_completed(outcome: WorkFailed { identity: "a", exit_code: 1, log_path: "/l" }) - && !compute_attempt_completed(outcome: WorkCancelled { identity: "a" }) + && !compute_attempt_completed(outcome: WorkCancelled { identity: "a", detail: "d" }) && !compute_attempt_completed(outcome: WorkRefusedByInfrastructure { identity: "a", cause: HostComputeCapacityFull { wire: "pool-full" } }) } From 17e06b6b2516e59c43563370382ee78837e8a683 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 03:54:44 +0000 Subject: [PATCH 27/44] The stale-lease claim forbade a word the repaired note legitimately keeps an_undropped_lease_is_recorded_and_never_becomes_the_works_verdict was RED at this head on one conjunct: it asserted the stale-lease note contains no "ProducerInFlight" at all. The repair that note actually received was to SCOPE the sentence, not to delete the word -- a caller reaching a published success attaches before the name is consulted, and the name still refuses a request that must PRODUCE again, which is true and worth telling the operator. So the claim forbade more than the contract requires (DESIGN 4d, over-prohibition) and went red against correct text. The assertion now names the two clauses that carry the scope, so a note that drops them and states the refusal unconditionally is still red, while the word itself is free to stay. Co-Authored-By: Claude Opus 5 (1M context) --- dag/test/claim/compute/work_request_witness_test.dag | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 34a1bf439f2..f02ff7144d1 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -332,8 +332,12 @@ test fn the_lifecycle_verdict_has_four_states_and_unknown_carries_why() -> Bool // in this change's history. (1) The undropped lease VANISHES -- LeaseNotDropped mapping to // HostRecordWritten. (2) The undropped lease becomes a VERDICT -- the outcome or the document // changing because a file could not be deleted. (3) The note tells the operator something the -// contract contradicts: it used to say the next request refuses as ProducerInFlight, which is -// false for a published success, because the attach is decided BEFORE the lease is consulted. +// contract contradicts: it used to say THE NEXT REQUEST refuses as ProducerInFlight, which is +// false for a published success, because the attach is decided BEFORE the lease is consulted. The +// repair was to SCOPE that sentence, not to delete the word: the note still names ProducerInFlight +// as what a request that must PRODUCE again will meet, which is true. So the assertion here is on +// the two clauses that carry the scope -- a note that drops them and states the refusal +// unconditionally goes red, while the word itself is free to stay. // // AND THE REMEDY IS ASSERTED TO NAME A ROUTE THAT EXISTS. A refusal whose remedy names a // capability nobody built decays into advice; this one names compute_recover_cli, and the @@ -359,6 +363,7 @@ test fn an_undropped_lease_is_recorded_and_never_becomes_the_works_verdict() -> && provide_result_document(r: carried) == document && !host_record_was_written(h: provide_result_host_record(r: carried)) - && !string_contains(s: compute_stale_lease_note(layout: layout, error: "EBUSY"), pattern: "ProducerInFlight") + && string_contains(s: compute_stale_lease_note(layout: layout, error: "EBUSY"), pattern: "the attach is decided before this name is consulted") + && string_contains(s: compute_stale_lease_note(layout: layout, error: "EBUSY"), pattern: "matters only to a request that must PRODUCE again") && string_contains(s: compute_stuck_reservation_remedy, pattern: "compute_recover_cli") } From ec72c34876427fce01352a8ae170359769e23054 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 04:35:30 +0000 Subject: [PATCH 28/44] The attempt record carries what it means: Kibibyte and EpochSecs on the record, bare numbers only at the JSON boundary, and one host clock Review 69915 on gunbc#12040 found two sites; the sweep found the population. granted_kib: Int stripped the carrier the producer held (work_request's own note names the defect); compute_now_seconds reproduced gunbc.fabric_event_log_host now_epoch_seconds body-for-body while the annotation beside it named that authority. Both were annotations written from the right model with the code going the other way. The record now holds granted: Kibibyte and EpochSecs instants; RecoveryTooEarly and recovery_trigger take EpochSecs; the decode re-admits an instant and a grant (negative is unreadable, not cast); the Int? clock fork is deleted and both callers consume now_epoch_seconds, as host_capacity already did. granted was never read for a decision -- only serialized and re-decoded -- so this was a DESIGN 3 fork, not a live miscalculation. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/attempt_lifecycle.dag | 49 ++++++++++++++----- dag/gunbc/compute/work_provider_local.dag | 21 +++----- .../attempt_lifecycle_wet_witness_test.dag | 21 +++++--- 3 files changed, 58 insertions(+), 33 deletions(-) diff --git a/dag/gunbc/compute/attempt_lifecycle.dag b/dag/gunbc/compute/attempt_lifecycle.dag index 3463d54d30a..03d2ece1c0d 100644 --- a/dag/gunbc/compute/attempt_lifecycle.dag +++ b/dag/gunbc/compute/attempt_lifecycle.dag @@ -1,6 +1,7 @@ module gunbc.compute.attempt_lifecycle -import std.types { String, Bool, Int, NonEmptyStr } +import std.types { String, Bool, Int, NonEmptyStr, EpochSecs } +import std.measure { Kibibyte, kibibyte, kibibyte_count } import std.content_hash { content_hash_of_value } import std.durable_compare_and_set { CasAttempt, cas_attempt, CasExpectation, ExpectSlotAbsent, ExpectSlotGeneration, @@ -120,13 +121,21 @@ fn attempt_state_wire(s: AttemptState) -> String { // that owns it; recovery never derives it from the identity, a path spelling or a directory listing. // That is not tidiness: a derived name is a second authority for the binding, and a recovery acting // on a name it invented can stop a unit that belongs to a different attempt. +// +// THE QUANTITIES ARE CARRIED AS WHAT THEY ARE. The grant is a Kibibyte (std.measure), not an Int +// with the unit in its field name -- gunbc.compute.work_request's note on the stripped carrier was +// written for exactly that defect, and this record held it anyway from gunbc#12040's first commit +// until review 69915 read it. The two instants are EpochSecs, the same denomination the pool's own +// lease terms and now_epoch_seconds carry, so the comparison recovery makes against the deadline is +// between two values of one type. The unit and the epoch appear as bare numbers only at the JSON +// boundary below, where the wire keeps its spelling (granted_kib) and the decode re-admits them. type AttemptRecord { identity: String reference: String unit: String - granted_kib: Int - opened_at: Int - launch_deadline: Int + granted: Kibibyte + opened_at: EpochSecs + launch_deadline: EpochSecs state: AttemptState } @@ -135,7 +144,7 @@ fn attempt_record_json(r: AttemptRecord) -> JsonValue { json_kv(key: "identity", value: json_string(s: r.identity)), json_kv(key: "reference", value: json_string(s: r.reference)), json_kv(key: "unit", value: json_string(s: r.unit)), - json_kv(key: "granted_kib", value: json_int(n: r.granted_kib)), + json_kv(key: "granted_kib", value: json_int(n: kibibyte_count(k: r.granted))), json_kv(key: "opened_at", value: json_int(n: r.opened_at)), json_kv(key: "launch_deadline", value: json_int(n: r.launch_deadline)), json_kv(key: "state", value: json_string(s: attempt_state_wire(s: r.state))), @@ -148,7 +157,7 @@ fn attempt_record_wire(r: AttemptRecord) -> String { } fn attempt_record_with_state(r: AttemptRecord, state: AttemptState) -> AttemptRecord { - AttemptRecord { identity: r.identity, reference: r.reference, unit: r.unit, granted_kib: r.granted_kib, opened_at: r.opened_at, launch_deadline: r.launch_deadline, state: state } + AttemptRecord { identity: r.identity, reference: r.reference, unit: r.unit, granted: r.granted, opened_at: r.opened_at, launch_deadline: r.launch_deadline, state: state } } // ── READING THE SLOT, AND UNREADABLE IS NEVER ABSENT ────────────────────────────────────────── @@ -181,6 +190,22 @@ fn attempt_member_int(doc: JsonValue, key: String) -> Int? { } } +// AN INSTANT ON THE WIRE IS RE-ADMITTED, NOT CAST: a negative number is not an epoch second, and a +// record carrying one is unreadable rather than a record whose deadline is before 1970. +fn attempt_member_epoch(doc: JsonValue, key: String) -> EpochSecs? { + match attempt_member_int(doc: doc, key: key) { + Absent => none + Present { value: n } => if n < 0 { none } else { Present { value: n } } + } +} + +fn attempt_member_kibibytes(doc: JsonValue, key: String) -> Kibibyte? { + match attempt_member_int(doc: doc, key: key) { + Absent => none + Present { value: n } => if n < 0 { none } else { Present { value: kibibyte(count: n) } } + } +} + // A RECORD THAT DOES NOT DECODE IS UNREADABLE, NOT EMPTY, for the same reason the slot is. The // state word is reconstructed from the same spelling attempt_state_wire emits, and a word this fold // does not know is unreadable rather than defaulted -- a future state must not read as Reserved, @@ -198,13 +223,13 @@ fn attempt_record_decode(text: String) -> AttemptRecord? { match attempt_member(doc: doc, key: "unit") { Absent => none Present { value: unit } => - match attempt_member_int(doc: doc, key: "granted_kib") { + match attempt_member_kibibytes(doc: doc, key: "granted_kib") { Absent => none Present { value: granted } => - match attempt_member_int(doc: doc, key: "opened_at") { + match attempt_member_epoch(doc: doc, key: "opened_at") { Absent => none Present { value: opened } => - match attempt_member_int(doc: doc, key: "launch_deadline") { + match attempt_member_epoch(doc: doc, key: "launch_deadline") { Absent => none Present { value: deadline } => match attempt_member(doc: doc, key: "state") { @@ -213,7 +238,7 @@ fn attempt_record_decode(text: String) -> AttemptRecord? { match attempt_state_decode(word: word, detail: match attempt_member(doc: doc, key: "detail") { Present { value: d } => d Absent => "" }) { Absent => none Present { value: state } => - Present { value: AttemptRecord { identity: identity, reference: reference, unit: unit, granted_kib: granted, opened_at: opened, launch_deadline: deadline, state: state } } + Present { value: AttemptRecord { identity: identity, reference: reference, unit: unit, granted: granted, opened_at: opened, launch_deadline: deadline, state: state } } } } } @@ -429,11 +454,11 @@ fn attempt_transition_advanced(t: AttemptTransition) -> Bool { // winner; it decides who may enter. type RecoveryTrigger = RecoveryMayTerminate { reason: String } - | RecoveryTooEarly { deadline: Int, now: Int } + | RecoveryTooEarly { deadline: EpochSecs, now: EpochSecs } | RecoveryUnitNotEnded { lifecycle: UnitLifecycle } | RecoveryStateNotRecoverable { state: String } -fn recovery_trigger(record: AttemptRecord, lifecycle: UnitLifecycle, now: Int) -> RecoveryTrigger { +fn recovery_trigger(record: AttemptRecord, lifecycle: UnitLifecycle, now: EpochSecs) -> RecoveryTrigger { match record.state { AttemptReserved => if now < record.launch_deadline { diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index fc53dde591c..b2dcdac94a4 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -1,6 +1,6 @@ module gunbc.compute.work_provider_local -import std.types { String, Bool, Int, List, NonEmptyStr, FilePath, GitRef } +import std.types { String, Bool, Int, List, NonEmptyStr, FilePath, GitRef, EpochSecs } import std.process { ProcessExit, ExitSuccess, ExitFailure } import std.algebra { trim } import std.measure { byte_size_count } @@ -18,7 +18,7 @@ import extdeps.linux.cgroup_v2 { } import extdeps.cache.sccache { sccache_installed_binary_path } import gunbc.clock_read { clock_now_probed_at_or_unknown } -import extdeps.clock { Clock } +import gunbc.fabric_event_log_host { now_epoch_seconds } import std.durable_compare_and_set { CasGeneration, cas_generation_count } import gunbc.compute.attempt_lifecycle { AttemptRecord, AttemptState, AttemptReserved, AttemptLaunching, AttemptTerminal, @@ -39,7 +39,7 @@ import gunbc.roadmap_dispatch_actuator { dispatch_capability_program } import gunbc.roadmap_execution_contract { ExecutionCapability, GitWorkspaceCapability, CargoCapability, RustcCapability, AttemptStateDirectoryCapability, } -import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size } +import std.measure { Kibibyte, kibibyte_to_byte_size } import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, @@ -752,22 +752,15 @@ type AttemptGate // there moves this declaration with it instead of silently leaving the producer at a stale bound. data compute_launch_window_seconds: Int = compute_pre_launch_ceiling_seconds -fn compute_now_seconds() -> Int? { - match parse_int(s: trim(s: Clock.UnixSecs().unix_secs)) { - Present { value: n } => if n < 0 { none } else { Present { value: n } } - Absent => none - } -} - // THE UNIT NAME IS WRITTEN INTO THE RECORD BY THE ATTEMPT THAT OWNS IT, from the same fold the // invocation uses. That is the binding criterion 2 is about: recovery reads this field rather than // rebuilding the name from the identity, so a recovery can never act on a unit it named itself. -fn compute_attempt_record(identity: String, reference: NonEmptyStr, granted: Kibibyte, now: Int) -> AttemptRecord { +fn compute_attempt_record(identity: String, reference: NonEmptyStr, granted: Kibibyte, now: EpochSecs) -> AttemptRecord { AttemptRecord { identity: identity, reference: reference as String, unit: compute_unit_name(identity_hex: identity) as String, - granted_kib: kibibyte_count(k: granted) as Int, + granted: granted, opened_at: now, launch_deadline: now + compute_launch_window_seconds, state: AttemptReserved, @@ -784,7 +777,7 @@ fn compute_gate_from_transition(t: AttemptTransition, record: AttemptRecord, wha } fn compute_open_attempt(layout: ComputeLayout, identity: String, reference: NonEmptyStr, granted: Kibibyte) -> AttemptGate { - match compute_now_seconds() { + match now_epoch_seconds() { Absent => AttemptGateRefused { detail: "the host clock could not be read as epoch seconds, and an attempt's opened_at and launch deadline are both instants -- a fabricated one would make this attempt look expired to recovery the moment it is written" } Present { value: now } => { let record = compute_attempt_record(identity: identity, reference: reference, granted: granted, now: now) @@ -1457,7 +1450,7 @@ fn compute_recover_attempt(instance: HostDashboardInstance, reference: NonEmptyS RecoveryRefused { detail: join(["no attempt slot exists for ", reference as String, "; there is no record binding that reference to a unit, and this door will not release a seat it cannot name the attempt of"], "") } AttemptSlotHeld { record: rec, generation: g } => { let lifecycle = observe_unit_lifecycle(unit: rec.unit as NonEmptyStr) - match compute_now_seconds() { + match now_epoch_seconds() { Absent => RecoveryRefused { detail: "the host clock could not be read as epoch seconds, and the reserved-state trigger is a comparison against a deadline" } Present { value: now } => match recovery_trigger(record: rec, lifecycle: lifecycle, now: now) { diff --git a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag index 721c5524ff4..983c48464f3 100644 --- a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag +++ b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag @@ -4,6 +4,7 @@ import extdeps.shell import extdeps.filesystem.filesystem_io { Filesystem } import std.types { Bool, Int, NonEmptyStr, String } import std.algebra { trim } +import std.measure { kibibyte, kibibyte_count } import std.durable_compare_and_set { CasGeneration, cas_generation_count } import gunbc.compute.attempt_lifecycle { AttemptRecord, AttemptState, AttemptReserved, AttemptLaunching, AttemptTerminal, @@ -30,7 +31,7 @@ fn record_for(reference: String) -> AttemptRecord { identity: "abcdef0123456789", reference: reference, unit: join(["gunbc-compute-abcdef0123456789-", reference], ""), - granted_kib: 27262976, + granted: kibibyte(count: 27262976), opened_at: 1800, launch_deadline: 2000, state: AttemptReserved, @@ -133,7 +134,7 @@ test fn the_unit_binding_is_read_back_from_the_record_the_attempt_wrote() -> Boo && recovered.unit == record.unit && recovered.identity == record.identity && recovered.reference == record.reference - && recovered.granted_kib == 27262976 + && kibibyte_count(k: recovered.granted) == 27262976 && recovered.unit != record.identity } @@ -200,12 +201,12 @@ test fn recovery_cannot_fire_inside_the_window_the_producer_declared() -> Bool { test fn a_launched_attempt_is_recovered_only_on_an_ended_unit() -> Bool { let launched = AttemptRecord { identity: "abcdef0123456789", reference: "attempt-launched", - unit: "gunbc-compute-abcdef0123456789-attempt-launched", granted_kib: 27262976, + unit: "gunbc-compute-abcdef0123456789-attempt-launched", granted: kibibyte(count: 27262976), opened_at: 1800, launch_deadline: 2000, state: AttemptLaunching, } let terminal = AttemptRecord { identity: "abcdef0123456789", reference: "attempt-done", - unit: "gunbc-compute-abcdef0123456789-attempt-done", granted_kib: 27262976, + unit: "gunbc-compute-abcdef0123456789-attempt-done", granted: kibibyte(count: 27262976), opened_at: 1800, launch_deadline: 2000, state: AttemptTerminal { verdict: "released", detail: "done" }, } recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitTerminatedSuccess { detail: "exit 0" }, now: 9999)) @@ -226,9 +227,15 @@ test fn an_undecodable_slot_is_unreadable_and_never_absent() -> Bool { let planted = Filesystem.Write(path: join([root as String, "/", reference, ".1"], ""), content: "this is not a json document at all") let reading = attempt_slot_read(root: root, reference: reference as NonEmptyStr) let absent_reading = attempt_slot_read(root: root, reference: "attempt-never-written" as NonEmptyStr) + let negative_instant = attempt_record_decode(text: "{\"identity\":\"abcdef0123456789\",\"reference\":\"attempt-negative\",\"unit\":\"u\",\"granted_kib\":1,\"opened_at\":-5,\"launch_deadline\":100,\"state\":\"reserved\"}") + let negative_grant = attempt_record_decode(text: "{\"identity\":\"abcdef0123456789\",\"reference\":\"attempt-negative\",\"unit\":\"u\",\"granted_kib\":-1,\"opened_at\":5,\"launch_deadline\":100,\"state\":\"reserved\"}") + let well_formed = attempt_record_decode(text: "{\"identity\":\"abcdef0123456789\",\"reference\":\"attempt-fine\",\"unit\":\"u\",\"granted_kib\":1,\"opened_at\":5,\"launch_deadline\":100,\"state\":\"reserved\"}") planted.success && (match reading { AttemptSlotUnreadable { detail: d } => string_contains(s: d, pattern: "cannot decode") AttemptSlotHeld { record: _, generation: _ } => false AttemptSlotAbsent => false }) && (match absent_reading { AttemptSlotAbsent => true AttemptSlotHeld { record: _, generation: _ } => false AttemptSlotUnreadable { detail: _ } => false }) + && (match negative_instant { Absent => true Present { value: _ } => false }) + && (match negative_grant { Absent => true Present { value: _ } => false }) + && (match well_formed { Present { value: r } => kibibyte_count(k: r.granted) == 1 && r.opened_at == 5 Absent => false }) } // THE SELF-DECLARED BOUND HAS A BOUND, AND AN OVER-LONG DECLARATION REFUSES RATHER THAN BEING @@ -245,15 +252,15 @@ test fn an_over_long_pre_launch_declaration_refuses_at_open_and_is_not_clamped() let root = fresh_root() as NonEmptyStr let base = record_for(reference: "attempt-greedy") let greedy = AttemptRecord { - identity: base.identity, reference: base.reference, unit: base.unit, granted_kib: base.granted_kib, + identity: base.identity, reference: base.reference, unit: base.unit, granted: base.granted, opened_at: 1800, launch_deadline: 1800 + compute_pre_launch_ceiling_seconds + 1, state: AttemptReserved, } let at_ceiling = AttemptRecord { - identity: base.identity, reference: "attempt-at-ceiling", unit: base.unit, granted_kib: base.granted_kib, + identity: base.identity, reference: "attempt-at-ceiling", unit: base.unit, granted: base.granted, opened_at: 1800, launch_deadline: 1800 + compute_pre_launch_ceiling_seconds, state: AttemptReserved, } let backwards = AttemptRecord { - identity: base.identity, reference: "attempt-backwards", unit: base.unit, granted_kib: base.granted_kib, + identity: base.identity, reference: "attempt-backwards", unit: base.unit, granted: base.granted, opened_at: 1800, launch_deadline: 1799, state: AttemptReserved, } let refused = attempt_open(root: root, record: greedy) From ec0c231197db48d3736d14fe3507119d578b057a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 04:50:43 +0000 Subject: [PATCH 29/44] The window constants are Seconds, and the wet witness is on the route that can reach it Two items, both from review 69915's population rather than its two sites. THE SWEEP'S THIRD INSTANCE. compute_pre_launch_ceiling_seconds and compute_launch_window_seconds carried the unit in the field NAME on a bare Int, which is the same defect finding 1 named: modeled twice, in the name and in the rendered text, and nowhere the compiler can see. std.types Seconds is the carrier and is consumed corpus-wide -- gunbc.auth.access_request holds the identical ceiling/requested shape -- so both constants take it. THE ROUTE GAP, which is not a code defect and is answered by the contract home its own package already used. The hermetic route has no arm for shell.Mktemp.DirWithTemplate, so the floor plans this witness and it reaches no subject. gunbc#11962's host_capacity_wet_witness_test.dag admits exactly this effect and is disposed of by a pair: a gunbc.ci.ci_layer_roots WitnessExclusionRow under excl_local_repo_wet_tempdir_write_reason, and every one of its functions on v2.workflow.local_repo_wet_terminal local_repo_wet_schedule. This file takes the same pair. Not a floor_route_gap enrolment: the exclusion keeps the identities out of the hermetic discovery corpus rather than recording a gap as held debt, which is what the neighbour does and what the effect warrants. ALL NINE FUNCTIONS ARE ENROLLED, NOT THE GAPPED SUBSET, and the two measurements are the argument. The #12040 CI run named five identities; a hermetic claim_batch over this tree ends six that way and reaches three verdicts. Which claims get a verdict hermetically is a property of the evaluation order, not of the claim, so a subset roster would be right today and wrong after an edit nobody connects to it. Measured wet over this tree before enrollment, which is the receipt the roster's own idiom requires: attempt_lifecycle_wet_witness 9/9 and work_request_witness 10/10 PASS, 19/19, no FAIL. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/ci/ci_layer_roots.dag | 5 ++ dag/gunbc/compute/attempt_lifecycle.dag | 4 +- dag/gunbc/compute/work_provider_local.dag | 4 +- src/v2/workflow/local_repo_wet_terminal.dag | 67 +++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/dag/gunbc/ci/ci_layer_roots.dag b/dag/gunbc/ci/ci_layer_roots.dag index 448b54f0df7..bbed91791a7 100644 --- a/dag/gunbc/ci/ci_layer_roots.dag +++ b/dag/gunbc/ci/ci_layer_roots.dag @@ -1031,6 +1031,11 @@ data witness_exclusion_frontier: List = [ classification: LocalRepoWetLane, reason: excl_local_repo_wet_tempdir_write_reason, dissolution: excl_local_repo_wet_dissolve}, + WitnessExclusionRow { + pattern: "attempt_lifecycle_wet_witness_test.dag", + classification: LocalRepoWetLane, + reason: excl_local_repo_wet_tempdir_write_reason, + dissolution: excl_local_repo_wet_dissolve}, WitnessExclusionRow { pattern: "review_sheet_legacy_declaration_wet_witness_test.dag", classification: LocalRepoWetLane, diff --git a/dag/gunbc/compute/attempt_lifecycle.dag b/dag/gunbc/compute/attempt_lifecycle.dag index 03d2ece1c0d..6d53cc8c3f2 100644 --- a/dag/gunbc/compute/attempt_lifecycle.dag +++ b/dag/gunbc/compute/attempt_lifecycle.dag @@ -1,6 +1,6 @@ module gunbc.compute.attempt_lifecycle -import std.types { String, Bool, Int, NonEmptyStr, EpochSecs } +import std.types { String, Bool, Int, NonEmptyStr, EpochSecs, Seconds } import std.measure { Kibibyte, kibibyte, kibibyte_count } import std.content_hash { content_hash_of_value } import std.durable_compare_and_set { @@ -342,7 +342,7 @@ fn attempt_commit(root: NonEmptyStr, record: AttemptRecord, expected: CasExpecta // a widen wearing a safety limit's clothes. The refusal is typed and names both numbers, so the // author of an over-long declaration learns what the ceiling is rather than discovering it as a // mysterious reclaim. -data compute_pre_launch_ceiling_seconds: Int = 300 +data compute_pre_launch_ceiling_seconds: Seconds = 300 // OPENING: the attempt declares itself Reserved, and it must be the FIRST writer of this reference. // ExpectSlotAbsent is what makes a duplicated reference a refusal rather than a silent overwrite of diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index b2dcdac94a4..c32cfb06cb2 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -1,6 +1,6 @@ module gunbc.compute.work_provider_local -import std.types { String, Bool, Int, List, NonEmptyStr, FilePath, GitRef, EpochSecs } +import std.types { String, Bool, Int, List, NonEmptyStr, FilePath, GitRef, EpochSecs, Seconds } import std.process { ProcessExit, ExitSuccess, ExitFailure } import std.algebra { trim } import std.measure { byte_size_count } @@ -750,7 +750,7 @@ type AttemptGate // ceiling in attempt_lifecycle is the authority on what is tolerable to the other contenders on // this host. Naming the ceiling's own row rather than restating 300 keeps one number: a change // there moves this declaration with it instead of silently leaving the producer at a stale bound. -data compute_launch_window_seconds: Int = compute_pre_launch_ceiling_seconds +data compute_launch_window_seconds: Seconds = compute_pre_launch_ceiling_seconds // THE UNIT NAME IS WRITTEN INTO THE RECORD BY THE ATTEMPT THAT OWNS IT, from the same fold the // invocation uses. That is the binding criterion 2 is about: recovery reads this field rather than diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index c59f66e7174..b5142de0b1c 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -242,6 +242,19 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // bin-witness wet class, whose standing excl_bin_wet_reason carries. Measured wet through // `gunbc run --claim-run` over this tree before enrollment: all ten held, and every metacharacter // probe failed without output rather than running its payload. +// NINE ATTEMPT-LIFECYCLE MEMBERS (2026-09-22, gunbc#12040). Same admitted effect as the seven +// host_capacity members directly below, and enrolled the same way: a real compare-and-set against a +// real generation-file store inside a directory the witness creates with shell.Mktemp.DirWithTemplate +// and removes -- no repository, no network, no cargo, no remote host, no install media. The floor +// PLANNED all nine and the hermetic route reached no subject for most of them: the #12040 run named +// five (cause route_gap, 'the hermetic route has no arm for DirWithTemplate'), and a hermetic +// claim_batch over this tree ended six that way with three verdicts. A changed witness ending in a +// route gap is admitted only when a wet terminal joins it. The whole file is enrolled rather than +// the gapped subset, and the two measurements disagreeing on which subset is itself the reason: a +// claim reaches a verdict hermetically only by not touching the directory on that path, which is a +// property of the evaluation order and not of the claim, and it moves under an edit. The file's exclusion row is gunbc.ci.ci_layer_roots +// excl_local_repo_wet_tempdir_write_reason, and it retires on the same mock coverage as its +// neighbours. Measured wet over this tree before enrollment: 9/9 held. fn local_repo_wet_schedule() -> List { [ WetScheduledClaim { @@ -286,6 +299,60 @@ fn local_repo_wet_schedule() -> List { function: "two_instances_on_one_host_resolve_to_one_pool_at_one_root", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "a_terminal_transition_refuses_a_launch_that_had_not_spawned_yet" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "a_terminal_transition_refuses_a_launch_that_had_not_spawned_yet", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "a_launch_that_won_refuses_a_recovery_arriving_behind_it" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "a_launch_that_won_refuses_a_recovery_arriving_behind_it", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "the_unit_binding_is_read_back_from_the_record_the_attempt_wrote" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "the_unit_binding_is_read_back_from_the_record_the_attempt_wrote", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "settling_records_why_and_does_not_erase_what_it_settled" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "settling_records_why_and_does_not_erase_what_it_settled", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "the_union_keeps_every_arm_either_list_carried_and_each_refuses_as_before" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "the_union_keeps_every_arm_either_list_carried_and_each_refuses_as_before", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "recovery_cannot_fire_inside_the_window_the_producer_declared" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "recovery_cannot_fire_inside_the_window_the_producer_declared", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "a_launched_attempt_is_recovered_only_on_an_ended_unit" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "a_launched_attempt_is_recovered_only_on_an_ended_unit", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "an_undecodable_slot_is_unreadable_and_never_absent" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "an_undecodable_slot_is_unreadable_and_never_absent", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "an_over_long_pre_launch_declaration_refuses_at_open_and_is_not_clamped" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "an_over_long_pre_launch_declaration_refuses_at_open_and_is_not_clamped", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.effect_plan_bash_materialize_real_execution_witness", function: "effect_plan_bash_two_declared_operations_execute" }, entry: "dag/test/claim/effect_plan_bash_materialize_real_execution_witness_test.dag", From 9d9f0747af4261f36b752104a930dec0560e795d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 04:52:15 +0000 Subject: [PATCH 30/44] The honest instrument conflates anyway: a second specimen where the cause is printed and the token still says FAIL The checkable question was whether the six route-gap lines carried their cause. THEY DID, verbatim: "FAIL (hermetic route has no arm for DirWithTemplate: operation declares no mock_response -- the claim never reached its subject, so this is a route gap and not a verdict)". So this is not the reporting layer distinguishing a gap on one path and conflating it on another; claim_batch says in words that the line is not a verdict. It says so AFTER the verdict token, which is the part worth filing: grep -c ^FAIL counts six reds, ^PASS reports three of nine, and the same nine are 9/9 with --wet over the same tree. FILED AS A RECEIPT ON AN EXISTING ROW RATHER THAN AS A NEW CLASS. gunbc.recurring_failure_mode an_unreached_measurement_renders_as_a_failed_one already owns this invalid state -- an unreached subject presented where adjudicated-and-lost is meant -- and a second row would be the section 3 fork the ledger exists to avoid. Only the grain moves, from a required lane at the merge surface to one identity inside a batch run. The two specimens bracket the remedy, which is why the second is worth keeping beside the first. The original has the distinction TYPED in a standing field the deciding surface never reads. This one has it RENDERED where every reader sees it and no reader can act on it mechanically. Neither is fixed by emitting more, which is what that row's trigger already says: the surface has to consume it. For a per-identity line the surface IS the token, so the remedy here is a token of its own, not a better parenthetical. Co-Authored-By: Claude Opus 5 (1M context) --- .../an_unreached_measurement_renders_as_a_failed_one.dag | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dag/gunbc/recurring_failure_mode/an_unreached_measurement_renders_as_a_failed_one.dag b/dag/gunbc/recurring_failure_mode/an_unreached_measurement_renders_as_a_failed_one.dag index 548be83c63a..44fb3d6d367 100644 --- a/dag/gunbc/recurring_failure_mode/an_unreached_measurement_renders_as_a_failed_one.dag +++ b/dag/gunbc/recurring_failure_mode/an_unreached_measurement_renders_as_a_failed_one.dag @@ -17,6 +17,10 @@ data an_unreached_measurement_renders_as_a_failed_one: RecurringFailureMode = Re "SPECIMEN (2026-09-11, gunbc main). Two consecutive main runs on two slots of one host reported required-witnesses-floor FAILURE with standing=measurement_unreached: srv4-04 on run 34570523344 (head d94bcf680) could not read a cargo registry source file mid-compile, and srv4-16 on run 34571176215 (head ec1571e89) could not write a target .rmeta. Both are ENOENT rather than ENOSPC, which is what rules out a full disk and is itself a distinguishing fact worth keeping. THE COST LANDED EXACTLY WHERE THIS CLASS PREDICTS: d94bcf680 IS #11032, the repair for a separate witness-budget defect that had main red, so the one run that could have told its lane whether the repair worked reported red while measuring nothing -- and the natural reading, `still red after the fix`, was unsupported in either direction. Three lanes were downstream of that misreading.", + "SECOND SPECIMEN, ONE LAYER DOWN AND WITH THE DISTINCTION PARTLY PRESENT (2026-09-22, gunbc#12040). `claim_batch` over dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag WITHOUT --wet, on the hermetic route, ended six of nine identities at shell.Mktemp.DirWithTemplate, which declares no mock_response. The line it prints is `FAIL (hermetic route has no arm for DirWithTemplate: operation declares no mock_response -- the claim never reached its subject, so this is a route gap and not a verdict)`. THE HARNESS IS HONEST AND STILL CONFLATES, which is the new fact this receipt carries: the cause is present, correct, and says IN WORDS that this is not a verdict -- and it is carried in PROSE AFTER THE VERDICT TOKEN, so every mechanical reader takes the token and drops the disclaimer. `grep -c ^FAIL` counts six reds, `grep -c ^PASS` reports three of nine, and a reader scanning the verdict column sees a witness that is mostly red. The same nine are 9/9 PASS with --wet over the same tree. So the remedy this specimen argues for is narrower than the first one and is NOT more log content: the token is the surface, and a route gap needs a token of its own rather than a parenthetical attached to FAIL.", + + "WHY THAT IS THE SAME CLASS AND NOT A SEPARATE ONE, since the instruments differ: the invalid state is identical -- an unreached subject presented in the position that means `this subject was adjudicated and lost` -- and only the grain moves, from a required lane at the merge surface to one identity in a batch run. The two specimens also bracket the remedy usefully. The first has the distinction typed in a field the deciding surface never reads; the second has it rendered where every reader sees it and no reader can act on it mechanically. Emitting the standing is not sufficient in either, which is what the next-rung trigger below already says.", + "NOT THE SAME AS A FLAKE, and the distinction matters for what to do next: a flake is a measurement that reached its subject and got the wrong answer, so re-running samples the same distribution. An unreached measurement has no answer to sample; re-running is only worth doing if the reason it could not reach is itself transient, and NOTHING IN THE RED SAYS WHETHER IT IS. Treating the two alike is how a broken runner gets re-run indefinitely.", "RUNG: found at 1, mitigatable. The distinction is not lost -- the standing field is typed and present in the log, so a reader who opens it can always recover which case they are in. What is lost is that the DECIDING surface does not carry it, so recovery depends on a human knowing to look, which is exactly the dependency a mitigation has and a guarantee does not.", From 3b2a04e083acc30f2a6547a9eadc1a4cfd742e57 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 04:55:34 +0000 Subject: [PATCH 31/44] The token is the defect, not the cause: name the remedy as a distinct terminal Per the parent's point on 9d9f0747. The row said the cause was printed and left a reader to infer what follows from that. It now says it outright: this is NOT the harness conflating a route gap with a verdict -- the cause is computed, correct and rendered -- and what fails is that it is carried under a token that reads as a verdict. The remedy is therefore a distinct terminal, not more annotation, and an instruction to read the parenthetical is a mitigation carrying exactly the dependency on a human knowing to look that the row's own RUNG note names. Held out of the push that is currently building: a push cancels the in-flight floor run, and this is prose on a data row. Co-Authored-By: Claude Opus 5 (1M context) --- .../an_unreached_measurement_renders_as_a_failed_one.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dag/gunbc/recurring_failure_mode/an_unreached_measurement_renders_as_a_failed_one.dag b/dag/gunbc/recurring_failure_mode/an_unreached_measurement_renders_as_a_failed_one.dag index 44fb3d6d367..b6f36ea27ad 100644 --- a/dag/gunbc/recurring_failure_mode/an_unreached_measurement_renders_as_a_failed_one.dag +++ b/dag/gunbc/recurring_failure_mode/an_unreached_measurement_renders_as_a_failed_one.dag @@ -17,7 +17,7 @@ data an_unreached_measurement_renders_as_a_failed_one: RecurringFailureMode = Re "SPECIMEN (2026-09-11, gunbc main). Two consecutive main runs on two slots of one host reported required-witnesses-floor FAILURE with standing=measurement_unreached: srv4-04 on run 34570523344 (head d94bcf680) could not read a cargo registry source file mid-compile, and srv4-16 on run 34571176215 (head ec1571e89) could not write a target .rmeta. Both are ENOENT rather than ENOSPC, which is what rules out a full disk and is itself a distinguishing fact worth keeping. THE COST LANDED EXACTLY WHERE THIS CLASS PREDICTS: d94bcf680 IS #11032, the repair for a separate witness-budget defect that had main red, so the one run that could have told its lane whether the repair worked reported red while measuring nothing -- and the natural reading, `still red after the fix`, was unsupported in either direction. Three lanes were downstream of that misreading.", - "SECOND SPECIMEN, ONE LAYER DOWN AND WITH THE DISTINCTION PARTLY PRESENT (2026-09-22, gunbc#12040). `claim_batch` over dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag WITHOUT --wet, on the hermetic route, ended six of nine identities at shell.Mktemp.DirWithTemplate, which declares no mock_response. The line it prints is `FAIL (hermetic route has no arm for DirWithTemplate: operation declares no mock_response -- the claim never reached its subject, so this is a route gap and not a verdict)`. THE HARNESS IS HONEST AND STILL CONFLATES, which is the new fact this receipt carries: the cause is present, correct, and says IN WORDS that this is not a verdict -- and it is carried in PROSE AFTER THE VERDICT TOKEN, so every mechanical reader takes the token and drops the disclaimer. `grep -c ^FAIL` counts six reds, `grep -c ^PASS` reports three of nine, and a reader scanning the verdict column sees a witness that is mostly red. The same nine are 9/9 PASS with --wet over the same tree. So the remedy this specimen argues for is narrower than the first one and is NOT more log content: the token is the surface, and a route gap needs a token of its own rather than a parenthetical attached to FAIL.", + "SECOND SPECIMEN, ONE LAYER DOWN AND WITH THE DISTINCTION PARTLY PRESENT (2026-09-22, gunbc#12040). `claim_batch` over dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag WITHOUT --wet, on the hermetic route, ended six of nine identities at shell.Mktemp.DirWithTemplate, which declares no mock_response. The line it prints is `FAIL (hermetic route has no arm for DirWithTemplate: operation declares no mock_response -- the claim never reached its subject, so this is a route gap and not a verdict)`. THE HARNESS IS HONEST AND STILL CONFLATES, which is the new fact this receipt carries: the cause is present, correct, and says IN WORDS that this is not a verdict -- and it is carried in PROSE AFTER THE VERDICT TOKEN, so every mechanical reader takes the token and drops the disclaimer. `grep -c ^FAIL` counts six reds, `grep -c ^PASS` reports three of nine, and a reader scanning the verdict column sees a witness that is mostly red. The same nine are 9/9 PASS with --wet over the same tree. THIS IS NOT THE HARNESS CONFLATING A ROUTE GAP WITH A VERDICT, and the distinction decides the remedy. The cause is computed, correct, and printed; what fails is that it is carried UNDER A TOKEN THAT READS AS A VERDICT. So the remedy this specimen argues for is narrower than the first one and is NOT more annotation: the token is the surface, and a route gap needs A DISTINCT TERMINAL of its own rather than a parenthetical attached to FAIL. A reader instructed to check the parenthetical is a mitigation with the same dependency on a human knowing to look that the RUNG note below names.", "WHY THAT IS THE SAME CLASS AND NOT A SEPARATE ONE, since the instruments differ: the invalid state is identical -- an unreached subject presented in the position that means `this subject was adjudicated and lost` -- and only the grain moves, from a required lane at the merge surface to one identity in a batch run. The two specimens also bracket the remedy usefully. The first has the distinction typed in a field the deciding surface never reads; the second has it rendered where every reader sees it and no reader can act on it mechanically. Emitting the standing is not sufficient in either, which is what the next-rung trigger below already says.", From abfcb8c9581cb59a8aad7eeb33d7387619d03d88 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 05:02:28 +0000 Subject: [PATCH 32/44] A duplicated roster identity was unrepresented by every claim over the roster Raised on the #12040/#12044 merge. Append-against-append resolves as a union, which is right when the two appended rows are independent and silently wrong when both sides appended the SAME identity -- git cannot see that, because both sides did the same legal thing, so the merge produces a green tree carrying a duplicate. The read that cleared that merge was an eye over 220 rows. NEITHER SIBLING CLAIM CAN SEE ONE, and the second is the interesting half. every_roster_row_spells_its_function_the_same_way_twice reads WITHIN a row. And the_local_roster_is_not_empty compares length(identities) == length(schedule), which is a map -- a map never changes a list's length, so that equality holds over a roster with a duplicate in it. It reads like a completeness check and is structurally incapable of being one. MEASURED RATHER THAN ARGUED. Duplicating one row in the real schedule: no_two_roster_rows_name_the_same_identity FAIL, the_local_roster_is_not_empty PASS, every_roster_row_spells_its_function_the_same_way_twice PASS. Restored tree: all three PASS. So the claim is wired to the roster and the reading above is the run's, not mine. the_duplicate_identity_predicate_discriminates is the second control and covers what the mutation cannot: the roster claim is a filter returning zero, and a filter over a predicate that never fires returns zero too. Two supplied lists differing only in whether one name repeats separate those. Co-Authored-By: Claude Opus 5 (1M context) --- .../claim/local_repo_wet_terminal_test.dag | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/v2/test/claim/local_repo_wet_terminal_test.dag b/src/v2/test/claim/local_repo_wet_terminal_test.dag index d52a40bed38..c5a425b7638 100644 --- a/src/v2/test/claim/local_repo_wet_terminal_test.dag +++ b/src/v2/test/claim/local_repo_wet_terminal_test.dag @@ -3,6 +3,7 @@ module v2.test.claim.local_repo_wet_terminal_test import v2.std.collection { List } import v2.std.logic { Bool } import v2.std.text { String } +import v2.std.integer { Int } import v2.std.algebra { length, filter } import v2.workflow.floor2_prepared_subject { WitnessIdentity, witness_identity_qualified_name } import v2.workflow.floor_terminal_ledger { @@ -332,3 +333,62 @@ test fn the_gate_predicate_admits_only_an_exact_identity_and_candidate() -> Bool candidate: fixture_candidate ) } + +// NO TWO ROWS NAME ONE IDENTITY, which the roster had no check for and which the sibling claims +// cannot see. `the_local_roster_is_not_empty` compares the length of the identity projection against +// the length of the schedule, and a map never changes a list's length, so that equality holds over a +// roster with a duplicate in it. `every_roster_row_spells_its_function_the_same_way_twice` reads +// WITHIN one row. So a repeated identity was unrepresented by every existing claim. +// +// THE POPULATION THIS EXISTS FOR IS A MERGE, not an author. Two branches each appending a row to +// this list conflict textually, and the correct resolution of append-against-append is a union -- +// which is right exactly when the two rows are independent and silently wrong when both sides +// appended the SAME identity. git cannot see that: both sides did the same legal thing, and the +// union it produces is a green tree carrying a duplicate. gunbc#12040 against gunbc#12044 was that +// merge, and the read that found it clean was an eye over 220 rows, which is the mitigation this +// claim replaces. +fn roster_identity_occurrences(name: String) -> Int { + length( + xs: filter( + xs: local_repo_wet_scheduled_identities(), + predicate: fn(n) { n == name } + ) + ) +} + +test fn no_two_roster_rows_name_the_same_identity() -> Bool { + length( + xs: filter( + xs: local_repo_wet_scheduled_identities(), + predicate: fn(n) { roster_identity_occurrences(name: n) != 1 } + ) + ) == 0 +} + +// THE DISCRIMINATING CONTROL, because the claim above is a filter that returns zero and a filter +// over a predicate that can never fire returns zero too. These two supplied lists differ only in +// whether one name repeats, so a predicate that stopped discriminating fails here while the roster +// claim stays green. +fn names_with_a_duplicate() -> List { + ["test.claim.x.a_holds", "test.claim.x.b_holds", "test.claim.x.a_holds"] +} + +fn names_all_distinct() -> List { + ["test.claim.x.a_holds", "test.claim.x.b_holds", "test.claim.x.c_holds"] +} + +fn every_name_occurs_once(names: List) -> Bool { + length( + xs: filter( + xs: names, + predicate: fn(n) { + length(xs: filter(xs: names, predicate: fn(m) { m == n })) != 1 + } + ) + ) == 0 +} + +test fn the_duplicate_identity_predicate_discriminates() -> Bool { + every_name_occurs_once(names: names_all_distinct()) + && !every_name_occurs_once(names: names_with_a_duplicate()) +} From d2be49872f63b85eae136bd889801920e72ca460 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 05:38:42 +0000 Subject: [PATCH 33/44] A refused gate settled nothing, and the receipt said it settled the slot Review 69967, and it is fabricated plausible output on a reachable path rather than a modeling point. compute_run_leased returns AttemptGateRefused when the launch CAS loses, compute_run_under_grant writes compute_attempt_settlement_wire into the consumption receipt, and the refusal mapped to HostRecordWritten -- so the operator-facing record of a LOST launch race asserted this producer settled the attempt slot, having settled nothing and observed nothing about the slot's final state. DESIGN section 5. THE CAUSE IS A THREE-VALUED FACT ON A TWO-ARMED CARRIER, so the third case had nowhere to go but the success arm. The fix is the third arm, and the argument is about WHOSE TYPE it belongs to. It is not on HostRecordStatus: that answers one question -- did this host record its own side facts, or is there a loss an operator must chase -- for five producers, and `nothing was owed here` is not an answer to it for any of the others. The arm would be uninhabitable everywhere but this call and would owe host_record_join and host_record_was_written a rule with no meaning to carry. So settlement gets AttemptSettlement (Settled / NotSettled / Unsettleable) and a total projection into the shared carrier. UNSETTLEABLE PROJECTS TO HostRecordWritten DELIBERATELY. A loss means this host owed a record and failed to write it, which is an operator action. A refused gate owed nothing: it never held the slot. Mapping it to HostRecordLost would manufacture an incident -- the same fabrication as the success arm, pointed the other way. CONTROL, driving the REAL race rather than describing it: two producers open against one store, the winner advances the generation, so the loser's begin_launch is refused BY THE STORE. PASS. Mutation restoring `AttemptGateRefused => AttemptSettled`: FAIL, on a clean run. An earlier mutation attempt was killed and is reported nowhere, because the tree was restored while it executed -- a run whose input changed under it has no verdict. ROUTE-GAP ENROLMENT, which is the other half of the CI red at 9d9f0747. The exclusion row keeps this file off the floor on every run that does not touch it; a CHANGED witness is selected regardless of discovery exclusion, so the PR that edits it plans the identities anyway. floor_route_gap_expectation_chunk_21 enrols exactly the six that gapped, and the row names what re-observes it. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 52 ++++++++++++++--- .../attempt_lifecycle_wet_witness_test.dag | 57 ++++++++++++++++++- src/v2/workflow/floor_route_gap.dag | 53 ++++++++++++++++- 3 files changed, 151 insertions(+), 11 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index c32cfb06cb2..9056f65efcd 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -803,29 +803,63 @@ fn compute_begin_launch(layout: ComputeLayout, gate: AttemptGate) -> AttemptGate // the history is kept), and if the transition does not land the attempt is still open in the store // -- which is a fact an operator needs, so it rides out on the same carrier as every other // recording failure this provider can suffer. -fn compute_settle_attempt(layout: ComputeLayout, gate: AttemptGate, verdict: String, detail: String) -> HostRecordStatus { +// SETTLEMENT IS A THREE-VALUED FACT AND IT GETS A THREE-ARMED CARRIER. Review 69967 found the third +// value forced onto HostRecordStatus, which has two arms, so "this producer never held the slot" +// landed in the arm that means "recorded". The consumption receipt then read `attempt: settled in +// the attempt slot` on a LOST LAUNCH RACE -- a reachable path, and the inverse of the argument the +// settlement line exists for: a line that says nothing on the good path cannot be read as evidence, +// and a line that says the good path happened where it did not is worse. +// +// WHY A NEW TYPE RATHER THAN A THIRD ARM ON HostRecordStatus, which was the other constructive +// option. HostRecordStatus answers ONE question -- did this host record its own side facts, or is +// there a loss an operator must hear about -- for five producers (the release, the lease drop, the +// measurement write, the joins over all of them). `nothing was owed here` is not an answer to that +// question for any of the others, so the arm would be uninhabitable everywhere but this call and +// would owe `host_record_join` and `host_record_was_written` a rule with no meaning to carry. The +// three-valued fact belongs to SETTLEMENT, so it gets settlement's own type, and the projection into +// the shared carrier below is total and says exactly why each arm maps where it does. +type AttemptSettlement + = AttemptSettled + | AttemptNotSettled { detail: String } + | AttemptUnsettleable { detail: String } + +fn compute_settle_attempt(layout: ComputeLayout, gate: AttemptGate, verdict: String, detail: String) -> AttemptSettlement { match gate { - AttemptGateRefused { detail: _ } => HostRecordWritten + AttemptGateRefused { detail: d } => AttemptUnsettleable { detail: d } AttemptGateReady { record: rec, generation: g } => match attempt_terminate(root: layout.attempts_dir as NonEmptyStr, record: rec, expected: g, verdict: verdict, detail: detail) { - AttemptAdvanced { state: _, generation: _ } => HostRecordWritten + AttemptAdvanced { state: _, generation: _ } => AttemptSettled AttemptLostRace { expected: e, observed: o } => - HostRecordLost { detail: join(["the attempt slot for ", rec.reference, " could not be closed: expected ", e, " and it holds ", o, "; the attempt stays open in the store and the seat's settlement is recorded only here"], "") } + AttemptNotSettled { detail: join(["the attempt slot for ", rec.reference, " could not be closed: expected ", e, " and it holds ", o, "; the attempt stays open in the store and the seat's settlement is recorded only here"], "") } AttemptTransitionRefused { detail: d } => - HostRecordLost { detail: join(["the attempt slot for ", rec.reference, " could not be closed: ", d, "; the attempt stays open in the store"], "") } + AttemptNotSettled { detail: join(["the attempt slot for ", rec.reference, " could not be closed: ", d, "; the attempt stays open in the store"], "") } } } } +// THE PROJECTION INTO THE HOST-RECORD CARRIER, AND UNSETTLEABLE IS DELIBERATELY NOT A LOSS. A loss +// means this host owed a record and failed to write it, which is an operator action. A producer +// whose gate was refused owed nothing: it never held the slot, so there is no missing record and +// nothing for an operator to chase. Mapping it to HostRecordLost would manufacture an incident, which +// is the same fabrication as the success arm did, pointed the other way. +fn attempt_settlement_host_record(s: AttemptSettlement) -> HostRecordStatus { + match s { + AttemptSettled => HostRecordWritten + AttemptNotSettled { detail: d } => HostRecordLost { detail: d } + AttemptUnsettleable { detail: _ } => HostRecordWritten + } +} + // THE SETTLEMENT LINE HAS ITS OWN WORDS BECAUSE host_record_wire RENDERS SUCCESS AS THE EMPTY // STRING -- it is built to be APPENDED to a sentence, so reusing it here produced a receipt line // reading "attempt: " with nothing after it. A line that says nothing on the good path cannot be // read as evidence that the good path happened, which is the whole reason the line is in the // receipt. Found in the first srv1 run at this head. -fn compute_attempt_settlement_wire(closed: HostRecordStatus) -> String { +fn compute_attempt_settlement_wire(closed: AttemptSettlement) -> String { match closed { - HostRecordWritten => "settled in the attempt slot" - HostRecordLost { detail: d } => join(["NOT SETTLED -- ", d], "") + AttemptSettled => "settled in the attempt slot" + AttemptNotSettled { detail: d } => join(["NOT SETTLED -- ", d], "") + AttemptUnsettleable { detail: d } => join(["NOT SETTLED BY THIS PRODUCER -- it never held the attempt slot: ", d, ". Whatever the slot holds was written by whoever did hold it, and this run observed none of it"], "") } } @@ -926,7 +960,7 @@ fn compute_run_under_grant( } else { compute_with_lost_host_record(settled: settled, layout: layout, dropped: dropped, error: measured.error) }, - carried: closed) + carried: attempt_settlement_host_record(s: closed)) } // THE HOST RECORD IS THE ONLY DURABLE CARRIER OF THREE FACTS, so losing it is reported rather than diff --git a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag index 983c48464f3..7b90bfea1aa 100644 --- a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag +++ b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag @@ -5,7 +5,14 @@ import extdeps.filesystem.filesystem_io { Filesystem } import std.types { Bool, Int, NonEmptyStr, String } import std.algebra { trim } import std.measure { kibibyte, kibibyte_count } -import std.durable_compare_and_set { CasGeneration, cas_generation_count } +import std.durable_compare_and_set { CasGeneration, cas_generation_count, cas_generation_first } +import gunbc.compute.work_provider_local { + ComputeLayout, + AttemptGate, AttemptGateReady, AttemptGateRefused, compute_begin_launch, + AttemptSettlement, AttemptSettled, AttemptNotSettled, AttemptUnsettleable, + compute_settle_attempt, compute_attempt_settlement_wire, attempt_settlement_host_record, + host_record_was_written, +} import gunbc.compute.attempt_lifecycle { AttemptRecord, AttemptState, AttemptReserved, AttemptLaunching, AttemptTerminal, AttemptSlotReading, AttemptSlotAbsent, AttemptSlotHeld, AttemptSlotUnreadable, attempt_slot_read, @@ -275,3 +282,51 @@ test fn an_over_long_pre_launch_declaration_refuses_at_open_and_is_not_clamped() && string_contains(s: attempt_transition_wire(t: closed_window), pattern: "already closed") && (match greedy_slot { AttemptSlotAbsent => true AttemptSlotHeld { record: _, generation: _ } => false AttemptSlotUnreadable { detail: _ } => false }) } + + +// THE LOSER OF A LAUNCH RACE MAY NOT REPORT THAT IT SETTLED THE SLOT (review 69967). This is the +// reachable path: compute_run_leased returns its gate from compute_begin_launch, so a producer whose +// launch CAS lost carries AttemptGateRefused into compute_settle_attempt, and the string that fold +// produces is written into the operator-facing consumption receipt. Before the third arm existed the +// refusal mapped to HostRecordWritten and the receipt read `attempt: settled in the attempt slot` for +// a producer that settled nothing and observed nothing about the slot's final state. +// +// THE RACE IS REAL AND NOT DESCRIBED. Two producers open against one store; the winner advances the +// generation, so the loser's begin_launch is refused BY THE STORE. The claim then asserts three +// things that a two-armed carrier could not have kept apart: the winner says settled, the loser's +// wire denies it and says who to believe instead, and the loser is NOT reported as a host-record +// LOSS -- because it owed no record, and calling it a loss would manufacture an incident for an +// operator to chase. +fn layout_at(root: String) -> ComputeLayout { + ComputeLayout { + root: root, work_dir: root, checkout: root, out_dir: root, + leases_dir: root, lease_path: root, store_dir: root, target_dir: root, + log_path: root, outcome_path: root, consumption_path: root, + attempts_dir: root, + } +} + +test fn the_loser_of_a_launch_race_never_reports_that_it_settled_the_slot() -> Bool { + let root = fresh_root() + let layout = layout_at(root: root) + let rec = record_for(reference: "attempt-race") + let opened = attempt_open(root: root as NonEmptyStr, record: rec) + let generation = match opened { + AttemptAdvanced { state: _, generation: g } => g + AttemptLostRace { expected: _, observed: _ } => cas_generation_first() + AttemptTransitionRefused { detail: _ } => cas_generation_first() + } + let winner = compute_begin_launch(layout: layout, gate: AttemptGateReady { record: rec, generation: generation }) + let loser = compute_begin_launch(layout: layout, gate: AttemptGateReady { record: rec, generation: generation }) + let winner_settled = compute_settle_attempt(layout: layout, gate: winner, verdict: "succeeded", detail: "the winner closes the slot") + let loser_settled = compute_settle_attempt(layout: layout, gate: loser, verdict: "succeeded", detail: "the loser has no slot to close") + attempt_transition_advanced(t: opened) + && (match winner { AttemptGateReady { record: _, generation: _ } => true AttemptGateRefused { detail: _ } => false }) + && (match loser { AttemptGateReady { record: _, generation: _ } => false AttemptGateRefused { detail: _ } => true }) + && (match winner_settled { AttemptSettled => true AttemptNotSettled { detail: _ } => false AttemptUnsettleable { detail: _ } => false }) + && (match loser_settled { AttemptSettled => false AttemptNotSettled { detail: _ } => false AttemptUnsettleable { detail: _ } => true }) + && compute_attempt_settlement_wire(closed: winner_settled) == "settled in the attempt slot" + && !string_contains(s: compute_attempt_settlement_wire(closed: loser_settled), pattern: "settled in the attempt slot") + && string_contains(s: compute_attempt_settlement_wire(closed: loser_settled), pattern: "NOT SETTLED BY THIS PRODUCER") + && host_record_was_written(h: attempt_settlement_host_record(s: loser_settled)) +} diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 26a0c5d52c1..0afac8aacf9 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1464,8 +1464,59 @@ fn floor_route_gap_expectation_chunk_20() -> List { } } +// test.claim.compute.attempt_lifecycle_wet_witness: these six reach shell.Mktemp.DirWithTemplate, +// which declares no mock_response, so the nominal floor refuses there and each identity is held. +// +// WHY THE ROSTER IS SIX AND NOT THE FILE'S NINE, which is the opposite of how the same file is +// enrolled on the wet lane and is not an inconsistency. An enrolled row whose identity does NOT gap +// is a STALE-ROUTE-GAP and refuses the run in the other direction, so this roster must name exactly +// the identities that gap -- it is an observation of one run, not a policy about a file. The wet +// schedule names all nine because it is a policy about an effect. The six are the ones the floor run +// on gunbc#12040 at 9d9f0747 reported, and a hermetic claim_batch over the same tree reported the +// same six; the CI run at the PREVIOUS head reported five, which is why the wet roster is not scoped +// this way. +// +// THE EXCLUSION ROW DOES NOT COVER THESE, and that is the fact that made the first remedy +// incomplete. gunbc.ci.ci_layer_roots excludes the file from DISCOVERY, which is what keeps the +// file's siblings off the nominal floor -- but a CHANGED witness is selected regardless of discovery +// exclusion, so a PR that edits this file plans its identities anyway. The exclusion holds for every +// run that does not touch the file, and this roster plus the wet terminal is what holds for the run +// that does. +// +// WHAT RE-OBSERVES THIS ROSTER, STATED BECAUSE NOTHING ELSE WOULD SAY IT. A six-row observation can +// go stale, and the staleness refuses from the other direction, so the obligation is named here +// rather than left for whoever meets the refusal. It is NARROWER than it first looks and the reason +// is the same exclusion: these identities are planned ONLY on a run that edits this file. An +// ordinary run never observes them, and an enrolled identity that does not execute does not red the +// floor -- the evidence is main itself, which carries ninety-nine wet-witness rows for files that +// are likewise discovery-excluded and is green. So this roster cannot rot quietly between heads. +// +// THE OBLIGATION IS THEREFORE ON THE NEXT CHANGE THAT TOUCHES THIS FILE, and it is one of two +// mechanical repairs, each named by the run that reports it: an identity that gaps and is not +// enrolled arrives as ROUTE-GAP and wants a row added; an identity enrolled here that reaches its +// verdict arrives as STALE-ROUTE-GAP and wants its row deleted. Both are the floor telling the +// author what the current subset is, which is the only authority for it -- the subset is a property +// of evaluation order, and the five-at-one-head against six-at-the-next measured on gunbc#12040 is +// what rules out predicting it. +fn floor_route_gap_expectation_chunk_21() -> List { + Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.a_launch_that_won_refuses_a_recovery_arriving_behind_it", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.a_terminal_transition_refuses_a_launch_that_had_not_spawned_yet", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.an_over_long_pre_launch_declaration_refuses_at_open_and_is_not_clamped", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.an_undecodable_slot_is_unreadable_and_never_absent", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.settling_records_why_and_does_not_erase_what_it_settled", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.the_unit_binding_is_read_back_from_the_record_the_attempt_wrote", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Empty {} + } } } } } } +} + fn floor_route_gap_expectation_chunks() -> List> { - Cons { head: floor_route_gap_expectation_chunk_00(), tail: Cons { head: floor_route_gap_expectation_chunk_01(), tail: Cons { head: floor_route_gap_expectation_chunk_02(), tail: Cons { head: floor_route_gap_expectation_chunk_03(), tail: Cons { head: floor_route_gap_expectation_chunk_04(), tail: Cons { head: floor_route_gap_expectation_chunk_05(), tail: Cons { head: floor_route_gap_expectation_chunk_06(), tail: Cons { head: floor_route_gap_expectation_chunk_07(), tail: Cons { head: floor_route_gap_expectation_chunk_08(), tail: Cons { head: floor_route_gap_expectation_chunk_09(), tail: Cons { head: floor_route_gap_expectation_chunk_10(), tail: Cons { head: floor_route_gap_expectation_chunk_11(), tail: Cons { head: floor_route_gap_expectation_chunk_12(), tail: Cons { head: floor_route_gap_expectation_chunk_13(), tail: Cons { head: floor_route_gap_expectation_chunk_14(), tail: Cons { head: floor_route_gap_expectation_chunk_15(), tail: Cons { head: floor_route_gap_expectation_chunk_16(), tail: Cons { head: floor_route_gap_expectation_chunk_17(), tail: Cons { head: floor_route_gap_expectation_chunk_18(), tail: Cons { head: floor_route_gap_expectation_chunk_19(), tail: Cons { head: floor_route_gap_expectation_chunk_20(), tail: Empty {} } } } } } } } } } } } } } } } } } } } } } + Cons { head: floor_route_gap_expectation_chunk_00(), tail: Cons { head: floor_route_gap_expectation_chunk_01(), tail: Cons { head: floor_route_gap_expectation_chunk_02(), tail: Cons { head: floor_route_gap_expectation_chunk_03(), tail: Cons { head: floor_route_gap_expectation_chunk_04(), tail: Cons { head: floor_route_gap_expectation_chunk_05(), tail: Cons { head: floor_route_gap_expectation_chunk_06(), tail: Cons { head: floor_route_gap_expectation_chunk_07(), tail: Cons { head: floor_route_gap_expectation_chunk_08(), tail: Cons { head: floor_route_gap_expectation_chunk_09(), tail: Cons { head: floor_route_gap_expectation_chunk_10(), tail: Cons { head: floor_route_gap_expectation_chunk_11(), tail: Cons { head: floor_route_gap_expectation_chunk_12(), tail: Cons { head: floor_route_gap_expectation_chunk_13(), tail: Cons { head: floor_route_gap_expectation_chunk_14(), tail: Cons { head: floor_route_gap_expectation_chunk_15(), tail: Cons { head: floor_route_gap_expectation_chunk_16(), tail: Cons { head: floor_route_gap_expectation_chunk_17(), tail: Cons { head: floor_route_gap_expectation_chunk_18(), tail: Cons { head: floor_route_gap_expectation_chunk_19(), tail: Cons { head: floor_route_gap_expectation_chunk_20(), tail: Cons { head: floor_route_gap_expectation_chunk_21(), tail: Empty {} } } } } } } } } } } } } } } } } } } } } } } } fn floor_route_gap_expectations() -> List { From 3a60e2c522a15a4e160cbb02239892c280823518 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 06:55:27 +0000 Subject: [PATCH 34/44] The discharge row failed the retirement condition it wrote for itself Round four on gunbc#12040, verified at head d2be4987 before acting on it. compute_recover_attempt occurs exactly twice in the tree: its definition, and one call from compute_recover_cli. The only test-side touch is a string_contains over the remedy TEXT, and DESIGN section 5 rules in those words that a typecheck and a .contains() grep are not consumers. The wet witnesses reach attempt_open, attempt_begin_launch and attempt_terminate; none reach this door. So the row's own trigger -- the capability CONSUMED BY A REAL CALLER -- was marked discharged while its own words went unmet. CURRENT RUNG GOES BACK TO MITIGATABLE, stated as a correction rather than left implied by the absence of a climb. Nothing regressed: the door is as good as it was and the evidence for it never existed. Section 4b(1) is the governing sentence, and rung inflation is worse than sitting low because an inflated class never ranks for climbing -- here the class that would stop being asked about is a compute seat nobody can reclaim. THE TRIGGER IS A CAPABILITY AND NOT A CHORE, because the control cannot be written today at any price. compute_recover_attempt takes a HostDashboardInstance and derives its attempts root from it, so no fixture can point the door anywhere but a real instance's real compute root -- a wet control over it would drive production, not the tempdir the harness already builds for every other attempt-lifecycle claim. The trigger is therefore a door whose STORE IS A PARAMETER, sufficient for an executing control to drive a held seat through discharge against a temporary store. Naming `write a test` would have been a trigger nobody could satisfy. The annotation at the door said the same thing in the second place and is corrected too. Leaving it is how the row got re-inflated once already. Also merges origin/main. The route-gap roster conflicted in the shape that is worth recording: main landed the SAME fix for host_capacity_wet_witness -- its header says gunbc#11962 shipped two of the three rows and the five identities blocked the merge queue for gunbc#11829 -- and both sides named the new function floor_route_gap_expectation_chunk_21. The two `chunks()` lines were textually identical, so git merged that line clean and left TWO definitions with ONE call site. Mine is renamed chunk_22 and both are registered; the enrolled identities were checked for duplicates separately. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 11 ++++++++++- ...a_held_compute_seat_has_no_in_corpus_discharge.dag | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 9056f65efcd..7da119d2119 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -1430,13 +1430,22 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent // ── THE RECOVERY DOOR (criteria 4, 5, 6) ───────────────────────────────────────────────────── // // THIS IS THE CAPABILITY gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge -// NAMED AS ITS TRIGGER, and PR 1's compute_release_on says in its own annotation what had to exist +// NAMES AS ITS TRIGGER, and PR 1's compute_release_on says in its own annotation what had to exist // before a discharge could ship: a termination decision bound to the unit, an attempt-to-unit // binding that cannot release another attempt's live reservation, and a settled state a retry // consumes. All three are here -- observe_unit_lifecycle decides from the manager, the unit name // comes from the attempt record rather than from an identity or a listing, and the terminal // generation IS the settled state. // +// THAT DOES NOT DISCHARGE THE TRIGGER AND THE ROW NO LONGER SAYS IT DOES (review round four on +// gunbc#12040). The trigger asks for the capability CONSUMED BY A REAL CALLER, and this door has one +// call site -- compute_recover_cli -- with nothing executing it: the wet witnesses reach +// attempt_open, attempt_begin_launch and attempt_terminate, never this fold, and the only test-side +// mention of the door is a string_contains over the remedy TEXT. The obstacle is visible in this +// function's own first line: the attempts root is DERIVED from a HostDashboardInstance, so no +// fixture can point the door at a temporary store, and the control cannot be written before the +// store is a parameter. That parameterization is the row's trigger now. +// // EVERY REFUSAL ARM BELOW EXISTS BECAUSE ITS OPPOSITE FREES A SEAT THAT SHOULD STAY HELD, so none // of them widens toward "release anyway". An unreadable slot is NOT an absent one (criterion 4): a // recovery that read "absent" from a failed read would release a seat whose attempt it never saw. diff --git a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag index f927e65ebf4..4d4687f45b5 100644 --- a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag +++ b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag @@ -18,17 +18,17 @@ data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = Recu "THERE IS NO SAFE MANUAL DISCHARGE UNDER LIVE ADMISSION, and saying so is part of the row rather than a caveat on it. Clearing the capacity partition by hand while this host is admitting work removes the very record the admission arithmetic folds, so the next requests are admitted against memory the stuck grant still holds -- trading a host that under-admits for one that over-promises, which is the direction this whole change exists to prevent. Any manual intervention requires compute admission STOPPED on the host and every gunbc-compute unit PROVEN gone, in that order.", - "THE TRIGGER FIRED, AND THIS ROW IS WHERE IT IS RECORDED RATHER THAN DELETED. The lifecycle capability the trigger names is gunbc.compute.work_provider_local compute_recover_cli, and it has all four properties the trigger asked for: (a) it decides termination from the manager and the cgroup through unit_lifecycle_decide, never from a state word; (b) it binds the unit from the attempt record the producer wrote, so it cannot stop a unit belonging to a different attempt; (c) it commits the terminal generation EXPECTING the generation it read, so a producer that advanced in the meantime wins and the recovery is refused by the store; (d) the terminal generation IS the settled state, carrying the verdict and why. The class does not stop existing -- a seat can still be held while the discharge has not been run -- so what changes is the RUNG, not the membership.", + "THE TRIGGER HAS NOT FIRED, AND AN EARLIER REVISION OF THIS ROW SAID IT HAD. The lifecycle capability the trigger names is gunbc.compute.work_provider_local compute_recover_cli, and the CORPUS half is real: it decides termination from the manager and the cgroup through unit_lifecycle_decide rather than from a state word; it binds the unit from the attempt record the producer wrote, so it cannot stop a unit belonging to a different attempt; it commits the terminal generation EXPECTING the generation it read, so a producer that advanced in the meantime wins and the recovery is refused by the store; and the terminal generation IS the settled state, carrying the verdict and why. What is missing is the half the trigger actually asked for -- CONSUMED BY A REAL CALLER. At gunbc#12040 head d2be4987 compute_recover_attempt appears exactly twice in the tree: its own definition, and one call from compute_recover_cli. The only test-side mention of the door is a string_contains over the remedy TEXT in work_request_witness_test, and DESIGN section 5 rules in those words that a typecheck and a .contains() grep are not consumers. The wet witnesses reach attempt_open, attempt_begin_launch, attempt_terminate and the provider gate; none of them reach this door. So the row failed the test it wrote for itself, which is the sharpest form of rung inflation: not an optimistic reading of the evidence, but a retirement condition marked discharged while its own words went unmet.", "AND THE DISCHARGE DOES NOT ANNOUNCE ITSELF ON A HALF-SUCCESS, which is the arm that would have quietly re-created this class under a name that reads as fixed. The terminal transition landing and the seat coming back are two facts: gunbc.compute.work_provider_local RecoverySettledButHeld is the arm where the attempt closed and the ledger refused the release, and it exits 2 saying the host still under-admits by that grant. A recovery that reported success on the transition alone would leave exactly the state this row describes while claiming the row was retired.", "RUNG FOUND AT: mitigatable. The state is reachable, its harm is bounded to one host's compute appropriation, it is visible in the outcome document and in the partition, and it never over-promises.", - "CURRENT RUNG: mechanically preventable, which is the CEILING for this class and is not structural impossibility. The invalid state is still writable -- a held seat whose release did not land is a state the store can be in -- and safety now depends on the discharge being RUN. What the discharge cannot do is free a seat it has not established is free, and that is the part held by construction: every arm that fails to establish an end keeps the charge.", + "CURRENT RUNG: mitigatable, the same rung this class was found at. It is stated here rather than left implied by the absence of a climb, because this row PREVIOUSLY REPORTED mechanically preventable and a reader who saw that number is owed the correction explicitly. Nothing about the code moved to justify the demotion and nothing has regressed: the door is as good as it was, and the evidence for it never existed. DESIGN section 4b(1) is the governing sentence -- the reported rung must equal the rung established by EXECUTED evidence, a discriminating RED on the real acceptance path plus an accepted positive control -- and a door with one call site and no executing control establishes nothing, however well it is written. Rung inflation is worse than sitting low because an inflated class never ranks for climbing, and the thing that stops being asked about here is a compute seat nobody can reclaim.", "CEILING: mechanically preventable, and the reason it goes no higher is worth stating rather than leaving as an unexplained stop. Structural impossibility would mean no reachable state in which a seat is held with nothing running, and that would require proving a systemd transient unit has stopped using memory from outside it -- an external fact at the boundary DESIGN 4b keeps OFF the ladder. So this class is at its ceiling, and the honest residual is that a seat stays charged until someone runs the door.", - "NEXT-RUNG TRIGGER: NONE, because the class is at its ceiling. The trigger this row carried -- the compute reservation LIFECYCLE capability, with all four properties, consumed by a real caller -- is discharged as described above. A row at its ceiling with no trigger is not a stalled row: DESIGN 4b(2) separates cannot-climb-further from can-climb-after-one-grounding, and this is the first.", + "NEXT-RUNG TRIGGER, A CAPABILITY AND NOT A CHORE: a door whose STORE IS A PARAMETER, sufficient for an executing control to drive a held seat through discharge against a temporary store. The distinction matters because write a test is not the obstacle here and naming it as the trigger would be a trigger nobody can satisfy. compute_recover_attempt takes a HostDashboardInstance and derives its attempts root from it through compute_attempts_root, so a fixture cannot point the door anywhere but the real compute root of a real instance -- there is no argument that would let a witness build the held slot in the tempdir the wet harness already creates for every other attempt-lifecycle claim. That is a design fact about this door and not a gap in the harness, which is why the trigger names the parameterization rather than the control. Once the store is a parameter, the control is within reach of machinery that already exists: the wet witness builds real CAS slots in a temporary directory today. THE CONTROL IS WHAT RETIRES THIS ROW, and the parameterization is what makes the control writable.", "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration, and gunbc.compute.work_provider_local compute_stuck_reservation_remedy carries the remedy sentence into the refusal a caller actually receives.", ], From 9eb96c459397526a7fa87d43b1e51ac7d2a24fab Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 07:38:48 +0000 Subject: [PATCH 35/44] The control for the settlement fix ran nowhere, and the census that should have said so was a count Review 70036 and the floor at 3a60e2c5 agree, which is as confirmed as a finding gets: the wet witness file holds TEN test fns and nine were enrolled. The tenth, the_loser_of_a_launch_race_never_reports_that_it_settled_the_slot, appeared only at its own definition -- and git grep AttemptUnsettleable returns exactly two files, so it is the SOLE EXECUTING CONTROL for the arm added in d2be4987. The fabricated-settlement finding had been answered with a control that does not run, which is 4b(1)'s inert-lens case with the sting that it LOOKS like evidence in the diff. CI reported it as route_gap AND changed_witness_planned_without_terminal_verdict. ENROLLED IN BOTH ROSTERS, and the route-gap row is a PREDICTION rather than an observation -- a departure from that roster's own rule, said in the row. The claim's first statement is fresh_root(), which is DirWithTemplate unconditionally, so it cannot reach its subject hermetically under any order. THE CENSUS IS NOW A RULE, NOT A COUNT. It said NINE ATTEMPT-LIFECYCLE MEMBERS over a file that had grown to ten, and that is how the gap stayed invisible. Changing NINE to TEN would reproduce it on the next addition, so membership is stated as every test fn in the file, with the admitting property being the EFFECT. Same class as the route-gap roster's six-against-nine, in the other direction: there the honest form is an observation, here it is a policy. THE ROSTER-UNIQUENESS CLAIM IS LIFTED OUT, and the reasoning stays behind in the module so nobody pays for the wall twice. It costs 346735 eval steps against a 72300 new-witness budget, the overage is the pairwise comparison itself, and no construction here makes it linear -- the substrate has no set and no sort over String. A declared ceiling or a reduced subject are both decisions about a shared budget and belong to their own change, not to a compute-lifecycle PR four rounds deep. What it guards is real and unguarded: a duplicated schedule row survives a union merge silently, wet_forward_causes walks SCHEDULED so it finds the same terminal twice with n == 1 and raises nothing, and both sibling claims are structurally blind to it. AND THE MEASUREMENT FALSIFIED MY OWN REPAIR, which is the part worth keeping. I read the first version as rebuilding the roster projection inside the inner predicate and hoisted it. The figure moved 343406 -> 346735, the difference being one row the roster gained. The rebuilt projection was never the expense, and the structural story I told about it was wrong. Co-Authored-By: Claude Opus 5 (1M context) --- .../claim/local_repo_wet_terminal_test.dag | 81 ++++++------------- src/v2/workflow/floor_route_gap.dag | 15 +++- src/v2/workflow/local_repo_wet_terminal.dag | 22 ++++- 3 files changed, 56 insertions(+), 62 deletions(-) diff --git a/src/v2/test/claim/local_repo_wet_terminal_test.dag b/src/v2/test/claim/local_repo_wet_terminal_test.dag index c5a425b7638..f9a32b9ab90 100644 --- a/src/v2/test/claim/local_repo_wet_terminal_test.dag +++ b/src/v2/test/claim/local_repo_wet_terminal_test.dag @@ -4,7 +4,7 @@ import v2.std.collection { List } import v2.std.logic { Bool } import v2.std.text { String } import v2.std.integer { Int } -import v2.std.algebra { length, filter } +import v2.std.algebra { length, filter, count_where } import v2.workflow.floor2_prepared_subject { WitnessIdentity, witness_identity_qualified_name } import v2.workflow.floor_terminal_ledger { ClaimAttemptTerminal, Returned, Panicked, RouteGap, @@ -334,61 +334,26 @@ test fn the_gate_predicate_admits_only_an_exact_identity_and_candidate() -> Bool ) } -// NO TWO ROWS NAME ONE IDENTITY, which the roster had no check for and which the sibling claims -// cannot see. `the_local_roster_is_not_empty` compares the length of the identity projection against -// the length of the schedule, and a map never changes a list's length, so that equality holds over a -// roster with a duplicate in it. `every_roster_row_spells_its_function_the_same_way_twice` reads -// WITHIN one row. So a repeated identity was unrepresented by every existing claim. +// THE ROSTER-UNIQUENESS CLAIM WAS WRITTEN HERE AND IS NOT LANDING IN THIS CHANGE, which is a +// deliberate call rather than an omission, recorded so the next reader does not rediscover the +// same wall and pay for it twice. // -// THE POPULATION THIS EXISTS FOR IS A MERGE, not an author. Two branches each appending a row to -// this list conflict textually, and the correct resolution of append-against-append is a union -- -// which is right exactly when the two rows are independent and silently wrong when both sides -// appended the SAME identity. git cannot see that: both sides did the same legal thing, and the -// union it produces is a green tree carrying a duplicate. gunbc#12040 against gunbc#12044 was that -// merge, and the read that found it clean was an eye over 220 rows, which is the mitigation this -// claim replaces. -fn roster_identity_occurrences(name: String) -> Int { - length( - xs: filter( - xs: local_repo_wet_scheduled_identities(), - predicate: fn(n) { n == name } - ) - ) -} - -test fn no_two_roster_rows_name_the_same_identity() -> Bool { - length( - xs: filter( - xs: local_repo_wet_scheduled_identities(), - predicate: fn(n) { roster_identity_occurrences(name: n) != 1 } - ) - ) == 0 -} - -// THE DISCRIMINATING CONTROL, because the claim above is a filter that returns zero and a filter -// over a predicate that can never fire returns zero too. These two supplied lists differ only in -// whether one name repeats, so a predicate that stopped discriminating fails here while the roster -// claim stays green. -fn names_with_a_duplicate() -> List { - ["test.claim.x.a_holds", "test.claim.x.b_holds", "test.claim.x.a_holds"] -} - -fn names_all_distinct() -> List { - ["test.claim.x.a_holds", "test.claim.x.b_holds", "test.claim.x.c_holds"] -} - -fn every_name_occurs_once(names: List) -> Bool { - length( - xs: filter( - xs: names, - predicate: fn(n) { - length(xs: filter(xs: names, predicate: fn(m) { m == n })) != 1 - } - ) - ) == 0 -} - -test fn the_duplicate_identity_predicate_discriminates() -> Bool { - every_name_occurs_once(names: names_all_distinct()) - && !every_name_occurs_once(names: names_with_a_duplicate()) -} +// WHAT IT GUARDS IS REAL AND NOTHING ELSE GUARDS IT. Two branches appending to this schedule +// conflict textually; the correct resolution is a union, which is right only when the appended +// rows are INDEPENDENT. A duplicated row survives that union silently, and the join cannot see +// it: wet_forward_causes walks SCHEDULED, so a row appearing twice finds the same single terminal +// twice, n == 1 both times, and no refusal is raised. The sibling claims cannot see it either -- +// every_roster_row_spells_its_function_the_same_way_twice reads WITHIN a row, and +// the_local_roster_is_not_empty compares length(identities) to length(schedule), which is a map +// and therefore equal over a roster WITH a duplicate in it. +// +// WHY IT IS NOT HERE: it costs 346735 eval steps against a 72300 new-witness budget, and the +// overage is the pairwise comparison itself. It is not made linear by any construction available +// -- the substrate offers no set and no sort over String -- so the honest routes are a declared +// ceiling or a reduced subject, and BOTH are decisions about a shared budget that belong to their +// own change rather than riding in on a compute-lifecycle PR that is already four rounds deep. +// +// ONE MEASUREMENT WORTH CARRYING, because it falsified the obvious repair: the first version +// rebuilt the roster projection inside the inner predicate, which looked like the cost. Hoisting +// it moved the figure from 343406 to 346735 -- the difference being one row the roster gained, not +// the repair. The rebuilt projection was never the expense. diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index cad2928d0a5..62f901d283e 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1537,6 +1537,17 @@ fn floor_route_gap_expectation_chunk_21() -> List { // run that does not touch the file, and this roster plus the wet terminal is what holds for the run // that does. // +// THE SEVENTH ROW IS the_loser_of_a_launch_race..., ADDED BY PREDICTION RATHER THAN BY OBSERVATION, +// which departs from this roster's own rule and is said rather than hidden. The six above were read +// off a floor run; this one was written after that run, so no run has reported it yet. Its first +// statement is `let root = fresh_root()`, which is DirWithTemplate unconditionally, so it cannot +// reach its subject hermetically under any evaluation order. If the floor disagrees it will say so +// as a STALE-ROUTE-GAP naming this identity -- the same self-correcting report the next paragraph +// describes -- and the alternative, leaving it out to be observed, spends a cycle learning what the +// call site already states. THE ROW IS ALSO WHY THIS ROSTER GREW: review 70036 found the claim +// enrolled NOWHERE, and it is the sole executing control for the AttemptUnsettleable arm, so the +// fabricated-settlement finding had been answered with a control that does not run. +// // WHAT RE-OBSERVES THIS ROSTER, STATED BECAUSE NOTHING ELSE WOULD SAY IT. A six-row observation can // go stale, and the staleness refuses from the other direction, so the obligation is named here // rather than left for whoever meets the refusal. It is NARROWER than it first looks and the reason @@ -1565,8 +1576,10 @@ fn floor_route_gap_expectation_chunk_22() -> List { head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.settling_records_why_and_does_not_erase_what_it_settled", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.the_unit_binding_is_read_back_from_the_record_the_attempt_wrote", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.the_loser_of_a_launch_race_never_reports_that_it_settled_the_slot", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Empty {} - } } } } } } + } } } } } } } } fn floor_route_gap_expectation_chunks() -> List> { diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index b5142de0b1c..324b39c54cf 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -242,8 +242,17 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // bin-witness wet class, whose standing excl_bin_wet_reason carries. Measured wet through // `gunbc run --claim-run` over this tree before enrollment: all ten held, and every metacharacter // probe failed without output rather than running its payload. -// NINE ATTEMPT-LIFECYCLE MEMBERS (2026-09-22, gunbc#12040). Same admitted effect as the seven -// host_capacity members directly below, and enrolled the same way: a real compare-and-set against a +// THE ATTEMPT-LIFECYCLE MEMBERS ARE EVERY TEST FN IN THAT FILE (2026-09-22, gunbc#12040), and the +// membership is stated as that RULE rather than as a count. It said NINE until review 70036, by +// which time the file held ten: the claim added for the fabricated-settlement finding was written, +// executed by hand, and enrolled NOWHERE -- and it was the sole executing control for the +// AttemptUnsettleable arm, so the finding was answered with a control that does not run. A count +// asserting completeness over a population that grows underneath it is what hid that, which is the +// same defect in the other direction from the one this file's route-gap sibling avoids by being an +// observation rather than a policy. Here the policy is the honest form: the admitting property is +// the EFFECT, every function in the file has it, so no member can be added without being covered. +// +// Same admitted effect as the seven host_capacity members directly below, and enrolled the same way: a real compare-and-set against a // real generation-file store inside a directory the witness creates with shell.Mktemp.DirWithTemplate // and removes -- no repository, no network, no cargo, no remote host, no install media. The floor // PLANNED all nine and the hermetic route reached no subject for most of them: the #12040 run named @@ -254,7 +263,8 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // claim reaches a verdict hermetically only by not touching the directory on that path, which is a // property of the evaluation order and not of the claim, and it moves under an edit. The file's exclusion row is gunbc.ci.ci_layer_roots // excl_local_repo_wet_tempdir_write_reason, and it retires on the same mock coverage as its -// neighbours. Measured wet over this tree before enrollment: 9/9 held. +// neighbours. Measured wet over this tree before enrollment: every member held -- 9/9 at the time +// the first nine were enrolled, and the tenth PASS with its mutation RED when it was added. fn local_repo_wet_schedule() -> List { [ WetScheduledClaim { @@ -353,6 +363,12 @@ fn local_repo_wet_schedule() -> List { function: "an_over_long_pre_launch_declaration_refuses_at_open_and_is_not_clamped", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "the_loser_of_a_launch_race_never_reports_that_it_settled_the_slot" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "the_loser_of_a_launch_race_never_reports_that_it_settled_the_slot", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.effect_plan_bash_materialize_real_execution_witness", function: "effect_plan_bash_two_declared_operations_execute" }, entry: "dag/test/claim/effect_plan_bash_materialize_real_execution_witness_test.dag", From 7412d23265287c810893b05ea88d71ae663b59e8 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 08:00:39 +0000 Subject: [PATCH 36/44] The note about a lifted claim was left at EOF, where an annotation names nothing CI on 9eb96c45: 20 parse errors, one block, all 'source annotation names no subject: no module item follows it'. The REQUIRED-FLOOR refusal ArmSetConsumerPlanningUnavailable in the same run is downstream of the parse failure rather than a second defect -- no parse-phase declaration index reaches the floor when the module will not parse. The block is the reasoning about the roster-uniqueness claim this change lifted out. It went at the end of the file, and annotations are module-item grain: a block at EOF has no subject. It now sits above the roster claims it is about, which is also where a reader looking for the missing check would start. The indented-annotation grep that has been circulating passes this cleanly, because the block is at column 1. Both shapes need checking: grep -Hn '^[[:space:]]\+//' awk '/^[[:space:]]*\/\//{c=NR;next} /^[[:space:]]*$/{next} {c=0} END{if(c) print FILENAME": trailing annotation at "c}' Both now run clean over every file in this diff. Co-Authored-By: Claude Opus 5 (1M context) --- .../claim/local_repo_wet_terminal_test.dag | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/v2/test/claim/local_repo_wet_terminal_test.dag b/src/v2/test/claim/local_repo_wet_terminal_test.dag index f9a32b9ab90..354338f43cc 100644 --- a/src/v2/test/claim/local_repo_wet_terminal_test.dag +++ b/src/v2/test/claim/local_repo_wet_terminal_test.dag @@ -127,6 +127,30 @@ fn terminal_matching() -> WetTerminalRow { ) } +// THE ROSTER-UNIQUENESS CLAIM BELONGS BESIDE THESE CLAIMS AND IS NOT LANDING IN THIS CHANGE, which is a +// deliberate call rather than an omission, recorded so the next reader does not rediscover the +// same wall and pay for it twice. +// +// WHAT IT GUARDS IS REAL AND NOTHING ELSE GUARDS IT. Two branches appending to this schedule +// conflict textually; the correct resolution is a union, which is right only when the appended +// rows are INDEPENDENT. A duplicated row survives that union silently, and the join cannot see +// it: wet_forward_causes walks SCHEDULED, so a row appearing twice finds the same single terminal +// twice, n == 1 both times, and no refusal is raised. The sibling claims cannot see it either -- +// every_roster_row_spells_its_function_the_same_way_twice reads WITHIN a row, and +// the_local_roster_is_not_empty compares length(identities) to length(schedule), which is a map +// and therefore equal over a roster WITH a duplicate in it. +// +// WHY IT IS NOT HERE: it costs 346735 eval steps against a 72300 new-witness budget, and the +// overage is the pairwise comparison itself. It is not made linear by any construction available +// -- the substrate offers no set and no sort over String -- so the honest routes are a declared +// ceiling or a reduced subject, and BOTH are decisions about a shared budget that belong to their +// own change rather than riding in on a compute-lifecycle PR that is already four rounds deep. +// +// ONE MEASUREMENT WORTH CARRYING, because it falsified the obvious repair: the first version +// rebuilt the roster projection inside the inner predicate, which looked like the cost. Hoisting +// it moved the figure from 343406 to 346735 -- the difference being one row the roster gained, not +// the repair. The rebuilt projection was never the expense. + // THE ROSTER DOES NOT LAND EMPTY. An empty roster would make this lane's coverage claim vacuously // true and leave nothing proving the executor runs anything at all -- permanently green by // construction, which DESIGN 4b calls a decoration rather than a wall. This is the witness that @@ -333,27 +357,3 @@ test fn the_gate_predicate_admits_only_an_exact_identity_and_candidate() -> Bool candidate: fixture_candidate ) } - -// THE ROSTER-UNIQUENESS CLAIM WAS WRITTEN HERE AND IS NOT LANDING IN THIS CHANGE, which is a -// deliberate call rather than an omission, recorded so the next reader does not rediscover the -// same wall and pay for it twice. -// -// WHAT IT GUARDS IS REAL AND NOTHING ELSE GUARDS IT. Two branches appending to this schedule -// conflict textually; the correct resolution is a union, which is right only when the appended -// rows are INDEPENDENT. A duplicated row survives that union silently, and the join cannot see -// it: wet_forward_causes walks SCHEDULED, so a row appearing twice finds the same single terminal -// twice, n == 1 both times, and no refusal is raised. The sibling claims cannot see it either -- -// every_roster_row_spells_its_function_the_same_way_twice reads WITHIN a row, and -// the_local_roster_is_not_empty compares length(identities) to length(schedule), which is a map -// and therefore equal over a roster WITH a duplicate in it. -// -// WHY IT IS NOT HERE: it costs 346735 eval steps against a 72300 new-witness budget, and the -// overage is the pairwise comparison itself. It is not made linear by any construction available -// -- the substrate offers no set and no sort over String -- so the honest routes are a declared -// ceiling or a reduced subject, and BOTH are decisions about a shared budget that belong to their -// own change rather than riding in on a compute-lifecycle PR that is already four rounds deep. -// -// ONE MEASUREMENT WORTH CARRYING, because it falsified the obvious repair: the first version -// rebuilt the roster projection inside the inner predicate, which looked like the cost. Hoisting -// it moved the figure from 343406 to 346735 -- the difference being one row the roster gained, not -// the repair. The rebuilt projection was never the expense. From 8efbc27b29105d2c56c6fc34eaa588490cf82a36 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 21:08:40 +0000 Subject: [PATCH 37/44] A seat the pool no longer holds was reported as a failed release compute_release_on folded SeatNotHeld into ComputeReleaseRefused through a catch-all arm, so a double release (recovery then producer) wrote outcome.json as STILL CHARGED with the stuck-reservation remedy. ComputeNothingToRelease is now its own arm, the three pre-grant arms use it, and every SeatRelease arm is named. The new wet claim releases one real seat twice and is enrolled in local_repo_wet_schedule; it holds, and goes red when SeatNotHeld is routed back to the refusal arm. Co-Authored-By: Claude Opus 5.5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 27 +++++++++-- dag/gunbc/compute/work_provider_local.dag | 7 ++- .../host_capacity_wet_witness_test.dag | 47 ++++++++++++++++++- src/v2/workflow/local_repo_wet_terminal.dag | 6 +++ 4 files changed, 81 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index 0441eb44451..9db18e2d28f 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -365,15 +365,31 @@ fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: Ep // // A RELEASE THAT DID NOT LAND IS NOT A RELEASE. The append is retried against a moving head and // exhaustion is its own arm; nothing here reports a return of capacity it did not observe. +// NOTHING-TO-RELEASE IS ITS OWN ARM, BECAUSE A SEAT THE POOL DOES NOT HOLD IS NOT A FAILED RELEASE. +// Three states, not two: this caller returned the seat; there was no seat to return; the ledger +// could not return it. Folding the middle into the last manufactures an incident -- outcome.json +// says the seat is STILL CHARGED and carries the stuck-reservation remedy, and an operator chases a +// seat that recovery already returned. It is the same distinction gunbc.compute.work_provider_local +// makes at compute_settle_attempt, where AttemptUnsettleable projects to HostRecordWritten +// deliberately: a loss means something was OWED and not done, and a producer that owed nothing +// cannot have lost anything. Here it was unmade, and by a CATCH-ALL rather than by a decision -- +// `other => ComputeReleaseRefused` in the fold below, which reads as exhaustive while being a +// default, so SeatNotHeld arrived at the refusal arm nobody wrote it into. +// +// THE THREE PRE-GRANT ARMS BELOW WERE ALREADY SAYING IT IN PROSE. Each rendered `nothing to +// release: ...` into a REFUSED detail, which is the right sentence in the wrong constructor: a +// reservation that was never granted has nothing to return, and every reader deciding on the arm +// rather than the text saw a failure. type ComputeRelease = ComputeReleased { partition: PartitionId, reference: NonEmptyStr, reason: NonEmptyStr } + | ComputeNothingToRelease { detail: String } | ComputeReleaseRefused { detail: String } fn compute_release(reservation: ComputeReservation, reason: NonEmptyStr) -> ComputeRelease { match reservation { - ComputeCapacityFull { wire: w } => ComputeReleaseRefused { detail: join(["nothing to release: ", w], "") } - ComputeHostBelowFloor { requested: _, available: _ } => ComputeReleaseRefused { detail: "nothing to release: the host was below its live memory floor" } - ComputeReservationRefused { detail: d } => ComputeReleaseRefused { detail: join(["nothing to release: ", d], "") } + ComputeCapacityFull { wire: w } => ComputeNothingToRelease { detail: join(["nothing to release: ", w], "") } + ComputeHostBelowFloor { requested: _, available: _ } => ComputeNothingToRelease { detail: "nothing to release: the host was below its live memory floor" } + ComputeReservationRefused { detail: d } => ComputeNothingToRelease { detail: join(["nothing to release: ", d], "") } ComputeReserved { grant: g, partition: p, amount: _, store: store } => compute_release_on(store: store, partition: p, appropriation: kibibyte(count: 0), reference: g.reference, reason: reason) } @@ -431,7 +447,10 @@ fn compute_release_on( budget: compute_partition_read_budget, ) { SeatReleased { reference: r, id: _ } => ComputeReleased { partition: partition, reference: r, reason: reason } - other => ComputeReleaseRefused { detail: seat_release_wire(r: other) } + SeatNotHeld { reference: r, wire: w } => + ComputeNothingToRelease { detail: join(["the pool is not holding ", r as String, ", so there is no seat to return: ", w, ". A double release reaches here -- recovery returning a seat the producer then releases again is the ordinary case -- and it is not a ledger failure"], "") } + SeatReleaseContended { attempts: a } => ComputeReleaseRefused { detail: seat_release_wire(r: SeatReleaseContended { attempts: a }) } + SeatReleaseRefused { step: st, reason: rs } => ComputeReleaseRefused { detail: seat_release_wire(r: SeatReleaseRefused { step: st, reason: rs }) } } } } diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 7da119d2119..1335b5b7439 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -42,7 +42,7 @@ import gunbc.roadmap_execution_contract { import std.measure { Kibibyte, kibibyte_to_byte_size } import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, + ComputeRelease, ComputeReleased, ComputeNothingToRelease, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, ComputeCapacitySubject, ComputeSubjectResolution, ComputeSubjectResolved, ComputeSubjectHostUnresolved, compute_capacity_subject, compute_capacity_store, compute_release_on, @@ -1262,6 +1262,10 @@ fn compute_refused_unreserved( fn compute_release_landed(r: ComputeRelease) -> Bool { match r { ComputeReleased { partition: _, reference: _, reason: _ } => true + // A SEAT NOBODY HOLDS IS NOT A CHARGE, so this arm LANDS. It is what gates the settled record + // against compute_unreleased_record, and routing a double release to the unreleased record is + // exactly the false incident this arm exists to stop. + ComputeNothingToRelease { detail: _ } => true ComputeReleaseRefused { detail: _ } => false } } @@ -1269,6 +1273,7 @@ fn compute_release_landed(r: ComputeRelease) -> Bool { fn compute_release_wire(r: ComputeRelease) -> String { match r { ComputeReleased { partition: _, reference: ref_, reason: why } => join(["released ", ref_ as String, " (", why as String, ")"], "") + ComputeNothingToRelease { detail: d } => join(["no seat to return -- ", d], "") ComputeReleaseRefused { detail: d } => join(["NOT released: ", d], "") } } diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index 0140266f9b2..45b15966a17 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -11,7 +11,7 @@ import gunbc.compute.host_capacity { compute_capacity_subject, compute_capacity_root, compute_memory_partition, HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - compute_reserve_on, compute_reserve_against, compute_release, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + compute_reserve_on, compute_reserve_against, compute_release, ComputeRelease, ComputeNothingToRelease, ComputeReleased, ComputeReleaseRefused, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, } @@ -90,10 +90,27 @@ fn is_ledger_refusal(r: ComputeReservation) -> Bool { fn released(r: ComputeRelease) -> Bool { match r { ComputeReleased { partition: _, reference: _, reason: _ } => true + ComputeNothingToRelease { detail: _ } => false ComputeReleaseRefused { detail: _ } => false } } +fn nothing_to_release(r: ComputeRelease) -> Bool { + match r { + ComputeReleased { partition: _, reference: _, reason: _ } => false + ComputeNothingToRelease { detail: _ } => true + ComputeReleaseRefused { detail: _ } => false + } +} + +fn release_refused(r: ComputeRelease) -> Bool { + match r { + ComputeReleased { partition: _, reference: _, reason: _ } => false + ComputeNothingToRelease { detail: _ } => false + ComputeReleaseRefused { detail: _ } => true + } +} + fn standing_committed(s: ComputeCapacityStanding) -> Int { match s { ComputeCapacityStandingRead { committed: c, ceiling: _, headroom: _, observed_available: _, observed_total: _ } => kibibyte_count(k: c) @@ -248,3 +265,31 @@ test fn the_real_observation_reads_this_machine() -> Bool { } } +// A SECOND RELEASE OF ONE SEAT IS NOT A FAILED RELEASE, and before this arm existed it was reported +// as one. The producer and the recovery door both come back through compute_release, so the seat +// can be returned twice for one reference -- recovery wins past the deadline, returns the seat, and +// the producer reaches its own release afterwards. The pool then answers SeatNotHeld, which a +// CATCH-ALL arm folded into ComputeReleaseRefused: compute_release_landed went false, the record +// fold wrote outcome.json saying the seat was STILL CHARGED, and the stuck-reservation remedy was +// attached to a seat recovery had already returned. +// +// THIS DRIVES THE DOUBLE RELEASE FOR REAL rather than describing it: one reservation against a real +// event log in a temporary directory, released twice. The capacity reading after the second release +// is what rules out the opposite error -- an arm that reported success by releasing the seat AGAIN +// would show the committed figure moving, and it does not. +test fn releasing_one_seat_twice_is_not_a_ledger_failure_and_charges_nothing() -> Bool { + let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes * 2, request_kib: one_gibibyte_in_kibibytes) + let held = compute_reserve_on(subject: subject, reference: "wet-double" as NonEmptyStr, term_seconds: 60) + let while_held = compute_capacity_standing_on(subject: subject) + let first = compute_release(reservation: held, reason: "the producer finished" as NonEmptyStr) + let after_first = compute_capacity_standing_on(subject: subject) + let second = compute_release(reservation: held, reason: "recovery got here too" as NonEmptyStr) + let after_second = compute_capacity_standing_on(subject: subject) + reserved_amount(r: held) == one_gibibyte_in_kibibytes + && standing_committed(s: while_held) == one_gibibyte_in_kibibytes + && released(r: first) + && standing_committed(s: after_first) == 0 + && nothing_to_release(r: second) + && !release_refused(r: second) + && standing_committed(s: after_second) == 0 +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 324b39c54cf..fccf1f5e6fb 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -309,6 +309,12 @@ fn local_repo_wet_schedule() -> List { function: "two_instances_on_one_host_resolve_to_one_pool_at_one_root", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "releasing_one_seat_twice_is_not_a_ledger_failure_and_charges_nothing" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "releasing_one_seat_twice_is_not_a_ledger_failure_and_charges_nothing", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "a_terminal_transition_refuses_a_launch_that_had_not_spawned_yet" }, entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", From f4d0bb1c4b1aecee2c11fe27f26d61b871f3aaa0 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 21:27:13 +0000 Subject: [PATCH 38/44] The note on the nothing-to-release arm sat inside a match body, where no annotation is admitted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moved to the fn it explains (DESIGN §4c: // attaches only at module-item grain); emit-build refused it as 'source annotation sits inside a declaration body'. Co-Authored-By: Claude Opus 5.5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 1335b5b7439..3207548f445 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -1259,12 +1259,12 @@ fn compute_refused_unreserved( } } +// A SEAT NOBODY HOLDS IS NOT A CHARGE, so ComputeNothingToRelease LANDS. This fn is what gates the +// settled record against compute_unreleased_record, and routing a double release to the unreleased +// record is exactly the false incident that arm exists to stop. fn compute_release_landed(r: ComputeRelease) -> Bool { match r { ComputeReleased { partition: _, reference: _, reason: _ } => true - // A SEAT NOBODY HOLDS IS NOT A CHARGE, so this arm LANDS. It is what gates the settled record - // against compute_unreleased_record, and routing a double release to the unreleased record is - // exactly the false incident this arm exists to stop. ComputeNothingToRelease { detail: _ } => true ComputeReleaseRefused { detail: _ } => false } From a2a19f93c4953e4a61777454a05fd6ca6f1e3023 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Tue, 22 Sep 2026 22:48:20 +0000 Subject: [PATCH 39/44] A terminal verdict was any word, and the decode that refuses unknown state words accepted it Review 70244. AttemptTerminal.verdict is now AttemptVerdict = AttemptWorkEnded { ending: WorkEnding } | AttemptRecovered -- the two writers the slot has. WorkEnding types the ending words work_outcome_ending already spelled (it now projects through work_ending_wire, so there is one spelling). attempt_state_decode refuses a verdict outside the set; RecoveryStateNotRecoverable carries the AttemptState. The witness's 'released' was never a word any producer wrote; it now uses real verdicts. Control a_terminal_verdict_outside_the_closed_set_is_unreadable (enrolled in the wet schedule) round-trips all six verdicts through the real record wire and decodes 'recoverd', 'released' and '' as unreadable. All 11 lifecycle claims hold; widening an unknown verdict to Recovered turns the new claim false. Co-Authored-By: Claude Opus 5.5 (1M context) --- dag/gunbc/compute/attempt_lifecycle.dag | 45 ++++++++++++--- dag/gunbc/compute/work_provider_local.dag | 10 ++-- dag/gunbc/compute/work_request.dag | 46 +++++++++++++-- .../attempt_lifecycle_wet_witness_test.dag | 56 ++++++++++++++++--- src/v2/workflow/local_repo_wet_terminal.dag | 6 ++ 5 files changed, 138 insertions(+), 25 deletions(-) diff --git a/dag/gunbc/compute/attempt_lifecycle.dag b/dag/gunbc/compute/attempt_lifecycle.dag index 6d53cc8c3f2..d26355f3574 100644 --- a/dag/gunbc/compute/attempt_lifecycle.dag +++ b/dag/gunbc/compute/attempt_lifecycle.dag @@ -16,6 +16,7 @@ import gunbc.durable_cas_file_store { CasAttemptKeyNotSlotAddressable, admit_cas_attempt, file_compare_and_set, DefaultAccessCreateOnly, observe_cas_slot_state, } +import gunbc.compute.work_request { WorkEnding, work_ending_wire, work_ending_decode } import extdeps.languages.json.emit { JsonValue, json_object, json_kv, json_string, json_int, serialize_json } import extdeps.languages.json.parse { parse_json_document, JsonDocumentParsed, JsonDocumentUnreadable, json_document_gap_text, @@ -107,13 +108,40 @@ fn unit_lifecycle_wire(u: UnitLifecycle) -> String { type AttemptState = AttemptReserved | AttemptLaunching - | AttemptTerminal { verdict: String, detail: String } + | AttemptTerminal { verdict: AttemptVerdict, detail: String } + +// WHO ENDED THE ATTEMPT AND HOW, AS A CLOSED SET. Two writers reach the terminal generation: the +// producer settling it with the work's own ending, and the recovery door terminating one it found +// dead. Anything else in that position is not a verdict this store ever wrote, so the decode below +// refuses it exactly as it refuses an unknown state word -- review 70244 found it accepting any word +// after "terminal:" while the comment above attempt_record_decode said the opposite. +type AttemptVerdict + = AttemptWorkEnded { ending: WorkEnding } + | AttemptRecovered + +fn attempt_verdict_wire(v: AttemptVerdict) -> String { + match v { + AttemptWorkEnded { ending: e } => work_ending_wire(e: e) + AttemptRecovered => "recovered" + } +} + +fn attempt_verdict_decode(word: String) -> AttemptVerdict? { + if word == attempt_verdict_wire(v: AttemptRecovered) { + Present { value: AttemptRecovered } + } else { + match work_ending_decode(word: word) { + Present { value: e } => Present { value: AttemptWorkEnded { ending: e } } + Absent => none + } + } +} fn attempt_state_wire(s: AttemptState) -> String { match s { AttemptReserved => "reserved" AttemptLaunching => "launching" - AttemptTerminal { verdict: v, detail: _ } => join(["terminal:", v], "") + AttemptTerminal { verdict: v, detail: _ } => join(["terminal:", attempt_verdict_wire(v: v)], "") } } @@ -256,7 +284,10 @@ fn attempt_state_decode(word: String, detail: String) -> AttemptState? { } else if word == "launching" { Present { value: AttemptLaunching } } else if starts_with(s: word, prefix: "terminal:") { - Present { value: AttemptTerminal { verdict: substring(s: word, start: length("terminal:"), end: length(word)), detail: detail } } + match attempt_verdict_decode(word: substring(s: word, start: length("terminal:"), end: length(word))) { + Present { value: v } => Present { value: AttemptTerminal { verdict: v, detail: detail } } + Absent => none + } } else { none } @@ -377,7 +408,7 @@ fn attempt_begin_launch(root: NonEmptyStr, record: AttemptRecord, expected: CasG // THE TERMINAL TRANSITION. Carries the verdict AND why, so the generation that records the end also // records the reason -- history the next reader can act on rather than a bare flag. -fn attempt_terminate(root: NonEmptyStr, record: AttemptRecord, expected: CasGeneration, verdict: String, detail: String) -> AttemptTransition { +fn attempt_terminate(root: NonEmptyStr, record: AttemptRecord, expected: CasGeneration, verdict: AttemptVerdict, detail: String) -> AttemptTransition { attempt_commit( root: root, record: attempt_record_with_state(r: record, state: AttemptTerminal { verdict: verdict, detail: detail }), @@ -456,7 +487,7 @@ type RecoveryTrigger = RecoveryMayTerminate { reason: String } | RecoveryTooEarly { deadline: EpochSecs, now: EpochSecs } | RecoveryUnitNotEnded { lifecycle: UnitLifecycle } - | RecoveryStateNotRecoverable { state: String } + | RecoveryStateNotRecoverable { state: AttemptState } fn recovery_trigger(record: AttemptRecord, lifecycle: UnitLifecycle, now: EpochSecs) -> RecoveryTrigger { match record.state { @@ -478,7 +509,7 @@ fn recovery_trigger(record: AttemptRecord, lifecycle: UnitLifecycle, now: EpochS } else { RecoveryUnitNotEnded { lifecycle: lifecycle } } - AttemptTerminal { verdict: v, detail: _ } => RecoveryStateNotRecoverable { state: join(["terminal:", v], "") } + AttemptTerminal { verdict: v, detail: d } => RecoveryStateNotRecoverable { state: AttemptTerminal { verdict: v, detail: d } } } } @@ -497,6 +528,6 @@ fn recovery_trigger_wire(t: RecoveryTrigger) -> String { RecoveryTooEarly { deadline: d, now: n } => join(["TOO EARLY: the attempt declared a launch deadline of ", to_string(value: d), " and it is only ", to_string(value: n), "; a producer inside its own declared window is not dead"], "") RecoveryUnitNotEnded { lifecycle: u } => join(["the unit has not ended: ", unit_lifecycle_wire(u: u)], "") - RecoveryStateNotRecoverable { state: s } => join(["the attempt is already ", s, ", so there is nothing to recover"], "") + RecoveryStateNotRecoverable { state: s } => join(["the attempt is already ", attempt_state_wire(s: s), ", so there is nothing to recover"], "") } } diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 3207548f445..b31f69247c5 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -21,7 +21,7 @@ import gunbc.clock_read { clock_now_probed_at_or_unknown } import gunbc.fabric_event_log_host { now_epoch_seconds } import std.durable_compare_and_set { CasGeneration, cas_generation_count } import gunbc.compute.attempt_lifecycle { - AttemptRecord, AttemptState, AttemptReserved, AttemptLaunching, AttemptTerminal, + AttemptRecord, AttemptState, AttemptReserved, AttemptLaunching, AttemptTerminal, AttemptVerdict, AttemptWorkEnded, AttemptRecovered, UnitLifecycle, UnitTerminatedSuccess, UnitTerminatedFailure, UnitTerminatedCauseUnknown, UnitStillRunning, UnitLifecycleUnknown, unit_lifecycle_ended, unit_lifecycle_wire, RecoveryTrigger, RecoveryMayTerminate, RecoveryTooEarly, RecoveryUnitNotEnded, RecoveryStateNotRecoverable, @@ -56,7 +56,7 @@ import gunbc.compute.work_request { WorkInfrastructureRefusal, ProducerInFlight, HostComputeCapacityFull, HostBelowMemoryFloor, ComputeCapacityLedgerUnavailable, CheckoutUnavailable, StoreUnavailable, ToolchainUnobserved, DependencyNotSucceeded, SubjectUnresolved, - work_outcome_wire, work_outcome_ending, StoredOutcomeRead, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_decode, stored_outcome_is_success, + work_outcome_wire, work_outcome_ending, work_outcome_ending_of, StoredOutcomeRead, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_decode, stored_outcome_is_success, } // THE LOCAL PROVIDER: one of N behind the exact work contract, serving the host it runs on. It is a @@ -823,7 +823,7 @@ type AttemptSettlement | AttemptNotSettled { detail: String } | AttemptUnsettleable { detail: String } -fn compute_settle_attempt(layout: ComputeLayout, gate: AttemptGate, verdict: String, detail: String) -> AttemptSettlement { +fn compute_settle_attempt(layout: ComputeLayout, gate: AttemptGate, verdict: AttemptVerdict, detail: String) -> AttemptSettlement { match gate { AttemptGateRefused { detail: d } => AttemptUnsettleable { detail: d } AttemptGateReady { record: rec, generation: g } => @@ -942,7 +942,7 @@ fn compute_run_under_grant( } let closed = compute_settle_attempt( layout: layout, gate: launched, - verdict: work_outcome_ending(o: outcome), + verdict: AttemptWorkEnded { ending: work_outcome_ending_of(o: outcome) }, detail: join([unit_termination_wire(t: termination), "; ", compute_release_wire(r: returned)], "")) let dropped = compute_drop_producer_lease(layout: layout) let measured = Filesystem.Write( @@ -1506,7 +1506,7 @@ fn compute_recover_attempt(instance: HostDashboardInstance, reference: NonEmptyS RecoveryUnitNotEnded { lifecycle: u } => RecoveryDeclined { reference: rec.reference, why: recovery_trigger_wire(t: RecoveryUnitNotEnded { lifecycle: u }) } RecoveryStateNotRecoverable { state: st } => RecoveryDeclined { reference: rec.reference, why: recovery_trigger_wire(t: RecoveryStateNotRecoverable { state: st }) } RecoveryMayTerminate { reason: why } => - match attempt_terminate(root: attempts_root, record: rec, expected: g, verdict: "recovered", detail: join([why, "; unit ", unit_lifecycle_wire(u: lifecycle)], "")) { + match attempt_terminate(root: attempts_root, record: rec, expected: g, verdict: AttemptRecovered, detail: join([why, "; unit ", unit_lifecycle_wire(u: lifecycle)], "")) { AttemptLostRace { expected: e, observed: o } => RecoveryRefused { detail: join(["the attempt moved while recovery was deciding: expected ", e, " and the slot holds ", o, "; the observation this recovery would act on is stale, so nothing is released"], "") } AttemptTransitionRefused { detail: d } => RecoveryRefused { detail: join(["the attempt slot would not take the terminal generation: ", d], "") } diff --git a/dag/gunbc/compute/work_request.dag b/dag/gunbc/compute/work_request.dag index befcf3a2f43..55753c06478 100644 --- a/dag/gunbc/compute/work_request.dag +++ b/dag/gunbc/compute/work_request.dag @@ -181,16 +181,50 @@ fn work_outcome_identity(o: WorkOutcome) -> String { } } -fn work_outcome_ending(o: WorkOutcome) -> String { +// HOW A WORK OUTCOME ENDED, WITH ITS PAYLOAD DROPPED. It is the closed set the ending words were +// always drawn from, typed once so a reader that stores an ending (the attempt slot's terminal +// generation) decodes against the set rather than accepting any word. work_outcome_ending is its +// spelling, and work_ending_decode is the inverse of that one spelling, not a second vocabulary. +type WorkEnding + = WorkEndedSucceeded + | WorkEndedFailed + | WorkEndedRefusedByInfrastructure + | WorkEndedCancelled + | WorkEndedOutputMismatch + +fn work_outcome_ending_of(o: WorkOutcome) -> WorkEnding { match o { - WorkSucceeded { identity: _, outputs: _, log_path: _ } => "succeeded" - WorkFailed { identity: _, exit_code: _, log_path: _ } => "failed" - WorkRefusedByInfrastructure { identity: _, cause: _ } => "refused_by_infrastructure" - WorkCancelled { identity: _, detail: _ } => "cancelled" - WorkOutputMismatch { identity: _, detail: _ } => "output_mismatch" + WorkSucceeded { identity: _, outputs: _, log_path: _ } => WorkEndedSucceeded + WorkFailed { identity: _, exit_code: _, log_path: _ } => WorkEndedFailed + WorkRefusedByInfrastructure { identity: _, cause: _ } => WorkEndedRefusedByInfrastructure + WorkCancelled { identity: _, detail: _ } => WorkEndedCancelled + WorkOutputMismatch { identity: _, detail: _ } => WorkEndedOutputMismatch + } +} + +fn work_ending_wire(e: WorkEnding) -> String { + match e { + WorkEndedSucceeded => "succeeded" + WorkEndedFailed => "failed" + WorkEndedRefusedByInfrastructure => "refused_by_infrastructure" + WorkEndedCancelled => "cancelled" + WorkEndedOutputMismatch => "output_mismatch" } } +fn work_ending_decode(word: String) -> WorkEnding? { + if word == work_ending_wire(e: WorkEndedSucceeded) { Present { value: WorkEndedSucceeded } } + else if word == work_ending_wire(e: WorkEndedFailed) { Present { value: WorkEndedFailed } } + else if word == work_ending_wire(e: WorkEndedRefusedByInfrastructure) { Present { value: WorkEndedRefusedByInfrastructure } } + else if word == work_ending_wire(e: WorkEndedCancelled) { Present { value: WorkEndedCancelled } } + else if word == work_ending_wire(e: WorkEndedOutputMismatch) { Present { value: WorkEndedOutputMismatch } } + else { none } +} + +fn work_outcome_ending(o: WorkOutcome) -> String { + work_ending_wire(e: work_outcome_ending_of(o: o)) +} + fn work_refusal_detail(r: WorkInfrastructureRefusal) -> String { match r { ProducerInFlight { lease_path } => join(["a producer already holds the lease at ", lease_path, "; attach later, do not start a second one"], "") diff --git a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag index 7b90bfea1aa..b643ec9e7a9 100644 --- a/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag +++ b/dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag @@ -13,8 +13,12 @@ import gunbc.compute.work_provider_local { compute_settle_attempt, compute_attempt_settlement_wire, attempt_settlement_host_record, host_record_was_written, } +import gunbc.compute.work_request { + WorkEnding, WorkEndedSucceeded, WorkEndedFailed, WorkEndedRefusedByInfrastructure, WorkEndedCancelled, WorkEndedOutputMismatch, +} import gunbc.compute.attempt_lifecycle { AttemptRecord, AttemptState, AttemptReserved, AttemptLaunching, AttemptTerminal, + AttemptVerdict, AttemptWorkEnded, AttemptRecovered, attempt_verdict_wire, AttemptSlotReading, AttemptSlotAbsent, AttemptSlotHeld, AttemptSlotUnreadable, attempt_slot_read, AttemptTransition, AttemptAdvanced, AttemptLostRace, AttemptTransitionRefused, attempt_open, attempt_begin_launch, attempt_terminate, attempt_transition_advanced, attempt_transition_wire, @@ -89,7 +93,7 @@ test fn a_terminal_transition_refuses_a_launch_that_had_not_spawned_yet() -> Boo let record = record_for(reference: "attempt-terminate-first") let opened = attempt_open(root: root, record: record) let after_open = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) - let terminated = attempt_terminate(root: root, record: record, expected: generation_of(r: after_open), verdict: "recovered", detail: "the producer never launched") + let terminated = attempt_terminate(root: root, record: record, expected: generation_of(r: after_open), verdict: AttemptRecovered, detail: "the producer never launched") let after_terminal = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) let late_launch = attempt_begin_launch(root: root, record: record, expected: generation_of(r: after_open)) let final_read = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) @@ -117,7 +121,7 @@ test fn a_launch_that_won_refuses_a_recovery_arriving_behind_it() -> Bool { let after_open = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) let launched = attempt_begin_launch(root: root, record: record, expected: generation_of(r: after_open)) let after_launch = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) - let late_recovery = attempt_terminate(root: root, record: record, expected: generation_of(r: after_open), verdict: "recovered", detail: "arrived after the launch") + let late_recovery = attempt_terminate(root: root, record: record, expected: generation_of(r: after_open), verdict: AttemptRecovered, detail: "arrived after the launch") let final_read = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) attempt_transition_advanced(t: opened) && attempt_transition_advanced(t: launched) @@ -156,12 +160,12 @@ test fn settling_records_why_and_does_not_erase_what_it_settled() -> Bool { let g1 = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) let launched = attempt_begin_launch(root: root, record: record, expected: generation_of(r: g1)) let g2 = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) - let settled = attempt_terminate(root: root, record: record, expected: generation_of(r: g2), verdict: "released", detail: "unit terminated-failure, exit 101") + let settled = attempt_terminate(root: root, record: record, expected: generation_of(r: g2), verdict: AttemptWorkEnded { ending: WorkEndedFailed }, detail: "unit terminated-failure, exit 101") let g3 = attempt_slot_read(root: root, reference: record.reference as NonEmptyStr) let terminal = held_record(r: g3) attempt_transition_advanced(t: opened) && attempt_transition_advanced(t: launched) && attempt_transition_advanced(t: settled) && generation_of(r: g1) == 1 && generation_of(r: g2) == 2 && generation_of(r: g3) == 3 - && (match terminal.state { AttemptTerminal { verdict: v, detail: d } => v == "released" && string_contains(s: d, pattern: "exit 101") AttemptReserved => false AttemptLaunching => false }) + && (match terminal.state { AttemptTerminal { verdict: v, detail: d } => attempt_verdict_wire(v: v) == "failed" && string_contains(s: d, pattern: "exit 101") AttemptReserved => false AttemptLaunching => false }) && Filesystem.Read(path: join([root as String, "/", record.reference, ".1"], "")).success && Filesystem.Read(path: join([root as String, "/", record.reference, ".2"], "")).success && string_contains(s: Filesystem.Read(path: join([root as String, "/", record.reference, ".1"], "")).content, pattern: "reserved") @@ -214,7 +218,7 @@ test fn a_launched_attempt_is_recovered_only_on_an_ended_unit() -> Bool { let terminal = AttemptRecord { identity: "abcdef0123456789", reference: "attempt-done", unit: "gunbc-compute-abcdef0123456789-attempt-done", granted: kibibyte(count: 27262976), - opened_at: 1800, launch_deadline: 2000, state: AttemptTerminal { verdict: "released", detail: "done" }, + opened_at: 1800, launch_deadline: 2000, state: AttemptTerminal { verdict: AttemptWorkEnded { ending: WorkEndedSucceeded }, detail: "done" }, } recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitTerminatedSuccess { detail: "exit 0" }, now: 9999)) && recovery_trigger_fires(t: recovery_trigger(record: launched, lifecycle: UnitTerminatedFailure { detail: "exit 101" }, now: 9999)) @@ -245,6 +249,44 @@ test fn an_undecodable_slot_is_unreadable_and_never_absent() -> Bool { && (match well_formed { Present { value: r } => kibibyte_count(k: r.granted) == 1 && r.opened_at == 5 Absent => false }) } +fn terminal_record_text(verdict_word: String) -> String { + join(["{\"identity\":\"abcdef0123456789\",\"reference\":\"attempt-verdict\",\"unit\":\"u\",\"granted_kib\":1,\"opened_at\":5,\"launch_deadline\":100,\"state\":\"terminal:", verdict_word, "\",\"detail\":\"why\"}"], "") +} + +fn verdict_round_trips(v: AttemptVerdict) -> Bool { + let written = attempt_record_wire(r: AttemptRecord { + identity: "abcdef0123456789", reference: "attempt-verdict", unit: "u", granted: kibibyte(count: 1), + opened_at: 5, launch_deadline: 100, state: AttemptTerminal { verdict: v, detail: "why" }, + }) + match attempt_record_decode(text: written) { + Present { value: r } => + match r.state { + AttemptTerminal { verdict: back, detail: d } => attempt_verdict_wire(v: back) == attempt_verdict_wire(v: v) && d == "why" + AttemptReserved => false + AttemptLaunching => false + } + Absent => false + } +} + +// A TERMINAL VERDICT IS A CLOSED SET AND A WORD OUTSIDE IT IS UNREADABLE, like an unknown state word +// (review 70244). The positive half writes every verdict the two writers can produce through the +// real record wire and reads it back; the RED half hands the real decode a misspelling and the word +// this file itself used to write, "released", which no producer ever wrote -- both must come back +// unreadable rather than as a terminal state some later reader trusts. +test fn a_terminal_verdict_outside_the_closed_set_is_unreadable() -> Bool { + verdict_round_trips(v: AttemptRecovered) + && verdict_round_trips(v: AttemptWorkEnded { ending: WorkEndedSucceeded }) + && verdict_round_trips(v: AttemptWorkEnded { ending: WorkEndedFailed }) + && verdict_round_trips(v: AttemptWorkEnded { ending: WorkEndedRefusedByInfrastructure }) + && verdict_round_trips(v: AttemptWorkEnded { ending: WorkEndedCancelled }) + && verdict_round_trips(v: AttemptWorkEnded { ending: WorkEndedOutputMismatch }) + && (match attempt_record_decode(text: terminal_record_text(verdict_word: "recovered")) { Present { value: _ } => true Absent => false }) + && (match attempt_record_decode(text: terminal_record_text(verdict_word: "recoverd")) { Present { value: _ } => false Absent => true }) + && (match attempt_record_decode(text: terminal_record_text(verdict_word: "released")) { Present { value: _ } => false Absent => true }) + && (match attempt_record_decode(text: terminal_record_text(verdict_word: "")) { Present { value: _ } => false Absent => true }) +} + // THE SELF-DECLARED BOUND HAS A BOUND, AND AN OVER-LONG DECLARATION REFUSES RATHER THAN BEING // CLAMPED. The deadline is declared by the party it protects, so nothing in the gate alone stops a // producer declaring an hour -- and the cost of that does not land on the producer, it lands on @@ -318,8 +360,8 @@ test fn the_loser_of_a_launch_race_never_reports_that_it_settled_the_slot() -> B } let winner = compute_begin_launch(layout: layout, gate: AttemptGateReady { record: rec, generation: generation }) let loser = compute_begin_launch(layout: layout, gate: AttemptGateReady { record: rec, generation: generation }) - let winner_settled = compute_settle_attempt(layout: layout, gate: winner, verdict: "succeeded", detail: "the winner closes the slot") - let loser_settled = compute_settle_attempt(layout: layout, gate: loser, verdict: "succeeded", detail: "the loser has no slot to close") + let winner_settled = compute_settle_attempt(layout: layout, gate: winner, verdict: AttemptWorkEnded { ending: WorkEndedSucceeded }, detail: "the winner closes the slot") + let loser_settled = compute_settle_attempt(layout: layout, gate: loser, verdict: AttemptWorkEnded { ending: WorkEndedSucceeded }, detail: "the loser has no slot to close") attempt_transition_advanced(t: opened) && (match winner { AttemptGateReady { record: _, generation: _ } => true AttemptGateRefused { detail: _ } => false }) && (match loser { AttemptGateReady { record: _, generation: _ } => false AttemptGateRefused { detail: _ } => true }) diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index fccf1f5e6fb..8663b99517b 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -363,6 +363,12 @@ fn local_repo_wet_schedule() -> List { function: "an_undecodable_slot_is_unreadable_and_never_absent", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "a_terminal_verdict_outside_the_closed_set_is_unreadable" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "a_terminal_verdict_outside_the_closed_set_is_unreadable", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "an_over_long_pre_launch_declaration_refuses_at_open_and_is_not_clamped" }, entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", From b6d0b764766878de5a16a1da75a4d9245f556579 Mon Sep 17 00:00:00 2001 From: "gunbai-bot[bot]" <289086189+gunbai-bot[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:29:39 -0400 Subject: [PATCH 40/44] A manager that cannot be asked is a property not queried, not an aborted settlement (#12044) * A manager that cannot be asked is a property not queried: ShowUserProperty opts into ShellOutcome, and the two unit readings fold it before the fold decides Co-Authored-By: Claude Opus 5 (1M context) * The check that replaced a decoration was a decoration too, and only the mutation said so Review 69970, both findings, plus one the review did not name and one my own first repair introduced. FINDING 1, the borrowed exclusion reason. The row reused excl_shell_spawn_refused_reason, which substantiates a DIFFERENT subject in its own words -- whether the host can spawn argv[0], one effect, a local program. This witness's subject is a systemd user manager, four unit properties and a cgroup read. DESIGN 3b: a home that resolves but owns only part of the scope a row claims is a scope mismatch on the row. It now has excl_manager_unaskable_reason and its own dissolution, which also states what a PARTIAL mock does not discharge. Returning the borrowed row repaired its own sentence: `this row and its four schedule rows delete together` is true again. FINDING 2, the decoration. On the one lane this cell runs (srv1, systemctl present) the claim was `if present { true }` and the pre-change behaviour was identical, so nothing on the executing path could turn it red. The subject was FUSED TO THE EFFECT -- observable only where systemd is absent. unit_property_ from_shell is now its own fold, so both ShellOutcome arms are authorable anywhere, and the wet claim keeps the inhabitance half with something real to assert on the present-manager branch. THE THIRD PROBLEM, WHICH IS FINDING 2 ONE MOVE ALONG. The new hermetic claim was sitting in a DISCOVERY-EXCLUDED file, so it would have executed on the change that wrote it and never again. Exclusion silently converts `always runs` into `runs when this file changes`, which is invisible at the call site and reads as coverage. The fold's claim moved to work_request_witness, discovered every run; the wet file keeps only what needs a host, and says why the split is where it is. THE FOURTH, AND IT IS MINE. The relocated claim PASSED against a fold mutated to `ShellSpawnRefused => { queried: success, value: value }` -- the wrong arm that reads the flag instead of the outcome. I had supplied that arm with success: false and an empty value, so the correct fold and the wrong one produced the SAME reading and the claim carried no information. Right interface, right layer, right assertions, zero discrimination. The separating input is a refusal with a FLATTERING FLAG -- success: true, value: "active" -- because the contract is that the OUTCOME decides: a spawn that produced no process is not a reading whatever the transport's other fields say. CONTROLS at this head: mutation FAIL, restored PASS, and the wet inhabitance cell PASS. The reasoning about the inputs is in the claim rather than here, because the next person to edit them is the one who needs it. Also merges the current gunbc#12040 head. Chunk names did not collide this time (21/22 against 23) and all four checks ran: no duplicate name, 24 definitions against 24 registered members, no duplicate identity, module resolves. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: gunbc-ci-auto-heal Co-authored-by: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemctl.dag | 2 + dag/gunbc/ci/ci_layer_roots.dag | 9 ++ dag/gunbc/compute/work_provider_local.dag | 79 ++++++++++++----- .../manager_unaskable_wet_witness_test.dag | 87 +++++++++++++++++++ .../compute/work_request_witness_test.dag | 39 +++++++++ src/v2/workflow/floor_route_gap.dag | 12 ++- src/v2/workflow/local_repo_wet_terminal.dag | 6 ++ 7 files changed, 209 insertions(+), 25 deletions(-) create mode 100644 dag/test/claim/compute/manager_unaskable_wet_witness_test.dag diff --git a/dag/extdeps/systemd/systemctl.dag b/dag/extdeps/systemd/systemctl.dag index 876ed292fbf..045e1271052 100644 --- a/dag/extdeps/systemd/systemctl.dag +++ b/dag/extdeps/systemd/systemctl.dag @@ -1,5 +1,6 @@ module extdeps.systemd.systemctl +import extdeps.transports.shell { ShellOutcome } import std.types { NonEmptyStr, List, Bool, Int, Unit } import std.string_type { String } import std.algebra { trim } @@ -595,6 +596,7 @@ service systemd.Systemctl { operation ShowUserProperty { input { unit: NonEmptyStr, property: NonEmptyStr } output { + outcome: ShellOutcome value: String from "stdout" success: Bool from "exit_success" } diff --git a/dag/gunbc/ci/ci_layer_roots.dag b/dag/gunbc/ci/ci_layer_roots.dag index bbed91791a7..e066acfc7ab 100644 --- a/dag/gunbc/ci/ci_layer_roots.dag +++ b/dag/gunbc/ci/ci_layer_roots.dag @@ -327,6 +327,10 @@ data excl_run_verdict_exit_status_dissolve: DissolutionCondition = unbound_disso data excl_shell_spawn_refused_reason: String = "SUBSTANTIATED per-row: the subject of test.claim.shell_spawn_refused_real_execution_witness is whether the host can spawn argv[0], and whether the seed's shell transport turns a failed spawn into ShellSpawnRefused rather than aborting the evaluation. Both are facts about the real spawn, so the file cannot be made hermetic: a mock_response for test.ShellSpawnProbe.Observed would be authored data, and the discriminating red (an absent binary yields ShellSpawnRefused) would pass against a fabricated refusal whether or not the transport realizes one. Its fns are scheduled in v2.workflow.local_repo_wet_terminal local_repo_wet_schedule, which the required floor executes: the only effect is spawning a local program (true, false, an absent name) -- no network, no cargo, no remote host, no writes." +data excl_manager_unaskable_reason: String = "SUBSTANTIATED per-row, and it is its OWN row rather than a second use of excl_shell_spawn_refused_reason because that reason substantiates a DIFFERENT SUBJECT AND A DIFFERENT EFFECT SET (review 69970): it says in terms that the subject of test.claim.shell_spawn_refused_real_execution_witness is whether the host can spawn argv[0], and that its only effect is spawning a local program. The subject here is a SYSTEMD USER MANAGER -- whether gunbc.compute.work_provider_local read_user_unit_property folds a refused spawn of systemctl into a property that was NOT QUERIED, so the two unit readings reach their own unavailable arm instead of the whole evaluation ending as a shell-spawn-refused runtime error with the seat charged and the producer lease held. The effects are the manager probe, systemctl ShowUserProperty for four unit properties, and a cgroup events read: a superset of the neighbour row's one effect, over a different boundary. DESIGN section 3b: a home that resolves but owns only part of the scope a row claims is a scope/ownership mismatch on the row. The negative half that admits it to the same LANE is the one every member shares: no network, no cargo, no remote host, no install media, and nothing written outside what the witness creates." + +data excl_manager_unaskable_dissolve: DissolutionCondition = unbound_dissolution(description: "mock_response coverage lands for the manager probe AND for systemd.Systemctl.ShowUserProperty, sufficient for the queried/not-queried fold to be re-checked hermetically against BOTH shell outcomes; then the fn re-enrols as an ordinary hermetic discovery row, drops off local_repo_wet_schedule and off floor_route_gap, and this row deletes. A mock for the probe ALONE does not discharge it: the subject is what the property reader does when the manager cannot be spawned, so a run that reaches ShowUserProperty through a fixture and never exercises the refusal arm re-checks the assertion against authored data rather than against the boundary.") + data excl_shell_spawn_refused_dissolve: DissolutionCondition = unbound_dissolution(description: "the spawn boundary of the shell transport becomes executable under the execution-corpus scope prefix as ExecutionWitnessKind rows in gunbc.commit_workflow, the same terminal excl_bin_wet_dissolve names; then this row and its four schedule rows delete together. Publishing a mock_response for ShellSpawnProbe.Observed does NOT discharge it: a mocked ShellSpawnRefused re-checks the assertions against a fixture, not against the spawn.") // bin-execution witness class re-homed to the falsifier wet cadence (CI floor endgame D6): the @@ -1031,6 +1035,11 @@ data witness_exclusion_frontier: List = [ classification: LocalRepoWetLane, reason: excl_local_repo_wet_tempdir_write_reason, dissolution: excl_local_repo_wet_dissolve}, + WitnessExclusionRow { + pattern: "manager_unaskable_wet_witness_test.dag", + classification: LocalRepoWetLane, + reason: excl_manager_unaskable_reason, + dissolution: excl_manager_unaskable_dissolve}, WitnessExclusionRow { pattern: "attempt_lifecycle_wet_witness_test.dag", classification: LocalRepoWetLane, diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index b31f69247c5..9a256bd0edf 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -12,6 +12,7 @@ import extdeps.git { shape_git_worktree_add_detached_argv } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex, git_object_id_wire_hex } import extdeps.systemd.systemd_run { systemd_run_user_wait_arguments, systemd_run_property } import extdeps.systemd.systemctl +import extdeps.transports.shell { ShellOutcome, ShellExited, ShellSpawnRefused } import extdeps.linux.cgroup_v2 { cgroup_v2_mount_point, cgroup_v2_events_path, CgroupEventsDecoding, CgroupEventsDecoded, CgroupEventsUnparseable, decode_cgroup_events_populated, @@ -571,23 +572,59 @@ fn unit_lifecycle_decide(o: UnitLifecycleObservations) -> UnitLifecycle { // it invented can stop a unit belonging to a different attempt, and the record is the only // authority for that binding. fn observe_unit_lifecycle(unit: NonEmptyStr) -> UnitLifecycle { - let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) - let active = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_active_state_property) - let result = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_result_property) - let cg = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_control_group_property) - let events = if cg.success && trim(s: cg.value) != "" { - linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) + let cg = read_user_unit_property(unit: unit, property: unit_control_group_property) + unit_lifecycle_decide(o: UnitLifecycleObservations { + load_state: read_user_unit_property(unit: unit, property: unit_load_state_property), + active_state: read_user_unit_property(unit: unit, property: unit_active_state_property), + result: read_user_unit_property(unit: unit, property: unit_result_property), + population: unit_population(control_group: cg, events: read_cgroup_events(control_group: cg)), + }) +} + +// ONE READER FOR A USER-UNIT PROPERTY, AND IT FOLDS THE TRANSPORT'S ANSWER BEFORE THE PROGRAM'S. +// systemctl answering nonzero and systemctl NOT RUNNING are different facts with different +// remedies (extdeps.transports.shell ShellOutcome): the first is the manager refusing a question, +// the second is a host with no manager to ask. Before this reader the second ended the whole +// evaluation as a shell-spawn-refused runtime error -- the settlement or recovery died mid-route, +// leaving the seat charged AND the producer lease held with no record saying why, which is the +// wedged state the recovery door exists to undo, manufactured by the driver itself. The fold's +// unavailable arm was never reached because the refusal pre-empted it. Now a spawn refusal is a +// property that was NOT QUERIED, and the fold decides exactly as it does for a manager that could +// not be read: the charge is kept and the outcome says the manager could not be asked. No caller +// branched on the difference before, because no field carried it; the difference is that the +// route now reaches its own unavailable arm instead of a diagnostic about a program. +fn read_user_unit_property(unit: NonEmptyStr, property: NonEmptyStr) -> UnitPropertyReading { + let read = systemd.Systemctl.ShowUserProperty(unit: unit, property: property) + unit_property_from_shell(outcome: read.outcome, success: read.success, value: read.value) +} + +// THE DECISION IS SEPARATED FROM THE EFFECT SO A CONTROL CAN REACH IT (review 69970). Fused to the +// ShowUserProperty call, this mapping was only observable on a host with NO systemd user manager, +// and the one lane the witness runs on HAS one -- so the claim over it was `if present { true }`, +// permanently green on the only route that executes it, which DESIGN 4b names as worse than absent +// because it will be cited as coverage. As a fold over a supplied ShellOutcome it is a claim at ONE +// INTERFACE with both arms authorable anywhere, and the wet witness keeps the inhabitance half: +// that the real producer on a host WITH a manager emits ShellExited and reaches a queried reading. +// +// A REFUSED SPAWN IS NOT A FALSE ANSWER, which is the whole content of the fold. systemctl +// answering nonzero means the manager refused the question; systemctl not running means there was +// no manager to ask. Only the first is a reading, so the second is queried: false and carries no +// value -- and the decide folds downstream already route a not-queried property to their own +// unavailable arm, keeping the charge and saying the manager could not be asked. +fn unit_property_from_shell(outcome: ShellOutcome, success: Bool, value: String) -> UnitPropertyReading { + match outcome { + ShellSpawnRefused { program: _, cause: _ } => UnitPropertyReading { queried: false, value: "" } + ShellExited => UnitPropertyReading { queried: success, value: value } + } +} + +fn read_cgroup_events(control_group: UnitPropertyReading) -> UnitPropertyReading { + let events = if control_group.queried && trim(s: control_group.value) != "" { + linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: control_group.value)], "") as NonEmptyStr)) } else { linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: cgroup_v2_mount_point)) } - unit_lifecycle_decide(o: UnitLifecycleObservations { - load_state: UnitPropertyReading { queried: load.success, value: load.value }, - active_state: UnitPropertyReading { queried: active.success, value: active.value }, - result: UnitPropertyReading { queried: result.success, value: result.value }, - population: unit_population( - control_group: UnitPropertyReading { queried: cg.success, value: cg.value }, - events: UnitPropertyReading { queried: events.success, value: events.value }), - }) + UnitPropertyReading { queried: events.success, value: events.value } } // THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is @@ -600,18 +637,12 @@ fn observe_unit_lifecycle(unit: NonEmptyStr) -> UnitLifecycle { // property become a conclusion in the first place. fn observe_unit_termination(identity_hex: String, attempt_completed: Bool) -> UnitTermination { let unit = compute_unit_name(identity_hex: identity_hex) - let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) - let cg = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_control_group_property) - let events = if cg.success && trim(s: cg.value) != "" { - linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) - } else { - linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: cgroup_v2_mount_point)) - } + let cg = read_user_unit_property(unit: unit, property: unit_control_group_property) unit_termination_decide(o: UnitObservations { attempt_completed: attempt_completed, - load_state: UnitPropertyReading { queried: load.success, value: load.value }, - control_group: UnitPropertyReading { queried: cg.success, value: cg.value }, - events: UnitPropertyReading { queried: events.success, value: events.value }, + load_state: read_user_unit_property(unit: unit, property: unit_load_state_property), + control_group: cg, + events: read_cgroup_events(control_group: cg), }) } diff --git a/dag/test/claim/compute/manager_unaskable_wet_witness_test.dag b/dag/test/claim/compute/manager_unaskable_wet_witness_test.dag new file mode 100644 index 00000000000..bc67237a52a --- /dev/null +++ b/dag/test/claim/compute/manager_unaskable_wet_witness_test.dag @@ -0,0 +1,87 @@ +module test.claim.compute.manager_unaskable_wet_witness + +import extdeps.shell +import std.types { Bool, NonEmptyStr, String } +import gunbc.compute.attempt_lifecycle { UnitLifecycle, UnitTerminatedSuccess, UnitTerminatedFailure, UnitTerminatedCauseUnknown, UnitStillRunning, UnitLifecycleUnknown, unit_lifecycle_ended } +import gunbc.compute.work_provider_local { + UnitTermination, UnitAttemptCompleted, UnitAbsentAuthoritatively, UnitPopulated, UnitTerminationUnavailable, + observe_unit_termination, observe_unit_lifecycle, unit_termination_frees_capacity, compute_unit_name, + UnitPropertyReading, unit_property_from_shell, +} +import extdeps.transports.shell { ShellExited, ShellSpawnRefused } + +// A HOST WITH NO USER MANAGER TO ASK, and the arm the settlement and recovery routes reach there. +// The subject is the boundary between the shell transport and the two readings that consume the +// manager: a spawn that never produced a process must arrive at the fold as a property NOT QUERIED +// (extdeps.transports.shell ShellSpawnRefused, opted into by systemd.Systemctl ShowUserProperty), +// so the fold decides UNAVAILABLE and keeps the charge. Before that opt-in the same host ended the +// evaluation as a shell-spawn-refused runtime error before any fold ran -- which is what this +// witness's mutation control reproduces: remove the opt-in and this cell does not go red, it +// ABORTS, because the arm under test was never reachable. +// +// THE ARM IS OBSERVABLE ONLY WHERE systemctl IS ABSENT, and the claim says which host it is on +// rather than choosing one. It reads the host with an independent probe (shell.PosixCommandV +// Check, a different program) and asserts the relation: where systemctl is absent, both readings +// reach their unavailable arm and nothing frees; where it is present, the claim asserts only that +// the readings RETURNED -- the manager may answer not-found (a transient unit that never existed) +// or refuse the bus (a user with no session), and both are that host's honest answers, not this +// subject. So on the local-repo wet lane (srv1, systemctl present) this cell establishes only +// non-abort; the discriminating half executed on a host without systemctl is the exact-head +// receipt on the change that landed it. A lane without systemctl is the declared gap. +// +// THE UNIT NAME IS FOR AN IDENTITY NO PRODUCER MINTS (all-f), so on a host WITH a manager the +// reading is of a unit that does not exist and nothing is stopped or released. +data unaskable_identity: String = "ffffffffffffffff" + +fn manager_present() -> Bool { + shell.PosixCommandV.Check(command: "systemctl" as NonEmptyStr).exists +} + +fn termination_unavailable(t: UnitTermination) -> Bool { + match t { + UnitTerminationUnavailable { detail: d } => string_contains(s: d, pattern: "could not be queried") + UnitAttemptCompleted { detail: _ } => false + UnitAbsentAuthoritatively { detail: _ } => false + UnitPopulated { detail: _ } => false + } +} + +fn lifecycle_unknown(u: UnitLifecycle) -> Bool { + match u { + UnitLifecycleUnknown { detail: d } => string_contains(s: d, pattern: "could not be queried") + UnitTerminatedSuccess { detail: _ } => false + UnitTerminatedFailure { detail: _ } => false + UnitTerminatedCauseUnknown { detail: _ } => false + UnitStillRunning { detail: _ } => false + } +} + +// THE DISCRIMINATING HALF OF THIS SUBJECT LIVES IN test.claim.compute.work_request_witness, +// NOT HERE, and the reason is this file's own exclusion row. unit_property_from_shell is a pure +// fold, so a claim over it needs no host effect -- but this file is excluded from discovery +// (gunbc.ci.ci_layer_roots excl_manager_unaskable_reason), and an excluded file's identities are +// planned only on a run that CHANGES it. A hermetic claim placed here would execute on the change +// that wrote it and never again, which is the same decoration review 69970 found, one move along. +// So the fold's claim sits in the hermetic witness that is discovered every run, and what stays +// here is the half that genuinely needs the host: the inhabitance claim below. +// +// THE INHABITANCE HALF (DESIGN section 3, the pairing obligation). The claim above supplies the +// boundary's inputs, so something must still establish that the REAL producer emits that shape and +// that the real route runs. On a host WITH a manager this asserts the route reaches a RETURNED +// reading rather than aborting, which is the half the wet lane can honestly carry; on a host +// without one it asserts the unavailable arm end to end. Deleting the ShellOutcome opt-in makes +// this cell ABORT rather than answer, which is the control the file's header describes. +test fn a_host_without_a_user_manager_reaches_the_unavailable_arm_instead_of_aborting() -> Bool { + let present = manager_present() + let termination = observe_unit_termination(identity_hex: unaskable_identity, attempt_completed: false) + let lifecycle = observe_unit_lifecycle(unit: compute_unit_name(identity_hex: unaskable_identity)) + if present { + !unit_termination_frees_capacity(t: termination) + && !unit_lifecycle_ended(u: lifecycle) + } else { + termination_unavailable(t: termination) + && !unit_termination_frees_capacity(t: termination) + && lifecycle_unknown(u: lifecycle) + && !unit_lifecycle_ended(u: lifecycle) + } +} diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index f02ff7144d1..fb7da19ce83 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -15,7 +15,9 @@ import gunbc.compute.work_provider_local { UnitLifecycleObservations, unit_lifecycle_decide, unit_population, UnitPopulation, PopulationOccupied, PopulationEmpty, PopulationUnknown, compute_kill_mode_property, compute_kill_mode_value, + UnitPropertyReading, unit_property_from_shell, } +import extdeps.transports.shell { ShellExited, ShellSpawnRefused } import gunbc.compute.attempt_lifecycle { UnitLifecycle, UnitTerminatedSuccess, UnitTerminatedFailure, UnitTerminatedCauseUnknown, UnitStillRunning, UnitLifecycleUnknown, unit_lifecycle_ended, unit_lifecycle_wire, @@ -367,3 +369,40 @@ test fn an_undropped_lease_is_recorded_and_never_becomes_the_works_verdict() -> && string_contains(s: compute_stale_lease_note(layout: layout, error: "EBUSY"), pattern: "matters only to a request that must PRODUCE again") && string_contains(s: compute_stuck_reservation_remedy, pattern: "compute_recover_cli") } + +// THE DISCRIMINATING HALF, AND IT RUNS EVERYWHERE (review 69970). The claim below used to be +// `if present { true }`, and the one lane it is enrolled on HAS a manager -- so on the only route +// that executes it, nothing could turn it red and the pre-change behaviour on that host was +// identical. DESIGN 4b: a check whose forbidden state cannot be expressed where the check runs is +// a decoration, permanently green by construction, and worse than absent because it is cited as +// coverage. The subject is the FOLD, which was fused to the effect and is now its own function, so +// both shell outcomes are authorable here and a host without systemctl is no longer required to +// observe the arm. +// +// THE RED THIS GOES TO: map ShellSpawnRefused to a queried reading, or delete the arm, and the +// first conjunct fails. That is the mapping the settlement and recovery routes depend on -- a +// refused spawn is NOT an answer, so it may not arrive as one. +test fn a_refused_spawn_is_a_property_not_queried_and_never_a_reading() -> Bool { + let refused = unit_property_from_shell( + outcome: ShellSpawnRefused { program: "systemctl" as NonEmptyStr, cause: "No such file or directory" as NonEmptyStr }, + success: false, value: "") + // THE INPUT THAT ACTUALLY DISCRIMINATES, and the first attempt at this claim did not carry it. + // Supplying success: false and an empty value alongside the refusal makes the correct arm and a + // WRONG arm that reads the flag instead of the outcome produce the SAME reading, so the claim + // passed against a fold mutated to `ShellSpawnRefused => { queried: success, value: value }`. + // The fold's contract is that the OUTCOME decides: a spawn that produced no process is not a + // reading whatever the transport's other fields say, so the separating input is a refusal + // carrying a successful-looking flag and a value. + let refused_with_a_flattering_flag = unit_property_from_shell( + outcome: ShellSpawnRefused { program: "systemctl" as NonEmptyStr, cause: "No such file or directory" as NonEmptyStr }, + success: true, value: "active") + let answered = unit_property_from_shell(outcome: ShellExited, success: true, value: "inactive") + let nonzero = unit_property_from_shell(outcome: ShellExited, success: false, value: "") + !refused.queried + && refused.value == "" + && !refused_with_a_flattering_flag.queried + && refused_with_a_flattering_flag.value == "" + && answered.queried + && answered.value == "inactive" + && !nonzero.queried +} diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 62f901d283e..b68fef2e06f 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1582,8 +1582,18 @@ fn floor_route_gap_expectation_chunk_22() -> List { } } } } } } } } +// test.claim.compute.manager_unaskable_wet_witness: its first shell operation on the hermetic route +// is the manager probe shell.PosixCommandV Check, which declares no mock_response, so the nominal +// floor refuses there and the identity is held; the local-repo wet schedule carries it. +fn floor_route_gap_expectation_chunk_23() -> List { + Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.manager_unaskable_wet_witness.a_host_without_a_user_manager_reaches_the_unavailable_arm_instead_of_aborting", operation: "Check", ground: NoMockResponse {} }, + tail: Empty {} + } +} + fn floor_route_gap_expectation_chunks() -> List> { - Cons { head: floor_route_gap_expectation_chunk_00(), tail: Cons { head: floor_route_gap_expectation_chunk_01(), tail: Cons { head: floor_route_gap_expectation_chunk_02(), tail: Cons { head: floor_route_gap_expectation_chunk_03(), tail: Cons { head: floor_route_gap_expectation_chunk_04(), tail: Cons { head: floor_route_gap_expectation_chunk_05(), tail: Cons { head: floor_route_gap_expectation_chunk_06(), tail: Cons { head: floor_route_gap_expectation_chunk_07(), tail: Cons { head: floor_route_gap_expectation_chunk_08(), tail: Cons { head: floor_route_gap_expectation_chunk_09(), tail: Cons { head: floor_route_gap_expectation_chunk_10(), tail: Cons { head: floor_route_gap_expectation_chunk_11(), tail: Cons { head: floor_route_gap_expectation_chunk_12(), tail: Cons { head: floor_route_gap_expectation_chunk_13(), tail: Cons { head: floor_route_gap_expectation_chunk_14(), tail: Cons { head: floor_route_gap_expectation_chunk_15(), tail: Cons { head: floor_route_gap_expectation_chunk_16(), tail: Cons { head: floor_route_gap_expectation_chunk_17(), tail: Cons { head: floor_route_gap_expectation_chunk_18(), tail: Cons { head: floor_route_gap_expectation_chunk_19(), tail: Cons { head: floor_route_gap_expectation_chunk_20(), tail: Cons { head: floor_route_gap_expectation_chunk_21(), tail: Cons { head: floor_route_gap_expectation_chunk_22(), tail: Empty {} } } } } } } } } } } } } } } } } } } } } } } } + Cons { head: floor_route_gap_expectation_chunk_00(), tail: Cons { head: floor_route_gap_expectation_chunk_01(), tail: Cons { head: floor_route_gap_expectation_chunk_02(), tail: Cons { head: floor_route_gap_expectation_chunk_03(), tail: Cons { head: floor_route_gap_expectation_chunk_04(), tail: Cons { head: floor_route_gap_expectation_chunk_05(), tail: Cons { head: floor_route_gap_expectation_chunk_06(), tail: Cons { head: floor_route_gap_expectation_chunk_07(), tail: Cons { head: floor_route_gap_expectation_chunk_08(), tail: Cons { head: floor_route_gap_expectation_chunk_09(), tail: Cons { head: floor_route_gap_expectation_chunk_10(), tail: Cons { head: floor_route_gap_expectation_chunk_11(), tail: Cons { head: floor_route_gap_expectation_chunk_12(), tail: Cons { head: floor_route_gap_expectation_chunk_13(), tail: Cons { head: floor_route_gap_expectation_chunk_14(), tail: Cons { head: floor_route_gap_expectation_chunk_15(), tail: Cons { head: floor_route_gap_expectation_chunk_16(), tail: Cons { head: floor_route_gap_expectation_chunk_17(), tail: Cons { head: floor_route_gap_expectation_chunk_18(), tail: Cons { head: floor_route_gap_expectation_chunk_19(), tail: Cons { head: floor_route_gap_expectation_chunk_20(), tail: Cons { head: floor_route_gap_expectation_chunk_21(), tail: Cons { head: floor_route_gap_expectation_chunk_22(), tail: Cons { head: floor_route_gap_expectation_chunk_23(), tail: Empty {} } } } } } } } } } } } } } } } } } } } } } } } } } fn floor_route_gap_expectations() -> List { diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 8663b99517b..7db00b7ee15 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -309,6 +309,12 @@ fn local_repo_wet_schedule() -> List { function: "two_instances_on_one_host_resolve_to_one_pool_at_one_root", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.manager_unaskable_wet_witness", function: "a_host_without_a_user_manager_reaches_the_unavailable_arm_instead_of_aborting" }, + entry: "dag/test/claim/compute/manager_unaskable_wet_witness_test.dag", + function: "a_host_without_a_user_manager_reaches_the_unavailable_arm_instead_of_aborting", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "releasing_one_seat_twice_is_not_a_ledger_failure_and_charges_nothing" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", From b7eab63dd95499b054bddf863193272e2273cb28 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 23 Sep 2026 00:47:10 +0000 Subject: [PATCH 41/44] The note on the refused-spawn claim sat inside its body, where no annotation is admitted Hoisted above the test fn (DESIGN 4c). Swept every .dag file in the PR's diff for an indented // line; this was the only one. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../claim/compute/work_request_witness_test.dag | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index fb7da19ce83..7b517964b66 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -382,17 +382,18 @@ test fn an_undropped_lease_is_recorded_and_never_becomes_the_works_verdict() -> // THE RED THIS GOES TO: map ShellSpawnRefused to a queried reading, or delete the arm, and the // first conjunct fails. That is the mapping the settlement and recovery routes depend on -- a // refused spawn is NOT an answer, so it may not arrive as one. +// +// THE INPUT THAT ACTUALLY DISCRIMINATES, and the first attempt at this claim did not carry it. +// Supplying success: false and an empty value alongside the refusal makes the correct arm and a +// WRONG arm that reads the flag instead of the outcome produce the SAME reading, so the claim +// passed against a fold mutated to `ShellSpawnRefused => { queried: success, value: value }`. +// The fold's contract is that the OUTCOME decides: a spawn that produced no process is not a +// reading whatever the transport's other fields say, so the separating input is a refusal +// carrying a successful-looking flag and a value. test fn a_refused_spawn_is_a_property_not_queried_and_never_a_reading() -> Bool { let refused = unit_property_from_shell( outcome: ShellSpawnRefused { program: "systemctl" as NonEmptyStr, cause: "No such file or directory" as NonEmptyStr }, success: false, value: "") - // THE INPUT THAT ACTUALLY DISCRIMINATES, and the first attempt at this claim did not carry it. - // Supplying success: false and an empty value alongside the refusal makes the correct arm and a - // WRONG arm that reads the flag instead of the outcome produce the SAME reading, so the claim - // passed against a fold mutated to `ShellSpawnRefused => { queried: success, value: value }`. - // The fold's contract is that the OUTCOME decides: a spawn that produced no process is not a - // reading whatever the transport's other fields say, so the separating input is a refusal - // carrying a successful-looking flag and a value. let refused_with_a_flattering_flag = unit_property_from_shell( outcome: ShellSpawnRefused { program: "systemctl" as NonEmptyStr, cause: "No such file or directory" as NonEmptyStr }, success: true, value: "active") From 9436fe2aa6fe3230cf33a6a0d1a683094fcb629e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 23 Sep 2026 02:48:46 +0000 Subject: [PATCH 42/44] Enrol the sixth host_capacity wet identity the floor refused as an unenrolled DirWithTemplate route gap releasing_one_seat_twice_is_not_a_ledger_failure_and_charges_nothing landed with its WetScheduledClaim row but no floor_route_gap row; floor job 106999407163 read route_gap_unenrolled=1 while the local-repo wet lane passed it. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/workflow/floor_route_gap.dag | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index b68fef2e06f..23238930bb1 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1499,6 +1499,13 @@ fn floor_route_gap_expectation_chunk_20() -> List { // hermetically in the same run (standing=planned-and-passed), so a row for either would be a stale // enrolment -- an expectation of a gap that does not occur -- which the floor counts as // stale_route_gap and which is its own red. +// +// A SIXTH ROW, releasing_one_seat_twice_is_not_a_ledger_failure_and_charges_nothing, joined when +// gunbc#12040 added it with its WetScheduledClaim row but not this one -- the incomplete triple +// again. Its floor run (job 106999407163) refused it with the same DirWithTemplate/no +// mock_response gap, route_gap_unenrolled=1, while the local-repo wet lane recorded it +// expected=passed observed=passed. It reaches fresh_root like the five above, so it is held the +// same way and discharged the same way. fn floor_route_gap_expectation_chunk_21() -> List { Cons { head: FloorRouteGapExpectation { identity: "test.claim.compute.host_capacity_wet_witness.a_request_the_machine_cannot_back_refuses_below_the_live_floor", operation: "DirWithTemplate", ground: NoMockResponse {} }, @@ -1510,12 +1517,15 @@ fn floor_route_gap_expectation_chunk_21() -> List { head: FloorRouteGapExpectation { identity: "test.claim.compute.host_capacity_wet_witness.competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.compute.host_capacity_wet_witness.two_requests_summing_past_the_observed_headroom_cannot_both_admit", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.host_capacity_wet_witness.releasing_one_seat_twice_is_not_a_ledger_failure_and_charges_nothing", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Empty {} } } } } } + } } // test.claim.compute.attempt_lifecycle_wet_witness: these six reach shell.Mktemp.DirWithTemplate, From bf63e3cbf49079eff9abec4d5c391c43f07593be Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 23 Sep 2026 04:18:09 +0000 Subject: [PATCH 43/44] Recovery release names the attempt's granted seat as the capacity subject's request main (#12021) gave compute_capacity_subject a request argument; the recovery path from #12040 still called it with one, and the floor refused the prepared subject on that call shape. Co-Authored-By: Claude Opus 5.5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 5f9ec0cf01e..0a018cdc76d 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -1960,7 +1960,7 @@ fn compute_recover_attempt(instance: HostDashboardInstance, reference: NonEmptyS compute_recovery_after_settlement( reference: rec.reference, verdict: join([why, "; unit ", unit_lifecycle_wire(u: lifecycle)], ""), - returned: compute_recover_release(instance: instance, reference: reference, reason: why)) + returned: compute_recover_release(instance: instance, reference: reference, granted: rec.granted, reason: why)) } } } @@ -1979,9 +1979,11 @@ fn compute_recovery_after_settlement(reference: String, verdict: String, returne // THE SEAT COMES BACK THROUGH PR 1'S OWN RELEASE PATH, never a second one. compute_release_on // reads, folds and PROPOSES against the folded pool, so a reference the ledger is not holding is // refused at the proposal rather than appended -- which is what keeps a recovery from poisoning the -// whole partition with an UnknownEncumbrance every later read would refuse on. -fn compute_recover_release(instance: HostDashboardInstance, reference: NonEmptyStr, reason: String) -> ComputeRelease { - match compute_capacity_subject(instance: instance) { +// whole partition with an UnknownEncumbrance every later read would refuse on. The subject's +// request is the seat the attempt record says it was granted -- the release fold consults no +// request, but the subject names the hold it is about rather than a figure invented for it. +fn compute_recover_release(instance: HostDashboardInstance, reference: NonEmptyStr, granted: Kibibyte, reason: String) -> ComputeRelease { + match compute_capacity_subject(instance: instance, request: granted) { ComputeSubjectHostUnresolved { host: h } => ComputeReleaseRefused { detail: join(["no capacity subject resolves for host ", h as String], "") } ComputeSubjectResolved { subject: subj } => compute_release_on( From a308481b50470ded491dcc4ac87ad64bf9c7ede0 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Wed, 23 Sep 2026 05:03:08 +0000 Subject: [PATCH 44/44] Enroll the ceiling-retry gate control: wet schedule row and DirWithTemplate route-gap row The floor at bf63e3cb reported it as an unenrolled route gap (fresh_root is DirWithTemplate); the file's LocalRepoWetLane exclusion already covers it by pattern. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/workflow/floor_route_gap.dag | 8 +++++++- src/v2/workflow/local_repo_wet_terminal.dag | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 52a5bd42bbd..a434e442e0e 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1579,6 +1579,10 @@ fn floor_route_gap_expectation_chunk_21() -> List { // author what the current subset is, which is the only authority for it -- the subset is a property // of evaluation order, and the five-at-one-head against six-at-the-next measured on gunbc#12040 is // what rules out predicting it. +// +// THE EIGHTH ROW, the ceiling retry's gate control, WAS OBSERVED rather than predicted: the floor at +// gunbc#12040 bf63e3cb reported it as an unenrolled ROUTE-GAP at DirWithTemplate. Its first +// statement is fresh_root(), so like the seventh it cannot reach its subject hermetically. fn floor_route_gap_expectation_chunk_22() -> List { Cons { head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.a_launch_that_won_refuses_a_recovery_arriving_behind_it", operation: "DirWithTemplate", ground: NoMockResponse {} }, @@ -1594,8 +1598,10 @@ fn floor_route_gap_expectation_chunk_22() -> List { head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.the_unit_binding_is_read_back_from_the_record_the_attempt_wrote", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.the_loser_of_a_launch_race_never_reports_that_it_settled_the_slot", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.compute.attempt_lifecycle_wet_witness.the_ceiling_retry_opens_its_own_attempt_and_a_held_slot_refuses_it_before_spawn", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Empty {} - } } } } } } } + } } } } } } } } } // test.claim.compute.manager_unaskable_wet_witness: its first shell operation on the hermetic route diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 56ca1c9c325..15e3ee7a9f9 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -395,6 +395,12 @@ fn local_repo_wet_schedule() -> List { function: "the_loser_of_a_launch_race_never_reports_that_it_settled_the_slot", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.attempt_lifecycle_wet_witness", function: "the_ceiling_retry_opens_its_own_attempt_and_a_held_slot_refuses_it_before_spawn" }, + entry: "dag/test/claim/compute/attempt_lifecycle_wet_witness_test.dag", + function: "the_ceiling_retry_opens_its_own_attempt_and_a_held_slot_refuses_it_before_spawn", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.effect_plan_bash_materialize_real_execution_witness", function: "effect_plan_bash_two_declared_operations_execute" }, entry: "dag/test/claim/effect_plan_bash_materialize_real_execution_witness_test.dag",