From 3e3b72c02fa44afbc76ffa4b4c859ebdcfcf5512 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 14:04:58 +0000 Subject: [PATCH 01/28] Compute admission reserves observed host memory atomically, not a count of leases The local compute provider admitted work by listing its lease directory and comparing the count against a policy budget of two. That is not atomic -- the list and the create are two operations on different names, and O_EXCL excludes a NAME, never a QUANTITY, so two requests with different identities both read one lease in flight and both proceeded -- and a count is not a capacity: two producers on a 125 GiB host and two on a 502 GiB host are the same policy applied to machines that differ fourfold. Admission is now a seat in a memory pool. The homes are the ones DESIGN 3b names for fabric/compute: product.capacity.pool owns the appropriation, encumbrance, term and release law; product.capacity.pool_events owns the transitions; gunbc.fabric_event_log linearizes each append with one compare-and-set against the partition head. gunbc.compute.host_capacity declares which pool, where it is linearized (this host, not the fleet DB -- a host's own memory is not a fact the fleet contends on, and routing it through another machine would refuse local builds on a network fault), and joins it to a real reading of /proc/meminfo. The unit's MemoryMax is the amount the pool granted, so the ledger's arithmetic is the enforcement rather than a parallel description of it. Every ending -- succeeded, failed, mismatch, infrastructure refusal -- releases, and a release that did not land is reported rather than assumed. Disposition: compute_cold_build_cap is DELETED, and HostAtColdBuildCap with it. Its fact is now HostComputeCapacityFull, measured in bytes; the two arms beside it are facts a count could not express (the machine's live availability, and an unreadable ledger). The appropriation is two worker ceilings, exactly the memory the deleted count admitted. Also: fabric_seat_acquire/observe generalize over the measure (memory is not a dimensionless count), and the observe/append/retry loop gunbc.fabric_quota held alone moves to the carrier as fabric_pool_event_append, with quota settlement migrated onto it rather than a second copy growing beside it. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/linux/proc_meminfo.dag | 87 ++++- dag/extdeps/linux/procfs.dag | 25 +- dag/gunbc/ci/ci_layer_roots.dag | 5 + dag/gunbc/compute/host_capacity.dag | 356 ++++++++++++++++++ dag/gunbc/compute/work_provider_local.dag | 150 ++++++-- dag/gunbc/compute/work_request.dag | 22 +- dag/gunbc/fabric/fabric_event_log.dag | 71 +++- dag/gunbc/fabric/fabric_quota.dag | 63 +--- dag/gunbc/harness/harness_guidance.dag | 2 +- dag/std/measure.dag | 10 + .../host_capacity_wet_witness_test.dag | 156 ++++++++ .../compute/work_request_witness_test.dag | 6 +- .../claim/os_proc_meminfo_witness_test.dag | 31 ++ src/v2/workflow/local_repo_wet_terminal.dag | 24 ++ 14 files changed, 913 insertions(+), 95 deletions(-) create mode 100644 dag/gunbc/compute/host_capacity.dag create mode 100644 dag/test/claim/compute/host_capacity_wet_witness_test.dag diff --git a/dag/extdeps/linux/proc_meminfo.dag b/dag/extdeps/linux/proc_meminfo.dag index f52806502d4..010a19d0381 100644 --- a/dag/extdeps/linux/proc_meminfo.dag +++ b/dag/extdeps/linux/proc_meminfo.dag @@ -1,8 +1,9 @@ module extdeps.linux.proc_meminfo -import std.types { NonEmptyStr } +import std.types { NonEmptyStr, String, Bool, Int } import std.types { List } -import std.measure { Kibibyte } +import std.measure { Kibibyte, kibibyte } +import std.algebra { trim } import extdeps.external_authority { ExternalAuthority } import extdeps.uri { Uri, Https } @@ -38,3 +39,85 @@ data meminfo_field_names: List = [ "SwapTotal", "SwapFree", ] + +// READING THE FILE'S TEXT BACK INTO THE SHAPE ABOVE. proc.5 describes /proc/meminfo as one metric +// per line, `Name:` followed by a decimal count and, for every field this module names, the unit +// `kB` -- which proc.5 spells in kilobytes and the kernel emits in KIBIBYTES, so the field type is +// Kibibyte and not a kilobyte. The parse lives here because this module owns the file's shape; the +// transport that produces the text is extdeps.linux.procfs, one of N (DESIGN 3). +// +// A MISSING FIELD IS ITS OWN ARM AND NEVER A ZERO. A host whose kernel does not publish +// MemAvailable (pre-3.14) would otherwise read as a host with no memory available, which is the +// state-space conflation an admission decision cannot survive: zero headroom and unknown headroom +// have opposite remedies, and only one of them is a refusal. +type MeminfoRead + = MeminfoParsed { meminfo: ProcMeminfo } + | MeminfoFieldAbsent { field: NonEmptyStr } + +fn meminfo_line_metric(line: String) -> MemoryMetric? { + let tokens = filter(split(s: line, delimiter: " "), t => trim(s: t) != "") + match first(tokens) { + Absent => none + Present { value: head } => + if !ends_with(s: head, suffix: ":") { + none + } else { + let key = substring(s: head, start: 0, end: length(head) - 1) + match first(tokens |> skip(n: 1)) { + Absent => none + Present { value: raw } => + match parse_int(s: trim(s: raw)) { + Absent => none + Present { value: n } => + if key == "" || n < 0 { none } else { Present { value: MemoryMetric { key: key as NonEmptyStr, value: kibibyte(count: n) } } } + } + } + } + } +} + +fn meminfo_metrics(text: String) -> List { + fold(split(s: text, delimiter: "\n"), init: [], f: (acc, line) => + match meminfo_line_metric(line: line) { + Absent => acc + Present { value: m } => concat(acc, [m]) + }) +} + +fn meminfo_metric_named(metrics: List, name: NonEmptyStr) -> Kibibyte? { + match first(filter(metrics, m => (m.key as String) == (name as String))) { + Absent => none + Present { value: m } => Present { value: m.value } + } +} + +// THE SEVEN NAMED FIELDS ARE REQUIRED TOGETHER, in the order meminfo_field_names declares them, so +// the roster above is the authority for what this parse demands rather than a second list here. +fn parse_proc_meminfo(text: String) -> MeminfoRead { + let metrics = meminfo_metrics(text: text) + match first(filter(meminfo_field_names, f => match meminfo_metric_named(metrics: metrics, name: f) { Absent => true Present { value: _ } => false })) { + Present { value: missing } => MeminfoFieldAbsent { field: missing } + Absent => + MeminfoParsed { + meminfo: ProcMeminfo { + mem_total: meminfo_required(metrics: metrics, name: "MemTotal" as NonEmptyStr), + mem_free: meminfo_required(metrics: metrics, name: "MemFree" as NonEmptyStr), + mem_available: meminfo_required(metrics: metrics, name: "MemAvailable" as NonEmptyStr), + buffers: meminfo_required(metrics: metrics, name: "Buffers" as NonEmptyStr), + cached: meminfo_required(metrics: metrics, name: "Cached" as NonEmptyStr), + swap_total: meminfo_required(metrics: metrics, name: "SwapTotal" as NonEmptyStr), + swap_free: meminfo_required(metrics: metrics, name: "SwapFree" as NonEmptyStr), + all_metrics: metrics, + }, + } + } +} + +// Reached only under the absence check above, which is why it can answer at all; the zero is the +// unreachable arm of an optional, not a fabricated reading. +fn meminfo_required(metrics: List, name: NonEmptyStr) -> Kibibyte { + match meminfo_metric_named(metrics: metrics, name: name) { + Present { value: k } => k + Absent => kibibyte(count: 0) + } +} diff --git a/dag/extdeps/linux/procfs.dag b/dag/extdeps/linux/procfs.dag index 7e2721ffc88..7fd793e0995 100644 --- a/dag/extdeps/linux/procfs.dag +++ b/dag/extdeps/linux/procfs.dag @@ -87,9 +87,11 @@ data procfs_paths_read_by_this_repository: List = [ // operation with a shell transport, so gunbc.host_operation_exec can carry it over whichever // transport the caller holds (DESIGN 3: the transport is a realization bound to this shape, never // a fact about procfs). `cat` is the whole realization: procfs files are read-once text and the -// kernel synthesizes them on open, so there is nothing to seek, follow or lock. Three operations -// and not one generic ReadFile, so that the operation's identity says which file it reads and an -// argv over an arbitrary path cannot be materialized from this service. +// kernel synthesizes them on open, so there is nothing to seek, follow or lock. ONE OPERATION PER +// FILE and not one generic ReadFile, so that the operation's identity says which file it reads and +// an argv over an arbitrary path cannot be materialized from this service. ReadMeminfo serves the +// host memory capacity and live availability a compute admission reads before it reserves +// (gunbc.compute.host_capacity). service linux.Procfs { operation ReadStat { input {} @@ -108,6 +110,23 @@ service linux.Procfs { } } + operation ReadMeminfo { + input {} + output { + value: String from "stdout" + success: Bool from "exit_success" + } + readonly + transport shell { argv: ["cat", "/proc/meminfo"] } + exit { + 0 => Unit + nonzero => String "/proc/meminfo read failed" + } + mock_response { + 0 => { value: "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: 8388608 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n", success: true } "hermetic linux.Procfs.ReadMeminfo" + } + } + operation ReadPidStat { input { pid: NonEmptyStr } output { diff --git a/dag/gunbc/ci/ci_layer_roots.dag b/dag/gunbc/ci/ci_layer_roots.dag index 5da6fa4460b..2503885cd4f 100644 --- a/dag/gunbc/ci/ci_layer_roots.dag +++ b/dag/gunbc/ci/ci_layer_roots.dag @@ -1026,6 +1026,11 @@ data witness_exclusion_frontier: List = [ classification: LocalRepoWetLane, reason: excl_local_repo_wet_tempdir_write_reason, dissolution: excl_local_repo_wet_dissolve}, + WitnessExclusionRow { + pattern: "host_capacity_wet_witness_test.dag", + classification: LocalRepoWetLane, + reason: excl_local_repo_wet_tempdir_write_reason, + dissolution: excl_local_repo_wet_dissolve}, WitnessExclusionRow { pattern: "durable_cas_file_store_wet_witness_test.dag", classification: LocalRepoWetLane, diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag new file mode 100644 index 00000000000..51a74c7550d --- /dev/null +++ b/dag/gunbc/compute/host_capacity.dag @@ -0,0 +1,356 @@ +module gunbc.compute.host_capacity + +import std.types { String, Bool, Int, NonEmptyStr, EpochSecs } +import std.nat { Nat } +import std.measure { Measure, Memory, Kibi, Kibibyte, kibibyte, kibibyte_count, kibibyte_from_byte_size_floor, ByteSize, byte_size_count } +import extdeps.linux.procfs +import extdeps.linux.proc_meminfo { MeminfoRead, MeminfoParsed, MeminfoFieldAbsent, parse_proc_meminfo } +import product.capacity.pool { Pool, pool_of, NoReplenishment, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at } +import product.capacity.event_chain { PartitionId } +import product.capacity.pool_events { SeatRequest, PoolReleased } +import product.capacity.lease { LeasePolicy, LeaseGrant, QuiescenceRequired } +import gunbc.fabric_storage_client { FabricStorageBinding, FabricStorageLocalFiles } +import gunbc.fabric_storage_file_store { fabric_storage_file_root, fabric_storage_file_root_ensure, FabricStorageRootStanding, FabricStorageRootReady, FabricStorageRootRefused } +import gunbc.fabric_event_log { + SeatAcquisition, SeatGranted, SeatFull, SeatContended, SeatAcquireRefused, SeatStoreRefused, + fabric_seat_acquire, seat_acquisition_wire, + SeatStandingObservation, SeatRoomObserved, SeatFullObserved, SeatStandingUnobserved, SeatStandingStoreRefused, fabric_seat_observe, + PoolEventAppend, PoolEventAppended, PoolEventContended, PoolEventAppendRefused, fabric_pool_event_append, pool_event_append_wire, +} +import gunbc.fabric_event_log_host { now_epoch_seconds } +import gunbc.roadmap_dashboard_instance { HostDashboardInstance, dashboard_instance_compute_root } + +// THE HOST'S COMPUTE MEMORY IS A POOL, AND A WORK REQUEST TAKES A SEAT IN IT BEFORE IT RUNS. +// +// gunbc.compute.work_provider_local admitted work by COUNTING LEASE FILES against a policy budget +// of two (compute_cold_build_cap): list the lease directory, compare the count, then create a lease +// for THIS identity. Two things were wrong with that and only the second is about arithmetic. +// +// FIRST, IT WAS NOT ATOMIC. The list and the create are two operations on different names, and the +// create excludes only a second producer of the SAME identity. Two requests with DIFFERENT +// identities both read one lease in flight, both compared 1 < 2, and both created their own lease: +// the cap admitted three. O_EXCL cannot close that, because the thing being over-committed is not a +// name -- it is a quantity, and no exclusive create of any path excludes a quantity. +// +// SECOND, A COUNT IS NOT A CAPACITY. Two producers on a 125 GiB host and two on a 502 GiB host are +// the same policy applied to hosts that differ by a factor of four, and a class that needs a +// gibibyte is charged the same as a release build. What the host actually has is memory, so what is +// reserved is memory -- and the ceiling is compared against a READING of the host rather than +// assumed. +// +// WHAT THIS IS NOT A SECOND AUTHORITY FOR. product.capacity.pool owns the appropriation, the +// encumbrance, the term and the release law; product.capacity.pool_events owns the transitions; +// gunbc.fabric_event_log owns the linearization -- one compare-and-set per append against the +// partition's head, which is what makes the reservation atomic across competing requests rather +// than merely checked. This module declares WHICH pool (the host's compute appropriation), WHERE it +// is linearized (this host, not the fleet), and joins it to the observation. It coins no capacity +// vocabulary of its own. + +// ── WHERE THE LEDGER LIVES, AND WHY NOT THE FLEET'S ──────────────────────────────────────────── +// +// gunbc.fabric_storage_placement places the fabric DB on ONE host and every other host reaches it +// over its served endpoint, which is correct for a fact the FLEET contends on -- an upstream's rate +// limit is metered on the credential, so every host must linearize against one partition. +// +// A HOST'S OWN MEMORY IS NOT SUCH A FACT. The contenders for it are all local, and routing their +// reservation through another machine would mean a local build refusing because a remote host was +// unreachable -- a fail-open in the other direction, since the refusal would not be about capacity +// at all. So the store is the SAME realization (gunbc.fabric_storage_file_store, whose exclusive +// create supplies the linearization) rooted under this host's own compute root. The partition names +// the host so a ledger copied between machines does not silently answer for the wrong one. +// THE SUBJECT OF A RESERVATION: which ledger, which partition, how much the pool appropriates and +// how much this request takes. It is separated from the instance deliberately, and not only to be +// testable: everything below is arithmetic and transaction over these four facts, while reading +// them off a HostDashboardInstance is a JOIN with the deployment's authority. Fusing the two would +// have made every reservation carry a whole deployment record in order to add two numbers, and +// would have left no seam at which the transaction can be exercised on a host other than the one +// the deployment describes. +type ComputeCapacitySubject { + root: NonEmptyStr + partition: PartitionId + appropriation: Kibibyte + request: Kibibyte +} + +fn compute_capacity_root(instance: HostDashboardInstance) -> NonEmptyStr { + join([dashboard_instance_compute_root(instance: instance) as String, "/capacity"], "") as NonEmptyStr +} + +fn compute_memory_partition(instance: HostDashboardInstance) -> PartitionId { + join(["compute-memory-", instance.instance_id as String], "") as PartitionId +} + +// THE JOIN WITH THE DEPLOYMENT, and the only function in this module that knows what an instance is. +fn compute_capacity_subject(instance: HostDashboardInstance) -> ComputeCapacitySubject { + ComputeCapacitySubject { + root: compute_capacity_root(instance: instance), + partition: compute_memory_partition(instance: instance), + appropriation: compute_memory_appropriation(instance: instance), + request: compute_request_memory(instance: instance), + } +} + +fn compute_capacity_store(subject: ComputeCapacitySubject) -> FabricStorageBinding { + FabricStorageLocalFiles { root: fabric_storage_file_root(root: subject.root) } +} + +fn compute_capacity_store_ensure(subject: ComputeCapacitySubject) -> FabricStorageRootStanding { + fabric_storage_file_root_ensure(root: fabric_storage_file_root(root: subject.root)) +} + +// ── WHAT THE HOST HAS, READ RATHER THAN SPELLED ──────────────────────────────────────────────── +// +// TWO FIGURES AND THEY ANSWER DIFFERENT QUESTIONS. MemTotal is the machine's installed memory and +// does not move, which is what a ledger needs: an appropriation whose ceiling changed between two +// folds would make the same chain of events add up differently each time it was read. MemAvailable +// is what is free RIGHT NOW including everything outside this pool -- the runner slices, the +// sessions, the page cache the kernel can hand back -- and it is a LIVE FLOOR, not a ceiling. +// +// Both are needed and neither substitutes for the other. The pool alone would admit work while +// another tenant had eaten the machine; MemAvailable alone would admit work whose own ledger says +// the memory is already promised, because memory a reservation holds but has not yet touched is +// still available to read. +type HostMemoryObservation + = HostMemoryObserved { total: Kibibyte, available: Kibibyte } + | HostMemoryUnobserved { detail: String } + +fn observe_host_memory() -> HostMemoryObservation { + let read = linux.Procfs.ReadMeminfo() + if !read.success { + HostMemoryUnobserved { detail: "/proc/meminfo could not be read on this host" } + } else { + match parse_proc_meminfo(text: read.value) { + MeminfoFieldAbsent { field: f } => HostMemoryUnobserved { detail: join(["/proc/meminfo carries no readable ", f as String, " line"], "") } + MeminfoParsed { meminfo: m } => HostMemoryObserved { total: m.mem_total, available: m.mem_available } + } + } +} + +// ── THE APPROPRIATION, AND WHAT IT REPLACES ──────────────────────────────────────────────────── +// +// THE SHARE OF THE HOST THIS PROVIDER MAY COMMIT, denominated in the bytes the old count was +// implicitly worth: compute_cold_build_cap was two producers, and each producer runs under the +// instance's worker memory ceiling, so two ceilings is EXACTLY the memory the deleted row admitted. +// Nothing is widened by this change and a class that needs less than a ceiling now admits more +// requests than a count ever could. +// +// DECLARED FRONTIER (DESIGN 3c), STATED BECAUSE IT IS THE HONEST LIMIT OF THIS ROW: charging this +// appropriation against the host's OTHER commitments -- the runner slice, the sessions slice, the +// fixed overhead -- is gunbc.ci_runner_placement's conservation wall, and this appropriation is +// DECLARED rather than derived from it. What keeps that from being fail-open is the pair of +// observations below: an appropriation larger than the machine refuses outright, and a request is +// refused when the machine's live availability cannot back it whatever the ledger says. +data compute_pool_slots: Nat = 2 + +fn compute_memory_appropriation(instance: HostDashboardInstance) -> Kibibyte { + kibibyte(count: kibibyte_count(k: kibibyte_from_byte_size_floor(b: instance.dispatch_worker_memory_max)) * compute_pool_slots) +} + +// THE UNIT'S CEILING IS THE RESERVED AMOUNT, and that identity is the whole point rather than a +// convenience. A reservation of X that caps the unit at Y is arithmetic about a number nothing +// enforces: if Y > X the pool's total is a fiction, and if Y < X the pool refuses work the host +// could have run. So one figure serves both, and it is the instance's worker memory ceiling -- +// the existing safety limit, unchanged and now also the thing being counted. +// +// DECLARED FRONTIER: a per-work-class amount (a claims run does not need a release build's +// ceiling) needs per-class peak measurements, which the consumption reading this change starts +// recording is the first producer of. Until those exist every class reserves the ceiling, which +// over-reserves and never under-reserves. +fn compute_request_memory(instance: HostDashboardInstance) -> Kibibyte { + kibibyte_from_byte_size_floor(b: instance.dispatch_worker_memory_max) +} + +// A COMPUTE SEAT IS QUIESCENCE-REQUIRED AND THAT IS THE SAFE ARM, NOT A LIMITATION. A running cargo +// cannot be told its lease lapsed -- there is no fence a compute unit honours -- so a term that runs +// out frees nothing, and product.capacity.pool holds the encumbrance until the seat is RELEASED. +// A worker that dies wedges its own share rather than silently handing it to the next arrival, +// which is the only direction that cannot oversubscribe a machine. Recovering a wedged seat is an +// explicit release naming why, which is the same entry completion and cancellation take. +fn compute_lease_policy(term_seconds: Nat) -> LeasePolicy { + LeasePolicy { maximum_duration_seconds: term_seconds, release_law: QuiescenceRequired } +} + +fn compute_memory_root_pool(subject: ComputeCapacitySubject) -> Pool { + pool_of( + identity: subject.partition as String as NonEmptyStr, + ceiling: subject.appropriation, + replenishment: NoReplenishment, + release_law: QuiescenceRequired, + ) +} + +// ── THE RESERVATION ──────────────────────────────────────────────────────────────────────────── +// +// EVERY ARM IS TYPED AND NONE OF THEM WIDENS. Insufficient capacity is ComputeCapacityFull and says +// so; a machine whose live availability cannot back the request is ComputeHostBelowFloor and is a +// DIFFERENT fact with a different remedy (wait, or find out who else is on the host); an +// unobservable host reserves nothing at all rather than reserving against a guess. +type ComputeReservation + = ComputeReserved { grant: LeaseGrant, partition: PartitionId, amount: Kibibyte, store: FabricStorageBinding } + | ComputeCapacityFull { wire: String } + | ComputeHostBelowFloor { requested: Kibibyte, available: Kibibyte } + | ComputeReservationRefused { detail: String } + +data compute_seat_attempts: Nat = 3 +data compute_partition_read_budget: Nat = 4096 + +// THE APPROPRIATION IS CHECKED AGAINST THE MACHINE BEFORE ANY SEAT IS TAKEN. A host too small for +// the declared share is a configuration error that would otherwise present as an OOM kill during a +// build, which is the same information arriving hours later and destructively. +fn compute_reserve(instance: HostDashboardInstance, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { + compute_reserve_on(subject: compute_capacity_subject(instance: instance), reference: reference, term_seconds: term_seconds) +} + +fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { + match observe_host_memory() { + HostMemoryUnobserved { detail: d } => ComputeReservationRefused { detail: join(["the host's memory could not be observed, so nothing was reserved: ", d], "") } + HostMemoryObserved { total: total, available: available } => + if kibibyte_count(k: subject.appropriation) > kibibyte_count(k: total) { + ComputeReservationRefused { + detail: join([ + "the compute appropriation is ", to_string(value: kibibyte_count(k: subject.appropriation)), + " KiB and this host has ", to_string(value: kibibyte_count(k: total)), + " KiB installed, so the share was never backed by the machine", + ], ""), + } + } else if kibibyte_count(k: available) < kibibyte_count(k: subject.request) { + ComputeHostBelowFloor { requested: subject.request, available: available } + } else { + match compute_capacity_store_ensure(subject: subject) { + FabricStorageRootRefused { area: a, detail: d } => ComputeReservationRefused { detail: join(["the capacity ledger could not be prepared at ", a, ": ", d], "") } + FabricStorageRootReady => + match now_epoch_seconds() { + Absent => ComputeReservationRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds) + } + } + } + } +} + +fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: EpochSecs, term_seconds: Nat) -> ComputeReservation { + let store = compute_capacity_store(subject: subject) + match fabric_seat_acquire( + store: store, + partition: subject.partition, + root: compute_memory_root_pool(subject: subject), + actor: reference, + request: SeatRequest { reference: reference, amount: kibibyte_count(k: subject.request), at: now, term_seconds: term_seconds }, + policy: compute_lease_policy(term_seconds: term_seconds), + attempts: compute_seat_attempts, + budget: compute_partition_read_budget, + ) { + SeatGranted { grant: g, generation: _, attempts_used: _ } => ComputeReserved { grant: g, partition: subject.partition, amount: subject.request, store: store } + SeatFull { wire: w, attempts_used: n } => ComputeCapacityFull { wire: join([w, " (attempt ", to_string(value: n), ")"], "") } + other => ComputeReservationRefused { detail: seat_acquisition_wire(a: other) } + } +} + +// ── THE RELEASE, WHICH IS THE ONLY WAY CAPACITY COMES BACK ───────────────────────────────────── +// +// COMPLETION AND CANCELLATION ARE THE SAME TRANSITION WITH DIFFERENT REASONS, which is why the +// reason is carried rather than encoded in two functions: the pool does not care why a holder +// stopped, and a caller that had to choose between two spellings would eventually pick the wrong +// one on the path it exercises least -- which is always the cancellation path. +// +// A RELEASE THAT DID NOT LAND IS NOT A RELEASE. The append is retried against a moving head and +// exhaustion is its own arm; nothing here reports a return of capacity it did not observe. +type ComputeRelease + = ComputeReleased { partition: PartitionId, reference: NonEmptyStr, reason: NonEmptyStr } + | ComputeReleaseRefused { detail: String } + +fn compute_release(reservation: ComputeReservation, reason: NonEmptyStr) -> ComputeRelease { + match reservation { + ComputeCapacityFull { wire: w } => ComputeReleaseRefused { detail: join(["nothing to release: ", w], "") } + ComputeHostBelowFloor { requested: _, available: _ } => ComputeReleaseRefused { detail: "nothing to release: the host was below its live memory floor" } + ComputeReservationRefused { detail: d } => ComputeReleaseRefused { detail: join(["nothing to release: ", d], "") } + ComputeReserved { grant: g, partition: p, amount: _, store: store } => + match now_epoch_seconds() { + Absent => ComputeReleaseRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match fabric_pool_event_append( + store: store, partition: p, actor: g.reference, at: now, + payload: PoolReleased { reference: g.reference, reason: reason }, attempts: compute_seat_attempts, + ) { + PoolEventAppended { id: _ } => ComputeReleased { partition: p, reference: g.reference, reason: reason } + other => ComputeReleaseRefused { detail: pool_event_append_wire(a: other) } + } + } + } +} + +// ── WHAT THE HOST IS CARRYING, WITHOUT TAKING FROM IT ────────────────────────────────────────── +// +// THE AGGREGATE READING A RECEIPT CARRIES: what the ledger says is committed on this host, beside +// what the machine says is free. The two are independent measurements of one quantity and a +// receipt that carried only the first would be a ledger agreeing with itself (DESIGN 5: a +// measurement copied from the same tree is not an oracle). Their DISAGREEMENT is the signal -- +// committed far above the machine's own shortfall means the pool is over-reserving each class, and +// the reverse means work is running outside the pool. +type ComputeCapacityStanding + = ComputeCapacityStandingRead { committed: Kibibyte, ceiling: Kibibyte, headroom: Nat, observed_available: Kibibyte, observed_total: Kibibyte } + | ComputeCapacityStandingUnread { detail: String } + +fn compute_capacity_standing(instance: HostDashboardInstance) -> ComputeCapacityStanding { + compute_capacity_standing_on(subject: compute_capacity_subject(instance: instance)) +} + +fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapacityStanding { + match observe_host_memory() { + HostMemoryUnobserved { detail: d } => ComputeCapacityStandingUnread { detail: d } + HostMemoryObserved { total: total, available: available } => + match now_epoch_seconds() { + Absent => ComputeCapacityStandingUnread { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match fabric_seat_observe( + store: compute_capacity_store(subject: subject), + partition: subject.partition, + root: compute_memory_root_pool(subject: subject), + at: now, + budget: compute_partition_read_budget, + ) { + SeatRoomObserved { headroom: h, generation: _ } => + ComputeCapacityStandingRead { + committed: kibibyte(count: kibibyte_count(k: subject.appropriation) - h), + ceiling: subject.appropriation, + headroom: h, + observed_available: available, + observed_total: total, + } + SeatFullObserved { generation: _ } => + ComputeCapacityStandingRead { + committed: subject.appropriation, + ceiling: subject.appropriation, + headroom: 0, + observed_available: available, + observed_total: total, + } + SeatStandingUnobserved { step: s, reason: why } => ComputeCapacityStandingUnread { detail: join(["the capacity ledger could not be read at ", s, ": ", why], "") } + SeatStandingStoreRefused { cause: _ } => ComputeCapacityStandingUnread { detail: "the capacity ledger refused the read" } + } + } + } +} + +fn compute_reservation_wire(r: ComputeReservation) -> String { + match r { + ComputeReserved { grant: g, partition: p, amount: a, store: _ } => + join(["reserved ", to_string(value: kibibyte_count(k: a)), " KiB on ", p as String, " as ", g.reference as String, " fence=", g.fence.grant as String, " expires_at=", to_string(value: g.expires_at)], "") + ComputeCapacityFull { wire: w } => join(["capacity full: ", w], "") + ComputeHostBelowFloor { requested: rq, available: av } => + join(["host below floor: ", to_string(value: kibibyte_count(k: rq)), " KiB requested, ", to_string(value: kibibyte_count(k: av)), " KiB available"], "") + ComputeReservationRefused { detail: d } => join(["refused: ", d], "") + } +} + +fn compute_capacity_standing_wire(s: ComputeCapacityStanding) -> String { + match s { + ComputeCapacityStandingRead { committed: c, ceiling: cl, headroom: h, observed_available: av, observed_total: tt } => + join([ + "committed ", to_string(value: kibibyte_count(k: c)), " KiB of ", to_string(value: kibibyte_count(k: cl)), + " KiB (headroom ", to_string(value: h), " KiB); host reports ", to_string(value: kibibyte_count(k: av)), + " KiB available of ", to_string(value: kibibyte_count(k: tt)), " KiB installed", + ], "") + ComputeCapacityStandingUnread { detail: d } => join(["unread: ", d], "") + } +} diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index fb1eea1c1eb..5d357f8598b 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -20,30 +20,39 @@ import gunbc.roadmap_dispatch_actuator { dispatch_capability_program } import gunbc.roadmap_execution_contract { ExecutionCapability, GitWorkspaceCapability, CargoCapability, RustcCapability, AttemptStateDirectoryCapability, } +import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size } +import gunbc.compute.host_capacity { + ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, + ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, + ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, +} import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoProfile, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, DeclaredOutput, WorkOutputRoot, CheckoutRoot, TargetRoot, WorkOutput, WorkOutcome, WorkSucceeded, WorkFailed, WorkRefusedByInfrastructure, WorkCancelled, WorkOutputMismatch, - WorkInfrastructureRefusal, ProducerInFlight, HostAtColdBuildCap, CheckoutUnavailable, StoreUnavailable, + WorkInfrastructureRefusal, ProducerInFlight, HostComputeCapacityFull, HostBelowMemoryFloor, ComputeCapacityLedgerUnavailable, + CheckoutUnavailable, StoreUnavailable, ToolchainUnobserved, DependencyNotSucceeded, SubjectUnresolved, work_outcome_wire, work_outcome_ending, StoredOutcomeRead, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_decode, stored_outcome_is_success, } // THE LOCAL PROVIDER: one of N behind the exact work contract, serving the host it runs on. It is a // realization and knows nothing the contract does not: it resolves the subject, observes the -// toolchain, takes a lease, checks the commit out, runs the operation inside a transient user unit -// with a memory bound, returns declared outputs by content into a store, and writes the outcome -// beside them. A second caller with the same identity attaches to the stored outcome and runs +// toolchain, takes a producer lease, RESERVES THE MEMORY THE WORK WILL USE out of the host's +// compute pool, checks the commit out, runs the operation inside a transient user unit capped at +// exactly that reservation, returns declared outputs by content into a store, writes the outcome +// beside them, and releases the reservation on every ending. A second caller with the same identity attaches to the stored outcome and runs // nothing (compute-deduplication-and-admission's first slice: two callers, one build). Placement // across the fabric is the next slice; this provider's boundary is the host. -// HOW MANY COLD BUILDS THIS HOST CARRIES AT ONCE. A policy budget, not a measurement: each producer -// runs under the instance's worker memory ceiling, and two of them fit beside the serve and belt -// units on the smaller build host (srv2, 125 GiB) without the page-cache eviction that hung srv1 in -// August. The dedup row's red control is that a caller refused for capacity is told so and counted; -// it is, as HostAtColdBuildCap. -data compute_cold_build_cap: Int = 2 +// HOW LONG A COMPUTE RESERVATION DECLARES IT WILL HOLD. Two hours covers the cold release build +// this provider's heaviest class runs, and it is a DECLARATION rather than a deadline anything +// enforces: the pool's release law for a compute seat is quiescence-required +// (gunbc.compute.host_capacity), so a term that runs out frees nothing and the seat comes back only +// by an explicit release. Saying so here rather than leaving the reader to infer a reclaim that +// does not happen. +data compute_reservation_term_seconds: Int = 7200 type ComputeLayout { root: String @@ -134,10 +143,6 @@ fn compute_ensure_dir(instance: HostDashboardInstance, path: String) -> Bool { } } -fn compute_count_lines(text: String) -> Int { - count(filter(text.split(sep: "\n"), l => trim(s: l) != "")) -} - // What a provide returns: a fresh run's outcome, or an attachment to a stored one (no run). type ProvideResult = ProvideFresh { outcome: WorkOutcome, document: String } @@ -199,9 +204,11 @@ fn compute_unit_name(identity_hex: String) -> NonEmptyStr { join(["gunbc-compute-", identity_hex], "") as NonEmptyStr } -// Run the operation as a transient user unit and wait for it: the unit carries the memory ceiling, -// the checkout as working directory, the shared per-host target directory and the sccache wrapper. -fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, identity_hex: String, argv: List) -> ComputeExec { +// Run the operation as a transient user unit and wait for it: the unit carries THE MEMORY THE POOL +// GRANTED -- not a ceiling read independently from the instance, which would let the reservation and +// the enforcement drift apart -- the checkout as working directory, the shared per-host target +// directory and the sccache wrapper. +fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, identity_hex: String, argv: List, granted: Kibibyte) -> ComputeExec { let systemd_run = match instance.toolchain.systemd_run { Present { value: p } => p as String Absent => "" } if systemd_run == "" { ComputeExecFailed { exit_code: 127, stdout: "", stderr: "this instance's toolchain declares no systemd-run; the local provider runs only under systemd" } @@ -212,7 +219,7 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden args: systemd_run_user_wait_arguments( unit: compute_unit_name(identity_hex: identity_hex), properties: [ - systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: instance.dispatch_worker_memory_max))), + systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: kibibyte_to_byte_size(k: granted)))), systemd_run_property(name: "WorkingDirectory", value: layout.checkout), ], setenv_bindings: [ @@ -283,29 +290,101 @@ fn compute_provide_leaf( } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { - let leases = Filesystem.List(path: layout.leases_dir) - let in_flight = if leases.success { compute_count_lines(text: leases.entries) } else { 0 } - if in_flight >= compute_cold_build_cap { - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: HostAtColdBuildCap { in_flight: in_flight, cap: compute_cold_build_cap } }, subject: subject, op: op) + // THE PRODUCER LEASE AND THE CAPACITY RESERVATION ARE TWO FACTS AND ARE TAKEN IN THIS ORDER. + // The lease excludes a SECOND PRODUCER OF THIS IDENTITY and is a name, so O_EXCL settles it; the + // reservation is a QUANTITY on the host and no exclusive name can settle that. Taking the lease + // first means a request that then loses on capacity releases a name it holds rather than + // reserving memory it may never use, and it is the cheaper of the two to undo. + let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) + if !lease.success { + compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) } else { - let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) - if !lease.success { - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) - } else { - let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store) - let document = work_outcome_wire(o: outcome, subject: subject, op: op) - let written = Filesystem.Write(path: layout.outcome_path, content: document) - let released = Filesystem.Delete(path: layout.lease_path) - if written.success && released.success { - ProvideFresh { outcome: outcome, document: document } - } else { - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["outcome written: ", if written.success { "yes" } else { written.error }, "; lease released: ", if released.success { "yes" } else { released.error }], "") } }, subject: subject, op: op) + let reservation = compute_reserve(instance: instance, reference: identity as NonEmptyStr, term_seconds: compute_reservation_term_seconds) + match reservation { + ComputeCapacityFull { wire: w } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) + ComputeHostBelowFloor { requested: rq, available: av } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested_kib: kibibyte_count(k: rq), available_kib: kibibyte_count(k: av) }) + ComputeReservationRefused { detail: d } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) + ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => { + // THE WORK RUNS UNDER THE GRANT: the unit's MemoryMax is the amount the pool reserved, so + // the ledger's arithmetic is the enforcement rather than a parallel description of it. + let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) + // AGGREGATE CONSUMPTION IS READ WHILE THE GRANT IS STILL HELD, which is the only instant + // at which the ledger's committed total and the machine's own availability describe the + // same population; after the release the reservation is gone from one side and the + // memory may not yet be gone from the other. + let standing = compute_capacity_standing(instance: instance) + // COMPLETION AND CANCELLATION RELEASE THE SAME WAY. Every ending of compute_run_leased -- + // succeeded, failed, output-mismatch, refused by infrastructure -- reaches here, so there + // is no ending on which the seat stays held. + let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) + let document = work_outcome_wire(o: outcome, subject: subject, op: op) + let written = Filesystem.Write(path: layout.outcome_path, content: join([document, "\n", compute_capacity_standing_wire(s: standing), "\n"], "")) + let released = Filesystem.Delete(path: layout.lease_path) + if written.success && released.success && compute_release_landed(r: returned) { + ProvideFresh { outcome: outcome, document: document } + } else { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { + detail: join([ + "outcome written: ", if written.success { "yes" } else { written.error }, + "; lease released: ", if released.success { "yes" } else { released.error }, + "; reservation released: ", compute_release_wire(r: returned), + "; work ended ", work_outcome_ending(o: outcome), + ], ""), + }, + }, + subject: subject, op: op) + } } } } } } +// A REQUEST REFUSED FOR CAPACITY HOLDS NOTHING. The producer lease is dropped on the way out, so a +// host that is full does not accumulate names that would refuse the next attempt for the wrong +// reason -- a capacity refusal reported as a producer already in flight is the state conflation +// this provider's refusal vocabulary exists to prevent. +fn compute_refused_unreserved( + instance: HostDashboardInstance, + identity: String, + subject: WorkSubject, + op: WorkOperation, + layout: ComputeLayout, + cause: WorkInfrastructureRefusal, +) -> ProvideResult { + let dropped = Filesystem.Delete(path: layout.lease_path) + if dropped.success { + compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) + } else { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { detail: join(["a capacity refusal could not drop its producer lease at ", layout.lease_path, ": ", dropped.error], "") }, + }, + subject: subject, op: op) + } +} + +fn compute_release_landed(r: ComputeRelease) -> Bool { + match r { + ComputeReleased { partition: _, reference: _, reason: _ } => true + ComputeReleaseRefused { detail: _ } => false + } +} + +fn compute_release_wire(r: ComputeRelease) -> String { + match r { + ComputeReleased { partition: _, reference: ref_, reason: why } => join(["released ", ref_ as String, " (", why as String, ")"], "") + ComputeReleaseRefused { detail: d } => join(["NOT released: ", d], "") + } +} + fn compute_fresh(outcome: WorkOutcome, subject: WorkSubject, op: WorkOperation) -> ProvideResult { ProvideFresh { outcome: outcome, document: work_outcome_wire(o: outcome, subject: subject, op: op) } } @@ -317,6 +396,7 @@ fn compute_run_leased( identity: String, op: WorkOperation, dependency_store: String, + granted: Kibibyte, ) -> WorkOutcome { let existing = Filesystem.List(path: layout.checkout) let checkout = if existing.success { @@ -332,7 +412,7 @@ fn compute_run_leased( ComputeExecFailed { exit_code, stdout: _, stderr } => WorkRefusedByInfrastructure { identity: identity, cause: CheckoutUnavailable { detail: join(["git worktree add exited ", to_string(value: exit_code), ": ", stderr], "") } } ComputeExecOk { stdout: _, stderr: _ } => { - let run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store)) + let run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store), granted: granted) let log = Filesystem.Write(path: layout.log_path, content: match run { ComputeExecOk { stdout, stderr } => join([stdout, stderr], "") ComputeExecFailed { exit_code: _, stdout, stderr } => join([stdout, stderr], "") diff --git a/dag/gunbc/compute/work_request.dag b/dag/gunbc/compute/work_request.dag index 53af4e64651..62e1231c219 100644 --- a/dag/gunbc/compute/work_request.dag +++ b/dag/gunbc/compute/work_request.dag @@ -134,12 +134,22 @@ type WorkOutcome | WorkOutputMismatch { identity: String, detail: String } // Infrastructure refusals are typed and counted, never a widen: a producer already holds the lease -// (the caller attaches later, it does not start a second build), the host is at its cold-build cap, -// the checkout or the store could not be prepared, or a dependency of this operation (the build the -// compile consumes) did not succeed. +// (the caller attaches later, it does not start a second build), the host cannot back the memory +// this class reserves, the checkout or the store could not be prepared, or a dependency of this +// operation (the build the compile consumes) did not succeed. +// +// THREE CAPACITY ARMS WHERE THERE USED TO BE ONE COUNT. HostAtColdBuildCap { in_flight, cap } is +// DELETED, and its disposition is that the fact it reported -- this host is already carrying as +// much as it may -- is now HostComputeCapacityFull, measured in the memory the pool actually +// appropriates rather than in producers. The other two are facts the count could not express at +// all and that a caller must not confuse with it: the machine's LIVE availability being below what +// this class needs is somebody else's memory, not this pool's commitments, and an unreadable +// ledger is a store repair. One arm for three remedies is the conflation DESIGN 5 forbids. type WorkInfrastructureRefusal = ProducerInFlight { lease_path: String } - | HostAtColdBuildCap { in_flight: Int, cap: Int } + | HostComputeCapacityFull { wire: String } + | HostBelowMemoryFloor { requested_kib: Int, available_kib: Int } + | ComputeCapacityLedgerUnavailable { detail: String } | CheckoutUnavailable { detail: String } | StoreUnavailable { detail: String } | ToolchainUnobserved { detail: String } @@ -169,7 +179,9 @@ fn work_outcome_ending(o: WorkOutcome) -> String { fn work_refusal_detail(r: WorkInfrastructureRefusal) -> String { match r { ProducerInFlight { lease_path } => join(["a producer already holds the lease at ", lease_path, "; attach later, do not start a second one"], "") - HostAtColdBuildCap { in_flight, cap } => join([to_string(value: in_flight), " producers in flight on this host, cap ", to_string(value: cap)], "") + HostComputeCapacityFull { wire } => join(["this host's compute memory pool has no room for another reservation: ", wire], "") + HostBelowMemoryFloor { requested_kib, available_kib } => join(["this class reserves ", to_string(value: requested_kib), " KiB and the host reports only ", to_string(value: available_kib), " KiB available, so the reservation would not have been backed by the machine"], "") + ComputeCapacityLedgerUnavailable { detail } => join(["the host's compute capacity ledger could not be transacted: ", detail], "") CheckoutUnavailable { detail } => join(["checkout unavailable: ", detail], "") StoreUnavailable { detail } => join(["store unavailable: ", detail], "") ToolchainUnobserved { detail } => join(["toolchain unobserved: ", detail], "") diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index 5225efab78a..ec7ee6ef88f 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -249,7 +249,7 @@ type SeatStandingObservation | SeatStandingUnobserved { step: String, reason: String } | SeatStandingStoreRefused { cause: EventLogRefusal } -fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root: Pool, at: EpochSecs, budget: Nat) -> SeatStandingObservation { +fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root: Pool, at: EpochSecs, budget: Nat) -> SeatStandingObservation { match event_log_read_partition(store: store, partition: partition, budget: budget) { PartitionReadRefused { cause: c } => SeatStandingStoreRefused { cause: c } PartitionReadOk { head: _, walk: walk } => @@ -275,7 +275,7 @@ fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root } } -fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, budget: Nat, attempt: Nat) -> SeatAcquisition? { +fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, budget: Nat, attempt: Nat) -> SeatAcquisition? { match event_log_read_partition(store: store, partition: partition, budget: budget) { PartitionReadRefused { cause: c } => Present { value: SeatStoreRefused { cause: c, attempts_used: attempt } } PartitionReadOk { head: head, walk: walk } => @@ -313,7 +313,7 @@ fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool< // and the drift is invisible because each side is internally consistent. A mismatch is refused before // anything is appended rather than resolved in favour of either side: neither is authoritative over // the other, and picking one silently would be the same fail-open in a different costume. -fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, attempts: Nat, budget: Nat) -> SeatAcquisition { +fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, attempts: Nat, budget: Nat) -> SeatAcquisition { if !release_law_eq(left: root.release_law, right: policy.release_law) { SeatAcquireRefused { step: "release-law", @@ -337,6 +337,71 @@ fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root } } +// APPENDING A POOL EVENT THAT IS NOT AN ACQUISITION -- a settlement or a release -- against the +// partition's current head, re-observing and retrying while the head moves under it. +// +// WHY IT IS HERE AND NOT AT EACH CALLER. Acquisition already lived here because it has to fold the +// pool to decide; settlement and release decide nothing, so each caller wrote its own +// observe/append/retry loop and gunbc.fabric_quota had the only copy. A second consumer +// (gunbc.compute.host_capacity, releasing a compute reservation) would have made it two spellings +// of one transaction, free to disagree about how many times a contended head is retried -- so the +// loop moves to the carrier that owns partition appends and both consumers call it. +// +// A RELEASE THAT LOSES ITS RACE IS RETRIED, NEVER ASSUMED. Nothing here reports a success it did +// not observe: exhausting the attempts is its own arm, and a caller holding it still holds the +// reservation. +type PoolEventAppend + = PoolEventAppended { id: EventId } + | PoolEventContended { attempts: Nat } + | PoolEventAppendRefused { detail: String } + +type PoolEventAttemptState { + done: PoolEventAppend? +} + +fn pool_event_append_attempt(store: FabricStorageBinding, partition: PartitionId, actor: NonEmptyStr, at: EpochSecs, payload: PoolEvent) -> PoolEventAppend? { + match event_log_observe_head(store: store, partition: partition) { + HeadObservationRefused { cause: c } => Present { value: PoolEventAppendRefused { detail: event_log_refusal_wire(cause: c) } } + HeadObserved { head: head } => + match event_log_append( + store: store, + partition: partition, + event: ChainEvent { + partition: partition, + parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, + recorded_at: at, + actor: actor, + payload: payload, + }, + expected: head, + ) { + EventAppended { id: h } => Present { value: PoolEventAppended { id: h } } + EventAppendStale { expected: _, observed: _ } => none + EventAppendRefused { cause: c } => Present { value: PoolEventAppendRefused { detail: event_log_refusal_wire(cause: c) } } + } + } +} + +fn fabric_pool_event_append(store: FabricStorageBinding, partition: PartitionId, actor: NonEmptyStr, at: EpochSecs, payload: PoolEvent, attempts: Nat) -> PoolEventAppend { + let st = fold(nat_range_inclusive(lo: 1, hi: attempts), init: PoolEventAttemptState { done: none }, f: (acc, _i) => + match acc.done { + Present { value: _ } => acc + Absent => PoolEventAttemptState { done: pool_event_append_attempt(store: store, partition: partition, actor: actor, at: at, payload: payload) } + }) + match st.done { + Present { value: d } => d + Absent => PoolEventContended { attempts: attempts } + } +} + +fn pool_event_append_wire(a: PoolEventAppend) -> String { + match a { + PoolEventAppended { id: h } => join(["appended ", h as String], "") + PoolEventContended { attempts: n } => join(["contended after ", to_string(n), " attempts"], "") + PoolEventAppendRefused { detail: d } => join(["refused ", d], "") + } +} + fn seat_acquisition_wire(a: SeatAcquisition) -> String { match a { SeatGranted { grant: g, generation: gen, attempts_used: n } => join(["granted ", g.reference as String, " fence=", g.fence.grant as String, "@", to_string(gen), " expires_at=", to_string(g.expires_at), " attempt=", to_string(n)], "") diff --git a/dag/gunbc/fabric/fabric_quota.dag b/dag/gunbc/fabric/fabric_quota.dag index 5fa0f3429a8..f6c87e0cb75 100644 --- a/dag/gunbc/fabric/fabric_quota.dag +++ b/dag/gunbc/fabric/fabric_quota.dag @@ -1,20 +1,17 @@ module gunbc.fabric_quota import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } -import std.nat { Nat, nat_range_inclusive } -import std.measure { Measure, Count, One } +import std.nat { Nat } import extdeps.api_rate_limit { UpstreamRateLimit, upstream_rate_limit_key } import product.capacity.quota { quota_partition, quota_root_pool } -import product.capacity.event_chain { PartitionId, HeadExpectation, HeadAbsent, HeadAt, ChainEvent } -import product.capacity.pool_events { PoolEvent, PoolSettled, SeatRequest } +import product.capacity.event_chain { PartitionId } +import product.capacity.pool_events { PoolSettled, SeatRequest } import product.capacity.lease { LeasePolicy, LeaseGrant, FencedResource } import gunbc.fabric_event_log_host { HostEventLogStore, HostStoreResolved, HostStoreRefused, event_log_store_for_host, now_epoch_seconds } import gunbc.fabric_storage_client { FabricStorageBinding } import gunbc.fabric_event_log { - event_log_refusal_wire, - HeadObservation, HeadObserved, HeadObservationRefused, event_log_observe_head, - EventAppend, EventAppended, EventAppendStale, EventAppendRefused, event_log_append, SeatAcquisition, SeatGranted, SeatFull, SeatContended, SeatAcquireRefused, fabric_seat_acquire, seat_acquisition_wire, + PoolEventAppend, PoolEventAppended, PoolEventContended, PoolEventAppendRefused, fabric_pool_event_append, pool_event_append_wire, } // A CALLER LEASES UPSTREAM QUOTA BEFORE IT SPENDS IT. The lease holds the calls it is about to @@ -61,46 +58,26 @@ type QuotaSettle = QuotaSettled { partition: PartitionId, actual: Nat } | QuotaSettleRefused { detail: String } -type SettleFold { - done: QuotaSettle? -} - -fn settle_attempt(lease: QuotaLease, actual: Nat, now: EpochSecs) -> QuotaSettle? { +// SETTLEMENT IS ONE PARTITION APPEND, and the observe/append/retry loop it needs is the carrier's +// (gunbc.fabric_event_log fabric_pool_event_append), not a second copy here. It used to be a copy: +// this module held the only one until a compute reservation needed to release, at which point the +// loop would have existed twice and been free to disagree with itself about a contended head. +fn fabric_quota_settle(lease: QuotaLease, actual: Nat) -> QuotaSettle { match lease { - QuotaFull { wire: w } => Present { value: QuotaSettleRefused { detail: join(["nothing to settle: ", w], "") } } - QuotaLeaseRefused { detail: d } => Present { value: QuotaSettleRefused { detail: join(["nothing to settle: ", d], "") } } + QuotaFull { wire: w } => QuotaSettleRefused { detail: join(["nothing to settle: ", w], "") } + QuotaLeaseRefused { detail: d } => QuotaSettleRefused { detail: join(["nothing to settle: ", d], "") } QuotaLeased { grant: g, partition: p, store: store } => - match event_log_observe_head(store: store, partition: p) { - HeadObservationRefused { cause: c } => Present { value: QuotaSettleRefused { detail: event_log_refusal_wire(cause: c) } } - HeadObserved { head: head } => { - let event = ChainEvent { - partition: p, - parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, - recorded_at: now, - actor: g.reference, - payload: PoolSettled { reference: g.reference, actual: actual }, - } - match event_log_append(store: store, partition: p, event: event, expected: head) { - EventAppended { id: _ } => Present { value: QuotaSettled { partition: p, actual: actual } } - EventAppendStale { expected: _, observed: _ } => none - EventAppendRefused { cause: c } => Present { value: QuotaSettleRefused { detail: event_log_refusal_wire(cause: c) } } + match now_epoch_seconds() { + Absent => QuotaSettleRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match fabric_pool_event_append( + store: store, partition: p, actor: g.reference, at: now, + payload: PoolSettled { reference: g.reference, actual: actual }, attempts: 3, + ) { + PoolEventAppended { id: _ } => QuotaSettled { partition: p, actual: actual } + other => QuotaSettleRefused { detail: pool_event_append_wire(a: other) } } - } - } - } -} - -fn fabric_quota_settle(lease: QuotaLease, actual: Nat) -> QuotaSettle { - match now_epoch_seconds() { - Absent => QuotaSettleRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => { - let st = fold(nat_range_inclusive(lo: 1, hi: 3), init: SettleFold { done: none }, f: (acc, _i) => - match acc.done { Present { value: _ } => acc Absent => SettleFold { done: settle_attempt(lease: lease, actual: actual, now: now) } }) - match st.done { - Present { value: r } => r - Absent => QuotaSettleRefused { detail: "settlement contended three times" } } - } } } diff --git a/dag/gunbc/harness/harness_guidance.dag b/dag/gunbc/harness/harness_guidance.dag index 563571b4166..599e907dabb 100644 --- a/dag/gunbc/harness/harness_guidance.dag +++ b/dag/gunbc/harness/harness_guidance.dag @@ -157,7 +157,7 @@ fn harness_worker_guidance_at(worktree: String, test_command: String, test_recei grounded(premise: TestProcessIs { command: test_command, receipt_path: test_receipt_path }, module_path: "gunbc.compute.test_run", decl_name: "test_pattern_cli"), grounded(premise: CapabilityWithheld { capability: BuildOrTestOutsideTheProcess }, - module_path: "gunbc.compute.work_provider_local", decl_name: "compute_cold_build_cap"), + module_path: "gunbc.compute.host_capacity", decl_name: "compute_reserve"), ]) } diff --git a/dag/std/measure.dag b/dag/std/measure.dag index e922d41da36..dad1d7a99f4 100644 --- a/dag/std/measure.dag +++ b/dag/std/measure.dag @@ -328,6 +328,16 @@ fn kibibyte_to_byte_size(k: Kibibyte) -> ByteSize { byte_size(count: kibibyte_count(k) * kibi_factor()) } +// THE INVERSE OF kibibyte_to_byte_size, FLOORING. A byte count that is not a whole number of +// kibibytes has no exact kibibyte reading, and the floor is the only rounding a CAPACITY may take: +// rounding up would report a ceiling the host does not have, and a reservation admitted against it +// would be admitted against memory that is not there. Named _floor for the same reason +// measure_scale_fraction_floor is -- the direction is part of what the caller is asking for, not an +// implementation detail it may discover later. +fn kibibyte_from_byte_size_floor(b: ByteSize) -> Kibibyte { + kibibyte(count: byte_size_count(b: b) / kibi_factor()) +} + fn mebibyte_to_byte_size(m: Mebibyte) -> ByteSize { byte_size(count: mebibyte_count(m) * mebibyte_scale_factor_bytes()) } diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag new file mode 100644 index 00000000000..235a64b7828 --- /dev/null +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -0,0 +1,156 @@ +module test.claim.compute.host_capacity_wet_witness + +import extdeps.shell +import std.types { Bool, Int, NonEmptyStr, String } +import std.measure { Kibibyte, kibibyte, kibibyte_count } +import std.algebra { trim } +import product.capacity.event_chain { PartitionId } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, dashboard_instance_compute_root } +import gunbc.compute.host_capacity { + ComputeCapacitySubject, compute_capacity_subject, compute_capacity_root, compute_memory_partition, + HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, + ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, + compute_reserve_on, compute_release, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, +} + +// WET CONTROLS FOR THE HOST'S COMPUTE MEMORY POOL. Every reservation below is a real append to a +// real event log on a real temporary directory, decided against a real reading of this machine's +// /proc/meminfo. Nothing here is a fixture standing in for the store: the linearization being +// asserted is the one gunbc.compute.work_provider_local runs. +// +// THE SUBJECT IS SUPPLIED AND THE JOIN THAT PRODUCES ONE IS ASSERTED SEPARATELY (DESIGN 3, the +// pairing obligation): a ComputeCapacitySubject is four values, and deriving it here would mean +// standing up a whole deployment record to exercise arithmetic that does not depend on one. The +// inhabitance claim is the_deployment_join_roots_the_ledger_under_the_instance_compute_root below, +// which runs the real producer. + +data one_gibibyte_in_kibibytes: Int = 1048576 + +fn fresh_root() -> String { + shell.Mktemp.DirWithTemplate(template: "/tmp/gunbc_compute_capacity.XXXXXX").path +} + +fn subject_at(root: String, appropriation_kib: Int, request_kib: Int) -> ComputeCapacitySubject { + ComputeCapacitySubject { + root: join([trim(s: root), "/capacity"], "") as NonEmptyStr, + partition: "compute-memory-wet-witness" as PartitionId, + appropriation: kibibyte(count: appropriation_kib), + request: kibibyte(count: request_kib), + } +} + +fn reserved_amount(r: ComputeReservation) -> Int { + match r { + ComputeReserved { grant: _, partition: _, amount: a, store: _ } => kibibyte_count(k: a) + ComputeCapacityFull { wire: _ } => 0 + ComputeHostBelowFloor { requested: _, available: _ } => 0 + ComputeReservationRefused { detail: _ } => 0 + } +} + +fn is_capacity_full(r: ComputeReservation) -> Bool { + match r { + ComputeCapacityFull { wire: w } => string_contains(s: w, pattern: "pool-full") + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeHostBelowFloor { requested: _, available: _ } => false + ComputeReservationRefused { detail: _ } => false + } +} + +fn released(r: ComputeRelease) -> Bool { + match r { + ComputeReleased { partition: _, reference: _, reason: _ } => true + ComputeReleaseRefused { detail: _ } => false + } +} + +fn standing_committed(s: ComputeCapacityStanding) -> Int { + match s { + ComputeCapacityStandingRead { committed: c, ceiling: _, headroom: _, observed_available: _, observed_total: _ } => kibibyte_count(k: c) + ComputeCapacityStandingUnread { detail: _ } => 0 - 1 + } +} + +// THE CONTROL THIS WHOLE CHANGE EXISTS FOR, AND IT IS RED UNDER THE THING IT REPLACED. A pool of two +// gibibytes admits two one-gibibyte reservations and REFUSES THE THIRD; the deleted admission -- +// count the lease directory, compare, then create -- admitted the third whenever the two lease +// files belonged to different identities, because a count of names excludes no quantity. Then the +// release returns exactly what it held and the next request is admitted again, which is the other +// half: capacity that never comes back is a wall, not a pool. +// +// MUTATION CONTROL: raise the appropriation above the sum of the three requests and the third +// reservation is admitted, so this cell goes red for the reason it names rather than for any +// property of the store. +test fn competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity() -> Bool { + let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes * 2, request_kib: one_gibibyte_in_kibibytes) + let first = compute_reserve_on(subject: subject, reference: "wet-first" as NonEmptyStr, term_seconds: 60) + let second = compute_reserve_on(subject: subject, reference: "wet-second" as NonEmptyStr, term_seconds: 60) + let full = compute_reserve_on(subject: subject, reference: "wet-third" as NonEmptyStr, term_seconds: 60) + let at_capacity = compute_capacity_standing_on(subject: subject) + let back = compute_release(reservation: first, reason: "witness completion" as NonEmptyStr) + let after_release = compute_capacity_standing_on(subject: subject) + let readmitted = compute_reserve_on(subject: subject, reference: "wet-fourth" as NonEmptyStr, term_seconds: 60) + reserved_amount(r: first) == one_gibibyte_in_kibibytes + && reserved_amount(r: second) == one_gibibyte_in_kibibytes + && is_capacity_full(r: full) + && standing_committed(s: at_capacity) == one_gibibyte_in_kibibytes * 2 + && released(r: back) + && standing_committed(s: after_release) == one_gibibyte_in_kibibytes + && reserved_amount(r: readmitted) == one_gibibyte_in_kibibytes +} + +// A RESERVATION THE MACHINE CANNOT BACK IS REFUSED BEFORE THE LEDGER IS TOUCHED, and it is a +// DIFFERENT refusal from a full pool: the pool's own commitments are irrelevant here (this ledger is +// empty), and the remedy is to find out what else is on the host rather than to wait for a seat. +// The request is the machine's own installed memory, which is strictly above what it reports +// available on any host that is running anything at all -- including the one running this witness. +test fn a_request_the_machine_cannot_back_refuses_below_the_live_floor() -> Bool { + match observe_host_memory() { + HostMemoryUnobserved { detail: _ } => false + HostMemoryObserved { total: total, available: available } => { + let installed = kibibyte_count(k: total) + let subject = subject_at(root: fresh_root(), appropriation_kib: installed, request_kib: installed) + kibibyte_count(k: available) < installed + && (match compute_reserve_on(subject: subject, reference: "wet-floor" as NonEmptyStr, term_seconds: 60) { + ComputeHostBelowFloor { requested: rq, available: av } => kibibyte_count(k: rq) == installed && kibibyte_count(k: av) > 0 + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeReservationRefused { detail: _ } => false + }) + } + } +} + +// AN APPROPRIATION LARGER THAN THE MACHINE IS A CONFIGURATION ERROR AND SAYS SO, rather than +// presenting hours later as a build the kernel killed. It is refused ahead of the floor check, +// because a share that was never backed is wrong whatever this instant's availability happens to be. +test fn an_appropriation_larger_than_the_machine_refuses_rather_than_admitting() -> Bool { + match observe_host_memory() { + HostMemoryUnobserved { detail: _ } => false + HostMemoryObserved { total: total, available: _ } => { + let subject = subject_at(root: fresh_root(), appropriation_kib: kibibyte_count(k: total) + 1, request_kib: one_gibibyte_in_kibibytes) + match compute_reserve_on(subject: subject, reference: "wet-oversized" as NonEmptyStr, term_seconds: 60) { + ComputeReservationRefused { detail: d } => string_contains(s: d, pattern: "never backed by the machine") + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeHostBelowFloor { requested: _, available: _ } => false + } + } + } +} + +// THE INHABITANCE CLAIM FOR EVERY SUPPLIED SUBJECT ABOVE (DESIGN 3). The real producer is the +// deployment join, and what it must establish is the ROUTE and not only the shape: the ledger a +// live instance reserves against is rooted under THAT INSTANCE'S compute root, and its partition +// names that instance, so two instances on one machine cannot silently transact on one pool. +test fn the_deployment_join_roots_the_ledger_under_the_instance_compute_root() -> Bool { + let instance = srv1_live_dashboard_instance() + let subject = compute_capacity_subject(instance: instance) + let compute_root = dashboard_instance_compute_root(instance: instance) as String + starts_with(s: subject.root as String, prefix: compute_root) + && (subject.root as String) == join([compute_root, "/capacity"], "") + && string_contains(s: subject.partition as String, pattern: instance.instance_id as String) + && kibibyte_count(k: subject.appropriation) == kibibyte_count(k: subject.request) * 2 + && kibibyte_count(k: subject.request) > 0 +} diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 6d57f7a0a71..385963a5879 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -6,7 +6,7 @@ import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, DeclaredOutput, WorkOutputRoot, CheckoutRoot, TargetRoot, - WorkOutput, WorkOutcome, WorkSucceeded, WorkFailed, WorkRefusedByInfrastructure, HostAtColdBuildCap, + WorkOutput, WorkOutcome, WorkSucceeded, WorkFailed, WorkRefusedByInfrastructure, HostComputeCapacityFull, work_outcome_wire, stored_outcome_decode, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_is_success, } @@ -64,12 +64,12 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool let subject = tree(hex: tree_a) let ok = work_outcome_wire(o: WorkSucceeded { identity: "abc", outputs: [WorkOutput { declared: "target/release/gunbc", blob: "deadbeef", store_path: "/s/gunbc" }], log_path: "/l" }, subject: subject, op: work_default_build()) let failed = work_outcome_wire(o: WorkFailed { identity: "abc", exit_code: 101, log_path: "/l" }, subject: subject, op: work_default_build()) - let refused = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: "abc", cause: HostAtColdBuildCap { in_flight: 2, cap: 2 } }, subject: subject, op: work_default_build()) + let refused = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: "abc", cause: HostComputeCapacityFull { wire: "pool-full (attempt 1)" } }, subject: subject, op: work_default_build()) (match stored_outcome_decode(text: ok) { StoredOutcomeFound { ending, identity, document: _ } => ending == "succeeded" && identity == "abc" StoredOutcomeUnreadable { reason: _ } => false }) && stored_outcome_is_success(r: stored_outcome_decode(text: ok)) && !stored_outcome_is_success(r: stored_outcome_decode(text: failed)) && (match stored_outcome_decode(text: refused) { StoredOutcomeFound { ending, identity: _, document: _ } => ending == "refused_by_infrastructure" StoredOutcomeUnreadable { reason: _ } => false }) - && string_contains(s: refused, pattern: "cap 2") + && string_contains(s: refused, pattern: "pool-full") && string_contains(s: ok, pattern: "\"blob\": \"deadbeef\"") && (match stored_outcome_decode(text: "{\"identity\": \"abc\"}") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) && (match stored_outcome_decode(text: "not json") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) diff --git a/dag/test/claim/os_proc_meminfo_witness_test.dag b/dag/test/claim/os_proc_meminfo_witness_test.dag index eb6c7b69ebb..b2dc1a40705 100644 --- a/dag/test/claim/os_proc_meminfo_witness_test.dag +++ b/dag/test/claim/os_proc_meminfo_witness_test.dag @@ -24,3 +24,34 @@ test fn proc_meminfo_witnesses() -> Bool { && measure_count(m: info.mem_available) <= measure_count(m: info.mem_total) && measure_count(m: metric.value) > 0 } + +// THE PARSE, AND THE TWO THINGS IT MUST NOT DO. A field the kernel did not publish may not read as +// a zero (a host with no MemAvailable line would otherwise present as a host with no memory +// available, and an admission would refuse forever for the wrong reason), and a line whose value is +// not a decimal count may not contribute a metric. Both are asserted against the same text with one +// line changed, so the assertion discriminates the parse rather than the fixture. +test fn parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one() -> Bool { + let text = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: 8388608 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nHugePages_Total: 0\n" + let without_available = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" + let unparseable = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: not-a-number kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" + (match parse_proc_meminfo(text: text) { + MeminfoParsed { meminfo: m } => + kibibyte_count(k: m.mem_total) == 16777216 + && kibibyte_count(k: m.mem_available) == 8388608 + && kibibyte_count(k: m.swap_total) == 0 + && count(m.all_metrics) == 8 + MeminfoFieldAbsent { field: _ } => false + }) + && (match parse_proc_meminfo(text: without_available) { + MeminfoFieldAbsent { field: f } => (f as String) == "MemAvailable" + MeminfoParsed { meminfo: _ } => false + }) + && (match parse_proc_meminfo(text: unparseable) { + MeminfoFieldAbsent { field: f } => (f as String) == "MemAvailable" + MeminfoParsed { meminfo: _ } => false + }) + && (match parse_proc_meminfo(text: "") { + MeminfoFieldAbsent { field: f } => (f as String) == "MemTotal" + MeminfoParsed { meminfo: _ } => false + }) +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 8357ffbf07e..60626c07334 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -244,6 +244,30 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // probe failed without output rather than running its payload. fn local_repo_wet_schedule() -> List { [ + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_request_the_machine_cannot_back_refuses_below_the_live_floor" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "a_request_the_machine_cannot_back_refuses_below_the_live_floor", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "an_appropriation_larger_than_the_machine_refuses_rather_than_admitting" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "an_appropriation_larger_than_the_machine_refuses_rather_than_admitting", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "the_deployment_join_roots_the_ledger_under_the_instance_compute_root" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "the_deployment_join_roots_the_ledger_under_the_instance_compute_root", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.effect_plan_bash_materialize_real_execution_witness", function: "effect_plan_bash_two_declared_operations_execute" }, entry: "dag/test/claim/effect_plan_bash_materialize_real_execution_witness_test.dag", From 9bf00d7a3b4c0cd8f8290ead44805f2f0ef75090 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 14:44:23 +0000 Subject: [PATCH 02/28] The meminfo parse resolves each field by the lookup that proves it present review 69561 on #11962. The presence pass walked meminfo_field_names and refused the first missing name; the record was then built from a SECOND spelling of those names, each falling back to kibibyte(count: 0) on an absence the presence pass had supposedly ruled out. Two lists for one concept is DESIGN 3's nicknaming, and the fork was load-bearing: the zero's unreachability was an invariant held AT A DISTANCE between roster and record, so renaming or dropping a roster entry made mem_total and mem_available read 0 KiB -- which compute_reserve_on turns into a permanent below-floor or appropriation-exceeds-total refusal ATTRIBUTED TO THE MACHINE rather than to the parse. A fabricated plausible output behind a guard that looked like a wall (DESIGN 5). The guard is deleted rather than strengthened. Each field is matched once, its absence names itself, and its value is the one that match just bound: meminfo_required and its zero are gone, so the fabricated reading has no constructor left (DESIGN 4b). meminfo_field_names no longer claims to be this fold's authority -- what the parse demands is exactly what ProcMeminfo has fields for, a fact the record carries and cannot drift from itself. The witness gains the control the absence arms cannot supply: every field carries a DISTINCT value and every field is asserted, so a name paired with the wrong field is red; and a field missing from the MIDDLE of the file names itself. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/linux/proc_meminfo.dag | 88 +++++++++++++------ .../claim/os_proc_meminfo_witness_test.dag | 21 ++++- 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/dag/extdeps/linux/proc_meminfo.dag b/dag/extdeps/linux/proc_meminfo.dag index 010a19d0381..abca8818e29 100644 --- a/dag/extdeps/linux/proc_meminfo.dag +++ b/dag/extdeps/linux/proc_meminfo.dag @@ -91,33 +91,71 @@ fn meminfo_metric_named(metrics: List, name: NonEmptyStr) -> Kibib } } -// THE SEVEN NAMED FIELDS ARE REQUIRED TOGETHER, in the order meminfo_field_names declares them, so -// the roster above is the authority for what this parse demands rather than a second list here. +// EVERY NAMED FIELD IS RESOLVED BY THE SAME LOOKUP THAT PROVES IT PRESENT, and that is the whole +// shape of this fold rather than a stylistic preference. +// +// IT WAS WRITTEN THE OTHER WAY FIRST AND THE OTHER WAY WAS WRONG (review 69561 on gunbc#11962). A +// presence pass walked meminfo_field_names and refused the first name it could not find; the record +// was then built from a SECOND spelling of those names, each one falling back to +// `kibibyte(count: 0)` on an absence the presence pass had supposedly ruled out. Two lists for one +// concept is DESIGN 3's nicknaming, and here the fork was load-bearing rather than cosmetic: the +// zero's unreachability was an invariant maintained AT A DISTANCE between the roster and the record, +// so renaming or dropping a roster entry made mem_total and mem_available read 0 KiB -- which a +// compute admission (gunbc.compute.host_capacity compute_reserve_on) turns into a permanent +// below-floor or appropriation-exceeds-total refusal ATTRIBUTED TO THE MACHINE rather than to the +// parse. A fabricated plausible output (DESIGN 5) behind a guard that looked like a wall. +// +// So the guard is deleted rather than strengthened. Each field is matched once, its absence names +// ITSELF, and its value is the one the match just bound: `meminfo_required` and its zero are gone, +// and the fabricated reading has no constructor left to be written in (DESIGN 4b, structurally +// impossible rather than validated). meminfo_field_names is consequently NOT this fold's authority +// and no longer claims to be -- it is the roster of fields this module models, read by consumers +// that want the names; what this parse demands is exactly what ProcMeminfo has fields for, which is +// a fact the record already carries and cannot drift from itself. +// +// The nesting is flat in meaning and deep in shape on purpose: there is no total lookup over seven +// independent optionals that does not either lose which one was missing or introduce a positional +// list, and a positional list would be the same fork one representation over. fn parse_proc_meminfo(text: String) -> MeminfoRead { let metrics = meminfo_metrics(text: text) - match first(filter(meminfo_field_names, f => match meminfo_metric_named(metrics: metrics, name: f) { Absent => true Present { value: _ } => false })) { - Present { value: missing } => MeminfoFieldAbsent { field: missing } - Absent => - MeminfoParsed { - meminfo: ProcMeminfo { - mem_total: meminfo_required(metrics: metrics, name: "MemTotal" as NonEmptyStr), - mem_free: meminfo_required(metrics: metrics, name: "MemFree" as NonEmptyStr), - mem_available: meminfo_required(metrics: metrics, name: "MemAvailable" as NonEmptyStr), - buffers: meminfo_required(metrics: metrics, name: "Buffers" as NonEmptyStr), - cached: meminfo_required(metrics: metrics, name: "Cached" as NonEmptyStr), - swap_total: meminfo_required(metrics: metrics, name: "SwapTotal" as NonEmptyStr), - swap_free: meminfo_required(metrics: metrics, name: "SwapFree" as NonEmptyStr), - all_metrics: metrics, - }, + match meminfo_metric_named(metrics: metrics, name: "MemTotal" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "MemTotal" as NonEmptyStr } + Present { value: mem_total } => + match meminfo_metric_named(metrics: metrics, name: "MemFree" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "MemFree" as NonEmptyStr } + Present { value: mem_free } => + match meminfo_metric_named(metrics: metrics, name: "MemAvailable" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "MemAvailable" as NonEmptyStr } + Present { value: mem_available } => + match meminfo_metric_named(metrics: metrics, name: "Buffers" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "Buffers" as NonEmptyStr } + Present { value: buffers } => + match meminfo_metric_named(metrics: metrics, name: "Cached" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "Cached" as NonEmptyStr } + Present { value: cached } => + match meminfo_metric_named(metrics: metrics, name: "SwapTotal" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "SwapTotal" as NonEmptyStr } + Present { value: swap_total } => + match meminfo_metric_named(metrics: metrics, name: "SwapFree" as NonEmptyStr) { + Absent => MeminfoFieldAbsent { field: "SwapFree" as NonEmptyStr } + Present { value: swap_free } => + MeminfoParsed { + meminfo: ProcMeminfo { + mem_total: mem_total, + mem_free: mem_free, + mem_available: mem_available, + buffers: buffers, + cached: cached, + swap_total: swap_total, + swap_free: swap_free, + all_metrics: metrics, + }, + } + } + } + } + } + } } } } - -// Reached only under the absence check above, which is why it can answer at all; the zero is the -// unreachable arm of an optional, not a fabricated reading. -fn meminfo_required(metrics: List, name: NonEmptyStr) -> Kibibyte { - match meminfo_metric_named(metrics: metrics, name: name) { - Present { value: k } => k - Absent => kibibyte(count: 0) - } -} diff --git a/dag/test/claim/os_proc_meminfo_witness_test.dag b/dag/test/claim/os_proc_meminfo_witness_test.dag index b2dc1a40705..debf04c5405 100644 --- a/dag/test/claim/os_proc_meminfo_witness_test.dag +++ b/dag/test/claim/os_proc_meminfo_witness_test.dag @@ -30,15 +30,25 @@ test fn proc_meminfo_witnesses() -> Bool { // available, and an admission would refuse forever for the wrong reason), and a line whose value is // not a decimal count may not contribute a metric. Both are asserted against the same text with one // line changed, so the assertion discriminates the parse rather than the fixture. +// +// EVERY FIELD CARRIES A DISTINCT VALUE AND EVERY FIELD IS ASSERTED, because with seven lookups +// written by hand a name paired with the WRONG field is the one defect the absence arms cannot +// catch. And a field missing from the MIDDLE of the file names itself: that arm used to be +// unreachable only by an invariant held between two lists, and is now unreachable by construction +// -- there is no fallback left that could answer 0 KiB for it (review 69561 on gunbc#11962). test fn parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one() -> Bool { - let text = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: 8388608 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\nHugePages_Total: 0\n" + let text = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: 8388608 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 2097152 kB\nSwapFree: 524288 kB\nHugePages_Total: 0\n" let without_available = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" let unparseable = "MemTotal: 16777216 kB\nMemFree: 1048576 kB\nMemAvailable: not-a-number kB\nBuffers: 65536 kB\nCached: 4194304 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n" (match parse_proc_meminfo(text: text) { MeminfoParsed { meminfo: m } => kibibyte_count(k: m.mem_total) == 16777216 + && kibibyte_count(k: m.mem_free) == 1048576 && kibibyte_count(k: m.mem_available) == 8388608 - && kibibyte_count(k: m.swap_total) == 0 + && kibibyte_count(k: m.buffers) == 65536 + && kibibyte_count(k: m.cached) == 4194304 + && kibibyte_count(k: m.swap_total) == 2097152 + && kibibyte_count(k: m.swap_free) == 524288 && count(m.all_metrics) == 8 MeminfoFieldAbsent { field: _ } => false }) @@ -54,4 +64,11 @@ test fn parsing_meminfo_reads_the_named_fields_and_refuses_a_missing_one() -> Bo MeminfoFieldAbsent { field: f } => (f as String) == "MemTotal" MeminfoParsed { meminfo: _ } => false }) + && (match parse_proc_meminfo(text: join([ + "MemTotal: 16777216 kB\n", "MemFree: 1048576 kB\n", "MemAvailable: 8388608 kB\n", + "Buffers: 65536 kB\n", "Cached: 4194304 kB\n", "SwapFree: 524288 kB\n", + ], "")) { + MeminfoFieldAbsent { field: f } => (f as String) == "SwapTotal" + MeminfoParsed { meminfo: _ } => false + }) } From 047bb3eade1f727d61d673959effbc3f972765c0 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 15:15:26 +0000 Subject: [PATCH 03/28] Three defects the first real run on srv1 found, and the controls for them The workload ran under the grant on srv1: a real cargo release build of v1-compiler as gunbc-compute-1bf2556cc4b015f9.service, capped at the 26 GiB the pool granted, binaries returned by content. It also surfaced three defects that no wet cell could see, because none of them touched the provider. 1. THE CONSUMPTION READING BROKE THE ATTACH PATH. It was appended to outcome.json, which the attach path PARSES (stored_outcome_decode), so every later caller of a SUCCEEDED identity read StoredOutcomeUnreadable and rebuilt -- the contract's "two callers, one build" silently dead. Measured: the provider's own dependent compile request refused on a dependency that had in fact succeeded. The reading moves to its own file beside the log; a realization's measurement does not change the shape of the contract's document. 2. THE RESERVATION REFERENCE WAS THE WORK IDENTITY. The identity is stable by construction -- that is its job -- so a second request for the same identity hit the encumbrance ledger's DuplicateReference. The pool's reference names one HOLD, so it is now per attempt; the identity stays in the receipt. 3. A LEDGER REFUSAL WAS REPORTED AS A FULL POOL. The seat carrier renders every pool refusal into SeatFull, so DuplicateReference reached an operator as "capacity full" on a host with 400 GiB free. product.capacity.pool_events names the marker it writes (pool_full_wire) so both sides read one authority, and anything else is carried through as a ledger refusal. The typed-arm fix widens SeatAcquisition and every match on SeatFull, so it is a declared frontier with its trigger rather than a silent widen here. New wet control, red under the conflation: a reused reference on a pool with room for both requests is a ledger refusal and never ComputeCapacityFull. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 14 +++++++-- dag/gunbc/compute/work_provider_local.dag | 26 +++++++++++++++-- dag/gunbc/product/capacity/pool_events.dag | 18 +++++++++++- .../host_capacity_wet_witness_test.dag | 29 +++++++++++++++++++ src/v2/workflow/local_repo_wet_terminal.dag | 6 ++++ 5 files changed, 88 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index 51a74c7550d..f386c1c890d 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -7,7 +7,7 @@ import extdeps.linux.procfs import extdeps.linux.proc_meminfo { MeminfoRead, MeminfoParsed, MeminfoFieldAbsent, parse_proc_meminfo } import product.capacity.pool { Pool, pool_of, NoReplenishment, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at } import product.capacity.event_chain { PartitionId } -import product.capacity.pool_events { SeatRequest, PoolReleased } +import product.capacity.pool_events { SeatRequest, PoolReleased, pool_full_wire } import product.capacity.lease { LeasePolicy, LeaseGrant, QuiescenceRequired } import gunbc.fabric_storage_client { FabricStorageBinding, FabricStorageLocalFiles } import gunbc.fabric_storage_file_store { fabric_storage_file_root, fabric_storage_file_root_ensure, FabricStorageRootStanding, FabricStorageRootReady, FabricStorageRootRefused } @@ -228,6 +228,11 @@ fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, t } } +// A REFUSED ACQUIRE IS NOT AUTOMATICALLY A FULL POOL. The carrier renders every pool refusal into +// SeatFull, so the reason survives only in the wire; a DuplicateReference reported as capacity sent +// an operator looking for a busy host while srv1 had 400 GiB free (2026-09-21). The marker is read +// from the authority that writes it (product.capacity.pool_events pool_full_wire), never re-spelled +// here, and anything else is a ledger refusal with its text carried through. fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: EpochSecs, term_seconds: Nat) -> ComputeReservation { let store = compute_capacity_store(subject: subject) match fabric_seat_acquire( @@ -241,7 +246,12 @@ fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: Ep budget: compute_partition_read_budget, ) { SeatGranted { grant: g, generation: _, attempts_used: _ } => ComputeReserved { grant: g, partition: subject.partition, amount: subject.request, store: store } - SeatFull { wire: w, attempts_used: n } => ComputeCapacityFull { wire: join([w, " (attempt ", to_string(value: n), ")"], "") } + SeatFull { wire: w, attempts_used: n } => + if starts_with(s: w, prefix: pool_full_wire) { + ComputeCapacityFull { wire: join([w, " (attempt ", to_string(value: n), ")"], "") } + } else { + ComputeReservationRefused { detail: join(["the capacity ledger refused the acquire: ", w, " (attempt ", to_string(value: n), ")"], "") } + } other => ComputeReservationRefused { detail: seat_acquisition_wire(a: other) } } } diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 5d357f8598b..433cb1bd7e2 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -65,6 +65,7 @@ type ComputeLayout { target_dir: String log_path: String outcome_path: String + consumption_path: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -81,6 +82,7 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp target_dir: join([root, "/target"], ""), log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), + consumption_path: join([work_dir, "/consumption"], ""), } } @@ -299,7 +301,7 @@ fn compute_provide_leaf( if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) } else { - let reservation = compute_reserve(instance: instance, reference: identity as NonEmptyStr, term_seconds: compute_reservation_term_seconds) + let reservation = compute_reserve(instance: instance, reference: compute_reservation_reference(identity: identity), term_seconds: compute_reservation_term_seconds) match reservation { ComputeCapacityFull { wire: w } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) @@ -321,7 +323,8 @@ fn compute_provide_leaf( // is no ending on which the seat stays held. let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) let document = work_outcome_wire(o: outcome, subject: subject, op: op) - let written = Filesystem.Write(path: layout.outcome_path, content: join([document, "\n", compute_capacity_standing_wire(s: standing), "\n"], "")) + let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) + let written = Filesystem.Write(path: layout.outcome_path, content: document) let released = Filesystem.Delete(path: layout.lease_path) if written.success && released.success && compute_release_landed(r: returned) { ProvideFresh { outcome: outcome, document: document } @@ -334,6 +337,7 @@ fn compute_provide_leaf( "outcome written: ", if written.success { "yes" } else { written.error }, "; lease released: ", if released.success { "yes" } else { released.error }, "; reservation released: ", compute_release_wire(r: returned), + "; consumption recorded: ", if measured.success { "yes" } else { measured.error }, "; work ended ", work_outcome_ending(o: outcome), ], ""), }, @@ -346,6 +350,24 @@ fn compute_provide_leaf( } } +// THE CONSUMPTION READING IS ITS OWN FILE AND MUST NOT JOIN THE OUTCOME DOCUMENT. It was appended +// to outcome.json first, and the first real run on srv1 (2026-09-21) is what said why that is +// wrong: the stored outcome is PARSED by the attach path (stored_outcome_decode), so a line +// appended after the JSON made every later caller of a SUCCEEDED identity read +// StoredOutcomeUnreadable and rebuild -- the contract's "two callers, one build" silently dead, and +// the provider's own dependent request refusing on a dependency that had in fact succeeded. A +// realization's measurement does not get to change the shape of the contract's document. + +// THE RESERVATION REFERENCE NAMES AN ATTEMPT, NOT A WORK IDENTITY, and the first real run on srv1 +// (2026-09-21) is what forced the distinction. The identity is stable by construction -- that is its +// whole job -- so using it as the encumbrance reference meant the SECOND request for the same +// identity hit the ledger's DuplicateReference refusal and was reported as a capacity refusal on a +// host with 400 GiB free. The pool's reference is the name of one HOLD, and two holds of one +// identity at different times are two holds; the identity stays in the receipt, where it belongs. +fn compute_reservation_reference(identity: String) -> NonEmptyStr { + join([identity, "@", clock_now_probed_at_or_unknown() as String], "") as NonEmptyStr +} + // A REQUEST REFUSED FOR CAPACITY HOLDS NOTHING. The producer lease is dropped on the way out, so a // host that is full does not accumulate names that would refuse the next attempt for the wrong // reason -- a capacity refusal reported as a producer already in flight is the state conflation diff --git a/dag/gunbc/product/capacity/pool_events.dag b/dag/gunbc/product/capacity/pool_events.dag index a7c765f7c2c..542f75c540f 100644 --- a/dag/gunbc/product/capacity/pool_events.dag +++ b/dag/gunbc/product/capacity/pool_events.dag @@ -186,11 +186,27 @@ type SeatProposal = SeatProposed { event: ChainEvent, expected_head: HeadExpectation } | SeatRefused { wire: String } +// THE ONE SPELLING OF "THE POOL HAD NO ROOM", named because a consumer has to be able to tell that +// refusal from every other reason an acquire can be refused and the wire is currently the only place +// the distinction survives. SeatRefused carries a rendered string for all of them, so a consumer +// that wants "full" versus "the ledger said no" reads THIS row rather than authoring a second +// literal of its own (DESIGN 3). The reason it matters is measured: gunbc.compute.host_capacity +// mapped every SeatRefused onto a capacity refusal, and a DuplicateReference -- a ledger fact with +// nothing to do with capacity -- was reported to an operator as a full pool on a host with 400 GiB +// free (srv1, 2026-09-21). +// +// THE HONEST FIX IS A TYPED ARM AND IT IS NOT THIS. SeatRefused / SeatFull should carry the +// distinction in their constructors instead of in rendered text; that widens +// gunbc.fabric_event_log.SeatAcquisition and every existing match on SeatFull, so it is its own +// change. DECLARED FRONTIER (DESIGN 3c), trigger: the next consumer that needs a third +// classification, or the first one that must branch on a refusal this row cannot name. +data pool_full_wire: String = "pool-full" + fn seat_refusal_wire(r: PoolRefusal) -> String { match r { LedgerRefusal { refusal: lr } => match lr { - ExceedsAppropriation { requested: _, committed: _, ceiling: _ } => "pool-full" + ExceedsAppropriation { requested: _, committed: _, ceiling: _ } => pool_full_wire _ => pool_refusal_wire(r: r) } _ => pool_refusal_wire(r: r) diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index 235a64b7828..a751587d64f 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -58,6 +58,15 @@ fn is_capacity_full(r: ComputeReservation) -> Bool { } } +fn is_ledger_refusal(r: ComputeReservation) -> Bool { + match r { + ComputeReservationRefused { detail: d } => string_contains(s: d, pattern: "the capacity ledger refused the acquire") + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeHostBelowFloor { requested: _, available: _ } => false + } +} + fn released(r: ComputeRelease) -> Bool { match r { ComputeReleased { partition: _, reference: _, reason: _ } => true @@ -154,3 +163,23 @@ test fn the_deployment_join_roots_the_ledger_under_the_instance_compute_root() - && kibibyte_count(k: subject.appropriation) == kibibyte_count(k: subject.request) * 2 && kibibyte_count(k: subject.request) > 0 } + +// A LEDGER REFUSAL IS NOT A CAPACITY REFUSAL, and this cell exists because the two were conflated in +// production and the conflation was only visible on a real host. The reservation reference names one +// HOLD; re-using it is a DuplicateReference from the encumbrance ledger, which the seat carrier +// renders into the same SeatFull arm a genuinely exhausted pool reaches. Reported as capacity, it +// sent a reader looking for a busy machine while srv1 had 400 GiB free (2026-09-21) -- and the +// provider's own dependent request refused on a dependency that had succeeded. +// +// The pool here has room for BOTH requests, so nothing about capacity can explain the second +// answer: if this cell ever reports ComputeCapacityFull again, the conflation is back. +test fn a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool() -> Bool { + let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes * 4, request_kib: one_gibibyte_in_kibibytes) + let first = compute_reserve_on(subject: subject, reference: "wet-same-reference" as NonEmptyStr, term_seconds: 60) + let again = compute_reserve_on(subject: subject, reference: "wet-same-reference" as NonEmptyStr, term_seconds: 60) + let other = compute_reserve_on(subject: subject, reference: "wet-other-reference" as NonEmptyStr, term_seconds: 60) + reserved_amount(r: first) == one_gibibyte_in_kibibytes + && is_ledger_refusal(r: again) + && !is_capacity_full(r: again) + && reserved_amount(r: other) == one_gibibyte_in_kibibytes +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 60626c07334..2ee01ca1795 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -244,6 +244,12 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // probe failed without output rather than running its payload. fn local_repo_wet_schedule() -> List { [ + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "competing_reservations_cannot_over_reserve_and_a_release_returns_the_capacity" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", From dd240f0d171f7e8049c309cc8f40e7e6444282da Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 15:40:36 +0000 Subject: [PATCH 04/28] Body-position annotations refused by gunbc compile; move them to module grain The fourth thing the srv1 run found, and the one that makes the case for running it. DESIGN 4c admits standalone leading blocks attached to MODULE-SCOPE declarations only. The witness route this provider is covered by tolerates a block inside a function body; `gunbc compile` refuses it outright. So work_provider_local carried 14 blocking errors while every witness over it reported green, and the real compile request on srv1 is what said so -- the provider had stopped compiling and nothing in the wet or hermetic evidence could see it. The prose is not deleted: it moves above compute_provide_leaf, which is the declaration it was describing. Verified: `gunbc compile --entry dag/gunbc/compute/work_provider_local.dag` reports 0 blocking errors at this head, against 14 before. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 37 ++++++++++++++--------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 433cb1bd7e2..e06833d05e2 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -272,6 +272,29 @@ fn compute_basename(path: String) -> String { // makes the retry cheap; what must never happen is a caller attaching to a failure it did not ask for); refuse at the cap or behind a lease; take the lease; check out; // run; return outputs; write the outcome; release the lease. A refusal before the lease leaves no // state; a refusal after it leaves the lease released and the outcome written. +// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease +// excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the +// reservation is a QUANTITY on the host, and no exclusive create of any name settles that. The +// lease is taken first because a request that then loses on capacity releases a name it holds +// rather than memory it may never use, and a name is the cheaper of the two to undo. +// +// THE WORK RUNS UNDER THE GRANT: the transient unit's MemoryMax is the amount the pool reserved, so +// the ledger's arithmetic IS the enforcement rather than a parallel description of it. +// +// AGGREGATE CONSUMPTION IS READ WHILE THE GRANT IS STILL HELD, which is the only instant at which +// the ledger's committed total and the machine's own availability describe the same population; +// after the release the reservation is gone from one side and the memory may not yet be gone from +// the other. +// +// COMPLETION AND CANCELLATION RELEASE THE SAME WAY. Every ending of compute_run_leased -- succeeded, +// failed, output-mismatch, refused by infrastructure -- reaches the release, so there is no ending +// on which the seat stays held. +// +// THE COMMENTS ABOVE USED TO SIT INSIDE THIS FUNCTION'S BODY, and the first real compile of this +// module on srv1 (2026-09-21) is what moved them: DESIGN 4c admits standalone leading blocks +// attached to MODULE-SCOPE declarations only, and the interpreter route this provider's witnesses +// run on tolerates a body-position block that `gunbc compile` refuses outright -- 14 blocking +// errors on a module every witness had reported green. fn compute_provide_leaf( instance: HostDashboardInstance, commit: String, @@ -292,11 +315,6 @@ fn compute_provide_leaf( } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { - // THE PRODUCER LEASE AND THE CAPACITY RESERVATION ARE TWO FACTS AND ARE TAKEN IN THIS ORDER. - // The lease excludes a SECOND PRODUCER OF THIS IDENTITY and is a name, so O_EXCL settles it; the - // reservation is a QUANTITY on the host and no exclusive name can settle that. Taking the lease - // first means a request that then loses on capacity releases a name it holds rather than - // reserving memory it may never use, and it is the cheaper of the two to undo. let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) @@ -310,17 +328,8 @@ fn compute_provide_leaf( ComputeReservationRefused { detail: d } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => { - // THE WORK RUNS UNDER THE GRANT: the unit's MemoryMax is the amount the pool reserved, so - // the ledger's arithmetic is the enforcement rather than a parallel description of it. let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) - // AGGREGATE CONSUMPTION IS READ WHILE THE GRANT IS STILL HELD, which is the only instant - // at which the ledger's committed total and the machine's own availability describe the - // same population; after the release the reservation is gone from one side and the - // memory may not yet be gone from the other. let standing = compute_capacity_standing(instance: instance) - // COMPLETION AND CANCELLATION RELEASE THE SAME WAY. Every ending of compute_run_leased -- - // succeeded, failed, output-mismatch, refused by infrastructure -- reaches here, so there - // is no ending on which the seat stays held. let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) let document = work_outcome_wire(o: outcome, subject: subject, op: op) let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) From 725d9c58620b1fe993ee43a52e9081b00e7ea316 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 16:33:22 +0000 Subject: [PATCH 05/28] Four design defects: the double promise, the instance-keyed pool, the record that disagreed with its caller, and the recovery route that did not exist Side-chat review of #11962 at dd240f0d, plus review 69595. All four are real and all four were verified against the code before anything was changed. A. THE DOUBLE PROMISE. Pool capacity and live MemAvailable were checked separately, so with a 52 GiB appropriation, 30 GiB available and two 26 GiB requests, each request saw 26 <= 30 and each fit the appropriation, and BOTH admitted -- neither could see the other's granted-but-not-yet-consumed memory. The compare-and-set was doing its job perfectly on a question that was missing a term. The fix is not a third check. The pool already computes the committed total inside the linearized fold, so the machine's reading belongs in the SAME comparison: the ceiling an attempt is adjudicated against is the smaller of the declared appropriation and what the host says is free. This is worst-case-sound and conservative, and says so: MemAvailable already excludes what outstanding grants have touched, so a resident grant is counted twice. The exact relation needs the resident share, which is observable per unit and is a declared frontier with its trigger. product.capacity.pool already ruled that replay never adjudicates the ceiling, so a moving ceiling cannot poison the fold -- it makes the pool over-committed, which refuses new acquires until it drains. B. THE POOL WAS KEYED BY THE INSTANCE, NOT THE HOST. srv1 declares two instances -- srv1-live and srv1-lab -- with different ids and roots, and the memory they spend is ONE machine's. Both halves are now host-keyed: the partition names the host identity, and the ledger is rooted under the instance that host resolves to through dashboard_instance_for_host, the same host-to-instance authority the fabric event log already uses. A host that authority cannot resolve reserves nothing rather than defaulting to the caller. C. THE RECORD DISAGREED WITH THE CALLER. A run whose work succeeded but whose release failed wrote WorkSucceeded to outcome.json and handed its caller a refusal -- and the next caller ATTACHED to that success, so the leaked seat had no consumer that would ever look for it. The ending and the unresolved obligation are two facts and are recorded as two: the obligation is its own durable file, and the outcome document carries the refusal the caller was given, so a later caller attaches to exactly what this caller was told. D. THE RECOVERY ROUTE DID NOT EXIST (review 69595). The module's prose said a wedged seat is recovered by an explicit release naming why; compute_release took a ComputeReservation whose only constructor lives and dies inside one compute_provide_leaf, so under QuiescenceRequired -- which never lapses -- nothing in the corpus could discharge it, and two such events would consume the whole appropriation and turn every later request into a capacity refusal on a host with its memory free. A claimed route no declaration reaches is the dangling consumption DESIGN 3c refuses. compute_release_reference takes the reference, which is all a recovering operator has, and compute_release_cli is its executing consumer. Both doors go through fabric_seat_release, the checked mirror of fabric_seat_acquire: read, fold, PROPOSE against the folded pool, append only what it admits. That check is load-bearing rather than defensive -- an unheld reference appended anyway would not fail at the append, it would fail at every later READ, when the fold hits UnknownEncumbrance and refuses the whole partition. Controls, all executing wet, seven cells: two requests summing past the observed headroom cannot both admit (RED when the backing ceiling is reverted to the appropriation); two instances on one host resolve to one pool at one root; a held seat is recoverable by reference and an unheld one writes nothing, asserted beside a later successful reservation so the ledger is shown still readable. gunbc compile on the provider closure: 0 blocking errors. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 192 ++++++++++++++--- dag/gunbc/compute/work_provider_local.dag | 193 +++++++++++++++--- dag/gunbc/fabric/fabric_event_log.dag | 98 ++++++++- .../host_capacity_wet_witness_test.dag | 93 +++++++-- src/v2/workflow/local_repo_wet_terminal.dag | 16 +- 5 files changed, 520 insertions(+), 72 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index f386c1c890d..f54b9e10398 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -15,10 +15,14 @@ import gunbc.fabric_event_log { SeatAcquisition, SeatGranted, SeatFull, SeatContended, SeatAcquireRefused, SeatStoreRefused, fabric_seat_acquire, seat_acquisition_wire, SeatStandingObservation, SeatRoomObserved, SeatFullObserved, SeatStandingUnobserved, SeatStandingStoreRefused, fabric_seat_observe, + SeatRelease, SeatReleased, SeatNotHeld, SeatReleaseContended, SeatReleaseRefused, fabric_seat_release, seat_release_wire, PoolEventAppend, PoolEventAppended, PoolEventContended, PoolEventAppendRefused, fabric_pool_event_append, pool_event_append_wire, } import gunbc.fabric_event_log_host { now_epoch_seconds } -import gunbc.roadmap_dashboard_instance { HostDashboardInstance, dashboard_instance_compute_root } +import gunbc.roadmap_dashboard_instance { + HostDashboardInstance, dashboard_instance_compute_root, + DashboardInstanceByHost, DashboardInstanceForHost, DashboardInstanceHostUnknown, dashboard_instance_for_host, +} // THE HOST'S COMPUTE MEMORY IS A POOL, AND A WORK REQUEST TAKES A SEAT IN IT BEFORE IT RUNS. // @@ -72,21 +76,47 @@ type ComputeCapacitySubject { request: Kibibyte } -fn compute_capacity_root(instance: HostDashboardInstance) -> NonEmptyStr { - join([dashboard_instance_compute_root(instance: instance) as String, "/capacity"], "") as NonEmptyStr +// THE POOL IS A HOST'S, AND KEYING IT BY THE INSTANCE WAS A MIS-KEYING (side-chat review of +// gunbc#11962 at dd240f0d, 2026-09-21). srv1 declares TWO dashboard instances -- srv1-live and +// srv1-lab -- with different ids and different instance roots, and the memory they would spend is +// ONE machine's. An instance-keyed partition gives each of them its own appropriation and its own +// ledger, so two pools each promise the same bytes and neither can see the other: the exclusion the +// compare-and-set provides is perfect and is about the wrong subject. +// +// So both halves are keyed by the HOST. The partition names the host identity, and the ledger is +// rooted under the instance THAT HOST resolves to -- gunbc.roadmap_dashboard_instance +// dashboard_instance_for_host, the same host-to-instance authority the fabric event log already +// uses to decide where a host's log lives. Two instances on one host therefore resolve to one +// partition at one root, which is the single host-scoped authority this fact needs; inventing a new +// host-scoped path would have been a second one. +// +// A HOST THAT AUTHORITY CANNOT RESOLVE RESERVES NOTHING. It is not defaulted to the calling +// instance: that is exactly the fork this fix removes, re-entering through a fallback. +type ComputeSubjectResolution + = ComputeSubjectResolved { subject: ComputeCapacitySubject } + | ComputeSubjectHostUnresolved { host: NonEmptyStr } + +fn compute_capacity_root(host_instance: HostDashboardInstance) -> NonEmptyStr { + join([dashboard_instance_compute_root(instance: host_instance) as String, "/capacity"], "") as NonEmptyStr } -fn compute_memory_partition(instance: HostDashboardInstance) -> PartitionId { - join(["compute-memory-", instance.instance_id as String], "") as PartitionId +fn compute_memory_partition(host: NonEmptyStr) -> PartitionId { + join(["compute-memory-", host as String], "") as PartitionId } // THE JOIN WITH THE DEPLOYMENT, and the only function in this module that knows what an instance is. -fn compute_capacity_subject(instance: HostDashboardInstance) -> ComputeCapacitySubject { - ComputeCapacitySubject { - root: compute_capacity_root(instance: instance), - partition: compute_memory_partition(instance: instance), - appropriation: compute_memory_appropriation(instance: instance), - request: compute_request_memory(instance: instance), +fn compute_capacity_subject(instance: HostDashboardInstance) -> ComputeSubjectResolution { + match dashboard_instance_for_host(host: instance.host_identity as String) { + DashboardInstanceHostUnknown { host: h } => ComputeSubjectHostUnresolved { host: h as NonEmptyStr } + DashboardInstanceForHost { instance: host_instance } => + ComputeSubjectResolved { + subject: ComputeCapacitySubject { + root: compute_capacity_root(host_instance: host_instance), + partition: compute_memory_partition(host: instance.host_identity), + appropriation: compute_memory_appropriation(instance: host_instance), + request: compute_request_memory(instance: instance), + }, + } } } @@ -170,10 +200,45 @@ fn compute_lease_policy(term_seconds: Nat) -> LeasePolicy { LeasePolicy { maximum_duration_seconds: term_seconds, release_law: QuiescenceRequired } } -fn compute_memory_root_pool(subject: ComputeCapacitySubject) -> Pool { +// THE BACKING RELATION, AND IT IS THE WHOLE REASON THE TWO OBSERVATIONS ARE NOT INDEPENDENT CHECKS. +// +// The first revision compared the pool's committed total against the appropriation in one place and +// the machine's live availability against ONE request in another. Both checks passed and the pair +// still double-promised (side-chat review of gunbc#11962 at dd240f0d, 2026-09-21): with a 52 GiB +// appropriation, 30 GiB available and two 26 GiB requests, each request sees 26 <= 30 and each fits +// the appropriation, so BOTH admit -- because neither sees the other's granted-but-not-yet-consumed +// memory. The compare-and-set was doing its job perfectly on a question that was missing a term. +// +// The fix is not a third check. The pool already computes the committed total INSIDE the linearized +// fold, so the machine's reading belongs in the SAME comparison: the ceiling this attempt is +// adjudicated against is the smaller of the declared appropriation and what the host says is free. +// Then "everything this pool has promised, plus what I am asking for, must fit in what the machine +// actually has" is one predicate evaluated in one CAS'd step, and the second 26 GiB request is +// refused as a full pool because 26 + 26 > 30. +// +// WORST-CASE-SOUND, AND CONSERVATIVE, AND SAYING SO. MemAvailable already excludes whatever the +// pool's outstanding grants have actually touched, so a grant that is fully resident is counted +// twice -- once in the machine's reading and once in the committed total. That errs toward refusing +// work the host could have run, which is the direction a safety admission must err in when the term +// it needs is unobserved. The exact relation needs the RESIDENT share of outstanding grants, which +// is observable -- each grant runs in a named transient unit with its own cgroup -- and is not +// observed here. DECLARED FRONTIER (DESIGN 3c), trigger: a per-host read of the compute units' +// current usage, whose first consumer is this fold; until it lands the conservative bound stands +// and the cost is stated rather than hidden. +// +// A CEILING THAT MOVES BETWEEN ATTEMPTS IS SAFE BY CONSTRUCTION HERE, which is why this is a ceiling +// and not a pre-check. product.capacity.pool ruled that replay does not adjudicate the ceiling -- +// an event's admission happened when it was written -- so a dip in availability cannot make a +// historical acquire refuse the fold and poison the partition. It makes the pool over-committed, +// which refuses new acquires until it drains. That is the correct consequence of a reduction. +fn compute_backing_ceiling(subject: ComputeCapacitySubject, available: Kibibyte) -> Kibibyte { + if kibibyte_count(k: available) < kibibyte_count(k: subject.appropriation) { available } else { subject.appropriation } +} + +fn compute_memory_root_pool(subject: ComputeCapacitySubject, ceiling: Kibibyte) -> Pool { pool_of( identity: subject.partition as String as NonEmptyStr, - ceiling: subject.appropriation, + ceiling: ceiling, replenishment: NoReplenishment, release_law: QuiescenceRequired, ) @@ -198,7 +263,11 @@ data compute_partition_read_budget: Nat = 4096 // the declared share is a configuration error that would otherwise present as an OOM kill during a // build, which is the same information arriving hours later and destructively. fn compute_reserve(instance: HostDashboardInstance, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { - compute_reserve_on(subject: compute_capacity_subject(instance: instance), reference: reference, term_seconds: term_seconds) + match compute_capacity_subject(instance: instance) { + ComputeSubjectHostUnresolved { host: h } => + ComputeReservationRefused { detail: join(["no dashboard instance is declared for host ", h as String, ", so this host has no compute capacity authority to reserve against"], "") } + ComputeSubjectResolved { subject: subject } => compute_reserve_on(subject: subject, reference: reference, term_seconds: term_seconds) + } } fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { @@ -221,7 +290,7 @@ fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, t FabricStorageRootReady => match now_epoch_seconds() { Absent => ComputeReservationRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds) + Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds, ceiling: compute_backing_ceiling(subject: subject, available: available)) } } } @@ -233,12 +302,12 @@ fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, t // an operator looking for a busy host while srv1 had 400 GiB free (2026-09-21). The marker is read // from the authority that writes it (product.capacity.pool_events pool_full_wire), never re-spelled // here, and anything else is a ledger refusal with its text carried through. -fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: EpochSecs, term_seconds: Nat) -> ComputeReservation { +fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: EpochSecs, term_seconds: Nat, ceiling: Kibibyte) -> ComputeReservation { let store = compute_capacity_store(subject: subject) match fabric_seat_acquire( store: store, partition: subject.partition, - root: compute_memory_root_pool(subject: subject), + root: compute_memory_root_pool(subject: subject, ceiling: ceiling), actor: reference, request: SeatRequest { reference: reference, amount: kibibyte_count(k: subject.request), at: now, term_seconds: term_seconds }, policy: compute_lease_policy(term_seconds: term_seconds), @@ -275,16 +344,78 @@ fn compute_release(reservation: ComputeReservation, reason: NonEmptyStr) -> Comp ComputeHostBelowFloor { requested: _, available: _ } => ComputeReleaseRefused { detail: "nothing to release: the host was below its live memory floor" } ComputeReservationRefused { detail: d } => ComputeReleaseRefused { detail: join(["nothing to release: ", d], "") } ComputeReserved { grant: g, partition: p, amount: _, store: store } => - match now_epoch_seconds() { - Absent => ComputeReleaseRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => - match fabric_pool_event_append( - store: store, partition: p, actor: g.reference, at: now, - payload: PoolReleased { reference: g.reference, reason: reason }, attempts: compute_seat_attempts, - ) { - PoolEventAppended { id: _ } => ComputeReleased { partition: p, reference: g.reference, reason: reason } - other => ComputeReleaseRefused { detail: pool_event_append_wire(a: other) } - } + compute_release_on(store: store, partition: p, appropriation: kibibyte(count: 0), reference: g.reference, reason: reason) + } +} + +// THE RECOVERY ROUTE, AND IT EXISTS BECAUSE THE MODULE CLAIMED IT (review 69595 on gunbc#11962). +// +// The prose above says a wedged seat is recovered by an explicit release naming why. Until this +// function that was FALSE: compute_release took a ComputeReservation, whose only constructor is +// compute_reserve_on, whose result lives and dies inside one compute_provide_leaf invocation. So a +// release that failed to land left an encumbrance that -- under QuiescenceRequired, which by design +// never lapses -- nothing in the corpus could ever discharge. Two of those consume the whole host +// appropriation, and every later request then gets a CAPACITY refusal on a host with its memory +// free: the exact conflation this change added a witness to prevent, arriving through a different +// door. A claimed route that no declaration reaches is the dangling consumption DESIGN 3c refuses. +// +// IT TAKES A REFERENCE, because that is all a recovering operator has: the reservation value is +// gone with the process that held it, and the reference is what the ledger and the obligation file +// both name. It is CHECKED against the folded pool before anything is appended -- a reference the +// pool is not holding writes nothing and refuses by name, because appending a release for an unheld +// reference would make the partition unfoldable for every later reader. +fn compute_release_reference(instance: HostDashboardInstance, reference: NonEmptyStr, reason: NonEmptyStr) -> ComputeRelease { + match compute_capacity_subject(instance: instance) { + ComputeSubjectHostUnresolved { host: h } => + ComputeReleaseRefused { detail: join(["no dashboard instance is declared for host ", h as String], "") } + ComputeSubjectResolved { subject: subject } => + compute_release_on( + store: compute_capacity_store(subject: subject), + partition: subject.partition, + appropriation: subject.appropriation, + reference: reference, + reason: reason, + ) + } +} + +// ONE RELEASE PATH FOR BOTH DOORS. The provider's own completion and the operator's recovery append +// the same event against the same check; a second spelling of "give the seat back" is the fork this +// module would otherwise grow the moment recovery existed. +// +// THE APPROPRIATION PASSED HERE IS NOT LOAD-BEARING AND THE ZERO IS NOT A FABRICATED READING. product.capacity.pool +// ruled that replay never adjudicates the ceiling, so a release is decided by whether the ledger +// HOLDS the reference and by nothing about capacity; a reservation released from its own provider +// carries no subject to read an appropriation from, and inventing one for it would be a number with +// no consequence. Where a subject is at hand it is passed; where none is, zero is passed, and zero +// cannot admit anything -- which is the right direction for a value nothing should be deciding on. +fn compute_release_on( + store: FabricStorageBinding, + partition: PartitionId, + appropriation: Kibibyte, + reference: NonEmptyStr, + reason: NonEmptyStr, +) -> ComputeRelease { + match now_epoch_seconds() { + Absent => ComputeReleaseRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => + match fabric_seat_release( + store: store, + partition: partition, + root: pool_of( + identity: partition as String as NonEmptyStr, + ceiling: appropriation, + replenishment: NoReplenishment, + release_law: QuiescenceRequired, + ), + reference: reference, + reason: reason, + at: now, + attempts: compute_seat_attempts, + budget: compute_partition_read_budget, + ) { + SeatReleased { reference: r, id: _ } => ComputeReleased { partition: partition, reference: r, reason: reason } + other => ComputeReleaseRefused { detail: seat_release_wire(r: other) } } } } @@ -302,7 +433,10 @@ type ComputeCapacityStanding | ComputeCapacityStandingUnread { detail: String } fn compute_capacity_standing(instance: HostDashboardInstance) -> ComputeCapacityStanding { - compute_capacity_standing_on(subject: compute_capacity_subject(instance: instance)) + match compute_capacity_subject(instance: instance) { + ComputeSubjectHostUnresolved { host: h } => ComputeCapacityStandingUnread { detail: join(["no dashboard instance is declared for host ", h as String], "") } + ComputeSubjectResolved { subject: subject } => compute_capacity_standing_on(subject: subject) + } } fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapacityStanding { @@ -315,7 +449,7 @@ fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapac match fabric_seat_observe( store: compute_capacity_store(subject: subject), partition: subject.partition, - root: compute_memory_root_pool(subject: subject), + root: compute_memory_root_pool(subject: subject, ceiling: subject.appropriation), at: now, budget: compute_partition_read_budget, ) { diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index e06833d05e2..0fb60514c62 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -23,7 +23,7 @@ import gunbc.roadmap_execution_contract { import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size } import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, + ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_release_reference, compute_reservation_wire, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, } import gunbc.compute.work_request { @@ -66,6 +66,7 @@ type ComputeLayout { log_path: String outcome_path: String consumption_path: String + obligation_path: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -83,6 +84,7 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), consumption_path: join([work_dir, "/consumption"], ""), + obligation_path: join([work_dir, "/unreleased-reservation"], ""), } } @@ -272,6 +274,127 @@ fn compute_basename(path: String) -> String { // makes the retry cheap; what must never happen is a caller attaching to a failure it did not ask for); refuse at the cap or behind a lease; take the lease; check out; // run; return outputs; write the outcome; release the lease. A refusal before the lease leaves no // state; a refusal after it leaves the lease released and the outcome written. +// SETTLING A RUN THAT HAS FINISHED, AND THE ONE RULE THAT GOVERNS IT: THE DURABLE RECORD AND THE +// CALLER MUST AGREE ABOUT WHAT HAPPENED. +// +// They did not. The outcome document was written unconditionally, so a run whose work SUCCEEDED but +// whose reservation failed to release wrote WorkSucceeded to outcome.json and handed its caller a +// WorkRefusedByInfrastructure -- and the next caller of that identity ATTACHED to the stored +// success, which is the contract behaving correctly on a record that had quietly become a lie about +// the infrastructure's state. The leaked seat then had no consumer that would ever look for it +// (side-chat review of gunbc#11962 at dd240f0d, 2026-09-21). +// +// THE RESOLUTION IS NOT TO SUPPRESS THE SUCCESS. The work really did succeed and its outputs really +// are in the store; deleting that fact would make every later caller rebuild and would be a second +// lie in the other direction. The run's ending and the infrastructure's unresolved obligation are +// TWO FACTS, so they are recorded as two: the obligation is written as its own durable file, and +// the outcome document carries the refusal the caller is given -- so a later caller attaches to +// EXACTLY what this caller was told, and neither of them attaches to a success whose reservation is +// still outstanding. The run's own ending is never lost: it is named in the obligation and in the +// refusal, and the outputs stay in the store under their content identity. +// +// THE SEAT STAYS HELD, AND THAT IS THE SAFE DIRECTION rather than an omission. The pool's release +// law for a compute seat is quiescence-required precisely because nothing here can prove a unit +// stopped using memory; a release that did not land leaves the capacity encumbered, so the host +// under-admits until the obligation is discharged rather than handing the same bytes to the next +// arrival. Discharging it is an explicit release naming why -- the same entry completion and +// cancellation take. +fn compute_run_and_settle( + instance: HostDashboardInstance, + commit: String, + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + dependency_store: String, + granted: Kibibyte, + reservation: ComputeReservation, +) -> ProvideResult { + let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) + let standing = compute_capacity_standing(instance: instance) + let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) + let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) + let released = Filesystem.Delete(path: layout.lease_path) + let notes = join([ + "; producer lease released: ", if released.success { "yes" } else { released.error }, + "; consumption recorded: ", if measured.success { "yes" } else { measured.error }, + ], "") + if compute_release_landed(r: returned) { + compute_settled_record( + layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, + lease_released: released.success, notes: notes) + } else { + compute_unreleased_record( + layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, + reservation: reservation, returned: returned, notes: notes) + } +} + +// THE ORDINARY ENDING: the reservation came back, so the record is the run's own ending and the +// caller is told the same thing. +fn compute_settled_record( + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + outcome: WorkOutcome, + lease_released: Bool, + notes: String, +) -> ProvideResult { + let document = work_outcome_wire(o: outcome, subject: subject, op: op) + let written = Filesystem.Write(path: layout.outcome_path, content: document) + if written.success && lease_released { + ProvideFresh { outcome: outcome, document: document } + } else { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { + detail: join([ + "the work ended ", work_outcome_ending(o: outcome), + " and its reservation was released, but the record could not be completed -- outcome written: ", + if written.success { "yes" } else { written.error }, notes, + ], ""), + }, + }, + subject: subject, op: op) + } +} + +// THE ENDING WITH AN OUTSTANDING OBLIGATION: one document, written once, given to this caller and +// read by the next one. +fn compute_unreleased_record( + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + outcome: WorkOutcome, + reservation: ComputeReservation, + returned: ComputeRelease, + notes: String, +) -> ProvideResult { + let obligation = join([ + "unreleased reservation: ", compute_reservation_wire(r: reservation), + "; release attempt: ", compute_release_wire(r: returned), + "; the work itself ended ", work_outcome_ending(o: outcome), + " and any declared outputs are in the store; the seat stays held until an explicit release ", + "discharges it, so this host under-admits rather than promising the same memory twice", + ], "") + let recorded = Filesystem.Write(path: layout.obligation_path, content: join([obligation, "\n"], "")) + let cause = ComputeCapacityLedgerUnavailable { + detail: join([ + obligation, "; obligation recorded: ", if recorded.success { layout.obligation_path } else { recorded.error }, notes, + ], ""), + } + let document = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) + let written = Filesystem.Write(path: layout.outcome_path, content: document) + if written.success { + ProvideFresh { outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, document: document } + } else { + compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) + } +} + // THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the // reservation is a QUANTITY on the host, and no exclusive create of any name settles that. The @@ -327,33 +450,10 @@ fn compute_provide_leaf( compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested_kib: kibibyte_count(k: rq), available_kib: kibibyte_count(k: av) }) ComputeReservationRefused { detail: d } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) - ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => { - let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) - let standing = compute_capacity_standing(instance: instance) - let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) - let document = work_outcome_wire(o: outcome, subject: subject, op: op) - let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) - let written = Filesystem.Write(path: layout.outcome_path, content: document) - let released = Filesystem.Delete(path: layout.lease_path) - if written.success && released.success && compute_release_landed(r: returned) { - ProvideFresh { outcome: outcome, document: document } - } else { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: StoreUnavailable { - detail: join([ - "outcome written: ", if written.success { "yes" } else { written.error }, - "; lease released: ", if released.success { "yes" } else { released.error }, - "; reservation released: ", compute_release_wire(r: returned), - "; consumption recorded: ", if measured.success { "yes" } else { measured.error }, - "; work ended ", work_outcome_ending(o: outcome), - ], ""), - }, - }, - subject: subject, op: op) - } - } + ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => + compute_run_and_settle( + instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, + dependency_store: dependency_store, granted: granted, reservation: reservation) } } } @@ -529,6 +629,43 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent } } +// THE OPERATOR'S RECOVERY DOOR, and the executing consumer of compute_release_reference. A +// reservation whose release did not land leaves its reference in +// /unreleased-reservation and in the partition; this is how it is given back. It is a +// verb on the same CLI rather than a script, so the refusal vocabulary is the module's: a reference +// the pool is not holding writes nothing and exits non-zero saying so, which is the answer a +// mistyped reference must get -- appending a release for an unheld reference would make the +// partition unfoldable for every later reader. +fn compute_release_cli(repo_root: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { + match dashboard_instance_for_repo_root(repo_root: repo_root) { + DashboardInstanceRootUnknown { repo_root: r } => ExitFailure { code: 2, reason: join(["compute: no dashboard instance is rooted at ", r], "") } + DashboardInstanceResolved { instance } => + if reference == "" || reason == "" { + ExitFailure { code: 2, reason: "compute: release needs a non-empty reference and a reason naming why the seat is being given back" } + } else { + let released = compute_release_reference(instance: instance, reference: reference as NonEmptyStr, reason: reason as NonEmptyStr) + let wire = compute_release_cli_wire(r: released) + let receipt = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) + if !receipt.success { + ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; release was ", wire], "") } + } else { + match released { + ComputeReleased { partition: _, reference: _, reason: _ } => ExitSuccess + ComputeReleaseRefused { detail: _ } => ExitFailure { code: 1, reason: join(["compute release refused: ", wire], "") } + } + } + } + } +} + +fn compute_release_cli_wire(r: ComputeRelease) -> String { + match r { + ComputeReleased { partition: p, reference: ref_, reason: why } => + join(["released ", ref_ as String, " on ", p as String, " (", why as String, ")"], "") + ComputeReleaseRefused { detail: d } => join(["refused: ", d], "") + } +} + fn compute_parse_operation(operation: String, entry: String, functions: String, hermetic: String) -> WorkOperation? { if operation == "build" { Present { value: work_default_build() } diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index ec7ee6ef88f..d7ae49970ac 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -16,7 +16,7 @@ import std.fabric_storage { import gunbc.fabric_storage_placement { fabric_storage_placement, FabricStoragePlaced, FabricStorageUnplaced } import gunbc.fabric_storage_client { FabricStorageBinding, fabric_storage_binding_for, fabric_storage_head, fabric_storage_put, fabric_storage_advance, fabric_storage_closure } import gunbc.fabric_storage_file_store { fabric_storage_file_root, fabric_storage_file_root_ensure, FabricStorageRootReady, FabricStorageRootRefused } -import product.capacity.pool { Pool, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at } +import product.capacity.pool { Pool, PoolReading, PoolRead, PoolReadingRefused, pool_reading_at, PoolOutcome, PoolAdvanced, PoolRefused, pool_release, pool_refusal_wire } import product.capacity.event_chain { PartitionId, EventId, ChainEvent, ChainEnvelope, HeadExpectation, HeadAbsent, HeadAt, head_expectation_eq, event_parent_expectation, AppendDecision, AppendAdmitted, AppendStale, ChainWalk, ChainWalked, ChainIncomplete, ChainBudgetExhausted, chain_from_head, @@ -25,6 +25,7 @@ import product.capacity.event_chain { import product.capacity.pool_events { PoolEvent, pool_event_wire_text, pool_event_decode, PoolFold, PoolFolded, PoolFoldRefused, pool_fold, SeatRequest, SeatProposal, SeatProposed, SeatRefused, propose_acquire, grant_from_admission, + PoolReleased, } import product.capacity.lease { LeasePolicy, LeaseGrant, release_law_eq, release_law_wire } @@ -402,6 +403,101 @@ fn pool_event_append_wire(a: PoolEventAppend) -> String { } } +// RELEASING A SEAT, AND IT IS CHECKED AGAINST THE FOLDED POOL BEFORE ANYTHING IS APPENDED. +// +// fabric_pool_event_append is unchecked by construction -- it writes the event a caller hands it -- +// which is correct for a settlement whose amount the caller already knows and WRONG for a release, +// because a release names a reference the ledger may not be holding. An unheld reference appended +// anyway does not fail at the append: it fails at the next READ, when pool_fold reaches the event, +// gets UnknownEncumbrance, and answers PoolFoldRefused for the WHOLE PARTITION. One mistyped +// reference in a recovery command would make a host's capacity ledger permanently unreadable, and +// the damage would surface at an unrelated caller's next acquire. +// +// So this is the mirror of fabric_seat_acquire and not of the settlement helper: read the partition, +// fold it, PROPOSE the release against the folded pool, and append only what the pool admits. A +// reference the pool is not holding is a typed refusal that writes nothing. +// +// ITS CONSUMER IS THE RECOVERY ROUTE A HELD SEAT NEEDS (gunbc.compute.host_capacity +// compute_release_reference, reached from the operator CLI compute_release_cli). Under +// QuiescenceRequired a term never frees capacity, so an explicit release is the ONLY way a seat +// comes back, and a module claiming that route owes one that executes (review 69595 on gunbc#11962, +// DESIGN 3c). +type SeatRelease + = SeatReleased { reference: NonEmptyStr, id: EventId } + | SeatNotHeld { reference: NonEmptyStr, wire: String } + | SeatReleaseContended { attempts: Nat } + | SeatReleaseRefused { step: String, reason: String } + +type SeatReleaseState { + done: SeatRelease? +} + +fn seat_release_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, reference: NonEmptyStr, reason: NonEmptyStr, at: EpochSecs, budget: Nat) -> SeatRelease? { + match event_log_read_partition(store: store, partition: partition, budget: budget) { + PartitionReadRefused { cause: c } => Present { value: SeatReleaseRefused { step: "store", reason: event_log_refusal_wire(cause: c) } } + PartitionReadOk { head: head, walk: walk } => + match walk { + ChainIncomplete { missing: m, newest_first_so_far: _ } => Present { value: SeatReleaseRefused { step: "chain", reason: join(["partition chain is missing event ", m as String], "") } } + ChainBudgetExhausted { at: a } => Present { value: SeatReleaseRefused { step: "chain", reason: join(["partition chain exceeded the read budget at ", a as String], "") } } + ChainWalked { oldest_first: xs } => + match pool_fold(root: root, oldest_first: xs) { + PoolFoldRefused { at_event: e, wire: w } => Present { value: SeatReleaseRefused { step: "fold", reason: join(["event ", e as String, " refused: ", w], "") } } + PoolFolded { pool: p, generation: _ } => + match pool_release(pool: p, reference: reference, reason: reason, at: at) { + PoolRefused { refusal: r } => Present { value: SeatNotHeld { reference: reference, wire: pool_refusal_wire(r: r) } } + PoolAdvanced { pool: _ } => + match event_log_append( + store: store, + partition: partition, + event: ChainEvent { + partition: partition, + parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, + recorded_at: at, + actor: reference, + payload: PoolReleased { reference: reference, reason: reason }, + }, + expected: head, + ) { + EventAppended { id: h } => Present { value: SeatReleased { reference: reference, id: h } } + EventAppendStale { expected: _, observed: _ } => none + EventAppendRefused { cause: c } => Present { value: SeatReleaseRefused { step: "store", reason: event_log_refusal_wire(cause: c) } } + } + } + } + } + } +} + +fn fabric_seat_release(store: FabricStorageBinding, partition: PartitionId, root: Pool, reference: NonEmptyStr, reason: NonEmptyStr, at: EpochSecs, attempts: Nat, budget: Nat) -> SeatRelease { + let st = fold(nat_range_inclusive(lo: 1, hi: attempts), init: SeatReleaseState { done: none }, f: (acc, _i) => + match acc.done { + Present { value: _ } => acc + Absent => SeatReleaseState { done: seat_release_attempt(store: store, partition: partition, root: root, reference: reference, reason: reason, at: at, budget: budget) } + }) + match st.done { + Present { value: d } => d + Absent => SeatReleaseContended { attempts: attempts } + } +} + +fn seat_release_wire(r: SeatRelease) -> String { + match r { + SeatReleased { reference: ref_, id: h } => join(["released ", ref_ as String, " at ", h as String], "") + SeatNotHeld { reference: ref_, wire: w } => join(["the pool is not holding ", ref_ as String, ": ", w], "") + SeatReleaseContended { attempts: n } => join(["contended after ", to_string(n), " attempts"], "") + SeatReleaseRefused { step: s, reason: why } => join(["refused at ", s, ": ", why], "") + } +} + +fn seat_release_landed(r: SeatRelease) -> Bool { + match r { + SeatReleased { reference: _, id: _ } => true + SeatNotHeld { reference: _, wire: _ } => false + SeatReleaseContended { attempts: _ } => false + SeatReleaseRefused { step: _, reason: _ } => false + } +} + fn seat_acquisition_wire(a: SeatAcquisition) -> String { match a { SeatGranted { grant: g, generation: gen, attempts_used: n } => join(["granted ", g.reference as String, " fence=", g.fence.grant as String, "@", to_string(gen), " expires_at=", to_string(g.expires_at), " attempt=", to_string(n)], "") diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index a751587d64f..34c3d13f302 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -5,12 +5,14 @@ import std.types { Bool, Int, NonEmptyStr, String } import std.measure { Kibibyte, kibibyte, kibibyte_count } import std.algebra { trim } import product.capacity.event_chain { PartitionId } -import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, dashboard_instance_compute_root } +import gunbc.compute.host_capacity { compute_capacity_store } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, srv1_lab_dashboard_instance, dashboard_instance_compute_root } import gunbc.compute.host_capacity { - ComputeCapacitySubject, compute_capacity_subject, compute_capacity_root, compute_memory_partition, + ComputeCapacitySubject, ComputeSubjectResolution, ComputeSubjectResolved, ComputeSubjectHostUnresolved, + compute_capacity_subject, compute_capacity_root, compute_memory_partition, HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - compute_reserve_on, compute_release, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + compute_reserve_on, compute_release, compute_release_on, ComputeRelease, ComputeReleased, ComputeReleaseRefused, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, } @@ -153,17 +155,30 @@ test fn an_appropriation_larger_than_the_machine_refuses_rather_than_admitting() // deployment join, and what it must establish is the ROUTE and not only the shape: the ledger a // live instance reserves against is rooted under THAT INSTANCE'S compute root, and its partition // names that instance, so two instances on one machine cannot silently transact on one pool. -test fn the_deployment_join_roots_the_ledger_under_the_instance_compute_root() -> Bool { - let instance = srv1_live_dashboard_instance() - let subject = compute_capacity_subject(instance: instance) - let compute_root = dashboard_instance_compute_root(instance: instance) as String - starts_with(s: subject.root as String, prefix: compute_root) - && (subject.root as String) == join([compute_root, "/capacity"], "") - && string_contains(s: subject.partition as String, pattern: instance.instance_id as String) - && kibibyte_count(k: subject.appropriation) == kibibyte_count(k: subject.request) * 2 - && kibibyte_count(k: subject.request) > 0 +test fn two_instances_on_one_host_resolve_to_one_pool_at_one_root() -> Bool { + let live = srv1_live_dashboard_instance() + let lab = srv1_lab_dashboard_instance() + match compute_capacity_subject(instance: live) { + ComputeSubjectHostUnresolved { host: _ } => false + ComputeSubjectResolved { subject: a } => + match compute_capacity_subject(instance: lab) { + ComputeSubjectHostUnresolved { host: _ } => false + ComputeSubjectResolved { subject: b } => + (live.instance_id as String) != (lab.instance_id as String) + && (live.instance_root as String) != (lab.instance_root as String) + && (live.host_identity as String) == (lab.host_identity as String) + && (a.partition as String) == (b.partition as String) + && (a.root as String) == (b.root as String) + && string_contains(s: a.partition as String, pattern: live.host_identity as String) + && !string_contains(s: a.partition as String, pattern: lab.instance_id as String) + && (a.root as String) == join([dashboard_instance_compute_root(instance: live) as String, "/capacity"], "") + && kibibyte_count(k: a.appropriation) == kibibyte_count(k: a.request) * 2 + && kibibyte_count(k: a.request) > 0 + } + } } + // A LEDGER REFUSAL IS NOT A CAPACITY REFUSAL, and this cell exists because the two were conflated in // production and the conflation was only visible on a real host. The reservation reference names one // HOLD; re-using it is a DuplicateReference from the encumbrance ledger, which the seat carrier @@ -183,3 +198,57 @@ test fn a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool && !is_capacity_full(r: again) && reserved_amount(r: other) == one_gibibyte_in_kibibytes } + +// THE DOUBLE PROMISE, AND IT IS THE CELL THE COMPARE-AND-SET COULD NOT SUPPLY. Two requests whose +// SUM exceeds what the machine says is free, against a pool whose declared appropriation is large +// enough for both: the first admits, and the second must be refused because the pool's committed +// total and the host's own reading are adjudicated in ONE step. Before that, each request compared +// itself alone against the reading, both passed, and both fit the appropriation -- so both admitted +// and the host was promised memory it does not have. The linearization was never the defect. +// +// The appropriation here is deliberately FOUR requests wide, so nothing about the declared share +// can explain the refusal; only the machine's reading can. The backing ceiling is derived from the +// live observation, so the fixture states the relation rather than a number: it asks for slightly +// more than half of what the host reports free, twice. +test fn two_requests_summing_past_the_observed_headroom_cannot_both_admit() -> Bool { + match observe_host_memory() { + HostMemoryUnobserved { detail: _ } => false + HostMemoryObserved { total: _, available: available } => { + let each = kibibyte_count(k: available) / 2 + 1024 + let subject = subject_at(root: fresh_root(), appropriation_kib: each * 4, request_kib: each) + let first = compute_reserve_on(subject: subject, reference: "wet-headroom-a" as NonEmptyStr, term_seconds: 60) + let second = compute_reserve_on(subject: subject, reference: "wet-headroom-b" as NonEmptyStr, term_seconds: 60) + reserved_amount(r: first) == each + && is_capacity_full(r: second) + && each * 2 > kibibyte_count(k: available) + && each * 4 > each * 2 + } + } +} + +// A WEDGED SEAT CAN ACTUALLY BE RECOVERED, BY REFERENCE, BY SOMEONE WHO NEVER HELD THE RESERVATION. +// Under QuiescenceRequired a term frees nothing, so this route is the ONLY way capacity returns +// after a release fails to land -- and until review 69595 the module claimed it in prose while no +// declaration reached it. The cell holds the whole appropriation, releases it with nothing but the +// reference (the reservation value is deliberately not used), and shows the capacity is admissible +// again. +// +// AND A REFERENCE THE POOL IS NOT HOLDING WRITES NOTHING. That is the sharper half: an unheld +// reference appended anyway would not fail at the append, it would fail at every later READ, when +// the fold hits UnknownEncumbrance and refuses the WHOLE partition. So the refusal is asserted +// BESIDE a subsequent successful reservation, which is what establishes the ledger is still +// readable after the bad attempt. +test fn a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing() -> Bool { + let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes, request_kib: one_gibibyte_in_kibibytes) + let store = compute_capacity_store(subject: subject) + let held = compute_reserve_on(subject: subject, reference: "wet-wedged" as NonEmptyStr, term_seconds: 60) + let full = compute_reserve_on(subject: subject, reference: "wet-blocked" as NonEmptyStr, term_seconds: 60) + let bogus = compute_release_on(store: store, partition: subject.partition, appropriation: subject.appropriation, reference: "wet-never-held" as NonEmptyStr, reason: "operator recovery" as NonEmptyStr) + let recovered = compute_release_on(store: store, partition: subject.partition, appropriation: subject.appropriation, reference: "wet-wedged" as NonEmptyStr, reason: "operator recovery" as NonEmptyStr) + let readmitted = compute_reserve_on(subject: subject, reference: "wet-after-recovery" as NonEmptyStr, term_seconds: 60) + reserved_amount(r: held) == one_gibibyte_in_kibibytes + && is_capacity_full(r: full) + && !released(r: bogus) + && released(r: recovered) + && reserved_amount(r: readmitted) == one_gibibyte_in_kibibytes +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 2ee01ca1795..6675dbf5577 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -244,6 +244,18 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // probe failed without output rather than running its payload. fn local_repo_wet_schedule() -> List { [ + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing", + expectation: ExpectedToHold {} + }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "two_requests_summing_past_the_observed_headroom_cannot_both_admit" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "two_requests_summing_past_the_observed_headroom_cannot_both_admit", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", @@ -269,9 +281,9 @@ fn local_repo_wet_schedule() -> List { expectation: ExpectedToHold {} }, WetScheduledClaim { - identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "the_deployment_join_roots_the_ledger_under_the_instance_compute_root" }, + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "two_instances_on_one_host_resolve_to_one_pool_at_one_root" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", - function: "the_deployment_join_roots_the_ledger_under_the_instance_compute_root", + function: "two_instances_on_one_host_resolve_to_one_pool_at_one_root", expectation: ExpectedToHold {} }, WetScheduledClaim { From 9172cf7da57abc1fc6bbced1b028663a77b41b8a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 17:11:20 +0000 Subject: [PATCH 06/28] Quiescence is observed, obligations are per attempt, and the admission cells supply their own inputs Side-chat review of #11962 at 725d9c58. All three blockers verified against the code first; all three were real. 1. QUIESCENCE WAS DECLARED AND NEVER OBSERVED. compute_release ran unconditionally on what compute_run_leased returned, and that is the exit of `systemd-run --wait` -- a LAUNCHER, not the unit's parent. A launcher killed, timed out, or disconnected from the bus yields ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, so the seat's memory would be handed to the next arrival on top of a live cargo. product.capacity.lease already models this exactly (QuiescenceFact) and this provider was not consulting it. Settlement and recovery now both consume a termination decision bound to the actual unit, read through the existing systemd.Systemctl.ShowUserProperty. Only two arms free capacity: the manager has no record of the unit (never started, or ran and was collected -- the same fact for this decision), or it reports it stopped. Running keeps the charge, and so does UNOBSERVED, which is the arm a fail-open would swallow and the one a lost launcher produces. An ActiveState this fold has never seen is unobserved, not permission. The recovery CLI observes it too: an operator releasing "because the worker looked dead" was asserting the one fact nothing checked. A running unit is refused by name with the state that refused it. 2. OBLIGATIONS WERE KEYED BY IDENTITY WHILE RESERVATIONS ARE KEYED BY ATTEMPT, so a retry overwrote the first obligation and the first seat became unrecoverable -- the reference that was the only way to release it was gone. One obligation file per attempt, named by the reference, created exclusively so a second writer cannot replace one. A retry now CONSUMES that lifecycle state: an identity with an outstanding obligation refuses by name and points at the release verb rather than reserving on top of a charged seat and publishing over the record carrying it. And the producer lease is dropped LAST -- it was deleted before the outcome was written, so the exclusion held for the name and not for the publication it exists to protect. 3. THE HEADROOM FIXTURE DERIVED ITSELF FROM THE THING UNDER TEST. appropriation = 4 x (available/2 + 1MiB) exceeds MemTotal on a lightly loaded host, so the cell refused before reaching the comparison it existed to make -- and on a busy host it passed for unrelated reasons. It was flaky in both directions. compute_reserve_against takes the observation as a value and compute_reserve supplies the real one, so the admission cells state the relation exactly: each request fits available, two exceed it, two fit the appropriation, and the appropriation fits MemTotal. Not a skip -- the pairing obligation is discharged by the_real_observation_reads_this_machine, which asserts the real producer reads THIS machine, and by the srv1 receipts. Also, per the accepted (A): the relation is described as admission against an observed MemAvailable ESTIMATE, and explicitly not a no-OOM guarantee. The unit's MemoryMax bounds one workload; this stops the POOL from over-promising, and conflating the two would be the rung inflation DESIGN 4b forbids. Eight wet cells and the termination fold all pass; gunbc compile on the provider closure: 0 blocking errors. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 69 +++-- dag/gunbc/compute/work_provider_local.dag | 258 +++++++++++++++--- .../host_capacity_wet_witness_test.dag | 78 +++--- .../compute/work_request_witness_test.dag | 34 +++ src/v2/workflow/local_repo_wet_terminal.dag | 6 + 5 files changed, 348 insertions(+), 97 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index f54b9e10398..51ac97b922a 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -216,6 +216,14 @@ fn compute_lease_policy(term_seconds: Nat) -> LeasePolicy { // actually has" is one predicate evaluated in one CAS'd step, and the second 26 GiB request is // refused as a full pool because 26 + 26 > 30. // +// WHAT THIS ADMITS AGAINST IS AN ESTIMATE, AND IT IS NOT A NO-OOM GUARANTEE. MemAvailable is the +// kernel's own ESTIMATE of what a new workload could obtain without swapping, it moves between the +// reading and the grant, and nothing here constrains the processes outside this pool. So the claim +// this relation supports is "this pool does not knowingly promise more than the machine reported", +// which is a real property and is strictly weaker than "no admitted workload will be OOM-killed". +// The unit's own MemoryMax is what bounds a single workload; this is what stops the POOL from +// over-promising. Conflating the two would be the rung inflation DESIGN 4b forbids. +// // WORST-CASE-SOUND, AND CONSERVATIVE, AND SAYING SO. MemAvailable already excludes whatever the // pool's outstanding grants have actually touched, so a grant that is fully resident is counted // twice -- once in the machine's reading and once in the committed total. That errs toward refusing @@ -270,30 +278,53 @@ fn compute_reserve(instance: HostDashboardInstance, reference: NonEmptyStr, term } } +// THE DECISION TAKES THE OBSERVATION AS A VALUE, AND THE DOOR ABOVE TAKES THE READING. +// +// It used to read /proc/meminfo itself, which made the admission arithmetic untestable except +// against whatever the host happened to be doing: the headroom control had to DERIVE its fixture +// from the live reading, and a fixture derived from the thing under test is not a fixture. It was +// wrong in both directions -- on a lightly loaded host the derived appropriation exceeded MemTotal +// and the cell refused before reaching the comparison it existed to make, and on a busy one it +// passed for reasons unrelated to the relation (side-chat review of gunbc#11962 at 725d9c58). +// +// So the boundary is here: this fold decides admission from a supplied observation, and +// compute_reserve supplies the real one. DESIGN 3's pairing obligation is discharged by an +// inhabitance claim that observe_host_memory really produces a well-formed reading of this machine, +// and by the srv1 receipts, where the whole path runs for real. fn compute_reserve_on(subject: ComputeCapacitySubject, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { match observe_host_memory() { HostMemoryUnobserved { detail: d } => ComputeReservationRefused { detail: join(["the host's memory could not be observed, so nothing was reserved: ", d], "") } HostMemoryObserved { total: total, available: available } => - if kibibyte_count(k: subject.appropriation) > kibibyte_count(k: total) { - ComputeReservationRefused { - detail: join([ - "the compute appropriation is ", to_string(value: kibibyte_count(k: subject.appropriation)), - " KiB and this host has ", to_string(value: kibibyte_count(k: total)), - " KiB installed, so the share was never backed by the machine", - ], ""), - } - } else if kibibyte_count(k: available) < kibibyte_count(k: subject.request) { - ComputeHostBelowFloor { requested: subject.request, available: available } - } else { - match compute_capacity_store_ensure(subject: subject) { - FabricStorageRootRefused { area: a, detail: d } => ComputeReservationRefused { detail: join(["the capacity ledger could not be prepared at ", a, ": ", d], "") } - FabricStorageRootReady => - match now_epoch_seconds() { - Absent => ComputeReservationRefused { detail: "the clock could not be read as epoch seconds" } - Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds, ceiling: compute_backing_ceiling(subject: subject, available: available)) - } + compute_reserve_against(subject: subject, reference: reference, term_seconds: term_seconds, total: total, available: available) + } +} + +fn compute_reserve_against( + subject: ComputeCapacitySubject, + reference: NonEmptyStr, + term_seconds: Nat, + total: Kibibyte, + available: Kibibyte, +) -> ComputeReservation { + if kibibyte_count(k: subject.appropriation) > kibibyte_count(k: total) { + ComputeReservationRefused { + detail: join([ + "the compute appropriation is ", to_string(value: kibibyte_count(k: subject.appropriation)), + " KiB and this host has ", to_string(value: kibibyte_count(k: total)), + " KiB installed, so the share was never backed by the machine", + ], ""), + } + } else if kibibyte_count(k: available) < kibibyte_count(k: subject.request) { + ComputeHostBelowFloor { requested: subject.request, available: available } + } else { + match compute_capacity_store_ensure(subject: subject) { + FabricStorageRootRefused { area: a, detail: d } => ComputeReservationRefused { detail: join(["the capacity ledger could not be prepared at ", a, ": ", d], "") } + FabricStorageRootReady => + match now_epoch_seconds() { + Absent => ComputeReservationRefused { detail: "the clock could not be read as epoch seconds" } + Present { value: now } => compute_seat(subject: subject, reference: reference, now: now, term_seconds: term_seconds, ceiling: compute_backing_ceiling(subject: subject, available: available)) } - } + } } } diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 0fb60514c62..6a44e60a36c 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -7,9 +7,11 @@ import std.measure { byte_size_count } import extdeps.filesystem.filesystem_io { Filesystem } import gunbc.output_policy { OutcomeIsData } import extdeps.gunbc +import extdeps.shell import extdeps.git { shape_git_worktree_add_detached_argv } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex, git_object_id_wire_hex } import extdeps.systemd.systemd_run { systemd_run_user_wait_arguments, systemd_run_property } +import extdeps.systemd.systemctl import extdeps.cache.sccache { sccache_installed_binary_path } import gunbc.clock_read { clock_now_probed_at_or_unknown } import gunbc.roadmap_dashboard_instance { @@ -66,7 +68,7 @@ type ComputeLayout { log_path: String outcome_path: String consumption_path: String - obligation_path: String + obligations_dir: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -84,7 +86,7 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), consumption_path: join([work_dir, "/consumption"], ""), - obligation_path: join([work_dir, "/unreleased-reservation"], ""), + obligations_dir: join([root, "/obligations"], ""), } } @@ -236,6 +238,75 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden } } +// ── WHETHER THE UNIT IS STILL USING THE MEMORY ──────────────────────────────────────────────── +// +// THE RELEASE LAW SAID "QUIESCENCE REQUIRED" AND NOTHING OBSERVED QUIESCENCE. That was the gap: +// compute_release ran unconditionally on whatever compute_run_leased returned, and what it returns +// is the exit of `systemd-run --wait`, which is NOT the unit. systemd-run is a launcher, not the +// unit's parent -- the transient service is owned by the manager -- so a launcher that is killed, +// times out, or loses its bus connection yields ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, and +// the seat's memory would be handed to the next arrival on top of a live cargo. product.capacity.lease +// already models this exactly (QuiescenceFact = Quiescent | QuiescenceUnobserved) and the corpus was +// simply not consulting it (side-chat review of gunbc#11962 at 725d9c58). +// +// SO A RELEASE NOW CONSUMES A TERMINATION DECISION BOUND TO THE ACTUAL UNIT, and only two arms may +// free capacity: the manager knows nothing about the unit, or it knows it has stopped. Running and +// UNOBSERVED both keep the commitment -- an unobserved unit is the case a fail-open would swallow, +// and it is the one that matters, because "I could not ask" is exactly what a lost launcher looks +// like. +// +// AN EMPTY ActiveState IS QUIESCENT AND THAT IS NOT A GUESS. `systemctl --user show` prints an empty +// value for a unit the manager has no record of, which for a transient --wait unit means it ran and +// was collected, and for a unit that never started means there is nothing consuming memory. Both are +// the same fact for THIS decision -- no process of ours is holding that memory -- and the reviewer's +// own formulation admits both: never started, or demonstrated quiescent. +type UnitTermination + = UnitQuiescent { state: String } + | UnitStillActive { state: String } + | UnitTerminationUnobserved { detail: String } + +data unit_active_state_property: NonEmptyStr = "ActiveState" as NonEmptyStr + +// THE CLOSED SET IS UPSTREAM'S (systemd.unit(5) ActiveState), and an unrecognised word is UNOBSERVED +// rather than assumed dead. A new systemd state that this fold has never seen must not be read as +// permission to take the memory back. +fn unit_state_termination(state: String) -> UnitTermination { + let s = trim(s: state) + if s == "" || s == "inactive" || s == "failed" { + UnitQuiescent { state: s } + } else if s == "active" || s == "activating" || s == "deactivating" || s == "reloading" || s == "maintenance" { + UnitStillActive { state: s } + } else { + UnitTerminationUnobserved { detail: join(["systemd reported an ActiveState this provider does not model: ", s], "") } + } +} + +fn observe_unit_termination(identity_hex: String) -> UnitTermination { + let unit = compute_unit_name(identity_hex: identity_hex) + let r = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_active_state_property) + if !r.success { + UnitTerminationUnobserved { detail: join(["systemctl --user show ", unit as String, " could not be read"], "") } + } else { + unit_state_termination(state: r.value) + } +} + +fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { + match t { + UnitQuiescent { state: _ } => true + UnitStillActive { state: _ } => false + UnitTerminationUnobserved { detail: _ } => false + } +} + +fn unit_termination_wire(t: UnitTermination) -> String { + match t { + UnitQuiescent { state: s } => join(["quiescent (ActiveState=", if s == "" { "" } else { s }, ")"], "") + UnitStillActive { state: s } => join(["STILL ACTIVE (ActiveState=", s, "), so the seat stays charged"], "") + UnitTerminationUnobserved { detail: d } => join(["termination unobserved, so the seat stays charged: ", d], "") + } +} + // Return declared outputs by content: hash each with git (a blob object id is a content identity // every subject already uses) and install it into the identity's store directory. An absent or // unreadable output is a mismatch, never a success with a hole. @@ -299,6 +370,10 @@ fn compute_basename(path: String) -> String { // under-admits until the obligation is discharged rather than handing the same bytes to the next // arrival. Discharging it is an explicit release naming why -- the same entry completion and // cancellation take. +// THE PRODUCER LEASE IS DROPPED LAST, AND THE ORDER IS THE POINT. It used to be deleted before the +// outcome and the obligation were written, so a second producer could take the lease and start +// against a work directory whose record was still being published -- the exclusion held for the +// name and not for the publication it exists to protect. fn compute_run_and_settle( instance: HostDashboardInstance, commit: String, @@ -308,26 +383,50 @@ fn compute_run_and_settle( op: WorkOperation, dependency_store: String, granted: Kibibyte, + reference: NonEmptyStr, reservation: ComputeReservation, ) -> ProvideResult { let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) let standing = compute_capacity_standing(instance: instance) - let returned = compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome)], "") as NonEmptyStr) + let termination = observe_unit_termination(identity_hex: identity) + let returned = if unit_termination_frees_capacity(t: termination) { + compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome), ", unit ", unit_termination_wire(t: termination)], "") as NonEmptyStr) + } else { + ComputeReleaseRefused { detail: join(["the seat was NOT released: ", unit_termination_wire(t: termination)], "") } + } let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) - let released = Filesystem.Delete(path: layout.lease_path) - let notes = join([ - "; producer lease released: ", if released.success { "yes" } else { released.error }, - "; consumption recorded: ", if measured.success { "yes" } else { measured.error }, - ], "") - if compute_release_landed(r: returned) { - compute_settled_record( - layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, - lease_released: released.success, notes: notes) + let notes = join(["; consumption recorded: ", if measured.success { "yes" } else { measured.error }], "") + let settled = if compute_release_landed(r: returned) { + compute_settled_record(layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, notes: notes) } else { compute_unreleased_record( layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, - reservation: reservation, returned: returned, notes: notes) + reservation: reservation, returned: returned, reference: reference, termination: termination, notes: notes) } + let dropped = Filesystem.Delete(path: layout.lease_path) + if dropped.success { settled } else { compute_lease_undropped(result: settled, layout: layout, identity: identity, subject: subject, op: op, error: dropped.error) } +} + +fn compute_lease_undropped( + result: ProvideResult, + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + error: String, +) -> ProvideResult { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { + detail: join([ + "the record was published as ", provide_result_ending(r: result), + " but the producer lease at ", layout.lease_path, " could not be dropped: ", error, + "; the next request for this identity will refuse as ProducerInFlight until it is removed", + ], ""), + }, + }, + subject: subject, op: op) } // THE ORDINARY ENDING: the reservation came back, so the record is the run's own ending and the @@ -338,12 +437,11 @@ fn compute_settled_record( subject: WorkSubject, op: WorkOperation, outcome: WorkOutcome, - lease_released: Bool, notes: String, ) -> ProvideResult { let document = work_outcome_wire(o: outcome, subject: subject, op: op) let written = Filesystem.Write(path: layout.outcome_path, content: document) - if written.success && lease_released { + if written.success { ProvideFresh { outcome: outcome, document: document } } else { compute_fresh( @@ -353,7 +451,7 @@ fn compute_settled_record( detail: join([ "the work ended ", work_outcome_ending(o: outcome), " and its reservation was released, but the record could not be completed -- outcome written: ", - if written.success { "yes" } else { written.error }, notes, + written.error, notes, ], ""), }, }, @@ -371,19 +469,27 @@ fn compute_unreleased_record( outcome: WorkOutcome, reservation: ComputeReservation, returned: ComputeRelease, + reference: NonEmptyStr, + termination: UnitTermination, notes: String, ) -> ProvideResult { let obligation = join([ - "unreleased reservation: ", compute_reservation_wire(r: reservation), - "; release attempt: ", compute_release_wire(r: returned), - "; the work itself ended ", work_outcome_ending(o: outcome), - " and any declared outputs are in the store; the seat stays held until an explicit release ", - "discharges it, so this host under-admits rather than promising the same memory twice", + "identity=", identity, "\n", + "reference=", reference as String, "\n", + "unit=", unit_termination_wire(t: termination), "\n", + "work_ending=", work_outcome_ending(o: outcome), "\n", + "reservation=", compute_reservation_wire(r: reservation), "\n", + "release_attempt=", compute_release_wire(r: returned), "\n", ], "") - let recorded = Filesystem.Write(path: layout.obligation_path, content: join([obligation, "\n"], "")) + let prepared = compute_ensure_dir_at(path: layout.obligations_dir) + let recorded = Filesystem.WriteCreateNew(path: compute_obligation_path(layout: layout, reference: reference), content: obligation) let cause = ComputeCapacityLedgerUnavailable { detail: join([ - obligation, "; obligation recorded: ", if recorded.success { layout.obligation_path } else { recorded.error }, notes, + "the seat for ", reference as String, " is still charged; the work itself ended ", + work_outcome_ending(o: outcome), " and any declared outputs are in the store. ", + compute_release_wire(r: returned), + ". obligation recorded: ", if prepared && recorded.success { compute_obligation_path(layout: layout, reference: reference) } else { recorded.error }, + notes, ], ""), } let document = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) @@ -395,7 +501,20 @@ fn compute_unreleased_record( } } -// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease +// ONE OBLIGATION FILE PER ATTEMPT, NOT PER IDENTITY. The obligation used to live at a path derived +// from the work identity while the reservation it describes is keyed by the ATTEMPT, so a retry +// overwrote the first obligation and the first seat became unrecoverable -- the reference that was +// the only way to release it was gone. The reference is the name of the hold, so it is the name of +// the obligation, and the create is EXCLUSIVE so a second writer cannot silently replace one. +fn compute_obligation_path(layout: ComputeLayout, reference: NonEmptyStr) -> String { + join([layout.obligations_dir, "/", reference as String], "") +} + +fn compute_ensure_dir_at(path: String) -> Bool { + shell.Mkdir.Parents(path: path as FilePath).success +} + +// THE ORDER THE TWO HOLDS ARE TAKEN IN// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the // reservation is a QUANTITY on the host, and no exclusive create of any name settles that. The // lease is taken first because a request that then loses on capacity releases a name it holds @@ -438,11 +557,28 @@ fn compute_provide_leaf( } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { + let outstanding = compute_outstanding_obligation(layout: layout, identity: identity) + if outstanding != "" { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: ComputeCapacityLedgerUnavailable { + detail: join([ + "this identity has an unreleased reservation from an earlier attempt and will not take a second one: ", + outstanding, + ". Discharge it with the release verb once the unit is demonstrably gone; until then a retry would ", + "reserve on top of a seat that is still charged and could publish an outcome over the obligation.", + ], ""), + }, + }, + subject: subject, op: op) + } else { let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) } else { - let reservation = compute_reserve(instance: instance, reference: compute_reservation_reference(identity: identity), term_seconds: compute_reservation_term_seconds) + let reference = compute_reservation_reference(identity: identity) + let reservation = compute_reserve(instance: instance, reference: reference, term_seconds: compute_reservation_term_seconds) match reservation { ComputeCapacityFull { wire: w } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) @@ -453,9 +589,34 @@ fn compute_provide_leaf( ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => compute_run_and_settle( instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, - dependency_store: dependency_store, granted: granted, reservation: reservation) + dependency_store: dependency_store, granted: granted, reference: reference, reservation: reservation) } } + } + } +} + +// WHAT A RETRY MUST LOOK AT BEFORE IT RESERVES AGAIN, and the reason this is a read rather than a +// convention. An obligation names a seat that is still charged; a retry of the same identity that +// ignored it would take a SECOND reservation while the first is outstanding, and -- because the +// outcome document is keyed by identity -- could publish its own ending over the record that was +// carrying the obligation. So the lifecycle state is consumed, not remembered: the obligations +// directory is listed, and any entry naming this identity refuses the request by name and points at +// the release verb. Empty means nothing outstanding; an unreadable directory means nothing +// outstanding either, because the directory does not exist until the first obligation is written. +fn compute_outstanding_obligation(layout: ComputeLayout, identity: String) -> String { + let listing = Filesystem.List(path: layout.obligations_dir) + if !listing.success { + "" + } else { + fold(split(s: listing.entries, delimiter: "\n"), init: "", f: (acc, entry) => + if acc != "" || trim(s: entry) == "" { + acc + } else if !starts_with(s: trim(s: entry), prefix: identity) { + acc + } else { + join([layout.obligations_dir, "/", trim(s: entry)], "") + }) } } @@ -636,28 +797,45 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent // the pool is not holding writes nothing and exits non-zero saying so, which is the answer a // mistyped reference must get -- appending a release for an unheld reference would make the // partition unfoldable for every later reader. -fn compute_release_cli(repo_root: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { +fn compute_release_cli(repo_root: String, identity: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { match dashboard_instance_for_repo_root(repo_root: repo_root) { DashboardInstanceRootUnknown { repo_root: r } => ExitFailure { code: 2, reason: join(["compute: no dashboard instance is rooted at ", r], "") } DashboardInstanceResolved { instance } => - if reference == "" || reason == "" { - ExitFailure { code: 2, reason: "compute: release needs a non-empty reference and a reason naming why the seat is being given back" } + if identity == "" || reference == "" || reason == "" { + ExitFailure { code: 2, reason: "compute: release needs the identity and the reference the obligation file names, and a reason saying why the seat is being given back" } } else { - let released = compute_release_reference(instance: instance, reference: reference as NonEmptyStr, reason: reason as NonEmptyStr) - let wire = compute_release_cli_wire(r: released) - let receipt = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) - if !receipt.success { - ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; release was ", wire], "") } - } else { - match released { - ComputeReleased { partition: _, reference: _, reason: _ } => ExitSuccess - ComputeReleaseRefused { detail: _ } => ExitFailure { code: 1, reason: join(["compute release refused: ", wire], "") } - } - } + compute_release_observed(instance: instance, identity: identity, reference: reference, reason: reason, receipt_path: receipt_path) } } } +// THE RECOVERY DOOR OBSERVES THE UNIT TOO, and this is the half a reason string cannot supply. An +// operator releasing "because the worker looked dead" is asserting exactly the fact that nothing +// checked -- and releasing a seat whose cargo is still resident hands its memory to the next +// arrival. So recovery consumes the same termination decision settlement does: only a unit the +// manager reports stopped, or has no record of, may be released. A running unit is refused BY NAME +// with the state that refused it, so the operator's next move is to stop the unit rather than to +// argue with the tool. +fn compute_release_observed(instance: HostDashboardInstance, identity: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { + let termination = observe_unit_termination(identity_hex: identity) + let wire = if unit_termination_frees_capacity(t: termination) { + compute_release_cli_wire(r: compute_release_reference( + instance: instance, + reference: reference as NonEmptyStr, + reason: join([reason, " (unit ", unit_termination_wire(t: termination), ")"], "") as NonEmptyStr)) + } else { + join(["refused: ", unit_termination_wire(t: termination), "; stop the unit first, then release"], "") + } + let receipt = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) + if !receipt.success { + ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; release was ", wire], "") } + } else if starts_with(s: wire, prefix: "released ") { + ExitSuccess + } else { + ExitFailure { code: 1, reason: join(["compute release refused: ", wire], "") } + } +} + fn compute_release_cli_wire(r: ComputeRelease) -> String { match r { ComputeReleased { partition: p, reference: ref_, reason: why } => diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index 34c3d13f302..6051e698a4f 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -12,7 +12,7 @@ import gunbc.compute.host_capacity { compute_capacity_subject, compute_capacity_root, compute_memory_partition, HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - compute_reserve_on, compute_release, compute_release_on, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + compute_reserve_on, compute_reserve_against, compute_release, compute_release_on, ComputeRelease, ComputeReleased, ComputeReleaseRefused, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, } @@ -117,19 +117,15 @@ test fn competing_reservations_cannot_over_reserve_and_a_release_returns_the_cap // The request is the machine's own installed memory, which is strictly above what it reports // available on any host that is running anything at all -- including the one running this witness. test fn a_request_the_machine_cannot_back_refuses_below_the_live_floor() -> Bool { - match observe_host_memory() { - HostMemoryUnobserved { detail: _ } => false - HostMemoryObserved { total: total, available: available } => { - let installed = kibibyte_count(k: total) - let subject = subject_at(root: fresh_root(), appropriation_kib: installed, request_kib: installed) - kibibyte_count(k: available) < installed - && (match compute_reserve_on(subject: subject, reference: "wet-floor" as NonEmptyStr, term_seconds: 60) { - ComputeHostBelowFloor { requested: rq, available: av } => kibibyte_count(k: rq) == installed && kibibyte_count(k: av) > 0 - ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false - ComputeCapacityFull { wire: _ } => false - ComputeReservationRefused { detail: _ } => false - }) - } + let total = kibibyte(count: 64 * one_gibibyte_in_kibibytes) + let available = kibibyte(count: 4 * one_gibibyte_in_kibibytes) + let subject = subject_at(root: fresh_root(), appropriation_kib: 52 * one_gibibyte_in_kibibytes, request_kib: 26 * one_gibibyte_in_kibibytes) + match compute_reserve_against(subject: subject, reference: "wet-floor" as NonEmptyStr, term_seconds: 60, total: total, available: available) { + ComputeHostBelowFloor { requested: rq, available: av } => + kibibyte_count(k: rq) == 26 * one_gibibyte_in_kibibytes && kibibyte_count(k: av) == 4 * one_gibibyte_in_kibibytes + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeReservationRefused { detail: _ } => false } } @@ -137,17 +133,13 @@ test fn a_request_the_machine_cannot_back_refuses_below_the_live_floor() -> Bool // presenting hours later as a build the kernel killed. It is refused ahead of the floor check, // because a share that was never backed is wrong whatever this instant's availability happens to be. test fn an_appropriation_larger_than_the_machine_refuses_rather_than_admitting() -> Bool { - match observe_host_memory() { - HostMemoryUnobserved { detail: _ } => false - HostMemoryObserved { total: total, available: _ } => { - let subject = subject_at(root: fresh_root(), appropriation_kib: kibibyte_count(k: total) + 1, request_kib: one_gibibyte_in_kibibytes) - match compute_reserve_on(subject: subject, reference: "wet-oversized" as NonEmptyStr, term_seconds: 60) { - ComputeReservationRefused { detail: d } => string_contains(s: d, pattern: "never backed by the machine") - ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false - ComputeCapacityFull { wire: _ } => false - ComputeHostBelowFloor { requested: _, available: _ } => false - } - } + let total = kibibyte(count: 64 * one_gibibyte_in_kibibytes) + let subject = subject_at(root: fresh_root(), appropriation_kib: 64 * one_gibibyte_in_kibibytes + 1, request_kib: one_gibibyte_in_kibibytes) + match compute_reserve_against(subject: subject, reference: "wet-oversized" as NonEmptyStr, term_seconds: 60, total: total, available: total) { + ComputeReservationRefused { detail: d } => string_contains(s: d, pattern: "never backed by the machine") + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => false + ComputeCapacityFull { wire: _ } => false + ComputeHostBelowFloor { requested: _, available: _ } => false } } @@ -211,27 +203,37 @@ test fn a_reused_reference_is_a_ledger_refusal_and_never_reported_as_a_full_pool // live observation, so the fixture states the relation rather than a number: it asks for slightly // more than half of what the host reports free, twice. test fn two_requests_summing_past_the_observed_headroom_cannot_both_admit() -> Bool { + let total = kibibyte(count: 64 * one_gibibyte_in_kibibytes) + let available = kibibyte(count: 30 * one_gibibyte_in_kibibytes) + let subject = subject_at(root: fresh_root(), appropriation_kib: 52 * one_gibibyte_in_kibibytes, request_kib: 26 * one_gibibyte_in_kibibytes) + let first = compute_reserve_against(subject: subject, reference: "wet-headroom-a" as NonEmptyStr, term_seconds: 60, total: total, available: available) + let second = compute_reserve_against(subject: subject, reference: "wet-headroom-b" as NonEmptyStr, term_seconds: 60, total: total, available: available) + kibibyte_count(k: subject.request) <= kibibyte_count(k: available) + && kibibyte_count(k: subject.request) * 2 > kibibyte_count(k: available) + && kibibyte_count(k: subject.appropriation) >= kibibyte_count(k: subject.request) * 2 + && kibibyte_count(k: subject.appropriation) <= kibibyte_count(k: total) + && reserved_amount(r: first) == kibibyte_count(k: subject.request) + && is_capacity_full(r: second) +} + +// THE PAIRING OBLIGATION FOR EVERY SUPPLIED OBSERVATION ABOVE (DESIGN 3): the real producer emits a +// reading of THIS machine, and it is a reading rather than a shape -- installed memory is positive, +// availability does not exceed it, and the two are not the same number on a host that is running +// anything. A fixture cannot establish that and the supplied cells do not claim to. +test fn the_real_observation_reads_this_machine() -> Bool { match observe_host_memory() { HostMemoryUnobserved { detail: _ } => false - HostMemoryObserved { total: _, available: available } => { - let each = kibibyte_count(k: available) / 2 + 1024 - let subject = subject_at(root: fresh_root(), appropriation_kib: each * 4, request_kib: each) - let first = compute_reserve_on(subject: subject, reference: "wet-headroom-a" as NonEmptyStr, term_seconds: 60) - let second = compute_reserve_on(subject: subject, reference: "wet-headroom-b" as NonEmptyStr, term_seconds: 60) - reserved_amount(r: first) == each - && is_capacity_full(r: second) - && each * 2 > kibibyte_count(k: available) - && each * 4 > each * 2 - } + HostMemoryObserved { total: total, available: available } => + kibibyte_count(k: total) > 0 + && kibibyte_count(k: available) > 0 + && kibibyte_count(k: available) < kibibyte_count(k: total) } } // A WEDGED SEAT CAN ACTUALLY BE RECOVERED, BY REFERENCE, BY SOMEONE WHO NEVER HELD THE RESERVATION. // Under QuiescenceRequired a term frees nothing, so this route is the ONLY way capacity returns // after a release fails to land -- and until review 69595 the module claimed it in prose while no -// declaration reached it. The cell holds the whole appropriation, releases it with nothing but the -// reference (the reservation value is deliberately not used), and shows the capacity is admissible -// again. +// declaration reached it. // // AND A REFERENCE THE POOL IS NOT HOLDING WRITES NOTHING. That is the sharper half: an unheld // reference appended anyway would not fail at the append, it would fail at every later READ, when diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 385963a5879..21b15320449 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -2,6 +2,10 @@ module test.claim.compute.work_request_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } +import gunbc.compute.work_provider_local { + UnitTermination, UnitQuiescent, UnitStillActive, UnitTerminationUnobserved, + unit_state_termination, unit_termination_frees_capacity, +} import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, @@ -74,3 +78,33 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool && (match stored_outcome_decode(text: "{\"identity\": \"abc\"}") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) && (match stored_outcome_decode(text: "not json") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) } + +// ONLY A UNIT THAT IS DEMONSTRABLY GONE MAY FREE ITS SEAT, and the arm that matters is the one in +// the middle. `systemd-run --wait` is a LAUNCHER, not the unit's parent, so a launcher that dies or +// times out returns failure while the transient service keeps running; releasing on that signal +// would hand a live cargo's memory to the next arrival. Empty and inactive and failed are all +// "nothing of ours is holding that memory" -- never started and already collected are the same fact +// for this decision. Everything systemd calls live keeps the charge, and a word this fold has never +// seen keeps it too: a future ActiveState must not read as permission to take memory back. +test fn only_a_gone_unit_frees_its_seat_and_an_unknown_state_does_not() -> Bool { + all(["", "inactive", "failed", " inactive "], s => + match unit_state_termination(state: s) { + UnitQuiescent { state: _ } => true + UnitStillActive { state: _ } => false + UnitTerminationUnobserved { detail: _ } => false + }) + && all(["active", "activating", "deactivating", "reloading", "maintenance"], s => + match unit_state_termination(state: s) { + UnitStillActive { state: _ } => true + UnitQuiescent { state: _ } => false + UnitTerminationUnobserved { detail: _ } => false + }) + && all(["quantum-superposed", "ACTIVE", "1"], s => + match unit_state_termination(state: s) { + UnitTerminationUnobserved { detail: _ } => true + UnitQuiescent { state: _ } => false + UnitStillActive { state: _ } => false + }) + && all(["", "inactive", "failed"], s => unit_termination_frees_capacity(t: unit_state_termination(state: s))) + && all(["active", "activating", "deactivating", "quantum-superposed"], s => !unit_termination_frees_capacity(t: unit_state_termination(state: s))) +} diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 6675dbf5577..6e8d09c7ff7 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -244,6 +244,12 @@ fn local_repo_wet_requirement(candidate: String) -> WetEvidenceRequirement { // probe failed without output rather than running its payload. fn local_repo_wet_schedule() -> List { [ + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "the_real_observation_reads_this_machine" }, + entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", + function: "the_real_observation_reads_this_machine", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", From 5ee0cceea8922117e09a0e86670527e20864c9e2 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 18:04:19 +0000 Subject: [PATCH 07/28] PR 1 rescoped to admission+pool: conservative release, declared discharge gap, and the memory carriers the refusals were stripping Split approved by the manager. All seven of the side chat's blockers at 9172cf7 are lifecycle/recovery; none touches admission or the pool. The argument for splitting is safety, not workload: WITHOUT a recovery route every one of those defects degrades to "the seat stays charged" -- the host under-admits and nothing is over-promised. WITH a half-built one they FREE SEATS THAT SHOULD NOT BE FREED. Shipping recovery half-built is worse than not shipping it. OUT OF THIS PR, and named rather than quietly dropped: compute_release_cli, compute_release_reference, the obligation files and the retry-blocking read. BLOCKER (1) IS NOT RECOVERY-ONLY AND IS FIXED HERE, per the manager's correction: automatic settlement releases too. ActiveState alone is not the evidence -- `failed` can coexist with live processes while a unit's stop reaches its SIGKILL timeout, and an empty ActiveState is the manager declining to answer, not authoritative absence. The decision is now four-valued over evidence the repo already models: authoritative absence from LoadState=not-found (the wire value this corpus reads positively), observed termination from the unit's own cgroup reporting populated 0, populated, and unavailable. Only the first two free capacity; unavailable keeps the charge, and that is the arm that matters, because a lost launcher, an unreadable manager and a missing cgroup all look like it. BLOCKER (7) STILL REACHED PR 1 AND IS FIXED: the lease-undropped arm handed the caller a refusal while the outcome document already said the run SUCCEEDED, then claimed the next request would refuse as ProducerInFlight -- three claims that cannot all be true, since the stored record is what the next caller attaches to. One contract now: the stored record is the result, the caller is told what the next caller will read, and the stale lease is recorded as an operational fact rather than converted into a verdict about the work. Blockers (2)-(6) are no longer reached: each was about the recovery door or the obligation files, and both are out. THE GAP IS CLASSIFIED AS THE MANAGER ASKED. No discharge route existed on main -- admission was a count of lease files and there were no reservations at all -- so this is not a rung that fell and is not a 4b(3) drop. It is a new class: gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge, rostered, with a trigger naming PR 2's CAPABILITY (termination from absence or an empty cgroup; attempt-to-unit binding; coordination with a pending producer; a settled state a retry consumes) rather than an artifact. Its remedy sentence reaches the caller through compute_stuck_reservation_remedy. review 69624, unit modeling: HostBelowMemoryFloor carried Int fields NAMED _kib while its producer already had Kibibyte and the provider called kibibyte_count to strip the carrier on the way in -- the unit modeled twice and checked nowhere. It carries Kibibyte now. And SeatRoomObserved.headroom was the boundary THIS PR moved: a bare Nat was right while the pool was fixed at seats, and generalizing fabric_seat_observe to Pool made the same field mean KiB for one caller and seats for another. SeatStandingObservation is now generic and headroom carries Measure. Seven wet cells plus the termination fold pass; gunbc compile on the provider closure: 0 blocking errors. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 54 +-- dag/gunbc/compute/work_provider_local.dag | 330 +++++++----------- dag/gunbc/compute/work_request.dag | 13 +- dag/gunbc/fabric/fabric_event_log.dag | 17 +- dag/gunbc/harness/harness_seat.dag | 4 +- ...ompute_seat_has_no_in_corpus_discharge.dag | 34 ++ .../host_capacity_wet_witness_test.dag | 27 +- .../compute/work_request_witness_test.dag | 50 ++- ...serving_d0_real_execution_witness_test.dag | 4 +- src/v2/workflow/local_repo_wet_terminal.dag | 6 - 10 files changed, 228 insertions(+), 311 deletions(-) create mode 100644 dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index 51ac97b922a..3248b01262d 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -379,40 +379,20 @@ fn compute_release(reservation: ComputeReservation, reason: NonEmptyStr) -> Comp } } -// THE RECOVERY ROUTE, AND IT EXISTS BECAUSE THE MODULE CLAIMED IT (review 69595 on gunbc#11962). +// THE ONE RELEASE PATH, CHECKED. It reads, folds, PROPOSES the release against the folded pool and +// appends only what the pool admits -- the mirror of the acquire. That check is load-bearing rather +// than defensive: a reference the pool is not holding, appended anyway, would not fail at the +// append; it would fail at every later READ, when the fold hits UnknownEncumbrance and refuses the +// WHOLE partition. // -// The prose above says a wedged seat is recovered by an explicit release naming why. Until this -// function that was FALSE: compute_release took a ComputeReservation, whose only constructor is -// compute_reserve_on, whose result lives and dies inside one compute_provide_leaf invocation. So a -// release that failed to land left an encumbrance that -- under QuiescenceRequired, which by design -// never lapses -- nothing in the corpus could ever discharge. Two of those consume the whole host -// appropriation, and every later request then gets a CAPACITY refusal on a host with its memory -// free: the exact conflation this change added a witness to prevent, arriving through a different -// door. A claimed route that no declaration reaches is the dangling consumption DESIGN 3c refuses. -// -// IT TAKES A REFERENCE, because that is all a recovering operator has: the reservation value is -// gone with the process that held it, and the reference is what the ledger and the obligation file -// both name. It is CHECKED against the folded pool before anything is appended -- a reference the -// pool is not holding writes nothing and refuses by name, because appending a release for an unheld -// reference would make the partition unfoldable for every later reader. -fn compute_release_reference(instance: HostDashboardInstance, reference: NonEmptyStr, reason: NonEmptyStr) -> ComputeRelease { - match compute_capacity_subject(instance: instance) { - ComputeSubjectHostUnresolved { host: h } => - ComputeReleaseRefused { detail: join(["no dashboard instance is declared for host ", h as String], "") } - ComputeSubjectResolved { subject: subject } => - compute_release_on( - store: compute_capacity_store(subject: subject), - partition: subject.partition, - appropriation: subject.appropriation, - reference: reference, - reason: reason, - ) - } -} - -// ONE RELEASE PATH FOR BOTH DOORS. The provider's own completion and the operator's recovery append -// the same event against the same check; a second spelling of "give the seat back" is the fork this -// module would otherwise grow the moment recovery existed. +// THE OPERATOR'S RECOVERY DOOR IS NOT HERE, and its absence is declared rather than implied. The +// discharge of a seat whose release did not land needs a termination decision bound to the unit, an +// attempt-to-unit binding that cannot release another attempt's live reservation, and a settled +// state a retry consumes -- see gunbc.recurring_failure_mode +// a_held_compute_seat_has_no_in_corpus_discharge, whose trigger names that capability. Shipping a +// recovery route with any of those missing would FREE SEATS THAT SHOULD NOT BE FREED, which is +// worse than not having one: without it every failure here keeps the charge and the host merely +// under-admits. // // THE APPROPRIATION PASSED HERE IS NOT LOAD-BEARING AND THE ZERO IS NOT A FABRICATED READING. product.capacity.pool // ruled that replay never adjudicates the ceiling, so a release is decided by whether the ledger @@ -460,7 +440,7 @@ fn compute_release_on( // committed far above the machine's own shortfall means the pool is over-reserving each class, and // the reverse means work is running outside the pool. type ComputeCapacityStanding - = ComputeCapacityStandingRead { committed: Kibibyte, ceiling: Kibibyte, headroom: Nat, observed_available: Kibibyte, observed_total: Kibibyte } + = ComputeCapacityStandingRead { committed: Kibibyte, ceiling: Kibibyte, headroom: Kibibyte, observed_available: Kibibyte, observed_total: Kibibyte } | ComputeCapacityStandingUnread { detail: String } fn compute_capacity_standing(instance: HostDashboardInstance) -> ComputeCapacityStanding { @@ -486,7 +466,7 @@ fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapac ) { SeatRoomObserved { headroom: h, generation: _ } => ComputeCapacityStandingRead { - committed: kibibyte(count: kibibyte_count(k: subject.appropriation) - h), + committed: kibibyte(count: kibibyte_count(k: subject.appropriation) - kibibyte_count(k: h)), ceiling: subject.appropriation, headroom: h, observed_available: available, @@ -496,7 +476,7 @@ fn compute_capacity_standing_on(subject: ComputeCapacitySubject) -> ComputeCapac ComputeCapacityStandingRead { committed: subject.appropriation, ceiling: subject.appropriation, - headroom: 0, + headroom: kibibyte(count: 0), observed_available: available, observed_total: total, } @@ -523,7 +503,7 @@ fn compute_capacity_standing_wire(s: ComputeCapacityStanding) -> String { ComputeCapacityStandingRead { committed: c, ceiling: cl, headroom: h, observed_available: av, observed_total: tt } => join([ "committed ", to_string(value: kibibyte_count(k: c)), " KiB of ", to_string(value: kibibyte_count(k: cl)), - " KiB (headroom ", to_string(value: h), " KiB); host reports ", to_string(value: kibibyte_count(k: av)), + " KiB (headroom ", to_string(value: kibibyte_count(k: h)), " KiB); host reports ", to_string(value: kibibyte_count(k: av)), " KiB available of ", to_string(value: kibibyte_count(k: tt)), " KiB installed", ], "") ComputeCapacityStandingUnread { detail: d } => join(["unread: ", d], "") diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 6a44e60a36c..3e19b143212 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -12,6 +12,10 @@ import extdeps.git { shape_git_worktree_add_detached_argv } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex, git_object_id_wire_hex } import extdeps.systemd.systemd_run { systemd_run_user_wait_arguments, systemd_run_property } import extdeps.systemd.systemctl +import extdeps.linux.cgroup_v2 { + cgroup_v2_mount_point, cgroup_v2_events_path, + CgroupEventsDecoding, CgroupEventsDecoded, CgroupEventsUnparseable, decode_cgroup_events_populated, +} import extdeps.cache.sccache { sccache_installed_binary_path } import gunbc.clock_read { clock_now_probed_at_or_unknown } import gunbc.roadmap_dashboard_instance { @@ -25,7 +29,7 @@ import gunbc.roadmap_execution_contract { import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size } import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_release_reference, compute_reservation_wire, + ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, } import gunbc.compute.work_request { @@ -68,7 +72,6 @@ type ComputeLayout { log_path: String outcome_path: String consumption_path: String - obligations_dir: String } fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> ComputeLayout { @@ -86,7 +89,6 @@ fn compute_layout(instance: HostDashboardInstance, identity_hex: String) -> Comp log_path: join([work_dir, "/log"], ""), outcome_path: join([work_dir, "/outcome.json"], ""), consumption_path: join([work_dir, "/consumption"], ""), - obligations_dir: join([root, "/obligations"], ""), } } @@ -240,70 +242,105 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden // ── WHETHER THE UNIT IS STILL USING THE MEMORY ──────────────────────────────────────────────── // -// THE RELEASE LAW SAID "QUIESCENCE REQUIRED" AND NOTHING OBSERVED QUIESCENCE. That was the gap: -// compute_release ran unconditionally on whatever compute_run_leased returned, and what it returns -// is the exit of `systemd-run --wait`, which is NOT the unit. systemd-run is a launcher, not the -// unit's parent -- the transient service is owned by the manager -- so a launcher that is killed, -// times out, or loses its bus connection yields ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, and -// the seat's memory would be handed to the next arrival on top of a live cargo. product.capacity.lease -// already models this exactly (QuiescenceFact = Quiescent | QuiescenceUnobserved) and the corpus was -// simply not consulting it (side-chat review of gunbc#11962 at 725d9c58). +// THE RELEASE LAW SAID "QUIESCENCE REQUIRED" AND NOTHING OBSERVED QUIESCENCE. compute_release ran +// on whatever compute_run_leased returned, and what it returns is the exit of `systemd-run --wait`, +// which is NOT the unit. systemd-run is a launcher, not the unit's parent -- the transient service +// is owned by the manager -- so a launcher killed, timed out, or disconnected from the bus yields +// ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, and the seat's memory would be handed to the next +// arrival on top of a live cargo. product.capacity.lease already models this (QuiescenceFact) and +// this provider was not consulting it. +// +// ActiveState ALONE IS NOT THE EVIDENCE, AND THE FIRST FIX OF THIS THOUGHT IT WAS. Two arms it +// treated as free are not (side-chat review of gunbc#11962 at 9172cf7): // -// SO A RELEASE NOW CONSUMES A TERMINATION DECISION BOUND TO THE ACTUAL UNIT, and only two arms may -// free capacity: the manager knows nothing about the unit, or it knows it has stopped. Running and -// UNOBSERVED both keep the commitment -- an unobserved unit is the case a fail-open would swallow, -// and it is the one that matters, because "I could not ask" is exactly what a lost launcher looks -// like. +// `failed` CAN COEXIST WITH LIVE PROCESSES. A unit whose stop reaches the SIGKILL timeout is +// reported failed while its processes are still being killed, and a task wedged in +// uninterruptible sleep outlives the state word entirely. // -// AN EMPTY ActiveState IS QUIESCENT AND THAT IS NOT A GUESS. `systemctl --user show` prints an empty -// value for a unit the manager has no record of, which for a transient --wait unit means it ran and -// was collected, and for a unit that never started means there is nothing consuming memory. Both are -// the same fact for THIS decision -- no process of ours is holding that memory -- and the reviewer's -// own formulation admits both: never started, or demonstrated quiescent. +// AN EMPTY ActiveState IS NOT AUTHORITATIVE ABSENCE. It is the manager declining to answer in a +// form this provider cannot distinguish from a unit it has never heard of. The repo's own +// systemctl authority already names the observation that proves absence, and it is LoadState: +// `systemctl show` exits 0 for a unit the manager has never heard of and prints not-found, so a +// nonzero exit means the MANAGER was unreadable, which is a different refusal from an absent unit +// and must not be folded into it. +// +// SO THE DECISION IS FOUR-VALUED AND ONLY THE FIRST TWO FREE CAPACITY. Authoritative absence, from +// the one wire value that proves it. Observed termination, from the cgroup this unit owns reporting +// no processes -- the kernel's own answer to "is anything still running here", which is the question +// the memory is actually about. Populated keeps the charge. And UNAVAILABLE keeps it too, which is +// the arm that matters most: "I could not ask" is what a lost launcher, an unreadable manager and a +// missing cgroup all look like, and reading any of them as permission to take the memory back is the +// absorbing fallback DESIGN 5 forbids. type UnitTermination - = UnitQuiescent { state: String } - | UnitStillActive { state: String } - | UnitTerminationUnobserved { detail: String } - -data unit_active_state_property: NonEmptyStr = "ActiveState" as NonEmptyStr - -// THE CLOSED SET IS UPSTREAM'S (systemd.unit(5) ActiveState), and an unrecognised word is UNOBSERVED -// rather than assumed dead. A new systemd state that this fold has never seen must not be read as -// permission to take the memory back. -fn unit_state_termination(state: String) -> UnitTermination { - let s = trim(s: state) - if s == "" || s == "inactive" || s == "failed" { - UnitQuiescent { state: s } - } else if s == "active" || s == "activating" || s == "deactivating" || s == "reloading" || s == "maintenance" { - UnitStillActive { state: s } + = UnitAbsentAuthoritatively { detail: String } + | UnitTerminatedObserved { detail: String } + | UnitPopulated { detail: String } + | UnitTerminationUnavailable { detail: String } + +data unit_load_state_property: NonEmptyStr = "LoadState" as NonEmptyStr +data unit_control_group_property: NonEmptyStr = "ControlGroup" as NonEmptyStr +data unit_load_state_absent_wire: String = "not-found" + +// THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is +// system-scope; these are transient --user units, so asking the system manager would answer +// not-found for EVERY one of them -- a fabricated authoritative absence, which is the single worst +// answer this fold can produce. The property goes through the user-scope show operation for that +// reason and no other. +fn observe_unit_termination(identity_hex: String) -> UnitTermination { + let unit = compute_unit_name(identity_hex: identity_hex) + let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) + if !load.success { + UnitTerminationUnavailable { detail: join(["the user manager could not be read for ", unit as String], "") } + } else if trim(s: load.value) == unit_load_state_absent_wire { + UnitAbsentAuthoritatively { detail: join([unit as String, " LoadState=", unit_load_state_absent_wire], "") } } else { - UnitTerminationUnobserved { detail: join(["systemd reported an ActiveState this provider does not model: ", s], "") } + observe_unit_cgroup(unit: unit, load_state: trim(s: load.value)) } } -fn observe_unit_termination(identity_hex: String) -> UnitTermination { - let unit = compute_unit_name(identity_hex: identity_hex) - let r = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_active_state_property) - if !r.success { - UnitTerminationUnobserved { detail: join(["systemctl --user show ", unit as String, " could not be read"], "") } +// THE UNIT EXISTS, SO THE QUESTION IS WHETHER ANYTHING IS STILL RUNNING IN IT, and the kernel +// answers that directly. ControlGroup is empty for a unit that is loaded and holds no processes -- +// systemd removes the cgroup when the last one exits -- so an empty value here is observed +// termination rather than a failed read, and it is the ordinary answer after a clean run. +fn observe_unit_cgroup(unit: NonEmptyStr, load_state: String) -> UnitTermination { + let cg = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_control_group_property) + if !cg.success { + UnitTerminationUnavailable { detail: join(["the control group of ", unit as String, " could not be read"], "") } + } else if trim(s: cg.value) == "" { + UnitTerminatedObserved { detail: join([unit as String, " LoadState=", load_state, " holds no control group, so no process of this unit survives"], "") } } else { - unit_state_termination(state: r.value) + let events = linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) + if !events.success { + UnitTerminationUnavailable { detail: join(["the cgroup events of ", unit as String, " could not be read at ", trim(s: cg.value)], "") } + } else { + match decode_cgroup_events_populated(body: events.value) { + CgroupEventsUnparseable { cause: c } => UnitTerminationUnavailable { detail: join(["the cgroup events of ", unit as String, " could not be decoded: ", c], "") } + CgroupEventsDecoded { populated: p } => + if p { + UnitPopulated { detail: join([unit as String, " still has processes in ", trim(s: cg.value)], "") } + } else { + UnitTerminatedObserved { detail: join([unit as String, " cgroup ", trim(s: cg.value), " reports populated 0"], "") } + } + } + } } } fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { match t { - UnitQuiescent { state: _ } => true - UnitStillActive { state: _ } => false - UnitTerminationUnobserved { detail: _ } => false + UnitAbsentAuthoritatively { detail: _ } => true + UnitTerminatedObserved { detail: _ } => true + UnitPopulated { detail: _ } => false + UnitTerminationUnavailable { detail: _ } => false } } fn unit_termination_wire(t: UnitTermination) -> String { match t { - UnitQuiescent { state: s } => join(["quiescent (ActiveState=", if s == "" { "" } else { s }, ")"], "") - UnitStillActive { state: s } => join(["STILL ACTIVE (ActiveState=", s, "), so the seat stays charged"], "") - UnitTerminationUnobserved { detail: d } => join(["termination unobserved, so the seat stays charged: ", d], "") + UnitAbsentAuthoritatively { detail: d } => join(["absent (", d, ")"], "") + UnitTerminatedObserved { detail: d } => join(["terminated (", d, ")"], "") + UnitPopulated { detail: d } => join(["STILL POPULATED (", d, "), so the seat stays charged"], "") + UnitTerminationUnavailable { detail: d } => join(["termination UNAVAILABLE (", d, "), so the seat stays charged"], "") } } @@ -394,39 +431,38 @@ fn compute_run_and_settle( } else { ComputeReleaseRefused { detail: join(["the seat was NOT released: ", unit_termination_wire(t: termination)], "") } } - let measured = Filesystem.Write(path: layout.consumption_path, content: join([compute_capacity_standing_wire(s: standing), "\n"], "")) - let notes = join(["; consumption recorded: ", if measured.success { "yes" } else { measured.error }], "") let settled = if compute_release_landed(r: returned) { - compute_settled_record(layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, notes: notes) + compute_settled_record(layout: layout, identity: identity, subject: subject, op: op, outcome: outcome) } else { compute_unreleased_record( layout: layout, identity: identity, subject: subject, op: op, outcome: outcome, - reservation: reservation, returned: returned, reference: reference, termination: termination, notes: notes) + reservation: reservation, returned: returned, reference: reference, termination: termination) } let dropped = Filesystem.Delete(path: layout.lease_path) - if dropped.success { settled } else { compute_lease_undropped(result: settled, layout: layout, identity: identity, subject: subject, op: op, error: dropped.error) } -} - -fn compute_lease_undropped( - result: ProvideResult, - layout: ComputeLayout, - identity: String, - subject: WorkSubject, - op: WorkOperation, - error: String, -) -> ProvideResult { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: StoreUnavailable { - detail: join([ - "the record was published as ", provide_result_ending(r: result), - " but the producer lease at ", layout.lease_path, " could not be dropped: ", error, - "; the next request for this identity will refuse as ProducerInFlight until it is removed", - ], ""), - }, - }, - subject: subject, op: op) + let measured = Filesystem.Write( + path: layout.consumption_path, + content: join([ + compute_capacity_standing_wire(s: standing), "\n", + "unit: ", unit_termination_wire(t: termination), "\n", + "reservation: ", compute_release_wire(r: returned), "\n", + if dropped.success { "" } else { compute_stale_lease_note(layout: layout, error: dropped.error) }, + ], "")) + if measured.success { settled } else { settled } +} + +// BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the +// caller a refusal while the outcome document already said the run had SUCCEEDED, and then state +// that the next request would refuse as ProducerInFlight -- three claims that cannot all be true, +// because the stored record is exactly what the next caller attaches to. So the stored record +// decides: a published success IS the result, the caller is told the same thing the next caller +// will read, and the stale lease is recorded as the operational fact it is rather than converted +// into a verdict about the work. An operator removes the name; nothing about the run changed. +fn compute_stale_lease_note(layout: ComputeLayout, error: String) -> String { + join([ + "STALE PRODUCER LEASE: ", layout.lease_path, " could not be dropped: ", error, + "; the record for this identity is published and is authoritative, and the next request will ", + "refuse as ProducerInFlight until an operator removes the name\n", + ], "") } // THE ORDINARY ENDING: the reservation came back, so the record is the run's own ending and the @@ -437,7 +473,6 @@ fn compute_settled_record( subject: WorkSubject, op: WorkOperation, outcome: WorkOutcome, - notes: String, ) -> ProvideResult { let document = work_outcome_wire(o: outcome, subject: subject, op: op) let written = Filesystem.Write(path: layout.outcome_path, content: document) @@ -451,7 +486,7 @@ fn compute_settled_record( detail: join([ "the work ended ", work_outcome_ending(o: outcome), " and its reservation was released, but the record could not be completed -- outcome written: ", - written.error, notes, + written.error, ], ""), }, }, @@ -471,25 +506,13 @@ fn compute_unreleased_record( returned: ComputeRelease, reference: NonEmptyStr, termination: UnitTermination, - notes: String, ) -> ProvideResult { - let obligation = join([ - "identity=", identity, "\n", - "reference=", reference as String, "\n", - "unit=", unit_termination_wire(t: termination), "\n", - "work_ending=", work_outcome_ending(o: outcome), "\n", - "reservation=", compute_reservation_wire(r: reservation), "\n", - "release_attempt=", compute_release_wire(r: returned), "\n", - ], "") - let prepared = compute_ensure_dir_at(path: layout.obligations_dir) - let recorded = Filesystem.WriteCreateNew(path: compute_obligation_path(layout: layout, reference: reference), content: obligation) let cause = ComputeCapacityLedgerUnavailable { detail: join([ - "the seat for ", reference as String, " is still charged; the work itself ended ", - work_outcome_ending(o: outcome), " and any declared outputs are in the store. ", - compute_release_wire(r: returned), - ". obligation recorded: ", if prepared && recorded.success { compute_obligation_path(layout: layout, reference: reference) } else { recorded.error }, - notes, + "the seat for ", reference as String, " is STILL CHARGED and this provider has no way to discharge it: ", + unit_termination_wire(t: termination), ". ", compute_release_wire(r: returned), + ". The work itself ended ", work_outcome_ending(o: outcome), + " and any declared outputs are in the store. ", compute_stuck_reservation_remedy, ], ""), } let document = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) @@ -501,18 +524,17 @@ fn compute_unreleased_record( } } -// ONE OBLIGATION FILE PER ATTEMPT, NOT PER IDENTITY. The obligation used to live at a path derived -// from the work identity while the reservation it describes is keyed by the ATTEMPT, so a retry -// overwrote the first obligation and the first seat became unrecoverable -- the reference that was -// the only way to release it was gone. The reference is the name of the hold, so it is the name of -// the obligation, and the create is EXCLUSIVE so a second writer cannot silently replace one. -fn compute_obligation_path(layout: ComputeLayout, reference: NonEmptyStr) -> String { - join([layout.obligations_dir, "/", reference as String], "") -} - -fn compute_ensure_dir_at(path: String) -> Bool { - shell.Mkdir.Parents(path: path as FilePath).success -} +// THE REMEDY SENTENCE FOR A SEAT THIS PROVIDER CANNOT GIVE BACK, and it is a declared absence +// rather than a route claimed in prose. No discharge existed on main -- there were no reservations +// at all, only a count of lease files -- so this is not a rung that dropped; it is a failure mode +// this change makes reachable for the first time, rostered as +// gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge, whose trigger names +// the capability that closes it. +// +// WHAT KEEPS IT FAIL-CLOSED MEANWHILE: every arm that cannot prove the unit is gone keeps the +// charge, so the host UNDER-admits. Nothing is over-promised; the pool simply shrinks until an +// operator clears the partition by hand. That is the direction a capacity ledger must fail in. +data compute_stuck_reservation_remedy: String = "Until the compute lifecycle capability lands (gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge), the only discharge is an operator clearing this host's compute capacity partition by hand; this host will under-admit until then, which is the fail-closed direction." // THE ORDER THE TWO HOLDS ARE TAKEN IN// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the @@ -557,22 +579,6 @@ fn compute_provide_leaf( } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: join(["could not create the compute directories under ", layout.root], "") } }, subject: subject, op: op) } else { - let outstanding = compute_outstanding_obligation(layout: layout, identity: identity) - if outstanding != "" { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: ComputeCapacityLedgerUnavailable { - detail: join([ - "this identity has an unreleased reservation from an earlier attempt and will not take a second one: ", - outstanding, - ". Discharge it with the release verb once the unit is demonstrably gone; until then a retry would ", - "reserve on top of a seat that is still charged and could publish an outcome over the obligation.", - ], ""), - }, - }, - subject: subject, op: op) - } else { let lease = Filesystem.WriteCreateNew(path: layout.lease_path, content: join([commit, " ", clock_now_probed_at_or_unknown() as String, "\n"], "")) if !lease.success { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) @@ -583,7 +589,7 @@ fn compute_provide_leaf( ComputeCapacityFull { wire: w } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) ComputeHostBelowFloor { requested: rq, available: av } => - compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested_kib: kibibyte_count(k: rq), available_kib: kibibyte_count(k: av) }) + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested: rq, available: av }) ComputeReservationRefused { detail: d } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => @@ -592,33 +598,9 @@ fn compute_provide_leaf( dependency_store: dependency_store, granted: granted, reference: reference, reservation: reservation) } } - } } } -// WHAT A RETRY MUST LOOK AT BEFORE IT RESERVES AGAIN, and the reason this is a read rather than a -// convention. An obligation names a seat that is still charged; a retry of the same identity that -// ignored it would take a SECOND reservation while the first is outstanding, and -- because the -// outcome document is keyed by identity -- could publish its own ending over the record that was -// carrying the obligation. So the lifecycle state is consumed, not remembered: the obligations -// directory is listed, and any entry naming this identity refuses the request by name and points at -// the release verb. Empty means nothing outstanding; an unreadable directory means nothing -// outstanding either, because the directory does not exist until the first obligation is written. -fn compute_outstanding_obligation(layout: ComputeLayout, identity: String) -> String { - let listing = Filesystem.List(path: layout.obligations_dir) - if !listing.success { - "" - } else { - fold(split(s: listing.entries, delimiter: "\n"), init: "", f: (acc, entry) => - if acc != "" || trim(s: entry) == "" { - acc - } else if !starts_with(s: trim(s: entry), prefix: identity) { - acc - } else { - join([layout.obligations_dir, "/", trim(s: entry)], "") - }) - } -} // THE CONSUMPTION READING IS ITS OWN FILE AND MUST NOT JOIN THE OUTCOME DOCUMENT. It was appended // to outcome.json first, and the first real run on srv1 (2026-09-21) is what said why that is @@ -790,60 +772,6 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent } } -// THE OPERATOR'S RECOVERY DOOR, and the executing consumer of compute_release_reference. A -// reservation whose release did not land leaves its reference in -// /unreleased-reservation and in the partition; this is how it is given back. It is a -// verb on the same CLI rather than a script, so the refusal vocabulary is the module's: a reference -// the pool is not holding writes nothing and exits non-zero saying so, which is the answer a -// mistyped reference must get -- appending a release for an unheld reference would make the -// partition unfoldable for every later reader. -fn compute_release_cli(repo_root: String, identity: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { - match dashboard_instance_for_repo_root(repo_root: repo_root) { - DashboardInstanceRootUnknown { repo_root: r } => ExitFailure { code: 2, reason: join(["compute: no dashboard instance is rooted at ", r], "") } - DashboardInstanceResolved { instance } => - if identity == "" || reference == "" || reason == "" { - ExitFailure { code: 2, reason: "compute: release needs the identity and the reference the obligation file names, and a reason saying why the seat is being given back" } - } else { - compute_release_observed(instance: instance, identity: identity, reference: reference, reason: reason, receipt_path: receipt_path) - } - } -} - -// THE RECOVERY DOOR OBSERVES THE UNIT TOO, and this is the half a reason string cannot supply. An -// operator releasing "because the worker looked dead" is asserting exactly the fact that nothing -// checked -- and releasing a seat whose cargo is still resident hands its memory to the next -// arrival. So recovery consumes the same termination decision settlement does: only a unit the -// manager reports stopped, or has no record of, may be released. A running unit is refused BY NAME -// with the state that refused it, so the operator's next move is to stop the unit rather than to -// argue with the tool. -fn compute_release_observed(instance: HostDashboardInstance, identity: String, reference: String, reason: String, receipt_path: String) -> ProcessExit { - let termination = observe_unit_termination(identity_hex: identity) - let wire = if unit_termination_frees_capacity(t: termination) { - compute_release_cli_wire(r: compute_release_reference( - instance: instance, - reference: reference as NonEmptyStr, - reason: join([reason, " (unit ", unit_termination_wire(t: termination), ")"], "") as NonEmptyStr)) - } else { - join(["refused: ", unit_termination_wire(t: termination), "; stop the unit first, then release"], "") - } - let receipt = Filesystem.Write(path: receipt_path, content: join([wire, "\n"], "")) - if !receipt.success { - ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; release was ", wire], "") } - } else if starts_with(s: wire, prefix: "released ") { - ExitSuccess - } else { - ExitFailure { code: 1, reason: join(["compute release refused: ", wire], "") } - } -} - -fn compute_release_cli_wire(r: ComputeRelease) -> String { - match r { - ComputeReleased { partition: p, reference: ref_, reason: why } => - join(["released ", ref_ as String, " on ", p as String, " (", why as String, ")"], "") - ComputeReleaseRefused { detail: d } => join(["refused: ", d], "") - } -} - fn compute_parse_operation(operation: String, entry: String, functions: String, hermetic: String) -> WorkOperation? { if operation == "build" { Present { value: work_default_build() } diff --git a/dag/gunbc/compute/work_request.dag b/dag/gunbc/compute/work_request.dag index 62e1231c219..16883a0830b 100644 --- a/dag/gunbc/compute/work_request.dag +++ b/dag/gunbc/compute/work_request.dag @@ -1,6 +1,7 @@ module gunbc.compute.work_request import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } +import std.measure { Kibibyte, kibibyte_count } import std.content_hash { Fnv1a64Structural, content_hash_atom, content_hash_tagged_structural } import extdeps.git.object_store { GitObjectId, git_object_id_wire_hex } import extdeps.languages.json.emit { @@ -138,6 +139,14 @@ type WorkOutcome // this class reserves, the checkout or the store could not be prepared, or a dependency of this // operation (the build the compile consumes) did not succeed. // +// THE MEMORY ARMS CARRY Kibibyte, NOT Int FIELDS NAMED _kib. A unit in a field NAME on a bare Int +// is the unit modeled twice -- once in the name and once in the " KiB" this module then renders -- +// and modeled nowhere the compiler can see. The producer already had it right +// (gunbc.compute.host_capacity ComputeHostBelowFloor carries Kibibyte) and the provider was calling +// kibibyte_count to STRIP the carrier on the way in, which is the tell (review 69624). The deleted +// arm this replaced, HostAtColdBuildCap { in_flight, cap }, was a dimensionless COUNT, so Int was +// right there and is not right here. +// // THREE CAPACITY ARMS WHERE THERE USED TO BE ONE COUNT. HostAtColdBuildCap { in_flight, cap } is // DELETED, and its disposition is that the fact it reported -- this host is already carrying as // much as it may -- is now HostComputeCapacityFull, measured in the memory the pool actually @@ -148,7 +157,7 @@ type WorkOutcome type WorkInfrastructureRefusal = ProducerInFlight { lease_path: String } | HostComputeCapacityFull { wire: String } - | HostBelowMemoryFloor { requested_kib: Int, available_kib: Int } + | HostBelowMemoryFloor { requested: Kibibyte, available: Kibibyte } | ComputeCapacityLedgerUnavailable { detail: String } | CheckoutUnavailable { detail: String } | StoreUnavailable { detail: String } @@ -180,7 +189,7 @@ fn work_refusal_detail(r: WorkInfrastructureRefusal) -> String { match r { ProducerInFlight { lease_path } => join(["a producer already holds the lease at ", lease_path, "; attach later, do not start a second one"], "") HostComputeCapacityFull { wire } => join(["this host's compute memory pool has no room for another reservation: ", wire], "") - HostBelowMemoryFloor { requested_kib, available_kib } => join(["this class reserves ", to_string(value: requested_kib), " KiB and the host reports only ", to_string(value: available_kib), " KiB available, so the reservation would not have been backed by the machine"], "") + HostBelowMemoryFloor { requested, available } => join(["this class reserves ", to_string(value: kibibyte_count(k: requested)), " KiB and the host reports only ", to_string(value: kibibyte_count(k: available)), " KiB available, so the reservation would not have been backed by the machine"], "") ComputeCapacityLedgerUnavailable { detail } => join(["the host's compute capacity ledger could not be transacted: ", detail], "") CheckoutUnavailable { detail } => join(["checkout unavailable: ", detail], "") StoreUnavailable { detail } => join(["store unavailable: ", detail], "") diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index d7ae49970ac..ee33b3251b6 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -2,7 +2,7 @@ module gunbc.fabric_event_log import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat, nat_range_inclusive } -import std.measure { Count, One } +import std.measure { Count, One, Measure } import product.placement_supply { HostIdentity } import std.process { ProcessExit, ExitSuccess, exit_failure } import std.fabric_storage { @@ -244,13 +244,20 @@ type SeatAttemptState { // and take a reading. Every step is the one seat_attempt already runs, which is what keeps this a // second QUESTION rather than a second authority -- there is no separate notion here of what a seat // is or when a pool is full. -type SeatStandingObservation - = SeatRoomObserved { headroom: Nat, generation: Nat } +// THE HEADROOM CARRIES ITS DIMENSION, AND THIS PR IS WHY IT HAS TO. A bare Nat was legitimate while +// this fold was fixed at Pool: seats are dimensionless and the field meant seats. The +// generalization to Pool above made the same field mean KiB for a memory pool and seats for a +// seat pool, with nothing in the type to say which -- so the unit moved into the consumer's head and +// into rendered strings, which is the unit modeled twice and checked nowhere (review 69624). The +// earliest unjustified boundary is this declaration, not its consumers, and it is one this change +// moved itself (DESIGN 6b). +type SeatStandingObservation + = SeatRoomObserved { headroom: Measure, generation: Nat } | SeatFullObserved { generation: Nat } | SeatStandingUnobserved { step: String, reason: String } | SeatStandingStoreRefused { cause: EventLogRefusal } -fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root: Pool, at: EpochSecs, budget: Nat) -> SeatStandingObservation { +fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId, root: Pool, at: EpochSecs, budget: Nat) -> SeatStandingObservation { match event_log_read_partition(store: store, partition: partition, budget: budget) { PartitionReadRefused { cause: c } => SeatStandingStoreRefused { cause: c } PartitionReadOk { head: _, walk: walk } => @@ -268,7 +275,7 @@ fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId PoolReadingRefused { at: t, anchor: a } => SeatStandingUnobserved { step: "reading", reason: join(["instant ", to_string(t), " precedes the pool anchor ", to_string(a)], "") } PoolRead { committed: _, ceiling: _, headroom: h, over_committed: _ } => - if h.count > 0 { SeatRoomObserved { headroom: h.count, generation: g } } + if h.count > 0 { SeatRoomObserved { headroom: h, generation: g } } else { SeatFullObserved { generation: g } } } } diff --git a/dag/gunbc/harness/harness_seat.dag b/dag/gunbc/harness/harness_seat.dag index 4b6de567bae..7fb2d6751a5 100644 --- a/dag/gunbc/harness/harness_seat.dag +++ b/dag/gunbc/harness/harness_seat.dag @@ -3,7 +3,7 @@ module gunbc.harness.harness_seat import product.placement_supply { HostIdentity } import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat, nat_range_inclusive, nat_min } -import std.measure { Measure, Count, One, money_amount_micro, second } +import std.measure { Measure, Count, One, money_amount_micro, second, measure_count } import gunbc.spark.fabric_switch_observed { FabricGroup, FabricGroupA, FabricGroupB, fabric_group_wire, fabric_group_hosts } import gunbc.serving.serving_enrollment { ServingRoute, ServingRouteResolved, ServingRouteUnresolved, ServingRouteEndpoint, @@ -334,7 +334,7 @@ fn harness_class_held(route: ServingRoute, class: ServingCoTenancyClass, store: } SeatFullObserved { generation: _ } => ClassSeatsHeld { held: ceiling } SeatRoomObserved { headroom: h, generation: _ } => - ClassSeatsHeld { held: ceiling - nat_min(a: h, b: ceiling) } + ClassSeatsHeld { held: ceiling - nat_min(a: measure_count(h), b: ceiling) } } } } diff --git a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag new file mode 100644 index 00000000000..2857c2a58ff --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag @@ -0,0 +1,34 @@ +module gunbc.recurring_failure_mode.a_held_compute_seat_has_no_in_corpus_discharge + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, decl_ref } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = RecurringFailureMode { + identity: "a_held_compute_seat_has_no_in_corpus_discharge" as NonEmptyStr, + + receipts: [ + "INVALID STATE: a quantitative reservation is taken under a release law that never lapses, and the only path that returns it is the settlement of the run that took it. When that settlement cannot prove the unit is gone -- or cannot append at all -- the encumbrance stays Held and NO DECLARATION IN THE CORPUS CAN DISCHARGE IT. HARM: the host's compute pool shrinks by one grant per occurrence and never grows back; at gunbc.compute.host_capacity compute_pool_slots = 2, two occurrences consume the whole appropriation, after which every request is refused for capacity on a host with its memory free.", + + "THIS IS A NEW CLASS RATHER THAN A DECLARED RUNG DROP, and the distinction is the one DESIGN 4b(3) turns on: a drop is a rung that FELL. Before gunbc#11962 this host admitted work by counting lease files (gunbc.compute.work_provider_local compute_cold_build_cap, deleted), so there were no reservations at all and nothing to discharge. The state becomes reachable for the first time with the pool, so it is filed here and not in gunbc.rung_drop.", + + "WHY THE RESERVATION IS NOT SIMPLY FREED ON DOUBT, which is the repair that suggests itself and is the wrong one: the pool's release law for a compute seat is QuiescenceRequired precisely because nothing can prove a systemd transient unit stopped using memory from outside it. Freeing a seat whose cargo is still resident hands its memory to the next arrival on top of a live build. Every arm that cannot prove termination therefore keeps the charge, so this class's harm is UNDER-admission -- the fail-closed direction -- and its opposite would be over-admission on a shared host.", + + "SPECIMEN, EXECUTED ON srv1 (2026-09-21, gunbc#11962): a real release build was started under a grant of 27262976 KiB and the gunbc driver was SIGKILLed mid-run. The unit stayed active, the partition carried the acquire and no release, and the producer lease stayed held -- the commitment correctly stayed charged, and nothing in the corpus could return it.", + + "RUNG FOUND AT: mitigatable. The state is reachable, its harm is bounded to one host's compute appropriation, it is visible in the outcome document and in the partition, and it never over-promises.", + + "CEILING: mechanically preventable. A discharge route exists to be built -- the evidence needed is a termination decision bound to the unit plus a settled lifecycle state that a retry consumes -- so this is not a class that must stay a ratchet.", + + "NEXT-RUNG TRIGGER, NAMING THE CAPABILITY AND NOT AN ARTIFACT: the compute reservation LIFECYCLE capability -- a discharge that (a) decides termination from authoritative absence or an observed-empty cgroup rather than from a state word, (b) binds the attempt and its unit from the authoritative attempt record so releasing one cannot release another's live reservation, (c) coordinates with a pending producer so a release cannot race a launch that has not happened yet, and (d) produces a settled state a retry consumes, with history kept. Until a discharge with all four properties is consumed by a real caller, this row stands.", + + "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration, and gunbc.compute.work_provider_local compute_stuck_reservation_remedy carries the remedy sentence into the refusal a caller actually receives.", + ], + + evidence: [ + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "compute_stuck_reservation_remedy"), + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "unit_termination_frees_capacity"), + decl_ref(module_path: "gunbc.compute.host_capacity", decl_name: "compute_release"), + decl_ref(module_path: "test.claim.compute.work_request_witness", decl_name: "only_proven_absence_or_an_empty_cgroup_frees_a_seat"), + ], +} diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index 6051e698a4f..490ef641cd8 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -5,14 +5,13 @@ import std.types { Bool, Int, NonEmptyStr, String } import std.measure { Kibibyte, kibibyte, kibibyte_count } import std.algebra { trim } import product.capacity.event_chain { PartitionId } -import gunbc.compute.host_capacity { compute_capacity_store } import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, srv1_lab_dashboard_instance, dashboard_instance_compute_root } import gunbc.compute.host_capacity { ComputeCapacitySubject, ComputeSubjectResolution, ComputeSubjectResolved, ComputeSubjectHostUnresolved, compute_capacity_subject, compute_capacity_root, compute_memory_partition, HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - compute_reserve_on, compute_reserve_against, compute_release, compute_release_on, ComputeRelease, ComputeReleased, ComputeReleaseRefused, + compute_reserve_on, compute_reserve_against, compute_release, ComputeRelease, ComputeReleased, ComputeReleaseRefused, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing_on, } @@ -230,27 +229,3 @@ test fn the_real_observation_reads_this_machine() -> Bool { } } -// A WEDGED SEAT CAN ACTUALLY BE RECOVERED, BY REFERENCE, BY SOMEONE WHO NEVER HELD THE RESERVATION. -// Under QuiescenceRequired a term frees nothing, so this route is the ONLY way capacity returns -// after a release fails to land -- and until review 69595 the module claimed it in prose while no -// declaration reached it. -// -// AND A REFERENCE THE POOL IS NOT HOLDING WRITES NOTHING. That is the sharper half: an unheld -// reference appended anyway would not fail at the append, it would fail at every later READ, when -// the fold hits UnknownEncumbrance and refuses the WHOLE partition. So the refusal is asserted -// BESIDE a subsequent successful reservation, which is what establishes the ledger is still -// readable after the bad attempt. -test fn a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing() -> Bool { - let subject = subject_at(root: fresh_root(), appropriation_kib: one_gibibyte_in_kibibytes, request_kib: one_gibibyte_in_kibibytes) - let store = compute_capacity_store(subject: subject) - let held = compute_reserve_on(subject: subject, reference: "wet-wedged" as NonEmptyStr, term_seconds: 60) - let full = compute_reserve_on(subject: subject, reference: "wet-blocked" as NonEmptyStr, term_seconds: 60) - let bogus = compute_release_on(store: store, partition: subject.partition, appropriation: subject.appropriation, reference: "wet-never-held" as NonEmptyStr, reason: "operator recovery" as NonEmptyStr) - let recovered = compute_release_on(store: store, partition: subject.partition, appropriation: subject.appropriation, reference: "wet-wedged" as NonEmptyStr, reason: "operator recovery" as NonEmptyStr) - let readmitted = compute_reserve_on(subject: subject, reference: "wet-after-recovery" as NonEmptyStr, term_seconds: 60) - reserved_amount(r: held) == one_gibibyte_in_kibibytes - && is_capacity_full(r: full) - && !released(r: bogus) - && released(r: recovered) - && reserved_amount(r: readmitted) == one_gibibyte_in_kibibytes -} diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 21b15320449..ac8dbabde15 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -3,8 +3,8 @@ module test.claim.compute.work_request_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } import gunbc.compute.work_provider_local { - UnitTermination, UnitQuiescent, UnitStillActive, UnitTerminationUnobserved, - unit_state_termination, unit_termination_frees_capacity, + UnitTermination, UnitAbsentAuthoritatively, UnitTerminatedObserved, UnitPopulated, UnitTerminationUnavailable, + unit_termination_frees_capacity, unit_termination_wire, } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, @@ -79,32 +79,22 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool && (match stored_outcome_decode(text: "not json") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) } -// ONLY A UNIT THAT IS DEMONSTRABLY GONE MAY FREE ITS SEAT, and the arm that matters is the one in -// the middle. `systemd-run --wait` is a LAUNCHER, not the unit's parent, so a launcher that dies or -// times out returns failure while the transient service keeps running; releasing on that signal -// would hand a live cargo's memory to the next arrival. Empty and inactive and failed are all -// "nothing of ours is holding that memory" -- never started and already collected are the same fact -// for this decision. Everything systemd calls live keeps the charge, and a word this fold has never -// seen keeps it too: a future ActiveState must not read as permission to take memory back. -test fn only_a_gone_unit_frees_its_seat_and_an_unknown_state_does_not() -> Bool { - all(["", "inactive", "failed", " inactive "], s => - match unit_state_termination(state: s) { - UnitQuiescent { state: _ } => true - UnitStillActive { state: _ } => false - UnitTerminationUnobserved { detail: _ } => false - }) - && all(["active", "activating", "deactivating", "reloading", "maintenance"], s => - match unit_state_termination(state: s) { - UnitStillActive { state: _ } => true - UnitQuiescent { state: _ } => false - UnitTerminationUnobserved { detail: _ } => false - }) - && all(["quantum-superposed", "ACTIVE", "1"], s => - match unit_state_termination(state: s) { - UnitTerminationUnobserved { detail: _ } => true - UnitQuiescent { state: _ } => false - UnitStillActive { state: _ } => false - }) - && all(["", "inactive", "failed"], s => unit_termination_frees_capacity(t: unit_state_termination(state: s))) - && all(["active", "activating", "deactivating", "quantum-superposed"], s => !unit_termination_frees_capacity(t: unit_state_termination(state: s))) +// ONLY PROVEN ABSENCE OR AN OBSERVED-EMPTY CGROUP MAY FREE A SEAT, and the two arms this cell +// exists to pin RED are the ones an earlier revision got wrong: `failed` can coexist with live +// processes while a unit's stop reaches its SIGKILL timeout, and an empty ActiveState is the +// manager declining to answer rather than authoritative absence. Neither may free capacity. The +// unavailable arm is the one that matters most -- it is what a lost launcher, an unreadable manager +// and a missing cgroup all look like, and reading any of them as permission is the absorbing +// fallback DESIGN 5 forbids. +test fn only_proven_absence_or_an_empty_cgroup_frees_a_seat() -> Bool { + let absent = UnitAbsentAuthoritatively { detail: "LoadState=not-found" } + let terminated = UnitTerminatedObserved { detail: "populated 0" } + let populated = UnitPopulated { detail: "populated 1" } + let unavailable = UnitTerminationUnavailable { detail: "the manager could not be read" } + unit_termination_frees_capacity(t: absent) + && unit_termination_frees_capacity(t: terminated) + && !unit_termination_frees_capacity(t: populated) + && !unit_termination_frees_capacity(t: unavailable) + && string_contains(s: unit_termination_wire(t: populated), pattern: "STILL POPULATED") + && string_contains(s: unit_termination_wire(t: unavailable), pattern: "UNAVAILABLE") } diff --git a/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag b/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag index d8b4c0c4ebf..5ffe31d285e 100644 --- a/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag +++ b/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag @@ -3,7 +3,7 @@ module test.claim.spark.pair_serving_d0_real_execution import std.logic { Bool } import std.types { String, NonEmptyStr, FilePath, List, Int } import v2.std.optional { Present, Absent } -import std.measure { second } +import std.measure { second, measure_count } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import extdeps.shell import std.content_hash { ContentHash, Sha256Hash, sha256_hex_digest, compare_content_hash, ContentHashEqual, ContentHashDifferent, ContentHashCrossFamilyIncomparable, serialize_content_hash, content_hash_equal } @@ -390,7 +390,7 @@ test fn a_seat_granted_after_the_suspension_landed_is_released_by_real_execution _ => false }) && (match fabric_seat_observe(store: store, partition: partition, root: root, at: 1002, budget: 64) { - SeatRoomObserved { headroom: room, generation: _ } => room == ceiling + SeatRoomObserved { headroom: room, generation: _ } => measure_count(room) == ceiling _ => false }) _ => false diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 6e8d09c7ff7..2115d42e0af 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -250,12 +250,6 @@ fn local_repo_wet_schedule() -> List { function: "the_real_observation_reads_this_machine", expectation: ExpectedToHold {} }, - WetScheduledClaim { - identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing" }, - entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", - function: "a_held_seat_is_recoverable_by_reference_and_an_unheld_one_writes_nothing", - expectation: ExpectedToHold {} - }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.compute.host_capacity_wet_witness", function: "two_requests_summing_past_the_observed_headroom_cannot_both_admit" }, entry: "dag/test/claim/compute/host_capacity_wet_witness_test.dag", From 773825cf91dd9eea6888a6845e5363c1ec18830f Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 18:49:51 +0000 Subject: [PATCH 08/28] The empty-ControlGroup release arm, the discarded host record, and two remedy texts that told a reader the wrong thing Side-chat review at 5ee0cce plus review 69645. All four verified against the code first. THE THIRD RELEASE ARM. observe_unit_cgroup read an empty ControlGroup as UnitTerminatedObserved without reading any cgroup at all, and an empty or unrecognized successful LoadState could reach it. An empty ControlGroup does not mean finished: systemd realizes the cgroup AT SPAWN, so it equally describes a start that is still PENDING -- which makes a whole unbacked launch reachable. Reserve; the start is pending; the LAUNCHER is lost while this driver lives on; settlement sees the empty property, releases; the job then starts against memory nobody holds. The arm is deleted and an empty required property is never promoted to a conclusion -- it is UNAVAILABLE like every other absence of evidence. Capacity is now freed on two facts that establish the attempt ENDED, plus one that requires a realized cgroup: THE ATTEMPT COMPLETED -- `systemd-run --wait` returned success, which it does only after the unit ran to completion. A lost launcher does not return success and a pending job has not returned at all. This is a SEPARATE fact from the work's ending, and conflating them is what let a lost launcher free a live unit's memory: a WorkFailed says the unit exited nonzero OR that the launcher never got an answer. THE MANAGER HAS NO RECORD -- LoadState=not-found. A REALIZED CGROUP REPORTING populated 0 -- which cannot be a pending start, because the cgroup exists only from spawn onward. THE DECISION IS NOW A FOLD OVER SUPPLIED READINGS (UnitObservations), so the controls sit at the OBSERVATION-TO-DECISION boundary rather than at the variant-to-release mapping, which cannot see any of the defects that actually occurred: a failed query whose text reads like an answer, an empty property, a pending start. A failed query is unavailable whatever it printed -- the exit status decides first and the text is never consulted. review 69645, THE DISCARDED HOST RECORD: `if measured.success { settled } else { settled }` had identical arms, so a failed write vanished by construction. The compound case was the sharp one -- the producer lease failing to drop AND that write failing left an identity whose lease is permanently held with NO RECORD ANYWHERE that it is stale, while the caller got an ordinary success. It now refuses, naming what was lost. This does not reopen blocker (7): that was a refusal DENYING a published success; this one affirms the run's ending, says the stored record is authoritative, and reports a host fault that happened after publication. review 69645, THE DEAD CITATION: fabric_seat_release's annotation named compute_release_reference and compute_release_cli, which the split removed. It now names its real consumer and states that the operator door is the capability PR 2 lands. TWO REMEDY TEXTS THAT MISLED. compute_stale_lease_note said the next request would refuse as ProducerInFlight, contradicting the contract the same block establishes: a caller reaching a published SUCCESS attaches, and the attach is decided before the lease is consulted -- the stale name only blocks a request that must PRODUCE again. And compute_stuck_reservation_remedy told an operator to clear the partition by hand, which under live admission removes the record the admission arithmetic folds and trades a host that under-admits for one that over-promises; it now refuses that and states the ordering any manual intervention requires. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 257 +++++++++++++----- dag/gunbc/fabric/fabric_event_log.dag | 17 +- ...ompute_seat_has_no_in_corpus_discharge.dag | 4 +- .../compute/work_request_witness_test.dag | 64 +++-- 4 files changed, 250 insertions(+), 92 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 3e19b143212..3fcd3b6b23f 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -240,39 +240,49 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden } } -// ── WHETHER THE UNIT IS STILL USING THE MEMORY ──────────────────────────────────────────────── +// ── WHETHER THE ATTEMPT HAS ENDED AND ITS UNIT IS GONE ──────────────────────────────────────── // // THE RELEASE LAW SAID "QUIESCENCE REQUIRED" AND NOTHING OBSERVED QUIESCENCE. compute_release ran -// on whatever compute_run_leased returned, and what it returns is the exit of `systemd-run --wait`, -// which is NOT the unit. systemd-run is a launcher, not the unit's parent -- the transient service -// is owned by the manager -- so a launcher killed, timed out, or disconnected from the bus yields -// ComputeExecFailed WHILE THE UNIT KEEPS RUNNING, and the seat's memory would be handed to the next -// arrival on top of a live cargo. product.capacity.lease already models this (QuiescenceFact) and -// this provider was not consulting it. +// on whatever compute_run_leased returned, and that is the exit of `systemd-run --wait`, which is +// NOT the unit: systemd-run is a launcher, the transient service is owned by the manager, and a +// launcher killed or disconnected yields a failure WHILE THE UNIT KEEPS RUNNING. // -// ActiveState ALONE IS NOT THE EVIDENCE, AND THE FIRST FIX OF THIS THOUGHT IT WAS. Two arms it -// treated as free are not (side-chat review of gunbc#11962 at 9172cf7): +// TWO EARLIER SHAPES OF THIS FOLD WERE WRONG, AND BOTH WERE WRONG IN THE SAME DIRECTION -- they +// found a way to say "gone" from evidence that does not establish it. // -// `failed` CAN COEXIST WITH LIVE PROCESSES. A unit whose stop reaches the SIGKILL timeout is -// reported failed while its processes are still being killed, and a task wedged in -// uninterruptible sleep outlives the state word entirely. +// ActiveState ALONE (first shape): `failed` coexists with live processes while a unit's stop +// reaches its SIGKILL timeout, and an empty ActiveState is the manager declining to answer. // -// AN EMPTY ActiveState IS NOT AUTHORITATIVE ABSENCE. It is the manager declining to answer in a -// form this provider cannot distinguish from a unit it has never heard of. The repo's own -// systemctl authority already names the observation that proves absence, and it is LoadState: -// `systemctl show` exits 0 for a unit the manager has never heard of and prints not-found, so a -// nonzero exit means the MANAGER was unreadable, which is a different refusal from an absent unit -// and must not be folded into it. +// AN EMPTY ControlGroup (second shape, side-chat review at 5ee0cce): it was read as observed +// termination without reading any cgroup at all, and it is not. systemd realizes the cgroup AT +// SPAWN, so an empty ControlGroup equally describes a unit whose start is still PENDING -- which +// makes a whole unbacked launch reachable: reserve, the start is pending, the launcher is lost +// while this driver lives on, settlement sees the empty property, releases, and THEN the job +// starts against memory nobody holds. An empty required property is now never promoted to a +// conclusion; it is UNAVAILABLE, like every other absence of evidence. // -// SO THE DECISION IS FOUR-VALUED AND ONLY THE FIRST TWO FREE CAPACITY. Authoritative absence, from -// the one wire value that proves it. Observed termination, from the cgroup this unit owns reporting -// no processes -- the kernel's own answer to "is anything still running here", which is the question -// the memory is actually about. Populated keeps the charge. And UNAVAILABLE keeps it too, which is -// the arm that matters most: "I could not ask" is what a lost launcher, an unreadable manager and a -// missing cgroup all look like, and reading any of them as permission to take the memory back is the -// absorbing fallback DESIGN 5 forbids. +// SO CAPACITY IS FREED ON TWO FACTS AND NO OTHERS, both of which establish that the attempt ENDED +// rather than that it looks quiet: +// +// THE ATTEMPT COMPLETED. `systemd-run --wait` returned success, which it does only after the +// unit it started ran to completion. A lost launcher does not return success, and a pending job +// has not returned at all. +// +// THE MANAGER HAS NO RECORD OF THE UNIT. LoadState=not-found is the wire value this corpus reads +// positively, and a transient --wait unit reaches it once it has run and been collected. +// +// A REALIZED CGROUP REPORTING populated 0 IS THE THIRD, AND IT IS EVIDENCE RATHER THAN A GUESS, +// which is why it survives where the empty property did not: the cgroup exists only from spawn +// onward, so a NON-EMPTY ControlGroup whose cgroup reports no processes cannot be a pending start. +// It spawned, and everything in it exited. +// +// EVERYTHING ELSE KEEPS THE CHARGE: populated, an unreadable manager, an unreadable cgroup, an +// undecodable body, a property that came back empty. "I could not establish it" is what a lost +// launcher, a pending job and a broken bus all look like, and reading any of them as permission to +// take the memory back is the absorbing fallback DESIGN 5 forbids. type UnitTermination - = UnitAbsentAuthoritatively { detail: String } + = UnitAttemptCompleted { detail: String } + | UnitAbsentAuthoritatively { detail: String } | UnitTerminatedObserved { detail: String } | UnitPopulated { detail: String } | UnitTerminationUnavailable { detail: String } @@ -281,53 +291,90 @@ data unit_load_state_property: NonEmptyStr = "LoadState" as NonEmptyStr data unit_control_group_property: NonEmptyStr = "ControlGroup" as NonEmptyStr data unit_load_state_absent_wire: String = "not-found" -// THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is -// system-scope; these are transient --user units, so asking the system manager would answer -// not-found for EVERY one of them -- a fabricated authoritative absence, which is the single worst -// answer this fold can produce. The property goes through the user-scope show operation for that -// reason and no other. -fn observe_unit_termination(identity_hex: String) -> UnitTermination { - let unit = compute_unit_name(identity_hex: identity_hex) - let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) - if !load.success { - UnitTerminationUnavailable { detail: join(["the user manager could not be read for ", unit as String], "") } - } else if trim(s: load.value) == unit_load_state_absent_wire { - UnitAbsentAuthoritatively { detail: join([unit as String, " LoadState=", unit_load_state_absent_wire], "") } +// ONE PROPERTY READ, AS READ. success is the query's, value is the manager's answer; the two are +// separate because a successful query that answers nothing is a different fact from a query that +// failed, and the fold below must be able to tell them apart without re-deriving either. +type UnitPropertyReading { + queried: Bool + value: String +} + +// THE RAW OBSERVATIONS THE DECISION IS MADE FROM, carried as a value so the decision is a fold over +// supplied readings rather than a fold over live service calls. That is what lets the controls +// drive the OBSERVATION-TO-DECISION boundary -- a failed query whose text looks like an answer, a +// property that came back empty, a pending start -- instead of only the variant-to-release mapping, +// which cannot see any of those (side-chat review at 5ee0cce). +type UnitObservations { + attempt_completed: Bool + load_state: UnitPropertyReading + control_group: UnitPropertyReading + events: UnitPropertyReading +} + +// A FAILED QUERY IS UNAVAILABLE WHATEVER IT PRINTED. systemctl writes diagnostics that read like +// answers -- "Failed to connect to bus", "not-found" inside a sentence -- so the exit status +// decides first and the text is never consulted on a failed query. +fn unit_termination_decide(o: UnitObservations) -> UnitTermination { + if o.attempt_completed { + UnitAttemptCompleted { detail: "systemd-run --wait returned success, so the unit it started ran to completion" } + } else if !o.load_state.queried { + UnitTerminationUnavailable { detail: "the user manager could not be queried for this unit, so nothing about it is established" } + } else if trim(s: o.load_state.value) == unit_load_state_absent_wire { + UnitAbsentAuthoritatively { detail: join(["LoadState=", unit_load_state_absent_wire], "") } + } else if trim(s: o.load_state.value) == "" { + UnitTerminationUnavailable { detail: "the manager answered the LoadState query with nothing, which establishes neither presence nor absence" } + } else if !o.control_group.queried { + UnitTerminationUnavailable { detail: join(["LoadState=", trim(s: o.load_state.value), " but the control group could not be queried"], "") } + } else if trim(s: o.control_group.value) == "" { + UnitTerminationUnavailable { + detail: join([ + "LoadState=", trim(s: o.load_state.value), + " and the unit holds no control group, which is EITHER a unit that has finished OR a start that is still pending -- the cgroup is realized at spawn -- so it establishes neither", + ], ""), + } + } else if !o.events.queried { + UnitTerminationUnavailable { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be read"], "") } } else { - observe_unit_cgroup(unit: unit, load_state: trim(s: load.value)) + match decode_cgroup_events_populated(body: o.events.value) { + CgroupEventsUnparseable { cause: c } => UnitTerminationUnavailable { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be decoded: ", c], "") } + CgroupEventsDecoded { populated: p } => + if p { + UnitPopulated { detail: join(["processes remain in ", trim(s: o.control_group.value)], "") } + } else { + UnitTerminatedObserved { detail: join(["the realized cgroup ", trim(s: o.control_group.value), " reports populated 0, so it spawned and everything in it exited"], "") } + } + } } } -// THE UNIT EXISTS, SO THE QUESTION IS WHETHER ANYTHING IS STILL RUNNING IN IT, and the kernel -// answers that directly. ControlGroup is empty for a unit that is loaded and holds no processes -- -// systemd removes the cgroup when the last one exits -- so an empty value here is observed -// termination rather than a failed read, and it is the ordinary answer after a clean run. -fn observe_unit_cgroup(unit: NonEmptyStr, load_state: String) -> UnitTermination { +// THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is +// system-scope; these are transient --user units, so asking the system manager would answer +// not-found for EVERY one of them -- a fabricated authoritative absence, the worst answer this fold +// can produce. The property goes through the user-scope show operation for that reason and no other. +// +// THE READS ARE TAKEN UNCONDITIONALLY AND THE FOLD DECIDES. Short-circuiting them here would put +// half the decision in this function and half in the fold, which is the split that let an empty +// property become a conclusion in the first place. +fn observe_unit_termination(identity_hex: String, attempt_completed: Bool) -> UnitTermination { + let unit = compute_unit_name(identity_hex: identity_hex) + let load = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_load_state_property) let cg = systemd.Systemctl.ShowUserProperty(unit: unit, property: unit_control_group_property) - if !cg.success { - UnitTerminationUnavailable { detail: join(["the control group of ", unit as String, " could not be read"], "") } - } else if trim(s: cg.value) == "" { - UnitTerminatedObserved { detail: join([unit as String, " LoadState=", load_state, " holds no control group, so no process of this unit survives"], "") } + let events = if cg.success && trim(s: cg.value) != "" { + linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) } else { - let events = linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: join([cgroup_v2_mount_point as String, trim(s: cg.value)], "") as NonEmptyStr)) - if !events.success { - UnitTerminationUnavailable { detail: join(["the cgroup events of ", unit as String, " could not be read at ", trim(s: cg.value)], "") } - } else { - match decode_cgroup_events_populated(body: events.value) { - CgroupEventsUnparseable { cause: c } => UnitTerminationUnavailable { detail: join(["the cgroup events of ", unit as String, " could not be decoded: ", c], "") } - CgroupEventsDecoded { populated: p } => - if p { - UnitPopulated { detail: join([unit as String, " still has processes in ", trim(s: cg.value)], "") } - } else { - UnitTerminatedObserved { detail: join([unit as String, " cgroup ", trim(s: cg.value), " reports populated 0"], "") } - } - } - } + linux.CgroupV2.ReadEvents(events_path: cgroup_v2_events_path(cgroup_path: cgroup_v2_mount_point)) } + unit_termination_decide(o: UnitObservations { + attempt_completed: attempt_completed, + load_state: UnitPropertyReading { queried: load.success, value: load.value }, + control_group: UnitPropertyReading { queried: cg.success, value: cg.value }, + events: UnitPropertyReading { queried: events.success, value: events.value }, + }) } fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { match t { + UnitAttemptCompleted { detail: _ } => true UnitAbsentAuthoritatively { detail: _ } => true UnitTerminatedObserved { detail: _ } => true UnitPopulated { detail: _ } => false @@ -337,10 +384,11 @@ fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { fn unit_termination_wire(t: UnitTermination) -> String { match t { + UnitAttemptCompleted { detail: d } => join(["attempt completed (", d, ")"], "") UnitAbsentAuthoritatively { detail: d } => join(["absent (", d, ")"], "") UnitTerminatedObserved { detail: d } => join(["terminated (", d, ")"], "") UnitPopulated { detail: d } => join(["STILL POPULATED (", d, "), so the seat stays charged"], "") - UnitTerminationUnavailable { detail: d } => join(["termination UNAVAILABLE (", d, "), so the seat stays charged"], "") + UnitTerminationUnavailable { detail: d } => join(["termination UNESTABLISHED (", d, "), so the seat stays charged"], "") } } @@ -411,6 +459,21 @@ fn compute_basename(path: String) -> String { // outcome and the obligation were written, so a second producer could take the lease and start // against a work directory whose record was still being published -- the exclusion held for the // name and not for the publication it exists to protect. +// THE ATTEMPT'S OWN COMPLETION IS A SEPARATE FACT FROM THE WORK'S ENDING, and only the first one +// licenses a release. A WorkFailed says the unit exited nonzero OR that the launcher never got an +// answer; a successful `systemd-run --wait` says the unit ran to completion, which is the fact the +// termination decision needs. So the ending is what the caller is told and this is what the seat is +// decided on; they are not the same question and were conflated before. +fn compute_attempt_completed(outcome: WorkOutcome) -> Bool { + match outcome { + WorkSucceeded { identity: _, outputs: _, log_path: _ } => true + WorkOutputMismatch { identity: _, detail: _ } => true + WorkFailed { identity: _, exit_code: _, log_path: _ } => false + WorkRefusedByInfrastructure { identity: _, cause: _ } => false + WorkCancelled { identity: _ } => false + } +} + fn compute_run_and_settle( instance: HostDashboardInstance, commit: String, @@ -425,7 +488,7 @@ fn compute_run_and_settle( ) -> ProvideResult { let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) let standing = compute_capacity_standing(instance: instance) - let termination = observe_unit_termination(identity_hex: identity) + let termination = observe_unit_termination(identity_hex: identity, attempt_completed: compute_attempt_completed(outcome: outcome)) let returned = if unit_termination_frees_capacity(t: termination) { compute_release(reservation: reservation, reason: join(["work ", work_outcome_ending(o: outcome), ", unit ", unit_termination_wire(t: termination)], "") as NonEmptyStr) } else { @@ -447,7 +510,58 @@ fn compute_run_and_settle( "reservation: ", compute_release_wire(r: returned), "\n", if dropped.success { "" } else { compute_stale_lease_note(layout: layout, error: dropped.error) }, ], "")) - if measured.success { settled } else { settled } + if measured.success { + settled + } else { + compute_host_record_lost( + layout: layout, identity: identity, subject: subject, op: op, settled: settled, + dropped_ok: dropped.success, dropped_error: dropped.error, error: measured.error) + } +} + +// THE HOST RECORD IS THE ONLY DURABLE CARRIER OF THREE FACTS, so losing it stops the line rather +// than being discarded. The arm used to read `if measured.success { settled } else { settled }` -- +// both arms identical, so a failed write vanished by construction (review 69645), and the +// COMPOUND case was the sharp one: the producer lease failing to drop AND this write failing left +// an identity whose lease is permanently held with NO RECORD ANYWHERE that it is stale, while the +// caller was handed an ordinary success. The module's own prose claimed that stale lease "is +// recorded as the operational fact it is"; the arm let that recording disappear. +// +// WHY THIS DOES NOT REOPEN BLOCKER (7). That blocker was a refusal that DENIED a published +// success -- the caller was told the run failed while the stored record said it succeeded and the +// next caller would attach to that success. This refusal denies nothing: it names the run's real +// ending, says the stored record is published and authoritative, and reports a HOST fault that +// happened after publication. The two callers still read the same thing about the work; what this +// one additionally learns is that the host has state no file describes. +fn compute_host_record_lost( + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + settled: ProvideResult, + dropped_ok: Bool, + dropped_error: String, + error: String, +) -> ProvideResult { + compute_fresh( + outcome: WorkRefusedByInfrastructure { + identity: identity, + cause: StoreUnavailable { + detail: join([ + "the work's own record is published at ", layout.outcome_path, " and is authoritative -- it ended ", + provide_result_ending(r: settled), + " and the next caller reads exactly that -- but this host's record at ", layout.consumption_path, + " could not be written: ", error, + "; so the consumption reading is lost, and ", + if dropped_ok { + "the producer lease was dropped, so nothing else is outstanding" + } else { + join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be requested again"], "") + }, + ], ""), + }, + }, + subject: subject, op: op) } // BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the @@ -457,11 +571,18 @@ fn compute_run_and_settle( // decides: a published success IS the result, the caller is told the same thing the next caller // will read, and the stale lease is recorded as the operational fact it is rather than converted // into a verdict about the work. An operator removes the name; nothing about the run changed. +// +// AND THE NOTE SAYS WHAT THE NAME ACTUALLY BLOCKS. It used to say the next request would refuse as +// ProducerInFlight, which contradicts the very contract this block establishes: a caller reaching a +// published SUCCESS attaches to it, and the attach is decided before the lease is consulted. The +// stale name only blocks a request that must PRODUCE again. fn compute_stale_lease_note(layout: ComputeLayout, error: String) -> String { join([ "STALE PRODUCER LEASE: ", layout.lease_path, " could not be dropped: ", error, - "; the record for this identity is published and is authoritative, and the next request will ", - "refuse as ProducerInFlight until an operator removes the name\n", + "; the record for this identity is published at ", layout.outcome_path, " and is authoritative, ", + "and a caller reaching that record ATTACHES to it -- the attach is decided before this name is ", + "consulted. The stale name matters only to a request that must PRODUCE again, which it will ", + "refuse as ProducerInFlight until an operator removes it\n", ], "") } @@ -534,7 +655,7 @@ fn compute_unreleased_record( // WHAT KEEPS IT FAIL-CLOSED MEANWHILE: every arm that cannot prove the unit is gone keeps the // charge, so the host UNDER-admits. Nothing is over-promised; the pool simply shrinks until an // operator clears the partition by hand. That is the direction a capacity ledger must fail in. -data compute_stuck_reservation_remedy: String = "Until the compute lifecycle capability lands (gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge), the only discharge is an operator clearing this host's compute capacity partition by hand; this host will under-admit until then, which is the fail-closed direction." +data compute_stuck_reservation_remedy: String = "There is NO safe discharge for this seat while this host is admitting work: clearing the capacity partition by hand under live admission removes the record the admission arithmetic is computed from, so the next requests would be admitted against memory this one still holds -- trading a host that under-admits for one that over-promises. The discharge is the compute lifecycle capability (gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge). Any manual intervention before then requires compute admission STOPPED on this host and every gunbc-compute unit PROVEN gone, in that order. Until then this host under-admits, which is the fail-closed direction and is not an incident." // THE ORDER THE TWO HOLDS ARE TAKEN IN// THE ORDER THE TWO HOLDS ARE TAKEN IN, and they are two facts rather than one. The producer lease // excludes a SECOND PRODUCER OF THIS IDENTITY and is a NAME, so an exclusive create settles it; the diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index ee33b3251b6..884a065cb3f 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -424,11 +424,18 @@ fn pool_event_append_wire(a: PoolEventAppend) -> String { // fold it, PROPOSE the release against the folded pool, and append only what the pool admits. A // reference the pool is not holding is a typed refusal that writes nothing. // -// ITS CONSUMER IS THE RECOVERY ROUTE A HELD SEAT NEEDS (gunbc.compute.host_capacity -// compute_release_reference, reached from the operator CLI compute_release_cli). Under -// QuiescenceRequired a term never frees capacity, so an explicit release is the ONLY way a seat -// comes back, and a module claiming that route owes one that executes (review 69595 on gunbc#11962, -// DESIGN 3c). +// ITS CONSUMER IS gunbc.compute.host_capacity compute_release_on, which every compute settlement +// reaches when a run ends and its unit is proven gone. Under QuiescenceRequired a term never frees +// capacity, so an explicit release is the ONLY way a seat comes back. +// +// AN OPERATOR-FACING RECOVERY DOOR IS NOT AMONG ITS CONSUMERS, and this annotation said it was +// until review 69645 caught the citation naming two symbols that do not exist -- they were removed +// when the compute lifecycle was split out, and the comment was not. A citation that does not +// resolve is a citation defect (DESIGN 3), and worse here because host_capacity's own annotation +// states the opposite. Discharging a seat whose release did not land is the capability named by +// gunbc.recurring_failure_mode a_held_compute_seat_has_no_in_corpus_discharge; when it lands it +// will reach this same function, which is why the check below is built to serve a caller that did +// not take the reservation. type SeatRelease = SeatReleased { reference: NonEmptyStr, id: EventId } | SeatNotHeld { reference: NonEmptyStr, wire: String } diff --git a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag index 2857c2a58ff..b137018ba21 100644 --- a/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag +++ b/dag/gunbc/recurring_failure_mode/a_held_compute_seat_has_no_in_corpus_discharge.dag @@ -16,6 +16,8 @@ data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = Recu "SPECIMEN, EXECUTED ON srv1 (2026-09-21, gunbc#11962): a real release build was started under a grant of 27262976 KiB and the gunbc driver was SIGKILLed mid-run. The unit stayed active, the partition carried the acquire and no release, and the producer lease stayed held -- the commitment correctly stayed charged, and nothing in the corpus could return it.", + "THERE IS NO SAFE MANUAL DISCHARGE UNDER LIVE ADMISSION, and saying so is part of the row rather than a caveat on it. Clearing the capacity partition by hand while this host is admitting work removes the very record the admission arithmetic folds, so the next requests are admitted against memory the stuck grant still holds -- trading a host that under-admits for one that over-promises, which is the direction this whole change exists to prevent. Any manual intervention requires compute admission STOPPED on the host and every gunbc-compute unit PROVEN gone, in that order.", + "RUNG FOUND AT: mitigatable. The state is reachable, its harm is bounded to one host's compute appropriation, it is visible in the outcome document and in the partition, and it never over-promises.", "CEILING: mechanically preventable. A discharge route exists to be built -- the evidence needed is a termination decision bound to the unit plus a settled lifecycle state that a retry consumes -- so this is not a class that must stay a ratchet.", @@ -29,6 +31,6 @@ data a_held_compute_seat_has_no_in_corpus_discharge: RecurringFailureMode = Recu decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "compute_stuck_reservation_remedy"), decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "unit_termination_frees_capacity"), decl_ref(module_path: "gunbc.compute.host_capacity", decl_name: "compute_release"), - decl_ref(module_path: "test.claim.compute.work_request_witness", decl_name: "only_proven_absence_or_an_empty_cgroup_frees_a_seat"), + decl_ref(module_path: "test.claim.compute.work_request_witness", decl_name: "only_proven_absence_or_an_ended_cgroup_frees_a_seat"), ], } diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index ac8dbabde15..93878ef609b 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -3,8 +3,9 @@ module test.claim.compute.work_request_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } import gunbc.compute.work_provider_local { - UnitTermination, UnitAbsentAuthoritatively, UnitTerminatedObserved, UnitPopulated, UnitTerminationUnavailable, - unit_termination_frees_capacity, unit_termination_wire, + UnitTermination, UnitAttemptCompleted, UnitAbsentAuthoritatively, UnitTerminatedObserved, UnitPopulated, UnitTerminationUnavailable, + UnitPropertyReading, UnitObservations, unit_termination_decide, + unit_termination_frees_capacity, unit_termination_wire, compute_attempt_completed, } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, @@ -79,22 +80,49 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool && (match stored_outcome_decode(text: "not json") { StoredOutcomeUnreadable { reason: _ } => true StoredOutcomeFound { ending: _, identity: _, document: _ } => false }) } -// ONLY PROVEN ABSENCE OR AN OBSERVED-EMPTY CGROUP MAY FREE A SEAT, and the two arms this cell -// exists to pin RED are the ones an earlier revision got wrong: `failed` can coexist with live -// processes while a unit's stop reaches its SIGKILL timeout, and an empty ActiveState is the -// manager declining to answer rather than authoritative absence. Neither may free capacity. The -// unavailable arm is the one that matters most -- it is what a lost launcher, an unreadable manager -// and a missing cgroup all look like, and reading any of them as permission is the absorbing -// fallback DESIGN 5 forbids. -test fn only_proven_absence_or_an_empty_cgroup_frees_a_seat() -> Bool { - let absent = UnitAbsentAuthoritatively { detail: "LoadState=not-found" } - let terminated = UnitTerminatedObserved { detail: "populated 0" } - let populated = UnitPopulated { detail: "populated 1" } - let unavailable = UnitTerminationUnavailable { detail: "the manager could not be read" } - unit_termination_frees_capacity(t: absent) - && unit_termination_frees_capacity(t: terminated) +// THE CONTROLS SIT AT THE OBSERVATION-TO-DECISION BOUNDARY, not at the variant-to-release mapping, +// because the mapping cannot see the defects that actually occurred: a failed query whose TEXT +// looks like an answer, a property that came back EMPTY, and a start that is still PENDING all +// arrive as readings and are wrong only in how they are read (side-chat review at 5ee0cce). +// +// THE PENDING-START ROW IS THE ONE THIS CELL EXISTS FOR. An empty ControlGroup was read as observed +// termination, and it equally describes a unit whose start has not spawned yet -- systemd realizes +// the cgroup AT SPAWN. That made a whole unbacked launch reachable: reserve, start pending, the +// LAUNCHER lost while this driver lives on, settlement sees the empty property, releases, and the +// job then starts against memory nobody holds. It must establish nothing. +test fn only_proven_absence_or_an_ended_cgroup_frees_a_seat() -> Bool { + let no_events = UnitPropertyReading { queried: false, value: "" } + let completed = unit_termination_decide(o: UnitObservations { attempt_completed: true, load_state: UnitPropertyReading { queried: false, value: "" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) + let absent = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "not-found" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) + let misleading = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: false, value: "Failed to connect to bus: not-found" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) + let empty_load = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 0\n" } }) + let pending = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "" }, events: no_events }) + let populated = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 1\n" } }) + let failed_unit = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "failed" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 1\n" } }) + let ended = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 0\n" } }) + let undecodable = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "garbage\n" } }) + unit_termination_frees_capacity(t: completed) + && unit_termination_frees_capacity(t: absent) + && unit_termination_frees_capacity(t: ended) + && !unit_termination_frees_capacity(t: misleading) + && !unit_termination_frees_capacity(t: empty_load) + && !unit_termination_frees_capacity(t: pending) && !unit_termination_frees_capacity(t: populated) - && !unit_termination_frees_capacity(t: unavailable) + && !unit_termination_frees_capacity(t: failed_unit) + && !unit_termination_frees_capacity(t: undecodable) + && string_contains(s: unit_termination_wire(t: pending), pattern: "pending") + && string_contains(s: unit_termination_wire(t: misleading), pattern: "UNESTABLISHED") && string_contains(s: unit_termination_wire(t: populated), pattern: "STILL POPULATED") - && string_contains(s: unit_termination_wire(t: unavailable), pattern: "UNAVAILABLE") +} + +// A COMPLETED ATTEMPT RELEASES, AND AN ATTEMPT THAT DID NOT COMPLETE DOES NOT RELEASE ON ITS OWN. +// The work's ENDING and the attempt's COMPLETION are different questions: a WorkFailed says the +// unit exited nonzero OR that the launcher never got an answer, and only the second is a lost +// launcher. The ending is what the caller is told; completion is what the seat is decided on. +test fn the_attempt_completion_fact_is_not_the_works_ending() -> Bool { + compute_attempt_completed(outcome: WorkSucceeded { identity: "a", outputs: [], log_path: "/l" }) + && compute_attempt_completed(outcome: WorkOutputMismatch { identity: "a", detail: "d" }) + && !compute_attempt_completed(outcome: WorkFailed { identity: "a", exit_code: 1, log_path: "/l" }) + && !compute_attempt_completed(outcome: WorkCancelled { identity: "a" }) + && !compute_attempt_completed(outcome: WorkRefusedByInfrastructure { identity: "a", cause: HostComputeCapacityFull { wire: "pool-full" } }) } From 73c570f9afa41f6b179367e5e1fac3b0dad4487d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 19:39:45 +0000 Subject: [PATCH 09/28] A completed wait never overrides a live cgroup; an uncertain completion keeps the charge; the record and the caller stop disagreeing Side-chat review at 773825cf, plus review 69663. All four verified first. (1) THE ORDER OF THE TESTS WAS THE DEFECT. attempt_completed was checked FIRST and returned immediately, so {completed, populated 1} RELEASED -- a launcher's success overriding the kernel saying processes are still there. `--wait` returns when the MAIN process exits, and under KillMode=process or none a forked child outlives it in the unit's cgroup. Both halves of the reviewer's either/or are taken, because each is independently right. KillMode=control-group is now BOUND on the invocation rather than inherited -- a default is not a declaration, and a drop-in could change it while this provider went on treating a completed wait as an empty cgroup. AND contrary evidence wins regardless: a positive populated=1 refuses the release before anything else is considered and is never discarded. (2) A MOMENTARILY EMPTY CGROUP IS NOT TERMINAL. populated=0 with completion NOT established used to release, and a realized cgroup can be empty between execs. In this PR an uncertain completion keeps the charge; the population reading now only CONFIRMS a completion or REFUSES one. UnitTerminatedObserved is gone as a releasing arm. Establishing termination without a completed attempt is the lifecycle capability's job. (3) THE RECORD AND THE CALLER DISAGREED AGAIN, one layer along: the lost-record repair published the real outcome and synthesized a WorkRefusedByInfrastructure for the caller, so the caller and the next attacher read different contracts -- the same fusion as before, in the opposite direction. The two facts are now returned as two. ProvideResult carries the stored outcome UNCHANGED plus a HostRecordStatus beside it, so re-requesting an identity yields exactly the ending the first caller was told. The CLI turns a lost record into a nonzero exit naming both halves, which is how the recording failure stays visible without becoming a verdict about work that really ran. review 69663: seat_release_landed had no consumer anywhere in the tree -- compute_release_on matches SeatRelease structurally and the provider uses its own compute_release_landed. Deleted rather than given a consumer, because the carrier already answers the question by match (DESIGN 3c). Controls, all at the observation-to-decision boundary: a completed wait with populated=1 keeps the charge; populated=0 without completion keeps the charge; a pending start establishes nothing; a failed query whose text reads like an answer is unavailable; and a lost host record changes no contract the next caller reads. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 227 ++++++++++++------ dag/gunbc/fabric/fabric_event_log.dag | 9 - .../compute/work_request_witness_test.dag | 79 ++++-- 3 files changed, 218 insertions(+), 97 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 3fcd3b6b23f..61bc72840c5 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -152,34 +152,72 @@ fn compute_ensure_dir(instance: HostDashboardInstance, path: String) -> Bool { } // What a provide returns: a fresh run's outcome, or an attachment to a stored one (no run). +// THE WORK'S RESULT AND THIS HOST'S RECORD ARE TWO FACTS AND ARE RETURNED AS TWO. +// +// They were fused twice, in opposite directions, and both were wrong. First a successful run whose +// release failed handed the caller a REFUSAL while outcome.json said it succeeded -- so the caller +// and the next attacher read different contracts. Then the repair for a lost host record did the +// same thing again: it published the real outcome and synthesized a WorkRefusedByInfrastructure for +// the caller (side-chat review at 773825cf). The stored document is what a later caller attaches +// to, so a returned outcome that differs from it is a disagreement no wording can fix. +// +// So the outcome carried here is ALWAYS the outcome that was stored, and whether this host managed +// to record its own side facts rides beside it. Both consumers get the same contract about the +// work; only this caller learns that the host lost its record, which is the only consumer that +// could act on it anyway. +type HostRecordStatus + = HostRecordWritten + | HostRecordLost { detail: String } + type ProvideResult - = ProvideFresh { outcome: WorkOutcome, document: String } + = ProvideFresh { outcome: WorkOutcome, document: String, host_record: HostRecordStatus } | ProvideAttached { identity: String, ending: String, document: String } +fn provide_result_host_record(r: ProvideResult) -> HostRecordStatus { + match r { + ProvideFresh { outcome: _, document: _, host_record: h } => h + ProvideAttached { identity: _, ending: _, document: _ } => HostRecordWritten + } +} + +fn host_record_wire(h: HostRecordStatus) -> String { + match h { + HostRecordWritten => "" + HostRecordLost { detail: d } => join(["; HOST RECORD LOST: ", d], "") + } +} + +fn host_record_was_written(h: HostRecordStatus) -> Bool { + match h { + HostRecordWritten => true + HostRecordLost { detail: _ } => false + } +} + fn provide_result_succeeded(r: ProvideResult) -> Bool { match r { - ProvideFresh { outcome, document: _ } => work_outcome_ending(o: outcome) == "succeeded" + ProvideFresh { outcome, document: _, host_record: _ } => work_outcome_ending(o: outcome) == "succeeded" ProvideAttached { identity: _, ending, document: _ } => ending == "succeeded" } } fn provide_result_identity(r: ProvideResult) -> String { match r { - ProvideFresh { outcome, document: _ } => gunbc.compute.work_request.work_outcome_identity(o: outcome) + ProvideFresh { outcome, document: _, host_record: _ } => gunbc.compute.work_request.work_outcome_identity(o: outcome) ProvideAttached { identity, ending: _, document: _ } => identity } } fn provide_result_ending(r: ProvideResult) -> String { match r { - ProvideFresh { outcome, document: _ } => work_outcome_ending(o: outcome) + ProvideFresh { outcome, document: _, host_record: _ } => work_outcome_ending(o: outcome) ProvideAttached { identity: _, ending, document: _ } => ending } } fn provide_result_document(r: ProvideResult) -> String { match r { - ProvideFresh { outcome: _, document } => document + ProvideFresh { outcome: _, document, host_record: _ } => document ProvideAttached { identity: _, ending: _, document } => document } } @@ -212,6 +250,20 @@ fn compute_unit_name(identity_hex: String) -> NonEmptyStr { join(["gunbc-compute-", identity_hex], "") as NonEmptyStr } +// THE TERMINATION POLICY IS BOUND, NOT INHERITED, and that is what makes a successful wait mean +// anything about the cgroup. `systemd-run --wait` returns when the MAIN process exits; under +// KillMode=process or none a forked child survives in the unit's cgroup and goes on using the +// memory this seat is accounting for. control-group is systemd's default, but a default is not a +// declaration -- a drop-in or a future default could change it, and this provider would keep +// treating a completed wait as an empty cgroup (side-chat review at 773825cf). Binding it makes the +// policy a fact of the invocation. +// +// IT IS BOUND *AND* THE EVIDENCE IS STILL REQUIRED. A positive populated=1 is never discarded in +// favour of a completed wait, because a bound policy says what systemd will do and the cgroup says +// what is true. +data compute_kill_mode_property: String = "KillMode" +data compute_kill_mode_value: String = "control-group" + // Run the operation as a transient user unit and wait for it: the unit carries THE MEMORY THE POOL // GRANTED -- not a ceiling read independently from the instance, which would let the reservation and // the enforcement drift apart -- the checkout as working directory, the shared per-host target @@ -228,6 +280,7 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden unit: compute_unit_name(identity_hex: identity_hex), properties: [ systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: kibibyte_to_byte_size(k: granted)))), + systemd_run_property(name: compute_kill_mode_property, value: compute_kill_mode_value), systemd_run_property(name: "WorkingDirectory", value: layout.checkout), ], setenv_bindings: [ @@ -283,7 +336,6 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden type UnitTermination = UnitAttemptCompleted { detail: String } | UnitAbsentAuthoritatively { detail: String } - | UnitTerminatedObserved { detail: String } | UnitPopulated { detail: String } | UnitTerminationUnavailable { detail: String } @@ -314,39 +366,71 @@ type UnitObservations { // A FAILED QUERY IS UNAVAILABLE WHATEVER IT PRINTED. systemctl writes diagnostics that read like // answers -- "Failed to connect to bus", "not-found" inside a sentence -- so the exit status // decides first and the text is never consulted on a failed query. -fn unit_termination_decide(o: UnitObservations) -> UnitTermination { - if o.attempt_completed { - UnitAttemptCompleted { detail: "systemd-run --wait returned success, so the unit it started ran to completion" } - } else if !o.load_state.queried { - UnitTerminationUnavailable { detail: "the user manager could not be queried for this unit, so nothing about it is established" } - } else if trim(s: o.load_state.value) == unit_load_state_absent_wire { - UnitAbsentAuthoritatively { detail: join(["LoadState=", unit_load_state_absent_wire], "") } - } else if trim(s: o.load_state.value) == "" { - UnitTerminationUnavailable { detail: "the manager answered the LoadState query with nothing, which establishes neither presence nor absence" } - } else if !o.control_group.queried { - UnitTerminationUnavailable { detail: join(["LoadState=", trim(s: o.load_state.value), " but the control group could not be queried"], "") } +// WHAT THE POPULATION READING ESTABLISHES, AS ITS OWN THREE-VALUED FACT. It has to be separable +// from the decision because the decision now consults it TWICE -- once to refuse a release outright, +// and once to confirm one -- and "not positively populated" is not the same as "positively empty". +type UnitPopulation + = PopulationOccupied { detail: String } + | PopulationEmpty { detail: String } + | PopulationUnknown { detail: String } + +fn unit_population(o: UnitObservations) -> UnitPopulation { + if !o.control_group.queried { + PopulationUnknown { detail: "the control group could not be queried" } } else if trim(s: o.control_group.value) == "" { - UnitTerminationUnavailable { - detail: join([ - "LoadState=", trim(s: o.load_state.value), - " and the unit holds no control group, which is EITHER a unit that has finished OR a start that is still pending -- the cgroup is realized at spawn -- so it establishes neither", - ], ""), - } + PopulationUnknown { detail: "the unit holds no control group, which is EITHER a unit that has finished OR a start that is still pending -- the cgroup is realized at spawn" } } else if !o.events.queried { - UnitTerminationUnavailable { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be read"], "") } + PopulationUnknown { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be read"], "") } } else { match decode_cgroup_events_populated(body: o.events.value) { - CgroupEventsUnparseable { cause: c } => UnitTerminationUnavailable { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be decoded: ", c], "") } + CgroupEventsUnparseable { cause: c } => PopulationUnknown { detail: join(["the cgroup events of ", trim(s: o.control_group.value), " could not be decoded: ", c], "") } CgroupEventsDecoded { populated: p } => if p { - UnitPopulated { detail: join(["processes remain in ", trim(s: o.control_group.value)], "") } + PopulationOccupied { detail: join(["processes remain in ", trim(s: o.control_group.value)], "") } } else { - UnitTerminatedObserved { detail: join(["the realized cgroup ", trim(s: o.control_group.value), " reports populated 0, so it spawned and everything in it exited"], "") } + PopulationEmpty { detail: join([trim(s: o.control_group.value), " reports populated 0"], "") } } } } } +// THE ORDER OF THESE TESTS IS THE SAFETY PROPERTY, and the previous order was the defect. Completion +// was checked FIRST and returned immediately, so {completed, populated 1} RELEASED -- a completed +// wait overriding the kernel telling us processes are still there. Contrary evidence now wins: +// a positive population refuses the release before anything else is considered, and is never +// discarded (side-chat review at 773825cf). +// +// AND A MOMENTARILY EMPTY CGROUP IS NOT TERMINAL ON ITS OWN. populated=0 with completion NOT +// established used to release; a realized cgroup can be empty between execs, and emptiness does not +// cover a pending job. So in this PR an uncertain completion KEEPS THE CHARGE, and the population +// reading serves only to CONFIRM a completion or to REFUSE one. Establishing termination without a +// completed attempt is the lifecycle capability's job, not this one's. +fn unit_termination_decide(o: UnitObservations) -> UnitTermination { + let population = unit_population(o: o) + match population { + PopulationOccupied { detail: d } => UnitPopulated { detail: d } + _ => + if o.load_state.queried && trim(s: o.load_state.value) == unit_load_state_absent_wire { + UnitAbsentAuthoritatively { detail: join(["LoadState=", unit_load_state_absent_wire, ", so the manager has no record of this unit"], "") } + } else if !o.attempt_completed { + UnitTerminationUnavailable { + detail: join([ + "the attempt did not complete, so its end is not established -- ", + match population { PopulationEmpty { detail: pd } => join([pd, ", but a realized cgroup can be empty between execs and emptiness does not cover a pending job"], "") PopulationUnknown { detail: pd } => pd PopulationOccupied { detail: pd } => pd }, + ], ""), + } + } else { + UnitAttemptCompleted { + detail: join([ + "systemd-run --wait returned success under ", compute_kill_mode_property, "=", compute_kill_mode_value, + ", so the unit ran to completion and its cgroup was torn down; ", + match population { PopulationEmpty { detail: pd } => pd PopulationUnknown { detail: pd } => join(["population unread (", pd, ") and not positively occupied"], "") PopulationOccupied { detail: pd } => pd }, + ], ""), + } + } + } +} + // THE USER MANAGER IS ASKED, NOT THE SYSTEM ONE. systemctl_show_load_state_argv exists and is // system-scope; these are transient --user units, so asking the system manager would answer // not-found for EVERY one of them -- a fabricated authoritative absence, the worst answer this fold @@ -376,7 +460,6 @@ fn unit_termination_frees_capacity(t: UnitTermination) -> Bool { match t { UnitAttemptCompleted { detail: _ } => true UnitAbsentAuthoritatively { detail: _ } => true - UnitTerminatedObserved { detail: _ } => true UnitPopulated { detail: _ } => false UnitTerminationUnavailable { detail: _ } => false } @@ -386,7 +469,6 @@ fn unit_termination_wire(t: UnitTermination) -> String { match t { UnitAttemptCompleted { detail: d } => join(["attempt completed (", d, ")"], "") UnitAbsentAuthoritatively { detail: d } => join(["absent (", d, ")"], "") - UnitTerminatedObserved { detail: d } => join(["terminated (", d, ")"], "") UnitPopulated { detail: d } => join(["STILL POPULATED (", d, "), so the seat stays charged"], "") UnitTerminationUnavailable { detail: d } => join(["termination UNESTABLISHED (", d, "), so the seat stays charged"], "") } @@ -513,57 +595,48 @@ fn compute_run_and_settle( if measured.success { settled } else { - compute_host_record_lost( - layout: layout, identity: identity, subject: subject, op: op, settled: settled, + compute_with_lost_host_record( + settled: settled, layout: layout, dropped_ok: dropped.success, dropped_error: dropped.error, error: measured.error) } } -// THE HOST RECORD IS THE ONLY DURABLE CARRIER OF THREE FACTS, so losing it stops the line rather -// than being discarded. The arm used to read `if measured.success { settled } else { settled }` -- -// both arms identical, so a failed write vanished by construction (review 69645), and the -// COMPOUND case was the sharp one: the producer lease failing to drop AND this write failing left -// an identity whose lease is permanently held with NO RECORD ANYWHERE that it is stale, while the -// caller was handed an ordinary success. The module's own prose claimed that stale lease "is -// recorded as the operational fact it is"; the arm let that recording disappear. +// THE HOST RECORD IS THE ONLY DURABLE CARRIER OF THREE FACTS, so losing it is reported rather than +// discarded -- and reported BESIDE the work's result, not instead of it. // -// WHY THIS DOES NOT REOPEN BLOCKER (7). That blocker was a refusal that DENIED a published -// success -- the caller was told the run failed while the stored record said it succeeded and the -// next caller would attach to that success. This refusal denies nothing: it names the run's real -// ending, says the stored record is published and authoritative, and reports a HOST fault that -// happened after publication. The two callers still read the same thing about the work; what this -// one additionally learns is that the host has state no file describes. -fn compute_host_record_lost( - layout: ComputeLayout, - identity: String, - subject: WorkSubject, - op: WorkOperation, +// TWO DEFECTS MET HERE. The arm read `if measured.success { settled } else { settled }`, so a failed +// write vanished by construction (review 69645); and the compound case was the sharp one, because +// the stale-lease note lives only in this file -- a lease that failed to drop AND a write that +// failed left an identity whose lease is permanently held with NO RECORD ANYWHERE. The repair then +// over-corrected: it synthesized a WorkRefusedByInfrastructure for the caller while outcome.json +// said the run had succeeded, which is the stored-record disagreement again, one layer along +// (side-chat review at 773825cf). +// +// So the outcome is untouched -- the caller reads exactly what the next attacher will read -- and +// the lost record rides as its own fact. The CLI turns that fact into a nonzero exit with both +// halves named, which is how a recording failure stays visible without becoming a verdict about +// work that really did run. +fn compute_with_lost_host_record( settled: ProvideResult, + layout: ComputeLayout, dropped_ok: Bool, dropped_error: String, error: String, ) -> ProvideResult { - compute_fresh( - outcome: WorkRefusedByInfrastructure { - identity: identity, - cause: StoreUnavailable { - detail: join([ - "the work's own record is published at ", layout.outcome_path, " and is authoritative -- it ended ", - provide_result_ending(r: settled), - " and the next caller reads exactly that -- but this host's record at ", layout.consumption_path, - " could not be written: ", error, - "; so the consumption reading is lost, and ", - if dropped_ok { - "the producer lease was dropped, so nothing else is outstanding" - } else { - join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be requested again"], "") - }, - ], ""), - }, + let detail = join([ + "this host's record at ", layout.consumption_path, " could not be written: ", error, + "; the consumption reading is lost, and ", + if dropped_ok { + "the producer lease was dropped, so nothing else is outstanding" + } else { + join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be produced again"], "") }, - subject: subject, op: op) + ], "") + match settled { + ProvideFresh { outcome: o, document: d, host_record: _ } => ProvideFresh { outcome: o, document: d, host_record: HostRecordLost { detail: detail } } + ProvideAttached { identity: i, ending: e, document: d } => ProvideAttached { identity: i, ending: e, document: d } + } } - // BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the // caller a refusal while the outcome document already said the run had SUCCEEDED, and then state // that the next request would refuse as ProducerInFlight -- three claims that cannot all be true, @@ -598,7 +671,7 @@ fn compute_settled_record( let document = work_outcome_wire(o: outcome, subject: subject, op: op) let written = Filesystem.Write(path: layout.outcome_path, content: document) if written.success { - ProvideFresh { outcome: outcome, document: document } + ProvideFresh { outcome: outcome, document: document, host_record: HostRecordWritten } } else { compute_fresh( outcome: WorkRefusedByInfrastructure { @@ -639,7 +712,7 @@ fn compute_unreleased_record( let document = work_outcome_wire(o: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) let written = Filesystem.Write(path: layout.outcome_path, content: document) if written.success { - ProvideFresh { outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, document: document } + ProvideFresh { outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, document: document, host_record: HostRecordWritten } } else { compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: cause }, subject: subject, op: op) } @@ -781,7 +854,7 @@ fn compute_release_wire(r: ComputeRelease) -> String { } fn compute_fresh(outcome: WorkOutcome, subject: WorkSubject, op: WorkOperation) -> ProvideResult { - ProvideFresh { outcome: outcome, document: work_outcome_wire(o: outcome, subject: subject, op: op) } + ProvideFresh { outcome: outcome, document: work_outcome_wire(o: outcome, subject: subject, op: op), host_record: HostRecordWritten } } fn compute_run_leased( @@ -880,9 +953,19 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent Absent => ExitFailure { code: 2, reason: join(["compute: operation must be build, compile or claims (compile and claims need entry; claims needs functions and hermetic=true|false); got ", operation], "") } Present { value: op } => { let result = compute_provide(instance: instance, commit: commit, op: op) - let receipt = Filesystem.Write(path: receipt_path, content: provide_result_document(r: result)) + let record = provide_result_host_record(r: result) + let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), host_record_wire(h: record), "\n"], "")) if !receipt.success { ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; outcome was ", provide_result_document(r: result)], "") } + } else if !host_record_was_written(h: record) { + ExitFailure { + code: 1, + reason: join([ + "compute: the work ended ", provide_result_ending(r: result), + " and its record at the store is authoritative -- a later request for this identity reads exactly that -- but ", + host_record_wire(h: record), + ], ""), + } } else if provide_result_succeeded(r: result) { ExitSuccess } else { diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index 884a065cb3f..5b109c419f3 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -503,15 +503,6 @@ fn seat_release_wire(r: SeatRelease) -> String { } } -fn seat_release_landed(r: SeatRelease) -> Bool { - match r { - SeatReleased { reference: _, id: _ } => true - SeatNotHeld { reference: _, wire: _ } => false - SeatReleaseContended { attempts: _ } => false - SeatReleaseRefused { step: _, reason: _ } => false - } -} - fn seat_acquisition_wire(a: SeatAcquisition) -> String { match a { SeatGranted { grant: g, generation: gen, attempts_used: n } => join(["granted ", g.reference as String, " fence=", g.fence.grant as String, "@", to_string(gen), " expires_at=", to_string(g.expires_at), " attempt=", to_string(n)], "") diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 93878ef609b..8392c547723 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -3,10 +3,14 @@ module test.claim.compute.work_request_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, FilePath } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex } import gunbc.compute.work_provider_local { - UnitTermination, UnitAttemptCompleted, UnitAbsentAuthoritatively, UnitTerminatedObserved, UnitPopulated, UnitTerminationUnavailable, + UnitTermination, UnitAttemptCompleted, UnitAbsentAuthoritatively, UnitPopulated, UnitTerminationUnavailable, UnitPropertyReading, UnitObservations, unit_termination_decide, unit_termination_frees_capacity, unit_termination_wire, compute_attempt_completed, + ProvideResult, ProvideFresh, ProvideAttached, HostRecordStatus, HostRecordWritten, HostRecordLost, + provide_result_host_record, provide_result_ending, provide_result_document, host_record_was_written, host_record_wire, + compute_with_lost_host_record, ComputeLayout, compute_layout, } +import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, WorkToolchainIdentity, work_identity, work_identity_hex, work_default_build, work_declared_outputs, @@ -85,34 +89,56 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool // looks like an answer, a property that came back EMPTY, and a start that is still PENDING all // arrive as readings and are wrong only in how they are read (side-chat review at 5ee0cce). // +// A COMPLETED WAIT NEVER OVERRIDES A POSITIVE POPULATION, which is the row the previous revision +// got backwards: completion was tested FIRST and returned immediately, so {completed, populated 1} +// RELEASED -- a launcher's success overriding the kernel saying processes are still there. Under +// KillMode=process or none a forked child outlives the main process exactly like that. Contrary +// evidence wins now, and a positive populated=1 is never discarded. +// +// AND A MOMENTARILY EMPTY CGROUP IS NOT TERMINAL ON ITS OWN: populated=0 WITHOUT an established +// completion used to release, and a realized cgroup can be empty between execs. Uncertain +// completion keeps the charge. +// // THE PENDING-START ROW IS THE ONE THIS CELL EXISTS FOR. An empty ControlGroup was read as observed // termination, and it equally describes a unit whose start has not spawned yet -- systemd realizes // the cgroup AT SPAWN. That made a whole unbacked launch reachable: reserve, start pending, the // LAUNCHER lost while this driver lives on, settlement sees the empty property, releases, and the // job then starts against memory nobody holds. It must establish nothing. +fn reading(queried: Bool, value: String) -> UnitPropertyReading { + UnitPropertyReading { queried: queried, value: value } +} + +fn decided(done: Bool, load: UnitPropertyReading, cg: UnitPropertyReading, ev: UnitPropertyReading) -> UnitTermination { + unit_termination_decide(o: UnitObservations { attempt_completed: done, load_state: load, control_group: cg, events: ev }) +} + test fn only_proven_absence_or_an_ended_cgroup_frees_a_seat() -> Bool { - let no_events = UnitPropertyReading { queried: false, value: "" } - let completed = unit_termination_decide(o: UnitObservations { attempt_completed: true, load_state: UnitPropertyReading { queried: false, value: "" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) - let absent = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "not-found" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) - let misleading = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: false, value: "Failed to connect to bus: not-found" }, control_group: UnitPropertyReading { queried: false, value: "" }, events: no_events }) - let empty_load = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 0\n" } }) - let pending = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "" }, events: no_events }) - let populated = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 1\n" } }) - let failed_unit = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "failed" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 1\n" } }) - let ended = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "populated 0\n" } }) - let undecodable = unit_termination_decide(o: UnitObservations { attempt_completed: false, load_state: UnitPropertyReading { queried: true, value: "loaded" }, control_group: UnitPropertyReading { queried: true, value: "/x" }, events: UnitPropertyReading { queried: true, value: "garbage\n" } }) - unit_termination_frees_capacity(t: completed) + let no_events = reading(queried: false, value: "") + let unread = reading(queried: false, value: "") + let completed_empty = decided(done: true, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 0\n")) + let completed_populated = decided(done: true, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 1\n")) + let completed_unknown = decided(done: true, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: ""), ev: no_events) + let absent = decided(done: false, load: reading(queried: true, value: "not-found"), cg: unread, ev: no_events) + let misleading = decided(done: false, load: reading(queried: false, value: "Failed to connect to bus: not-found"), cg: unread, ev: no_events) + let empty_load = decided(done: false, load: reading(queried: true, value: ""), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 0\n")) + let pending = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: ""), ev: no_events) + let empty_no_completion = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 0\n")) + let populated_no_completion = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 1\n")) + let undecodable = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "garbage\n")) + unit_termination_frees_capacity(t: completed_empty) + && unit_termination_frees_capacity(t: completed_unknown) && unit_termination_frees_capacity(t: absent) - && unit_termination_frees_capacity(t: ended) + && !unit_termination_frees_capacity(t: completed_populated) + && !unit_termination_frees_capacity(t: empty_no_completion) + && !unit_termination_frees_capacity(t: populated_no_completion) && !unit_termination_frees_capacity(t: misleading) && !unit_termination_frees_capacity(t: empty_load) && !unit_termination_frees_capacity(t: pending) - && !unit_termination_frees_capacity(t: populated) - && !unit_termination_frees_capacity(t: failed_unit) && !unit_termination_frees_capacity(t: undecodable) + && string_contains(s: unit_termination_wire(t: completed_populated), pattern: "STILL POPULATED") + && string_contains(s: unit_termination_wire(t: empty_no_completion), pattern: "did not complete") && string_contains(s: unit_termination_wire(t: pending), pattern: "pending") && string_contains(s: unit_termination_wire(t: misleading), pattern: "UNESTABLISHED") - && string_contains(s: unit_termination_wire(t: populated), pattern: "STILL POPULATED") } // A COMPLETED ATTEMPT RELEASES, AND AN ATTEMPT THAT DID NOT COMPLETE DOES NOT RELEASE ON ITS OWN. @@ -126,3 +152,24 @@ test fn the_attempt_completion_fact_is_not_the_works_ending() -> Bool { && !compute_attempt_completed(outcome: WorkCancelled { identity: "a" }) && !compute_attempt_completed(outcome: WorkRefusedByInfrastructure { identity: "a", cause: HostComputeCapacityFull { wire: "pool-full" } }) } + +// THE STORED RECORD AND THE RETURNED OUTCOME AGREE, WHATEVER HAPPENED TO THIS HOST'S OWN RECORD. +// The fusion was made twice in opposite directions -- a refusal handed to the caller over a stored +// success, then a synthesized refusal over a published outcome -- and both times the next attacher +// read something different from the caller. Losing the host record now changes the host_record +// fact and NOTHING about the work, so re-requesting the identity yields the same ending the first +// caller was told; what the first caller additionally gets is the recording failure, which is the +// only consumer that could act on it. +test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { + let succeeded = WorkSucceeded { identity: "abc", outputs: [], log_path: "/l" } + let document = work_outcome_wire(o: succeeded, subject: tree(hex: tree_a), op: work_default_build()) + let settled = ProvideFresh { outcome: succeeded, document: document, host_record: HostRecordWritten } + let layout = compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc") + let lost = compute_with_lost_host_record(settled: settled, layout: layout, dropped_ok: false, dropped_error: "EBUSY", error: "ENOSPC") + provide_result_ending(r: lost) == provide_result_ending(r: settled) + && provide_result_document(r: lost) == provide_result_document(r: settled) + && provide_result_ending(r: lost) == "succeeded" + && host_record_was_written(h: provide_result_host_record(r: settled)) + && !host_record_was_written(h: provide_result_host_record(r: lost)) + && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost)), pattern: "STILL HELD") +} From 1248df9502e3d37d1896aaba87df1b8522b8f32b Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 19:56:54 +0000 Subject: [PATCH 10/28] The bound termination policy is a fold a control can read, not a sentence A completed wait may confirm a release only because KillMode=control-group means systemd tears the cgroup down with the unit. Spelled inline at the call, that was an assertion nothing checked; the properties are a fold now, and the_unit_binds_the_termination_policy_the_release_decision_relies_on is what makes "bound" a fact. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 18 ++++++++++++----- .../compute/work_request_witness_test.dag | 20 ++++++++++++++++++- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 61bc72840c5..20abb8159b2 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -264,6 +264,18 @@ fn compute_unit_name(identity_hex: String) -> NonEmptyStr { data compute_kill_mode_property: String = "KillMode" data compute_kill_mode_value: String = "control-group" +// THE PROPERTIES ARE A FOLD SO THE BOUND POLICY IS CHECKABLE. A termination policy that is only +// asserted in prose is exactly the thing the release decision must not trust, and the whole point +// of binding it is that the decision leans on it -- so the invocation's properties are produced +// where a control can read them rather than spelled inline at the call. +fn compute_unit_properties(layout: ComputeLayout, granted: Kibibyte) -> List { + [ + systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: kibibyte_to_byte_size(k: granted)))), + systemd_run_property(name: compute_kill_mode_property, value: compute_kill_mode_value), + systemd_run_property(name: "WorkingDirectory", value: layout.checkout), + ] +} + // Run the operation as a transient user unit and wait for it: the unit carries THE MEMORY THE POOL // GRANTED -- not a ceiling read independently from the instance, which would let the reservation and // the enforcement drift apart -- the checkout as working directory, the shared per-host target @@ -278,11 +290,7 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden workdir: layout.checkout, args: systemd_run_user_wait_arguments( unit: compute_unit_name(identity_hex: identity_hex), - properties: [ - systemd_run_property(name: "MemoryMax", value: to_string(value: byte_size_count(b: kibibyte_to_byte_size(k: granted)))), - systemd_run_property(name: compute_kill_mode_property, value: compute_kill_mode_value), - systemd_run_property(name: "WorkingDirectory", value: layout.checkout), - ], + properties: compute_unit_properties(layout: layout, granted: granted), setenv_bindings: [ join(["CARGO_TARGET_DIR=", layout.target_dir], ""), join(["RUSTC_WRAPPER=", sccache_installed_binary_path], ""), diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 8392c547723..4704bc5ac96 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -8,8 +8,10 @@ import gunbc.compute.work_provider_local { unit_termination_frees_capacity, unit_termination_wire, compute_attempt_completed, ProvideResult, ProvideFresh, ProvideAttached, HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, provide_result_ending, provide_result_document, host_record_was_written, host_record_wire, - compute_with_lost_host_record, ComputeLayout, compute_layout, + compute_with_lost_host_record, ComputeLayout, compute_layout, compute_unit_properties, + compute_kill_mode_property, compute_kill_mode_value, } +import std.measure { kibibyte } import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance } import gunbc.compute.work_request { WorkSubject, ExactTree, WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, @@ -173,3 +175,19 @@ test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { && !host_record_was_written(h: provide_result_host_record(r: lost)) && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost)), pattern: "STILL HELD") } + +// THE TERMINATION POLICY THE RELEASE DECISION LEANS ON IS ACTUALLY BOUND ON THE INVOCATION. A +// completed wait is allowed to confirm a release only because KillMode=control-group means systemd +// tears the cgroup down with the unit; under process or none a forked child outlives the main +// process and goes on using the memory this seat accounts for. systemd's DEFAULT is +// control-group -- but a default is not a declaration, and a drop-in or a future default could +// change it while this provider kept reasoning from the old one. So it is bound, and this cell is +// what makes "bound" a fact rather than a sentence. +test fn the_unit_binds_the_termination_policy_the_release_decision_relies_on() -> Bool { + let props = compute_unit_properties( + layout: compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc"), + granted: kibibyte(count: 1048576)) + contains(props, join([compute_kill_mode_property, "=", compute_kill_mode_value], "")) + && compute_kill_mode_value == "control-group" + && count(filter(props, p => starts_with(s: p, prefix: "MemoryMax="))) == 1 +} From c901fc008c3dd97f27c8128b23bd9a1787cb7424 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 20:25:38 +0000 Subject: [PATCH 11/28] The release predicate is exactly four cases; a lost record reaches a terminal consumer; the KillMode control claims only what it checks Side-chat verdict at f5344b28. All three verified first. (1) THE PREDICATE READ "not occupied AND (absent OR completed)", so completed + UNKNOWN released -- and the witness ASSERTED that, which is the worse half: a control that pins the defect is not a control. It is now exactly: occupied keeps; authoritative absence releases; completed AND POSITIVELY EMPTY releases; everything else keeps. A bound KillMode says what systemd WILL do and is not a reading of what is true, so an unreadable cgroup establishes nothing beside a completed wait. The witness is flipped. The KillMode control now claims only what it checks: it reads the argv this provider CONSTRUCTS -- invocation wiring -- and does NOT establish what systemd applied. A drop-in, a rejected property, or a unit started by some other path are outside what any fold over our own argv can see, which is exactly why the release predicate does not lean on the binding. (2) HostRecordLost WAS ERASED BY THREE CONSUMERS and is carried through all of them now. compute_provide_dependent returned only the child's result, so a BUILD that lost its record followed by a successful dependent reported clean -- the status is joined onto the dependent's result, accumulating rather than letting the later arm win, because the two details name different identities. ProvideAttached carries the status too, so an attach cannot drop a carried loss. compile_entry_cli exits nonzero naming the loss, and run_selected_module carries it onto ModuleRan, into the emitted JSON, and into the module's verdict. None of this changes the stored work outcome: the ending and the document reported are exactly what was stored, and only whether the run is called clean moves. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/test_run.dag | 22 +++-- dag/gunbc/compute/work_provider_local.dag | 83 ++++++++++++++----- .../compute/work_request_witness_test.dag | 55 ++++++++++-- 3 files changed, 127 insertions(+), 33 deletions(-) diff --git a/dag/gunbc/compute/test_run.dag b/dag/gunbc/compute/test_run.dag index 3711f1b1f6e..0a338b0a4ce 100644 --- a/dag/gunbc/compute/test_run.dag +++ b/dag/gunbc/compute/test_run.dag @@ -19,6 +19,7 @@ import gunbc.roadmap_execution_contract { GitWorkspaceCapability } import gunbc.compute.work_request { WorkOperation, RunClaims, CompileEntry } import gunbc.compute.work_provider_local { ProvideResult, ProvideFresh, ProvideAttached, compute_provide, provide_result_succeeded, provide_result_ending, provide_result_identity, provide_result_document, + HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, host_record_wire, host_record_was_written, } import gunbc.compute.test_selection { SelectedFile, select_witness_files, classify_witness_module, planned_functions_joined, @@ -86,7 +87,7 @@ fn show_file(instance: HostDashboardInstance, worktree: String, commit: String, // One selected module's result in the report. type ModuleRun - = ModuleRan { label: String, module_path: String, planned: List, declined: List, ending: String, identity: String, document: String } + = ModuleRan { label: String, module_path: String, planned: List, declined: List, ending: String, identity: String, document: String, host_record: String } | ModuleAllDeclined { label: String, module_path: String, declined: List } | ModuleUnreadable { label: String, path: String, reason: String } | ModuleNotAWitness { label: String, path: String, reason: String } @@ -105,15 +106,22 @@ fn run_selected_module(instance: HostDashboardInstance, worktree: String, commit ModuleRan { label: render_label(l: label), module_path: module_path, planned: planned, declined: declined, ending: provide_result_ending(r: result), identity: provide_result_identity(r: result), document: provide_result_document(r: result), + host_record: host_record_wire(h: provide_result_host_record(r: result)), } } } } } +// A HOST-RECORD LOSS COUNTS AGAINST THE MODULE'S VERDICT, and that is the point of carrying it this +// far. The worker CLIs are where a compute result stops being a value and becomes an exit code, so a +// loss that reaches here and is not counted is a loss nobody ever hears about: the dependent request +// dropped it at one seam (compute_provide_dependent) and these two consumers dropped it at the next +// (side-chat review at f5344b28). The work's own ending is still reported verbatim in the receipt -- +// what changes is only whether the run is called clean. fn module_run_succeeded(r: ModuleRun) -> Bool { match r { - ModuleRan { label: _, module_path: _, planned: _, declined: _, ending, identity: _, document: _ } => ending == "succeeded" + ModuleRan { label: _, module_path: _, planned: _, declined: _, ending, identity: _, document: _, host_record: record } => ending == "succeeded" && record == "" ModuleAllDeclined { label: _, module_path: _, declined: _ } => true ModuleUnreadable { label: _, path: _, reason: _ } => false ModuleNotAWitness { label: _, path: _, reason: _ } => true @@ -129,7 +137,7 @@ fn standing_json(s: WitnessStanding) -> JsonValue { fn module_run_json(r: ModuleRun) -> JsonValue { match r { - ModuleRan { label, module_path, planned, declined, ending, identity, document: _ } => + ModuleRan { label, module_path, planned, declined, ending, identity, document: _, host_record: record } => json_object(members: [ json_kv(key: "label", value: json_string(s: label)), json_kv(key: "module", value: json_string(s: module_path)), @@ -137,6 +145,7 @@ fn module_run_json(r: ModuleRun) -> JsonValue { json_kv(key: "declined", value: json_array(elements: map(declined, standing_json))), json_kv(key: "ending", value: json_string(s: ending)), json_kv(key: "identity", value: json_string(s: identity)), + json_kv(key: "host_record", value: json_string(s: record)), ]) ModuleAllDeclined { label, module_path, declined } => json_object(members: [ @@ -164,7 +173,7 @@ fn test_run_report_json(r: TestRunReport) -> JsonValue { json_kv(key: "pattern", value: json_string(s: pattern)), json_kv(key: "snapshot_commit", value: json_string(s: commit)), json_kv(key: "selected", value: json_int(n: count(modules))), - json_kv(key: "succeeded", value: json_int(n: count(filter(modules, m => match m { ModuleRan { label: _, module_path: _, planned: _, declined: _, ending, identity: _, document: _ } => ending == "succeeded" _ => false })))), + json_kv(key: "succeeded", value: json_int(n: count(filter(modules, m => match m { ModuleRan { label: _, module_path: _, planned: _, declined: _, ending, identity: _, document: _, host_record: record } => ending == "succeeded" && record == "" _ => false })))), json_kv(key: "not_succeeded", value: json_int(n: count(filter(modules, m => !module_run_succeeded(r: m))))), json_kv(key: "modules", value: json_array(elements: map(modules, module_run_json))), ]) @@ -264,9 +273,12 @@ fn compile_entry_cli(repo_root: String, worktree: String, entry: String, receipt SnapshotRefused { step, detail } => ExitFailure { code: 2, reason: join(["gunbc compile: snapshot refused at ", step, ": ", detail], "") } SnapshotResolved { commit } => { let result = compute_provide(instance: instance, commit: commit, op: CompileEntry { entry: entry as FilePath }) - let receipt = Filesystem.Write(path: receipt_path, content: provide_result_document(r: result)) + let record = provide_result_host_record(r: result) + let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), host_record_wire(h: record)], "")) if !receipt.success { ExitFailure { code: 2, reason: join(["gunbc compile: receipt could not be written to ", receipt_path, ": ", receipt.error, "; outcome was ", provide_result_document(r: result)], "") } + } else if !host_record_was_written(h: record) { + ExitFailure { code: 1, reason: join(["gunbc compile: the work ended ", provide_result_ending(r: result), " and its stored record is authoritative, but ", host_record_wire(h: record)], "") } } else if provide_result_succeeded(r: result) { ExitSuccess } else { diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 20abb8159b2..f6a4070179d 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -171,12 +171,12 @@ type HostRecordStatus type ProvideResult = ProvideFresh { outcome: WorkOutcome, document: String, host_record: HostRecordStatus } - | ProvideAttached { identity: String, ending: String, document: String } + | ProvideAttached { identity: String, ending: String, document: String, host_record: HostRecordStatus } fn provide_result_host_record(r: ProvideResult) -> HostRecordStatus { match r { ProvideFresh { outcome: _, document: _, host_record: h } => h - ProvideAttached { identity: _, ending: _, document: _ } => HostRecordWritten + ProvideAttached { identity: _, ending: _, document: _, host_record: h } => h } } @@ -187,6 +187,29 @@ fn host_record_wire(h: HostRecordStatus) -> String { } } +// JOINING TWO HOST-RECORD FACTS: a loss anywhere in the chain is a loss the caller must hear about, +// so the arms accumulate rather than the later one winning. Both details are kept because they name +// different identities and an operator needs each one. +fn host_record_join(a: HostRecordStatus, b: HostRecordStatus) -> HostRecordStatus { + match a { + HostRecordWritten => b + HostRecordLost { detail: da } => + match b { + HostRecordWritten => HostRecordLost { detail: da } + HostRecordLost { detail: db } => HostRecordLost { detail: join([da, "; and ", db], "") } + } + } +} + +fn provide_result_with_host_record(r: ProvideResult, carried: HostRecordStatus) -> ProvideResult { + match r { + ProvideFresh { outcome: o, document: d, host_record: h } => + ProvideFresh { outcome: o, document: d, host_record: host_record_join(a: carried, b: h) } + ProvideAttached { identity: i, ending: e, document: d, host_record: h } => + ProvideAttached { identity: i, ending: e, document: d, host_record: host_record_join(a: carried, b: h) } + } +} + fn host_record_was_written(h: HostRecordStatus) -> Bool { match h { HostRecordWritten => true @@ -197,28 +220,28 @@ fn host_record_was_written(h: HostRecordStatus) -> Bool { fn provide_result_succeeded(r: ProvideResult) -> Bool { match r { ProvideFresh { outcome, document: _, host_record: _ } => work_outcome_ending(o: outcome) == "succeeded" - ProvideAttached { identity: _, ending, document: _ } => ending == "succeeded" + ProvideAttached { identity: _, ending, document: _, host_record: _ } => ending == "succeeded" } } fn provide_result_identity(r: ProvideResult) -> String { match r { ProvideFresh { outcome, document: _, host_record: _ } => gunbc.compute.work_request.work_outcome_identity(o: outcome) - ProvideAttached { identity, ending: _, document: _ } => identity + ProvideAttached { identity, ending: _, document: _, host_record: _ } => identity } } fn provide_result_ending(r: ProvideResult) -> String { match r { ProvideFresh { outcome, document: _, host_record: _ } => work_outcome_ending(o: outcome) - ProvideAttached { identity: _, ending, document: _ } => ending + ProvideAttached { identity: _, ending, document: _, host_record: _ } => ending } } fn provide_result_document(r: ProvideResult) -> String { match r { ProvideFresh { outcome: _, document, host_record: _ } => document - ProvideAttached { identity: _, ending: _, document } => document + ProvideAttached { identity: _, ending: _, document, host_record: _ } => document } } @@ -258,9 +281,12 @@ fn compute_unit_name(identity_hex: String) -> NonEmptyStr { // treating a completed wait as an empty cgroup (side-chat review at 773825cf). Binding it makes the // policy a fact of the invocation. // -// IT IS BOUND *AND* THE EVIDENCE IS STILL REQUIRED. A positive populated=1 is never discarded in -// favour of a completed wait, because a bound policy says what systemd will do and the cgroup says -// what is true. +// IT IS BOUND *AND* THE EVIDENCE IS STILL REQUIRED, IN BOTH DIRECTIONS. A positive populated=1 is +// never discarded in favour of a completed wait; and an UNREADABLE population is not licensed by one +// either. A bound policy says what systemd WILL do, the cgroup says what IS true, and "I could not +// look" is not the second (side-chat review at f5344b28). Completion confirms a release only beside +// a positively empty cgroup -- or, short-circuiting all of this, a manager that has no record of +// the unit at all. data compute_kill_mode_property: String = "KillMode" data compute_kill_mode_value: String = "control-group" @@ -428,12 +454,22 @@ fn unit_termination_decide(o: UnitObservations) -> UnitTermination { ], ""), } } else { - UnitAttemptCompleted { - detail: join([ - "systemd-run --wait returned success under ", compute_kill_mode_property, "=", compute_kill_mode_value, - ", so the unit ran to completion and its cgroup was torn down; ", - match population { PopulationEmpty { detail: pd } => pd PopulationUnknown { detail: pd } => join(["population unread (", pd, ") and not positively occupied"], "") PopulationOccupied { detail: pd } => pd }, - ], ""), + match population { + PopulationEmpty { detail: pd } => + UnitAttemptCompleted { + detail: join([ + "systemd-run --wait returned success under ", compute_kill_mode_property, "=", compute_kill_mode_value, + " and ", pd, + ], ""), + } + _ => + UnitTerminationUnavailable { + detail: join([ + "the attempt completed, but the population could not be read (", + match population { PopulationUnknown { detail: pd } => pd PopulationEmpty { detail: pd } => pd PopulationOccupied { detail: pd } => pd }, + ") -- a bound KillMode says what systemd WILL do, not what is true, so an unreadable cgroup establishes nothing even beside a completed wait", + ], ""), + } } } } @@ -642,7 +678,7 @@ fn compute_with_lost_host_record( ], "") match settled { ProvideFresh { outcome: o, document: d, host_record: _ } => ProvideFresh { outcome: o, document: d, host_record: HostRecordLost { detail: detail } } - ProvideAttached { identity: i, ending: e, document: d } => ProvideAttached { identity: i, ending: e, document: d } + ProvideAttached { identity: i, ending: e, document: d, host_record: h } => ProvideAttached { identity: i, ending: e, document: d, host_record: h } } } // BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the @@ -775,7 +811,7 @@ fn compute_provide_leaf( let attached = if stored.success { stored_outcome_decode(text: stored.content) } else { StoredOutcomeUnreadable { reason: "no stored outcome" } } if stored_outcome_is_success(r: attached) { match attached { - StoredOutcomeFound { ending, identity: stored_identity, document } => ProvideAttached { identity: stored_identity, ending: ending, document: document } + StoredOutcomeFound { ending, identity: stored_identity, document } => ProvideAttached { identity: stored_identity, ending: ending, document: document, host_record: HostRecordWritten } StoredOutcomeUnreadable { reason } => compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: StoreUnavailable { detail: reason } }, subject: subject, op: op) } } else if !(compute_ensure_dir(instance: instance, path: layout.work_dir) && compute_ensure_dir(instance: instance, path: layout.leases_dir) && compute_ensure_dir(instance: instance, path: layout.store_dir) && compute_ensure_dir(instance: instance, path: layout.target_dir)) { @@ -926,14 +962,23 @@ fn compute_provide(instance: HostDashboardInstance, commit: String, op: WorkOper } } +// THE DEPENDENCY'S LOST RECORD TRAVELS WITH THE RESULT, because the caller of a dependent request +// never sees the dependency's own result. Returning the second leaf's result alone silently dropped +// the fact that the BUILD failed to record its side facts -- the stuck lease or the lost consumption +// reading vanished at exactly the seam where nobody was left to report it (side-chat review at +// f5344b28). The work contract is untouched; only the host-record fact is joined. fn compute_provide_dependent(instance: HostDashboardInstance, commit: String, subject: WorkSubject, op: WorkOperation, toolchain: WorkToolchainIdentity) -> ProvideResult { let build = compute_provide_leaf(instance: instance, commit: commit, subject: subject, op: work_default_build(), toolchain: toolchain, dependency_store: "") if provide_result_succeeded(r: build) { let build_identity = provide_result_identity(r: build) - compute_provide_leaf(instance: instance, commit: commit, subject: subject, op: op, toolchain: toolchain, dependency_store: compute_layout(instance: instance, identity_hex: build_identity).store_dir) + provide_result_with_host_record( + r: compute_provide_leaf(instance: instance, commit: commit, subject: subject, op: op, toolchain: toolchain, dependency_store: compute_layout(instance: instance, identity_hex: build_identity).store_dir), + carried: provide_result_host_record(r: build)) } else { let identity = work_identity_hex(id: work_identity(subject: subject, op: op, toolchain: toolchain)) - compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: DependencyNotSucceeded { dependency_identity: provide_result_identity(r: build), ending: provide_result_ending(r: build) } }, subject: subject, op: op) + provide_result_with_host_record( + r: compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: DependencyNotSucceeded { dependency_identity: provide_result_identity(r: build), ending: provide_result_ending(r: build) } }, subject: subject, op: op), + carried: provide_result_host_record(r: build)) } } diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 4704bc5ac96..4f396da6242 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -8,7 +8,8 @@ import gunbc.compute.work_provider_local { unit_termination_frees_capacity, unit_termination_wire, compute_attempt_completed, ProvideResult, ProvideFresh, ProvideAttached, HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, provide_result_ending, provide_result_document, host_record_was_written, host_record_wire, - compute_with_lost_host_record, ComputeLayout, compute_layout, compute_unit_properties, + compute_with_lost_host_record, provide_result_with_host_record, host_record_join, + ComputeLayout, compute_layout, compute_unit_properties, compute_kill_mode_property, compute_kill_mode_value, } import std.measure { kibibyte } @@ -91,6 +92,14 @@ test fn the_stored_outcome_round_trips_and_a_broken_one_is_unreadable() -> Bool // looks like an answer, a property that came back EMPTY, and a start that is still PENDING all // arrive as readings and are wrong only in how they are read (side-chat review at 5ee0cce). // +// THE PREDICATE IS EXACTLY FOUR WORDS LONG AND EVERY OTHER READING KEEPS THE CHARGE: occupied +// keeps; authoritative absence releases; completed AND POSITIVELY EMPTY releases; everything else +// keeps. The revision before this one read "not occupied AND (absent OR completed)", so +// completed + UNKNOWN released -- and this cell ASSERTED that, which is the worse half: a control +// that pins the defect is not a control (side-chat verdict at f5344b28). A bound KillMode says what +// systemd WILL do; it is not a reading of what is true, and an unreadable cgroup establishes +// nothing beside a completed wait. +// // A COMPLETED WAIT NEVER OVERRIDES A POSITIVE POPULATION, which is the row the previous revision // got backwards: completion was tested FIRST and returned immediately, so {completed, populated 1} // RELEASED -- a launcher's success overriding the kernel saying processes are still there. Under @@ -128,8 +137,8 @@ test fn only_proven_absence_or_an_ended_cgroup_frees_a_seat() -> Bool { let populated_no_completion = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "populated 1\n")) let undecodable = decided(done: false, load: reading(queried: true, value: "loaded"), cg: reading(queried: true, value: "/x"), ev: reading(queried: true, value: "garbage\n")) unit_termination_frees_capacity(t: completed_empty) - && unit_termination_frees_capacity(t: completed_unknown) && unit_termination_frees_capacity(t: absent) + && !unit_termination_frees_capacity(t: completed_unknown) && !unit_termination_frees_capacity(t: completed_populated) && !unit_termination_frees_capacity(t: empty_no_completion) && !unit_termination_frees_capacity(t: populated_no_completion) @@ -138,6 +147,7 @@ test fn only_proven_absence_or_an_ended_cgroup_frees_a_seat() -> Bool { && !unit_termination_frees_capacity(t: pending) && !unit_termination_frees_capacity(t: undecodable) && string_contains(s: unit_termination_wire(t: completed_populated), pattern: "STILL POPULATED") + && string_contains(s: unit_termination_wire(t: completed_unknown), pattern: "establishes nothing even beside a completed wait") && string_contains(s: unit_termination_wire(t: empty_no_completion), pattern: "did not complete") && string_contains(s: unit_termination_wire(t: pending), pattern: "pending") && string_contains(s: unit_termination_wire(t: misleading), pattern: "UNESTABLISHED") @@ -176,13 +186,13 @@ test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost)), pattern: "STILL HELD") } -// THE TERMINATION POLICY THE RELEASE DECISION LEANS ON IS ACTUALLY BOUND ON THE INVOCATION. A -// completed wait is allowed to confirm a release only because KillMode=control-group means systemd -// tears the cgroup down with the unit; under process or none a forked child outlives the main -// process and goes on using the memory this seat accounts for. systemd's DEFAULT is -// control-group -- but a default is not a declaration, and a drop-in or a future default could -// change it while this provider kept reasoning from the old one. So it is bound, and this cell is -// what makes "bound" a fact rather than a sentence. +// THIS CHECKS INVOCATION WIRING, NOT A RUNTIME POLICY READBACK, and the distinction is the whole +// honesty of the cell. It reads the argv this provider CONSTRUCTS and establishes that the +// KillMode word is in it. It does NOT establish what systemd applied: a drop-in, a manager that +// rejected the property, or a unit started by some other path are all outside what any fold over +// our own argv can see. That is precisely why the release predicate does not lean on the binding -- +// completion must still be confirmed by a positively empty cgroup, and this row only ensures we are +// not ALSO shipping an invocation that opts into the dangerous mode. test fn the_unit_binds_the_termination_policy_the_release_decision_relies_on() -> Bool { let props = compute_unit_properties( layout: compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc"), @@ -191,3 +201,30 @@ test fn the_unit_binds_the_termination_policy_the_release_decision_relies_on() - && compute_kill_mode_value == "control-group" && count(filter(props, p => starts_with(s: p, prefix: "MemoryMax="))) == 1 } + +// A LOSS ANYWHERE IN THE CHAIN REACHES A TERMINAL CONSUMER, which is the property three separate +// seams were erasing: compute_provide_dependent returned only the child's result, so a BUILD that +// lost its record followed by a successful dependent reported clean; and compile_entry_cli and +// run_selected_module both read only the work's ending (side-chat verdict at f5344b28). +// +// The join is what carries it, and it accumulates rather than letting the later arm win -- both +// details are kept because they name different identities and an operator needs each. The work +// contract is untouched by all of this: the ending and the document are exactly what was stored. +test fn a_lost_record_on_a_dependency_survives_a_successful_dependent() -> Bool { + let succeeded = WorkSucceeded { identity: "child", outputs: [], log_path: "/l" } + let document = work_outcome_wire(o: succeeded, subject: tree(hex: tree_a), op: work_default_build()) + let clean_child = ProvideFresh { outcome: succeeded, document: document, host_record: HostRecordWritten } + let attached_child = ProvideAttached { identity: "child", ending: "succeeded", document: document, host_record: HostRecordWritten } + let build_lost = HostRecordLost { detail: "the build's lease is still held" } + let carried_fresh = provide_result_with_host_record(r: clean_child, carried: build_lost) + let carried_attached = provide_result_with_host_record(r: attached_child, carried: build_lost) + let both_lost = host_record_join(a: build_lost, b: HostRecordLost { detail: "and the dependent lost its reading" }) + !host_record_was_written(h: provide_result_host_record(r: carried_fresh)) + && !host_record_was_written(h: provide_result_host_record(r: carried_attached)) + && provide_result_ending(r: carried_fresh) == "succeeded" + && provide_result_document(r: carried_fresh) == document + && provide_result_ending(r: carried_attached) == "succeeded" + && host_record_was_written(h: provide_result_host_record(r: clean_child)) + && string_contains(s: host_record_wire(h: both_lost), pattern: "still held") + && string_contains(s: host_record_wire(h: both_lost), pattern: "lost its reading") +} From 1c670b8d4ea992e5b4b4456380799736a9848e6a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 20:54:26 +0000 Subject: [PATCH 12/28] The receipt stays parseable: the host-record fact gets its own file beside it Found by the srv1 fault-injection run for the re-attach control, and it is the same defect as the consumption line once appended to outcome.json -- committed a second time, one file along. Appending the host-record sentence to the receipt stopped it being parseable JSON, so a consumer reading it back got "Extra data" instead of an outcome. A document a consumer parses carries exactly what its contract says. The receipt is the document again; the loss is written to .host-record and the exit reason names where it went. Both CLIs. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/test_run.dag | 7 ++++--- dag/gunbc/compute/work_provider_local.dag | 25 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/compute/test_run.dag b/dag/gunbc/compute/test_run.dag index 0a338b0a4ce..4ad563fe459 100644 --- a/dag/gunbc/compute/test_run.dag +++ b/dag/gunbc/compute/test_run.dag @@ -19,7 +19,7 @@ import gunbc.roadmap_execution_contract { GitWorkspaceCapability } import gunbc.compute.work_request { WorkOperation, RunClaims, CompileEntry } import gunbc.compute.work_provider_local { ProvideResult, ProvideFresh, ProvideAttached, compute_provide, provide_result_succeeded, provide_result_ending, provide_result_identity, provide_result_document, - HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, host_record_wire, host_record_was_written, + HostRecordStatus, HostRecordWritten, HostRecordLost, provide_result_host_record, host_record_wire, host_record_was_written, compute_write_host_record_note, } import gunbc.compute.test_selection { SelectedFile, select_witness_files, classify_witness_module, planned_functions_joined, @@ -274,11 +274,12 @@ fn compile_entry_cli(repo_root: String, worktree: String, entry: String, receipt SnapshotResolved { commit } => { let result = compute_provide(instance: instance, commit: commit, op: CompileEntry { entry: entry as FilePath }) let record = provide_result_host_record(r: result) - let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), host_record_wire(h: record)], "")) + let receipt = Filesystem.Write(path: receipt_path, content: provide_result_document(r: result)) + let record_note = compute_write_host_record_note(receipt_path: receipt_path, record: record) if !receipt.success { ExitFailure { code: 2, reason: join(["gunbc compile: receipt could not be written to ", receipt_path, ": ", receipt.error, "; outcome was ", provide_result_document(r: result)], "") } } else if !host_record_was_written(h: record) { - ExitFailure { code: 1, reason: join(["gunbc compile: the work ended ", provide_result_ending(r: result), " and its stored record is authoritative, but ", host_record_wire(h: record)], "") } + ExitFailure { code: 1, reason: join(["gunbc compile: the work ended ", provide_result_ending(r: result), " and its stored record is authoritative, but ", host_record_wire(h: record), "; ", record_note], "") } } else if provide_result_succeeded(r: result) { ExitSuccess } else { diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index f6a4070179d..59e1eada9d1 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -1007,7 +1007,8 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent Present { value: op } => { let result = compute_provide(instance: instance, commit: commit, op: op) let record = provide_result_host_record(r: result) - let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), host_record_wire(h: record), "\n"], "")) + let receipt = Filesystem.Write(path: receipt_path, content: join([provide_result_document(r: result), "\n"], "")) + let record_note = compute_write_host_record_note(receipt_path: receipt_path, record: record) if !receipt.success { ExitFailure { code: 2, reason: join(["compute: receipt could not be written to ", receipt_path, ": ", receipt.error, "; outcome was ", provide_result_document(r: result)], "") } } else if !host_record_was_written(h: record) { @@ -1016,7 +1017,7 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent reason: join([ "compute: the work ended ", provide_result_ending(r: result), " and its record at the store is authoritative -- a later request for this identity reads exactly that -- but ", - host_record_wire(h: record), + host_record_wire(h: record), "; ", record_note, ], ""), } } else if provide_result_succeeded(r: result) { @@ -1029,6 +1030,26 @@ fn compute_request_cli(repo_root: String, commit: String, operation: String, ent } } +// THE RECEIPT STAYS THE DOCUMENT, AND THE HOST-RECORD FACT GETS ITS OWN FILE BESIDE IT. +// +// The first version of this appended the host-record sentence to the receipt, and the srv1 +// fault-injection run is what caught it: the receipt stopped being parseable JSON, so a consumer +// reading it back got "Extra data" instead of an outcome. That is the SAME defect as the +// consumption line once appended to outcome.json, committed a second time one file along -- a +// realization's side fact changing the shape of a document some consumer parses. The rule has to be +// the same in both places: a document a consumer parses carries exactly what its contract says, and +// everything else lives beside it. +fn compute_write_host_record_note(receipt_path: String, record: HostRecordStatus) -> String { + match record { + HostRecordWritten => "no host-record note was needed" + HostRecordLost { detail: d } => { + let path = join([receipt_path, ".host-record"], "") + let written = Filesystem.Write(path: path, content: join([d, "\n"], "")) + if written.success { join(["the loss is recorded at ", path], "") } else { join(["and the note could not be written to ", path, " either: ", written.error], "") } + } + } +} + fn compute_parse_operation(operation: String, entry: String, functions: String, hermetic: String) -> WorkOperation? { if operation == "build" { Present { value: work_default_build() } From bf58075f152847c1497f86066f72e02998f4b107 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 21:38:42 +0000 Subject: [PATCH 13/28] SeatRequest carries its dimension; the strip moves to where the pool proves it review 69724, and it is this change's own diagnosis unapplied one boundary over. gunbc.compute.host_capacity was calling kibibyte_count to STRIP its Kibibyte on the way into SeatRequest -- the exact tell work_request.dag names for HostBelowMemoryFloor and fabric_event_log.dag names for SeatRoomObserved. A bare Nat was right while every pool reachable through this fold was Pool: seats are dimensionless and the field meant seats. This PR generalized the carrier to Pool, which made the same field mean KiB for a memory pool and seats for a seat pool with nothing in the type to say which. So SeatRequest is generic and amount carries Measure. WHERE THE STRIP LEGITIMATELY HAPPENS IS ONE LAYER DOWN, and the difference is not cosmetic: propose_acquire holds the POOL, so the pool's own type parameters prove the request's dimension matches the appropriation it is adjudicated against. A consumer stripping the carrier ASSERTS that match; this fold CHECKS it. THE PERSISTED EVENT KEEPS A BARE Nat AND THAT IS NOT THE SAME DEFECT, stated rather than left to be re-found. PoolAcquired is a wire record: its dimension is the partition's, and it is restored at both ends from the pool's own parameters -- minted from a Measure the pool typed, and re-minted at replay by pool_apply_event into the Pool being folded. A JSON integer carrying a magnitude whose unit the surrounding type fixes is a serialization, not a second model of the unit. Call sites: harness_seat and fabric_quota wrap their counts, the capacity and spark witnesses likewise, and host_capacity passes its Kibibyte straight through. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/host_capacity.dag | 2 +- dag/gunbc/fabric/fabric_event_log.dag | 4 +-- dag/gunbc/fabric/fabric_quota.dag | 3 +- dag/gunbc/harness/harness_seat.dag | 2 +- dag/gunbc/product/capacity/pool_events.dag | 32 +++++++++++++++---- .../capacity_lease_chain_witness_test.dag | 12 +++---- ...serving_d0_real_execution_witness_test.dag | 4 +-- 7 files changed, 39 insertions(+), 20 deletions(-) diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index 3248b01262d..a4dbfb0a8a7 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -340,7 +340,7 @@ fn compute_seat(subject: ComputeCapacitySubject, reference: NonEmptyStr, now: Ep partition: subject.partition, root: compute_memory_root_pool(subject: subject, ceiling: ceiling), actor: reference, - request: SeatRequest { reference: reference, amount: kibibyte_count(k: subject.request), at: now, term_seconds: term_seconds }, + request: SeatRequest { reference: reference, amount: subject.request, at: now, term_seconds: term_seconds }, policy: compute_lease_policy(term_seconds: term_seconds), attempts: compute_seat_attempts, budget: compute_partition_read_budget, diff --git a/dag/gunbc/fabric/fabric_event_log.dag b/dag/gunbc/fabric/fabric_event_log.dag index 5b109c419f3..877a4368837 100644 --- a/dag/gunbc/fabric/fabric_event_log.dag +++ b/dag/gunbc/fabric/fabric_event_log.dag @@ -283,7 +283,7 @@ fn fabric_seat_observe(store: FabricStorageBinding, partition: PartitionId } } -fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, budget: Nat, attempt: Nat) -> SeatAcquisition? { +fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, budget: Nat, attempt: Nat) -> SeatAcquisition? { match event_log_read_partition(store: store, partition: partition, budget: budget) { PartitionReadRefused { cause: c } => Present { value: SeatStoreRefused { cause: c, attempts_used: attempt } } PartitionReadOk { head: head, walk: walk } => @@ -321,7 +321,7 @@ fn seat_attempt(store: FabricStorageBinding, partition: PartitionId, root: // and the drift is invisible because each side is internally consistent. A mismatch is refused before // anything is appended rather than resolved in favour of either side: neither is authoritative over // the other, and picking one silently would be the same fail-open in a different costume. -fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, attempts: Nat, budget: Nat) -> SeatAcquisition { +fn fabric_seat_acquire(store: FabricStorageBinding, partition: PartitionId, root: Pool, actor: NonEmptyStr, request: SeatRequest, policy: LeasePolicy, attempts: Nat, budget: Nat) -> SeatAcquisition { if !release_law_eq(left: root.release_law, right: policy.release_law) { SeatAcquireRefused { step: "release-law", diff --git a/dag/gunbc/fabric/fabric_quota.dag b/dag/gunbc/fabric/fabric_quota.dag index f6c87e0cb75..870172d2f17 100644 --- a/dag/gunbc/fabric/fabric_quota.dag +++ b/dag/gunbc/fabric/fabric_quota.dag @@ -2,6 +2,7 @@ module gunbc.fabric_quota import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat } +import std.measure { Measure } import extdeps.api_rate_limit { UpstreamRateLimit, upstream_rate_limit_key } import product.capacity.quota { quota_partition, quota_root_pool } import product.capacity.event_chain { PartitionId } @@ -42,7 +43,7 @@ fn fabric_quota_lease(short_hostname: String, limit: UpstreamRateLimit, amount: let partition = quota_partition(limit: limit) match fabric_seat_acquire( store: store, partition: partition, root: quota_root_pool(limit: limit), actor: actor, - request: SeatRequest { reference: join([actor as String, "@", to_string(now)], "") as NonEmptyStr, amount: amount, at: now, term_seconds: term_seconds }, + request: SeatRequest { reference: join([actor as String, "@", to_string(now)], "") as NonEmptyStr, amount: Measure { count: amount }, at: now, term_seconds: term_seconds }, policy: quota_policy(term_seconds: term_seconds), attempts: 3, budget: 4096, ) { SeatGranted { grant: g, generation: _, attempts_used: _ } => QuotaLeased { grant: g, partition: partition, store: store } diff --git a/dag/gunbc/harness/harness_seat.dag b/dag/gunbc/harness/harness_seat.dag index 7fb2d6751a5..dd4c79fc6cf 100644 --- a/dag/gunbc/harness/harness_seat.dag +++ b/dag/gunbc/harness/harness_seat.dag @@ -1225,7 +1225,7 @@ fn harness_acquire_under_authority(candidate: HarnessCandidate, class: ServingCo HeadObserved { head: h } => { let request = SeatRequest { reference: harness_seat_reference(attempt: attempt, offer_key: candidate.key, now: now, round: round, head: harness_head_word(head: h)), - amount: 1, + amount: Measure { count: 1 }, at: now, term_seconds: policy.maximum_duration_seconds, } diff --git a/dag/gunbc/product/capacity/pool_events.dag b/dag/gunbc/product/capacity/pool_events.dag index 542f75c540f..13ca830929a 100644 --- a/dag/gunbc/product/capacity/pool_events.dag +++ b/dag/gunbc/product/capacity/pool_events.dag @@ -2,7 +2,7 @@ module product.capacity.pool_events import std.types { String, Bool, Int, NonEmptyStr, List, EpochSecs } import std.nat { Nat } -import std.measure { Measure } +import std.measure { Measure, measure_count } import extdeps.languages.json.emit { JsonValue, JsonKeyValue, json_kv, json_string, json_int } import product.capacity.event_json { chain_event_wire_text, chain_event_decode, member_nonempty, member_nat } import product.capacity.pool { @@ -175,9 +175,27 @@ fn pool_fold(root: Pool, oldest_first: List // event with the head it was decided against. The carrier performs the compare-and-set; a stale // answer sends the caller back through this function with the new head. A full pool is a typed // refusal, never a queue. -type SeatRequest { +// THE REQUESTED AMOUNT CARRIES ITS DIMENSION, and this is the same repair SeatStandingObservation +// took for the same reason. A bare Nat was right while every pool reachable through this fold was +// Pool: seats are dimensionless and the field meant seats. Generalizing the carrier to +// Pool made the same field mean KiB for a memory pool and seats for a seat pool, with nothing +// in the type to say which -- so gunbc.compute.host_capacity had to call kibibyte_count to STRIP its +// Kibibyte on the way in, which is precisely the tell this change diagnoses one boundary over +// (review 69724 on gunbc#11962). +// +// WHERE THE STRIP LEGITIMATELY HAPPENS IS HERE, one layer down, and the difference is not cosmetic: +// propose_acquire holds the POOL, so the pool's own type parameters prove the request's dimension +// matches the appropriation it is being adjudicated against. A consumer stripping the carrier is +// asserting that match; this fold checks it. +// +// THE PERSISTED EVENT KEEPS A BARE Nat AND THAT IS NOT THE SAME DEFECT. PoolAcquired is a WIRE +// RECORD: its dimension is the partition's, and it is restored at both ends from the pool's own +// parameters -- minted here from a Measure the pool typed, and re-minted at replay by +// pool_apply_event into the Pool being folded. A JSON integer carrying a magnitude whose unit +// the surrounding type fixes is a serialization, not a second model of the unit. +type SeatRequest { reference: NonEmptyStr - amount: Nat + amount: Measure at: EpochSecs term_seconds: Nat } @@ -213,8 +231,8 @@ fn seat_refusal_wire(r: PoolRefusal) -> String { } } -fn propose_acquire(pool: Pool, generation: Nat, head: HeadExpectation, partition: PartitionId, actor: NonEmptyStr, request: SeatRequest) -> SeatProposal { - match pool_acquire(pool: pool, reference: request.reference, amount: Measure { count: request.amount }, at: request.at, term_seconds: request.term_seconds) { +fn propose_acquire(pool: Pool, generation: Nat, head: HeadExpectation, partition: PartitionId, actor: NonEmptyStr, request: SeatRequest) -> SeatProposal { + match pool_acquire(pool: pool, reference: request.reference, amount: request.amount, at: request.at, term_seconds: request.term_seconds) { PoolRefused { refusal: r } => SeatRefused { wire: seat_refusal_wire(r: r) } PoolAdvanced { pool: _ } => SeatProposed { event: ChainEvent { @@ -222,7 +240,7 @@ fn propose_acquire(pool: Pool, generation: Nat, head: HeadExpectatio parent: match head { HeadAbsent => none HeadAt { id: h } => Present { value: h } }, recorded_at: request.at, actor: actor, - payload: PoolAcquired { reference: request.reference, amount: request.amount, term_seconds: request.term_seconds }, + payload: PoolAcquired { reference: request.reference, amount: measure_count(request.amount), term_seconds: request.term_seconds }, }, expected_head: head, } @@ -231,7 +249,7 @@ fn propose_acquire(pool: Pool, generation: Nat, head: HeadExpectatio // THE GRANT IS MINTED FROM THE ADMISSION, never from the proposal: a proposal that lost the // compare-and-set mints nothing. -fn grant_from_admission(request: SeatRequest, admission: AppendDecision, policy: LeasePolicy) -> LeaseGrant? { +fn grant_from_admission(request: SeatRequest, admission: AppendDecision, policy: LeasePolicy) -> LeaseGrant? { match admission { AppendStale { expected: _, observed: _ } => none AppendAdmitted { new_head: h, generation: g } => diff --git a/dag/test/claim/capacity_lease_chain_witness_test.dag b/dag/test/claim/capacity_lease_chain_witness_test.dag index 825874a4632..50b263bff49 100644 --- a/dag/test/claim/capacity_lease_chain_witness_test.dag +++ b/dag/test/claim/capacity_lease_chain_witness_test.dag @@ -90,9 +90,9 @@ test fn two_writers_from_one_head_produce_one_admission_one_stale_and_then_a_ful PoolFoldRefused { at_event: _, wire: _ } => false PoolFolded { pool: p, generation: g } => { let a = propose_acquire(pool: p, generation: g, head: head, partition: w_partition(), actor: "srv1-harness" as NonEmptyStr, - request: SeatRequest { reference: "srv1-turn-7" as NonEmptyStr, amount: 1, at: 1100, term_seconds: 300 }) + request: SeatRequest { reference: "srv1-turn-7" as NonEmptyStr, amount: Measure { count: 1 }, at: 1100, term_seconds: 300 }) let b = propose_acquire(pool: p, generation: g, head: head, partition: w_partition(), actor: "srv2-harness" as NonEmptyStr, - request: SeatRequest { reference: "srv2-turn-3" as NonEmptyStr, amount: 1, at: 1100, term_seconds: 300 }) + request: SeatRequest { reference: "srv2-turn-3" as NonEmptyStr, amount: Measure { count: 1 }, at: 1100, term_seconds: 300 }) match a { SeatRefused { wire: _ } => false SeatProposed { event: ea, expected_head: xa } => @@ -109,7 +109,7 @@ test fn two_writers_from_one_head_produce_one_admission_one_stale_and_then_a_ful PoolFolded { pool: p2, generation: g2 } => g2 == 2 && w_headroom(p: p2, at: 1100) == 0 && (match propose_acquire(pool: p2, generation: g2, head: HeadAt { id: w_id(s: "e2") }, partition: w_partition(), actor: "srv2-harness" as NonEmptyStr, - request: SeatRequest { reference: "srv2-turn-3" as NonEmptyStr, amount: 1, at: 1101, term_seconds: 300 }) { + request: SeatRequest { reference: "srv2-turn-3" as NonEmptyStr, amount: Measure { count: 1 }, at: 1101, term_seconds: 300 }) { SeatRefused { wire: w } => w == "pool-full" SeatProposed { event: _, expected_head: _ } => false }) @@ -147,7 +147,7 @@ test fn the_fence_rejects_obsolete_and_foreign_grants_and_expiry_frees_only_unde // THE GRANT IS MINTED FROM THE ADMISSION, never from the proposal: its fence is the admitted event // and generation, and its deadline is the declared policy applied at the grant instant. test fn a_grant_carries_the_admitted_event_as_its_fence_and_the_declared_deadline() -> Bool { - let req = SeatRequest { reference: "srv1-turn-7" as NonEmptyStr, amount: 1, at: 1100, term_seconds: 300 } + let req = SeatRequest { reference: "srv1-turn-7" as NonEmptyStr, amount: Measure { count: 1 }, at: 1100, term_seconds: 300 } match grant_from_admission(request: req, admission: AppendAdmitted { new_head: w_id(s: "e2"), generation: 2 }, policy: w_policy()) { Absent => false Present { value: g } => @@ -212,7 +212,7 @@ test fn an_over_committed_pool_still_refuses_a_new_acquire() -> Bool { PoolFolded { pool: p, generation: g } => match propose_acquire(pool: p, generation: g, head: HeadAt { id: w_id(s: "o4") }, partition: w_partition(), actor: "srv1-harness" as NonEmptyStr, - request: SeatRequest { reference: "seat-5" as NonEmptyStr, amount: 1, at: 1100, term_seconds: 300 }) { + request: SeatRequest { reference: "seat-5" as NonEmptyStr, amount: Measure { count: 1 }, at: 1100, term_seconds: 300 }) { SeatProposed { event: _, expected_head: _ } => false SeatRefused { wire: w } => w == "pool-full" } @@ -311,7 +311,7 @@ test fn a_lapsed_quiescence_required_partition_still_refuses_the_next_seat() -> PoolFolded { pool: p, generation: g } => match propose_acquire(pool: p, generation: g, head: HeadAt { id: w_id(s: "l2") }, partition: w_partition(), actor: "srv1-harness" as NonEmptyStr, - request: SeatRequest { reference: "seat-arriving" as NonEmptyStr, amount: 1, at: 2001, term_seconds: 300 }) { + request: SeatRequest { reference: "seat-arriving" as NonEmptyStr, amount: Measure { count: 1 }, at: 2001, term_seconds: 300 }) { SeatProposed { event: _, expected_head: _ } => false SeatRefused { wire: w } => w == "pool-full" } diff --git a/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag b/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag index 1cce5d07974..00aea13fd0c 100644 --- a/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag +++ b/dag/test/claim/spark/pair_serving_d0_real_execution_witness_test.dag @@ -3,7 +3,7 @@ module test.claim.spark.pair_serving_d0_real_execution import std.logic { Bool } import std.types { String, NonEmptyStr, FilePath, List, Int } import v2.std.optional { Present, Absent } -import std.measure { second, measure_count } +import std.measure { second, measure_count, Measure } import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } import extdeps.shell import std.content_hash { ContentHash, Sha256Hash, sha256_hex_digest, compare_content_hash, ContentHashEqual, ContentHashDifferent, ContentHashCrossFamilyIncomparable, serialize_content_hash, content_hash_equal } @@ -383,7 +383,7 @@ test fn a_seat_granted_after_the_suspension_landed_is_released_by_real_execution AuthorityTransitionAppended { id: _, generation: g1, next: _ } => g1 == 1 _ => false }) - && (match fabric_seat_acquire(store: store, partition: partition, root: root, actor: "witness" as NonEmptyStr, request: SeatRequest { reference: "witness-seat-1" as NonEmptyStr, amount: 1, at: 1001, term_seconds: 300 }, policy: policy, attempts: 3, budget: 64) { + && (match fabric_seat_acquire(store: store, partition: partition, root: root, actor: "witness" as NonEmptyStr, request: SeatRequest { reference: "witness-seat-1" as NonEmptyStr, amount: Measure { count: 1 }, at: 1001, term_seconds: 300 }, policy: policy, attempts: 3, budget: 64) { SeatGranted { grant: g, generation: _, attempts_used: _ } => (match harness_seat_after_grant_authority(store: store, group: FabricGroupA, partition: partition, ceiling: ceiling, grant: g, now: 1001) { SeatReleasedAuthorityMoved { cause: _ } => true From afb46eb704e57c53a8fa241738286780ccd80143 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 21:54:10 +0000 Subject: [PATCH 14/28] The attached arm carried the loss it had just built into a discard review 69724 finding 2. compute_with_lost_host_record matched the arms itself, and the ProvideAttached arm returned the status it was handed while THROWING AWAY the detail it had just built -- the exact vanish this function exists to prevent, one arm along, in the function whose annotation is about a fact vanishing. It was latent only because the single call site passes a fresh result, which is the kind of "safe today" that stops being true at the next call site. Two spellings of one join is also the fork DESIGN 3 forbids, so there is now one: provide_result_with_host_record, which already handled both arms and accumulates rather than letting either side win. The control drives BOTH arms. Mutation, run and reverted: restore the arm-matching discard and a_lost_host_record_changes_no_contract_the_next_caller_reads goes FAIL. Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_provider_local.dag | 14 ++++++++++---- .../claim/compute/work_request_witness_test.dag | 11 +++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 59e1eada9d1..13ca10a3a44 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -660,6 +660,15 @@ fn compute_run_and_settle( // the lost record rides as its own fact. The CLI turns that fact into a nonzero exit with both // halves named, which is how a recording failure stays visible without becoming a verdict about // work that really did run. +// +// AND IT GOES THROUGH THE ONE JOIN RATHER THAN MATCHING THE ARMS ITSELF. It matched them, and the +// ProvideAttached arm returned the status it was handed while DISCARDING the detail it had just +// built -- the exact vanish this function exists to prevent, one arm along, in the function whose +// annotation describes the vanish (review 69724). It was latent only because the single call site +// passes a fresh result, which is the kind of "safe today" that stops being true at the next call +// site. Two spellings of one join is the fork DESIGN 3 forbids, so there is now one: +// provide_result_with_host_record, which already handled both arms and accumulates rather than +// letting either side win. fn compute_with_lost_host_record( settled: ProvideResult, layout: ComputeLayout, @@ -676,10 +685,7 @@ fn compute_with_lost_host_record( join(["THE PRODUCER LEASE AT ", layout.lease_path, " IS ALSO STILL HELD (", dropped_error, ") AND NOTHING NOW RECORDS THAT IT IS STALE: an operator must remove the name before this identity can be produced again"], "") }, ], "") - match settled { - ProvideFresh { outcome: o, document: d, host_record: _ } => ProvideFresh { outcome: o, document: d, host_record: HostRecordLost { detail: detail } } - ProvideAttached { identity: i, ending: e, document: d, host_record: h } => ProvideAttached { identity: i, ending: e, document: d, host_record: h } - } + provide_result_with_host_record(r: settled, carried: HostRecordLost { detail: detail }) } // BLOCKER (7): ONE CONTRACT, AND THE CONTRACT IS THE STORED RECORD. This arm used to hand the // caller a refusal while the outcome document already said the run had SUCCEEDED, and then state diff --git a/dag/test/claim/compute/work_request_witness_test.dag b/dag/test/claim/compute/work_request_witness_test.dag index 4f396da6242..c641d411fed 100644 --- a/dag/test/claim/compute/work_request_witness_test.dag +++ b/dag/test/claim/compute/work_request_witness_test.dag @@ -165,6 +165,11 @@ test fn the_attempt_completion_fact_is_not_the_works_ending() -> Bool { && !compute_attempt_completed(outcome: WorkRefusedByInfrastructure { identity: "a", cause: HostComputeCapacityFull { wire: "pool-full" } }) } +// AN ATTACHED RESULT CARRIES THE LOSS TOO, which is the arm that used to throw it away. The only +// call site passes a FRESH result, so the discard was latent -- and "latent" is the kind of safe +// that stops being true at the next call site, in the very function whose annotation is about a +// fact vanishing (review 69724). Both arms are driven here. +// // THE STORED RECORD AND THE RETURNED OUTCOME AGREE, WHATEVER HAPPENED TO THIS HOST'S OWN RECORD. // The fusion was made twice in opposite directions -- a refusal handed to the caller over a stored // success, then a synthesized refusal over a published outcome -- and both times the next attacher @@ -176,14 +181,20 @@ test fn a_lost_host_record_changes_no_contract_the_next_caller_reads() -> Bool { let succeeded = WorkSucceeded { identity: "abc", outputs: [], log_path: "/l" } let document = work_outcome_wire(o: succeeded, subject: tree(hex: tree_a), op: work_default_build()) let settled = ProvideFresh { outcome: succeeded, document: document, host_record: HostRecordWritten } + let attached = ProvideAttached { identity: "abc", ending: "succeeded", document: document, host_record: HostRecordWritten } let layout = compute_layout(instance: srv1_live_dashboard_instance(), identity_hex: "abc") let lost = compute_with_lost_host_record(settled: settled, layout: layout, dropped_ok: false, dropped_error: "EBUSY", error: "ENOSPC") + let lost_attached = compute_with_lost_host_record(settled: attached, layout: layout, dropped_ok: false, dropped_error: "EBUSY", error: "ENOSPC") provide_result_ending(r: lost) == provide_result_ending(r: settled) && provide_result_document(r: lost) == provide_result_document(r: settled) && provide_result_ending(r: lost) == "succeeded" && host_record_was_written(h: provide_result_host_record(r: settled)) && !host_record_was_written(h: provide_result_host_record(r: lost)) && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost)), pattern: "STILL HELD") + && provide_result_ending(r: lost_attached) == "succeeded" + && provide_result_document(r: lost_attached) == document + && !host_record_was_written(h: provide_result_host_record(r: lost_attached)) + && string_contains(s: host_record_wire(h: provide_result_host_record(r: lost_attached)), pattern: "STILL HELD") } // THIS CHECKS INVOCATION WIRING, NOT A RUNTIME POLICY READBACK, and the distinction is the whole From 2700b015ac9774392941f5889c8c9c184ec80387 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Mon, 21 Sep 2026 22:04:31 +0000 Subject: [PATCH 15/28] File the partition-identity class the PoolAcquired argument would have buried Ruled 2026-09-21 (loyal-swift-608): do NOT carry Measure through PoolAcquired and the codec -- the decoder would still mint the caller-requested Q,S because the wire has nothing to disagree with, so the RED would not be authorable anywhere a check could run, which DESIGN 4b names as worse than absent. But the error class the argument identified is real and nothing in the corpus holds it, so it does not leave with the argument. gunbc.recurring_failure_mode a_pool_partition_does_not_declare_its_quantity: a persisted capacity partition carries magnitudes and does not declare the QUANTITY they are denominated in. pool_apply_event mints every replayed amount into the Q and S OF THE POOL BEING FOLDED, so a seat partition replayed by a memory pool reads 1 seat as 1 KiB and a memory partition replayed by a seat pool reads 27262976 KiB as 27262976 seats -- arithmetically consistent, semantically unrelated, nothing refuses. THE CLASS BECAME REACHABLE IN THIS PR, which is why it is filed here: generalizing fabric_seat_acquire and fabric_seat_observe from Pool to Pool is what made two quantities share one linearization. Before it, every partition reachable through that fold was seats and the question could not be asked. What is NOT claimed: that it has occurred. The two partition namers are disjoint by construction today -- `quota-` over a rate-limit key, `compute-memory-` over a host -- so the row is filed because the discipline preventing it is a naming convention held by two authors, not a fact the substrate checks. Trigger names the CAPABILITY -- partition identity binds the pool quantity, so a mismatched fold refuses -- deliberately not "carry Measure through the codec", which could land in full while the class stayed exactly as dead. Rostered, and stated in the PR body beside the other declared gaps so a reader does not have to find it in a failure-mode file. Co-Authored-By: Claude Opus 5 (1M context) --- ...artition_does_not_declare_its_quantity.dag | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 dag/gunbc/recurring_failure_mode/a_pool_partition_does_not_declare_its_quantity.dag diff --git a/dag/gunbc/recurring_failure_mode/a_pool_partition_does_not_declare_its_quantity.dag b/dag/gunbc/recurring_failure_mode/a_pool_partition_does_not_declare_its_quantity.dag new file mode 100644 index 00000000000..138a4b7c99c --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/a_pool_partition_does_not_declare_its_quantity.dag @@ -0,0 +1,34 @@ +module gunbc.recurring_failure_mode.a_pool_partition_does_not_declare_its_quantity + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, decl_ref } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data a_pool_partition_does_not_declare_its_quantity: RecurringFailureMode = RecurringFailureMode { + identity: "a_pool_partition_does_not_declare_its_quantity" as NonEmptyStr, + + receipts: [ + "INVALID STATE: a persisted capacity partition carries magnitudes and does not declare the QUANTITY they are denominated in. product.capacity.pool_events pool_apply_event mints every replayed amount into the Q and S OF THE POOL BEING FOLDED, and the wire has nothing to disagree with -- so a partition written by a seat pool and replayed by a memory pool reinterprets 1 seat as 1 KiB, and a memory partition replayed by a seat pool reinterprets 27262976 KiB as 27262976 seats. HARM: an admission decision computed against a committed total that means something else; the numbers are arithmetically consistent and semantically unrelated, so nothing refuses and nothing looks wrong.", + + "THE CLASS BECAME REACHABLE WHEN THE POOL BECAME POLYMORPHIC. gunbc#11962 generalized gunbc.fabric_event_log fabric_seat_acquire and fabric_seat_observe from Pool to Pool so a host memory pool could use the same linearization as the seat pools. Before that every partition reachable through that fold was denominated in seats, the question could not be asked, and the answer could not be wrong. The generalization is what made two quantities share one carrier.", + + "WHY TYPING THE WIRE FIELD IS NOT THE REPAIR, ruled 2026-09-21 and recorded here because the wrong repair is the attractive one. Making PoolAcquired.amount a Measure looks like the fix and buys nothing: the DECODER would still mint the caller-requested Q and S, because there is nothing on the wire to disagree with. The RED would not be authorable anywhere a check could run, which DESIGN 4b names as worse than absent -- a decoration that gets cited as coverage. The admission boundary, where the hazard WAS reachable, is closed separately and by construction: SeatRequest must match Pool at propose_acquire, so a seat count can no longer be charged against a memory pool at the door.", + + "WHAT IS NOT CLAIMED: that this has occurred. The two partition namers in the corpus are disjoint by construction today -- product.capacity.quota quota_partition prefixes `quota-` over an upstream rate-limit key and gunbc.compute.host_capacity compute_memory_partition prefixes `compute-memory-` over a host identity -- so no name collides and no fold has been observed reading the wrong one. The class is filed because the DISCIPLINE preventing it is a naming convention held by two authors, not a fact the substrate checks.", + + "RUNG FOUND AT: mitigatable, and only by that convention. Nothing refuses a cross-quantity fold; what prevents it is that the two producers happen to choose non-overlapping prefixes and that each consumer happens to pass the pool it built the partition name from.", + + "CEILING: structurally guaranteed. A partition identity that BINDS the quantity it is denominated in, checked where the fold is assembled, makes the mismatched pairing refuse rather than reinterpret -- the root pool and the partition are already supplied together at every call site, so the join exists and is simply not made.", + + "NEXT-RUNG TRIGGER, NAMING THE CAPABILITY: partition identity binds the pool quantity, so assembling a fold from a partition and a root pool whose quantities disagree is a typed refusal rather than a silent reinterpretation. It is deliberately NOT `carry Measure through the codec` -- that artifact could land in full while this class stayed exactly as dead, which is the grain mismatch DESIGN 4b(3) warns about between a loss sentence and its trigger.", + + "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration.", + ], + + evidence: [ + decl_ref(module_path: "product.capacity.pool_events", decl_name: "pool_apply_event"), + decl_ref(module_path: "product.capacity.pool_events", decl_name: "pool_event_decode"), + decl_ref(module_path: "product.capacity.quota", decl_name: "quota_partition"), + decl_ref(module_path: "gunbc.compute.host_capacity", decl_name: "compute_memory_partition"), + ], +} From d5fc09b2bac060194a4c54f37cf5daaae6d5c1bc Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 21 Sep 2026 23:53:45 +0000 Subject: [PATCH 16/28] WIP: unit peak producer, typed grant-exceeded ending, per-class grant resolution (every class at ceiling) Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemd_run.dag | 190 ++++++++++++++ dag/gunbc/compute/host_capacity.dag | 24 +- dag/gunbc/compute/work_class_grant.dag | 166 ++++++++++++ dag/gunbc/compute/work_provider_local.dag | 237 ++++++++++++++++-- dag/gunbc/compute/work_request.dag | 11 + ...d_grant_is_discovered_below_worst_case.dag | 32 +++ .../host_capacity_wet_witness_test.dag | 6 +- .../compute/work_class_grant_witness_test.dag | 134 ++++++++++ 8 files changed, 767 insertions(+), 33 deletions(-) create mode 100644 dag/gunbc/compute/work_class_grant.dag create mode 100644 dag/gunbc/recurring_failure_mode/a_sampled_grant_is_discovered_below_worst_case.dag create mode 100644 dag/test/claim/compute/work_class_grant_witness_test.dag diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index 760f4aedc3a..c05192539d5 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -1,6 +1,9 @@ module extdeps.systemd.systemd_run import std.types { NonEmptyStr, List, Bool, Int, Unit } +import std.nat { Nat } +import std.measure { ByteSize, byte_size } +import std.algebra { trim } import std.string_type { String } import extdeps.external_authority { ExternalAuthority, ExternalModelScope, ExternalSubjectRef } @@ -196,3 +199,190 @@ service systemd.SystemdRun { } } } + +// ── WHAT `systemd-run --wait` REPORTS WHEN THE UNIT ENDS ────────────────────────────────────── +// +// THE SUMMARY IS THE ONLY INSTANT THE UNIT'S OWN PEAK AND RESULT ARE STILL READABLE, and that is why +// it is modeled rather than a property read afterwards. A `--wait --collect` transient unit is +// garbage-collected when it ends, so `systemctl show -p MemoryPeak` after the launcher returns +// answers for a unit the manager no longer has, and the cgroup whose memory.peak it mirrors is gone +// with it. systemd-run itself reads Result and MemoryPeak off the unit over D-Bus before it exits +// and logs them to stderr, unless --quiet was passed (src/run/run.c at v255, `if (arg_wait && +// !arg_quiet)`): "Finished with result: " when the result is non-empty, and "Memory peak: +// " when MemoryPeak is not UINT64_MAX. MemoryPeak itself arrives in v254 +// (extdeps.systemd.systemctl systemd_memory_peak_minimum_version), and an older manager simply +// omits the line -- so ABSENCE IS A READING, and it is never a peak of zero. +// +// WHY THE LAST LINE AND NOT THE FIRST. Under --pipe the unit's own stderr and the launcher's summary +// arrive on one stream. The summary is written after the unit has ended and the pipe has drained, +// so it is the TAIL; a command that happens to print "Memory peak: 1K" on its own stderr prints it +// BEFORE the summary. Taking the last occurrence of each prefix is what that ordering entitles. +// It is an ordering fact about the launcher, stated here so a reader can falsify it. + +// systemd.exec(5) "$SERVICE_RESULT": the closed vocabulary a service's Result property takes. +type SystemdServiceResult + = ServiceResultSuccess + | ServiceResultProtocol + | ServiceResultTimeout + | ServiceResultExitCode + | ServiceResultSignal + | ServiceResultCoreDump + | ServiceResultWatchdog + | ServiceResultStartLimitHit + | ServiceResultResources + | ServiceResultOomKill + +fn systemd_service_result_wire(r: SystemdServiceResult) -> String { + match r { + ServiceResultSuccess => "success" + ServiceResultProtocol => "protocol" + ServiceResultTimeout => "timeout" + ServiceResultExitCode => "exit-code" + ServiceResultSignal => "signal" + ServiceResultCoreDump => "core-dump" + ServiceResultWatchdog => "watchdog" + ServiceResultStartLimitHit => "start-limit-hit" + ServiceResultResources => "resources" + ServiceResultOomKill => "oom-kill" + } +} + +type SystemdServiceResultReading + = ServiceResultRead { result: SystemdServiceResult } + | ServiceResultUnrecognized { wire: String } + | ServiceResultNotReported + +fn systemd_service_result_parse(wire: String) -> SystemdServiceResultReading { + let w = trim(s: wire) + if w == "success" { ServiceResultRead { result: ServiceResultSuccess } } + else if w == "protocol" { ServiceResultRead { result: ServiceResultProtocol } } + else if w == "timeout" { ServiceResultRead { result: ServiceResultTimeout } } + else if w == "exit-code" { ServiceResultRead { result: ServiceResultExitCode } } + else if w == "signal" { ServiceResultRead { result: ServiceResultSignal } } + else if w == "core-dump" { ServiceResultRead { result: ServiceResultCoreDump } } + else if w == "watchdog" { ServiceResultRead { result: ServiceResultWatchdog } } + else if w == "start-limit-hit" { ServiceResultRead { result: ServiceResultStartLimitHit } } + else if w == "resources" { ServiceResultRead { result: ServiceResultResources } } + else if w == "oom-kill" { ServiceResultRead { result: ServiceResultOomKill } } + else { ServiceResultUnrecognized { wire: w } } +} + +// A FORMAT_BYTES READING IS AN INTERVAL, NOT A NUMBER. format_bytes (src/basic/format-util.h, v255) +// is format_bytes_full with USE_IEC | BELOW_POINT | TRAILING_B: below 1024 it prints the exact count +// with a trailing "B"; otherwise the largest 1024-power suffix K/M/G/T/P/E with the whole part and +// ONE TRUNCATED tenth, and no "B". The tenth is computed as floor(floor(t / F') * 10 / 1024) mod 10 +// with F' the next-smaller factor, which can read one tenth LOW, so the only sound upper bound for +// a printed W.d of factor F is (10W + d + 2) * F / 10. A capacity reading must carry the upper +// bound; carrying the printed figure would size a grant below the peak that produced it. +type SystemdFormattedBytes + = FormattedBytesExact { bytes: ByteSize } + | FormattedBytesBounded { printed: String, lower: ByteSize, upper: ByteSize } + +type SystemdFormattedBytesReading + = FormattedBytesRead { value: SystemdFormattedBytes } + | FormattedBytesUnparseable { text: String } + +fn systemd_iec_suffix_factor(suffix: String) -> Nat? { + if suffix == "K" { Present { value: 1024 } } + else if suffix == "M" { Present { value: 1048576 } } + else if suffix == "G" { Present { value: 1073741824 } } + else if suffix == "T" { Present { value: 1099511627776 } } + else if suffix == "P" { Present { value: 1125899906842624 } } + else if suffix == "E" { Present { value: 1152921504606846976 } } + else { none } +} + +fn systemd_formatted_bytes_parse(text: String) -> SystemdFormattedBytesReading { + let t = trim(s: text) + let n = string_length(s: t) + if n < 2 { + FormattedBytesUnparseable { text: t } + } else { + let suffix = substring(s: t, start: n - 1, end: n) + let body = substring(s: t, start: 0, end: n - 1) + if suffix == "B" { + match parse_int(s: body) { + Present { value: v } => if v < 0 { FormattedBytesUnparseable { text: t } } else { FormattedBytesRead { value: FormattedBytesExact { bytes: byte_size(count: v) } } } + Absent => FormattedBytesUnparseable { text: t } + } + } else { + match systemd_iec_suffix_factor(suffix: suffix) { + Absent => FormattedBytesUnparseable { text: t } + Present { value: factor } => + match split(body, ".") { + Cons { head: whole_text, tail: rest } => + match rest { + Cons { head: tenth_text, tail: more } => + match more { + Cons { head: _, tail: _ } => FormattedBytesUnparseable { text: t } + Empty {} => + match parse_int(s: whole_text) { + Absent => FormattedBytesUnparseable { text: t } + Present { value: w } => + match parse_int(s: tenth_text) { + Absent => FormattedBytesUnparseable { text: t } + Present { value: d } => + if w < 0 || d < 0 || d > 9 || string_length(s: tenth_text) != 1 { + FormattedBytesUnparseable { text: t } + } else { + FormattedBytesRead { value: FormattedBytesBounded { + printed: t, + lower: byte_size(count: (w * 10 + d) * factor / 10), + upper: byte_size(count: ((w * 10 + d + 2) * factor + 9) / 10), + } } + } + } + } + } + Empty {} => FormattedBytesUnparseable { text: t } + } + Empty {} => FormattedBytesUnparseable { text: t } + } + } + } + } +} + +fn systemd_formatted_bytes_upper(b: SystemdFormattedBytes) -> ByteSize { + match b { + FormattedBytesExact { bytes } => bytes + FormattedBytesBounded { printed: _, lower: _, upper } => upper + } +} + +type SystemdMemoryPeakReading + = MemoryPeakRead { peak: SystemdFormattedBytes } + | MemoryPeakUnparseable { text: String } + | MemoryPeakNotReported + +type SystemdRunWaitSummary { + result: SystemdServiceResultReading + memory_peak: SystemdMemoryPeakReading +} + +data systemd_run_summary_result_prefix: String = "Finished with result: " +data systemd_run_summary_memory_peak_prefix: String = "Memory peak: " + +fn systemd_run_summary_last_value(stderr: String, prefix: String) -> String? { + fold(split(stderr, "\n"), init: none, f: (acc, line) => + if starts_with(s: line, prefix: prefix) { + Present { value: substring(s: line, start: string_length(s: prefix), end: string_length(s: line)) } + } else { acc }) +} + +fn systemd_run_wait_summary(stderr: String) -> SystemdRunWaitSummary { + SystemdRunWaitSummary { + result: match systemd_run_summary_last_value(stderr: stderr, prefix: systemd_run_summary_result_prefix) { + Present { value: v } => systemd_service_result_parse(wire: v) + Absent => ServiceResultNotReported + }, + memory_peak: match systemd_run_summary_last_value(stderr: stderr, prefix: systemd_run_summary_memory_peak_prefix) { + Present { value: v } => + match systemd_formatted_bytes_parse(text: v) { + FormattedBytesRead { value: b } => MemoryPeakRead { peak: b } + FormattedBytesUnparseable { text: x } => MemoryPeakUnparseable { text: x } + } + Absent => MemoryPeakNotReported + }, + } +} diff --git a/dag/gunbc/compute/host_capacity.dag b/dag/gunbc/compute/host_capacity.dag index a4dbfb0a8a7..b0f12f1fb1c 100644 --- a/dag/gunbc/compute/host_capacity.dag +++ b/dag/gunbc/compute/host_capacity.dag @@ -105,7 +105,7 @@ fn compute_memory_partition(host: NonEmptyStr) -> PartitionId { } // THE JOIN WITH THE DEPLOYMENT, and the only function in this module that knows what an instance is. -fn compute_capacity_subject(instance: HostDashboardInstance) -> ComputeSubjectResolution { +fn compute_capacity_subject(instance: HostDashboardInstance, request: Kibibyte) -> ComputeSubjectResolution { match dashboard_instance_for_host(host: instance.host_identity as String) { DashboardInstanceHostUnknown { host: h } => ComputeSubjectHostUnresolved { host: h as NonEmptyStr } DashboardInstanceForHost { instance: host_instance } => @@ -114,7 +114,7 @@ fn compute_capacity_subject(instance: HostDashboardInstance) -> ComputeSubjectRe root: compute_capacity_root(host_instance: host_instance), partition: compute_memory_partition(host: instance.host_identity), appropriation: compute_memory_appropriation(instance: host_instance), - request: compute_request_memory(instance: instance), + request: request, }, } } @@ -182,11 +182,13 @@ fn compute_memory_appropriation(instance: HostDashboardInstance) -> Kibibyte { // could have run. So one figure serves both, and it is the instance's worker memory ceiling -- // the existing safety limit, unchanged and now also the thing being counted. // -// DECLARED FRONTIER: a per-work-class amount (a claims run does not need a release build's -// ceiling) needs per-class peak measurements, which the consumption reading this change starts -// recording is the first producer of. Until those exist every class reserves the ceiling, which -// over-reserves and never under-reserves. -fn compute_request_memory(instance: HostDashboardInstance) -> Kibibyte { +// THE CEILING IS WHAT A CLASS RESERVES WHEN IT HAS NO GRANT, and it is no longer what every request +// reserves. The per-work-class amount (a claims run does not need a release build's ceiling) is +// gunbc.compute.work_class_grant work_class_grant, which resolves to THIS figure for any class +// without enough samples on the provider's own route -- the fail-closed arm, which over-reserves +// and never under-reserves. A request's amount is therefore supplied to compute_reserve rather than +// read here; this function only names the ceiling the grant falls back to and never exceeds. +fn compute_memory_ceiling(instance: HostDashboardInstance) -> Kibibyte { kibibyte_from_byte_size_floor(b: instance.dispatch_worker_memory_max) } @@ -270,8 +272,8 @@ data compute_partition_read_budget: Nat = 4096 // THE APPROPRIATION IS CHECKED AGAINST THE MACHINE BEFORE ANY SEAT IS TAKEN. A host too small for // the declared share is a configuration error that would otherwise present as an OOM kill during a // build, which is the same information arriving hours later and destructively. -fn compute_reserve(instance: HostDashboardInstance, reference: NonEmptyStr, term_seconds: Nat) -> ComputeReservation { - match compute_capacity_subject(instance: instance) { +fn compute_reserve(instance: HostDashboardInstance, reference: NonEmptyStr, term_seconds: Nat, request: Kibibyte) -> ComputeReservation { + match compute_capacity_subject(instance: instance, request: request) { ComputeSubjectHostUnresolved { host: h } => ComputeReservationRefused { detail: join(["no dashboard instance is declared for host ", h as String, ", so this host has no compute capacity authority to reserve against"], "") } ComputeSubjectResolved { subject: subject } => compute_reserve_on(subject: subject, reference: reference, term_seconds: term_seconds) @@ -443,8 +445,10 @@ type ComputeCapacityStanding = ComputeCapacityStandingRead { committed: Kibibyte, ceiling: Kibibyte, headroom: Kibibyte, observed_available: Kibibyte, observed_total: Kibibyte } | ComputeCapacityStandingUnread { detail: String } +// THE STANDING IS A FACT ABOUT THE POOL, NOT ABOUT ANY REQUEST, so it reads the subject at the +// ceiling: no field of the standing fold consults the subject's request. fn compute_capacity_standing(instance: HostDashboardInstance) -> ComputeCapacityStanding { - match compute_capacity_subject(instance: instance) { + match compute_capacity_subject(instance: instance, request: compute_memory_ceiling(instance: instance)) { ComputeSubjectHostUnresolved { host: h } => ComputeCapacityStandingUnread { detail: join(["no dashboard instance is declared for host ", h as String], "") } ComputeSubjectResolved { subject: subject } => compute_capacity_standing_on(subject: subject) } diff --git a/dag/gunbc/compute/work_class_grant.dag b/dag/gunbc/compute/work_class_grant.dag new file mode 100644 index 00000000000..9fbb0f76d48 --- /dev/null +++ b/dag/gunbc/compute/work_class_grant.dag @@ -0,0 +1,166 @@ +module gunbc.compute.work_class_grant + +import std.types { String, Bool, Int, List, NonEmptyStr, EpochSecs } +import std.nat { Nat } +import std.measure { Kibibyte, kibibyte, kibibyte_count } +import gunbc.compute.work_request { WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoProfile, cargo_profile_wire } + +// A WORK REQUEST RESERVES AN AMOUNT PER CLASS, AND THE AMOUNT IS A GRANT WITH ITS OBSERVATIONS +// ATTACHED (operator ruling 2026-09-21: do not raise a global ceiling; size a job's reservation per +// class from that class's measured peak, admitted against observed host memory). +// +// THE GRANT IS ALSO THE CAP. gunbc.compute.host_capacity makes the reserved amount and the unit's +// MemoryMax one figure, so a grant sized at a measured peak would also KILL the first run that +// exceeds it. A peak is a SAMPLE, not a bound. That is why a grant is peak plus headroom, why the +// headroom is a named policy budget rather than a picked multiplier, and why exceeding a grant is a +// typed, counted ending (gunbc.compute.work_provider_local compute_grant_exceeded_record) rather +// than an OOM that renders as the work having failed. +// +// WHAT THIS CHANGE DOES AND DOES NOT DELIVER. It builds the producer (the unit's peak, read from +// the launcher's own summary), the typed ending, the per-class counter and this resolution. It +// ships NO grant rows: no class has a peak measured on the provider's own route yet, so every class +// resolves to the ceiling and no reserved amount moves. The relief is the next change, the one that +// adds rows from the samples this one starts recording. + +// THE CLASS GRAIN IS A CLAIM, STATED SO IT CAN BE WRONG. A class is the operation kind, split by +// cargo profile for builds. A compile's peak depends heavily on WHICH entry it compiles (the +// v2.compiler.compile closure is far above a single witness module), so two compiles of one class +// can differ by an order of magnitude. A grant over that grain is sampled, not derived, which is +// exactly the residual gunbc.recurring_failure_mode a_sampled_grant_is_discovered_below_worst_case +// rosters -- and the counted GrantExceeded ending is what reports a class whose grain is too coarse. +type WorkClass + = WorkClassBuild { profile: CargoProfile } + | WorkClassCompile + | WorkClassClaims + +fn work_class_of(op: WorkOperation) -> WorkClass { + match op { + BuildGunbcBinaries { profile, bins: _ } => WorkClassBuild { profile: profile } + CompileEntry { entry: _ } => WorkClassCompile + RunClaims { entry: _, functions: _, hermetic: _ } => WorkClassClaims + } +} + +fn work_class_wire(c: WorkClass) -> String { + match c { + WorkClassBuild { profile } => join(["build-", cargo_profile_wire(p: profile)], "") + WorkClassCompile => "compile" + WorkClassClaims => "claims" + } +} + +// ONE SAMPLE: one attempt of one work identity on one host, and the UPPER BOUND of the peak the +// unit's manager reported (extdeps.systemd.systemd_run systemd_formatted_bytes_upper -- the printed +// figure is truncated, so its lower reading would size a grant below the peak that produced it). +// The operation's canonical text says what ran, so "over what" is carried, not remembered. +type WorkClassPeakSample { + work_identity: NonEmptyStr + operation: NonEmptyStr + host: NonEmptyStr + observed_at: EpochSecs + peak_upper: Kibibyte +} + +// THE ROW: a policy budget grounded in observations. The headroom is the OPERATOR'S figure and is +// cited as such; what makes it admissible (DESIGN 5) is that it is FALSIFIABLE -- a non-zero +// GrantExceeded count for this class (compute_grant_exceeded_count) means the grant is wrong and +// the row is re-derived. A budget nobody can prove wrong is a guess; this one carries its counter. +type WorkClassGrantRow { + class: WorkClass + samples: List + headroom: Kibibyte + headroom_authority: NonEmptyStr +} + +// ONE OBSERVATION IS NOT A PEAK (operator ruling, 2026-09-21), so the smallest sample set a grant +// may be sized from is two. Below that the class reserves the ceiling and says how many it has. +data work_class_minimum_samples: Int = 2 + +// THE ROWS. Empty in this change, deliberately and not as a placeholder: a row lands with samples +// taken on the provider's route (the per-attempt record compute_run_and_settle writes), and none +// exist yet. neat-boar-16's 9.7 GB peak for the v2.compiler.compile closure is a real observation +// but was not taken under a provider unit, so it is cited in the next change's evidence and does +// not size a row here. +data work_class_grant_rows: List = [] + +type WorkClassCeilingReason + = ClassUnmeasured + | ClassTooFewSamples { samples: Int } + | ClassRowsAmbiguous { rows: Int } + | ClassRetryAfterGrantExceeded { exceeded: Kibibyte } + | ClassSizedAtOrAboveCeiling { samples: Int, sized: Kibibyte } + +// EVERY ARM NAMES ITS CLASS AND ITS EVIDENCE COUNT. The ceiling arm is the fail-closed one and it +// is the only arm this change's rows reach; it must stay reachable, so "no row" and "too few +// samples" are both ceiling, never a default amount. +type WorkClassGrant + = GrantAtCeiling { class: WorkClass, amount: Kibibyte, reason: WorkClassCeilingReason } + | GrantSized { class: WorkClass, amount: Kibibyte, samples: Int, peak_upper: Kibibyte, headroom: Kibibyte, headroom_authority: NonEmptyStr } + +fn work_class_grant_amount(g: WorkClassGrant) -> Kibibyte { + match g { + GrantAtCeiling { class: _, amount, reason: _ } => amount + GrantSized { class: _, amount, samples: _, peak_upper: _, headroom: _, headroom_authority: _ } => amount + } +} + +fn work_class_grant_is_ceiling(g: WorkClassGrant) -> Bool { + match g { + GrantAtCeiling { class: _, amount: _, reason: _ } => true + GrantSized { class: _, amount: _, samples: _, peak_upper: _, headroom: _, headroom_authority: _ } => false + } +} + +fn work_class_grant_wire(g: WorkClassGrant) -> String { + match g { + GrantAtCeiling { class, amount, reason } => + join(["class ", work_class_wire(c: class), " reserves the ceiling ", to_string(value: kibibyte_count(k: amount)), " KiB (", + match reason { + ClassUnmeasured => "no grant row" + ClassTooFewSamples { samples } => join([to_string(value: samples), " sample(s), fewer than ", to_string(value: work_class_minimum_samples)], "") + ClassRetryAfterGrantExceeded { exceeded } => join(["this attempt's grant of ", to_string(value: kibibyte_count(k: exceeded)), " KiB was exceeded, so it is retried at the ceiling"], "") + ClassRowsAmbiguous { rows } => join([to_string(value: rows), " grant rows name this class, so none of them decides"], "") + ClassSizedAtOrAboveCeiling { samples, sized } => join(["sized ", to_string(value: kibibyte_count(k: sized)), " KiB from ", to_string(value: samples), " samples, not below the ceiling"], "") + }, ")"], "") + GrantSized { class, amount, samples, peak_upper, headroom, headroom_authority } => + join(["class ", work_class_wire(c: class), " reserves ", to_string(value: kibibyte_count(k: amount)), " KiB = peak ", + to_string(value: kibibyte_count(k: peak_upper)), " KiB over ", to_string(value: samples), " samples + headroom ", + to_string(value: kibibyte_count(k: headroom)), " KiB (", headroom_authority as String, ")"], "") + } +} + +fn work_class_rows_for(rows: List, class: WorkClass) -> List { + filter(rows, r => work_class_wire(c: r.class) == work_class_wire(c: class)) +} + +fn work_class_samples_peak(samples: List) -> Nat { + fold(samples, init: kibibyte_count(k: kibibyte(count: 0)), f: (acc, s) => if kibibyte_count(k: s.peak_upper) > acc { kibibyte_count(k: s.peak_upper) } else { acc }) +} + +// THE RESOLUTION, taking the rows and the ceiling as values so every arm is exercisable with a +// supplied row while production passes work_class_grant_rows. A class with more than one row is +// refused to the ceiling rather than resolved by position: two rows for one class are two +// authorities for one fact, and picking one would be the silent choice DESIGN 3 forbids. +fn work_class_grant(class: WorkClass, rows: List, ceiling: Kibibyte) -> WorkClassGrant { + match work_class_rows_for(rows: rows, class: class) { + Empty {} => GrantAtCeiling { class: class, amount: ceiling, reason: ClassUnmeasured } + Cons { head: row, tail: more } => + match more { + Cons { head: _, tail: _ } => GrantAtCeiling { class: class, amount: ceiling, reason: ClassRowsAmbiguous { rows: length(work_class_rows_for(rows: rows, class: class)) } } + Empty {} => { + let n = length(row.samples) + if n < work_class_minimum_samples { + GrantAtCeiling { class: class, amount: ceiling, reason: ClassTooFewSamples { samples: n } } + } else { + let peak = work_class_samples_peak(samples: row.samples) + let sized = kibibyte(count: peak + kibibyte_count(k: row.headroom)) + if kibibyte_count(k: sized) >= kibibyte_count(k: ceiling) { + GrantAtCeiling { class: class, amount: ceiling, reason: ClassSizedAtOrAboveCeiling { samples: n, sized: sized } } + } else { + GrantSized { class: class, amount: sized, samples: n, peak_upper: kibibyte(count: peak), headroom: row.headroom, headroom_authority: row.headroom_authority } + } + } + } + } + } +} diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 13ca10a3a44..098d5a9fba3 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -10,7 +10,13 @@ import extdeps.gunbc import extdeps.shell import extdeps.git { shape_git_worktree_add_detached_argv } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex, git_object_id_wire_hex } -import extdeps.systemd.systemd_run { systemd_run_user_wait_arguments, systemd_run_property } +import extdeps.systemd.systemd_run { + systemd_run_user_wait_arguments, systemd_run_property, + SystemdRunWaitSummary, systemd_run_wait_summary, SystemdServiceResultReading, ServiceResultRead, ServiceResultUnrecognized, ServiceResultNotReported, + SystemdServiceResult, ServiceResultOomKill, systemd_service_result_wire, + SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, + SystemdFormattedBytes, FormattedBytesExact, FormattedBytesBounded, systemd_formatted_bytes_upper, +} import extdeps.systemd.systemctl import extdeps.linux.cgroup_v2 { cgroup_v2_mount_point, cgroup_v2_events_path, @@ -26,10 +32,14 @@ import gunbc.roadmap_dispatch_actuator { dispatch_capability_program } import gunbc.roadmap_execution_contract { ExecutionCapability, GitWorkspaceCapability, CargoCapability, RustcCapability, AttemptStateDirectoryCapability, } -import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size } +import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size, kibibyte, byte_size_count } +import gunbc.compute.work_class_grant { + WorkClass, work_class_of, work_class_wire, WorkClassGrant, GrantAtCeiling, GrantSized, ClassRetryAfterGrantExceeded, + work_class_grant, work_class_grant_rows, work_class_grant_amount, work_class_grant_is_ceiling, work_class_grant_wire, +} import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, - ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, + ComputeRelease, ComputeReleased, ComputeReleaseRefused, compute_reserve, compute_release, compute_reservation_wire, compute_memory_ceiling, ComputeCapacityStanding, ComputeCapacityStandingRead, ComputeCapacityStandingUnread, compute_capacity_standing, compute_capacity_standing_wire, } import gunbc.compute.work_request { @@ -39,8 +49,8 @@ import gunbc.compute.work_request { WorkOutput, WorkOutcome, WorkSucceeded, WorkFailed, WorkRefusedByInfrastructure, WorkCancelled, WorkOutputMismatch, WorkInfrastructureRefusal, ProducerInFlight, HostComputeCapacityFull, HostBelowMemoryFloor, ComputeCapacityLedgerUnavailable, CheckoutUnavailable, StoreUnavailable, - ToolchainUnobserved, DependencyNotSucceeded, SubjectUnresolved, - work_outcome_wire, work_outcome_ending, StoredOutcomeRead, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_decode, stored_outcome_is_success, + ToolchainUnobserved, DependencyNotSucceeded, SubjectUnresolved, UnitMemoryGrantExceeded, + work_outcome_wire, work_outcome_ending, work_operation_canonical, StoredOutcomeRead, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_decode, stored_outcome_is_success, } // THE LOCAL PROVIDER: one of N behind the exact work contract, serving the host it runs on. It is a @@ -600,6 +610,120 @@ fn compute_attempt_completed(outcome: WorkOutcome) -> Bool { } } +// ── A GRANT THAT WAS TOO SMALL: RETRIED ONCE AT THE CEILING, AND COUNTED ────────────────────── +// +// WHICH RETRY THIS IS, since the phrase admits two (loyal-swift-608, 2026-09-21). It is NOT the class +// reserving the ceiling forever after one OOM: that would let one bad run revert a class to the +// status quo silently and permanently. It is THIS ATTEMPT retried at the ceiling, with the event +// written to a per-class record whose count is readable -- so the work still completes, and the +// grant that was too small is reported rather than absorbed. +// +// UNCOUNTED, THIS WOULD BE THE ABSORBING FALLBACK IN ITS EXACT FORM: the grant is too small, the +// retry widens to the ceiling, the work succeeds, and the mechanism degrades to yesterday's +// behaviour while looking like it works. COUNTED, the retry is the falsifier of the grant's +// headroom budget: a class with a non-zero count has a wrong grant (gunbc.compute.work_class_grant +// WorkClassGrantRow). That is also why the count is written BEFORE the retry is attempted -- a retry +// that is refused for capacity still leaves the evidence that the grant was exceeded. +// +// A GRANT THAT WAS ALREADY THE CEILING IS NOT RETRIED: there is no larger reservation, so the +// attempt ends as the typed UnitMemoryGrantExceeded refusal. And a retry is taken only after the +// first seat is PROVEN released, by the same termination fact settlement uses; a seat that cannot be +// proven free is settled as it stands, so the retry can never run beside the attempt it replaces. +fn compute_grant_exceeded_dir(layout: ComputeLayout, class: WorkClass) -> String { + join([layout.root, "/grant-exceeded/", work_class_wire(c: class)], "") +} + +fn compute_grant_exceeded_record(instance: HostDashboardInstance, layout: ComputeLayout, op: WorkOperation, grant: WorkClassGrant, attempt: ComputeAttempt, reference: NonEmptyStr) -> Bool { + let dir = compute_grant_exceeded_dir(layout: layout, class: work_class_of(op: op)) + if !compute_ensure_dir(instance: instance, path: dir) { + false + } else { + Filesystem.WriteCreateNew( + path: join([dir, "/", reference as String], ""), + content: compute_peak_sample_wire(op: op, grant: grant, summary: attempt.summary), + ).success + } +} + +// THE READABLE COUNT: the number of recorded exceedances for a class on this host. An unreadable +// directory is reported as unreadable and never as zero -- a count that reads 0 when it could not +// look would be the one falsifier of the budget silently reporting the budget sound. +type GrantExceededCount + = GrantExceededCounted { count: Int } + | GrantExceededNeverRecorded + | GrantExceededUnreadable { detail: String } + +fn compute_grant_exceeded_count(instance: HostDashboardInstance, class: WorkClass) -> GrantExceededCount { + let dir = join([dashboard_instance_compute_root(instance: instance) as String, "/grant-exceeded/", work_class_wire(c: class)], "") + let listed = Filesystem.List(path: dir) + if listed.success { + GrantExceededCounted { count: length(filter(split(listed.entries, "\n"), n => trim(s: n) != "")) } + } else if !Filesystem.List(path: join([dashboard_instance_compute_root(instance: instance) as String, "/grant-exceeded"], "")).success { + GrantExceededNeverRecorded + } else { + GrantExceededUnreadable { detail: listed.error } + } +} + +fn compute_grant_exceeded_count_wire(instance: HostDashboardInstance, class: WorkClass) -> String { + match compute_grant_exceeded_count(instance: instance, class: class) { + GrantExceededCounted { count } => to_string(value: count) + GrantExceededNeverRecorded => "0 (no exceedance has ever been recorded on this host)" + GrantExceededUnreadable { detail } => join(["UNREADABLE (", detail, ")"], "") + } +} + +fn compute_attempt_retries_at_ceiling(grant: WorkClassGrant, attempt: ComputeAttempt) -> Bool { + !work_class_grant_is_ceiling(g: grant) && compute_summary_is_oom_kill(summary: attempt.summary) +} + +fn compute_run_granted( + instance: HostDashboardInstance, + commit: String, + layout: ComputeLayout, + identity: String, + subject: WorkSubject, + op: WorkOperation, + dependency_store: String, + grant: WorkClassGrant, + reference: NonEmptyStr, + reservation: ComputeReservation, +) -> ProvideResult { + let granted = work_class_grant_amount(g: grant) + let attempt = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted, grant_was_ceiling: work_class_grant_is_ceiling(g: grant)) + if !compute_attempt_retries_at_ceiling(grant: grant, attempt: attempt) { + compute_run_and_settle(instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, dependency_store: dependency_store, grant: grant, attempt: attempt, reference: reference, reservation: reservation) + } else { + let counted = compute_grant_exceeded_record(instance: instance, layout: layout, op: op, grant: grant, attempt: attempt, reference: reference) + let termination = observe_unit_termination(identity_hex: identity, attempt_completed: compute_attempt_completed(outcome: attempt.outcome)) + let returned = if counted && unit_termination_frees_capacity(t: termination) { + compute_release(reservation: reservation, reason: join(["grant of ", to_string(value: kibibyte_count(k: granted)), " KiB exceeded (oom-kill); retrying this attempt at the ceiling"], "") as NonEmptyStr) + } else { + ComputeReleaseRefused { detail: if counted { join(["not retried: ", unit_termination_wire(t: termination)], "") } else { "not retried: the exceedance could not be recorded, and an uncounted retry is the absorbing fallback" } } + } + if !compute_release_landed(r: returned) { + compute_run_and_settle(instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, dependency_store: dependency_store, grant: grant, attempt: attempt, reference: reference, reservation: reservation) + } else { + let ceiling = compute_memory_ceiling(instance: instance) + let retry_grant = GrantAtCeiling { class: work_class_of(op: op), amount: ceiling, reason: ClassRetryAfterGrantExceeded { exceeded: granted } } + let retry_reference = compute_reservation_reference(identity: identity) + let retry_reservation = compute_reserve(instance: instance, reference: retry_reference, term_seconds: compute_reservation_term_seconds, request: ceiling) + match retry_reservation { + ComputeCapacityFull { wire: w } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) + ComputeHostBelowFloor { requested: rq, available: av } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested: rq, available: av }) + ComputeReservationRefused { detail: d } => + compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => { + let retry = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: ceiling, grant_was_ceiling: true) + compute_run_and_settle(instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, dependency_store: dependency_store, grant: retry_grant, attempt: retry, reference: retry_reference, reservation: retry_reservation) + } + } + } + } +} + fn compute_run_and_settle( instance: HostDashboardInstance, commit: String, @@ -608,11 +732,12 @@ fn compute_run_and_settle( subject: WorkSubject, op: WorkOperation, dependency_store: String, - granted: Kibibyte, + grant: WorkClassGrant, + attempt: ComputeAttempt, reference: NonEmptyStr, reservation: ComputeReservation, ) -> ProvideResult { - let outcome = compute_run_leased(instance: instance, commit: commit, layout: layout, identity: identity, op: op, dependency_store: dependency_store, granted: granted) + let outcome = attempt.outcome let standing = compute_capacity_standing(instance: instance) let termination = observe_unit_termination(identity_hex: identity, attempt_completed: compute_attempt_completed(outcome: outcome)) let returned = if unit_termination_frees_capacity(t: termination) { @@ -632,6 +757,8 @@ fn compute_run_and_settle( path: layout.consumption_path, content: join([ compute_capacity_standing_wire(s: standing), "\n", + compute_peak_sample_wire(op: op, grant: grant, summary: attempt.summary), + "grant exceeded, this class: ", compute_grant_exceeded_count_wire(instance: instance, class: work_class_of(op: op)), "\n", "unit: ", unit_termination_wire(t: termination), "\n", "reservation: ", compute_release_wire(r: returned), "\n", if dropped.success { "" } else { compute_stale_lease_note(layout: layout, error: dropped.error) }, @@ -828,7 +955,8 @@ fn compute_provide_leaf( compute_fresh(outcome: WorkRefusedByInfrastructure { identity: identity, cause: ProducerInFlight { lease_path: layout.lease_path } }, subject: subject, op: op) } else { let reference = compute_reservation_reference(identity: identity) - let reservation = compute_reserve(instance: instance, reference: reference, term_seconds: compute_reservation_term_seconds) + let grant = work_class_grant(class: work_class_of(op: op), rows: work_class_grant_rows, ceiling: compute_memory_ceiling(instance: instance)) + let reservation = compute_reserve(instance: instance, reference: reference, term_seconds: compute_reservation_term_seconds, request: work_class_grant_amount(g: grant)) match reservation { ComputeCapacityFull { wire: w } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostComputeCapacityFull { wire: w }) @@ -836,10 +964,10 @@ fn compute_provide_leaf( compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: HostBelowMemoryFloor { requested: rq, available: av }) ComputeReservationRefused { detail: d } => compute_refused_unreserved(instance: instance, identity: identity, subject: subject, op: op, layout: layout, cause: ComputeCapacityLedgerUnavailable { detail: d }) - ComputeReserved { grant: _, partition: _, amount: granted, store: _ } => - compute_run_and_settle( + ComputeReserved { grant: _, partition: _, amount: _, store: _ } => + compute_run_granted( instance: instance, commit: commit, layout: layout, identity: identity, subject: subject, op: op, - dependency_store: dependency_store, granted: granted, reference: reference, reservation: reservation) + dependency_store: dependency_store, grant: grant, reference: reference, reservation: reservation) } } } @@ -907,6 +1035,57 @@ fn compute_fresh(outcome: WorkOutcome, subject: WorkSubject, op: WorkOperation) ProvideFresh { outcome: outcome, document: work_outcome_wire(o: outcome, subject: subject, op: op), host_record: HostRecordWritten } } +// ONE ATTEMPT AND WHAT ITS UNIT'S MANAGER SAID ABOUT IT. The summary is read off the launcher's +// stderr because it is the only reading of the unit's peak and result that outlives the unit +// (extdeps.systemd.systemd_run systemd_run_wait_summary); a checkout or toolchain refusal never +// launched a unit, so its summary reports nothing, which is the honest reading of that attempt. +type ComputeAttempt { + outcome: WorkOutcome + summary: SystemdRunWaitSummary +} + +fn compute_summary_is_oom_kill(summary: SystemdRunWaitSummary) -> Bool { + match summary.result { + ServiceResultRead { result: r } => match r { ServiceResultOomKill => true _ => false } + ServiceResultUnrecognized { wire: _ } => false + ServiceResultNotReported => false + } +} + +fn compute_summary_peak_wire(summary: SystemdRunWaitSummary) -> String { + match summary.memory_peak { + MemoryPeakRead { peak: b } => + match b { + FormattedBytesExact { bytes } => join([to_string(value: byte_size_count(b: bytes)), " B"], "") + FormattedBytesBounded { printed, lower: _, upper } => join([printed, " (at most ", to_string(value: byte_size_count(b: upper)), " B)"], "") + } + MemoryPeakUnparseable { text } => join(["UNPARSEABLE (", text, ")"], "") + MemoryPeakNotReported => "NOT REPORTED (no Memory peak line: systemd below v254, or --wait summary suppressed)" + } +} + +fn compute_summary_result_wire(summary: SystemdRunWaitSummary) -> String { + match summary.result { + ServiceResultRead { result: r } => systemd_service_result_wire(r: r) + ServiceResultUnrecognized { wire } => join(["UNRECOGNIZED (", wire, ")"], "") + ServiceResultNotReported => "NOT REPORTED" + } +} + +// THE PEAK SAMPLE, and a peak that was not reported is NEVER written as zero: a recorded zero would +// size its class at headroom alone, which is the worst failure this mechanism can have. The sample +// line carries the class, the operation's canonical text and the unparsed reading, so a later grant +// row can cite exactly which runs it was sized from. +fn compute_peak_sample_wire(op: WorkOperation, grant: WorkClassGrant, summary: SystemdRunWaitSummary) -> String { + join([ + "class: ", work_class_wire(c: work_class_of(op: op)), "\n", + "operation: ", work_operation_canonical(op: op), "\n", + "grant: ", work_class_grant_wire(g: grant), "\n", + "unit result: ", compute_summary_result_wire(summary: summary), "\n", + "unit peak: ", compute_summary_peak_wire(summary: summary), "\n", + ], "") +} + fn compute_run_leased( instance: HostDashboardInstance, commit: String, @@ -915,7 +1094,8 @@ fn compute_run_leased( op: WorkOperation, dependency_store: String, granted: Kibibyte, -) -> WorkOutcome { + grant_was_ceiling: Bool, +) -> ComputeAttempt { let existing = Filesystem.List(path: layout.checkout) let checkout = if existing.success { ComputeExecOk { stdout: "", stderr: "" } @@ -928,20 +1108,37 @@ fn compute_run_leased( } match checkout { ComputeExecFailed { exit_code, stdout: _, stderr } => - WorkRefusedByInfrastructure { identity: identity, cause: CheckoutUnavailable { detail: join(["git worktree add exited ", to_string(value: exit_code), ": ", stderr], "") } } + ComputeAttempt { + outcome: WorkRefusedByInfrastructure { identity: identity, cause: CheckoutUnavailable { detail: join(["git worktree add exited ", to_string(value: exit_code), ": ", stderr], "") } }, + summary: SystemdRunWaitSummary { result: ServiceResultNotReported, memory_peak: MemoryPeakNotReported }, + } ComputeExecOk { stdout: _, stderr: _ } => { let run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store), granted: granted) let log = Filesystem.Write(path: layout.log_path, content: match run { ComputeExecOk { stdout, stderr } => join([stdout, stderr], "") ComputeExecFailed { exit_code: _, stdout, stderr } => join([stdout, stderr], "") }) - match run { - ComputeExecFailed { exit_code, stdout: _, stderr: _ } => WorkFailed { identity: identity, exit_code: exit_code, log_path: layout.log_path } - ComputeExecOk { stdout: _, stderr: _ } => - match compute_return_outputs(instance: instance, layout: layout, declared: work_declared_outputs(op: op)) { - OutputsMismatch { detail } => WorkOutputMismatch { identity: identity, detail: detail } - OutputsReturned { outputs } => WorkSucceeded { identity: identity, outputs: outputs, log_path: if log.success { layout.log_path } else { "" } } - } + let summary = systemd_run_wait_summary(stderr: match run { + ComputeExecOk { stdout: _, stderr } => stderr + ComputeExecFailed { exit_code: _, stdout: _, stderr } => stderr + }) + ComputeAttempt { + outcome: match run { + ComputeExecFailed { exit_code, stdout: _, stderr: _ } => + if compute_summary_is_oom_kill(summary: summary) { + WorkRefusedByInfrastructure { identity: identity, cause: UnitMemoryGrantExceeded { + class: work_class_wire(c: work_class_of(op: op)), granted: granted, peak: compute_summary_peak_wire(summary: summary), grant_was_ceiling: grant_was_ceiling, + } } + } else { + WorkFailed { identity: identity, exit_code: exit_code, log_path: layout.log_path } + } + ComputeExecOk { stdout: _, stderr: _ } => + match compute_return_outputs(instance: instance, layout: layout, declared: work_declared_outputs(op: op)) { + OutputsMismatch { detail } => WorkOutputMismatch { identity: identity, detail: detail } + OutputsReturned { outputs } => WorkSucceeded { identity: identity, outputs: outputs, log_path: if log.success { layout.log_path } else { "" } } + } + }, + summary: summary, } } } diff --git a/dag/gunbc/compute/work_request.dag b/dag/gunbc/compute/work_request.dag index 16883a0830b..63c0c0a89eb 100644 --- a/dag/gunbc/compute/work_request.dag +++ b/dag/gunbc/compute/work_request.dag @@ -154,6 +154,12 @@ type WorkOutcome // all and that a caller must not confuse with it: the machine's LIVE availability being below what // this class needs is somebody else's memory, not this pool's commitments, and an unreadable // ledger is a store repair. One arm for three remedies is the conflation DESIGN 5 forbids. +// A UNIT KILLED FOR MEMORY IS THE MACHINE ENDING THE WORK, NOT THE WORK FAILING, and it has its +// own arm for the reason this vocabulary has five endings rather than one: rendering an OOM kill as +// WorkFailed { exit_code: 137 } tells the caller to fix its code when the remedy is a larger +// reservation. The arm is reached only when no larger reservation remains -- a grant below the +// ceiling that is exceeded retries at the ceiling first (gunbc.compute.work_provider_local +// compute_run_granted), and that retry is counted per class. type WorkInfrastructureRefusal = ProducerInFlight { lease_path: String } | HostComputeCapacityFull { wire: String } @@ -164,6 +170,7 @@ type WorkInfrastructureRefusal | ToolchainUnobserved { detail: String } | DependencyNotSucceeded { dependency_identity: String, ending: String } | SubjectUnresolved { detail: String } + | UnitMemoryGrantExceeded { class: String, granted: Kibibyte, peak: String, grant_was_ceiling: Bool } fn work_outcome_identity(o: WorkOutcome) -> String { match o { @@ -196,6 +203,10 @@ fn work_refusal_detail(r: WorkInfrastructureRefusal) -> String { ToolchainUnobserved { detail } => join(["toolchain unobserved: ", detail], "") DependencyNotSucceeded { dependency_identity, ending } => join(["dependency ", dependency_identity, " ended ", ending], "") SubjectUnresolved { detail } => join(["subject unresolved: ", detail], "") + UnitMemoryGrantExceeded { class, granted, peak, grant_was_ceiling } => + join(["the unit was OOM-killed at its memory grant: class ", class, " was granted ", to_string(value: kibibyte_count(k: granted)), + " KiB and its manager reported a peak of ", peak, + if grant_was_ceiling { "; the grant was this host's ceiling, so no larger reservation exists to retry under" } else { "; the grant was below the ceiling and the retry at the ceiling was not taken -- the consumption record says why" }], "") } } diff --git a/dag/gunbc/recurring_failure_mode/a_sampled_grant_is_discovered_below_worst_case.dag b/dag/gunbc/recurring_failure_mode/a_sampled_grant_is_discovered_below_worst_case.dag new file mode 100644 index 00000000000..5cbc87f1ecc --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/a_sampled_grant_is_discovered_below_worst_case.dag @@ -0,0 +1,32 @@ +module gunbc.recurring_failure_mode.a_sampled_grant_is_discovered_below_worst_case + +import std.types { NonEmptyStr } +import std.decl_ref { DeclarationRef, decl_ref } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data a_sampled_grant_is_discovered_below_worst_case: RecurringFailureMode = RecurringFailureMode { + identity: "a_sampled_grant_is_discovered_below_worst_case" as NonEmptyStr, + + receipts: [ + "INVALID STATE: a work class's memory grant (gunbc.compute.work_class_grant work_class_grant) is sized from sampled peaks plus a policy-budget headroom, and some run of that class needs more than the grant. Because the reservation and the unit's MemoryMax are one figure (gunbc.compute.host_capacity), the grant is also the cap, so that run is OOM-killed by its own cgroup. HARM: the attempt's work is lost and repeated at the ceiling -- wall time and one ceiling-sized reservation -- and, if no ceiling seat is free, the request is refused for capacity after having already spent a run.", + + "WHAT KEEPS IT FROM BEING SILENT, which is the whole reason it may be admitted at all (DESIGN 5): the unit's own Result is read from the launcher's summary (extdeps.systemd.systemd_run systemd_run_wait_summary), an oom-kill is never rendered as WorkFailed, the attempt is retried at the ceiling ONCE, and every exceedance is written to a per-class record on the host whose count is readable (gunbc.compute.work_provider_local compute_grant_exceeded_count) and is printed in every consumption record of that class. A non-zero count FALSIFIES that class's headroom budget and is the signal to re-derive its row. The retry is not a permanent downgrade of the class: one bad run does not silently return a class to the ceiling forever.", + + "THE CLASS GRAIN IS PART OF THE HAZARD. A class is the operation kind (build per cargo profile, compile, claims), and a compile's peak depends heavily on the entry it compiles, so a grant sized from small-entry samples is below the peak of a large entry by construction. The count reports that; it does not prevent it.", + + "RUNG FOUND AT: mitigatable. The state is reachable, its harm is bounded (one lost attempt per exceedance, then a ceiling retry), and it is typed and counted rather than silent. It is not reachable in production at the change that files this row: work_class_grant_rows is empty, so every class reserves the ceiling and no grant can be below it.", + + "CEILING: structurally guaranteed for a class whose peak is DERIVED rather than sampled. Build peak is not a random variable to be estimated; it is roughly a function of the largest compilation unit and the parallelism the class runs at, so a class that declares its job count and the units it compiles admits a bound established by argument. Undeclared or data-dependent classes (a compile whose closure is chosen at request time) stay at this rung, where the counter is the honest mechanism.", + + "NEXT-RUNG TRIGGER, NAMING THE CAPABILITY AND NOT AN ARTIFACT: a STRUCTURAL PEAK BOUND for a work class -- a derivation, consumed by work_class_grant in place of sampled peaks plus budgeted headroom, that bounds the class's resident memory from its declared parallelism and its largest compilation unit (or the corpus it resolves), sufficient that a grant sized at that bound cannot be exceeded by any request the class admits. More samples do NOT satisfy this trigger: they would be satisfied while the class stayed at this rung forever.", + + "CONSUMPTION: the derived gunbc.recurring_failure_mode.roster recurring_failure_mode_roster includes this per-class declaration; gunbc.compute.work_class_grant cites it at the class grain it concerns.", + ], + + evidence: [ + decl_ref(module_path: "gunbc.compute.work_class_grant", decl_name: "work_class_grant"), + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "compute_run_granted"), + decl_ref(module_path: "gunbc.compute.work_provider_local", decl_name: "compute_grant_exceeded_count"), + decl_ref(module_path: "extdeps.systemd.systemd_run", decl_name: "systemd_run_wait_summary"), + ], +} diff --git a/dag/test/claim/compute/host_capacity_wet_witness_test.dag b/dag/test/claim/compute/host_capacity_wet_witness_test.dag index 490ef641cd8..a75ad969654 100644 --- a/dag/test/claim/compute/host_capacity_wet_witness_test.dag +++ b/dag/test/claim/compute/host_capacity_wet_witness_test.dag @@ -8,7 +8,7 @@ import product.capacity.event_chain { PartitionId } import gunbc.roadmap_dashboard_instance { srv1_live_dashboard_instance, srv1_lab_dashboard_instance, dashboard_instance_compute_root } import gunbc.compute.host_capacity { ComputeCapacitySubject, ComputeSubjectResolution, ComputeSubjectResolved, ComputeSubjectHostUnresolved, - compute_capacity_subject, compute_capacity_root, compute_memory_partition, + compute_capacity_subject, compute_capacity_root, compute_memory_partition, compute_memory_ceiling, HostMemoryObservation, HostMemoryObserved, HostMemoryUnobserved, observe_host_memory, ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, compute_reserve_on, compute_reserve_against, compute_release, ComputeRelease, ComputeReleased, ComputeReleaseRefused, @@ -149,10 +149,10 @@ test fn an_appropriation_larger_than_the_machine_refuses_rather_than_admitting() test fn two_instances_on_one_host_resolve_to_one_pool_at_one_root() -> Bool { let live = srv1_live_dashboard_instance() let lab = srv1_lab_dashboard_instance() - match compute_capacity_subject(instance: live) { + match compute_capacity_subject(instance: live, request: compute_memory_ceiling(instance: live)) { ComputeSubjectHostUnresolved { host: _ } => false ComputeSubjectResolved { subject: a } => - match compute_capacity_subject(instance: lab) { + match compute_capacity_subject(instance: lab, request: compute_memory_ceiling(instance: lab)) { ComputeSubjectHostUnresolved { host: _ } => false ComputeSubjectResolved { subject: b } => (live.instance_id as String) != (lab.instance_id as String) diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag new file mode 100644 index 00000000000..0c2d285cd1c --- /dev/null +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -0,0 +1,134 @@ +module test.claim.compute.work_class_grant_witness_test + +import std.types { String, Bool, Int, List, NonEmptyStr, EpochSecs } +import std.measure { Kibibyte, kibibyte, kibibyte_count, byte_size_count } +import extdeps.systemd.systemd_run { + SystemdRunWaitSummary, systemd_run_wait_summary, + SystemdServiceResultReading, ServiceResultRead, ServiceResultUnrecognized, ServiceResultNotReported, + SystemdServiceResult, ServiceResultOomKill, ServiceResultSuccess, ServiceResultExitCode, + SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, + SystemdFormattedBytes, FormattedBytesExact, FormattedBytesBounded, systemd_formatted_bytes_upper, +} +import gunbc.compute.work_request { WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, WorkSucceeded, WorkFailed } +import gunbc.compute.work_class_grant { + WorkClass, WorkClassBuild, WorkClassCompile, WorkClassClaims, work_class_of, work_class_wire, + WorkClassPeakSample, WorkClassGrantRow, work_class_grant_rows, + WorkClassGrant, GrantAtCeiling, GrantSized, + WorkClassCeilingReason, ClassUnmeasured, ClassTooFewSamples, ClassRowsAmbiguous, ClassSizedAtOrAboveCeiling, ClassRetryAfterGrantExceeded, + work_class_grant, work_class_grant_amount, +} +import gunbc.compute.work_provider_local { ComputeAttempt, compute_attempt_retries_at_ceiling, compute_summary_is_oom_kill } + +// ── THE PRODUCER: what systemd-run --wait said about the unit ───────────────────────────────── + +// The unit's own stderr precedes the launcher's summary under --pipe, so a command that prints a +// "Memory peak:" line of its own must not be read as the unit's peak. The summary text is the v255 +// run.c ordering: result, main process status, runtime, CPU, memory peak. +data summary_oom_after_spoof: String = "compiling...\nMemory peak: 1K\nFinished with result: success\nerror: killed\nFinished with result: oom-kill\nMain processes terminated with: code=killed/status=KILL\nService runtime: 10min 31.402s\nCPU time consumed: 1h 2min 3.100s\nMemory peak: 9.7G\nMemory swap peak: 0B\n" + +// 9.7G: whole 9, tenth 7, factor 1024^3 -- the sound upper bound is (97 + 2) * 1024^3 / 10 rounded +// up, because the printed tenth is truncated and computed through a second truncation. +data nine_point_seven_gib_upper_bytes: Int = 10630044058 +data nine_point_seven_gib_lower_bytes: Int = 10415295692 + +test fn the_units_peak_and_result_are_the_summarys_last_lines_not_the_commands() -> Bool { + let s = systemd_run_wait_summary(stderr: summary_oom_after_spoof) + compute_summary_is_oom_kill(summary: s) + && match s.memory_peak { + MemoryPeakRead { peak: b } => + match b { + FormattedBytesBounded { printed, lower, upper } => + printed == "9.7G" + && byte_size_count(b: lower) == nine_point_seven_gib_lower_bytes + && byte_size_count(b: upper) == nine_point_seven_gib_upper_bytes + FormattedBytesExact { bytes: _ } => false + } + MemoryPeakUnparseable { text: _ } => false + MemoryPeakNotReported => false + } +} + +// A MISSING PEAK IS NEVER A ZERO PEAK. A manager below v254 omits the line; so does a launcher run +// with --quiet. Either must read NotReported, because a recorded zero would size a class at its +// headroom alone. An exact small figure keeps its exact reading. +test fn an_absent_peak_is_not_reported_and_a_byte_figure_is_exact() -> Bool { + let absent = systemd_run_wait_summary(stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") + let small = systemd_run_wait_summary(stderr: "Finished with result: success\nMemory peak: 512B\n") + let garbled = systemd_run_wait_summary(stderr: "Finished with result: frobnicated\nMemory peak: lots\n") + match absent.memory_peak { MemoryPeakNotReported => true _ => false } + && !compute_summary_is_oom_kill(summary: absent) + && match small.memory_peak { + MemoryPeakRead { peak: b } => match b { FormattedBytesExact { bytes } => byte_size_count(b: bytes) == 512 _ => false } + _ => false + } + && match garbled.result { ServiceResultUnrecognized { wire } => wire == "frobnicated" _ => false } + && match garbled.memory_peak { MemoryPeakUnparseable { text } => text == "lots" _ => false } + && match systemd_run_wait_summary(stderr: "").result { ServiceResultNotReported => true _ => false } +} + +// ── THE RESOLUTION: rows to a grant ────────────────────────────────────────────────────────── + +data gib: Int = 1048576 + +fn ceiling() -> Kibibyte { kibibyte(count: 26 * gib) } + +fn sample(peak_gib: Int) -> WorkClassPeakSample { + WorkClassPeakSample { + work_identity: "0000000000000001" as NonEmptyStr, + operation: "compile:src/v2/compiler/compile.dag" as NonEmptyStr, + host: "fixture-host" as NonEmptyStr, + observed_at: 1790000000, + peak_upper: kibibyte(count: peak_gib * gib), + } +} + +fn row(class: WorkClass, samples: List) -> WorkClassGrantRow { + WorkClassGrantRow { class: class, samples: samples, headroom: kibibyte(count: 2 * gib), headroom_authority: "fixture policy budget" as NonEmptyStr } +} + +// EVERY ARM, from supplied rows. The sized arm is peak (the LARGEST sample, not the last or first) +// plus headroom; one sample is not a peak; a sized figure at or above the ceiling is the ceiling; +// two rows for one class decide nothing; a class with no row is the ceiling. +test fn a_grant_is_the_largest_sample_plus_headroom_and_every_shortfall_is_the_ceiling() -> Bool { + let sized = work_class_grant(class: WorkClassCompile, rows: [row(class: WorkClassCompile, samples: [sample(peak_gib: 4), sample(peak_gib: 9), sample(peak_gib: 6)])], ceiling: ceiling()) + let one = work_class_grant(class: WorkClassCompile, rows: [row(class: WorkClassCompile, samples: [sample(peak_gib: 4)])], ceiling: ceiling()) + let over = work_class_grant(class: WorkClassCompile, rows: [row(class: WorkClassCompile, samples: [sample(peak_gib: 24), sample(peak_gib: 20)])], ceiling: ceiling()) + let twice = work_class_grant(class: WorkClassCompile, rows: [row(class: WorkClassCompile, samples: [sample(peak_gib: 4), sample(peak_gib: 5)]), row(class: WorkClassCompile, samples: [sample(peak_gib: 4), sample(peak_gib: 5)])], ceiling: ceiling()) + let other = work_class_grant(class: WorkClassClaims, rows: [row(class: WorkClassCompile, samples: [sample(peak_gib: 4), sample(peak_gib: 5)])], ceiling: ceiling()) + match sized { GrantSized { class: _, amount, samples, peak_upper: _, headroom: _, headroom_authority: _ } => kibibyte_count(k: amount) == 11 * gib && samples == 3 _ => false } + && match one { GrantAtCeiling { class: _, amount, reason } => kibibyte_count(k: amount) == 26 * gib && match reason { ClassTooFewSamples { samples } => samples == 1 _ => false } _ => false } + && match over { GrantAtCeiling { class: _, amount, reason } => kibibyte_count(k: amount) == 26 * gib && match reason { ClassSizedAtOrAboveCeiling { samples: _, sized: s } => kibibyte_count(k: s) == 26 * gib _ => false } _ => false } + && match twice { GrantAtCeiling { class: _, amount: _, reason } => match reason { ClassRowsAmbiguous { rows } => rows == 2 _ => false } _ => false } + && match other { GrantAtCeiling { class: _, amount: _, reason } => match reason { ClassUnmeasured => true _ => false } _ => false } +} + +// THE INHABITANCE CLAIM OVER THE REAL ROWS, and it is expected to FLIP (DESIGN 4b(4)). This change +// ships no rows, so every class the provider can be asked for resolves to the ceiling and no +// reserved amount moves. When the first row lands from provider-route samples this claim reds for +// that class and becomes the regression control that production rows reach the sized arm. +test fn with_the_shipped_rows_every_class_reserves_the_ceiling() -> Bool { + let ops = [ + BuildGunbcBinaries { profile: CargoRelease, bins: ["gunbc" as NonEmptyStr] }, + BuildGunbcBinaries { profile: CargoDebug, bins: ["gunbc" as NonEmptyStr] }, + CompileEntry { entry: "src/v2/compiler/compile.dag" }, + RunClaims { entry: "dag/test/claim/x_test.dag", functions: "f" as NonEmptyStr, hermetic: true }, + ] + all(ops, op => match work_class_grant(class: work_class_of(op: op), rows: work_class_grant_rows, ceiling: ceiling()) { + GrantAtCeiling { class: _, amount, reason } => kibibyte_count(k: amount) == 26 * gib && match reason { ClassUnmeasured => true _ => false } + GrantSized { class: _, amount: _, samples: _, peak_upper: _, headroom: _, headroom_authority: _ } => false + }) +} + +// ── WHICH ATTEMPT IS RETRIED ───────────────────────────────────────────────────────────────── + +// Only an oom-kill under a grant BELOW the ceiling retries; the same kill at the ceiling ends as the +// typed refusal, and a grant below the ceiling that ended any other way does not retry. +test fn only_an_oom_kill_below_the_ceiling_retries_at_the_ceiling() -> Bool { + let below = GrantSized { class: WorkClassCompile, amount: kibibyte(count: 11 * gib), samples: 3, peak_upper: kibibyte(count: 9 * gib), headroom: kibibyte(count: 2 * gib), headroom_authority: "fixture" as NonEmptyStr } + let at = GrantAtCeiling { class: WorkClassCompile, amount: ceiling(), reason: ClassUnmeasured } + let oom = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(stderr: summary_oom_after_spoof) } + let clean = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(stderr: "Finished with result: exit-code\nMemory peak: 3.1G\n") } + compute_attempt_retries_at_ceiling(grant: below, attempt: oom) + && !compute_attempt_retries_at_ceiling(grant: at, attempt: oom) + && !compute_attempt_retries_at_ceiling(grant: below, attempt: clean) +} From f05497816d674e841b990337514a975267473d12 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Mon, 21 Sep 2026 23:58:09 +0000 Subject: [PATCH 17/28] Record the peak-route decision: display contract chosen on the tradeoff, fail-closed arm linked Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemd_run.dag | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index c05192539d5..bcd98b66ada 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -213,6 +213,36 @@ service systemd.SystemdRun { // (extdeps.systemd.systemctl systemd_memory_peak_minimum_version), and an older manager simply // omits the line -- so ABSENCE IS A READING, and it is never a peak of zero. // +// THE CONTRACT IS A DISPLAY FORMAT, CHOSEN ON THE TRADEOFF RATHER THAN INHERITED FROM --collect +// (fork put by loyal-swift-608, 2026-09-21). Three routes read a unit's peak, and this one is taken +// knowing it is the weakest KIND of contract -- a human-readable line, verified against run.c and +// format-util.c at systemd v255 (srv1's recorded version), which upstream is free to reword. +// +// DROPPING --collect DOES NOT KEEP THE UNIT. systemd.unit(5) CollectMode= (v255): the default, +// `inactive`, unloads a unit "in the inactive state and not referenced" and keeps only FAILED +// units. So a typed `systemctl show -p MemoryPeak` after the launcher returns would answer for +// failed units only -- the samples a grant is sized from are overwhelmingly successful runs, so +// this route loses the population that matters, and it adds a teardown (reset-failed) this +// provider would own and could leak. +// +// READING memory.peak FROM THE CGROUP means a process inside the unit at stop time, i.e. an +// ExecStopPost carrying shell that writes a file -- raw shell implementing semantics beside the +// modeled invocation (DESIGN 6), plus a second file protocol to parse. The number would be exact, +// but the route is more authored machinery for a fact the manager already reads over D-Bus. +// +// THIS ROUTE: systemd-run itself reads Result and MemoryPeak as TYPED D-Bus properties before the +// unit is collected, and prints them. What is display-shaped is only the last hop. +// +// WHAT MAKES THE DISPLAY CONTRACT ADMISSIBLE IS ITS FAILURE ARM, and the link is load-bearing: a +// reworded or absent line reads ServiceResultNotReported / MemoryPeakNotReported / +// MemoryPeakUnparseable -- never a peak of zero and never a default -- so a format change stops the +// class from ever gaining samples and it stays at the ceiling (gunbc.compute.work_class_grant), +// which is fail-closed. What it CANNOT do is detect a reworded result line that still parses as a +// different valid word; the result vocabulary is systemd.exec(5)'s closed $SERVICE_RESULT set, so +// that would be an upstream semantic change, not a formatting one. Retirement trigger for this +// choice: an upstream structured output for the --wait summary (e.g. a JSON mode), which this module +// would read instead. +// // WHY THE LAST LINE AND NOT THE FIRST. Under --pipe the unit's own stderr and the launcher's summary // arrive on one stream. The summary is written after the unit has ended and the pipe has drained, // so it is the TAIL; a command that happens to print "Memory peak: 1K" on its own stderr prints it From 4659691d250cd9c417cbac68611f800b4801055d Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 02:15:40 +0000 Subject: [PATCH 18/28] Review 69829: one authority for the exceedance path, class-grain zero, typed peak sample at its upper bound Co-Authored-By: Claude Opus 5 (1M context) --- dag/gunbc/compute/work_class_grant.dag | 58 ++++++++++++- dag/gunbc/compute/work_provider_local.dag | 84 +++++++++++++++---- dag/gunbc/compute/work_request.dag | 6 +- .../compute/work_class_grant_witness_test.dag | 49 ++++++++++- 4 files changed, 173 insertions(+), 24 deletions(-) diff --git a/dag/gunbc/compute/work_class_grant.dag b/dag/gunbc/compute/work_class_grant.dag index 9fbb0f76d48..253939b982a 100644 --- a/dag/gunbc/compute/work_class_grant.dag +++ b/dag/gunbc/compute/work_class_grant.dag @@ -2,8 +2,11 @@ module gunbc.compute.work_class_grant import std.types { String, Bool, Int, List, NonEmptyStr, EpochSecs } import std.nat { Nat } -import std.measure { Kibibyte, kibibyte, kibibyte_count } -import gunbc.compute.work_request { WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoProfile, cargo_profile_wire } +import std.measure { Kibibyte, kibibyte, kibibyte_count, byte_size_count } +import gunbc.compute.work_request { WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoProfile, cargo_profile_wire, work_operation_canonical } +import extdeps.systemd.systemd_run { + SystemdRunWaitSummary, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, systemd_formatted_bytes_upper, +} // A WORK REQUEST RESERVES AN AMOUNT PER CLASS, AND THE AMOUNT IS A GRANT WITH ITS OBSERVATIONS // ATTACHED (operator ruling 2026-09-21: do not raise a global ceiling; size a job's reservation per @@ -61,6 +64,57 @@ type WorkClassPeakSample { peak_upper: Kibibyte } +// THE SAMPLE PRODUCER: the launcher's summary to a typed sample (review 69829 -- the upper bound was +// computed and then written only as prose). The byte upper bound is rounded UP to kibibytes, the +// direction a capacity reading may round. A summary with no peak, an unparseable peak or an +// unreadable clock takes NO sample and says which; it never takes a zero. +fn work_class_summary_peak_upper(summary: SystemdRunWaitSummary) -> Kibibyte? { + match summary.memory_peak { + MemoryPeakRead { peak: b } => Present { value: kibibyte(count: (byte_size_count(b: systemd_formatted_bytes_upper(b: b)) + 1023) / 1024) } + MemoryPeakUnparseable { text: _ } => none + MemoryPeakNotReported => none + } +} + +type WorkClassPeakSampleReading + = PeakSampleTaken { class: WorkClass, sample: WorkClassPeakSample } + | PeakSampleNotTaken { reason: String } + +fn work_class_peak_sample(summary: SystemdRunWaitSummary, work_identity: String, op: WorkOperation, host: String, observed_at: EpochSecs?) -> WorkClassPeakSampleReading { + match work_class_summary_peak_upper(summary: summary) { + Absent => PeakSampleNotTaken { reason: match summary.memory_peak { + MemoryPeakUnparseable { text } => join(["the Memory peak line did not parse: ", text], "") + _ => "no Memory peak line was reported" + } } + Present { value: upper } => + match observed_at { + Absent => PeakSampleNotTaken { reason: "the clock could not be read as epoch seconds" } + Present { value: at } => + if work_identity == "" || host == "" { + PeakSampleNotTaken { reason: "the attempt carries no work identity or host" } + } else { + PeakSampleTaken { class: work_class_of(op: op), sample: WorkClassPeakSample { + work_identity: work_identity as NonEmptyStr, + operation: work_operation_canonical(op: op) as NonEmptyStr, + host: host as NonEmptyStr, + observed_at: at, + peak_upper: upper, + } } + } + } + } +} + +fn work_class_peak_sample_wire(r: WorkClassPeakSampleReading) -> String { + match r { + PeakSampleTaken { class, sample } => + join(["class=", work_class_wire(c: class), " identity=", sample.work_identity as String, " host=", sample.host as String, + " observed_at=", to_string(value: sample.observed_at), " peak_upper_kib=", to_string(value: kibibyte_count(k: sample.peak_upper)), + " operation=", sample.operation as String], "") + PeakSampleNotTaken { reason } => join(["NOT TAKEN (", reason, ")"], "") + } +} + // THE ROW: a policy budget grounded in observations. The headroom is the OPERATOR'S figure and is // cited as such; what makes it admissible (DESIGN 5) is that it is FALSIFIABLE -- a non-zero // GrantExceeded count for this class (compute_grant_exceeded_count) means the grant is wrong and diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 098d5a9fba3..e7b2e9190b6 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -4,7 +4,7 @@ import std.types { String, Bool, Int, List, NonEmptyStr, FilePath, GitRef } import std.process { ProcessExit, ExitSuccess, ExitFailure } import std.algebra { trim } import std.measure { byte_size_count } -import extdeps.filesystem.filesystem_io { Filesystem } +import extdeps.filesystem.filesystem_io { Filesystem, filesystem_listing_names_entry } import gunbc.output_policy { OutcomeIsData } import extdeps.gunbc import extdeps.shell @@ -24,6 +24,7 @@ import extdeps.linux.cgroup_v2 { } import extdeps.cache.sccache { sccache_installed_binary_path } import gunbc.clock_read { clock_now_probed_at_or_unknown } +import gunbc.fabric_event_log_host { now_epoch_seconds } import gunbc.roadmap_dashboard_instance { HostDashboardInstance, dashboard_instance_for_repo_root, DashboardInstanceResolved, DashboardInstanceRootUnknown, dashboard_instance_compute_root, @@ -36,6 +37,7 @@ import std.measure { Kibibyte, kibibyte_count, kibibyte_to_byte_size, kibibyte, import gunbc.compute.work_class_grant { WorkClass, work_class_of, work_class_wire, WorkClassGrant, GrantAtCeiling, GrantSized, ClassRetryAfterGrantExceeded, work_class_grant, work_class_grant_rows, work_class_grant_amount, work_class_grant_is_ceiling, work_class_grant_wire, + work_class_peak_sample, work_class_peak_sample_wire, work_class_summary_peak_upper, } import gunbc.compute.host_capacity { ComputeReservation, ComputeReserved, ComputeCapacityFull, ComputeHostBelowFloor, ComputeReservationRefused, @@ -50,7 +52,7 @@ import gunbc.compute.work_request { WorkInfrastructureRefusal, ProducerInFlight, HostComputeCapacityFull, HostBelowMemoryFloor, ComputeCapacityLedgerUnavailable, CheckoutUnavailable, StoreUnavailable, ToolchainUnobserved, DependencyNotSucceeded, SubjectUnresolved, UnitMemoryGrantExceeded, - work_outcome_wire, work_outcome_ending, work_operation_canonical, StoredOutcomeRead, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_decode, stored_outcome_is_success, + work_outcome_wire, work_outcome_ending, work_operation_canonical, work_outcome_identity, StoredOutcomeRead, StoredOutcomeFound, StoredOutcomeUnreadable, stored_outcome_decode, stored_outcome_is_success, } // THE LOCAL PROVIDER: one of N behind the exact work contract, serving the host it runs on. It is a @@ -629,18 +631,27 @@ fn compute_attempt_completed(outcome: WorkOutcome) -> Bool { // attempt ends as the typed UnitMemoryGrantExceeded refusal. And a retry is taken only after the // first seat is PROVEN released, by the same termination fact settlement uses; a seat that cannot be // proven free is settled as it stands, so the retry can never run beside the attempt it replaces. -fn compute_grant_exceeded_dir(layout: ComputeLayout, class: WorkClass) -> String { - join([layout.root, "/grant-exceeded/", work_class_wire(c: class)], "") +// ONE AUTHORITY FOR WHERE EXCEEDANCES LIVE. Both the writer and the reader call these two, keyed by +// the compute root, so the reader cannot list a path nothing writes (review 69829: the reader had +// re-spelled the join, and a drift would have read as a silent zero). +data compute_grant_exceeded_dirname: String = "grant-exceeded" + +fn compute_grant_exceeded_parent(root: String) -> String { + join([root, "/", compute_grant_exceeded_dirname], "") +} + +fn compute_grant_exceeded_dir(root: String, class: WorkClass) -> String { + join([compute_grant_exceeded_parent(root: root), "/", work_class_wire(c: class)], "") } fn compute_grant_exceeded_record(instance: HostDashboardInstance, layout: ComputeLayout, op: WorkOperation, grant: WorkClassGrant, attempt: ComputeAttempt, reference: NonEmptyStr) -> Bool { - let dir = compute_grant_exceeded_dir(layout: layout, class: work_class_of(op: op)) + let dir = compute_grant_exceeded_dir(root: layout.root, class: work_class_of(op: op)) if !compute_ensure_dir(instance: instance, path: dir) { false } else { Filesystem.WriteCreateNew( path: join([dir, "/", reference as String], ""), - content: compute_peak_sample_wire(op: op, grant: grant, summary: attempt.summary), + content: compute_peak_sample_wire(instance: instance, identity: work_outcome_identity(o: attempt.outcome), op: op, grant: grant, summary: attempt.summary), ).success } } @@ -648,23 +659,59 @@ fn compute_grant_exceeded_record(instance: HostDashboardInstance, layout: Comput // THE READABLE COUNT: the number of recorded exceedances for a class on this host. An unreadable // directory is reported as unreadable and never as zero -- a count that reads 0 when it could not // look would be the one falsifier of the budget silently reporting the budget sound. +// +// ZERO IS ESTABLISHED BY THE LEVEL ABOVE, AT CLASS GRAIN (review 69829). A class directory exists +// only once that class has exceeded, and Filesystem.List cannot tell an absent path from an +// unreadable one, so the class's absence is read positively from its parent's listing: parent listed +// and the class not in it is a counted zero; parent listed WITH the class in it but the class +// unlistable is unreadable. The same move one level up distinguishes a host that has never recorded +// any exceedance from a host whose record cannot be read. type GrantExceededCount = GrantExceededCounted { count: Int } | GrantExceededNeverRecorded | GrantExceededUnreadable { detail: String } -fn compute_grant_exceeded_count(instance: HostDashboardInstance, class: WorkClass) -> GrantExceededCount { - let dir = join([dashboard_instance_compute_root(instance: instance) as String, "/grant-exceeded/", work_class_wire(c: class)], "") - let listed = Filesystem.List(path: dir) - if listed.success { - GrantExceededCounted { count: length(filter(split(listed.entries, "\n"), n => trim(s: n) != "")) } - } else if !Filesystem.List(path: join([dashboard_instance_compute_root(instance: instance) as String, "/grant-exceeded"], "")).success { +type ComputeListing { + listed: Bool + entries: String + error: String +} + +fn compute_listing_count(entries: String) -> Int { + length(filter(split(entries, "\n"), n => trim(s: n) != "")) +} + +fn grant_exceeded_count_decide(root: ComputeListing, parent: ComputeListing, class_dir: ComputeListing, class: WorkClass) -> GrantExceededCount { + if class_dir.listed { + GrantExceededCounted { count: compute_listing_count(entries: class_dir.entries) } + } else if parent.listed { + if filesystem_listing_names_entry(listing: parent.entries, name: work_class_wire(c: class)) { + GrantExceededUnreadable { detail: class_dir.error } + } else { + GrantExceededCounted { count: 0 } + } + } else if root.listed && !filesystem_listing_names_entry(listing: root.entries, name: compute_grant_exceeded_dirname) { GrantExceededNeverRecorded } else { - GrantExceededUnreadable { detail: listed.error } + GrantExceededUnreadable { detail: if root.listed { parent.error } else { root.error } } } } +fn compute_list(path: String) -> ComputeListing { + let l = Filesystem.List(path: path) + ComputeListing { listed: l.success, entries: l.entries, error: l.error } +} + +fn compute_grant_exceeded_count(instance: HostDashboardInstance, class: WorkClass) -> GrantExceededCount { + let root = dashboard_instance_compute_root(instance: instance) as String + grant_exceeded_count_decide( + root: compute_list(path: root), + parent: compute_list(path: compute_grant_exceeded_parent(root: root)), + class_dir: compute_list(path: compute_grant_exceeded_dir(root: root, class: class)), + class: class, + ) +} + fn compute_grant_exceeded_count_wire(instance: HostDashboardInstance, class: WorkClass) -> String { match compute_grant_exceeded_count(instance: instance, class: class) { GrantExceededCounted { count } => to_string(value: count) @@ -757,7 +804,7 @@ fn compute_run_and_settle( path: layout.consumption_path, content: join([ compute_capacity_standing_wire(s: standing), "\n", - compute_peak_sample_wire(op: op, grant: grant, summary: attempt.summary), + compute_peak_sample_wire(instance: instance, identity: identity, op: op, grant: grant, summary: attempt.summary), "grant exceeded, this class: ", compute_grant_exceeded_count_wire(instance: instance, class: work_class_of(op: op)), "\n", "unit: ", unit_termination_wire(t: termination), "\n", "reservation: ", compute_release_wire(r: returned), "\n", @@ -1076,13 +1123,14 @@ fn compute_summary_result_wire(summary: SystemdRunWaitSummary) -> String { // size its class at headroom alone, which is the worst failure this mechanism can have. The sample // line carries the class, the operation's canonical text and the unparsed reading, so a later grant // row can cite exactly which runs it was sized from. -fn compute_peak_sample_wire(op: WorkOperation, grant: WorkClassGrant, summary: SystemdRunWaitSummary) -> String { +fn compute_peak_sample_wire(instance: HostDashboardInstance, identity: String, op: WorkOperation, grant: WorkClassGrant, summary: SystemdRunWaitSummary) -> String { join([ "class: ", work_class_wire(c: work_class_of(op: op)), "\n", - "operation: ", work_operation_canonical(op: op), "\n", "grant: ", work_class_grant_wire(g: grant), "\n", "unit result: ", compute_summary_result_wire(summary: summary), "\n", - "unit peak: ", compute_summary_peak_wire(summary: summary), "\n", + "unit peak (as printed): ", compute_summary_peak_wire(summary: summary), "\n", + "sample: ", work_class_peak_sample_wire(r: work_class_peak_sample( + summary: summary, work_identity: identity, op: op, host: instance.host_identity as String, observed_at: now_epoch_seconds())), "\n", ], "") } @@ -1127,7 +1175,7 @@ fn compute_run_leased( ComputeExecFailed { exit_code, stdout: _, stderr: _ } => if compute_summary_is_oom_kill(summary: summary) { WorkRefusedByInfrastructure { identity: identity, cause: UnitMemoryGrantExceeded { - class: work_class_wire(c: work_class_of(op: op)), granted: granted, peak: compute_summary_peak_wire(summary: summary), grant_was_ceiling: grant_was_ceiling, + class: work_class_wire(c: work_class_of(op: op)), granted: granted, peak_upper: work_class_summary_peak_upper(summary: summary), grant_was_ceiling: grant_was_ceiling, } } } else { WorkFailed { identity: identity, exit_code: exit_code, log_path: layout.log_path } diff --git a/dag/gunbc/compute/work_request.dag b/dag/gunbc/compute/work_request.dag index 63c0c0a89eb..c5d8bae7583 100644 --- a/dag/gunbc/compute/work_request.dag +++ b/dag/gunbc/compute/work_request.dag @@ -170,7 +170,7 @@ type WorkInfrastructureRefusal | ToolchainUnobserved { detail: String } | DependencyNotSucceeded { dependency_identity: String, ending: String } | SubjectUnresolved { detail: String } - | UnitMemoryGrantExceeded { class: String, granted: Kibibyte, peak: String, grant_was_ceiling: Bool } + | UnitMemoryGrantExceeded { class: String, granted: Kibibyte, peak_upper: Kibibyte?, grant_was_ceiling: Bool } fn work_outcome_identity(o: WorkOutcome) -> String { match o { @@ -203,9 +203,9 @@ fn work_refusal_detail(r: WorkInfrastructureRefusal) -> String { ToolchainUnobserved { detail } => join(["toolchain unobserved: ", detail], "") DependencyNotSucceeded { dependency_identity, ending } => join(["dependency ", dependency_identity, " ended ", ending], "") SubjectUnresolved { detail } => join(["subject unresolved: ", detail], "") - UnitMemoryGrantExceeded { class, granted, peak, grant_was_ceiling } => + UnitMemoryGrantExceeded { class, granted, peak_upper, grant_was_ceiling } => join(["the unit was OOM-killed at its memory grant: class ", class, " was granted ", to_string(value: kibibyte_count(k: granted)), - " KiB and its manager reported a peak of ", peak, + " KiB and its manager reported a peak of ", match peak_upper { Present { value: k } => join(["at most ", to_string(value: kibibyte_count(k: k)), " KiB"], "") Absent => "NOTHING (no Memory peak line was reported)" }, if grant_was_ceiling { "; the grant was this host's ceiling, so no larger reservation exists to retry under" } else { "; the grant was below the ceiling and the retry at the ceiling was not taken -- the consumption record says why" }], "") } } diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag index 0c2d285cd1c..adb877e46a4 100644 --- a/dag/test/claim/compute/work_class_grant_witness_test.dag +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -16,8 +16,12 @@ import gunbc.compute.work_class_grant { WorkClassGrant, GrantAtCeiling, GrantSized, WorkClassCeilingReason, ClassUnmeasured, ClassTooFewSamples, ClassRowsAmbiguous, ClassSizedAtOrAboveCeiling, ClassRetryAfterGrantExceeded, work_class_grant, work_class_grant_amount, + WorkClassPeakSampleReading, PeakSampleTaken, PeakSampleNotTaken, work_class_peak_sample, +} +import gunbc.compute.work_provider_local { + ComputeAttempt, compute_attempt_retries_at_ceiling, compute_summary_is_oom_kill, + ComputeListing, GrantExceededCount, GrantExceededCounted, GrantExceededNeverRecorded, GrantExceededUnreadable, grant_exceeded_count_decide, } -import gunbc.compute.work_provider_local { ComputeAttempt, compute_attempt_retries_at_ceiling, compute_summary_is_oom_kill } // ── THE PRODUCER: what systemd-run --wait said about the unit ───────────────────────────────── @@ -132,3 +136,46 @@ test fn only_an_oom_kill_below_the_ceiling_retries_at_the_ceiling() -> Bool { && !compute_attempt_retries_at_ceiling(grant: at, attempt: oom) && !compute_attempt_retries_at_ceiling(grant: below, attempt: clean) } + +// ── THE TYPED SAMPLE ───────────────────────────────────────────────────────────────────────── + +// The summary becomes a WorkClassPeakSample carrying the UPPER bound in KiB, rounded up -- not the +// printed figure, and not prose. 9.7G's byte upper bound 10630044058 is 10380902.4 KiB, so 10380903. +// No peak line, or no clock, takes no sample. +test fn a_summary_becomes_a_typed_sample_at_its_upper_bound_or_no_sample() -> Bool { + let op = CompileEntry { entry: "src/v2/compiler/compile.dag" } + let s = systemd_run_wait_summary(stderr: summary_oom_after_spoof) + let taken = work_class_peak_sample(summary: s, work_identity: "abc", op: op, host: "srv1", observed_at: Present { value: 1790000000 }) + let no_peak = work_class_peak_sample(summary: systemd_run_wait_summary(stderr: "Finished with result: success\n"), work_identity: "abc", op: op, host: "srv1", observed_at: Present { value: 1790000000 }) + let no_clock = work_class_peak_sample(summary: s, work_identity: "abc", op: op, host: "srv1", observed_at: none) + match taken { + PeakSampleTaken { class, sample } => + work_class_wire(c: class) == "compile" && kibibyte_count(k: sample.peak_upper) == 10380903 + && (sample.operation as String) == "compile:src/v2/compiler/compile.dag" && (sample.host as String) == "srv1" + PeakSampleNotTaken { reason: _ } => false + } + && match no_peak { PeakSampleNotTaken { reason: _ } => true _ => false } + && match no_clock { PeakSampleNotTaken { reason: _ } => true _ => false } +} + +// ── THE COUNTER ────────────────────────────────────────────────────────────────────────────── + +fn listed(entries: String) -> ComputeListing { ComputeListing { listed: true, entries: entries, error: "" } } +fn unlisted(error: String) -> ComputeListing { ComputeListing { listed: false, entries: "", error: error } } + +// Zero is established by the level above, at class grain: a class absent from a listed parent is a +// counted zero even when another class has exceedances; a class present in the parent but unlistable +// is unreadable; a root listing without the parent is a host that never recorded; an unlistable root +// is unreadable. None of the failure arms reads as zero. +test fn the_exceedance_count_is_zero_only_when_absence_is_positively_read() -> Bool { + let counted = grant_exceeded_count_decide(root: listed(entries: "grant-exceeded\nwork"), parent: listed(entries: "claims"), class_dir: listed(entries: "r1\nr2\n"), class: WorkClassClaims) + let healthy = grant_exceeded_count_decide(root: listed(entries: "grant-exceeded\nwork"), parent: listed(entries: "claims"), class_dir: unlisted(error: "ENOENT"), class: WorkClassCompile) + let broken = grant_exceeded_count_decide(root: listed(entries: "grant-exceeded\nwork"), parent: listed(entries: "compile"), class_dir: unlisted(error: "EACCES"), class: WorkClassCompile) + let never = grant_exceeded_count_decide(root: listed(entries: "work\nleases"), parent: unlisted(error: "ENOENT"), class_dir: unlisted(error: "ENOENT"), class: WorkClassCompile) + let blind = grant_exceeded_count_decide(root: unlisted(error: "EIO"), parent: unlisted(error: "EIO"), class_dir: unlisted(error: "EIO"), class: WorkClassCompile) + match counted { GrantExceededCounted { count } => count == 2 _ => false } + && match healthy { GrantExceededCounted { count } => count == 0 _ => false } + && match broken { GrantExceededUnreadable { detail } => detail == "EACCES" _ => false } + && match never { GrantExceededNeverRecorded => true _ => false } + && match blind { GrantExceededUnreadable { detail: _ } => true _ => false } +} From 6a53f39f921a959418d328519cecb8e5462d2f24 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 02:43:11 +0000 Subject: [PATCH 19/28] Bind the peak reader to an invocation that can print a peak, and name who owes the truncated read Two responses to loyal-swift-608 (2026-09-22). THE --quiet TRAP IS NOW UNWRITABLE RATHER THAN MERELY UNUSED. systemd-run prints the result and peak lines only under `if (arg_wait && !arg_quiet)` (run.c, v255), so anything routed through systemd.SystemdRun RunTransientAndWait -- which carries --quiet -- reads MemoryPeakNotReported forever: a dead producer indistinguishable from a healthy launcher whose manager is below v254, reachable by a one-word difference between two operations in one file. The provider did not use it, and that is exactly the protection that decays. So systemd_run_wait_summary no longer takes a loose String. It takes a SystemdSummaryPrintingWait, minted in exactly one place -- systemd_run_user_wait_arguments, the only wait builder here that omits --quiet -- so a launcher that cannot print a peak cannot be handed to the thing whose contract is to observe one. RunTransientAndWait keeps --quiet, with the reason stated: its callers want the child's stdio unpolluted, which is a real and different need; the fix is that the two cannot be confused, not that one loses its flag. Two consequences, both improvements. The provider reads the summary inside compute_run_unit, where the invocation is in scope (ComputeUnitRun { exec, summary }), so no site further out holds a bare stderr String a peak could be read from. And every fixture claim now names the invocation it is pretending came from. WHO OWES THE TRUNCATED READ, since the parser may not guard it. A truncation under-reports -- "11G" cut to "1G" parses cleanly as a tenth -- and under-reporting is the direction that OOMs. No parser guard can catch it: gunbc.machine_intake_host_capture_envelope HostCaptureText settled that line-wholeness is the CALLER's fact, never inferred from bytes, and this stderr is a completed transcript the seed's shell observation trims. The reader is gunbc.WitnessBin.Run and it CAN return a prefix: stderr_capture defaults to BoundedTail { bytes: 16384 }. What closes the hazard is the direction of that policy -- a bounded TAIL discards the oldest bytes and the summary is the youngest, so a cut can only slice the first retained line, which is the unit's own output. Stated as a named dependency on that operation's contract, not as a check this module performs. ABSENCE CLAIMS. NotReported is reachable by three absence shapes, all constructible without a host: empty stderr, a failed unit whose stderr carries only its own output, and the pre-v254 omission. None reads as a peak of zero, and an unterminated summary still READS -- pinned so no one adds the termination guard that would drop every real run. the_peak_is_unreported_by_absence_and_an_untrimmed_summary_still_reads returned true; whole closure typechecked with no other error. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemd_run.dag | 35 +++++++- dag/gunbc/compute/work_provider_local.dag | 79 ++++++++++++++----- .../compute/work_class_grant_witness_test.dag | 70 +++++++++++++--- 3 files changed, 153 insertions(+), 31 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index bcd98b66ada..e327fee0151 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -98,13 +98,30 @@ fn systemd_run_user_transient_arguments( // stdio (`--wait --pipe`), so a build or a check runs under the unit's resource bounds while the // process that asked for it observes the result directly. `--collect` unloads the unit afterwards // whatever its result, so a failed compute unit does not accumulate as a failed unit to reap. +// WHETHER THERE IS A SUMMARY TO READ IS A FACT ABOUT THE INVOCATION, SO THE INVOCATION CARRIES IT. +// systemd-run prints the result and peak lines only under `if (arg_wait && !arg_quiet)` (run.c, +// v255), so --quiet suppresses the summary ENTIRELY: a reader handed a quiet launcher's stderr +// finds no peak, reports MemoryPeakNotReported forever, and is indistinguishable from a healthy +// launcher whose manager is below v254. A DEAD PRODUCER THAT LOOKS LIKE A HEALTHY ONE WITH NOTHING +// TO REPORT is the failure this file is one word away from: RunTransientAndWait and this builder +// differ by exactly "--quiet", and the next author adds a third by copying whichever they reached +// first. So the summary reader does not take a loose String. It takes this token, which is minted +// in exactly one place -- the builder below, the only wait invocation in this module that omits +// --quiet -- and a launcher that cannot print a peak therefore cannot be handed to the thing whose +// contract is to observe one. +type SystemdSummaryPrintingWait { argv: List } + +fn systemd_summary_printing_wait_argv(w: SystemdSummaryPrintingWait) -> List { + w.argv +} + fn systemd_run_user_wait_arguments( unit: NonEmptyStr, properties: List, setenv_bindings: List, command_argv: List, -) -> List { - concat(["--wait", "--pipe", "--collect"], systemd_run_user_transient_arguments(unit: unit, properties: properties, setenv_bindings: setenv_bindings, command_argv: command_argv)) +) -> SystemdSummaryPrintingWait { + SystemdSummaryPrintingWait { argv: concat(["--wait", "--pipe", "--collect"], systemd_run_user_transient_arguments(unit: unit, properties: properties, setenv_bindings: setenv_bindings, command_argv: command_argv)) } } fn systemd_run_property(name: String, value: String) -> String { @@ -185,6 +202,14 @@ service systemd.SystemdRun { } } + // THIS OPERATION CANNOT SERVE PEAK OBSERVATION, AND THE REASON IS THE --quiet BELOW. run.c at v255 + // prints "Finished with result:" and "Memory peak:" only under `if (arg_wait && !arg_quiet)`, so + // this invocation's stderr never carries the summary and a peak read from it is NotReported for + // every run. That is why systemd_run_wait_summary takes a SystemdSummaryPrintingWait token this + // operation cannot produce: a caller needing the unit's peak takes + // systemd_run_user_wait_arguments instead. --quiet stays here because this operation's callers + // want the child's own stdio unpolluted by the launcher's chatter, which is a real and different + // need -- the fix is that the two cannot be confused, not that one of them loses its flag. operation RunTransientAndWait { input { unit: NonEmptyStr, property_argv: List, command_argv: List } output { @@ -400,7 +425,11 @@ fn systemd_run_summary_last_value(stderr: String, prefix: String) -> String? { } else { acc }) } -fn systemd_run_wait_summary(stderr: String) -> SystemdRunWaitSummary { +// THE READER TAKES THE LAUNCHER, NOT JUST ITS OUTPUT. The launcher argument is not inspected and is +// not meant to be: it is the evidence that this stderr came from an invocation that prints a +// summary at all, and its only inhabitant is minted by systemd_run_user_wait_arguments. Dropping it +// would restore a reader that accepts a quiet launcher's silence as "no peak reported". +fn systemd_run_wait_summary(launcher: SystemdSummaryPrintingWait, stderr: String) -> SystemdRunWaitSummary { SystemdRunWaitSummary { result: match systemd_run_summary_last_value(stderr: stderr, prefix: systemd_run_summary_result_prefix) { Present { value: v } => systemd_service_result_parse(wire: v) diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index e7b2e9190b6..abcf4e8454b 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -11,7 +11,7 @@ import extdeps.shell import extdeps.git { shape_git_worktree_add_detached_argv } import extdeps.git.object_store { GitObjectId, git_object_id_from_untagged_hex, git_object_id_wire_hex } import extdeps.systemd.systemd_run { - systemd_run_user_wait_arguments, systemd_run_property, + systemd_run_user_wait_arguments, systemd_run_property, SystemdSummaryPrintingWait, systemd_summary_printing_wait_argv, SystemdRunWaitSummary, systemd_run_wait_summary, SystemdServiceResultReading, ServiceResultRead, ServiceResultUnrecognized, ServiceResultNotReported, SystemdServiceResult, ServiceResultOomKill, systemd_service_result_wire, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, @@ -318,24 +318,69 @@ fn compute_unit_properties(layout: ComputeLayout, granted: Kibibyte) -> List, granted: Kibibyte) -> ComputeExec { +// WHO OWES THE READ, AND WHY A TRUNCATION CANNOT UNDER-REPORT A PEAK HERE. A truncated stderr is the +// worst-directed hazard in this mechanism: it makes the number SMALLER -- "Memory peak: 11G" cut to +// "Memory peak: 1G" parses cleanly as a tenth of the real figure -- and under-reporting is the +// direction that OOMs. No guard in the parser can catch it (gunbc.machine_intake_host_capture_envelope +// HostCaptureText: line-wholeness is the CALLER's fact, never inferred from the bytes), so the +// obligation is NAMED here rather than dropped. +// +// THE READER IS gunbc.WitnessBin.Run, AND IT CAN RETURN A PREFIX -- its stderr_capture input +// defaults to BoundedTail { bytes: 16384 }, so a unit that writes more than 16 KiB of stderr has +// some of it discarded. What closes the hazard is the DIRECTION of that policy: a bounded TAIL +// discards the OLDEST bytes, and the launcher's summary is the youngest -- it is written after the +// unit has ended and the pipe has drained (extdeps.systemd.systemd_run, "WHY THE LAST LINE AND NOT +// THE FIRST"). So what a cut can slice mid-line is the FIRST retained line, which is the unit's own +// output, never the summary; and a fragment that happened to begin "Memory peak: " would still lose +// to the real line, because the fold takes the LAST occurrence of each prefix. A head-bounded or +// middle-eliding policy would NOT have this property, which is why the dependency is on the policy +// and not on the operation's name. +// +// SO THIS IS A NAMED DEPENDENCY ON A CONTRACT SOMEONE ELSE OWNS: if gunbc.WitnessBin.Run's default +// capture ever becomes anything other than a tail, this provider's peaks may silently under-report +// and every grant sized from them is undersized. The reading survives because that operation +// promises a suffix, not because this module checks anything. + +// THE UNIT RUN AND ITS SUMMARY ARE ONE RESULT, because the summary may only be read where the +// INVOCATION is known. extdeps.systemd.systemd_run systemd_run_wait_summary takes a +// SystemdSummaryPrintingWait, minted only by the builder that omits --quiet, so reading the peak +// from stderr somewhere further out -- where all that survives is a String -- is no longer +// expressible. The no-systemd arm reports NOTHING rather than a peak of zero: there was no unit, so +// there is no unit peak, and no launcher token to read one with. +type ComputeUnitRun { + exec: ComputeExec + summary: SystemdRunWaitSummary +} + +fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, identity_hex: String, argv: List, granted: Kibibyte) -> ComputeUnitRun { let systemd_run = match instance.toolchain.systemd_run { Present { value: p } => p as String Absent => "" } if systemd_run == "" { - ComputeExecFailed { exit_code: 127, stdout: "", stderr: "this instance's toolchain declares no systemd-run; the local provider runs only under systemd" } + ComputeUnitRun { + exec: ComputeExecFailed { exit_code: 127, stdout: "", stderr: "this instance's toolchain declares no systemd-run; the local provider runs only under systemd" }, + summary: SystemdRunWaitSummary { result: ServiceResultNotReported, memory_peak: MemoryPeakNotReported }, + } } else { - compute_exec( + let launcher = systemd_run_user_wait_arguments( + unit: compute_unit_name(identity_hex: identity_hex), + properties: compute_unit_properties(layout: layout, granted: granted), + setenv_bindings: [ + join(["CARGO_TARGET_DIR=", layout.target_dir], ""), + join(["RUSTC_WRAPPER=", sccache_installed_binary_path], ""), + ], + command_argv: argv, + ) + let r = compute_exec( program: systemd_run, workdir: layout.checkout, - args: systemd_run_user_wait_arguments( - unit: compute_unit_name(identity_hex: identity_hex), - properties: compute_unit_properties(layout: layout, granted: granted), - setenv_bindings: [ - join(["CARGO_TARGET_DIR=", layout.target_dir], ""), - join(["RUSTC_WRAPPER=", sccache_installed_binary_path], ""), - ], - command_argv: argv, - ), + args: systemd_summary_printing_wait_argv(w: launcher), ) + ComputeUnitRun { + exec: r, + summary: systemd_run_wait_summary(launcher: launcher, stderr: match r { + ComputeExecOk { stdout: _, stderr } => stderr + ComputeExecFailed { exit_code: _, stdout: _, stderr } => stderr + }), + } } } @@ -1161,15 +1206,13 @@ fn compute_run_leased( summary: SystemdRunWaitSummary { result: ServiceResultNotReported, memory_peak: MemoryPeakNotReported }, } ComputeExecOk { stdout: _, stderr: _ } => { - let run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store), granted: granted) + let unit_run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store), granted: granted) + let run = unit_run.exec let log = Filesystem.Write(path: layout.log_path, content: match run { ComputeExecOk { stdout, stderr } => join([stdout, stderr], "") ComputeExecFailed { exit_code: _, stdout, stderr } => join([stdout, stderr], "") }) - let summary = systemd_run_wait_summary(stderr: match run { - ComputeExecOk { stdout: _, stderr } => stderr - ComputeExecFailed { exit_code: _, stdout: _, stderr } => stderr - }) + let summary = unit_run.summary ComputeAttempt { outcome: match run { ComputeExecFailed { exit_code, stdout: _, stderr: _ } => diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag index adb877e46a4..11907495a18 100644 --- a/dag/test/claim/compute/work_class_grant_witness_test.dag +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -3,11 +3,12 @@ module test.claim.compute.work_class_grant_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, EpochSecs } import std.measure { Kibibyte, kibibyte, kibibyte_count, byte_size_count } import extdeps.systemd.systemd_run { - SystemdRunWaitSummary, systemd_run_wait_summary, + SystemdRunWaitSummary, systemd_run_wait_summary, systemd_run_user_wait_arguments, SystemdServiceResultReading, ServiceResultRead, ServiceResultUnrecognized, ServiceResultNotReported, SystemdServiceResult, ServiceResultOomKill, ServiceResultSuccess, ServiceResultExitCode, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, SystemdFormattedBytes, FormattedBytesExact, FormattedBytesBounded, systemd_formatted_bytes_upper, + SystemdSummaryPrintingWait, } import gunbc.compute.work_request { WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoRelease, CargoDebug, WorkSucceeded, WorkFailed } import gunbc.compute.work_class_grant { @@ -25,6 +26,13 @@ import gunbc.compute.work_provider_local { // ── THE PRODUCER: what systemd-run --wait said about the unit ───────────────────────────────── +// EVERY FIXTURE BELOW NAMES THE INVOCATION IT IS PRETENDING CAME FROM. systemd_run_wait_summary +// takes a SystemdSummaryPrintingWait, minted only by the builder that omits --quiet, so a claim +// cannot read a peak out of stderr that no launcher in this corpus could have printed. Supplying +// the launcher here is DESIGN 3's supplied input at one interface; the real builder is exercised by +// the provider's own route (gunbc.compute.work_provider_local compute_run_unit). +data fixture_launcher: SystemdSummaryPrintingWait = systemd_run_user_wait_arguments(unit: "gunbc-fixture", properties: [], setenv_bindings: [], command_argv: ["true"]) + // The unit's own stderr precedes the launcher's summary under --pipe, so a command that prints a // "Memory peak:" line of its own must not be read as the unit's peak. The summary text is the v255 // run.c ordering: result, main process status, runtime, CPU, memory peak. @@ -36,7 +44,7 @@ data nine_point_seven_gib_upper_bytes: Int = 10630044058 data nine_point_seven_gib_lower_bytes: Int = 10415295692 test fn the_units_peak_and_result_are_the_summarys_last_lines_not_the_commands() -> Bool { - let s = systemd_run_wait_summary(stderr: summary_oom_after_spoof) + let s = systemd_run_wait_summary(launcher: fixture_launcher, stderr: summary_oom_after_spoof) compute_summary_is_oom_kill(summary: s) && match s.memory_peak { MemoryPeakRead { peak: b } => @@ -56,9 +64,9 @@ test fn the_units_peak_and_result_are_the_summarys_last_lines_not_the_commands() // with --quiet. Either must read NotReported, because a recorded zero would size a class at its // headroom alone. An exact small figure keeps its exact reading. test fn an_absent_peak_is_not_reported_and_a_byte_figure_is_exact() -> Bool { - let absent = systemd_run_wait_summary(stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") - let small = systemd_run_wait_summary(stderr: "Finished with result: success\nMemory peak: 512B\n") - let garbled = systemd_run_wait_summary(stderr: "Finished with result: frobnicated\nMemory peak: lots\n") + let absent = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") + let small = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\nMemory peak: 512B\n") + let garbled = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: frobnicated\nMemory peak: lots\n") match absent.memory_peak { MemoryPeakNotReported => true _ => false } && !compute_summary_is_oom_kill(summary: absent) && match small.memory_peak { @@ -67,7 +75,7 @@ test fn an_absent_peak_is_not_reported_and_a_byte_figure_is_exact() -> Bool { } && match garbled.result { ServiceResultUnrecognized { wire } => wire == "frobnicated" _ => false } && match garbled.memory_peak { MemoryPeakUnparseable { text } => text == "lots" _ => false } - && match systemd_run_wait_summary(stderr: "").result { ServiceResultNotReported => true _ => false } + && match systemd_run_wait_summary(launcher: fixture_launcher, stderr: "").result { ServiceResultNotReported => true _ => false } } // ── THE RESOLUTION: rows to a grant ────────────────────────────────────────────────────────── @@ -130,8 +138,8 @@ test fn with_the_shipped_rows_every_class_reserves_the_ceiling() -> Bool { test fn only_an_oom_kill_below_the_ceiling_retries_at_the_ceiling() -> Bool { let below = GrantSized { class: WorkClassCompile, amount: kibibyte(count: 11 * gib), samples: 3, peak_upper: kibibyte(count: 9 * gib), headroom: kibibyte(count: 2 * gib), headroom_authority: "fixture" as NonEmptyStr } let at = GrantAtCeiling { class: WorkClassCompile, amount: ceiling(), reason: ClassUnmeasured } - let oom = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(stderr: summary_oom_after_spoof) } - let clean = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(stderr: "Finished with result: exit-code\nMemory peak: 3.1G\n") } + let oom = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(launcher: fixture_launcher, stderr: summary_oom_after_spoof) } + let clean = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: exit-code\nMemory peak: 3.1G\n") } compute_attempt_retries_at_ceiling(grant: below, attempt: oom) && !compute_attempt_retries_at_ceiling(grant: at, attempt: oom) && !compute_attempt_retries_at_ceiling(grant: below, attempt: clean) @@ -144,9 +152,9 @@ test fn only_an_oom_kill_below_the_ceiling_retries_at_the_ceiling() -> Bool { // No peak line, or no clock, takes no sample. test fn a_summary_becomes_a_typed_sample_at_its_upper_bound_or_no_sample() -> Bool { let op = CompileEntry { entry: "src/v2/compiler/compile.dag" } - let s = systemd_run_wait_summary(stderr: summary_oom_after_spoof) + let s = systemd_run_wait_summary(launcher: fixture_launcher, stderr: summary_oom_after_spoof) let taken = work_class_peak_sample(summary: s, work_identity: "abc", op: op, host: "srv1", observed_at: Present { value: 1790000000 }) - let no_peak = work_class_peak_sample(summary: systemd_run_wait_summary(stderr: "Finished with result: success\n"), work_identity: "abc", op: op, host: "srv1", observed_at: Present { value: 1790000000 }) + let no_peak = work_class_peak_sample(summary: systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\n"), work_identity: "abc", op: op, host: "srv1", observed_at: Present { value: 1790000000 }) let no_clock = work_class_peak_sample(summary: s, work_identity: "abc", op: op, host: "srv1", observed_at: none) match taken { PeakSampleTaken { class, sample } => @@ -179,3 +187,45 @@ test fn the_exceedance_count_is_zero_only_when_absence_is_positively_read() -> B && match never { GrantExceededNeverRecorded => true _ => false } && match blind { GrantExceededUnreadable { detail: _ } => true _ => false } } + +// ── HOW ELSE THE PEAK GOES UNREPORTED, AND THE GUARD THAT MUST NOT BE ADDED ─────────────────── +// +// NotReported is reachable by three ABSENCE shapes, not only by a manager below v254 omitting the +// line, and absence is the one thing this parser can be interrogated about without ever having seen +// a real summary: these claims are about the ABSENCE of the format rather than its content +// (loyal-swift-608, 2026-09-22). Empty stderr; a unit that failed and whose stderr carries only its +// own output; and neither may read as a peak of zero, because a zero would size the class at its +// headroom alone. +// +// THE THIRD CASE IS THE INVERSE, AND IT IS WHY NO TERMINATION GUARD BELONGS HERE. A truncated read +// would under-report a peak silently -- "Memory peak: 11G" cut to "Memory peak: 1G" parses cleanly +// as a tenth of the real figure -- so a trailing-newline requirement suggests itself. It is REFUSED, +// on the authority that already settled this question: gunbc.machine_intake_host_capture_envelope +// HostCaptureText rules that whether a capture's last line is whole is the CALLER's fact and is +// never inferred from the bytes, which cannot tell a truncation from a transport that trimmed. This +// stderr is a COMPLETED TRANSCRIPT -- systemd-run has exited before the provider reads it -- and the +// seed's shell observation trims, so a termination guard would drop the summary of every real run +// while still not detecting a mid-line cut. The claim below pins that an unterminated summary READS. +// 4.0G against the same factor and rounding as the 9.7G rows above: lower 40 * 1024^3 / 10, upper +// (40 + 2) * 1024^3 / 10 rounded up. +data four_gib_lower_bytes: Int = 4294967296 +data four_gib_upper_bytes: Int = 4509715661 + +test fn the_peak_is_unreported_by_absence_and_an_untrimmed_summary_still_reads() -> Bool { + let empty = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "") + let unit_output_only = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "error: linking with `cc` failed\nrustc exited 1\n") + let trimmed = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\nMemory peak: 4.0G") + match empty.memory_peak { MemoryPeakNotReported => true _ => false } + && match unit_output_only.memory_peak { MemoryPeakNotReported => true _ => false } + && match unit_output_only.result { ServiceResultNotReported => true _ => false } + && match trimmed.result { ServiceResultRead { result: r } => match r { ServiceResultSuccess => true _ => false } _ => false } + && match trimmed.memory_peak { + MemoryPeakRead { peak: b } => + match b { + FormattedBytesBounded { printed, lower, upper } => + printed == "4.0G" && byte_size_count(b: lower) == four_gib_lower_bytes && byte_size_count(b: upper) == four_gib_upper_bytes + FormattedBytesExact { bytes: _ } => false + } + _ => false + } +} From c88e6906362026b64e4f102d0f95153d99f85d2e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 03:13:42 +0000 Subject: [PATCH 20/28] An absence carries its cause, so three absence claims can no longer pass on one defect loyal-swift-608, 2026-09-22: assert the mechanism, not the outcome. A bare NotReported is an OUTCOME several causes produce -- a pre-v254 manager, a --quiet launcher, a unit never started, an empty capture, a dead parser -- so a defect that makes EVERYTHING unreported satisfied all three absence claims at once. That is DESIGN section 5's absorbing fallback in miniature: the failure arm answers where it should refuse precisely. SummaryAbsence types the cause (NoInvocation, SuppressedByQuietLauncher, CaptureEmpty, LineAbsent), shared by both readings because both go missing for the same reasons, and every claim now asserts its own arm. Also lands the demoted wall's regression control. The intent was that SystemdSummaryPrintingWait, minted only by the builder omitting --quiet, makes a quiet launcher unexpressible as the reader's argument. Measured otherwise: a record parameter accepted a List on the evaluation path and read a peak out of stderr no quiet launcher could have printed -- rung 2, not rung 4, with the fold as the actual wall and parameter-type enforcement as the next-rung trigger. Evidence, nine claims in work_class_grant_witness_test: clean tree 9 PASS quiet check -> `if false` reds exactly a_quiet_launcher_reports_no_peak... absence collapsed to one reds exactly the two claims asserting LineAbsent Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemd_run.dag | 61 ++++++++++++++++--- dag/gunbc/compute/work_class_grant.dag | 6 +- dag/gunbc/compute/work_provider_local.dag | 11 ++-- .../compute/work_class_grant_witness_test.dag | 51 +++++++++++++--- 4 files changed, 107 insertions(+), 22 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index e327fee0151..9369f0811a6 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -302,10 +302,33 @@ fn systemd_service_result_wire(r: SystemdServiceResult) -> String { } } +// WHY THERE IS NOTHING TO READ, BECAUSE "NOTHING" IS AN OUTCOME SEVERAL DIFFERENT CAUSES PRODUCE. +// A bare NotReported answers identically for a manager below v254, a launcher run with --quiet, a +// unit that was never started, an empty capture and a parser that has stopped working -- so a defect +// that makes EVERYTHING unreported satisfies every absence claim at once, and the producer cannot +// tell an old manager from a broken read. That is DESIGN 5's absorbing fallback in miniature: the +// failure arm answers rather than refuses precisely. So the absence carries its cause, one vocabulary +// shared by both readings because both go missing for the same reasons (loyal-swift-608, 2026-09-22: +// assert the mechanism, not the outcome). +type SummaryAbsence + = SummaryNoInvocation + | SummarySuppressedByQuietLauncher + | SummaryCaptureEmpty + | SummaryLineAbsent + +fn summary_absence_wire(a: SummaryAbsence) -> String { + match a { + SummaryNoInvocation => "no launcher ran, so there is no unit summary" + SummarySuppressedByQuietLauncher => "the launcher carries --quiet, which suppresses the summary entirely" + SummaryCaptureEmpty => "the launcher ran and its captured stderr was empty" + SummaryLineAbsent => "the captured stderr carries no such line (a manager below v254 omits the peak)" + } +} + type SystemdServiceResultReading = ServiceResultRead { result: SystemdServiceResult } | ServiceResultUnrecognized { wire: String } - | ServiceResultNotReported + | ServiceResultNotReported { absence: SummaryAbsence } fn systemd_service_result_parse(wire: String) -> SystemdServiceResultReading { let w = trim(s: wire) @@ -408,7 +431,7 @@ fn systemd_formatted_bytes_upper(b: SystemdFormattedBytes) -> ByteSize { type SystemdMemoryPeakReading = MemoryPeakRead { peak: SystemdFormattedBytes } | MemoryPeakUnparseable { text: String } - | MemoryPeakNotReported + | MemoryPeakNotReported { absence: SummaryAbsence } type SystemdRunWaitSummary { result: SystemdServiceResultReading @@ -425,15 +448,36 @@ fn systemd_run_summary_last_value(stderr: String, prefix: String) -> String? { } else { acc }) } -// THE READER TAKES THE LAUNCHER, NOT JUST ITS OUTPUT. The launcher argument is not inspected and is -// not meant to be: it is the evidence that this stderr came from an invocation that prints a -// summary at all, and its only inhabitant is minted by systemd_run_user_wait_arguments. Dropping it -// would restore a reader that accepts a quiet launcher's silence as "no peak reported". +// A WORD THIS ARGV CARRIES IS A FACT ABOUT WHETHER THERE IS ANYTHING TO READ, so it is read rather +// than assumed. --quiet is the single word that decides it (run.c v255, `if (arg_wait && +// !arg_quiet)`); its presence is checked here rather than trusted from the parameter's type for the +// reason measured below. +fn systemd_argv_suppresses_summary(argv: List) -> Bool { + fold(argv, init: false, f: (acc, w) => acc || w == "--quiet") +} + +// THE READER TAKES THE LAUNCHER AND INSPECTS IT, AND THE SECOND HALF IS NOT REDUNDANT -- IT IS +// MEASURED. The intent was construction: SystemdSummaryPrintingWait is minted only by +// systemd_run_user_wait_arguments, the one wait builder here that omits --quiet, so a quiet +// launcher should not be expressible as this argument at all. IT IS. A probe passing the +// List returned by systemd_run_transient_wait_unit_argv (which carries --quiet) as this +// parameter COMPILED AND EVALUATED, reading a peak out of stderr no quiet launcher could have +// printed. A declared record parameter does not refuse a list on this path, so the type alone is a +// decoration -- DESIGN 4b(1) rung inflation, and it would have been reported as a wall. +// +// SO THE HONEST RUNG IS 2, MECHANICALLY PREVENTABLE, NOT 4. The check below is what actually holds: +// a launcher whose argv carries --quiet reports NOTHING RATHER THAN A PEAK OF ZERO, which is the +// same fail-closed reading as a manager below v254, and the probe that compiled now goes red +// through it. Next-rung trigger: parameter-type enforcement for record types on the evaluation +// path, at which point the mint becomes the wall and this fold retires as its regression control. fn systemd_run_wait_summary(launcher: SystemdSummaryPrintingWait, stderr: String) -> SystemdRunWaitSummary { + if systemd_argv_suppresses_summary(argv: systemd_summary_printing_wait_argv(w: launcher)) { + SystemdRunWaitSummary { result: ServiceResultNotReported { absence: SummarySuppressedByQuietLauncher }, memory_peak: MemoryPeakNotReported { absence: SummarySuppressedByQuietLauncher } } + } else { SystemdRunWaitSummary { result: match systemd_run_summary_last_value(stderr: stderr, prefix: systemd_run_summary_result_prefix) { Present { value: v } => systemd_service_result_parse(wire: v) - Absent => ServiceResultNotReported + Absent => ServiceResultNotReported { absence: if stderr == "" { SummaryCaptureEmpty } else { SummaryLineAbsent } } }, memory_peak: match systemd_run_summary_last_value(stderr: stderr, prefix: systemd_run_summary_memory_peak_prefix) { Present { value: v } => @@ -441,7 +485,8 @@ fn systemd_run_wait_summary(launcher: SystemdSummaryPrintingWait, stderr: String FormattedBytesRead { value: b } => MemoryPeakRead { peak: b } FormattedBytesUnparseable { text: x } => MemoryPeakUnparseable { text: x } } - Absent => MemoryPeakNotReported + Absent => MemoryPeakNotReported { absence: if stderr == "" { SummaryCaptureEmpty } else { SummaryLineAbsent } } }, } + } } diff --git a/dag/gunbc/compute/work_class_grant.dag b/dag/gunbc/compute/work_class_grant.dag index 253939b982a..24f865cf966 100644 --- a/dag/gunbc/compute/work_class_grant.dag +++ b/dag/gunbc/compute/work_class_grant.dag @@ -6,6 +6,7 @@ import std.measure { Kibibyte, kibibyte, kibibyte_count, byte_size_count } import gunbc.compute.work_request { WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoProfile, cargo_profile_wire, work_operation_canonical } import extdeps.systemd.systemd_run { SystemdRunWaitSummary, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, systemd_formatted_bytes_upper, + SummaryAbsence, summary_absence_wire, } // A WORK REQUEST RESERVES AN AMOUNT PER CLASS, AND THE AMOUNT IS A GRANT WITH ITS OBSERVATIONS @@ -72,7 +73,7 @@ fn work_class_summary_peak_upper(summary: SystemdRunWaitSummary) -> Kibibyte? { match summary.memory_peak { MemoryPeakRead { peak: b } => Present { value: kibibyte(count: (byte_size_count(b: systemd_formatted_bytes_upper(b: b)) + 1023) / 1024) } MemoryPeakUnparseable { text: _ } => none - MemoryPeakNotReported => none + MemoryPeakNotReported { absence: _ } => none } } @@ -84,7 +85,8 @@ fn work_class_peak_sample(summary: SystemdRunWaitSummary, work_identity: String, match work_class_summary_peak_upper(summary: summary) { Absent => PeakSampleNotTaken { reason: match summary.memory_peak { MemoryPeakUnparseable { text } => join(["the Memory peak line did not parse: ", text], "") - _ => "no Memory peak line was reported" + MemoryPeakNotReported { absence } => summary_absence_wire(a: absence) + MemoryPeakRead { peak: _ } => "the peak was read but produced no upper bound" } } Present { value: upper } => match observed_at { diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index abcf4e8454b..c3f1d6c0c65 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -15,6 +15,7 @@ import extdeps.systemd.systemd_run { SystemdRunWaitSummary, systemd_run_wait_summary, SystemdServiceResultReading, ServiceResultRead, ServiceResultUnrecognized, ServiceResultNotReported, SystemdServiceResult, ServiceResultOomKill, systemd_service_result_wire, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, + SummaryAbsence, SummaryNoInvocation, SummarySuppressedByQuietLauncher, SummaryCaptureEmpty, SummaryLineAbsent, summary_absence_wire, SystemdFormattedBytes, FormattedBytesExact, FormattedBytesBounded, systemd_formatted_bytes_upper, } import extdeps.systemd.systemctl @@ -357,7 +358,7 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden if systemd_run == "" { ComputeUnitRun { exec: ComputeExecFailed { exit_code: 127, stdout: "", stderr: "this instance's toolchain declares no systemd-run; the local provider runs only under systemd" }, - summary: SystemdRunWaitSummary { result: ServiceResultNotReported, memory_peak: MemoryPeakNotReported }, + summary: SystemdRunWaitSummary { result: ServiceResultNotReported { absence: SummaryNoInvocation }, memory_peak: MemoryPeakNotReported { absence: SummaryNoInvocation } }, } } else { let launcher = systemd_run_user_wait_arguments( @@ -1140,7 +1141,7 @@ fn compute_summary_is_oom_kill(summary: SystemdRunWaitSummary) -> Bool { match summary.result { ServiceResultRead { result: r } => match r { ServiceResultOomKill => true _ => false } ServiceResultUnrecognized { wire: _ } => false - ServiceResultNotReported => false + ServiceResultNotReported { absence: _ } => false } } @@ -1152,7 +1153,7 @@ fn compute_summary_peak_wire(summary: SystemdRunWaitSummary) -> String { FormattedBytesBounded { printed, lower: _, upper } => join([printed, " (at most ", to_string(value: byte_size_count(b: upper)), " B)"], "") } MemoryPeakUnparseable { text } => join(["UNPARSEABLE (", text, ")"], "") - MemoryPeakNotReported => "NOT REPORTED (no Memory peak line: systemd below v254, or --wait summary suppressed)" + MemoryPeakNotReported { absence } => join(["NOT REPORTED (", summary_absence_wire(a: absence), ")"], "") } } @@ -1160,7 +1161,7 @@ fn compute_summary_result_wire(summary: SystemdRunWaitSummary) -> String { match summary.result { ServiceResultRead { result: r } => systemd_service_result_wire(r: r) ServiceResultUnrecognized { wire } => join(["UNRECOGNIZED (", wire, ")"], "") - ServiceResultNotReported => "NOT REPORTED" + ServiceResultNotReported { absence } => join(["NOT REPORTED (", summary_absence_wire(a: absence), ")"], "") } } @@ -1203,7 +1204,7 @@ fn compute_run_leased( ComputeExecFailed { exit_code, stdout: _, stderr } => ComputeAttempt { outcome: WorkRefusedByInfrastructure { identity: identity, cause: CheckoutUnavailable { detail: join(["git worktree add exited ", to_string(value: exit_code), ": ", stderr], "") } }, - summary: SystemdRunWaitSummary { result: ServiceResultNotReported, memory_peak: MemoryPeakNotReported }, + summary: SystemdRunWaitSummary { result: ServiceResultNotReported { absence: SummaryNoInvocation }, memory_peak: MemoryPeakNotReported { absence: SummaryNoInvocation } }, } ComputeExecOk { stdout: _, stderr: _ } => { let unit_run = compute_run_unit(instance: instance, layout: layout, identity_hex: identity, argv: compute_command_argv(instance: instance, layout: layout, op: op, dependency_store: dependency_store), granted: granted) diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag index 11907495a18..9a73eb7545c 100644 --- a/dag/test/claim/compute/work_class_grant_witness_test.dag +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -3,10 +3,11 @@ module test.claim.compute.work_class_grant_witness_test import std.types { String, Bool, Int, List, NonEmptyStr, EpochSecs } import std.measure { Kibibyte, kibibyte, kibibyte_count, byte_size_count } import extdeps.systemd.systemd_run { - SystemdRunWaitSummary, systemd_run_wait_summary, systemd_run_user_wait_arguments, + SystemdRunWaitSummary, systemd_run_wait_summary, systemd_run_user_wait_arguments, systemd_run_transient_wait_unit_argv, SystemdServiceResultReading, ServiceResultRead, ServiceResultUnrecognized, ServiceResultNotReported, SystemdServiceResult, ServiceResultOomKill, ServiceResultSuccess, ServiceResultExitCode, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, + SummaryAbsence, SummaryNoInvocation, SummarySuppressedByQuietLauncher, SummaryCaptureEmpty, SummaryLineAbsent, SystemdFormattedBytes, FormattedBytesExact, FormattedBytesBounded, systemd_formatted_bytes_upper, SystemdSummaryPrintingWait, } @@ -56,7 +57,7 @@ test fn the_units_peak_and_result_are_the_summarys_last_lines_not_the_commands() FormattedBytesExact { bytes: _ } => false } MemoryPeakUnparseable { text: _ } => false - MemoryPeakNotReported => false + MemoryPeakNotReported { absence: _ } => false } } @@ -67,7 +68,7 @@ test fn an_absent_peak_is_not_reported_and_a_byte_figure_is_exact() -> Bool { let absent = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") let small = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\nMemory peak: 512B\n") let garbled = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: frobnicated\nMemory peak: lots\n") - match absent.memory_peak { MemoryPeakNotReported => true _ => false } + match absent.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } && !compute_summary_is_oom_kill(summary: absent) && match small.memory_peak { MemoryPeakRead { peak: b } => match b { FormattedBytesExact { bytes } => byte_size_count(b: bytes) == 512 _ => false } @@ -75,7 +76,7 @@ test fn an_absent_peak_is_not_reported_and_a_byte_figure_is_exact() -> Bool { } && match garbled.result { ServiceResultUnrecognized { wire } => wire == "frobnicated" _ => false } && match garbled.memory_peak { MemoryPeakUnparseable { text } => text == "lots" _ => false } - && match systemd_run_wait_summary(launcher: fixture_launcher, stderr: "").result { ServiceResultNotReported => true _ => false } + && match systemd_run_wait_summary(launcher: fixture_launcher, stderr: "").result { ServiceResultNotReported { absence } => match absence { SummaryCaptureEmpty => true _ => false } _ => false } } // ── THE RESOLUTION: rows to a grant ────────────────────────────────────────────────────────── @@ -190,6 +191,13 @@ test fn the_exceedance_count_is_zero_only_when_absence_is_positively_read() -> B // ── HOW ELSE THE PEAK GOES UNREPORTED, AND THE GUARD THAT MUST NOT BE ADDED ─────────────────── // +// EACH CLAIM ASSERTS ITS OWN CAUSE, NOT THE SHARED OUTCOME (loyal-swift-608, 2026-09-22: assert the +// mechanism, not the outcome). Written against a bare NotReported these three claims would all have +// passed on ONE defect -- a parser that answers "nothing" for every input satisfies every absence +// assertion at once, which is exactly the dead producer this lane is at risk of. SummaryAbsence is +// what makes them discriminate: empty capture, absent line and suppressed-by-quiet are three +// different answers now, so a collapse to any single cause reds two of the three. +// // NotReported is reachable by three ABSENCE shapes, not only by a manager below v254 omitting the // line, and absence is the one thing this parser can be interrogated about without ever having seen // a real summary: these claims are about the ABSENCE of the format rather than its content @@ -215,9 +223,9 @@ test fn the_peak_is_unreported_by_absence_and_an_untrimmed_summary_still_reads() let empty = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "") let unit_output_only = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "error: linking with `cc` failed\nrustc exited 1\n") let trimmed = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\nMemory peak: 4.0G") - match empty.memory_peak { MemoryPeakNotReported => true _ => false } - && match unit_output_only.memory_peak { MemoryPeakNotReported => true _ => false } - && match unit_output_only.result { ServiceResultNotReported => true _ => false } + match empty.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryCaptureEmpty => true _ => false } _ => false } + && match unit_output_only.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } + && match unit_output_only.result { ServiceResultNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } && match trimmed.result { ServiceResultRead { result: r } => match r { ServiceResultSuccess => true _ => false } _ => false } && match trimmed.memory_peak { MemoryPeakRead { peak: b } => @@ -229,3 +237,32 @@ test fn the_peak_is_unreported_by_absence_and_an_untrimmed_summary_still_reads() _ => false } } + + +// ── A LAUNCHER THAT CANNOT PRINT A PEAK REPORTS NOTHING, NOT A PEAK ─────────────────────────── +// +// THIS IS THE DEMOTED WALL'S REGRESSION CONTROL, AND IT IS HERE BECAUSE THE TYPE WAS NOT ENOUGH. +// The intent was that SystemdSummaryPrintingWait, minted only by the builder that omits --quiet, +// makes a quiet launcher unexpressible as the reader's argument. Measured: passing the List +// from systemd_run_transient_wait_unit_argv (which carries --quiet) as that parameter COMPILED and +// read 9.7G out of stderr no quiet launcher could have printed. A declared record parameter does not +// refuse a list on the evaluation path, so the mint is not the wall -- DESIGN 4b(1), reported as +// rung 2 rather than rung 4. +// +// TWO THINGS HOLD NOW AND THIS CLAIM PINS THE SECOND. A wrongly typed argument refuses at the field +// access with a located TypeError naming the accessor and the call chain -- fail-closed, but not a +// fact a Bool claim can carry. What a claim CAN carry is the fold: a token whose argv does contain +// --quiet, constructed here directly because no builder will mint one, reports NotReported rather +// than the peak sitting in the fixture. Deleting the check in systemd_run_wait_summary reds this. +data quiet_launcher: SystemdSummaryPrintingWait = SystemdSummaryPrintingWait { argv: systemd_run_transient_wait_unit_argv(unit: "gunbc-quiet", properties: [], command_argv: ["true"]) } + +test fn a_quiet_launcher_reports_no_peak_rather_than_the_one_in_the_text() -> Bool { + let s = systemd_run_wait_summary(launcher: quiet_launcher, stderr: summary_oom_after_spoof) + match s.memory_peak { MemoryPeakNotReported { absence } => match absence { SummarySuppressedByQuietLauncher => true _ => false } _ => false } + && match s.result { ServiceResultNotReported { absence } => match absence { SummarySuppressedByQuietLauncher => true _ => false } _ => false } + && !compute_summary_is_oom_kill(summary: s) + && match work_class_peak_sample(summary: s, work_identity: "abc", op: CompileEntry { entry: "e" }, host: "srv1", observed_at: Present { value: 1790000000 }) { + PeakSampleNotTaken { reason: _ } => true + PeakSampleTaken { class: _, sample: _ } => false + } +} From 030d9b69a1ca8cebb8cff63569735c8de096230b Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 03:38:27 +0000 Subject: [PATCH 21/28] One claim per absence cause, so a red names which cause collapsed loyal-swift-608, 2026-09-22: run the other collapse arm. It falsified what I expected and the finding is worth more than the run. Typing the absence made three claims stop passing on ONE defect, but it did NOT pin each claim to one cause: both absence claims asserted CaptureEmpty AND LineAbsent, so collapsing every absence to EITHER cause reds THE SAME TWO CLAIMS. Pairwise discrimination held; per-claim separability did not, and the second is the property worth having -- a red that names which cause collapsed, not one that says some cause did. Each cause is now asserted in exactly one claim. No fixture was dropped: the failed-unit-without-a-peak-line case moved to the LineAbsent claim, and the empty capture got its own. Evidence, ten claims, four runs: clean 10 PASS collapse -> CaptureEmpty reds ONLY the_peak_is_unreported_by_absence... collapse -> LineAbsent reds ONLY an_empty_capture_is_reported_as_capture_empty... quiet fold deleted reds ONLY a_quiet_launcher_reports_no_peak... Co-Authored-By: Claude Opus 5 (1M context) --- .../compute/work_class_grant_witness_test.dag | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag index 9a73eb7545c..0868e41c819 100644 --- a/dag/test/claim/compute/work_class_grant_witness_test.dag +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -68,15 +68,13 @@ test fn an_absent_peak_is_not_reported_and_a_byte_figure_is_exact() -> Bool { let absent = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") let small = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\nMemory peak: 512B\n") let garbled = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: frobnicated\nMemory peak: lots\n") - match absent.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } - && !compute_summary_is_oom_kill(summary: absent) + !compute_summary_is_oom_kill(summary: absent) && match small.memory_peak { MemoryPeakRead { peak: b } => match b { FormattedBytesExact { bytes } => byte_size_count(b: bytes) == 512 _ => false } _ => false } && match garbled.result { ServiceResultUnrecognized { wire } => wire == "frobnicated" _ => false } && match garbled.memory_peak { MemoryPeakUnparseable { text } => text == "lots" _ => false } - && match systemd_run_wait_summary(launcher: fixture_launcher, stderr: "").result { ServiceResultNotReported { absence } => match absence { SummaryCaptureEmpty => true _ => false } _ => false } } // ── THE RESOLUTION: rows to a grant ────────────────────────────────────────────────────────── @@ -220,10 +218,10 @@ data four_gib_lower_bytes: Int = 4294967296 data four_gib_upper_bytes: Int = 4509715661 test fn the_peak_is_unreported_by_absence_and_an_untrimmed_summary_still_reads() -> Bool { - let empty = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "") + let no_peak_line = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") let unit_output_only = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "error: linking with `cc` failed\nrustc exited 1\n") let trimmed = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\nMemory peak: 4.0G") - match empty.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryCaptureEmpty => true _ => false } _ => false } + match no_peak_line.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } && match unit_output_only.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } && match unit_output_only.result { ServiceResultNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } && match trimmed.result { ServiceResultRead { result: r } => match r { ServiceResultSuccess => true _ => false } _ => false } @@ -266,3 +264,23 @@ test fn a_quiet_launcher_reports_no_peak_rather_than_the_one_in_the_text() -> Bo PeakSampleTaken { class: _, sample: _ } => false } } + + +// ── ONE CLAIM PER ABSENCE CAUSE, BECAUSE MIXING THEM MADE TWO CLAIMS INSEPARABLE ────────────── +// +// MEASURED, AND IT FALSIFIED WHAT I EXPECTED (loyal-swift-608, 2026-09-22: run the other collapse +// arm). Typing the absence made three claims stop passing on ONE defect, but it did not make each +// claim pinned to ONE cause: the two absence claims each asserted BOTH CaptureEmpty and LineAbsent, +// so collapsing every absence to either cause reds THE SAME TWO CLAIMS. Pairwise discrimination +// held; per-claim separability did not, and the second is the property worth having -- a red that +// names which cause collapsed, rather than a red that says some cause did. +// +// So each cause is now asserted in exactly one place: LineAbsent above, CaptureEmpty here, +// SuppressedByQuietLauncher below. A collapse to any one cause now reds exactly the claims for the +// OTHER causes, which is what makes the red legible. +test fn an_empty_capture_is_reported_as_capture_empty_not_as_a_missing_line() -> Bool { + let empty = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "") + match empty.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryCaptureEmpty => true _ => false } _ => false } + && match empty.result { ServiceResultNotReported { absence } => match absence { SummaryCaptureEmpty => true _ => false } _ => false } + && !compute_summary_is_oom_kill(summary: empty) +} From 023912ff96dfed9b876b127eed65d7a846c54239 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 03:50:02 +0000 Subject: [PATCH 22/28] Roster the class, and correct two mint annotations that still claimed the wall TWO THINGS, both rung honesty. FIRST, the class is rostered rather than left in a commit message: a_claim_asserting_several_causes_cannot_name_which_one_went. Its tell is greppable and needs no knowledge of the subject -- a conjunction, inside one claim, over two or more sibling arms of the same cause type. If a claim asserts N causes, its red cannot name which one went, however precisely the OUTCOME is typed. Verification is a matrix, not a mutation: collapse to each cause in turn, require each direction to red exactly one claim. Bounded against executed_conjunct_discriminates_nothing, which owns the red that never fires; this is its complement, where the red fires and the loss is attribution. SECOND, the correction neat-boar-16's scoping makes me owe. I demoted the reader's wall to rung 2 when I measured a List accepted for a record formal, but TWO OTHER ANNOTATIONS still stated the mint as construction -- that a quiet launcher was "no longer expressible" and that a claim "cannot read a peak no launcher could have printed". Both false by the same measurement, both corrected to name the argv fold as the wall. The scope is narrow and is now recorded where the rung is: declared parameter types ARE enforced generally (Int<-String, Int<-List, Rec<-Other, Rec<-Optional all refuse); the hole is ONE relation, a collection handed to a nominal product and its dual, in v1.compiler.infer declared_type_inhabitance -- a third specimen of declared_type_wall_keyed_on_the_value_being_a_kernel, not a new class. The trigger now names that relation and names the PATH: the interpreter's evaluation path was measured, emission was not, and DESIGN 4b(1) takes the minimum across paths. Evidence: ten claims PASS; the derived roster regenerated with the new row and the closure typechecks with it. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemd_run.dag | 16 +++++++++++-- dag/gunbc/compute/work_provider_local.dag | 15 ++++++++---- ...eral_causes_cannot_name_which_one_went.dag | 24 +++++++++++++++++++ .../compute/work_class_grant_witness_test.dag | 12 ++++++---- 4 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 dag/gunbc/recurring_failure_mode/a_claim_asserting_several_causes_cannot_name_which_one_went.dag diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index 9369f0811a6..7633d51df27 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -465,11 +465,23 @@ fn systemd_argv_suppresses_summary(argv: List) -> Bool { // printed. A declared record parameter does not refuse a list on this path, so the type alone is a // decoration -- DESIGN 4b(1) rung inflation, and it would have been reported as a wall. // +// SCOPED SINCE, AND THE SCOPE MATTERS BECAUSE IT IS NARROW (neat-boar-16, grid on main ff2110b7e). +// Declared parameter types ARE enforced generally -- Int from String, Int from List, a record +// from another record and from an Optional all REFUSE. The hole is ONE relation, a collection +// handed to a nominal product and its dual, because v1.compiler.infer declared_type_inhabitance +// only refuses in its collection arm when the declared base is a kernel scalar and otherwise +// reaches Inhabits by fallthrough. So this is a third specimen of the rostered +// declared_type_wall_keyed_on_the_value_being_a_kernel and not a new class, and no claim resting on +// a kernel or record-vs-record formal is affected by it. +// // SO THE HONEST RUNG IS 2, MECHANICALLY PREVENTABLE, NOT 4. The check below is what actually holds: // a launcher whose argv carries --quiet reports NOTHING RATHER THAN A PEAK OF ZERO, which is the // same fail-closed reading as a manager below v254, and the probe that compiled now goes red -// through it. Next-rung trigger: parameter-type enforcement for record types on the evaluation -// path, at which point the mint becomes the wall and this fold retires as its regression control. +// through it. Next-rung trigger: parameter-type enforcement for the collection-to-nominal relation +// ON THE EVALUATION PATH, at which point the mint becomes the wall and this fold retires as its +// regression control. THE PATH IS NAMED DELIBERATELY: DESIGN 4b(1) makes a class's rung the minimum +// across its in-scope paths, and the EMISSION path has not been measured, so closing the +// interpreter does not by itself restore a rung claimed source-to-emission. fn systemd_run_wait_summary(launcher: SystemdSummaryPrintingWait, stderr: String) -> SystemdRunWaitSummary { if systemd_argv_suppresses_summary(argv: systemd_summary_printing_wait_argv(w: launcher)) { SystemdRunWaitSummary { result: ServiceResultNotReported { absence: SummarySuppressedByQuietLauncher }, memory_peak: MemoryPeakNotReported { absence: SummarySuppressedByQuietLauncher } } diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index c3f1d6c0c65..9d7303ebd07 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -344,10 +344,17 @@ fn compute_unit_properties(layout: ComputeLayout, granted: Kibibyte) -> List carrying --quiet was ACCEPTED for this record formal on the evaluation path +// (measured, and scoped by neat-boar-16 on main ff2110b7e to ONE relation -- a collection handed to +// a nominal product, and its dual -- in v1.compiler.infer declared_type_inhabitance, a third +// specimen of declared_type_wall_keyed_on_the_value_being_a_kernel, fix lane dispatched). So the +// wall that actually refuses a quiet launcher is the argv fold inside systemd_run_wait_summary, +// rung 2, not this type. The no-systemd arm reports NOTHING rather than a peak of zero: there was +// no unit, so there is no unit peak, and no launcher token to read one with. type ComputeUnitRun { exec: ComputeExec summary: SystemdRunWaitSummary diff --git a/dag/gunbc/recurring_failure_mode/a_claim_asserting_several_causes_cannot_name_which_one_went.dag b/dag/gunbc/recurring_failure_mode/a_claim_asserting_several_causes_cannot_name_which_one_went.dag new file mode 100644 index 00000000000..8253ad7c7f7 --- /dev/null +++ b/dag/gunbc/recurring_failure_mode/a_claim_asserting_several_causes_cannot_name_which_one_went.dag @@ -0,0 +1,24 @@ +module gunbc.recurring_failure_mode.a_claim_asserting_several_causes_cannot_name_which_one_went + +import std.types { NonEmptyStr } +import gunbc.recurring_failure_mode { RecurringFailureMode } + +data a_claim_asserting_several_causes_cannot_name_which_one_went: RecurringFailureMode = RecurringFailureMode { + identity: "a_claim_asserting_several_causes_cannot_name_which_one_went" as NonEmptyStr, + + receipts: [ + "a claim conjoins assertions about SEVERAL SIBLING ARMS of one cause type, so it reddens when any of them collapses and its red cannot name which (INVALID STATE: a producer's outcome is refined from one arm into N typed causes, which is the repair that stops N claims passing on one defect; the CLAIMS are then written as conjunctions spanning those arms, so collapsing the producer to ANY single cause reddens THE SAME SET of claims. Pairwise discrimination is restored -- the claims no longer share one defect -- while per-claim attribution is not: the red says SOME cause collapsed and never which, and two such claims are indistinguishable from each other under every collapse. THE CONFLATION THE TYPE JUST REMOVED IS PUT BACK ONE LAYER UP, in the evidence rather than in the producer, which is why typing the outcome reads as having finished the job)", + + "THE MECHANICAL TELL IS GREPPABLE AND NEEDS NO KNOWLEDGE OF THE SUBJECT: A CONJUNCTION, INSIDE ONE CLAIM, OVER TWO OR MORE SIBLING ARMS OF THE SAME CAUSE TYPE. If a claim asserts N causes, its red cannot name which one went, no matter how precisely the OUTCOME is typed. The count is the whole check -- N greater than one is the defect -- and it is decidable from the claim's own text, unlike the belief-shaped membership tests elsewhere in this roster.", + + "THE VERIFICATION IS A MATRIX AND NOT A MUTATION: collapse the producer to EACH cause in turn, and require that each direction reddens EXACTLY ONE claim with every other claim green in all runs. That is a strictly stronger statement than 'the claims discriminate', and it is the one a reviewer can check without re-deriving the argument. ONE COLLAPSE DIRECTION IS NOT EVIDENCE, and this is the half most likely to be skipped: the first arm agrees with the author's expectation and is read as confirmation, which is mutation testing's general failure mode -- mutate toward what the assertions already tolerate, then read the survivors as coverage.", + + "MEASURED, gunbc#12021, and the second arm falsified a belief the author had already stated IN WRITING, which is the receipt that makes this a class rather than a preference. extdeps.systemd.systemd_run had a bare NotReported reading that several causes produced -- a manager below v254, a --quiet launcher, a unit never started, an empty capture, a dead parser -- so one broken parser satisfied every absence claim at once. The repair typed it: SummaryAbsence with NoInvocation, SuppressedByQuietLauncher, CaptureEmpty and LineAbsent. Collapsing every absence to CaptureEmpty then reddened exactly two claims, which was reported as the claims discriminating. THE OPPOSITE COLLAPSE, to LineAbsent, REDDENED THE SAME TWO. Both claims asserted BOTH causes, so neither collapse distinguished them and no red could name the cause that went. The repair was one cause per claim, with no fixture dropped -- the failed-unit-without-a-peak-line case moved into the LineAbsent claim and the empty capture got its own -- after which the four-run matrix separates cleanly: clean, collapse to CaptureEmpty, collapse to LineAbsent, and delete the quiet fold each redden exactly one claim.", + + "NEIGHBOUR, named so the boundary is decidable. gunbc.recurring_failure_mode.executed_conjunct_discriminates_nothing owns the conjunct whose red NEVER FIRES -- counterfactually independent of the property, deletable with every witness still green. This class is its complement and the cases are disjoint: here the red DOES fire, reliably and for a real defect, and the loss is ATTRIBUTION rather than detection. Its remedy half 'confirm EXACTLY ONE witness reddens, not that a witness reddens' is the same instinct one layer down, asked of a wall rather than of a cause partition; this row states the cause-partition form, where the population of collapses is enumerable from the type's own arms and the matrix is therefore finite and cheap.", + + "RUNG: MITIGATABLE, and the ceiling is mechanically preventable rather than higher. What exists today is a method an author must choose to run; nothing refuses a claim that conjoins sibling arms, and the tell above is greppable but ungated. NEXT-RUNG TRIGGER, a capability and not an artifact: a check that derives, from a claim's own text, the set of cause-type arms it asserts over, and refuses a merge-blocking claim whose set has more than one member unless the claim declares why the conjunction is its subject. That is sufficient because the arms of a closed cause type are enumerable from the declaration, so the collapse matrix is derivable rather than hand-written -- which is what retires the method. Enumerating collapse directions by hand does not retire it; it executes this row's recognition rule once per cause type.", + ], + + evidence: [], +} diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag index 0868e41c819..999a3f98abe 100644 --- a/dag/test/claim/compute/work_class_grant_witness_test.dag +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -28,10 +28,14 @@ import gunbc.compute.work_provider_local { // ── THE PRODUCER: what systemd-run --wait said about the unit ───────────────────────────────── // EVERY FIXTURE BELOW NAMES THE INVOCATION IT IS PRETENDING CAME FROM. systemd_run_wait_summary -// takes a SystemdSummaryPrintingWait, minted only by the builder that omits --quiet, so a claim -// cannot read a peak out of stderr that no launcher in this corpus could have printed. Supplying -// the launcher here is DESIGN 3's supplied input at one interface; the real builder is exercised by -// the provider's own route (gunbc.compute.work_provider_local compute_run_unit). +// takes a SystemdSummaryPrintingWait, so a claim must say which launcher its stderr came from +// rather than reading a peak out of a bare String. IT DOES NOT FOLLOW -- AND THIS ANNOTATION USED +// TO SAY IT DID -- that a claim cannot read a peak no launcher in this corpus could have printed: +// the record formal ACCEPTS a List carrying --quiet on the evaluation path, so the mint is +// not the wall. The argv fold is, at rung 2, and a_quiet_launcher_reports_no_peak... below is its +// regression control. Supplying the launcher here is DESIGN 3's supplied input at one interface; +// the real builder is exercised by the provider's own route (gunbc.compute.work_provider_local +// compute_run_unit). data fixture_launcher: SystemdSummaryPrintingWait = systemd_run_user_wait_arguments(unit: "gunbc-fixture", properties: [], setenv_bindings: [], command_argv: ["true"]) // The unit's own stderr precedes the launcher's summary under --pipe, so a command that prints a From 59b977211a05f588ec457d16ce432c58fdde90be Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 03:59:20 +0000 Subject: [PATCH 23/28] A rung demotion has a population, not a site -- filed on the row that owns it loyal-swift-608, 2026-09-22, generalizing this lane's own receipt. Appended to stale_claim_survives_its_own_correct_edit rather than forked into a new class: the specimens there have the falsifying edit made ELSEWHERE by someone else, which is what makes the join hard to demand. This variant removes that excuse -- the correcting edit was the AUTHOR'S OWN, minutes earlier -- and is still missed. The addition that is not a restatement: WHY THE SURVIVORS ARE WORSE THAN THE ORIGINAL OVERCLAIM. Before, N annotations agreed and all were wrong, so a reader got one consistent wrong story. After a site-local demotion, N-1 assert the wall and 1 denies it, each individually honest-looking, and the majority is the stale side -- the one place that tells the truth is outvoted. The sweep is free: grep the ASSERTION, and the count is the population the demotion owes. Also records the complement, because a roster with no green arm trains readers to ignore its red: a harness that says "route gap and not a verdict" IN THE VERDICT LINE. The failure it avoids -- a route gap reported as a red -- is the one that teaches people to triage reds as infrastructure. Evidence: ledger_row_coherence witness 3 PASS, so the appended receipt parses and the row still renders coherently. Co-Authored-By: Claude Opus 5 (1M context) --- .../stale_claim_survives_its_own_correct_edit.dag | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dag/gunbc/recurring_failure_mode/stale_claim_survives_its_own_correct_edit.dag b/dag/gunbc/recurring_failure_mode/stale_claim_survives_its_own_correct_edit.dag index 0f0ed1d17dd..094eb18a09a 100644 --- a/dag/gunbc/recurring_failure_mode/stale_claim_survives_its_own_correct_edit.dag +++ b/dag/gunbc/recurring_failure_mode/stale_claim_survives_its_own_correct_edit.dag @@ -85,6 +85,12 @@ data stale_claim_survives_its_own_correct_edit: RecurringFailureMode = Recurring "TERM (b) SPLITS ON THIS LENS RATHER THAN BEING UNMODELED. VARIANT ARMS are not in `decl_facts`. The lens SAYS SO in `bare_name_fork_lens_coverage_bound`: `Variant constructor names are NOT in this population and can fork undetected. Next-rung trigger: variant constructors carried as decl facts.` That bound is CONSTRUCTOR-NAME COLLISION. It is not exhaustiveness. A stale match on a grown coproduct (`accepted_of` after `ProviderUseRefusal` grew) is missing arms on a resolved match; carrying variant constructors as decl facts would not make this lens detect missing match arms. The capability that closes THAT hole is `v1.compiler.infer` `check_match_exhaustiveness` invoked over the module that contains the match -- which the required-floor context does not do for a keep-set-absent identity. Named on [[check_reachable_only_from_an_entry_point_the_context_never_calls]] and [[witness_that_fails_to_compile_is_absent_rather_than_red]], not as a widening of this lens's bound. Merge-path standing of the lens itself is the specimen receipt above, not a coverage claim here. The lens trigger (`at which point this bound is deleted rather than widened here`) remains the collision hole. Not built.", + + "**A RUNG DEMOTION HAS A POPULATION, NOT A SITE -- and the correcting edit here was the AUTHOR'S OWN, in the same session (gunbc#12021, 2026-09-22, named by loyal-swift-608 from this lane's own receipt).** The specimens above have the falsifying edit made ELSEWHERE, by someone else, which is what makes the join hard to demand. This variant removes that excuse and is still not caught: an author MEASURES that a wall does not hold, correctly demotes the rung AT THE SITE THEY PROBED, and leaves every OTHER annotation asserting the same wall standing. THE MOMENT A MEASUREMENT FALSIFIES A WALL, EVERY ANNOTATION ASSERTING THAT WALL IS STALE, WHEREVER IT SITS, and the demotion is incomplete until that population is swept. Measured instance: SystemdSummaryPrintingWait was believed unexpressible from a quiet launcher; a probe passed a List carrying --quiet to that record formal and it was ACCEPTED, so the reader's annotation was demoted to rung 2 with the measurement beside it -- while `gunbc.compute.work_provider_local` still said a quiet launcher was \"no longer expressible\" and the witness header still said a claim \"cannot read a peak no launcher in this corpus could have printed\". Both were false by the author's own measurement, taken minutes earlier.", + + "**WHY THE SURVIVORS ARE WORSE THAN THE ORIGINAL OVERCLAIM, which is this receipt's addition rather than a restatement: THE ONE PLACE THAT TELLS THE TRUTH IS OUTVOTED.** Before the demotion, three annotations agreed and all three were wrong -- a reader got one consistent, wrong story. After a site-local demotion, two assert the wall and one denies it, each individually honest-looking, and the majority is the stale side. A reviewer who never reaches the demoted site reads a wall; a reviewer who reaches only the demoted site cannot tell whether the others are a different subject or the same one unswept. So a PARTIAL correction can leave the corpus harder to read than no correction, and the tell is available to the author at zero cost: after demoting a rung, grep the ASSERTION -- the mint, the type name, the phrase that stated the impossibility -- and count the sites, because the count is the population the demotion owes.", + + "**THE COMPLEMENT WORTH NAMING, since this roster is short on good cases and a class with no green arm trains people to ignore its red: A HARNESS THAT DISTINGUISHES A ROUTE GAP FROM A VERDICT IN THE VERDICT LINE ITSELF.** In the same session, two wet claims run under `claim_batch` reported `FAIL ... (hermetic route has no arm for Dir: operation declares no mock_response -- the claim never reached its subject, so this is a route gap and not a verdict)`. The failure mode avoided is precise and is the one that does lasting damage: A ROUTE GAP REPORTED AS A RED trains readers that reds are noise, after which a real red is triaged as infrastructure. Reporting the distinction where the verdict is read -- not in a log line further out that a summary drops -- is what makes it load-bearing, and it is the same discipline this roster demands of a refusal: say WHICH cause, at the place the answer is consumed.", ], evidence: [], From 207946513dcf55a4e5d3d216c64ce19704c96bb0 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 04:20:03 +0000 Subject: [PATCH 24/28] The summary format is version-keyed, so an unread major refuses instead of parsing loyal-swift-608, 2026-09-22: the version was an assumption sitting outside the type. It is now inside it -- systemd_run_wait_summary takes a SummaryFormatStanding, and only SummaryFormatGrounded reaches the parser. WHY A REFUSAL AND NOT A BEST-EFFORT PARSE. Everything this module knows about the summary was read at ONE major, and the load-bearing part is invisible in the output: FORMAT_BYTES passes USE_IEC, so the figures are 1024-based, while format_bytes_full DEFAULTS to SI 1000-based factors. A major that dropped the flag would print a well-formed line this module reads ~7% high and would size every grant BELOW the peak that produced it -- silent, unsafe, undetectable from the line, and it would have survived every mutation here because the fixtures and the parser would carry the same wrong base. The grounded set is a LIST, not a floor: a newer major is not safer, only unread. Deliberately NOT the same question as systemctl systemd_memory_peak_support, which asks whether a peak is reported at all (>= 254, open-ended). A manager can report a peak in a format nobody has read; that is the case that must refuse. The version is observed from systemd-run itself -- the binary whose output is parsed -- not from systemctl, which would answer about a different executable. Evidence, thirteen claims, four runs: clean 13 PASS grounded set widened to 257 reds ONLY an_unread_major_refuses... ungrounded major parsed reds ONLY an_unread_major_refuses... unreadable version parsed reds ONLY an_unreadable_version_line_refuses... One honest limit: the Grounded arm has no single-claim collapse, because negating it means emptying the grounded set, which disables every fixture and reds broadly by construction -- a capability removal, not a collapse to a sibling cause. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemd_run.dag | 62 ++++++++++++++- dag/gunbc/compute/work_class_grant.dag | 3 + dag/gunbc/compute/work_provider_local.dag | 18 ++++- .../compute/work_class_grant_witness_test.dag | 76 +++++++++++++++---- 4 files changed, 144 insertions(+), 15 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index 7633d51df27..bf9c745b289 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -10,6 +10,9 @@ import extdeps.external_authority { ExternalAuthority, ExternalModelScope, Exter import std.decl_ref { DeclarationRef, WholeDeclaration } import extdeps.uri { Uri, Https } import extdeps.systemd { SystemdUnitProperty, systemd_unit_property_wire } +// The manager's version line has ONE spelling for the whole systemd upstream, so the major is read +// with the existing parser rather than a second one (DESIGN section 3: no fork of one concept). +import extdeps.systemd.systemctl { systemd_major_version } type SystemdRunProperty { property: SystemdUnitProperty, @@ -325,10 +328,56 @@ fn summary_absence_wire(a: SummaryAbsence) -> String { } } +// THE FORMAT IS VERSION-KEYED, SO A VERSION THIS MODULE HAS NOT READ IS AN UNMODELED EXTERNAL FACT +// AND THE ANSWER IS A REFUSAL, NOT A BEST-EFFORT PARSE (loyal-swift-608, 2026-09-22). +// +// Everything this module knows about the summary comes from reading systemd's source AT ONE MAJOR: +// the guard, the line spellings, the emission order, and -- the expensive one -- that FORMAT_BYTES +// passes USE_IEC, so the figures are 1024-based. format_bytes_full DEFAULTS TO SI 1000-BASED +// FACTORS and only that flag makes it IEC, so a major that dropped the flag would print figures +// this module reads about 7% HIGH in the whole part and would size every grant BELOW the peak that +// produced it -- silent, and in the unsafe direction. None of that is checkable from the printed +// line: "4.0G" is well-formed under either base. +// +// So the grounded set is the majors whose run.c and format-util.h were actually read, and it is a +// LIST rather than a floor: a newer major is not safer, it is merely unread. This is deliberately +// NOT the same question as extdeps.systemd.systemctl systemd_memory_peak_support, which asks +// whether the manager REPORTS a peak at all (>= 254, an open-ended feature floor). A manager can +// report a peak in a format this module has never read; that is precisely the case that must refuse. +data grounded_summary_format_majors: List = [255] + +type SummaryFormatStanding + = SummaryFormatGrounded { major: Int } + | SummaryFormatUngroundedMajor { major: Int } + | SummaryFormatVersionUnreadable { line: String } + +// systemd-run --version opens "systemd ()" -- the same binary whose output is +// parsed, so this is the producer's own version rather than a sibling tool's. +fn summary_format_standing(version_line: String) -> SummaryFormatStanding { + match systemd_major_version(version_line: version_line) { + Absent => SummaryFormatVersionUnreadable { line: trim(s: version_line) } + Present { value: v } => + if fold(grounded_summary_format_majors, init: false, f: (acc, g) => acc || g == v) { + SummaryFormatGrounded { major: v } + } else { + SummaryFormatUngroundedMajor { major: v } + } + } +} + +fn summary_format_standing_wire(s: SummaryFormatStanding) -> String { + match s { + SummaryFormatGrounded { major } => join(["systemd ", to_string(value: major), ", a major this module has read"], "") + SummaryFormatUngroundedMajor { major } => join(["systemd ", to_string(value: major), " is outside the majors whose summary format this module has read, so its figures are not interpretable here"], "") + SummaryFormatVersionUnreadable { line } => join(["the manager's version line did not parse (", line, "), so the summary format is unknown"], "") + } +} + type SystemdServiceResultReading = ServiceResultRead { result: SystemdServiceResult } | ServiceResultUnrecognized { wire: String } | ServiceResultNotReported { absence: SummaryAbsence } + | ServiceResultFormatNotGrounded { standing: SummaryFormatStanding } fn systemd_service_result_parse(wire: String) -> SystemdServiceResultReading { let w = trim(s: wire) @@ -432,6 +481,7 @@ type SystemdMemoryPeakReading = MemoryPeakRead { peak: SystemdFormattedBytes } | MemoryPeakUnparseable { text: String } | MemoryPeakNotReported { absence: SummaryAbsence } + | MemoryPeakFormatNotGrounded { standing: SummaryFormatStanding } type SystemdRunWaitSummary { result: SystemdServiceResultReading @@ -482,7 +532,17 @@ fn systemd_argv_suppresses_summary(argv: List) -> Bool { // regression control. THE PATH IS NAMED DELIBERATELY: DESIGN 4b(1) makes a class's rung the minimum // across its in-scope paths, and the EMISSION path has not been measured, so closing the // interpreter does not by itself restore a rung claimed source-to-emission. -fn systemd_run_wait_summary(launcher: SystemdSummaryPrintingWait, stderr: String) -> SystemdRunWaitSummary { +fn systemd_run_wait_summary(launcher: SystemdSummaryPrintingWait, standing: SummaryFormatStanding, stderr: String) -> SystemdRunWaitSummary { + match standing { + SummaryFormatUngroundedMajor { major: _ } => + SystemdRunWaitSummary { result: ServiceResultFormatNotGrounded { standing: standing }, memory_peak: MemoryPeakFormatNotGrounded { standing: standing } } + SummaryFormatVersionUnreadable { line: _ } => + SystemdRunWaitSummary { result: ServiceResultFormatNotGrounded { standing: standing }, memory_peak: MemoryPeakFormatNotGrounded { standing: standing } } + SummaryFormatGrounded { major: _ } => systemd_run_grounded_wait_summary(launcher: launcher, stderr: stderr) + } +} + +fn systemd_run_grounded_wait_summary(launcher: SystemdSummaryPrintingWait, stderr: String) -> SystemdRunWaitSummary { if systemd_argv_suppresses_summary(argv: systemd_summary_printing_wait_argv(w: launcher)) { SystemdRunWaitSummary { result: ServiceResultNotReported { absence: SummarySuppressedByQuietLauncher }, memory_peak: MemoryPeakNotReported { absence: SummarySuppressedByQuietLauncher } } } else { diff --git a/dag/gunbc/compute/work_class_grant.dag b/dag/gunbc/compute/work_class_grant.dag index 24f865cf966..ba28b14abb6 100644 --- a/dag/gunbc/compute/work_class_grant.dag +++ b/dag/gunbc/compute/work_class_grant.dag @@ -6,6 +6,7 @@ import std.measure { Kibibyte, kibibyte, kibibyte_count, byte_size_count } import gunbc.compute.work_request { WorkOperation, BuildGunbcBinaries, CompileEntry, RunClaims, CargoProfile, cargo_profile_wire, work_operation_canonical } import extdeps.systemd.systemd_run { SystemdRunWaitSummary, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, systemd_formatted_bytes_upper, + MemoryPeakFormatNotGrounded, summary_format_standing_wire, SummaryAbsence, summary_absence_wire, } @@ -74,6 +75,7 @@ fn work_class_summary_peak_upper(summary: SystemdRunWaitSummary) -> Kibibyte? { MemoryPeakRead { peak: b } => Present { value: kibibyte(count: (byte_size_count(b: systemd_formatted_bytes_upper(b: b)) + 1023) / 1024) } MemoryPeakUnparseable { text: _ } => none MemoryPeakNotReported { absence: _ } => none + MemoryPeakFormatNotGrounded { standing: _ } => none } } @@ -86,6 +88,7 @@ fn work_class_peak_sample(summary: SystemdRunWaitSummary, work_identity: String, Absent => PeakSampleNotTaken { reason: match summary.memory_peak { MemoryPeakUnparseable { text } => join(["the Memory peak line did not parse: ", text], "") MemoryPeakNotReported { absence } => summary_absence_wire(a: absence) + MemoryPeakFormatNotGrounded { standing } => summary_format_standing_wire(s: standing) MemoryPeakRead { peak: _ } => "the peak was read but produced no upper bound" } } Present { value: upper } => diff --git a/dag/gunbc/compute/work_provider_local.dag b/dag/gunbc/compute/work_provider_local.dag index 9d7303ebd07..6e2d11f18d9 100644 --- a/dag/gunbc/compute/work_provider_local.dag +++ b/dag/gunbc/compute/work_provider_local.dag @@ -15,6 +15,9 @@ import extdeps.systemd.systemd_run { SystemdRunWaitSummary, systemd_run_wait_summary, SystemdServiceResultReading, ServiceResultRead, ServiceResultUnrecognized, ServiceResultNotReported, SystemdServiceResult, ServiceResultOomKill, systemd_service_result_wire, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, + SummaryFormatStanding, SummaryFormatGrounded, SummaryFormatUngroundedMajor, SummaryFormatVersionUnreadable, + summary_format_standing, summary_format_standing_wire, + ServiceResultFormatNotGrounded, MemoryPeakFormatNotGrounded, SummaryAbsence, SummaryNoInvocation, SummarySuppressedByQuietLauncher, SummaryCaptureEmpty, SummaryLineAbsent, summary_absence_wire, SystemdFormattedBytes, FormattedBytesExact, FormattedBytesBounded, systemd_formatted_bytes_upper, } @@ -368,6 +371,16 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden summary: SystemdRunWaitSummary { result: ServiceResultNotReported { absence: SummaryNoInvocation }, memory_peak: MemoryPeakNotReported { absence: SummaryNoInvocation } }, } } else { + // THE VERSION IS OBSERVED FROM THE SAME BINARY WHOSE OUTPUT IS PARSED, not from a sibling tool + // and not assumed. extdeps.systemd.systemd_run grounds the summary format at specific majors -- + // down to the USE_IEC flag that makes the figures 1024-based -- so a manager outside that set + // must refuse rather than have its figures read against a format nobody has checked. Reading + // `systemctl --version` here would answer about a DIFFERENT executable than the one that prints + // the line. + let standing = summary_format_standing(version_line: match compute_exec(program: systemd_run, workdir: layout.checkout, args: ["--version"]) { + ComputeExecOk { stdout: v, stderr: _ } => v + ComputeExecFailed { exit_code: _, stdout: _, stderr: _ } => "" + }) let launcher = systemd_run_user_wait_arguments( unit: compute_unit_name(identity_hex: identity_hex), properties: compute_unit_properties(layout: layout, granted: granted), @@ -384,7 +397,7 @@ fn compute_run_unit(instance: HostDashboardInstance, layout: ComputeLayout, iden ) ComputeUnitRun { exec: r, - summary: systemd_run_wait_summary(launcher: launcher, stderr: match r { + summary: systemd_run_wait_summary(launcher: launcher, standing: standing, stderr: match r { ComputeExecOk { stdout: _, stderr } => stderr ComputeExecFailed { exit_code: _, stdout: _, stderr } => stderr }), @@ -1149,6 +1162,7 @@ fn compute_summary_is_oom_kill(summary: SystemdRunWaitSummary) -> Bool { ServiceResultRead { result: r } => match r { ServiceResultOomKill => true _ => false } ServiceResultUnrecognized { wire: _ } => false ServiceResultNotReported { absence: _ } => false + ServiceResultFormatNotGrounded { standing: _ } => false } } @@ -1161,6 +1175,7 @@ fn compute_summary_peak_wire(summary: SystemdRunWaitSummary) -> String { } MemoryPeakUnparseable { text } => join(["UNPARSEABLE (", text, ")"], "") MemoryPeakNotReported { absence } => join(["NOT REPORTED (", summary_absence_wire(a: absence), ")"], "") + MemoryPeakFormatNotGrounded { standing } => join(["REFUSED (", summary_format_standing_wire(s: standing), ")"], "") } } @@ -1169,6 +1184,7 @@ fn compute_summary_result_wire(summary: SystemdRunWaitSummary) -> String { ServiceResultRead { result: r } => systemd_service_result_wire(r: r) ServiceResultUnrecognized { wire } => join(["UNRECOGNIZED (", wire, ")"], "") ServiceResultNotReported { absence } => join(["NOT REPORTED (", summary_absence_wire(a: absence), ")"], "") + ServiceResultFormatNotGrounded { standing } => join(["REFUSED (", summary_format_standing_wire(s: standing), ")"], "") } } diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag index 999a3f98abe..a15f640543f 100644 --- a/dag/test/claim/compute/work_class_grant_witness_test.dag +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -7,6 +7,8 @@ import extdeps.systemd.systemd_run { SystemdServiceResultReading, ServiceResultRead, ServiceResultUnrecognized, ServiceResultNotReported, SystemdServiceResult, ServiceResultOomKill, ServiceResultSuccess, ServiceResultExitCode, SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, + SummaryFormatStanding, SummaryFormatGrounded, SummaryFormatUngroundedMajor, SummaryFormatVersionUnreadable, + summary_format_standing, MemoryPeakFormatNotGrounded, ServiceResultFormatNotGrounded, SummaryAbsence, SummaryNoInvocation, SummarySuppressedByQuietLauncher, SummaryCaptureEmpty, SummaryLineAbsent, SystemdFormattedBytes, FormattedBytesExact, FormattedBytesBounded, systemd_formatted_bytes_upper, SystemdSummaryPrintingWait, @@ -38,6 +40,11 @@ import gunbc.compute.work_provider_local { // compute_run_unit). data fixture_launcher: SystemdSummaryPrintingWait = systemd_run_user_wait_arguments(unit: "gunbc-fixture", properties: [], setenv_bindings: [], command_argv: ["true"]) +// THE FIXTURES ALSO NAME THE MANAGER THEY ARE PRETENDING CAME FROM, because the summary format is +// version-keyed and the reader refuses a major it has not read. Grounded here so the claims below +// are about PARSING; the refusal itself is claimed separately at the bottom of this module. +data grounded_standing: SummaryFormatStanding = summary_format_standing(version_line: "systemd 255 (255.4-1ubuntu8)") + // The unit's own stderr precedes the launcher's summary under --pipe, so a command that prints a // "Memory peak:" line of its own must not be read as the unit's peak. The summary text is the v255 // run.c ordering: result, main process status, runtime, CPU, memory peak. @@ -49,7 +56,7 @@ data nine_point_seven_gib_upper_bytes: Int = 10630044058 data nine_point_seven_gib_lower_bytes: Int = 10415295692 test fn the_units_peak_and_result_are_the_summarys_last_lines_not_the_commands() -> Bool { - let s = systemd_run_wait_summary(launcher: fixture_launcher, stderr: summary_oom_after_spoof) + let s = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: summary_oom_after_spoof) compute_summary_is_oom_kill(summary: s) && match s.memory_peak { MemoryPeakRead { peak: b } => @@ -62,6 +69,7 @@ test fn the_units_peak_and_result_are_the_summarys_last_lines_not_the_commands() } MemoryPeakUnparseable { text: _ } => false MemoryPeakNotReported { absence: _ } => false + MemoryPeakFormatNotGrounded { standing: _ } => false } } @@ -69,9 +77,9 @@ test fn the_units_peak_and_result_are_the_summarys_last_lines_not_the_commands() // with --quiet. Either must read NotReported, because a recorded zero would size a class at its // headroom alone. An exact small figure keeps its exact reading. test fn an_absent_peak_is_not_reported_and_a_byte_figure_is_exact() -> Bool { - let absent = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") - let small = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\nMemory peak: 512B\n") - let garbled = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: frobnicated\nMemory peak: lots\n") + let absent = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") + let small = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "Finished with result: success\nMemory peak: 512B\n") + let garbled = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "Finished with result: frobnicated\nMemory peak: lots\n") !compute_summary_is_oom_kill(summary: absent) && match small.memory_peak { MemoryPeakRead { peak: b } => match b { FormattedBytesExact { bytes } => byte_size_count(b: bytes) == 512 _ => false } @@ -141,8 +149,8 @@ test fn with_the_shipped_rows_every_class_reserves_the_ceiling() -> Bool { test fn only_an_oom_kill_below_the_ceiling_retries_at_the_ceiling() -> Bool { let below = GrantSized { class: WorkClassCompile, amount: kibibyte(count: 11 * gib), samples: 3, peak_upper: kibibyte(count: 9 * gib), headroom: kibibyte(count: 2 * gib), headroom_authority: "fixture" as NonEmptyStr } let at = GrantAtCeiling { class: WorkClassCompile, amount: ceiling(), reason: ClassUnmeasured } - let oom = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(launcher: fixture_launcher, stderr: summary_oom_after_spoof) } - let clean = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: exit-code\nMemory peak: 3.1G\n") } + let oom = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: summary_oom_after_spoof) } + let clean = ComputeAttempt { outcome: WorkFailed { identity: "i", exit_code: 1, log_path: "" }, summary: systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "Finished with result: exit-code\nMemory peak: 3.1G\n") } compute_attempt_retries_at_ceiling(grant: below, attempt: oom) && !compute_attempt_retries_at_ceiling(grant: at, attempt: oom) && !compute_attempt_retries_at_ceiling(grant: below, attempt: clean) @@ -155,9 +163,9 @@ test fn only_an_oom_kill_below_the_ceiling_retries_at_the_ceiling() -> Bool { // No peak line, or no clock, takes no sample. test fn a_summary_becomes_a_typed_sample_at_its_upper_bound_or_no_sample() -> Bool { let op = CompileEntry { entry: "src/v2/compiler/compile.dag" } - let s = systemd_run_wait_summary(launcher: fixture_launcher, stderr: summary_oom_after_spoof) + let s = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: summary_oom_after_spoof) let taken = work_class_peak_sample(summary: s, work_identity: "abc", op: op, host: "srv1", observed_at: Present { value: 1790000000 }) - let no_peak = work_class_peak_sample(summary: systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\n"), work_identity: "abc", op: op, host: "srv1", observed_at: Present { value: 1790000000 }) + let no_peak = work_class_peak_sample(summary: systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "Finished with result: success\n"), work_identity: "abc", op: op, host: "srv1", observed_at: Present { value: 1790000000 }) let no_clock = work_class_peak_sample(summary: s, work_identity: "abc", op: op, host: "srv1", observed_at: none) match taken { PeakSampleTaken { class, sample } => @@ -222,9 +230,9 @@ data four_gib_lower_bytes: Int = 4294967296 data four_gib_upper_bytes: Int = 4509715661 test fn the_peak_is_unreported_by_absence_and_an_untrimmed_summary_still_reads() -> Bool { - let no_peak_line = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") - let unit_output_only = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "error: linking with `cc` failed\nrustc exited 1\n") - let trimmed = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "Finished with result: success\nMemory peak: 4.0G") + let no_peak_line = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "Finished with result: exit-code\nMain processes terminated with: code=exited/status=1\n") + let unit_output_only = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "error: linking with `cc` failed\nrustc exited 1\n") + let trimmed = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "Finished with result: success\nMemory peak: 4.0G") match no_peak_line.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } && match unit_output_only.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } && match unit_output_only.result { ServiceResultNotReported { absence } => match absence { SummaryLineAbsent => true _ => false } _ => false } @@ -259,7 +267,7 @@ test fn the_peak_is_unreported_by_absence_and_an_untrimmed_summary_still_reads() data quiet_launcher: SystemdSummaryPrintingWait = SystemdSummaryPrintingWait { argv: systemd_run_transient_wait_unit_argv(unit: "gunbc-quiet", properties: [], command_argv: ["true"]) } test fn a_quiet_launcher_reports_no_peak_rather_than_the_one_in_the_text() -> Bool { - let s = systemd_run_wait_summary(launcher: quiet_launcher, stderr: summary_oom_after_spoof) + let s = systemd_run_wait_summary(launcher: quiet_launcher, standing: grounded_standing, stderr: summary_oom_after_spoof) match s.memory_peak { MemoryPeakNotReported { absence } => match absence { SummarySuppressedByQuietLauncher => true _ => false } _ => false } && match s.result { ServiceResultNotReported { absence } => match absence { SummarySuppressedByQuietLauncher => true _ => false } _ => false } && !compute_summary_is_oom_kill(summary: s) @@ -283,8 +291,50 @@ test fn a_quiet_launcher_reports_no_peak_rather_than_the_one_in_the_text() -> Bo // SuppressedByQuietLauncher below. A collapse to any one cause now reds exactly the claims for the // OTHER causes, which is what makes the red legible. test fn an_empty_capture_is_reported_as_capture_empty_not_as_a_missing_line() -> Bool { - let empty = systemd_run_wait_summary(launcher: fixture_launcher, stderr: "") + let empty = systemd_run_wait_summary(launcher: fixture_launcher, standing: grounded_standing, stderr: "") match empty.memory_peak { MemoryPeakNotReported { absence } => match absence { SummaryCaptureEmpty => true _ => false } _ => false } && match empty.result { ServiceResultNotReported { absence } => match absence { SummaryCaptureEmpty => true _ => false } _ => false } && !compute_summary_is_oom_kill(summary: empty) } + + +// ── A MANAGER WHOSE FORMAT WAS NEVER READ REFUSES, RATHER THAN BEING PARSED OPTIMISTICALLY ──── +// +// WHY THIS IS A REFUSAL AND NOT A BEST-EFFORT PARSE. Everything this corpus knows about the summary +// was read from systemd's source at ONE major, and the load-bearing part is invisible in the output: +// FORMAT_BYTES passes USE_IEC, so "4.0G" is 1024-based, while format_bytes_full DEFAULTS to +// 1000-based factors. A major that dropped the flag would print a well-formed line this module reads +// about 7% high, sizing every grant BELOW the peak that produced it -- silent, unsafe, and not +// detectable from the line. So an unread major is an unmodeled external fact (DESIGN 4b) and the +// answer is a typed, located refusal (DESIGN 5). +// +// THE THREE STANDINGS ARE ASSERTED SEPARATELY, one cause per claim, because a claim that asserted +// several of them could not name which collapsed -- the class this lane filed as +// a_claim_asserting_several_causes_cannot_name_which_one_went. +test fn a_grounded_major_is_the_only_one_that_parses() -> Bool { + match summary_format_standing(version_line: "systemd 255 (255.4-1ubuntu8)") { + SummaryFormatGrounded { major } => major == 255 + _ => false + } +} + +test fn an_unread_major_refuses_instead_of_reading_its_figures() -> Bool { + let newer = summary_format_standing(version_line: "systemd 257 (257.1-1)") + let s = systemd_run_wait_summary(launcher: fixture_launcher, standing: newer, stderr: summary_oom_after_spoof) + match newer { SummaryFormatUngroundedMajor { major } => major == 257 _ => false } + && match s.memory_peak { MemoryPeakFormatNotGrounded { standing: _ } => true _ => false } + && match s.result { ServiceResultFormatNotGrounded { standing: _ } => true _ => false } + && !compute_summary_is_oom_kill(summary: s) + && match work_class_peak_sample(summary: s, work_identity: "abc", op: CompileEntry { entry: "e" }, host: "srv1", observed_at: Present { value: 1790000000 }) { + PeakSampleNotTaken { reason: _ } => true + PeakSampleTaken { class: _, sample: _ } => false + } +} + +test fn an_unreadable_version_line_refuses_rather_than_assuming_the_grounded_major() -> Bool { + let unknown = summary_format_standing(version_line: "") + let s = systemd_run_wait_summary(launcher: fixture_launcher, standing: unknown, stderr: summary_oom_after_spoof) + match unknown { SummaryFormatVersionUnreadable { line } => line == "" _ => false } + && match s.memory_peak { MemoryPeakFormatNotGrounded { standing: _ } => true _ => false } + && !compute_summary_is_oom_kill(summary: s) +} From 7cf660c11fbb6dceb43760e7cd9f62b73202896e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 04:30:35 +0000 Subject: [PATCH 25/28] The refusal names its own remedy, and a claim proves it reaches the consumer loyal-swift-608, 2026-09-22. Every new systemd major refuses until someone reads it, which is correct and will happen -- so the diagnostic carries the procedure instead of making the next reader rediscover it: read run.c and format-util.h at that major, confirm the guard, the emit and its UINT64_MAX condition, that FORMAT_BYTES still passes USE_IEC | BELOW_POINT | TRAILING_B, and the line order, then add the major to grounded_summary_format_majors. Same discipline as a next-rung trigger naming the capability rather than an artifact. The third fact is named because it is the one that would be skipped: the flags are invisible in the output, and USE_IEC is the difference between 1024-based and 1000-based figures. A REMEDY NOTHING CARRIES IS PROSE NOBODY READS AT THE MOMENT THEY HIT THE WALL, so the claim asserts the JOIN: the refusal reaches work_class_peak_sample's reason with the remedy on it. Keyed to the DECLARATION, not to a literal -- rewording the remedy stays green because the wording is not the claim; dropping it from the consumer's path goes red because the join is. Evidence: 13 PASS; dropping the remedy from the ungrounded wire reds ONLY an_unread_major_refuses_instead_of_reading_its_figures. Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemd_run.dag | 13 +++++++++++-- .../claim/compute/work_class_grant_witness_test.dag | 7 ++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index bf9c745b289..a94463ea3f9 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -365,11 +365,20 @@ fn summary_format_standing(version_line: String) -> SummaryFormatStanding { } } +// A REFUSAL NAMES ITS OWN REMEDY (loyal-swift-608, 2026-09-22). Every new systemd major refuses +// here until someone reads it, which is correct and WILL happen -- so the diagnostic carries the +// procedure rather than leaving the next reader to rediscover it. This is the same discipline +// DESIGN 4b(2) applies to a next-rung trigger: name the capability, not the symptom. The four +// facts are named because each one is separately load-bearing and the third is the one that would +// be skipped -- the flags are invisible in the output, and USE_IEC is the difference between +// 1024-based and 1000-based figures. +data summary_format_grounding_remedy: String = "to ground it: read src/run/run.c and src/basic/format-util.h at that major and confirm four things -- the `if (arg_wait && !arg_quiet)` guard, the `Memory peak: %s` emit and its UINT64_MAX condition, that FORMAT_BYTES still passes USE_IEC | BELOW_POINT | TRAILING_B (SI vs IEC is ~7% and silent), and the line order -- then add the major to grounded_summary_format_majors" + fn summary_format_standing_wire(s: SummaryFormatStanding) -> String { match s { SummaryFormatGrounded { major } => join(["systemd ", to_string(value: major), ", a major this module has read"], "") - SummaryFormatUngroundedMajor { major } => join(["systemd ", to_string(value: major), " is outside the majors whose summary format this module has read, so its figures are not interpretable here"], "") - SummaryFormatVersionUnreadable { line } => join(["the manager's version line did not parse (", line, "), so the summary format is unknown"], "") + SummaryFormatUngroundedMajor { major } => join(["systemd ", to_string(value: major), " is outside the majors whose summary format this module has read, so its figures are not interpretable here; ", summary_format_grounding_remedy], "") + SummaryFormatVersionUnreadable { line } => join(["the manager's version line did not parse (", line, "), so the summary format is unknown; establish the major first (systemd-run --version opens \"systemd \"), then ", summary_format_grounding_remedy], "") } } diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag index a15f640543f..a79525ab375 100644 --- a/dag/test/claim/compute/work_class_grant_witness_test.dag +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -9,6 +9,7 @@ import extdeps.systemd.systemd_run { SystemdMemoryPeakReading, MemoryPeakRead, MemoryPeakUnparseable, MemoryPeakNotReported, SummaryFormatStanding, SummaryFormatGrounded, SummaryFormatUngroundedMajor, SummaryFormatVersionUnreadable, summary_format_standing, MemoryPeakFormatNotGrounded, ServiceResultFormatNotGrounded, + summary_format_grounding_remedy, SummaryAbsence, SummaryNoInvocation, SummarySuppressedByQuietLauncher, SummaryCaptureEmpty, SummaryLineAbsent, SystemdFormattedBytes, FormattedBytesExact, FormattedBytesBounded, systemd_formatted_bytes_upper, SystemdSummaryPrintingWait, @@ -325,8 +326,12 @@ test fn an_unread_major_refuses_instead_of_reading_its_figures() -> Bool { && match s.memory_peak { MemoryPeakFormatNotGrounded { standing: _ } => true _ => false } && match s.result { ServiceResultFormatNotGrounded { standing: _ } => true _ => false } && !compute_summary_is_oom_kill(summary: s) + // THE REFUSAL MUST REACH THE CONSUMER CARRYING ITS REMEDY, or the remedy is prose nobody reads + // at the moment they hit the wall. Keyed to the DECLARATION rather than to a literal: rewording + // the remedy stays green, because the wording is not the claim -- dropping it from the path a + // consumer actually reads goes red, because the join is. && match work_class_peak_sample(summary: s, work_identity: "abc", op: CompileEntry { entry: "e" }, host: "srv1", observed_at: Present { value: 1790000000 }) { - PeakSampleNotTaken { reason: _ } => true + PeakSampleNotTaken { reason } => reason.contains(summary_format_grounding_remedy) PeakSampleTaken { class: _, sample: _ } => false } } From 511b20bb9525b56266a65fc151ba4ec932c90904 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 04:36:15 +0000 Subject: [PATCH 26/28] Name the base case of the grounding recursion as the base case loyal-swift-608, 2026-09-22, and the rule rather than the preference: a claim's NAME is what the next reader uses to decide whether it may be retargeted, narrowed or deleted, so a name that understates what depends on a claim is a decay the annotation cannot repair -- the annotation is read AFTER the decision to touch it. an_unreadable_version_line_refuses_rather_than_assuming_the_grounded_major read as edge-case tidying. It is not: deciding whether the format may be parsed requires READING A VERSION LINE, so "parse the version to decide whether you may parse" is circular unless the version read can itself refuse. Ordinary edge-case cleanup would have deleted the base case and left the recursion open. Renamed to the_grounding_recursion_terminates_because_an_unreadable_version_refuses. The rule's exception did not apply: this claim is new on this branch, in no roster and behind no declared split, so the rename is cheap and the annotate-and-cite substitute was not needed. The two-way citation is there anyway because it was free -- each end of the grounding relation now names the other. Evidence: 13 PASS after the rename. Co-Authored-By: Claude Opus 5 (1M context) --- .../compute/work_class_grant_witness_test.dag | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/dag/test/claim/compute/work_class_grant_witness_test.dag b/dag/test/claim/compute/work_class_grant_witness_test.dag index a79525ab375..b6051540e5d 100644 --- a/dag/test/claim/compute/work_class_grant_witness_test.dag +++ b/dag/test/claim/compute/work_class_grant_witness_test.dag @@ -312,6 +312,9 @@ test fn an_empty_capture_is_reported_as_capture_empty_not_as_a_missing_line() -> // THE THREE STANDINGS ARE ASSERTED SEPARATELY, one cause per claim, because a claim that asserted // several of them could not name which collapsed -- the class this lane filed as // a_claim_asserting_several_causes_cannot_name_which_one_went. +// The other end of the grounding relation: only a major this module has read proceeds to the +// parser. Its base case is the_grounding_recursion_terminates_because_an_unreadable_version_refuses +// below -- without that, the version read this check depends on could fail open. test fn a_grounded_major_is_the_only_one_that_parses() -> Bool { match summary_format_standing(version_line: "systemd 255 (255.4-1ubuntu8)") { SummaryFormatGrounded { major } => major == 255 @@ -336,7 +339,19 @@ test fn an_unread_major_refuses_instead_of_reading_its_figures() -> Bool { } } -test fn an_unreadable_version_line_refuses_rather_than_assuming_the_grounded_major() -> Bool { +// THIS IS THE BASE CASE OF THE GROUNDING RECURSION, WHICH IS WHY IT IS NOT NAMED FOR AN EDGE CASE +// (loyal-swift-608, 2026-09-22). Deciding whether the format may be parsed requires READING A +// VERSION LINE, so "parse the version to decide whether you may parse" is circular unless the +// version read can ITSELF refuse. That is this claim, and the recursion terminates here and +// nowhere else. Its counterpart is a_grounded_major_is_the_only_one_that_parses, which is the +// other end of the same relation: that one says only a read major proceeds, this one says an +// unread VERSION LINE does not silently become a read major. +// +// The earlier name, an_unreadable_version_line_refuses_rather_than_assuming_the_grounded_major, +// read as edge-case tidying, so ordinary edge-case cleanup would have deleted the base case and +// left the recursion open. A name that understates what depends on a claim is a decay an +// annotation cannot fully repair, because the annotation is read AFTER the decision to touch it. +test fn the_grounding_recursion_terminates_because_an_unreadable_version_refuses() -> Bool { let unknown = summary_format_standing(version_line: "") let s = systemd_run_wait_summary(launcher: fixture_launcher, standing: unknown, stderr: summary_oom_after_spoof) match unknown { SummaryFormatVersionUnreadable { line } => line == "" _ => false } From a8809fa67e9fbe862b6baf1816584e0c3b267e8f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 22 Sep 2026 05:09:07 +0000 Subject: [PATCH 27/28] Annotations at module-item grain: the floor's parse phase is a gate my local loop cannot see CI floor red since 6a53f39f, three annotation blocks of mine sitting INSIDE declaration bodies -- a fn body, an else branch, and a nested `operation` within a service. DESIGN 4c admits standalone leading `//` only above a MODULE-SCOPE declaration; body grain refuses until separately modeled. main carried none of these, so all three are mine. WHY THIRTEEN GREEN CLAIMS DID NOT CATCH IT, which is the part worth recording: claim_batch over a witness entry resolves the module closure and NEVER RUNS THE DAG PARSE SWEEP, so a grain defect in a touched module is invisible to it while every witness reports PASS. That is the rostered class annotation_inside_a_declaration_body_is_invisible_to_the_authoring_gate, and I walked into it by verifying with the only instrument I had running. Each block moved above its enclosing module-scope declaration AND REWORDED to name its subject, because it is no longer adjacent to what it describes -- an annotation that says "this operation" from a site two levels out is a citation by position. The service-level one states why it is authored at that grain. Evidence, and the green is falsified rather than trusted: v1_src_dag_parse 6591 files parse-clean, exit 0 planted body annotation LOCATED at work_provider_local.dag:373:3, exit 1 plant removed parse-clean again, exit 0 claim_batch 13 PASS (text moved across declaration boundaries) Co-Authored-By: Claude Opus 5 (1M context) --- dag/extdeps/systemd/systemd_run.dag | 20 +++++++++++-------- dag/gunbc/compute/work_provider_local.dag | 12 +++++------ .../compute/work_class_grant_witness_test.dag | 9 +++++---- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/dag/extdeps/systemd/systemd_run.dag b/dag/extdeps/systemd/systemd_run.dag index a94463ea3f9..9c898a1ba2b 100644 --- a/dag/extdeps/systemd/systemd_run.dag +++ b/dag/extdeps/systemd/systemd_run.dag @@ -169,6 +169,18 @@ fn systemd_run_transient_wait_unit_argv(unit: NonEmptyStr, properties: List