From 6b4ce85efa5ec0bcb49b6441d3646b2435dbb01a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 02:16:10 +0000 Subject: [PATCH 01/30] Add a sealed held-session store op and named k=2 hold-slot reads. Per-commit family-hold brackets listed the whole store root on every write. A session acquires once for many lookups and commits; check and release verify the known generation by path so they stay O(1) in store size. Unknown-head acquire still lists, because a windowed slot is not a presence prefix. Co-authored-by: Cursor --- .../materialization_store_local.dag | 40 ++++++++++++-- dag/gunbc/durable_cas_file_store.dag | 48 ++++++++++++++++ .../durable_exclusive_hold_file_store.dag | 41 +++++++++++++- dag/gunbc/typecheck_module_store.dag | 33 +++++++++++ ...lization_store_local_seal_witness_test.dag | 12 ++++ ...alization_store_local_wet_witness_test.dag | 55 +++++++++++++++++++ src/v2/workflow/local_repo_wet_terminal.dag | 6 ++ 7 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 dag/gunbc/typecheck_module_store.dag diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag index 9551444a2ec..95246684322 100644 --- a/dag/extdeps/realization/materialization_store_local.dag +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -91,10 +91,10 @@ import std.durable_exclusive_hold { import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquireOutcome, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable, - file_hold_release_assess_windowed, FileHoldReleasePlanned, FileHoldReleaseNotEligible, + file_hold_release_assess_windowed_at, FileHoldReleasePlanned, FileHoldReleaseNotEligible, file_hold_release_commit, FileHoldReleased, FileHoldReleaseLost, FileHoldReleaseStoreUnavailable, FileHoldReleaseKeyNotSlotAddressable, file_hold_recovery_assess_windowed, FileHoldRecoveryPlanned, FileHoldRecoveryNotEligible, - observe_file_hold_windowed, + observe_file_hold_windowed_at, } import gunbc.process_hold_identity { process_hold_text, process_hold_marker, process_hold_decode, ProcessHoldHolder, @@ -1033,6 +1033,18 @@ fn local_store_write_index_settled(held: LocalStoreFamilyHeld, expected: CasExpe // on this host, and only a holder observed dead is recovered through file_hold_recovery_assess. An // unobservable holder stays held -- a family that cannot be shown free is not written. The hold slot // is windowed like the occupancy index (k = 2), so its bookkeeping does not grow with commits. +// +// HOLD CHECK AND RELEASE READ THE SLOT BY NAME. The windowed verify step is: generation G present +// and G+1 absent. LocalStoreFamilyHeld already names G, so those two reads are cas_file_slot_path +// Filesystem.Read -- O(1) in the number of objects in the store root. That preserves the existing +// argument (a head is never guessed; a successor present means the slot moved; an absent G is not +// this hold). +// +// ACQUIRE STILL LISTS. Windowed presence is not a prefix: generations at or below head-k are +// deleted, so a named gallop from generation 1 would read NotFound and treat a live head at 100 as +// an empty slot, then create-new generation 1 beside it. Listing the root to discover this key's +// generation files is the unknown-head observation cas_observe_window already uses. A held session +// pays that list once, not once per commit. data local_store_hold_owner_prefix: String = "materialization-store" @@ -1196,7 +1208,7 @@ fn local_store_hold_check(held: LocalStoreFamilyHeld) -> LocalStoreHoldCheck decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_commit_verified"), ] { - let observed = observe_file_hold_windowed(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, key: local_store_hold_key(family: held.family), window: held.window) + let observed = observe_file_hold_windowed_at(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, key: local_store_hold_key(family: held.family), window: held.window, generation: held.generation) match observed { HoldSlotHeld { generation: g, owner: o } => if cas_generation_count(g: g) == cas_generation_count(g: held.generation) && durable_hold_owner_equal(left: o, right: held.owner) { LocalStoreHoldCurrent } else { LocalStoreHoldNotCurrent { observed: observed } } @@ -1223,6 +1235,7 @@ fn local_store_with_family_of(binding: LocalStoreBinding, family: ArtifactKin admit_callers: [ decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_with_family"), decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_record_recency"), + decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_held_session"), ] { match local_store_acquire_family_hold(binding: binding, family: family, recovered: false) { @@ -1234,6 +1247,25 @@ fn local_store_with_family_of(binding: LocalStoreBinding, family: ArtifactKin } } +// A HELD SESSION: acquire the family once, run many lookups and commits under LocalStoreFamilyHeld, +// release once. Distinct from local_store_commit, which still brackets every write; this operation +// does not change that signature. Sealed to gunbc.typecheck_module_store (and the inhabitance +// witness that proves N commits acquire once). +// +// DEADLINE AND RENEWAL follow std.durable_exclusive_hold, not a clock. That protocol recovers a +// hold only when its holder is OBSERVED dead -- there is no timestamp, duration or silent expiry. +// This process is the holder (gunbc.process_hold_identity). Renewal is the named re-observation +// local_store_hold_check performs before every mutation: the slot must still be held at the +// generation and by the owner the session recorded, or the mutation refuses as stale. A session +// whose process has died is recovered by the next writer's liveness observation, never by age. +fn local_store_held_session(capability: LocalStoreCapability, family: ArtifactKindId, run: fn(LocalStoreFamilyHeld) -> T) -> LocalStoreBracket + admit_callers: [ + decl_ref(module_path: "gunbc.typecheck_module_store", decl_name: "typecheck_module_store_session"), + ] +{ + local_store_with_family_of(binding: capability.binding, family: family, run: run) +} + // The release is reported, never fatal to the write it closes: a hold that could not be freed stays // held by this process, and once this process is gone the next writer recovers it by observation. type LocalStoreHoldRelease @@ -1250,7 +1282,7 @@ fn local_store_release_family_hold(held: LocalStoreFamilyHeld) -> LocalStoreHold decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_acquire_all"), ] { - match file_hold_release_assess_windowed(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, slot_key: local_store_hold_key(family: held.family), owner: held.owner, acquired_generation: held.generation, released_by: local_store_hold_release_ref, window: held.window) { + match file_hold_release_assess_windowed_at(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, slot_key: local_store_hold_key(family: held.family), owner: held.owner, acquired_generation: held.generation, released_by: local_store_hold_release_ref, window: held.window) { FileHoldReleaseNotEligible { slot_key: _, refusal: r } => LocalStoreHoldReleaseNotEligible { refusal: r } FileHoldReleasePlanned { plan: pl } => match file_hold_release_commit(plan: pl) { diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index 29889384b6d..09a66bcea69 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -682,6 +682,54 @@ fn observe_cas_slot_state_windowed(root: NonEmptyStr, key: NonEmptyStr, window: observe_cas_slot_state_retained(root: root, key: key, retention: KeepGenerationWindow { window: window }) } +// A WINDOWED HEAD READ BY CONSTRUCTED NAME, NOT BY LISTING THE SLOT ROOT. +// +// cas_observe_window finds the head by listing every name under the store root and keeping this +// key's generation files. That is the correct discovery when the generation is unknown, because a +// windowed slot is NOT a presence prefix: generations at or below head-k are deleted, so galloping +// from generation 1 would read NotFound and report an empty slot while the live head sat at 100. +// Listing is therefore retained for unknown-head observation (cas_observe_window). +// +// When the caller already names the generation it believes is head -- a hold it just acquired, a +// release at the generation the protocol recorded -- the existing correctness argument does not +// need the listing. The windowed contract's verify step is: the named generation reads present and +// its successor reads absent. Those two reads are Filesystem.Read of cas_file_slot_path, O(1) in +// the store's object count. A successor that is present means the head moved; an absent named +// generation is not this head. The observation is never guessed. +fn cas_verify_named_window_head(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> CasSlotProbe + admit_callers: [ + decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "observe_cas_slot_state_windowed_at"), + ] +{ + match cas_read_generation(root: root, key: key, generation: generation) { + GenerationRefused { probe: p } => p + GenerationAbsent => ProbedAbsent + GenerationPresent { content: c } => + match cas_generation_successor(g: generation) { + CasSuccessorExhausted { head: _ } => ProbedHead { generation: generation, value: c } + CasSuccessorGeneration { generation: next } => + match cas_read_generation(root: root, key: key, generation: next) { + GenerationRefused { probe: p } => p + GenerationAbsent => ProbedHead { generation: generation, value: c } + GenerationPresent { content: _ } => ProbedWindowHeadUnsettled { attempts: 1 } + } + } + } +} + +fn observe_cas_slot_state_windowed_at(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationWindow, generation: CasGeneration) -> CasSlotObservation + admit_callers: [ + decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "observe_file_hold_windowed_at"), + ] +{ + let _window = window + if !cas_key_is_slot_addressable(key: key) { + CasObservedUnreadable { cause: CasUnreadableReadRefused { detail: cas_key_not_slot_addressable_detail } } + } else { + cas_probe_as_observation(probe: cas_verify_named_window_head(root: root, key: key, generation: generation)) + } +} + data cas_key_not_slot_addressable_detail: NonEmptyStr = "key is not slot-addressable: it carries a path separator" diff --git a/dag/gunbc/durable_exclusive_hold_file_store.dag b/dag/gunbc/durable_exclusive_hold_file_store.dag index fa5efa93c9d..f31697e15be 100644 --- a/dag/gunbc/durable_exclusive_hold_file_store.dag +++ b/dag/gunbc/durable_exclusive_hold_file_store.dag @@ -42,9 +42,10 @@ import std.durable_exclusive_hold { DurableHoldHolderReport, DurableHoldRecoveryRefusal, HoldRecoveryEligible, HoldRecoveryRefused, durable_hold_recovery_assess, } -import gunbc.durable_cas_file_store { KeepAllGenerations, CasSlotRetention, - DefaultAccessCreateOnly, DerivedPayloadAdmitted, DerivedPayloadKeyNotSlotAddressable, - admit_cas_attempt_for_derived_payload, file_compare_and_set, observe_cas_slot_state, +import gunbc.durable_cas_file_store { KeepAllGenerations, KeepGenerationWindow, CasSlotRetention, + CasGenerationWindow, DefaultAccessCreateOnly, DerivedPayloadAdmitted, DerivedPayloadKeyNotSlotAddressable, + admit_cas_attempt_for_derived_payload, file_compare_and_set, file_compare_and_set_retained, + observe_cas_slot_state, observe_cas_slot_state_retained, observe_cas_slot_state_windowed_at, } // THE TAGS ARE NAMED ONCE AND THE BODY IS DERIVED FROM THE TAG THAT MATCHED. @@ -163,6 +164,15 @@ fn observe_file_hold_windowed(root: NonEmptyStr, key: NonEmptyStr, window: CasGe observe_file_hold_retained(root: root, key: key, retention: KeepGenerationWindow { window: window }) } +fn observe_file_hold_windowed_at(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationWindow, generation: CasGeneration) -> DurableHoldObservation + admit_callers: [ + decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_hold_check"), + decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess_windowed_at"), + ] +{ + hold_observation_of_cas(observed: observe_cas_slot_state_windowed_at(root: root, key: key, window: window, generation: generation)) +} + // EVERY EFFECTFUL HOLD WRITE IS LIVE READ -> SEALED PLAN -> CAS COMMIT, AND NO // WRITER ACCEPTS AN OBSERVATION FROM ITS CALLER. // @@ -375,6 +385,31 @@ fn file_hold_release_assess_windowed(root: NonEmptyStr, slot_key: NonEmptyStr, o file_hold_release_assess_retained(root: root, slot_key: slot_key, owner: owner, acquired_generation: acquired_generation, released_by: released_by, retention: KeepGenerationWindow { window: window }) } +fn file_hold_release_assess_windowed_at(root: NonEmptyStr, slot_key: NonEmptyStr, owner: DurableHoldOwnerRef, acquired_generation: CasGeneration, released_by: DurableHoldReleaseRef, window: CasGenerationWindow) -> FileHoldReleaseAssessment + admit_callers: [ + decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_release_family_hold"), + ] +{ + match durable_hold_release_assess( + observed: observe_file_hold_windowed_at(root: root, key: slot_key, window: window, generation: acquired_generation), + owner: owner, + acquired_generation: acquired_generation, + ) { + HoldReleaseRefused { refusal: r } => FileHoldReleaseNotEligible { slot_key: slot_key, refusal: r } + HoldReleaseEligible { generation: g } => + FileHoldReleasePlanned { + plan: FileHoldReleasePlan { + root: root, + slot_key: slot_key, + owner: owner, + generation: g, + free_payload: hold_slot_payload(state: DurableHoldFree { released_by: released_by }), + retention: KeepGenerationWindow { window: window }, + }, + } + } +} + // The commit still tells a lost race from an unavailable store. A release // assessed at generation N and committed after the slot moved is // FileHoldReleaseLost -- the hold this plan described no longer exists -- and diff --git a/dag/gunbc/typecheck_module_store.dag b/dag/gunbc/typecheck_module_store.dag new file mode 100644 index 00000000000..9394092c982 --- /dev/null +++ b/dag/gunbc/typecheck_module_store.dag @@ -0,0 +1,33 @@ +module gunbc.typecheck_module_store + +// THE TYPECHECK MODULE STORE SESSION: the admitted consumer of +// extdeps.realization.materialization_store_local local_store_held_session. +// neat-carp-75 (C2) will drive lookups and commits through this fold. This +// module is the seal's inhabitant, not a second store. + +import std.types { List } +import std.cache_identity { ArtifactKindId } +import std.materialization_provider { ArtifactRequest } +import std.materialization_object { ArtifactPayload, StoreLookup } +import extdeps.realization.materialization_store_local { + LocalStoreCapability, LocalStoreFamilyHeld, LocalStoreBracket, LocalStoreCommit, + LocalStoreStagedPart, + local_store_held_session, local_store_commit_under, local_store_lookup, +} + +fn typecheck_module_store_session(capability: LocalStoreCapability, family: ArtifactKindId, run: fn(LocalStoreFamilyHeld) -> T) -> LocalStoreBracket { + local_store_held_session(capability: capability, family: family, run: run) +} + +fn typecheck_module_store_lookup(capability: LocalStoreCapability, req: ArtifactRequest) -> StoreLookup { + local_store_lookup(capability: capability, req: req) +} + +fn typecheck_module_store_commit_under( + held: LocalStoreFamilyHeld, + req: ArtifactRequest, + payloads: List, + parts: List +) -> LocalStoreCommit { + local_store_commit_under(held: held, req: req, payloads: payloads, parts: parts) +} diff --git a/dag/test/claim/materialization_store_local_seal_witness_test.dag b/dag/test/claim/materialization_store_local_seal_witness_test.dag index d8411bace7a..3f49969d04e 100644 --- a/dag/test/claim/materialization_store_local_seal_witness_test.dag +++ b/dag/test/claim/materialization_store_local_seal_witness_test.dag @@ -142,3 +142,15 @@ test fn a_direct_raw_observe_call_is_an_admission_refusal() -> Bool { test fn a_direct_raw_reclaim_call_is_an_admission_refusal() -> Bool { admission_refusals(source: direct_raw_reclaim_source, callee: "cas_reclaim_below") == 1 } + +data direct_held_session_source: String = "module probe_direct_held_session\nimport std.cache_identity { ArtifactKindId }\nimport extdeps.realization.materialization_store_local { LocalStoreCapability, LocalStoreFamilyHeld, LocalStoreBracket, local_store_held_session }\nfn direct(c: LocalStoreCapability, f: ArtifactKindId) -> LocalStoreBracket {\n local_store_held_session(capability: c, family: f, run: h => h)\n}\n" + +data direct_named_window_observe_source: String = "module probe_direct_named_window_observe\nimport std.types { NonEmptyStr }\nimport std.durable_compare_and_set { CasGeneration, CasSlotObservation }\nimport gunbc.durable_cas_file_store { CasGenerationWindow, observe_cas_slot_state_windowed_at }\nfn direct(r: NonEmptyStr, k: NonEmptyStr, w: CasGenerationWindow, g: CasGeneration) -> CasSlotObservation {\n observe_cas_slot_state_windowed_at(root: r, key: k, window: w, generation: g)\n}\n" + +test fn a_direct_held_session_call_is_an_admission_refusal() -> Bool { + admission_refusals(source: direct_held_session_source, callee: "local_store_held_session") == 1 +} + +test fn a_direct_named_window_observe_call_is_an_admission_refusal() -> Bool { + admission_refusals(source: direct_named_window_observe_source, callee: "observe_cas_slot_state_windowed_at") == 1 +} diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 3390043414a..e45a22af760 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -48,6 +48,9 @@ import extdeps.realization.materialization_store_local { local_store_index_window, LocalStoreRecordDeleteAttempt, LocalStoreRecordGone, LocalStoreRecordFileRefused, LocalStoreRecordPartsRefused, LocalStoreRecordPartsUnlisted, LocalStoreStagedPart, LocalStorePartsLookup, local_store_lookup_parts, local_store_part_name, } +import gunbc.typecheck_module_store { + typecheck_module_store_session, typecheck_module_store_commit_under, typecheck_module_store_lookup, +} import gunbc.process_hold_identity { self_process_identity, SelfIdentified, SelfUnidentified, process_hold_text } import gunbc.build_cache_instance { ProcessIdentity } import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable } @@ -1339,3 +1342,55 @@ test fn a_leftover_copy_that_cannot_be_removed_leaves_the_store_unwritable_by_re let removed = shell.Remove.RecursiveForce(path: root_path) verdict && removed.success } + +fn hold_slot_head_generation(root_path: String) -> Int { + let key = local_store_hold_key(family: native_module_verdict_bundle_artifact_kind) as String + fold([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], init: 0, f: (best, n) => + if Filesystem.Read(path: store_path(root_path: root_path, name: concat(concat(key, "."), n as String))).success { n } else { best } + ) +} + +fn session_commit_ok(c: LocalStoreCommit) -> Bool { + commit_tag(c: c.commit) == "committed" +} + +// N COMMITS UNDER ONE SESSION ACQUIRE THE HOLD ONCE. The hold slot's generation is the acquire +// count: one acquire and one release leave head 2. The same N commits through local_store_commit +// (the per-commit bracket) leave head 2N. That second measurement is the RED: if the session were +// secretly the per-commit bracket, both heads would match and this claim would fail. Lookups run +// under the same session and do not take a second hold. Dummy files in the root make a per-commit +// whole-root List O(store size); the session still finishes with head 2. +test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution() -> Bool { + let n = 6 + let session_root_path = fresh_root_path() + let bracket_root_path = fresh_root_path() + let session_root = WitnessScratchRoot { root: session_root_path } + let bracket_root = WitnessScratchRoot { root: bracket_root_path } + let session_ok = match opened(root: session_root) { + Absent => false + Present { value: store } => { + let planted = fold([1, 2, 3, 4, 5, 6, 7, 8], init: true, f: (ok, i) => + ok && Filesystem.WriteCreateNew(path: store_path(root_path: session_root_path, name: concat("dummy-", i as String)), content: "x").success + ) + match typecheck_module_store_session(capability: store, family: native_module_verdict_bundle_artifact_kind, run: h => { + let writes = [1, 2, 3, 4, 5, 6] |> map(i => typecheck_module_store_commit_under(held: h, req: bundle_req(module: concat("s", i as String)), payloads: verdicts(text: "s pass"), parts: [])) + let hits = [1, 2, 3, 4, 5, 6] |> map(i => store_lookup_is_hit(l: typecheck_module_store_lookup(capability: store, req: bundle_req(module: concat("s", i as String))))) + writes |> all(c => session_commit_ok(c: c)) && hits |> all(x => x) + }) { + LocalStoreBracketRefused { refusal: _ } => false + LocalStoreBracketRan { value: body_ok, hold_release: _ } => planted && body_ok && hold_slot_head_generation(root_path: session_root_path) == 2 + } + } + } + let bracket_ok = match opened(root: bracket_root) { + Absent => false + Present { value: store } => { + let writes = [1, 2, 3, 4, 5, 6] |> map(i => local_store_commit(capability: store, req: bundle_req(module: concat("b", i as String)), payloads: verdicts(text: "b pass"), parts: [])) + writes |> all(c => session_commit_ok(c: c)) && hold_slot_head_generation(root_path: bracket_root_path) == (2 * n) + } + } + let removed_s = shell.Remove.RecursiveForce(path: session_root_path) + let removed_b = shell.Remove.RecursiveForce(path: bracket_root_path) + session_ok && bracket_ok && removed_s.success && removed_b.success +} + diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 6b23bfdabf8..b025d33af2a 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -2185,6 +2185,12 @@ fn local_repo_wet_schedule() -> List { function: "a_leftover_copy_that_cannot_be_removed_leaves_the_store_unwritable_by_real_execution", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.pre_os_capture_replay_witness", function: "w_early_exit_refuses_absence_on_real_bytes" }, entry: "dag/test/claim/machine_intake/pre_os_capture_replay_witness_test.dag", From 0af5c2ed036936a887116ef1844ddf1325775d1e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 03:36:56 +0000 Subject: [PATCH 02/30] Measure hold-slot advance from after open, not from zero. Initialize/open already acquires and releases the family hold for the sweep, so an absolute head of 2 was never the session's signature. The control is the delta: +2 for one session, +2N for N per-commit brackets. Co-authored-by: Cursor --- ...alization_store_local_wet_witness_test.dag | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index e45a22af760..17a7a55a5ed 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -1345,7 +1345,7 @@ test fn a_leftover_copy_that_cannot_be_removed_leaves_the_store_unwritable_by_re fn hold_slot_head_generation(root_path: String) -> Int { let key = local_store_hold_key(family: native_module_verdict_bundle_artifact_kind) as String - fold([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], init: 0, f: (best, n) => + fold([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24], init: 0, f: (best, n) => if Filesystem.Read(path: store_path(root_path: root_path, name: concat(concat(key, "."), n as String))).success { n } else { best } ) } @@ -1354,12 +1354,12 @@ fn session_commit_ok(c: LocalStoreCommit) -> Bool { commit_tag(c: c.commit) == "committed" } -// N COMMITS UNDER ONE SESSION ACQUIRE THE HOLD ONCE. The hold slot's generation is the acquire -// count: one acquire and one release leave head 2. The same N commits through local_store_commit -// (the per-commit bracket) leave head 2N. That second measurement is the RED: if the session were -// secretly the per-commit bracket, both heads would match and this claim would fail. Lookups run -// under the same session and do not take a second hold. Dummy files in the root make a per-commit -// whole-root List O(store size); the session still finishes with head 2. +// N COMMITS UNDER ONE SESSION ACQUIRE THE HOLD ONCE. Open already takes and releases the family +// hold (the sweep), so the discriminator is the DELTA after open: one session acquire+release +// advances the slot by 2; N per-commit brackets advance it by 2N. If the session were secretly +// the per-commit path, the two deltas would match and this claim would fail. Lookups run under +// the same session. Dummy names in the root make a whole-root List grow with store size; they +// do not change the hold generation. test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution() -> Bool { let n = 6 let session_root_path = fresh_root_path() @@ -1369,6 +1369,7 @@ test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution let session_ok = match opened(root: session_root) { Absent => false Present { value: store } => { + let before = hold_slot_head_generation(root_path: session_root_path) let planted = fold([1, 2, 3, 4, 5, 6, 7, 8], init: true, f: (ok, i) => ok && Filesystem.WriteCreateNew(path: store_path(root_path: session_root_path, name: concat("dummy-", i as String)), content: "x").success ) @@ -1378,15 +1379,17 @@ test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution writes |> all(c => session_commit_ok(c: c)) && hits |> all(x => x) }) { LocalStoreBracketRefused { refusal: _ } => false - LocalStoreBracketRan { value: body_ok, hold_release: _ } => planted && body_ok && hold_slot_head_generation(root_path: session_root_path) == 2 + LocalStoreBracketRan { value: body_ok, hold_release: _ } => + planted && body_ok && (hold_slot_head_generation(root_path: session_root_path) - before) == 2 } } } let bracket_ok = match opened(root: bracket_root) { Absent => false Present { value: store } => { + let before = hold_slot_head_generation(root_path: bracket_root_path) let writes = [1, 2, 3, 4, 5, 6] |> map(i => local_store_commit(capability: store, req: bundle_req(module: concat("b", i as String)), payloads: verdicts(text: "b pass"), parts: [])) - writes |> all(c => session_commit_ok(c: c)) && hold_slot_head_generation(root_path: bracket_root_path) == (2 * n) + writes |> all(c => session_commit_ok(c: c)) && (hold_slot_head_generation(root_path: bracket_root_path) - before) == (2 * n) } } let removed_s = shell.Remove.RecursiveForce(path: session_root_path) From b55f08d7464faf15c2c18b863080a974f6aed0e3 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 05:11:09 +0000 Subject: [PATCH 03/30] Read hold-slot head from one root listing, like the other wet controls. Probing generations with per-file Reads was a second observation route and showed up as host_effect_refused / route-gap on the changed-witness wet lane. The store's own listing fold already runs in this file. Co-authored-by: Cursor --- ...alization_store_local_wet_witness_test.dag | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 17a7a55a5ed..02e27e62588 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -45,7 +45,7 @@ import extdeps.realization.materialization_store_local { LocalStoreHolderUnobservable, LocalStoreHoldRecoveryRefused, LocalStoreHoldContended, LocalStoreHoldStoreUnavailable, LocalStoreHoldSlotUndecodable, LocalStoreHoldKeyNotAddressable, LocalStoreHoldSelfUnidentified, LocalStoreHoldWindowTooNarrow, LocalStoreHoldUnsettled, LocalStoreCatalogMismatch, LocalStoreCatalogInadmissible, LocalStoreSweepIncomplete, LocalStoreUnavailableCause, - local_store_index_window, + local_store_index_window, local_store_digits_int, LocalStoreRecordDeleteAttempt, LocalStoreRecordGone, LocalStoreRecordFileRefused, LocalStoreRecordPartsRefused, LocalStoreRecordPartsUnlisted, LocalStoreStagedPart, LocalStorePartsLookup, local_store_lookup_parts, local_store_part_name, } import gunbc.typecheck_module_store { @@ -1344,10 +1344,21 @@ test fn a_leftover_copy_that_cannot_be_removed_leaves_the_store_unwritable_by_re } fn hold_slot_head_generation(root_path: String) -> Int { - let key = local_store_hold_key(family: native_module_verdict_bundle_artifact_kind) as String - fold([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24], init: 0, f: (best, n) => - if Filesystem.Read(path: store_path(root_path: root_path, name: concat(concat(key, "."), n as String))).success { n } else { best } - ) + let prefix = concat(local_store_hold_key(family: native_module_verdict_bundle_artifact_kind) as String, ".") + let listing = Filesystem.List(path: root_path) + match filesystem_listing_observation(directory: root_path, success: listing.success, entries: listing.entries, error: listing.error) { + FilesystemDirectoryListed(listed) => + fold(filesystem_listing_entry_names(listing: listed), init: 0, f: (best, n) => + if !starts_with(s: n, prefix: prefix) { best } else { + match local_store_digits_int(s: substring(s: n, start: string_length(s: prefix), end: string_length(s: n))) { + Absent => best + Present { value: g } => if g > best { g } else { best } + } + } + ) + FilesystemDirectoryListingRefused { directory: _, error: _ } => 0 + FilesystemDirectorySubjectRefused { directory: _, cause: _ } => 0 + } } fn session_commit_ok(c: LocalStoreCommit) -> Bool { From bfbcfe316420501944ea75f88d83b873ab9428c4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 06:58:51 +0000 Subject: [PATCH 04/30] Enroll the held-session wet claim as a DirWithTemplate hermetic gap. Changed-witness admission treated the unenrolled Mktemp as route-gap-before-verdict and never joined the local-repo wet terminal. The rest of this file's wet identities are already that triple: gap row, wet schedule, LocalRepoWetLane exclusion. Co-authored-by: Cursor --- src/v2/workflow/floor_route_gap.dag | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index df6a8b6f0fb..5c6fa11626c 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1149,6 +1149,8 @@ fn floor_route_gap_expectation_chunk_09() -> List { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_refused_part_publish_leaves_no_temporary_copy_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_leftover_copy_that_cannot_be_removed_leaves_the_store_unwritable_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Empty {} } } @@ -1180,6 +1182,7 @@ fn floor_route_gap_expectation_chunk_09() -> List { } } } + } } } } From f209e9f8680c7638e26093f294c40dc293d2fa67 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 08:57:51 +0000 Subject: [PATCH 05/30] Delete the listing-based windowed hold observe/release this PR replaced. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check and release already go through the named-generation ops; leaving the listing twins in tree left two answers for one question. Named observe also dropped the unused window parameter — the two-file verify does not read it. Acquire still lists, because a reclaimed prefix is not a presence sequence from gen 1. Co-authored-by: Cursor --- .../materialization_store_local.dag | 2 +- dag/gunbc/durable_cas_file_store.dag | 3 +-- .../durable_exclusive_hold_file_store.dag | 24 +++---------------- ...lization_store_local_seal_witness_test.dag | 2 +- ...alization_store_local_wet_witness_test.dag | 9 +++---- 5 files changed, 9 insertions(+), 31 deletions(-) diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag index 95246684322..68ae22cfdc3 100644 --- a/dag/extdeps/realization/materialization_store_local.dag +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -1208,7 +1208,7 @@ fn local_store_hold_check(held: LocalStoreFamilyHeld) -> LocalStoreHoldCheck decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_commit_verified"), ] { - let observed = observe_file_hold_windowed_at(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, key: local_store_hold_key(family: held.family), window: held.window, generation: held.generation) + let observed = observe_file_hold_windowed_at(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, key: local_store_hold_key(family: held.family), generation: held.generation) match observed { HoldSlotHeld { generation: g, owner: o } => if cas_generation_count(g: g) == cas_generation_count(g: held.generation) && durable_hold_owner_equal(left: o, right: held.owner) { LocalStoreHoldCurrent } else { LocalStoreHoldNotCurrent { observed: observed } } diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index 09a66bcea69..ff6de2f35f1 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -717,12 +717,11 @@ fn cas_verify_named_window_head(root: NonEmptyStr, key: NonEmptyStr, generation: } } -fn observe_cas_slot_state_windowed_at(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationWindow, generation: CasGeneration) -> CasSlotObservation +fn observe_cas_slot_state_windowed_at(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> CasSlotObservation admit_callers: [ decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "observe_file_hold_windowed_at"), ] { - let _window = window if !cas_key_is_slot_addressable(key: key) { CasObservedUnreadable { cause: CasUnreadableReadRefused { detail: cas_key_not_slot_addressable_detail } } } else { diff --git a/dag/gunbc/durable_exclusive_hold_file_store.dag b/dag/gunbc/durable_exclusive_hold_file_store.dag index f31697e15be..0c473aba462 100644 --- a/dag/gunbc/durable_exclusive_hold_file_store.dag +++ b/dag/gunbc/durable_exclusive_hold_file_store.dag @@ -143,7 +143,6 @@ fn hold_observation_of_cas(observed: CasSlotObservation) -> Durable fn observe_file_hold_retained(root: NonEmptyStr, key: NonEmptyStr, retention: CasSlotRetention) -> DurableHoldObservation admit_callers: [ decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "observe_file_hold"), - decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "observe_file_hold_windowed"), decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_acquire_prepare_retained"), decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess_retained"), decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_recovery_assess_retained"), @@ -156,21 +155,13 @@ fn observe_file_hold(root: NonEmptyStr, key: NonEmptyStr) -> DurableHoldObservat observe_file_hold_retained(root: root, key: key, retention: KeepAllGenerations) } -fn observe_file_hold_windowed(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationWindow) -> DurableHoldObservation - admit_callers: [ - decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_hold_check"), - ] -{ - observe_file_hold_retained(root: root, key: key, retention: KeepGenerationWindow { window: window }) -} - -fn observe_file_hold_windowed_at(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationWindow, generation: CasGeneration) -> DurableHoldObservation +fn observe_file_hold_windowed_at(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> DurableHoldObservation admit_callers: [ decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_hold_check"), decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess_windowed_at"), ] { - hold_observation_of_cas(observed: observe_cas_slot_state_windowed_at(root: root, key: key, window: window, generation: generation)) + hold_observation_of_cas(observed: observe_cas_slot_state_windowed_at(root: root, key: key, generation: generation)) } // EVERY EFFECTFUL HOLD WRITE IS LIVE READ -> SEALED PLAN -> CAS COMMIT, AND NO @@ -350,7 +341,6 @@ fn file_hold_release_assess_retained( ) -> FileHoldReleaseAssessment admit_callers: [ decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess"), - decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess_windowed"), ] { match durable_hold_release_assess( @@ -377,21 +367,13 @@ fn file_hold_release_assess(root: NonEmptyStr, slot_key: NonEmptyStr, owner: Dur file_hold_release_assess_retained(root: root, slot_key: slot_key, owner: owner, acquired_generation: acquired_generation, released_by: released_by, retention: KeepAllGenerations) } -fn file_hold_release_assess_windowed(root: NonEmptyStr, slot_key: NonEmptyStr, owner: DurableHoldOwnerRef, acquired_generation: CasGeneration, released_by: DurableHoldReleaseRef, window: CasGenerationWindow) -> FileHoldReleaseAssessment - admit_callers: [ - decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_release_family_hold"), - ] -{ - file_hold_release_assess_retained(root: root, slot_key: slot_key, owner: owner, acquired_generation: acquired_generation, released_by: released_by, retention: KeepGenerationWindow { window: window }) -} - fn file_hold_release_assess_windowed_at(root: NonEmptyStr, slot_key: NonEmptyStr, owner: DurableHoldOwnerRef, acquired_generation: CasGeneration, released_by: DurableHoldReleaseRef, window: CasGenerationWindow) -> FileHoldReleaseAssessment admit_callers: [ decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_release_family_hold"), ] { match durable_hold_release_assess( - observed: observe_file_hold_windowed_at(root: root, key: slot_key, window: window, generation: acquired_generation), + observed: observe_file_hold_windowed_at(root: root, key: slot_key, generation: acquired_generation), owner: owner, acquired_generation: acquired_generation, ) { diff --git a/dag/test/claim/materialization_store_local_seal_witness_test.dag b/dag/test/claim/materialization_store_local_seal_witness_test.dag index 3f49969d04e..7ae335b501b 100644 --- a/dag/test/claim/materialization_store_local_seal_witness_test.dag +++ b/dag/test/claim/materialization_store_local_seal_witness_test.dag @@ -145,7 +145,7 @@ test fn a_direct_raw_reclaim_call_is_an_admission_refusal() -> Bool { data direct_held_session_source: String = "module probe_direct_held_session\nimport std.cache_identity { ArtifactKindId }\nimport extdeps.realization.materialization_store_local { LocalStoreCapability, LocalStoreFamilyHeld, LocalStoreBracket, local_store_held_session }\nfn direct(c: LocalStoreCapability, f: ArtifactKindId) -> LocalStoreBracket {\n local_store_held_session(capability: c, family: f, run: h => h)\n}\n" -data direct_named_window_observe_source: String = "module probe_direct_named_window_observe\nimport std.types { NonEmptyStr }\nimport std.durable_compare_and_set { CasGeneration, CasSlotObservation }\nimport gunbc.durable_cas_file_store { CasGenerationWindow, observe_cas_slot_state_windowed_at }\nfn direct(r: NonEmptyStr, k: NonEmptyStr, w: CasGenerationWindow, g: CasGeneration) -> CasSlotObservation {\n observe_cas_slot_state_windowed_at(root: r, key: k, window: w, generation: g)\n}\n" +data direct_named_window_observe_source: String = "module probe_direct_named_window_observe\nimport std.types { NonEmptyStr }\nimport std.durable_compare_and_set { CasGeneration, CasSlotObservation }\nimport gunbc.durable_cas_file_store { observe_cas_slot_state_windowed_at }\nfn direct(r: NonEmptyStr, k: NonEmptyStr, g: CasGeneration) -> CasSlotObservation {\n observe_cas_slot_state_windowed_at(root: r, key: k, generation: g)\n}\n" test fn a_direct_held_session_call_is_an_admission_refusal() -> Bool { admission_refusals(source: direct_held_session_source, callee: "local_store_held_session") == 1 diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 02e27e62588..31720e52e9a 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -1369,8 +1369,8 @@ fn session_commit_ok(c: LocalStoreCommit) -> Bool { // hold (the sweep), so the discriminator is the DELTA after open: one session acquire+release // advances the slot by 2; N per-commit brackets advance it by 2N. If the session were secretly // the per-commit path, the two deltas would match and this claim would fail. Lookups run under -// the same session. Dummy names in the root make a whole-root List grow with store size; they -// do not change the hold generation. +// the same session. This measures hold-acquire count, not root-List count: occupancy writes still +// list the store. test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution() -> Bool { let n = 6 let session_root_path = fresh_root_path() @@ -1381,9 +1381,6 @@ test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution Absent => false Present { value: store } => { let before = hold_slot_head_generation(root_path: session_root_path) - let planted = fold([1, 2, 3, 4, 5, 6, 7, 8], init: true, f: (ok, i) => - ok && Filesystem.WriteCreateNew(path: store_path(root_path: session_root_path, name: concat("dummy-", i as String)), content: "x").success - ) match typecheck_module_store_session(capability: store, family: native_module_verdict_bundle_artifact_kind, run: h => { let writes = [1, 2, 3, 4, 5, 6] |> map(i => typecheck_module_store_commit_under(held: h, req: bundle_req(module: concat("s", i as String)), payloads: verdicts(text: "s pass"), parts: [])) let hits = [1, 2, 3, 4, 5, 6] |> map(i => store_lookup_is_hit(l: typecheck_module_store_lookup(capability: store, req: bundle_req(module: concat("s", i as String))))) @@ -1391,7 +1388,7 @@ test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution }) { LocalStoreBracketRefused { refusal: _ } => false LocalStoreBracketRan { value: body_ok, hold_release: _ } => - planted && body_ok && (hold_slot_head_generation(root_path: session_root_path) - before) == 2 + body_ok && (hold_slot_head_generation(root_path: session_root_path) - before) == 2 } } } From 94f20a7f352e9a5d76dfad3878733d851c2bf7ec Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 09:25:48 +0000 Subject: [PATCH 06/30] Bound windowed slot observation so a successor-present retry cannot run forever. cas_observe_window recursed on listed-max G when G+1 still read present; under a TCO lowering that does not increment attempt that is the hang after hold-typed_module.1507/.1508. Fold at most eight list-and-verify passes instead. Co-authored-by: Cursor --- dag/gunbc/durable_cas_file_store.dag | 74 +++++++++++-------- ...urable_cas_file_store_wet_witness_test.dag | 31 ++++++++ src/v2/workflow/floor_route_gap.dag | 3 + src/v2/workflow/local_repo_wet_terminal.dag | 6 ++ 4 files changed, 85 insertions(+), 29 deletions(-) diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index ff6de2f35f1..f0cc31e30a5 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -491,7 +491,7 @@ fn cas_key_generation_of_name(key: NonEmptyStr, name: String) -> CasGeneration? fn cas_window_generations(root: NonEmptyStr, key: NonEmptyStr) -> CasWindowGenerations admit_callers: [ decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_settle_slot"), - decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "cas_observe_window"), + decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "cas_observe_window_once"), decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "cas_reclaim_below"), ] { @@ -516,37 +516,52 @@ fn cas_window_max_generation(generations: List) -> CasGeneration? }) } -fn cas_observe_window(root: NonEmptyStr, key: NonEmptyStr, attempt: Int) -> CasSlotProbe +data cas_window_observation_attempts: List = [1, 2, 3, 4, 5, 6, 7, 8] + +// ONE LIST-AND-VERIFY PASS. Recursion here was the unbounded arm: a window whose listed max is +// strictly below a generation that still reads present (stale listing, or a TCO lowering that does +// not increment `attempt`) re-entered cas_observe_window forever, re-reading G and G+1 with no +// further List. A fold over a finite attempt roster cannot fail to stop. +fn cas_observe_window_once(root: NonEmptyStr, key: NonEmptyStr) -> CasSlotProbe? admit_callers: [ - decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "cas_observe_retained_slot"), decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "cas_observe_window"), ] { - if attempt > cas_window_observation_attempt_bound { - ProbedWindowHeadUnsettled { attempts: cas_window_observation_attempt_bound } - } else { - match cas_window_generations(root: root, key: key) { - CasWindowGenerationsUnlisted { cause: c } => ProbedAbsenceUnestablished { cause: c } - CasWindowGenerationsListed { generations: gs } => - match cas_window_max_generation(generations: gs) { - Absent => ProbedAbsent - Present { value: head } => - match cas_read_generation(root: root, key: key, generation: head) { - GenerationRefused { probe: p } => p - GenerationAbsent => cas_observe_window(root: root, key: key, attempt: attempt + 1) - GenerationPresent { content: c } => - match cas_generation_successor(g: head) { - CasSuccessorExhausted { head: _ } => ProbedHead { generation: head, value: c } - CasSuccessorGeneration { generation: next } => - match cas_read_generation(root: root, key: key, generation: next) { - GenerationRefused { probe: p } => p - GenerationAbsent => ProbedHead { generation: head, value: c } - GenerationPresent { content: _ } => cas_observe_window(root: root, key: key, attempt: attempt + 1) - } - } - } - } - } + match cas_window_generations(root: root, key: key) { + CasWindowGenerationsUnlisted { cause: c } => Present { value: ProbedAbsenceUnestablished { cause: c } } + CasWindowGenerationsListed { generations: gs } => + match cas_window_max_generation(generations: gs) { + Absent => Present { value: ProbedAbsent } + Present { value: head } => + match cas_read_generation(root: root, key: key, generation: head) { + GenerationRefused { probe: p } => Present { value: p } + GenerationAbsent => none + GenerationPresent { content: c } => + match cas_generation_successor(g: head) { + CasSuccessorExhausted { head: _ } => Present { value: ProbedHead { generation: head, value: c } } + CasSuccessorGeneration { generation: next } => + match cas_read_generation(root: root, key: key, generation: next) { + GenerationRefused { probe: p } => Present { value: p } + GenerationAbsent => Present { value: ProbedHead { generation: head, value: c } } + GenerationPresent { content: _ } => none + } + } + } + } + } +} + +fn cas_observe_window(root: NonEmptyStr, key: NonEmptyStr) -> CasSlotProbe + admit_callers: [ + decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "cas_observe_retained_slot"), + ] +{ + match fold(cas_window_observation_attempts, init: none, f: (acc, _) => match acc { + Present { value: p } => Present { value: p } + Absent => cas_observe_window_once(root: root, key: key) + }) { + Present { value: p } => p + Absent => ProbedWindowHeadUnsettled { attempts: cas_window_observation_attempt_bound } } } @@ -560,7 +575,7 @@ fn cas_observe_retained_slot(root: NonEmptyStr, key: NonEmptyStr, retention: Cas { match retention { KeepAllGenerations => cas_observe_slot(root: root, key: key) - KeepGenerationWindow { window: _ } => cas_observe_window(root: root, key: key, attempt: 1) + KeepGenerationWindow { window: _ } => cas_observe_window(root: root, key: key) } } @@ -677,6 +692,7 @@ fn observe_cas_slot_state_windowed(root: NonEmptyStr, key: NonEmptyStr, window: admit_callers: [ decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_read_index"), decl_ref(module_path: "test.claim.durable_cas_file_store_wet_witness", decl_name: "a_window_slot_holds_at_most_k_plus_one_generations_by_real_execution"), + decl_ref(module_path: "test.claim.durable_cas_file_store_wet_witness", decl_name: "a_window_observation_at_generation_1500_terminates_by_real_execution"), ] { observe_cas_slot_state_retained(root: root, key: key, retention: KeepGenerationWindow { window: window }) diff --git a/dag/test/claim/durable_cas_file_store_wet_witness_test.dag b/dag/test/claim/durable_cas_file_store_wet_witness_test.dag index b0e73e9924d..3fffc5be42a 100644 --- a/dag/test/claim/durable_cas_file_store_wet_witness_test.dag +++ b/dag/test/claim/durable_cas_file_store_wet_witness_test.dag @@ -9,6 +9,7 @@ import std.durable_compare_and_set { CasOutcome, CasCommitted, CasPreconditionFailed, CasStoreRefused, CasReadableAbsent, CasReadablePresent, CasObservedReadable, CasObservedUnreadable, CasGenerationPublicationRefused, + CasGeneration, cas_generation_count, } import std.checked_arithmetic { int_inclusive_max } @@ -291,6 +292,36 @@ test fn a_racing_writer_under_a_window_retries_and_loses_no_commit_by_real_execu && removed.success } +fn plant_generation(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration, payload: String) -> Bool { + Filesystem.WriteCreateNew(path: cas_file_slot_path(root: root, key: key, generation: generation), content: payload).success +} + +// HIGH HEAD, SUCCESSOR PRESENT ON DISK. The recursive cas_observe_window retry (listed max G, +// named G+1 present) did not bound its attempts under a TCO lowering that left `attempt` unchanged, +// so one observation at hold-typed_module.1507/.1508 never returned. A fold over eight attempts +// must. The fixture is two files at 1500 and 1501: if listing reports both, the head is 1501; if +// listing omits the true max (the 757-file trace), G+1 still reads present and the fold answers +// Unsettled rather than looping. +test fn a_window_observation_at_generation_1500_terminates_by_real_execution() -> Bool { + let root_path = fresh_root() + let root = root_path as NonEmptyStr + let w = window_of(k: 2) + let planted = plant_generation(root: root, key: "hold-typed_module", generation: 1500, payload: "g1500") + && plant_generation(root: root, key: "hold-typed_module", generation: 1501, payload: "g1501") + let observed = match w { + Absent => false + Present { value: win } => match observe_cas_slot_state_windowed(root: root, key: "hold-typed_module", window: win) { + CasObservedReadable { readable: r } => match r { + CasReadablePresent { version: v } => v.value == "g1501" && cas_generation_count(g: v.generation) == 1501 + CasReadableAbsent => false + } + CasObservedUnreadable { cause: _ } => true + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + planted && observed && removed.success +} + // The window's floor is construction: k < 2 has no admitted window. test fn a_window_narrower_than_two_is_not_admitted() -> Bool { match cas_generation_window(k: 1) { diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 5c6fa11626c..715b9673f65 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1295,6 +1295,8 @@ fn floor_route_gap_expectation_chunk_12() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.durable_cas_file_store_wet_witness.a_racing_writer_under_a_window_retries_and_loses_no_commit_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.durable_cas_file_store_wet_witness.a_window_observation_at_generation_1500_terminates_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.filesystem_link_create_new_wet_witness.a_link_publishes_the_source_bytes_and_keeps_the_source_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.filesystem_link_create_new_wet_witness.an_existing_target_refuses_as_occupied_and_is_unchanged_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, @@ -1310,6 +1312,7 @@ fn floor_route_gap_expectation_chunk_12() -> List { } } } + } } } } diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index b025d33af2a..037825b4bc5 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -1843,6 +1843,12 @@ fn local_repo_wet_schedule() -> List { function: "a_racing_writer_under_a_window_retries_and_loses_no_commit_by_real_execution", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.durable_cas_file_store_wet_witness", function: "a_window_observation_at_generation_1500_terminates_by_real_execution" }, + entry: "dag/test/claim/durable_cas_file_store_wet_witness_test.dag", + function: "a_window_observation_at_generation_1500_terminates_by_real_execution", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.filesystem_link_create_new_wet_witness", function: "a_link_publishes_the_source_bytes_and_keeps_the_source_by_real_execution" }, entry: "dag/test/claim/filesystem_link_create_new_wet_witness_test.dag", From 32a76a07d1c31a246f93eb03171d31cce57f78e2 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 11:32:10 +0000 Subject: [PATCH 07/30] Stop windowed reclaim at the first failed generation so it cannot punch a hole. Independent deletes could remove G+1 while G survived, which made named window verify report G as head. Reclaim now walks eligible generations in ascending order and leaves later ones in place after a host refusal. Co-authored-by: Cursor --- dag/gunbc/durable_cas_file_store.dag | 51 ++++++++++++++++--- ...urable_cas_file_store_wet_witness_test.dag | 32 +++++++++++- src/v2/workflow/floor_route_gap.dag | 3 ++ src/v2/workflow/local_repo_wet_terminal.dag | 6 +++ 4 files changed, 84 insertions(+), 8 deletions(-) diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index f0cc31e30a5..6ab28d269db 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -516,6 +516,13 @@ fn cas_window_max_generation(generations: List) -> CasGeneration? }) } +fn cas_window_min_generation(generations: List) -> CasGeneration? { + fold(generations, init: none, f: (best, g) => match best { + Absent => Present { value: g } + Present { value: b } => if cas_generation_count(g: g) < cas_generation_count(g: b) { Present { value: g } } else { best } + }) +} + data cas_window_observation_attempts: List = [1, 2, 3, 4, 5, 6, 7, 8] // ONE LIST-AND-VERIFY PASS. Recursion here was the unbounded arm: a window whose listed max is @@ -1098,23 +1105,53 @@ type CasFileWrite { reclamation: CasWindowReclamation } +type CasReclaimProgress { + remaining: List + removed: List + failed: List + stopped: Bool +} + +// ASCENDING, AND STOP AT THE FIRST HOST REFUSAL. Independent deletes of every generation at or +// below the floor could remove G+1 while G survived, which punches a hole: named window verify +// (G present and G+1 absent) would then report G as head while a later generation was live. +// Deleting in generation order and leaving every later eligible generation in place when one +// delete fails keeps the surviving prefix contiguous, so that two-read proof cannot mint a +// stale head. fn cas_reclaim_below(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationWindow, head: CasGeneration) -> CasWindowReclamation admit_callers: [ decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "file_compare_and_set_retained"), decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_settle_slot"), + decl_ref(module_path: "test.claim.durable_cas_file_store_wet_witness", decl_name: "reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution"), ] { match cas_window_generations(root: root, key: key) { CasWindowGenerationsUnlisted { cause: c } => CasReclamationUnlisted { cause: c } CasWindowGenerationsListed { generations: gs } => { let floor = cas_generation_count(g: head) - window.k - let attempts = gs |> filter(g => cas_generation_count(g: g) <= floor) |> map(g => { - let d = Filesystem.Delete(path: cas_file_slot_path(root: root, key: key, generation: g)) - CasGenerationDeleteAttempt { generation: g, success: d.success, error: d.error } - }) - let removed = attempts |> filter(t => t.success) |> map(t => t.generation) - let failed = attempts |> filter(t => !t.success) |> map(t => CasGenerationReclaimFailure { generation: t.generation, host_error: t.error }) - if count(failed) == 0 { CasReclaimed { removed: removed } } else { CasReclamationIncomplete { removed: removed, failed: failed } } + let eligible = gs |> filter(g => cas_generation_count(g: g) <= floor) + let walk = fold(eligible, init: CasReclaimProgress { remaining: eligible, removed: [], failed: [], stopped: false }, f: (acc, _) => + if acc.stopped { acc } else { + match cas_window_min_generation(generations: acc.remaining) { + Absent => acc + Present { value: g } => { + let rest = acc.remaining |> filter(x => cas_generation_count(g: x) != cas_generation_count(g: g)) + let d = Filesystem.Delete(path: cas_file_slot_path(root: root, key: key, generation: g)) + if d.success { + CasReclaimProgress { remaining: rest, removed: acc.removed |> list_push(g), failed: acc.failed, stopped: false } + } else { + CasReclaimProgress { + remaining: rest, + removed: acc.removed, + failed: [CasGenerationReclaimFailure { generation: g, host_error: d.error }], + stopped: true, + } + } + } + } + } + ) + if count(walk.failed) == 0 { CasReclaimed { removed: walk.removed } } else { CasReclamationIncomplete { removed: walk.removed, failed: walk.failed } } } } } diff --git a/dag/test/claim/durable_cas_file_store_wet_witness_test.dag b/dag/test/claim/durable_cas_file_store_wet_witness_test.dag index 3fffc5be42a..39ae958cff5 100644 --- a/dag/test/claim/durable_cas_file_store_wet_witness_test.dag +++ b/dag/test/claim/durable_cas_file_store_wet_witness_test.dag @@ -18,7 +18,8 @@ import gunbc.durable_cas_file_store { admit_cas_attempt, file_compare_and_set, DefaultAccessCreateOnly, observe_cas_slot_state, cas_file_slot_path, file_compare_and_set_windowed, observe_cas_slot_state_windowed, CasGenerationWindow, cas_generation_window, CasGenerationWindowAdmitted, CasGenerationWindowTooNarrow, - CasSlotProbe, ProbedHead, cas_probe_gallop, + CasSlotProbe, ProbedHead, cas_probe_gallop, cas_reclaim_below, + CasReclaimed, CasReclamationIncomplete, CasReclamationUnlisted, CasReclamationNotDeclared, CasReclamationNotCommitted, } // Wet controls for the file-backed CAS provider. Hermetic mint and classification @@ -322,6 +323,35 @@ test fn a_window_observation_at_generation_1500_terminates_by_real_execution() - planted && observed && removed.success } +// A DELETE THAT FAILS MUST NOT PUNCH A HOLE. Independent reclaim of every generation at or below +// the floor could remove G+1 while G stayed, so named window verify would report G as head. This +// plants G as a nonempty directory (unlink refuses) and G+1 as a file, then reclaims at head 4 +// with k=2. The later eligible generation must still read present. +test fn reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution() -> Bool { + let root_path = fresh_root() + let root = root_path as NonEmptyStr + let key = "hold-typed_module" as NonEmptyStr + let path1 = cas_file_slot_path(root: root, key: key, generation: 1) + let planted = shell.Mkdir.NewOwnerOnly(path: path1).success + && Filesystem.WriteCreateNew(path: concat(path1, "/stuck"), content: "x").success + && plant_generation(root: root, key: key, generation: 2, payload: "g2") + && plant_generation(root: root, key: key, generation: 3, payload: "g3") + && plant_generation(root: root, key: key, generation: 4, payload: "g4") + let reclaimed = match window_of(k: 2) { + Absent => false + Present { value: w } => match cas_reclaim_below(root: root, key: key, window: w, head: 4) { + CasReclamationIncomplete { removed: _, failed: f } => count(f) == 1 + CasReclaimed { removed: _ } => false + CasReclamationUnlisted { cause: _ } => false + CasReclamationNotDeclared => false + CasReclamationNotCommitted => false + } + } + let successor_still_there = Filesystem.Read(path: cas_file_slot_path(root: root, key: key, generation: 2)).success + let removed = shell.Remove.RecursiveForce(path: root_path) + planted && reclaimed && successor_still_there && removed.success +} + // The window's floor is construction: k < 2 has no admitted window. test fn a_window_narrower_than_two_is_not_admitted() -> Bool { match cas_generation_window(k: 1) { diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 715b9673f65..848a9c57392 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1297,6 +1297,8 @@ fn floor_route_gap_expectation_chunk_12() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.durable_cas_file_store_wet_witness.a_window_observation_at_generation_1500_terminates_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.durable_cas_file_store_wet_witness.reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.filesystem_link_create_new_wet_witness.a_link_publishes_the_source_bytes_and_keeps_the_source_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.filesystem_link_create_new_wet_witness.an_existing_target_refuses_as_occupied_and_is_unchanged_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, @@ -1313,6 +1315,7 @@ fn floor_route_gap_expectation_chunk_12() -> List { } } } + } } } } diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 037825b4bc5..579caa283c6 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -1849,6 +1849,12 @@ fn local_repo_wet_schedule() -> List { function: "a_window_observation_at_generation_1500_terminates_by_real_execution", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.durable_cas_file_store_wet_witness", function: "reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution" }, + entry: "dag/test/claim/durable_cas_file_store_wet_witness_test.dag", + function: "reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.filesystem_link_create_new_wet_witness", function: "a_link_publishes_the_source_bytes_and_keeps_the_source_by_real_execution" }, entry: "dag/test/claim/filesystem_link_create_new_wet_witness_test.dag", From 6a4621cb041dc4833f10b2863d91d88208ce2515 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 11:52:04 +0000 Subject: [PATCH 08/30] Add the leaked-hold inhabitance control for partial window reclaim. The no-holes reclaim already stops at the first failed generation. This pins that generation against unlink, runs another acquire/release, and requires reuse of the leaked hold to refuse stale with no object or index write. Co-authored-by: Cursor --- ...alization_store_local_wet_witness_test.dag | 40 +++++++++++++++++++ src/v2/workflow/floor_route_gap.dag | 3 ++ src/v2/workflow/local_repo_wet_terminal.dag | 6 +++ 3 files changed, 49 insertions(+) diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 31720e52e9a..9f67fc3b4f2 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -54,6 +54,7 @@ import gunbc.typecheck_module_store { import gunbc.process_hold_identity { self_process_identity, SelfIdentified, SelfUnidentified, process_hold_text } import gunbc.build_cache_instance { ProcessIdentity } import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable } +import gunbc.durable_cas_file_store { cas_file_slot_path } import std.durable_exclusive_hold { DurableHoldOwnerRef } import std.fabric_blob { fabric_blob_reading_path } import std.cache_identity { native_module_verdict_bundle_artifact_kind, typed_module_artifact_kind } @@ -708,6 +709,45 @@ test fn a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution() verdict && removed.success } +data pin_ext2_immutable_script: String = "import fcntl, os, struct, sys\nGET = 0x80086601\nSET = 0x40086602\nIMM = 0x10\npath = sys.argv[1]\non = sys.argv[2] == \"1\"\nfd = os.open(path, os.O_RDONLY)\ncur = struct.unpack(\"I\", fcntl.ioctl(fd, GET, struct.pack(\"I\", 0)))[0]\nfcntl.ioctl(fd, SET, struct.pack(\"I\", (cur | IMM) if on else (cur & ~IMM)))\nos.close(fd)\n" + +fn pin_generation_unlink(path: String, on: Bool) -> Bool { + shell.Exec.RunArgv(program: "python3", arguments: ["-c", pin_ext2_immutable_script, path, if on { "1" } else { "0" }]).exit_code == 0 +} + +// PARTIAL RECLAIM MUST NOT RE-AUTHORIZE A LEAKED HOLD. Independent deletes could remove G+1 while +// G survived, so named verify would treat leaked G as head. Pin G against unlink, run another +// acquire/release (the reclaim that used to punch that hole), then reuse G: check must refuse +// stale and write nothing. +test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let rs = bundle_req(module: "s") + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => + match local_store_with_family(capability: store, of: rs, run: h => h) { + LocalStoreBracketRefused { refusal: _ } => false + LocalStoreBracketRan { value: leaked, hold_release: _ } => { + let pinned = cas_file_slot_path(root: root_path as NonEmptyStr, key: local_store_hold_key(family: leaked.family), generation: leaked.generation) + let immutable = pin_generation_unlink(path: pinned, on: true) + let advanced = match local_store_with_family(capability: store, of: rs, run: _ => true) { + LocalStoreBracketRefused { refusal: _ } => false + LocalStoreBracketRan { value: ok, hold_release: _ } => ok + } + let c = local_store_commit_under(held: leaked, req: rs, payloads: verdicts(text: "s pass"), parts: []) + let _unlock = pin_generation_unlink(path: pinned, on: false) + immutable && advanced + && commit_tag(c: c.commit) == "occupancy_unavailable" && hold_refusal_is_stale(r: c.hold_refusal) + && !Filesystem.Read(path: object_path(root_path: root_path, req: rs)).success + && store_lookup_is_miss(l: local_store_lookup(capability: store, req: rs)) + } + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} + // A HOLDER THAT DIED HOLDING THE FAMILY IS FREED BY OBSERVATION, NEVER BY AGE. The family is held by an // owner naming this host's current boot and namespace but a pid with no process; the next commit looks // the holder up, observes it dead, recovers the hold at the generation it saw, and lands. diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 848a9c57392..43dc07b914e 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1124,6 +1124,8 @@ fn floor_route_gap_expectation_chunk_09() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.reopening_with_a_larger_budget_than_the_store_was_created_with_refuses_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_refused_orphan_delete_leaves_the_store_unwritable_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, @@ -1191,6 +1193,7 @@ fn floor_route_gap_expectation_chunk_09() -> List { } } } + } } // MegaRAC committed-artifact sha256sum: first host effect is shell.Mktemp.Dir (checksum diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index 579caa283c6..b62a3efc096 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -2119,6 +2119,12 @@ fn local_repo_wet_schedule() -> List { function: "a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution", expectation: ExpectedToHold {} }, + WetScheduledClaim { + identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution" }, + entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", + function: "a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution", + expectation: ExpectedToHold {} + }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "reopening_with_a_larger_budget_than_the_store_was_created_with_refuses_by_real_execution" }, entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", From 28c4fc815509ca0a017f64d5602ba4b0cf1490c8 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 12:20:43 +0000 Subject: [PATCH 09/30] Name the reclaim stop as the hole-freedom for named hold verify. The two-read check is sound only while reclaim cannot drop G+1 and leave G; the previous comment treated that as already true. Co-authored-by: Cursor --- dag/extdeps/realization/materialization_store_local.dag | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag index 68ae22cfdc3..7e1dca9d050 100644 --- a/dag/extdeps/realization/materialization_store_local.dag +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -1036,9 +1036,11 @@ fn local_store_write_index_settled(held: LocalStoreFamilyHeld, expected: CasExpe // // HOLD CHECK AND RELEASE READ THE SLOT BY NAME. The windowed verify step is: generation G present // and G+1 absent. LocalStoreFamilyHeld already names G, so those two reads are cas_file_slot_path -// Filesystem.Read -- O(1) in the number of objects in the store root. That preserves the existing -// argument (a head is never guessed; a successor present means the slot moved; an absent G is not -// this hold). +// Filesystem.Read -- O(1) in the number of objects in the store root. That two-read proof is sound +// only while a surviving eligible prefix cannot have holes: cas_reclaim_below deletes in generation +// order and stops at the first host refusal, so it cannot remove G+1 while G stays. A successor +// present still means the slot moved; an absent G is not this hold. Unknown-head observation +// (acquire, index) still lists, as DESIGN §3b states for a window whose generation is not named. // // ACQUIRE STILL LISTS. Windowed presence is not a prefix: generations at or below head-k are // deleted, so a named gallop from generation 1 would read NotFound and treat a live head at 100 as From 054db538e166be9a4d44eb886e4b4bd453ddc021 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 12:33:25 +0000 Subject: [PATCH 10/30] Walk windowed reclaim over one sorted eligible list. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-step min-and-filter of remaining was O(n²); DESIGN §6 requires that cost shape fixed regardless of n. Co-authored-by: Cursor --- dag/gunbc/durable_cas_file_store.dag | 40 +++++++++------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index 6ab28d269db..1dc74725f41 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -516,13 +516,6 @@ fn cas_window_max_generation(generations: List) -> CasGeneration? }) } -fn cas_window_min_generation(generations: List) -> CasGeneration? { - fold(generations, init: none, f: (best, g) => match best { - Absent => Present { value: g } - Present { value: b } => if cas_generation_count(g: g) < cas_generation_count(g: b) { Present { value: g } } else { best } - }) -} - data cas_window_observation_attempts: List = [1, 2, 3, 4, 5, 6, 7, 8] // ONE LIST-AND-VERIFY PASS. Recursion here was the unbounded arm: a window whose listed max is @@ -1106,7 +1099,6 @@ type CasFileWrite { } type CasReclaimProgress { - remaining: List removed: List failed: List stopped: Bool @@ -1115,9 +1107,8 @@ type CasReclaimProgress { // ASCENDING, AND STOP AT THE FIRST HOST REFUSAL. Independent deletes of every generation at or // below the floor could remove G+1 while G survived, which punches a hole: named window verify // (G present and G+1 absent) would then report G as head while a later generation was live. -// Deleting in generation order and leaving every later eligible generation in place when one -// delete fails keeps the surviving prefix contiguous, so that two-read proof cannot mint a -// stale head. +// One sort_by of the eligible set, then one walk that leaves every later generation in place when +// a delete fails, keeps that prefix contiguous without a min-and-filter per step. fn cas_reclaim_below(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationWindow, head: CasGeneration) -> CasWindowReclamation admit_callers: [ decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "file_compare_and_set_retained"), @@ -1129,24 +1120,17 @@ fn cas_reclaim_below(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationW CasWindowGenerationsUnlisted { cause: c } => CasReclamationUnlisted { cause: c } CasWindowGenerationsListed { generations: gs } => { let floor = cas_generation_count(g: head) - window.k - let eligible = gs |> filter(g => cas_generation_count(g: g) <= floor) - let walk = fold(eligible, init: CasReclaimProgress { remaining: eligible, removed: [], failed: [], stopped: false }, f: (acc, _) => + let ordered = gs |> filter(g => cas_generation_count(g: g) <= floor) |> sort_by(g => cas_generation_count(g: g)) + let walk = fold(ordered, init: CasReclaimProgress { removed: [], failed: [], stopped: false }, f: (acc, g) => if acc.stopped { acc } else { - match cas_window_min_generation(generations: acc.remaining) { - Absent => acc - Present { value: g } => { - let rest = acc.remaining |> filter(x => cas_generation_count(g: x) != cas_generation_count(g: g)) - let d = Filesystem.Delete(path: cas_file_slot_path(root: root, key: key, generation: g)) - if d.success { - CasReclaimProgress { remaining: rest, removed: acc.removed |> list_push(g), failed: acc.failed, stopped: false } - } else { - CasReclaimProgress { - remaining: rest, - removed: acc.removed, - failed: [CasGenerationReclaimFailure { generation: g, host_error: d.error }], - stopped: true, - } - } + let d = Filesystem.Delete(path: cas_file_slot_path(root: root, key: key, generation: g)) + if d.success { + CasReclaimProgress { removed: acc.removed |> list_push(g), failed: acc.failed, stopped: false } + } else { + CasReclaimProgress { + removed: acc.removed, + failed: [CasGenerationReclaimFailure { generation: g, host_error: d.error }], + stopped: true, } } } From 6b39172d15e858f053a08c0d771317fc0ffc9d16 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 12:36:32 +0000 Subject: [PATCH 11/30] Assert leaked-hold refusal is named-window Unsettled, not occupancy-first. commit_under re-observes the hold before any index write; the pin control must match that Unsettled cause, and a failed IMMUTABLE ioctl cannot green the claim. Co-authored-by: Cursor --- ...alization_store_local_wet_witness_test.dag | 50 +++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 9f67fc3b4f2..7319edb555e 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -55,7 +55,13 @@ import gunbc.process_hold_identity { self_process_identity, SelfIdentified, Self import gunbc.build_cache_instance { ProcessIdentity } import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable } import gunbc.durable_cas_file_store { cas_file_slot_path } -import std.durable_exclusive_hold { DurableHoldOwnerRef } +import std.durable_exclusive_hold { + DurableHoldOwnerRef, + HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, +} +import std.durable_compare_and_set { + CasUnreadableMalformed, CasUnreadableContentMissing, CasUnreadableReadRefused, CasUnreadableObservationBoundExceeded, +} import std.fabric_blob { fabric_blob_reading_path } import std.cache_identity { native_module_verdict_bundle_artifact_kind, typed_module_artifact_kind } import gunbc.materialization_store_budgets { @@ -665,6 +671,38 @@ test fn a_commit_while_the_family_is_held_refuses_and_writes_nothing_by_real_exe verdict && removed.success } +fn hold_refusal_is_named_window_unsettled(r: LocalStoreHoldRefusal?) -> Bool { + match r { + Present { value: x } => match x { + LocalStoreHoldStale { observed: o } => match o { + HoldObservationUnavailable { cause: c } => match c { + CasUnreadableReadRefused { detail: d } => + d as String == "the window head moved during 1 observations and was not settled; a head is never guessed" + CasUnreadableMalformed { detail: _ } => false + CasUnreadableContentMissing { referenced: _ } => false + CasUnreadableObservationBoundExceeded { bound: _ } => false + } + HoldSlotAbsent => false + HoldSlotFree { generation: _, released_by: _ } => false + HoldSlotHeld { generation: _, owner: _ } => false + HoldSlotUndecodable { generation: _, detail: _ } => false + } + LocalStoreFamilyBusy { holder: _, evidence: _ } => false + LocalStoreHolderUnobservable { holder: _, cause: _ } => false + LocalStoreHoldRecoveryRefused { refusal: _ } => false + LocalStoreHoldContended => false + LocalStoreHoldStoreUnavailable { cause: _ } => false + LocalStoreHoldSlotUndecodable { detail: _ } => false + LocalStoreHoldKeyNotAddressable => false + LocalStoreHoldSelfUnidentified { cause: _ } => false + LocalStoreHoldWindowTooNarrow { k: _ } => false + LocalStoreHoldUnsettled { settlement: _ } => false + LocalStoreHoldWrongFamily { held: _, requested: _ } => false + } + Absent => false + } +} + fn hold_refusal_is_stale(r: LocalStoreHoldRefusal?) -> Bool { match r { Present { value: x } => match x { @@ -717,8 +755,11 @@ fn pin_generation_unlink(path: String, on: Bool) -> Bool { // PARTIAL RECLAIM MUST NOT RE-AUTHORIZE A LEAKED HOLD. Independent deletes could remove G+1 while // G survived, so named verify would treat leaked G as head. Pin G against unlink, run another -// acquire/release (the reclaim that used to punch that hole), then reuse G: check must refuse -// stale and write nothing. +// acquire/release (the reclaim that used to punch that hole), then reuse G. commit_under observes +// the hold first: named verify must answer Unsettled (G present, G+1 still there), mapped to +// HoldStale via HoldObservationUnavailable, before any occupancy write. occupancy_unavailable on +// that path is the hold-refuse wrapper, not an occupancy fault from the pin. ioctl-set IMMUTABLE +// must succeed (CAP_LINUX_IMMUTABLE); without it this claim is false, not vacuously green. test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution() -> Bool { let root_path = fresh_root_path() let root = WitnessScratchRoot { root: root_path } @@ -738,7 +779,8 @@ test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_r let c = local_store_commit_under(held: leaked, req: rs, payloads: verdicts(text: "s pass"), parts: []) let _unlock = pin_generation_unlink(path: pinned, on: false) immutable && advanced - && commit_tag(c: c.commit) == "occupancy_unavailable" && hold_refusal_is_stale(r: c.hold_refusal) + && commit_tag(c: c.commit) == "occupancy_unavailable" + && hold_refusal_is_named_window_unsettled(r: c.hold_refusal) && !Filesystem.Read(path: object_path(root_path: root_path, req: rs)).success && store_lookup_is_miss(l: local_store_lookup(capability: store, req: rs)) } From d90105072448198bf4a60712ac8b3f762557b3b4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 12:37:59 +0000 Subject: [PATCH 12/30] Delete lookup/commit aliases on the typecheck store session. They were second names for unsealed local_store_lookup and local_store_commit_under; the seal is only local_store_held_session. Co-authored-by: Cursor --- dag/gunbc/typecheck_module_store.dag | 21 ++----------------- ...alization_store_local_wet_witness_test.dag | 6 +++--- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/dag/gunbc/typecheck_module_store.dag b/dag/gunbc/typecheck_module_store.dag index 9394092c982..8de9d70ddc4 100644 --- a/dag/gunbc/typecheck_module_store.dag +++ b/dag/gunbc/typecheck_module_store.dag @@ -5,29 +5,12 @@ module gunbc.typecheck_module_store // neat-carp-75 (C2) will drive lookups and commits through this fold. This // module is the seal's inhabitant, not a second store. -import std.types { List } import std.cache_identity { ArtifactKindId } -import std.materialization_provider { ArtifactRequest } -import std.materialization_object { ArtifactPayload, StoreLookup } import extdeps.realization.materialization_store_local { - LocalStoreCapability, LocalStoreFamilyHeld, LocalStoreBracket, LocalStoreCommit, - LocalStoreStagedPart, - local_store_held_session, local_store_commit_under, local_store_lookup, + LocalStoreCapability, LocalStoreFamilyHeld, LocalStoreBracket, + local_store_held_session, } fn typecheck_module_store_session(capability: LocalStoreCapability, family: ArtifactKindId, run: fn(LocalStoreFamilyHeld) -> T) -> LocalStoreBracket { local_store_held_session(capability: capability, family: family, run: run) } - -fn typecheck_module_store_lookup(capability: LocalStoreCapability, req: ArtifactRequest) -> StoreLookup { - local_store_lookup(capability: capability, req: req) -} - -fn typecheck_module_store_commit_under( - held: LocalStoreFamilyHeld, - req: ArtifactRequest, - payloads: List, - parts: List -) -> LocalStoreCommit { - local_store_commit_under(held: held, req: req, payloads: payloads, parts: parts) -} diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 7319edb555e..c302a97ade3 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -49,7 +49,7 @@ import extdeps.realization.materialization_store_local { LocalStoreRecordDeleteAttempt, LocalStoreRecordGone, LocalStoreRecordFileRefused, LocalStoreRecordPartsRefused, LocalStoreRecordPartsUnlisted, LocalStoreStagedPart, LocalStorePartsLookup, local_store_lookup_parts, local_store_part_name, } import gunbc.typecheck_module_store { - typecheck_module_store_session, typecheck_module_store_commit_under, typecheck_module_store_lookup, + typecheck_module_store_session, } import gunbc.process_hold_identity { self_process_identity, SelfIdentified, SelfUnidentified, process_hold_text } import gunbc.build_cache_instance { ProcessIdentity } @@ -1464,8 +1464,8 @@ test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution Present { value: store } => { let before = hold_slot_head_generation(root_path: session_root_path) match typecheck_module_store_session(capability: store, family: native_module_verdict_bundle_artifact_kind, run: h => { - let writes = [1, 2, 3, 4, 5, 6] |> map(i => typecheck_module_store_commit_under(held: h, req: bundle_req(module: concat("s", i as String)), payloads: verdicts(text: "s pass"), parts: [])) - let hits = [1, 2, 3, 4, 5, 6] |> map(i => store_lookup_is_hit(l: typecheck_module_store_lookup(capability: store, req: bundle_req(module: concat("s", i as String))))) + let writes = [1, 2, 3, 4, 5, 6] |> map(i => local_store_commit_under(held: h, req: bundle_req(module: concat("s", i as String)), payloads: verdicts(text: "s pass"), parts: [])) + let hits = [1, 2, 3, 4, 5, 6] |> map(i => store_lookup_is_hit(l: local_store_lookup(capability: store, req: bundle_req(module: concat("s", i as String))))) writes |> all(c => session_commit_ok(c: c)) && hits |> all(x => x) }) { LocalStoreBracketRefused { refusal: _ } => false From 2ecde15b2d8672ca974c145b609bdfd290194680 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 12:39:16 +0000 Subject: [PATCH 13/30] Call gen-1500 a termination regression, not the hang's red. The two-file plant is GREEN on main's recursive observe; it only asserts the fold returns. Co-authored-by: Cursor --- .../durable_cas_file_store_wet_witness_test.dag | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dag/test/claim/durable_cas_file_store_wet_witness_test.dag b/dag/test/claim/durable_cas_file_store_wet_witness_test.dag index 39ae958cff5..3a8ca38f843 100644 --- a/dag/test/claim/durable_cas_file_store_wet_witness_test.dag +++ b/dag/test/claim/durable_cas_file_store_wet_witness_test.dag @@ -297,12 +297,13 @@ fn plant_generation(root: NonEmptyStr, key: NonEmptyStr, generation: CasGenerati Filesystem.WriteCreateNew(path: cas_file_slot_path(root: root, key: key, generation: generation), content: payload).success } -// HIGH HEAD, SUCCESSOR PRESENT ON DISK. The recursive cas_observe_window retry (listed max G, -// named G+1 present) did not bound its attempts under a TCO lowering that left `attempt` unchanged, -// so one observation at hold-typed_module.1507/.1508 never returned. A fold over eight attempts -// must. The fixture is two files at 1500 and 1501: if listing reports both, the head is 1501; if -// listing omits the true max (the 757-file trace), G+1 still reads present and the fold answers -// Unsettled rather than looping. +// TERMINATION REGRESSION, NOT THE HANG'S RED. Measured against main's recursive +// cas_observe_window this two-file plant is GREEN (listed max 1501, successor 1502 absent, retry +// arm never runs): BuildBuddy invocation 492a33d7-ff9a-4fcf-8007-4fd7029aa139. It does not +// reproduce the production hang at hold-typed_module.1507/.1508. It asserts that a fold over eight +// cas_observe_window_once passes cannot fail to return. Persistent List-omits-true-max is a +// frontier measured by gunbc#13482 after this lands; ProbedWindowHeadUnsettled there is a counted +// store Unavailable, not a kill. test fn a_window_observation_at_generation_1500_terminates_by_real_execution() -> Bool { let root_path = fresh_root() let root = root_path as NonEmptyStr From a7900ee9953835fb4204ae6dd17cd99c95430ae9 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 12:40:33 +0000 Subject: [PATCH 14/30] Derive the window Unsettled attempt count from the fold roster. The Int 8 row was a second source for the same bound; the diagnostic now reports the list length. Co-authored-by: Cursor --- dag/gunbc/durable_cas_file_store.dag | 5 +---- dag/gunbc/typecheck_module_store.dag | 6 ++++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index 1dc74725f41..a5c50f39c24 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -456,9 +456,6 @@ type CasSlotRetention = KeepAllGenerations | KeepGenerationWindow { window: CasGenerationWindow } -// A window read that keeps observing a moving head stops here and refuses with the attempt count. -data cas_window_observation_attempt_bound: Int = 8 - // "" -> the generation it names; none for anything else, for zero, and for a value past the // Int maximum. The multiplication is pre-checked (std.checked_arithmetic: check before, never after). fn cas_generation_of_digits(s: String) -> CasGeneration? { @@ -561,7 +558,7 @@ fn cas_observe_window(root: NonEmptyStr, key: NonEmptyStr) -> CasSlotProbe Absent => cas_observe_window_once(root: root, key: key) }) { Present { value: p } => p - Absent => ProbedWindowHeadUnsettled { attempts: cas_window_observation_attempt_bound } + Absent => ProbedWindowHeadUnsettled { attempts: count(cas_window_observation_attempts) } } } diff --git a/dag/gunbc/typecheck_module_store.dag b/dag/gunbc/typecheck_module_store.dag index 8de9d70ddc4..5b5455286c4 100644 --- a/dag/gunbc/typecheck_module_store.dag +++ b/dag/gunbc/typecheck_module_store.dag @@ -2,8 +2,10 @@ module gunbc.typecheck_module_store // THE TYPECHECK MODULE STORE SESSION: the admitted consumer of // extdeps.realization.materialization_store_local local_store_held_session. -// neat-carp-75 (C2) will drive lookups and commits through this fold. This -// module is the seal's inhabitant, not a second store. +// Inhabitance is n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution. +// Production cutover trigger: the typecheck compile driver that still brackets each write +// through local_store_commit switches those writes to typecheck_module_store_session. +// This module is the seal's inhabitant, not a second store. import std.cache_identity { ArtifactKindId } import extdeps.realization.materialization_store_local { From 1d8f95b329b4c0402c5875807b547020d8c2734e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 13:16:35 +0000 Subject: [PATCH 15/30] Classify window exhaustion as BoundExceeded and named G+1 as head-moved. Eight-pass cas_observe_window was a refused read by sentence; named verify with G+1 present was the same Unsettled arm. The leaked-hold control now matches HoldNamedWindowHeadMoved, and the attempt bound remains count of cas_window_observation_attempts. Co-authored-by: Cursor --- .../materialization_store_local.dag | 3 ++- dag/gunbc/durable_cas_file_store.dag | 18 +++++++++++++----- .../durable_exclusive_hold_file_store.dag | 17 ++++++++++++++--- dag/gunbc/host/managed_host_unit_hold.dag | 4 +++- dag/gunbc/workspace_allocation_plan.dag | 6 +++++- dag/gunbc/workspace_allocation_supply.dag | 3 ++- ...rkspace_commissioning_admission_observe.dag | 3 ++- .../workspace_commissioning_executor_fence.dag | 3 ++- ...space_commissioning_executor_submission.dag | 3 ++- dag/gunbc/workspace_commissioning_fleet.dag | 3 ++- dag/gunbc/workspace_slot_mutation.dag | 7 ++++++- dag/std/durable_compare_and_set.dag | 10 ++++++---- dag/std/durable_exclusive_hold.dag | 10 ++++++++++ .../durable_exclusive_hold_witness_test.dag | 8 ++++++-- ...ialization_store_local_wet_witness_test.dag | 17 +++++------------ 15 files changed, 80 insertions(+), 35 deletions(-) diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag index 7e1dca9d050..d8bf6a7ebb3 100644 --- a/dag/extdeps/realization/materialization_store_local.dag +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -85,7 +85,7 @@ import gunbc.durable_cas_file_store { import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldHolderReport, DurableHoldReleaseRefusal, DurableHoldRecoveryRefusal, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, - DurableHoldObservation, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, + DurableHoldObservation, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, durable_hold_owner_equal, } import gunbc.durable_exclusive_hold_file_store { @@ -1218,6 +1218,7 @@ fn local_store_hold_check(held: LocalStoreFamilyHeld) -> LocalStoreHoldCheck HoldSlotFree { generation: _, released_by: _ } => LocalStoreHoldNotCurrent { observed: observed } HoldSlotUndecodable { generation: _, detail: _ } => LocalStoreHoldNotCurrent { observed: observed } HoldObservationUnavailable { cause: _ } => LocalStoreHoldNotCurrent { observed: observed } + HoldNamedWindowHeadMoved { generation: _ } => LocalStoreHoldNotCurrent { observed: observed } } } diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index a5c50f39c24..9e88e626766 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -5,7 +5,7 @@ import std.algebra { trim } import std.decimal { decimal_digits_only } import std.list { distinct_by_key } import std.content_hash { ContentHash, content_hash_of_value, compare_content_hash, ContentHashEqual, ContentHashDifferent, ContentHashCrossFamilyIncomparable } -import std.durable_compare_and_set { CasAttempt, cas_attempt, CasOutcome, CasCommitted, CasPreconditionFailed, CasStoreRefused, CasExpectation, ExpectSlotAbsent, ExpectSlotGeneration, CasGeneration, CasSlotVersion, CasReadableAbsent, CasReadablePresent, CasSlotObservation, CasObservedReadable, CasObservedUnreadable, CasUnreadableMalformed, CasUnreadableReadRefused, CasStoreFailure, CasSlotObservationRefused, CasGenerationPublicationRefused, cas_generation_exhausted, cas_generation_first, cas_generation_count, cas_generation_successor, CasSuccessorGeneration, CasSuccessorExhausted } +import std.durable_compare_and_set { CasAttempt, cas_attempt, CasOutcome, CasCommitted, CasPreconditionFailed, CasStoreRefused, CasExpectation, ExpectSlotAbsent, ExpectSlotGeneration, CasGeneration, CasSlotVersion, CasReadableAbsent, CasReadablePresent, CasSlotObservation, CasObservedReadable, CasObservedUnreadable, CasUnreadableMalformed, CasUnreadableReadRefused, CasUnreadableObservationBoundExceeded, CasStoreFailure, CasSlotObservationRefused, CasGenerationPublicationRefused, cas_generation_exhausted, cas_generation_first, cas_generation_count, cas_generation_successor, CasSuccessorGeneration, CasSuccessorExhausted } import std.checked_arithmetic { int_inclusive_max } import v2.std.algebra { skip } import extdeps.access.posix { FileMode, file_mode_bits } @@ -307,6 +307,7 @@ type CasSlotProbe | ProbedUnreadable { generation: CasGeneration, kind: FilesystemFailureKind } | ProbedReadKindUnrecognized { generation: CasGeneration, observed: String } | ProbedWindowHeadUnsettled { attempts: Int } + | ProbedNamedWindowHeadMoved { generation: CasGeneration } // One generation read, classified: present with its content, absent, or a typed refusal. type CasGenerationRead @@ -513,6 +514,8 @@ fn cas_window_max_generation(generations: List) -> CasGeneration? }) } +// THE ONE AUTHORITY FOR THE WINDOWED OBSERVATION BOUND: the roster is the bound. Exhaustion +// reports CasUnreadableObservationBoundExceeded { bound: count(cas_window_observation_attempts) }. data cas_window_observation_attempts: List = [1, 2, 3, 4, 5, 6, 7, 8] // ONE LIST-AND-VERIFY PASS. Recursion here was the unbounded arm: a window whose listed max is @@ -577,9 +580,7 @@ fn cas_observe_retained_slot(root: NonEmptyStr, key: NonEmptyStr, retention: Cas } fn cas_window_head_unsettled(attempts: Int) -> CasUnreadableSlot { - CasUnreadableReadRefused { - detail: concat(concat("the window head moved during ", attempts as String), " observations and was not settled; a head is never guessed") as NonEmptyStr - } + CasUnreadableObservationBoundExceeded { bound: attempts } } // THE PROJECTION TAKES A HEAD, NOT A PROBE, AND THE DELETED ARM IS THE REPAIR. @@ -640,6 +641,8 @@ fn cas_probe_as_observation(probe: CasSlotProbe) -> CasSlotObservation CasObservedReadable { readable: CasReadableAbsent } ProbedAbsenceUnestablished { cause: c } => CasObservedUnreadable { cause: cas_absence_unestablished(cause: c) } ProbedWindowHeadUnsettled { attempts: a } => CasObservedUnreadable { cause: cas_window_head_unsettled(attempts: a) } + ProbedNamedWindowHeadMoved { generation: _ } => + CasObservedUnreadable { cause: CasUnreadableObservationBoundExceeded { bound: 1 } } ProbedHead { generation: h, value: v } => CasObservedReadable { readable: cas_head_as_readable(generation: h, value: v) } } @@ -712,6 +715,7 @@ fn observe_cas_slot_state_windowed(root: NonEmptyStr, key: NonEmptyStr, window: fn cas_verify_named_window_head(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> CasSlotProbe admit_callers: [ decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "observe_cas_slot_state_windowed_at"), + decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "observe_file_hold_windowed_at"), ] { match cas_read_generation(root: root, key: key, generation: generation) { @@ -724,7 +728,7 @@ fn cas_verify_named_window_head(root: NonEmptyStr, key: NonEmptyStr, generation: match cas_read_generation(root: root, key: key, generation: next) { GenerationRefused { probe: p } => p GenerationAbsent => ProbedHead { generation: generation, value: c } - GenerationPresent { content: _ } => ProbedWindowHeadUnsettled { attempts: 1 } + GenerationPresent { content: _ } => ProbedNamedWindowHeadMoved { generation: generation } } } } @@ -988,6 +992,8 @@ fn cas_owner_only_failed_publication( } ProbedWindowHeadUnsettled { attempts: a } => CasStoreRefused { cause: CasSlotObservationRefused { cause: cas_window_head_unsettled(attempts: a) } } + ProbedNamedWindowHeadMoved { generation: _ } => + CasStoreRefused { cause: CasSlotObservationRefused { cause: CasUnreadableObservationBoundExceeded { bound: 1 } } } } } @@ -1027,6 +1033,8 @@ fn cas_compare_and_set_outcome( CasStoreRefused { cause: CasSlotObservationRefused { cause: cas_absence_unestablished(cause: c) } } ProbedWindowHeadUnsettled { attempts: a } => CasStoreRefused { cause: CasSlotObservationRefused { cause: cas_window_head_unsettled(attempts: a) } } + ProbedNamedWindowHeadMoved { generation: _ } => + CasStoreRefused { cause: CasSlotObservationRefused { cause: CasUnreadableObservationBoundExceeded { bound: 1 } } } ProbedAbsent => match attempt.expected { ExpectSlotAbsent => diff --git a/dag/gunbc/durable_exclusive_hold_file_store.dag b/dag/gunbc/durable_exclusive_hold_file_store.dag index 0c473aba462..760d91326d6 100644 --- a/dag/gunbc/durable_exclusive_hold_file_store.dag +++ b/dag/gunbc/durable_exclusive_hold_file_store.dag @@ -28,7 +28,7 @@ import std.durable_compare_and_set { CasExpectation, ExpectSlotGeneration, CasCommitted, CasPreconditionFailed, CasStoreRefused, CasStoreFailure, CasSlotObservationRefused, CasSlotObservation, CasObservedReadable, CasObservedUnreadable, - CasReadableAbsent, CasReadablePresent, CasGeneration, + CasReadableAbsent, CasReadablePresent, CasGeneration, CasUnreadableReadRefused, } import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, @@ -37,6 +37,7 @@ import std.durable_exclusive_hold { HoldSlotUndecodable, HoldObservationUnavailable, DurableHoldAcquireRefusal, HoldAcquireEligible, HoldAcquireRefused, HoldAcquireOccupied, HoldAcquireSlotUndecodable, HoldAcquireObservationUnavailable, + HoldAcquireNamedWindowHeadMoved, HoldNamedWindowHeadMoved, HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseRefusal, durable_hold_acquire_assess, durable_hold_release_assess, DurableHoldHolderReport, DurableHoldRecoveryRefusal, HoldRecoveryEligible, HoldRecoveryRefused, @@ -45,7 +46,9 @@ import std.durable_exclusive_hold { import gunbc.durable_cas_file_store { KeepAllGenerations, KeepGenerationWindow, CasSlotRetention, CasGenerationWindow, DefaultAccessCreateOnly, DerivedPayloadAdmitted, DerivedPayloadKeyNotSlotAddressable, admit_cas_attempt_for_derived_payload, file_compare_and_set, file_compare_and_set_retained, - observe_cas_slot_state, observe_cas_slot_state_retained, observe_cas_slot_state_windowed_at, + observe_cas_slot_state, observe_cas_slot_state_retained, + cas_verify_named_window_head, cas_probe_as_observation, cas_key_is_slot_addressable, + cas_key_not_slot_addressable_detail, ProbedNamedWindowHeadMoved, } // THE TAGS ARE NAMED ONCE AND THE BODY IS DERIVED FROM THE TAG THAT MATCHED. @@ -161,7 +164,14 @@ fn observe_file_hold_windowed_at(root: NonEmptyStr, key: NonEmptyStr, generation decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess_windowed_at"), ] { - hold_observation_of_cas(observed: observe_cas_slot_state_windowed_at(root: root, key: key, generation: generation)) + if !cas_key_is_slot_addressable(key: key) { + HoldObservationUnavailable { cause: CasUnreadableReadRefused { detail: cas_key_not_slot_addressable_detail } } + } else { + match cas_verify_named_window_head(root: root, key: key, generation: generation) { + ProbedNamedWindowHeadMoved { generation: g } => HoldNamedWindowHeadMoved { generation: g } + p => hold_observation_of_cas(observed: cas_probe_as_observation(probe: p)) + } + } } // EVERY EFFECTFUL HOLD WRITE IS LIVE READ -> SEALED PLAN -> CAS COMMIT, AND NO @@ -289,6 +299,7 @@ fn file_hold_acquire_retained( FileHoldSlotUndecodable { slot_key: slot_key, generation: g, detail: d } HoldAcquireObservationUnavailable { cause: c } => FileHoldAcquireStoreUnavailable { slot_key: slot_key, cause: CasSlotObservationRefused { cause: c } } + HoldAcquireNamedWindowHeadMoved { generation: _ } => FileHoldAcquireLost { slot_key: slot_key } } } } diff --git a/dag/gunbc/host/managed_host_unit_hold.dag b/dag/gunbc/host/managed_host_unit_hold.dag index e249d59d7f0..ac1a1315ed3 100644 --- a/dag/gunbc/host/managed_host_unit_hold.dag +++ b/dag/gunbc/host/managed_host_unit_hold.dag @@ -11,7 +11,7 @@ import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldObservation, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, - HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, + HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, DurableHoldHolderLiveness, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, DurableHoldHolderReport, DurableHoldRecoveryRefusal, HoldRecoveryNothingHeld, HoldRecoveryHolderLive, HoldRecoveryHolderUnobservable, HoldRecoverySlotMoved, @@ -616,6 +616,7 @@ fn probe_hold_release_decision(observed: DurableHoldObservation, run_id: NonEmpt HoldSlotFree { generation: _, released_by: _ } => ProbeHoldFreeNoop HoldSlotUndecodable { generation: _, detail: d } => ProbeHoldUnobservable { cause: concat("slot undecodable: ", d as String) } HoldObservationUnavailable { cause: _ } => ProbeHoldUnobservable { cause: "the store could not be read" } + HoldNamedWindowHeadMoved { generation: _ } => ProbeHoldUnobservable { cause: "the store could not be read" } HoldSlotHeld { generation: g, owner: o } => match decode_unit_hold_owner(owner: o) { UnitHoldOwnerDecoded { owner: KvmObserverProbe { run_id: r, process: p, pid_namespace: n } } => @@ -879,6 +880,7 @@ fn maintenance_hold_release_on_store_host(executor: UnitHoldStoreExecutor) -> Pr HoldSlotFree { generation: _, released_by: b } => exit_failure(reason: join([label, ": already free (released by ", b as String, ")"], "")) HoldSlotUndecodable { generation: _, detail: d } => exit_failure(reason: join([label, ": slot undecodable: ", d as String], "")) HoldObservationUnavailable { cause: _ } => exit_failure(reason: concat(label, ": store unavailable")) + HoldNamedWindowHeadMoved { generation: _ } => exit_failure(reason: concat(label, ": store unavailable")) HoldSlotHeld { generation: g, owner: o } => match decode_unit_hold_owner(owner: o) { UnitHoldOwnerUnrecognized { raw: r } => diff --git a/dag/gunbc/workspace_allocation_plan.dag b/dag/gunbc/workspace_allocation_plan.dag index 14841a634e8..a932825a016 100644 --- a/dag/gunbc/workspace_allocation_plan.dag +++ b/dag/gunbc/workspace_allocation_plan.dag @@ -13,7 +13,7 @@ import std.types { NonEmptyStr, String, Int } import std.measure { money_amount_micro } import std.content_hash { content_hash_of_value, serialize_content_hash } import std.durable_compare_and_set { ExpectSlotAbsent, ExpectSlotGeneration, cas_generation_count, cas_generation_successor, CasSuccessorGeneration, CasSuccessorExhausted } -import std.durable_exclusive_hold { DurableHoldObservation, DurableHoldOwnerRef, HoldSlotHeld, HoldSlotAbsent, HoldSlotFree, HoldSlotUndecodable, HoldObservationUnavailable, HoldAcquireOccupied, HoldAcquireSlotUndecodable, HoldAcquireObservationUnavailable } +import std.durable_exclusive_hold { DurableHoldObservation, DurableHoldOwnerRef, HoldSlotHeld, HoldSlotAbsent, HoldSlotFree, HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldAcquireOccupied, HoldAcquireSlotUndecodable, HoldAcquireObservationUnavailable, HoldAcquireNamedWindowHeadMoved } import extdeps.languages.json.emit { JsonValue, json_object, json_kv, json_string } import product.fabric.demand { Demand } import product.fabric.identity { FabricIdentity, ExecutionAttemptKey, fabric_identity_eq } @@ -85,6 +85,7 @@ fn workspace_reservation_prepare( HoldAcquireOccupied { generation: _, holder: _ } => WorkspaceReservationPreparationRefused { reason: "selected slot is occupied; no acquisition plan issued" } HoldAcquireSlotUndecodable { generation: _, detail } => WorkspaceReservationPreparationRefused { reason: detail } HoldAcquireObservationUnavailable { cause: _ } => WorkspaceReservationPreparationRefused { reason: "allocation store observation unavailable; no acquisition plan issued" } + HoldAcquireNamedWindowHeadMoved { generation: _ } => WorkspaceReservationPreparationRefused { reason: "selected slot head moved; no acquisition plan issued" } } } MoneyRefused { refusal: _ } => WorkspaceReservationPreparationRefused { reason: "monetary reservation refused" } @@ -139,6 +140,7 @@ fn workspace_reservation_apply_under_exclusion(root: NonEmptyStr, plan: Workspac HoldSlotFree { generation: _, released_by: _ } => workspace_reservation_acquire(root: root, key: key, owner: owner, reservation: reservation) HoldSlotUndecodable { generation: _, detail: _ } => workspace_reservation_acquire(root: root, key: key, owner: owner, reservation: reservation) HoldObservationUnavailable { cause: _ } => workspace_reservation_acquire(root: root, key: key, owner: owner, reservation: reservation) + HoldNamedWindowHeadMoved { generation: _ } => workspace_reservation_acquire(root: root, key: key, owner: owner, reservation: reservation) } } } @@ -192,6 +194,7 @@ fn workspace_actor_takeover_decision(plan: WorkspaceReservationPlan, hold: Durab else { TakeoverRetain { reason: "slot is held by another reservation or incarnation" } } HoldSlotUndecodable { generation: _, detail: _ } => TakeoverRetain { reason: "slot hold is undecodable; the acquisition obligation is retained" } HoldObservationUnavailable { cause: _ } => TakeoverRetain { reason: "slot hold is unreadable; the acquisition obligation is retained" } + HoldNamedWindowHeadMoved { generation: _ } => TakeoverRetain { reason: "slot hold is unreadable; the acquisition obligation is retained" } HoldSlotAbsent => workspace_actor_takeover_unheld(prior: prior, intent: intent) HoldSlotFree { generation: _, released_by: _ } => workspace_actor_takeover_unheld(prior: prior, intent: intent) } @@ -228,6 +231,7 @@ fn workspace_reservation_recover(root: NonEmptyStr, plan: WorkspaceReservationPl HoldSlotFree { generation: _, released_by: _ } => WorkspaceReservationApplyRefused { reason: "no matching held incarnation observed; do not acquire retired intent" } HoldSlotUndecodable { generation: _, detail: _ } => WorkspaceReservationApplyRefused { reason: "no matching held incarnation observed; do not acquire retired intent" } HoldObservationUnavailable { cause: _ } => WorkspaceReservationApplyRefused { reason: "no matching held incarnation observed; do not acquire retired intent" } + HoldNamedWindowHeadMoved { generation: _ } => WorkspaceReservationApplyRefused { reason: "no matching held incarnation observed; do not acquire retired intent" } } } } diff --git a/dag/gunbc/workspace_allocation_supply.dag b/dag/gunbc/workspace_allocation_supply.dag index 0a197a47db1..5025b49ac01 100644 --- a/dag/gunbc/workspace_allocation_supply.dag +++ b/dag/gunbc/workspace_allocation_supply.dag @@ -8,7 +8,7 @@ import extdeps.shell.exec { ProcessOutputPresent, ProcessOutputAbsent, ProcessRe import gunbc.command_runner { run_shell_command_capture } import std.decl_ref { DeclarationRef } import std.durable_compare_and_set { cas_generation_count, cas_generation_successor, CasSuccessorGeneration, CasSuccessorExhausted } -import std.durable_exclusive_hold { HoldSlotAbsent, HoldSlotFree, HoldObservationUnavailable, HoldSlotHeld, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotAbsent, HoldSlotFree, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotHeld, HoldSlotUndecodable } import product.host_identity { HostIdentity } import product.fabric.identity { fabric_identity_eq } import product.fabric.provider_route { CompliancePostureRevision } @@ -102,6 +102,7 @@ fn workspace_slot_observation_refusal(root: NonEmptyStr, slot: RunnerSlotIdentit HoldSlotHeld { generation: _, owner: _ } => none HoldSlotUndecodable { generation: _, detail: _ } => none HoldObservationUnavailable { cause: _ } => none + HoldNamedWindowHeadMoved { generation: _ } => none } match next { Absent => Present { value: "allocation slot is held, unreadable, or generation exhausted" } diff --git a/dag/gunbc/workspace_commissioning_admission_observe.dag b/dag/gunbc/workspace_commissioning_admission_observe.dag index ecc4695d0a8..67b4e0613a2 100644 --- a/dag/gunbc/workspace_commissioning_admission_observe.dag +++ b/dag/gunbc/workspace_commissioning_admission_observe.dag @@ -19,7 +19,7 @@ import gunbc.workspace_initial_sanitation { workspace_observe_initial_sanitation import gunbc.runner_microvm_network { SlotNetworkSanitationSubject } import gunbc.fabric_control_plane { fabric_control_plane_store_root, fabric_cell_allocation_key } import gunbc.durable_exclusive_hold_file_store { observe_file_hold } -import std.durable_exclusive_hold { HoldSlotAbsent, HoldObservationUnavailable, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotAbsent, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable } import gunbc.runner_microvm_cell_readiness { StoreRootOwned, StoreNotRootOwned } import gunbc.runner_microvm_cell_readiness_store { read_store_ownership, OwnershipRead, OwnershipUnread } import gunbc.fleet_host_budget { HostAllocationConserves } @@ -75,6 +75,7 @@ fn workspace_commissioning_no_hold_refusal(provenance: WorkspaceCommissioningPro HoldSlotHeld { generation: _, owner: _ } => Present { value: "slot has held, released or unreadable allocation history; initial commissioning cannot reset it" } HoldSlotUndecodable { generation: _, detail: _ } => Present { value: "slot has held, released or unreadable allocation history; initial commissioning cannot reset it" } HoldObservationUnavailable { cause: _ } => Present { value: "slot has held, released or unreadable allocation history; initial commissioning cannot reset it" } + HoldNamedWindowHeadMoved { generation: _ } => Present { value: "slot has held, released or unreadable allocation history; initial commissioning cannot reset it" } } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => Present { value: "allocation history root is missing, unreadable or outside root custody" } OwnershipUnread { reason: _ } => Present { value: "allocation history root is missing, unreadable or outside root custody" } diff --git a/dag/gunbc/workspace_commissioning_executor_fence.dag b/dag/gunbc/workspace_commissioning_executor_fence.dag index 5481576869f..44fdb924f79 100644 --- a/dag/gunbc/workspace_commissioning_executor_fence.dag +++ b/dag/gunbc/workspace_commissioning_executor_fence.dag @@ -4,7 +4,7 @@ import gunbc.workspace_commissioning_start_identity { workspace_commissioning_sn import std.types { NonEmptyStr, String, Bool } import std.decl_ref { decl_ref } -import std.durable_exclusive_hold { HoldSlotHeld, HoldObservationUnavailable, HoldSlotFree, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotHeld, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotFree, HoldSlotUndecodable } import gunbc.build_cache_instance { runner_slot_instance_name } import gunbc.workspace_commissioning_location { workspace_commissioning_store_location } import gunbc.workspace_slot_mutation { workspace_slot_mutation_key, workspace_commissioning_mutation_owner } @@ -39,6 +39,7 @@ fn workspace_executor_mutation_matches(record: WorkspaceExecutorRecord) -> Bool HoldSlotFree { generation: _, released_by: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false + HoldNamedWindowHeadMoved { generation: _ } => false } } diff --git a/dag/gunbc/workspace_commissioning_executor_submission.dag b/dag/gunbc/workspace_commissioning_executor_submission.dag index ecb46d8ab9b..d3e1a763af1 100644 --- a/dag/gunbc/workspace_commissioning_executor_submission.dag +++ b/dag/gunbc/workspace_commissioning_executor_submission.dag @@ -4,7 +4,7 @@ import gunbc.workspace_commissioning_apply_readback { workspace_commissioning_co import gunbc.runner_microvm_slot_start { MicrovmInvocationWait, InvocationStillRunning, InvocationTerminated, InvocationWaitRefused, microvm_invocation_observe, microvm_unit_start_observed, MicrovmSlotStartRun, SlotStartCompleted, SlotStartWrongHost, SlotStartRefused, SlotStartAlreadyActive, SlotStartInvocationUnread, SlotStartInvocationUnchanged, SlotStartStillActiveAtBound } import std.types { String, NonEmptyStr, Bool } import std.decl_ref { decl_ref } -import std.durable_exclusive_hold { HoldSlotHeld, HoldSlotAbsent, HoldObservationUnavailable, HoldSlotFree, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotHeld, HoldSlotAbsent, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotFree, HoldSlotUndecodable } import std.content_hash { content_hash_validate_lower_hex_length } import std.durable_compare_and_set { CasGeneration } import extdeps.filesystem.filesystem_io { Filesystem, filesystem_exact_read, FilesystemExactPathRead, FilesystemExactPathAbsent, FilesystemExactPathKindUnrecognized, FilesystemExactPathUnreadable } @@ -112,6 +112,7 @@ fn workspace_executor_initial_guard(slot: NonEmptyStr, plan: NonEmptyStr) -> Cas HoldSlotFree { generation: _, released_by: _ } => none HoldSlotUndecodable { generation: _, detail: _ } => none HoldObservationUnavailable { cause: _ } => none + HoldNamedWindowHeadMoved { generation: _ } => none } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => none OwnershipUnread { reason: _ } => none diff --git a/dag/gunbc/workspace_commissioning_fleet.dag b/dag/gunbc/workspace_commissioning_fleet.dag index 14839d08417..bd8557d8220 100644 --- a/dag/gunbc/workspace_commissioning_fleet.dag +++ b/dag/gunbc/workspace_commissioning_fleet.dag @@ -35,7 +35,7 @@ import gunbc.workspace_commissioning_executor_input { workspace_executor_install import gunbc.workspace_commissioning_location { workspace_commissioning_store_location } import gunbc.workspace_slot_mutation { workspace_slot_mutation_key } import gunbc.durable_exclusive_hold_file_store { observe_file_hold } -import std.durable_exclusive_hold { HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldObservationUnavailable, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotUndecodable } import gunbc.workspace_slot_mutation { workspace_commissioning_mutation_owner, workspace_mutation_recovery_standing, WorkspaceMutationRecoveryReleased } import gunbc.runner_microvm_cell_readiness_store { read_cell_readiness_store } import gunbc.workspace_commissioning_apply_readback { workspace_commissioning_initial_ready, workspace_commissioning_credential_absent } @@ -158,6 +158,7 @@ fn workspace_commissioning_basis_observe(provenance: WorkspaceCommissioningProve HoldSlotFree { generation: _, released_by: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false + HoldNamedWindowHeadMoved { generation: _ } => false }) if !committed && !initial { WorkspaceCommissioningFleetRefused { reason: "commissioning has existing history; resume its exact reviewed executor rather than minting a replacement plan" } } else { match resolve_network_subject(slot: slot) { diff --git a/dag/gunbc/workspace_slot_mutation.dag b/dag/gunbc/workspace_slot_mutation.dag index c1f3c9c0fac..cc37daf3fcd 100644 --- a/dag/gunbc/workspace_slot_mutation.dag +++ b/dag/gunbc/workspace_slot_mutation.dag @@ -3,7 +3,7 @@ module gunbc.workspace_slot_mutation import std.types { Bool, NonEmptyStr, String } import std.decl_ref { decl_ref } import std.durable_compare_and_set { CasGeneration, cas_generation_successor, CasSuccessorGeneration, CasSuccessorExhausted } -import std.durable_exclusive_hold { DurableHoldObservation, DurableHoldOwnerRef, DurableHoldReleaseRef, HoldSlotFree, HoldSlotHeld, HoldObservationUnavailable, HoldSlotUndecodable, HoldSlotAbsent } +import std.durable_exclusive_hold { DurableHoldObservation, DurableHoldOwnerRef, DurableHoldReleaseRef, HoldSlotFree, HoldSlotHeld, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotUndecodable, HoldSlotAbsent } import gunbc.build_cache_instance { runner_slot_instance_name } import gunbc.runner_slot_allocation { workspace_execution_slot_identities } import gunbc.workspace_commissioning_location { workspace_commissioning_store_location } @@ -100,6 +100,7 @@ fn workspace_slot_mutation_release_fenced(lease: WorkspaceSlotMutationLease) -> HoldSlotHeld { generation: _, owner: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false + HoldNamedWindowHeadMoved { generation: _ } => false } FileHoldReleaseLost { slot_key: _ } => false FileHoldReleaseStoreUnavailable { slot_key: _, cause: _ } => false @@ -127,6 +128,7 @@ fn workspace_commissioning_mutation_held(slot_key: NonEmptyStr, plan_hash: NonEm HoldSlotFree { generation: _, released_by: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false + HoldNamedWindowHeadMoved { generation: _ } => false } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => false OwnershipUnread { reason: _ } => false @@ -144,6 +146,7 @@ fn workspace_slot_mutation_supply_open(slot_key: NonEmptyStr) -> Bool { HoldSlotHeld { generation: _, owner: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false + HoldNamedWindowHeadMoved { generation: _ } => false } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => false OwnershipUnread { reason: _ } => false @@ -159,6 +162,7 @@ fn workspace_slot_mutation_still_owned(held: WorkspaceSlotMutationHeld) -> Bool HoldSlotFree { generation: _, released_by: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false + HoldNamedWindowHeadMoved { generation: _ } => false } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => false OwnershipUnread { reason: _ } => false @@ -188,6 +192,7 @@ fn workspace_mutation_recovery_standing(observed: DurableHoldObservation, HoldSlotAbsent => WorkspaceMutationRecoveryRefused HoldSlotUndecodable { generation: _, detail: _ } => WorkspaceMutationRecoveryRefused HoldObservationUnavailable { cause: _ } => WorkspaceMutationRecoveryRefused + HoldNamedWindowHeadMoved { generation: _ } => WorkspaceMutationRecoveryRefused } } diff --git a/dag/std/durable_compare_and_set.dag b/dag/std/durable_compare_and_set.dag index 04e02b7ca70..5b850827ecf 100644 --- a/dag/std/durable_compare_and_set.dag +++ b/dag/std/durable_compare_and_set.dag @@ -129,10 +129,12 @@ type CasSlotObservation // // THE PLACEMENT WAS PUT TO REVIEW AND RULED ON, and the ruling also corrected the NAME, which is the // part worth keeping. The arm stays here: an interface outcome need not be producible by every -// realization, it needs a real producer and a stable caller-visible meaning, and a realization that -// must discover its head under a budget supplies both (gunbc.fabric.durable_cas_fabric_storage does; the -// file store no longer does, because its head search is total over the generation line). Moving it into the realization would either erase the typed remedy or force the whole -// store-failure carrier to become realization-specific. +// realization, it needs a real producer and a stable caller-visible meaning. Two producers supply +// both: gunbc.fabric.durable_cas_fabric_storage (generation-gallop budget) and +// gunbc.durable_cas_file_store cas_observe_window (windowed list-and-verify attempt roster). A +// named-generation verify that finds G+1 present is not this arm -- that is a moved head, not an +// exhausted observation budget. Moving BoundExceeded into a realization would either erase the +// typed remedy or force the whole store-failure carrier to become realization-specific. // // But the first spelling was `CasUnreadableBeyondProbeBound`, and PROBE IS THE FILE REALIZATION'S // DISCOVERY METHOD while EXHAUSTED OBSERVATION CAPACITY IS THE INTERFACE FACT. Naming the interface diff --git a/dag/std/durable_exclusive_hold.dag b/dag/std/durable_exclusive_hold.dag index 54e7072dfa7..e3bd08bc706 100644 --- a/dag/std/durable_exclusive_hold.dag +++ b/dag/std/durable_exclusive_hold.dag @@ -70,6 +70,7 @@ type DurableHoldObservation | HoldSlotHeld { generation: CasGeneration, owner: DurableHoldOwnerRef } | HoldSlotUndecodable { generation: CasGeneration, detail: NonEmptyStr } | HoldObservationUnavailable { cause: CasUnreadableSlot } + | HoldNamedWindowHeadMoved { generation: CasGeneration } // ACQUIRE DERIVES ITS EXPECTATION FROM THE OBSERVATION -- THE FIRST AMENDMENT. // @@ -91,6 +92,7 @@ type DurableHoldAcquireRefusal = HoldAcquireOccupied { generation: CasGeneration, holder: DurableHoldOwnerRef } | HoldAcquireSlotUndecodable { generation: CasGeneration, detail: NonEmptyStr } | HoldAcquireObservationUnavailable { cause: CasUnreadableSlot } + | HoldAcquireNamedWindowHeadMoved { generation: CasGeneration } type DurableHoldAcquireAssessment = HoldAcquireEligible { expected: CasExpectation } @@ -107,6 +109,8 @@ fn durable_hold_acquire_assess(observed: DurableHoldObservation) -> DurableHoldA HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: g, detail: d } } HoldObservationUnavailable { cause: c } => HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: c } } + HoldNamedWindowHeadMoved { generation: g } => + HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: g } } } } @@ -155,6 +159,10 @@ fn durable_hold_release_assess( HoldReleaseRefused { refusal: HoldReleaseSlotUndecodable { generation: g, detail: d } } HoldObservationUnavailable { cause: c } => HoldReleaseRefused { refusal: HoldReleaseObservationUnavailable { cause: c } } + HoldNamedWindowHeadMoved { generation: g } => + HoldReleaseRefused { + refusal: HoldReleaseAgainstAnotherGeneration { acquired: acquired_generation, observed: g }, + } HoldSlotHeld { generation: g, owner: holder } => if !durable_hold_owner_equal(left: holder, right: owner) { HoldReleaseRefused { refusal: HoldReleaseHeldByAnotherOwner { generation: g, holder: holder } } @@ -221,6 +229,8 @@ fn durable_hold_recovery_assess(observed: DurableHoldObservation, report: Durabl HoldRecoveryRefused { refusal: HoldRecoverySlotUndecodable { generation: g, detail: d } } HoldObservationUnavailable { cause: c } => HoldRecoveryRefused { refusal: HoldRecoveryObservationUnavailable { cause: c } } + HoldNamedWindowHeadMoved { generation: g } => + HoldRecoveryRefused { refusal: HoldRecoverySlotMoved { reported: report.generation, observed: g } } HoldSlotHeld { generation: g, owner: holder } => if !durable_hold_owner_equal(left: holder, right: report.holder) { HoldRecoveryRefused { refusal: HoldRecoveryReportForAnotherHolder { holder: holder, reported: report.holder } } diff --git a/dag/test/claim/durable_exclusive_hold_witness_test.dag b/dag/test/claim/durable_exclusive_hold_witness_test.dag index fc2efbd5b7f..0c95c55df49 100644 --- a/dag/test/claim/durable_exclusive_hold_witness_test.dag +++ b/dag/test/claim/durable_exclusive_hold_witness_test.dag @@ -13,10 +13,10 @@ import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldState, DurableHoldHeld, DurableHoldFree, DurableHoldObservation, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, - HoldSlotUndecodable, HoldObservationUnavailable, + HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, DurableHoldAcquireAssessment, HoldAcquireEligible, HoldAcquireRefused, DurableHoldAcquireRefusal, HoldAcquireOccupied, HoldAcquireSlotUndecodable, - HoldAcquireObservationUnavailable, + HoldAcquireObservationUnavailable, HoldAcquireNamedWindowHeadMoved, DurableHoldReleaseAssessment, HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, @@ -117,6 +117,7 @@ test fn a_bare_free_tag_is_not_an_acquirable_free_slot() -> Bool { HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: g, detail: _ } } => cas_generation_count(g: g) == 3 HoldAcquireRefused { refusal: HoldAcquireOccupied { generation: _, holder: _ } } => false HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: _ } } => false + HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: _ } } => false HoldAcquireEligible { expected: _ } => false } } @@ -189,6 +190,7 @@ fn eh_acquire_occupied_by(a: DurableHoldAcquireAssessment, holder: String) -> Bo HoldAcquireRefused { refusal: HoldAcquireOccupied { generation: _, holder: h } } => (h as String) == holder HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: _, detail: _ } } => false HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: _ } } => false + HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: _ } } => false HoldAcquireEligible { expected: _ } => false } } @@ -215,6 +217,7 @@ test fn an_undecodable_slot_is_neither_acquirable_nor_occupied() -> Bool { HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: g, detail: _ } } => cas_generation_count(g: g) == 2 HoldAcquireRefused { refusal: HoldAcquireOccupied { generation: _, holder: _ } } => false HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: _ } } => false + HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: _ } } => false HoldAcquireEligible { expected: _ } => false } } @@ -224,6 +227,7 @@ test fn an_unavailable_observation_is_not_acquirable() -> Bool { HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: _ } } => true HoldAcquireRefused { refusal: HoldAcquireOccupied { generation: _, holder: _ } } => false HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: _, detail: _ } } => false + HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: _ } } => false HoldAcquireEligible { expected: _ } => false } } diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index c302a97ade3..064edcd6589 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -58,9 +58,7 @@ import gunbc.durable_cas_file_store { cas_file_slot_path } import std.durable_exclusive_hold { DurableHoldOwnerRef, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, -} -import std.durable_compare_and_set { - CasUnreadableMalformed, CasUnreadableContentMissing, CasUnreadableReadRefused, CasUnreadableObservationBoundExceeded, + HoldNamedWindowHeadMoved, } import std.fabric_blob { fabric_blob_reading_path } import std.cache_identity { native_module_verdict_bundle_artifact_kind, typed_module_artifact_kind } @@ -675,13 +673,8 @@ fn hold_refusal_is_named_window_unsettled(r: LocalStoreHoldRefusal?) -> Bool { match r { Present { value: x } => match x { LocalStoreHoldStale { observed: o } => match o { - HoldObservationUnavailable { cause: c } => match c { - CasUnreadableReadRefused { detail: d } => - d as String == "the window head moved during 1 observations and was not settled; a head is never guessed" - CasUnreadableMalformed { detail: _ } => false - CasUnreadableContentMissing { referenced: _ } => false - CasUnreadableObservationBoundExceeded { bound: _ } => false - } + HoldNamedWindowHeadMoved { generation: _ } => true + HoldObservationUnavailable { cause: _ } => false HoldSlotAbsent => false HoldSlotFree { generation: _, released_by: _ } => false HoldSlotHeld { generation: _, owner: _ } => false @@ -756,8 +749,8 @@ fn pin_generation_unlink(path: String, on: Bool) -> Bool { // PARTIAL RECLAIM MUST NOT RE-AUTHORIZE A LEAKED HOLD. Independent deletes could remove G+1 while // G survived, so named verify would treat leaked G as head. Pin G against unlink, run another // acquire/release (the reclaim that used to punch that hole), then reuse G. commit_under observes -// the hold first: named verify must answer Unsettled (G present, G+1 still there), mapped to -// HoldStale via HoldObservationUnavailable, before any occupancy write. occupancy_unavailable on +// the hold first: named verify must answer HoldNamedWindowHeadMoved (G present, G+1 still there), mapped to +// LocalStoreHoldStale, before any occupancy write. occupancy_unavailable on // that path is the hold-refuse wrapper, not an occupancy fault from the pin. ioctl-set IMMUTABLE // must succeed (CAP_LINUX_IMMUTABLE); without it this claim is false, not vacuously green. test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution() -> Bool { From b99601fdeac95c395b08c4ff4f5e49aef1cec4a4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 13:36:53 +0000 Subject: [PATCH 16/30] Keep named-head-moved off BoundExceeded and delete the unused CAS observe. ProbedNamedWindowHeadMoved had been projected as CasUnreadableObservationBoundExceeded { bound: 1 }, which is the eight-pass budget arm. Named verify now returns CasNamedWindowVerify; observe_cas_slot_state_windowed_at is deleted rather than left sealed with no caller. Co-authored-by: Cursor --- dag/gunbc/durable_cas_file_store.dag | 41 +++++++------------ .../durable_exclusive_hold_file_store.dag | 6 +-- ...lization_store_local_seal_witness_test.dag | 4 +- 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index 9e88e626766..dfd7c7a72b3 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -307,7 +307,6 @@ type CasSlotProbe | ProbedUnreadable { generation: CasGeneration, kind: FilesystemFailureKind } | ProbedReadKindUnrecognized { generation: CasGeneration, observed: String } | ProbedWindowHeadUnsettled { attempts: Int } - | ProbedNamedWindowHeadMoved { generation: CasGeneration } // One generation read, classified: present with its content, absent, or a typed refusal. type CasGenerationRead @@ -641,8 +640,6 @@ fn cas_probe_as_observation(probe: CasSlotProbe) -> CasSlotObservation CasObservedReadable { readable: CasReadableAbsent } ProbedAbsenceUnestablished { cause: c } => CasObservedUnreadable { cause: cas_absence_unestablished(cause: c) } ProbedWindowHeadUnsettled { attempts: a } => CasObservedUnreadable { cause: cas_window_head_unsettled(attempts: a) } - ProbedNamedWindowHeadMoved { generation: _ } => - CasObservedUnreadable { cause: CasUnreadableObservationBoundExceeded { bound: 1 } } ProbedHead { generation: h, value: v } => CasObservedReadable { readable: cas_head_as_readable(generation: h, value: v) } } @@ -712,40 +709,34 @@ fn observe_cas_slot_state_windowed(root: NonEmptyStr, key: NonEmptyStr, window: // its successor reads absent. Those two reads are Filesystem.Read of cas_file_slot_path, O(1) in // the store's object count. A successor that is present means the head moved; an absent named // generation is not this head. The observation is never guessed. -fn cas_verify_named_window_head(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> CasSlotProbe +// Named verify is not a CasSlotProbe: G+1 present is a moved head, not an observation-budget +// exhaustion, and CasSlotProbe's Unsettled arm is the eight-pass list-and-verify bound. Folding the +// two together is the meaning-fork DESIGN section 3 names; the hold consumer maps HeadMoved itself. +type CasNamedWindowVerify + = NamedWindowVerified { probe: CasSlotProbe } + | NamedWindowHeadMoved { generation: CasGeneration } + +fn cas_verify_named_window_head(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> CasNamedWindowVerify admit_callers: [ - decl_ref(module_path: "gunbc.durable_cas_file_store", decl_name: "observe_cas_slot_state_windowed_at"), decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "observe_file_hold_windowed_at"), ] { match cas_read_generation(root: root, key: key, generation: generation) { - GenerationRefused { probe: p } => p - GenerationAbsent => ProbedAbsent + GenerationRefused { probe: p } => NamedWindowVerified { probe: p } + GenerationAbsent => NamedWindowVerified { probe: ProbedAbsent } GenerationPresent { content: c } => match cas_generation_successor(g: generation) { - CasSuccessorExhausted { head: _ } => ProbedHead { generation: generation, value: c } + CasSuccessorExhausted { head: _ } => NamedWindowVerified { probe: ProbedHead { generation: generation, value: c } } CasSuccessorGeneration { generation: next } => match cas_read_generation(root: root, key: key, generation: next) { - GenerationRefused { probe: p } => p - GenerationAbsent => ProbedHead { generation: generation, value: c } - GenerationPresent { content: _ } => ProbedNamedWindowHeadMoved { generation: generation } + GenerationRefused { probe: p } => NamedWindowVerified { probe: p } + GenerationAbsent => NamedWindowVerified { probe: ProbedHead { generation: generation, value: c } } + GenerationPresent { content: _ } => NamedWindowHeadMoved { generation: generation } } } } } -fn observe_cas_slot_state_windowed_at(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> CasSlotObservation - admit_callers: [ - decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "observe_file_hold_windowed_at"), - ] -{ - if !cas_key_is_slot_addressable(key: key) { - CasObservedUnreadable { cause: CasUnreadableReadRefused { detail: cas_key_not_slot_addressable_detail } } - } else { - cas_probe_as_observation(probe: cas_verify_named_window_head(root: root, key: key, generation: generation)) - } -} - data cas_key_not_slot_addressable_detail: NonEmptyStr = "key is not slot-addressable: it carries a path separator" @@ -992,8 +983,6 @@ fn cas_owner_only_failed_publication( } ProbedWindowHeadUnsettled { attempts: a } => CasStoreRefused { cause: CasSlotObservationRefused { cause: cas_window_head_unsettled(attempts: a) } } - ProbedNamedWindowHeadMoved { generation: _ } => - CasStoreRefused { cause: CasSlotObservationRefused { cause: CasUnreadableObservationBoundExceeded { bound: 1 } } } } } @@ -1033,8 +1022,6 @@ fn cas_compare_and_set_outcome( CasStoreRefused { cause: CasSlotObservationRefused { cause: cas_absence_unestablished(cause: c) } } ProbedWindowHeadUnsettled { attempts: a } => CasStoreRefused { cause: CasSlotObservationRefused { cause: cas_window_head_unsettled(attempts: a) } } - ProbedNamedWindowHeadMoved { generation: _ } => - CasStoreRefused { cause: CasSlotObservationRefused { cause: CasUnreadableObservationBoundExceeded { bound: 1 } } } ProbedAbsent => match attempt.expected { ExpectSlotAbsent => diff --git a/dag/gunbc/durable_exclusive_hold_file_store.dag b/dag/gunbc/durable_exclusive_hold_file_store.dag index 760d91326d6..86167e13620 100644 --- a/dag/gunbc/durable_exclusive_hold_file_store.dag +++ b/dag/gunbc/durable_exclusive_hold_file_store.dag @@ -48,7 +48,7 @@ import gunbc.durable_cas_file_store { KeepAllGenerations, KeepGenerationWindow, admit_cas_attempt_for_derived_payload, file_compare_and_set, file_compare_and_set_retained, observe_cas_slot_state, observe_cas_slot_state_retained, cas_verify_named_window_head, cas_probe_as_observation, cas_key_is_slot_addressable, - cas_key_not_slot_addressable_detail, ProbedNamedWindowHeadMoved, + cas_key_not_slot_addressable_detail, NamedWindowVerified, NamedWindowHeadMoved, } // THE TAGS ARE NAMED ONCE AND THE BODY IS DERIVED FROM THE TAG THAT MATCHED. @@ -168,8 +168,8 @@ fn observe_file_hold_windowed_at(root: NonEmptyStr, key: NonEmptyStr, generation HoldObservationUnavailable { cause: CasUnreadableReadRefused { detail: cas_key_not_slot_addressable_detail } } } else { match cas_verify_named_window_head(root: root, key: key, generation: generation) { - ProbedNamedWindowHeadMoved { generation: g } => HoldNamedWindowHeadMoved { generation: g } - p => hold_observation_of_cas(observed: cas_probe_as_observation(probe: p)) + NamedWindowHeadMoved { generation: g } => HoldNamedWindowHeadMoved { generation: g } + NamedWindowVerified { probe: p } => hold_observation_of_cas(observed: cas_probe_as_observation(probe: p)) } } } diff --git a/dag/test/claim/materialization_store_local_seal_witness_test.dag b/dag/test/claim/materialization_store_local_seal_witness_test.dag index 7ae335b501b..1ce58e244b0 100644 --- a/dag/test/claim/materialization_store_local_seal_witness_test.dag +++ b/dag/test/claim/materialization_store_local_seal_witness_test.dag @@ -145,12 +145,12 @@ test fn a_direct_raw_reclaim_call_is_an_admission_refusal() -> Bool { data direct_held_session_source: String = "module probe_direct_held_session\nimport std.cache_identity { ArtifactKindId }\nimport extdeps.realization.materialization_store_local { LocalStoreCapability, LocalStoreFamilyHeld, LocalStoreBracket, local_store_held_session }\nfn direct(c: LocalStoreCapability, f: ArtifactKindId) -> LocalStoreBracket {\n local_store_held_session(capability: c, family: f, run: h => h)\n}\n" -data direct_named_window_observe_source: String = "module probe_direct_named_window_observe\nimport std.types { NonEmptyStr }\nimport std.durable_compare_and_set { CasGeneration, CasSlotObservation }\nimport gunbc.durable_cas_file_store { observe_cas_slot_state_windowed_at }\nfn direct(r: NonEmptyStr, k: NonEmptyStr, g: CasGeneration) -> CasSlotObservation {\n observe_cas_slot_state_windowed_at(root: r, key: k, generation: g)\n}\n" +data direct_named_window_observe_source: String = "module probe_direct_named_window_observe\nimport std.types { NonEmptyStr }\nimport std.durable_compare_and_set { CasGeneration }\nimport gunbc.durable_cas_file_store { cas_verify_named_window_head, CasNamedWindowVerify }\nfn direct(r: NonEmptyStr, k: NonEmptyStr, g: CasGeneration) -> CasNamedWindowVerify {\n cas_verify_named_window_head(root: r, key: k, generation: g)\n}\n" test fn a_direct_held_session_call_is_an_admission_refusal() -> Bool { admission_refusals(source: direct_held_session_source, callee: "local_store_held_session") == 1 } test fn a_direct_named_window_observe_call_is_an_admission_refusal() -> Bool { - admission_refusals(source: direct_named_window_observe_source, callee: "observe_cas_slot_state_windowed_at") == 1 + admission_refusals(source: direct_named_window_observe_source, callee: "cas_verify_named_window_head") == 1 } From 662ca06d8111128e41715e5320ac4c811c6c851f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 13:40:09 +0000 Subject: [PATCH 17/30] Give named-head-moved its own release and recovery refusals. HoldReleaseAgainstAnotherGeneration and HoldRecoverySlotMoved name an observed current generation. Named verify only established that a successor of G exists, so those constructors were lying when acquired/reported equalled G. The leaked-hold helper is named for Moved, not Unsettled. Co-authored-by: Cursor --- dag/gunbc/fabric/fabric_control_plane.dag | 9 ++++++++ dag/gunbc/host/managed_host_unit_hold.dag | 4 ++++ dag/std/durable_exclusive_hold.dag | 8 +++---- .../durable_exclusive_hold_witness_test.dag | 22 +++++++++++++++++++ ...alization_store_local_wet_witness_test.dag | 4 ++-- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/dag/gunbc/fabric/fabric_control_plane.dag b/dag/gunbc/fabric/fabric_control_plane.dag index 185ecc3d22a..7bd9149192a 100644 --- a/dag/gunbc/fabric/fabric_control_plane.dag +++ b/dag/gunbc/fabric/fabric_control_plane.dag @@ -19,6 +19,7 @@ import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, + HoldReleaseNamedWindowHeadMoved, HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, } import product.fabric.identity { FabricIdentity, OfferKey, DemandKey, WorkKey, fabric_identity_eq } @@ -736,6 +737,14 @@ fn end_cell_hold( to_string(cas_generation_count(g: a)), ) as NonEmptyStr, } + HoldReleaseNamedWindowHeadMoved { named: n } => + HoldEndSlotNotHeld { + slot_key: slot_key, + detail: concat( + "named window head moved beyond generation ", + to_string(cas_generation_count(g: n)), + ) as NonEmptyStr, + } HoldReleaseSlotUndecodable { generation: _, detail: d } => HoldEndSlotNotHeld { slot_key: slot_key, detail: d } } diff --git a/dag/gunbc/host/managed_host_unit_hold.dag b/dag/gunbc/host/managed_host_unit_hold.dag index ac1a1315ed3..f91af4a6c99 100644 --- a/dag/gunbc/host/managed_host_unit_hold.dag +++ b/dag/gunbc/host/managed_host_unit_hold.dag @@ -10,11 +10,13 @@ import std.optional { Present, Absent } import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldObservation, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, + HoldReleaseNamedWindowHeadMoved, HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, DurableHoldHolderLiveness, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, DurableHoldHolderReport, DurableHoldRecoveryRefusal, HoldRecoveryNothingHeld, HoldRecoveryHolderLive, HoldRecoveryHolderUnobservable, HoldRecoverySlotMoved, + HoldRecoveryNamedWindowHeadMoved, HoldRecoveryReportForAnotherHolder, HoldRecoverySlotUndecodable, HoldRecoveryObservationUnavailable, } import std.nat { Nat } @@ -448,6 +450,7 @@ fn recovery_refusal_text(key: NonEmptyStr, r: DurableHoldRecoveryRefusal) -> Non HoldRecoveryHolderLive { generation: _, holder: h, evidence: e } => join(["is held by '", h as String, "', which is alive (", e as String, "); no BMC actuation until it is released"], "") HoldRecoveryHolderUnobservable { generation: _, holder: h, cause: c } => join(["is held by '", h as String, "', whose liveness could not be observed (", c as String, "); an unobserved holder is not a dead one, so the hold stands"], "") HoldRecoverySlotMoved { reported: _, observed: _ } => "moved while its holder was being observed; re-run" + HoldRecoveryNamedWindowHeadMoved { named: _ } => "named window head moved; the current head was not observed; re-run" HoldRecoveryReportForAnotherHolder { holder: h, reported: _ } => join(["changed hands to '", h as String, "' while its holder was being observed; re-run"], "") HoldRecoverySlotUndecodable { generation: _, detail: d } => concat("became undecodable: ", d as String) HoldRecoveryObservationUnavailable { cause: _ } => "could not be re-read for recovery; an unreadable interlock is not a released one" @@ -643,6 +646,7 @@ fn probe_hold_release_refused(refusal: DurableHoldReleaseRefusal) -> ProbeHoldRe HoldReleaseAlreadyFree { generation: _, released_by: _ } => ProbeHoldAlreadyFree HoldReleaseHeldByAnotherOwner { generation: _, holder: h } => ProbeHoldNotOurs { holder: h as String } HoldReleaseAgainstAnotherGeneration { acquired: _, observed: _ } => ProbeHoldStale { detail: "the slot moved to another generation after it was observed" } + HoldReleaseNamedWindowHeadMoved { named: _ } => ProbeHoldStale { detail: "the named window head moved; the current head was not observed" } HoldReleaseSlotUndecodable { generation: _, detail: d } => ProbeHoldUnreleased { cause: concat("slot undecodable: ", d as String) } HoldReleaseObservationUnavailable { cause: _ } => ProbeHoldUnreleased { cause: "the store could not be read" } } diff --git a/dag/std/durable_exclusive_hold.dag b/dag/std/durable_exclusive_hold.dag index e3bd08bc706..877c4494393 100644 --- a/dag/std/durable_exclusive_hold.dag +++ b/dag/std/durable_exclusive_hold.dag @@ -136,6 +136,7 @@ type DurableHoldReleaseRefusal | HoldReleaseAlreadyFree { generation: CasGeneration, released_by: DurableHoldReleaseRef } | HoldReleaseHeldByAnotherOwner { generation: CasGeneration, holder: DurableHoldOwnerRef } | HoldReleaseAgainstAnotherGeneration { acquired: CasGeneration, observed: CasGeneration } + | HoldReleaseNamedWindowHeadMoved { named: CasGeneration } | HoldReleaseSlotUndecodable { generation: CasGeneration, detail: NonEmptyStr } | HoldReleaseObservationUnavailable { cause: CasUnreadableSlot } @@ -160,9 +161,7 @@ fn durable_hold_release_assess( HoldObservationUnavailable { cause: c } => HoldReleaseRefused { refusal: HoldReleaseObservationUnavailable { cause: c } } HoldNamedWindowHeadMoved { generation: g } => - HoldReleaseRefused { - refusal: HoldReleaseAgainstAnotherGeneration { acquired: acquired_generation, observed: g }, - } + HoldReleaseRefused { refusal: HoldReleaseNamedWindowHeadMoved { named: g } } HoldSlotHeld { generation: g, owner: holder } => if !durable_hold_owner_equal(left: holder, right: owner) { HoldReleaseRefused { refusal: HoldReleaseHeldByAnotherOwner { generation: g, holder: holder } } @@ -210,6 +209,7 @@ type DurableHoldRecoveryRefusal | HoldRecoveryHolderLive { generation: CasGeneration, holder: DurableHoldOwnerRef, evidence: NonEmptyStr } | HoldRecoveryHolderUnobservable { generation: CasGeneration, holder: DurableHoldOwnerRef, cause: NonEmptyStr } | HoldRecoverySlotMoved { reported: CasGeneration, observed: CasGeneration } + | HoldRecoveryNamedWindowHeadMoved { named: CasGeneration } | HoldRecoveryReportForAnotherHolder { holder: DurableHoldOwnerRef, reported: DurableHoldOwnerRef } | HoldRecoverySlotUndecodable { generation: CasGeneration, detail: NonEmptyStr } | HoldRecoveryObservationUnavailable { cause: CasUnreadableSlot } @@ -230,7 +230,7 @@ fn durable_hold_recovery_assess(observed: DurableHoldObservation, report: Durabl HoldObservationUnavailable { cause: c } => HoldRecoveryRefused { refusal: HoldRecoveryObservationUnavailable { cause: c } } HoldNamedWindowHeadMoved { generation: g } => - HoldRecoveryRefused { refusal: HoldRecoverySlotMoved { reported: report.generation, observed: g } } + HoldRecoveryRefused { refusal: HoldRecoveryNamedWindowHeadMoved { named: g } } HoldSlotHeld { generation: g, owner: holder } => if !durable_hold_owner_equal(left: holder, right: report.holder) { HoldRecoveryRefused { refusal: HoldRecoveryReportForAnotherHolder { holder: holder, reported: report.holder } } diff --git a/dag/test/claim/durable_exclusive_hold_witness_test.dag b/dag/test/claim/durable_exclusive_hold_witness_test.dag index 0c95c55df49..22c1a15d0b9 100644 --- a/dag/test/claim/durable_exclusive_hold_witness_test.dag +++ b/dag/test/claim/durable_exclusive_hold_witness_test.dag @@ -20,11 +20,13 @@ import std.durable_exclusive_hold { DurableHoldReleaseAssessment, HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, + HoldReleaseNamedWindowHeadMoved, HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, durable_hold_acquire_assess, durable_hold_release_assess, durable_hold_owner_equal, DurableHoldHolderReport, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, DurableHoldRecoveryAssessment, HoldRecoveryEligible, HoldRecoveryRefused, HoldRecoveryNothingHeld, HoldRecoveryHolderLive, HoldRecoveryHolderUnobservable, HoldRecoverySlotMoved, + HoldRecoveryNamedWindowHeadMoved, HoldRecoveryReportForAnotherHolder, durable_hold_recovery_assess, } import gunbc.durable_exclusive_hold_file_store { @@ -252,6 +254,7 @@ fn eh_release_refusal_label(a: DurableHoldReleaseAssessment) -> String { HoldReleaseAlreadyFree { generation: _, released_by: _ } => "already-free" HoldReleaseHeldByAnotherOwner { generation: _, holder: _ } => "another-owner" HoldReleaseAgainstAnotherGeneration { acquired: _, observed: _ } => "another-generation" + HoldReleaseNamedWindowHeadMoved { named: _ } => "named-head-moved" HoldReleaseSlotUndecodable { generation: _, detail: _ } => "undecodable" HoldReleaseObservationUnavailable { cause: _ } => "unavailable" } @@ -371,3 +374,22 @@ test fn a_dead_verdict_recovers_only_the_hold_it_was_taken_of() -> Bool { _ => false }) } + +test fn named_window_head_moved_release_does_not_claim_an_observed_current_generation() -> Bool { + match eh_release_assess(observed: HoldNamedWindowHeadMoved { generation: 3 }, owner: "demand-7|offer-1", acquired: 3) { + HoldReleaseRefused { refusal: HoldReleaseNamedWindowHeadMoved { named: n } } => + cas_generation_count(g: n) == 3 + HoldReleaseRefused { refusal: HoldReleaseAgainstAnotherGeneration { acquired: _, observed: _ } } => false + _ => false + } +} + +test fn named_window_head_moved_recovery_does_not_claim_an_observed_current_generation() -> Bool { + let dead = eh_report(holder: "mtcollins1-boot:4242", generation: 3, dead: 1) + match durable_hold_recovery_assess(observed: HoldNamedWindowHeadMoved { generation: 3 }, report: dead) { + HoldRecoveryRefused { refusal: HoldRecoveryNamedWindowHeadMoved { named: n } } => + cas_generation_count(g: n) == 3 + HoldRecoveryRefused { refusal: HoldRecoverySlotMoved { reported: _, observed: _ } } => false + _ => false + } +} diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 064edcd6589..88f96c30f85 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -669,7 +669,7 @@ test fn a_commit_while_the_family_is_held_refuses_and_writes_nothing_by_real_exe verdict && removed.success } -fn hold_refusal_is_named_window_unsettled(r: LocalStoreHoldRefusal?) -> Bool { +fn hold_refusal_is_named_window_head_moved(r: LocalStoreHoldRefusal?) -> Bool { match r { Present { value: x } => match x { LocalStoreHoldStale { observed: o } => match o { @@ -773,7 +773,7 @@ test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_r let _unlock = pin_generation_unlink(path: pinned, on: false) immutable && advanced && commit_tag(c: c.commit) == "occupancy_unavailable" - && hold_refusal_is_named_window_unsettled(r: c.hold_refusal) + && hold_refusal_is_named_window_head_moved(r: c.hold_refusal) && !Filesystem.Read(path: object_path(root_path: root_path, req: rs)).success && store_lookup_is_miss(l: local_store_lookup(capability: store, req: rs)) } From f96456dacfcff62e92939cf047909dcc05a0d2e4 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 14:08:07 +0000 Subject: [PATCH 18/30] Make named-window verify realization-private and drop it from hold observation. local_store_hold_check and windowed release/recovery consume NamedWindowHeadCurrent | NamedWindowHeadMoved directly. Dedicated release/recovery refusals stay; DurableHoldObservation and CasSlotProbe no longer carry a moved-head arm. Co-authored-by: Cursor --- .../materialization_store_local.dag | 37 ++++++++-- dag/gunbc/durable_cas_file_store.dag | 46 +++++++----- .../durable_exclusive_hold_file_store.dag | 74 ++++++++++++++----- dag/gunbc/host/managed_host_unit_hold.dag | 4 +- dag/gunbc/workspace_allocation_plan.dag | 6 +- dag/gunbc/workspace_allocation_supply.dag | 3 +- ...kspace_commissioning_admission_observe.dag | 3 +- ...workspace_commissioning_executor_fence.dag | 3 +- ...pace_commissioning_executor_submission.dag | 3 +- dag/gunbc/workspace_commissioning_fleet.dag | 3 +- dag/gunbc/workspace_slot_mutation.dag | 7 +- dag/std/durable_exclusive_hold.dag | 8 -- .../durable_exclusive_hold_witness_test.dag | 18 +++-- ...alization_store_local_wet_witness_test.dag | 20 ++--- 14 files changed, 137 insertions(+), 98 deletions(-) diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag index d8bf6a7ebb3..0d257ccdee4 100644 --- a/dag/extdeps/realization/materialization_store_local.dag +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -81,11 +81,12 @@ import gunbc.durable_cas_file_store { CasWindowReclamation, CasReclamationNotCommitted, CasAttemptAdmission, cas_window_generations, CasWindowGenerations, CasWindowGenerationsListed, CasWindowGenerationsUnlisted, cas_reclaim_below, CasGenerationWindow, + cas_verify_named_window_head, NamedWindowHeadCurrent, NamedWindowHeadMoved, } import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldHolderReport, DurableHoldReleaseRefusal, DurableHoldRecoveryRefusal, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, - DurableHoldObservation, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, + DurableHoldObservation, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, durable_hold_owner_equal, } import gunbc.durable_exclusive_hold_file_store { @@ -94,7 +95,7 @@ import gunbc.durable_exclusive_hold_file_store { file_hold_release_assess_windowed_at, FileHoldReleasePlanned, FileHoldReleaseNotEligible, file_hold_release_commit, FileHoldReleased, FileHoldReleaseLost, FileHoldReleaseStoreUnavailable, FileHoldReleaseKeyNotSlotAddressable, file_hold_recovery_assess_windowed, FileHoldRecoveryPlanned, FileHoldRecoveryNotEligible, - observe_file_hold_windowed_at, + hold_observation_of_cas, } import gunbc.process_hold_identity { process_hold_text, process_hold_marker, process_hold_decode, ProcessHoldHolder, @@ -907,6 +908,7 @@ type LocalStoreIndexRefusal | LocalStoreIndexContended { attempts: Int } | LocalStoreIndexUnsettled { settlement: LocalStoreSlotSettlement } | LocalStoreIndexHoldStale { observed: DurableHoldObservation } + | LocalStoreIndexHoldNamedWindowHeadMoved { named: CasGeneration } fn local_store_index_refusal_text(refusal: LocalStoreIndexRefusal) -> String { match refusal { @@ -927,6 +929,7 @@ fn local_store_index_refusal_text(refusal: LocalStoreIndexRefusal) -> String { LocalStoreIndexContended { attempts: n } => concat("the occupancy index kept moving under concurrent writers past the attempt bound: ", n as String) LocalStoreIndexUnsettled { settlement: st } => concat("the occupancy index cannot advance: ", local_store_settlement_text(s: st)) LocalStoreIndexHoldStale { observed: _ } => "the occupancy index write was attempted under a hold that is no longer live" + LocalStoreIndexHoldNamedWindowHeadMoved { named: _ } => "the occupancy index write was attempted after the named window head moved" } } @@ -974,6 +977,7 @@ fn local_store_write_index(held: LocalStoreFamilyHeld, expected: CasExpectation, { match local_store_hold_check(held: held) { LocalStoreHoldNotCurrent { observed: o } => LocalStoreIndexWriteRefused { refusal: LocalStoreIndexHoldStale { observed: o } } + LocalStoreHoldNamedWindowMoved { named: n } => LocalStoreIndexWriteRefused { refusal: LocalStoreIndexHoldNamedWindowHeadMoved { named: n } } LocalStoreHoldCurrent => local_store_write_index_current(held: held, expected: expected, index: index) } } @@ -1070,6 +1074,7 @@ type LocalStoreHoldRefusal | LocalStoreHoldWindowTooNarrow { k: Int } | LocalStoreHoldUnsettled { settlement: LocalStoreSlotSettlement } | LocalStoreHoldStale { observed: DurableHoldObservation } + | LocalStoreHoldNamedWindowHeadMoved { named: CasGeneration } | LocalStoreHoldWrongFamily { held: ArtifactKindId, requested: ArtifactKindId } fn local_store_hold_refusal_text(refusal: LocalStoreHoldRefusal) -> String { @@ -1085,6 +1090,7 @@ fn local_store_hold_refusal_text(refusal: LocalStoreHoldRefusal) -> String { LocalStoreHoldWindowTooNarrow { k: k } => concat("the family hold window is narrower than its floor: k = ", k as String) LocalStoreHoldUnsettled { settlement: st } => concat("the family hold slot cannot advance: ", local_store_settlement_text(s: st)) LocalStoreHoldStale { observed: _ } => "the hold this write was given is no longer the family's live hold" + LocalStoreHoldNamedWindowHeadMoved { named: _ } => "the named window head moved; this hold is no longer the family's live hold" LocalStoreHoldWrongFamily { held: h, requested: r } => concat(concat("the hold is for family ", h as String), concat(", the write is for ", r as String)) } } @@ -1200,6 +1206,7 @@ fn local_store_hold_occupied(binding: LocalStoreBinding, family: ArtifactKindId, type LocalStoreHoldCheck = LocalStoreHoldCurrent | LocalStoreHoldNotCurrent { observed: DurableHoldObservation } + | LocalStoreHoldNamedWindowMoved { named: CasGeneration } fn local_store_hold_check(held: LocalStoreFamilyHeld) -> LocalStoreHoldCheck admit_callers: [ @@ -1210,15 +1217,29 @@ fn local_store_hold_check(held: LocalStoreFamilyHeld) -> LocalStoreHoldCheck decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_commit_verified"), ] { - let observed = observe_file_hold_windowed_at(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, key: local_store_hold_key(family: held.family), generation: held.generation) + match cas_verify_named_window_head(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, key: local_store_hold_key(family: held.family), generation: held.generation) { + NamedWindowHeadMoved { named_generation: n } => LocalStoreHoldNamedWindowMoved { named: n } + NamedWindowHeadCurrent { observation: o } => + local_store_hold_from_current_observation(held: held, observed: hold_observation_of_cas(observed: o)) + } +} + +fn local_store_hold_from_current_observation(held: LocalStoreFamilyHeld, observed: DurableHoldObservation) -> LocalStoreHoldCheck + admit_callers: [ + decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_hold_check"), + ] +{ match observed { - HoldSlotHeld { generation: g, owner: o } => - if cas_generation_count(g: g) == cas_generation_count(g: held.generation) && durable_hold_owner_equal(left: o, right: held.owner) { LocalStoreHoldCurrent } else { LocalStoreHoldNotCurrent { observed: observed } } + HoldSlotHeld { generation: g, owner: owner } => + if cas_generation_count(g: g) == cas_generation_count(g: held.generation) && durable_hold_owner_equal(left: owner, right: held.owner) { + LocalStoreHoldCurrent + } else { + LocalStoreHoldNotCurrent { observed: observed } + } HoldSlotAbsent => LocalStoreHoldNotCurrent { observed: observed } HoldSlotFree { generation: _, released_by: _ } => LocalStoreHoldNotCurrent { observed: observed } HoldSlotUndecodable { generation: _, detail: _ } => LocalStoreHoldNotCurrent { observed: observed } HoldObservationUnavailable { cause: _ } => LocalStoreHoldNotCurrent { observed: observed } - HoldNamedWindowHeadMoved { generation: _ } => LocalStoreHoldNotCurrent { observed: observed } } } @@ -1679,6 +1700,7 @@ fn local_store_sweep_held(all: LocalStoreAllFamiliesHeld) -> LocalStoreSweep Absent => match local_store_hold_check(held: h) { LocalStoreHoldCurrent => none LocalStoreHoldNotCurrent { observed: o } => Present { value: LocalStoreSweepHoldRefused { family: h.family, refusal: LocalStoreHoldStale { observed: o }, released: [] } } + LocalStoreHoldNamedWindowMoved { named: n } => Present { value: LocalStoreSweepHoldRefused { family: h.family, refusal: LocalStoreHoldNamedWindowHeadMoved { named: n }, released: [] } } } }) match stale { @@ -1759,6 +1781,7 @@ fn local_store_record_recency_held(held: LocalStoreFamilyHeld, served: List LocalStoreRecencyHoldRefused { family: family, refusal: LocalStoreHoldStale { observed: o } } + LocalStoreHoldNamedWindowMoved { named: n } => LocalStoreRecencyHoldRefused { family: family, refusal: LocalStoreHoldNamedWindowHeadMoved { named: n } } LocalStoreHoldCurrent => local_store_record_recency_current(held: held, served: served) } } @@ -2019,6 +2042,7 @@ fn local_store_commit_verified( let root_path = materialization_store_root_path(root: root) match local_store_hold_check(held: held) { LocalStoreHoldNotCurrent { observed: o } => local_store_commit_hold_refused(key: key, refusal: LocalStoreHoldStale { observed: o }) + LocalStoreHoldNamedWindowMoved { named: n } => local_store_commit_hold_refused(key: key, refusal: LocalStoreHoldNamedWindowHeadMoved { named: n }) LocalStoreHoldCurrent => local_store_commit_refused(commit: store_commit_settle( store: held.binding.store, @@ -2285,6 +2309,7 @@ fn local_store_commit_under( } else { match local_store_hold_check(held: held) { LocalStoreHoldNotCurrent { observed: o } => local_store_commit_hold_refused(key: request_key(req: req), refusal: LocalStoreHoldStale { observed: o }) + LocalStoreHoldNamedWindowMoved { named: n } => local_store_commit_hold_refused(key: request_key(req: req), refusal: LocalStoreHoldNamedWindowHeadMoved { named: n }) LocalStoreHoldCurrent => { let staged = parts |> map(p => local_store_read_part(id: p.id, path: p.staged)) match local_store_first_unreadable(readings: staged) { diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index dfd7c7a72b3..d0b4dd16796 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -709,31 +709,39 @@ fn observe_cas_slot_state_windowed(root: NonEmptyStr, key: NonEmptyStr, window: // its successor reads absent. Those two reads are Filesystem.Read of cas_file_slot_path, O(1) in // the store's object count. A successor that is present means the head moved; an absent named // generation is not this head. The observation is never guessed. -// Named verify is not a CasSlotProbe: G+1 present is a moved head, not an observation-budget -// exhaustion, and CasSlotProbe's Unsettled arm is the eight-pass list-and-verify bound. Folding the -// two together is the meaning-fork DESIGN section 3 names; the hold consumer maps HeadMoved itself. +// Realization-private: named verify is not CasSlotProbe and not DurableHoldObservation. G+1 present +// is a moved head, not eight-pass BoundExceeded. Consumers are the windowed hold check and the +// windowed release assessment; they match HeadMoved directly. type CasNamedWindowVerify - = NamedWindowVerified { probe: CasSlotProbe } - | NamedWindowHeadMoved { generation: CasGeneration } + = NamedWindowHeadCurrent { observation: CasSlotObservation } + | NamedWindowHeadMoved { named_generation: CasGeneration } fn cas_verify_named_window_head(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> CasNamedWindowVerify admit_callers: [ - decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "observe_file_hold_windowed_at"), + decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_hold_check"), + decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess_windowed_at"), + decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_recovery_assess_windowed"), ] { - match cas_read_generation(root: root, key: key, generation: generation) { - GenerationRefused { probe: p } => NamedWindowVerified { probe: p } - GenerationAbsent => NamedWindowVerified { probe: ProbedAbsent } - GenerationPresent { content: c } => - match cas_generation_successor(g: generation) { - CasSuccessorExhausted { head: _ } => NamedWindowVerified { probe: ProbedHead { generation: generation, value: c } } - CasSuccessorGeneration { generation: next } => - match cas_read_generation(root: root, key: key, generation: next) { - GenerationRefused { probe: p } => NamedWindowVerified { probe: p } - GenerationAbsent => NamedWindowVerified { probe: ProbedHead { generation: generation, value: c } } - GenerationPresent { content: _ } => NamedWindowHeadMoved { generation: generation } - } - } + if !cas_key_is_slot_addressable(key: key) { + NamedWindowHeadCurrent { observation: CasObservedUnreadable { cause: CasUnreadableReadRefused { detail: cas_key_not_slot_addressable_detail } } } + } else { + match cas_read_generation(root: root, key: key, generation: generation) { + GenerationRefused { probe: p } => NamedWindowHeadCurrent { observation: cas_probe_as_observation(probe: p) } + GenerationAbsent => NamedWindowHeadCurrent { observation: cas_probe_as_observation(probe: ProbedAbsent) } + GenerationPresent { content: c } => + match cas_generation_successor(g: generation) { + CasSuccessorExhausted { head: _ } => + NamedWindowHeadCurrent { observation: cas_probe_as_observation(probe: ProbedHead { generation: generation, value: c }) } + CasSuccessorGeneration { generation: next } => + match cas_read_generation(root: root, key: key, generation: next) { + GenerationRefused { probe: p } => NamedWindowHeadCurrent { observation: cas_probe_as_observation(probe: p) } + GenerationAbsent => + NamedWindowHeadCurrent { observation: cas_probe_as_observation(probe: ProbedHead { generation: generation, value: c }) } + GenerationPresent { content: _ } => NamedWindowHeadMoved { named_generation: generation } + } + } + } } } diff --git a/dag/gunbc/durable_exclusive_hold_file_store.dag b/dag/gunbc/durable_exclusive_hold_file_store.dag index 86167e13620..f97be05abc2 100644 --- a/dag/gunbc/durable_exclusive_hold_file_store.dag +++ b/dag/gunbc/durable_exclusive_hold_file_store.dag @@ -28,7 +28,7 @@ import std.durable_compare_and_set { CasExpectation, ExpectSlotGeneration, CasCommitted, CasPreconditionFailed, CasStoreRefused, CasStoreFailure, CasSlotObservationRefused, CasSlotObservation, CasObservedReadable, CasObservedUnreadable, - CasReadableAbsent, CasReadablePresent, CasGeneration, CasUnreadableReadRefused, + CasReadableAbsent, CasReadablePresent, CasGeneration, } import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, @@ -37,18 +37,17 @@ import std.durable_exclusive_hold { HoldSlotUndecodable, HoldObservationUnavailable, DurableHoldAcquireRefusal, HoldAcquireEligible, HoldAcquireRefused, HoldAcquireOccupied, HoldAcquireSlotUndecodable, HoldAcquireObservationUnavailable, - HoldAcquireNamedWindowHeadMoved, HoldNamedWindowHeadMoved, - HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseRefusal, + HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseAssessment, DurableHoldReleaseRefusal, HoldReleaseNamedWindowHeadMoved, durable_hold_acquire_assess, durable_hold_release_assess, - DurableHoldHolderReport, DurableHoldRecoveryRefusal, HoldRecoveryEligible, HoldRecoveryRefused, + DurableHoldHolderReport, DurableHoldRecoveryRefusal, DurableHoldRecoveryAssessment, HoldRecoveryEligible, HoldRecoveryRefused, + HoldRecoveryNamedWindowHeadMoved, durable_hold_recovery_assess, } import gunbc.durable_cas_file_store { KeepAllGenerations, KeepGenerationWindow, CasSlotRetention, CasGenerationWindow, DefaultAccessCreateOnly, DerivedPayloadAdmitted, DerivedPayloadKeyNotSlotAddressable, admit_cas_attempt_for_derived_payload, file_compare_and_set, file_compare_and_set_retained, observe_cas_slot_state, observe_cas_slot_state_retained, - cas_verify_named_window_head, cas_probe_as_observation, cas_key_is_slot_addressable, - cas_key_not_slot_addressable_detail, NamedWindowVerified, NamedWindowHeadMoved, + cas_verify_named_window_head, CasNamedWindowVerify, NamedWindowHeadCurrent, NamedWindowHeadMoved, } // THE TAGS ARE NAMED ONCE AND THE BODY IS DERIVED FROM THE TAG THAT MATCHED. @@ -158,19 +157,38 @@ fn observe_file_hold(root: NonEmptyStr, key: NonEmptyStr) -> DurableHoldObservat observe_file_hold_retained(root: root, key: key, retention: KeepAllGenerations) } -fn observe_file_hold_windowed_at(root: NonEmptyStr, key: NonEmptyStr, generation: CasGeneration) -> DurableHoldObservation +fn windowed_hold_release_from_verify( + verify: CasNamedWindowVerify, + owner: DurableHoldOwnerRef, + acquired_generation: CasGeneration, +) -> DurableHoldReleaseAssessment admit_callers: [ - decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_hold_check"), decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess_windowed_at"), + decl_ref(module_path: "test.claim.durable_exclusive_hold_witness", decl_name: "named_window_head_moved_release_does_not_claim_an_observed_current_generation"), ] { - if !cas_key_is_slot_addressable(key: key) { - HoldObservationUnavailable { cause: CasUnreadableReadRefused { detail: cas_key_not_slot_addressable_detail } } - } else { - match cas_verify_named_window_head(root: root, key: key, generation: generation) { - NamedWindowHeadMoved { generation: g } => HoldNamedWindowHeadMoved { generation: g } - NamedWindowVerified { probe: p } => hold_observation_of_cas(observed: cas_probe_as_observation(probe: p)) - } + match verify { + NamedWindowHeadMoved { named_generation: n } => + HoldReleaseRefused { refusal: HoldReleaseNamedWindowHeadMoved { named: n } } + NamedWindowHeadCurrent { observation: o } => + durable_hold_release_assess(observed: hold_observation_of_cas(observed: o), owner: owner, acquired_generation: acquired_generation) + } +} + +fn windowed_hold_recovery_from_verify( + verify: CasNamedWindowVerify, + report: DurableHoldHolderReport, +) -> DurableHoldRecoveryAssessment + admit_callers: [ + decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_recovery_assess_windowed"), + decl_ref(module_path: "test.claim.durable_exclusive_hold_witness", decl_name: "named_window_head_moved_recovery_does_not_claim_an_observed_current_generation"), + ] +{ + match verify { + NamedWindowHeadMoved { named_generation: n } => + HoldRecoveryRefused { refusal: HoldRecoveryNamedWindowHeadMoved { named: n } } + NamedWindowHeadCurrent { observation: o } => + durable_hold_recovery_assess(observed: hold_observation_of_cas(observed: o), report: report) } } @@ -299,7 +317,6 @@ fn file_hold_acquire_retained( FileHoldSlotUndecodable { slot_key: slot_key, generation: g, detail: d } HoldAcquireObservationUnavailable { cause: c } => FileHoldAcquireStoreUnavailable { slot_key: slot_key, cause: CasSlotObservationRefused { cause: c } } - HoldAcquireNamedWindowHeadMoved { generation: _ } => FileHoldAcquireLost { slot_key: slot_key } } } } @@ -383,8 +400,8 @@ fn file_hold_release_assess_windowed_at(root: NonEmptyStr, slot_key: NonEmptyStr decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_release_family_hold"), ] { - match durable_hold_release_assess( - observed: observe_file_hold_windowed_at(root: root, key: slot_key, generation: acquired_generation), + match windowed_hold_release_from_verify( + verify: cas_verify_named_window_head(root: root, key: slot_key, generation: acquired_generation), owner: owner, acquired_generation: acquired_generation, ) { @@ -453,7 +470,6 @@ fn file_hold_recovery_assess_retained( ) -> FileHoldRecoveryAssessment admit_callers: [ decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_recovery_assess"), - decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_recovery_assess_windowed"), ] { match durable_hold_recovery_assess(observed: observe_file_hold_retained(root: root, key: slot_key, retention: retention), report: report) { @@ -487,5 +503,23 @@ fn file_hold_recovery_assess_windowed(root: NonEmptyStr, slot_key: NonEmptyStr, decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_hold_occupied"), ] { - file_hold_recovery_assess_retained(root: root, slot_key: slot_key, report: report, released_by: released_by, retention: KeepGenerationWindow { window: window }) + match windowed_hold_recovery_from_verify( + verify: cas_verify_named_window_head(root: root, key: slot_key, generation: report.generation), + report: report, + ) { + HoldRecoveryRefused { refusal: r } => FileHoldRecoveryNotEligible { slot_key: slot_key, refusal: r } + HoldRecoveryEligible { generation: g, dead_holder: h, evidence: e } => + FileHoldRecoveryPlanned { + plan: FileHoldReleasePlan { + root: root, + slot_key: slot_key, + owner: h, + generation: g, + free_payload: hold_slot_payload(state: DurableHoldFree { released_by: released_by }), + retention: KeepGenerationWindow { window: window }, + }, + dead_holder: h, + evidence: e, + } + } } diff --git a/dag/gunbc/host/managed_host_unit_hold.dag b/dag/gunbc/host/managed_host_unit_hold.dag index f91af4a6c99..1e99a71f336 100644 --- a/dag/gunbc/host/managed_host_unit_hold.dag +++ b/dag/gunbc/host/managed_host_unit_hold.dag @@ -12,7 +12,7 @@ import std.durable_exclusive_hold { HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, HoldReleaseNamedWindowHeadMoved, HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, - HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, + HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, DurableHoldHolderLiveness, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, DurableHoldHolderReport, DurableHoldRecoveryRefusal, HoldRecoveryNothingHeld, HoldRecoveryHolderLive, HoldRecoveryHolderUnobservable, HoldRecoverySlotMoved, @@ -619,7 +619,6 @@ fn probe_hold_release_decision(observed: DurableHoldObservation, run_id: NonEmpt HoldSlotFree { generation: _, released_by: _ } => ProbeHoldFreeNoop HoldSlotUndecodable { generation: _, detail: d } => ProbeHoldUnobservable { cause: concat("slot undecodable: ", d as String) } HoldObservationUnavailable { cause: _ } => ProbeHoldUnobservable { cause: "the store could not be read" } - HoldNamedWindowHeadMoved { generation: _ } => ProbeHoldUnobservable { cause: "the store could not be read" } HoldSlotHeld { generation: g, owner: o } => match decode_unit_hold_owner(owner: o) { UnitHoldOwnerDecoded { owner: KvmObserverProbe { run_id: r, process: p, pid_namespace: n } } => @@ -884,7 +883,6 @@ fn maintenance_hold_release_on_store_host(executor: UnitHoldStoreExecutor) -> Pr HoldSlotFree { generation: _, released_by: b } => exit_failure(reason: join([label, ": already free (released by ", b as String, ")"], "")) HoldSlotUndecodable { generation: _, detail: d } => exit_failure(reason: join([label, ": slot undecodable: ", d as String], "")) HoldObservationUnavailable { cause: _ } => exit_failure(reason: concat(label, ": store unavailable")) - HoldNamedWindowHeadMoved { generation: _ } => exit_failure(reason: concat(label, ": store unavailable")) HoldSlotHeld { generation: g, owner: o } => match decode_unit_hold_owner(owner: o) { UnitHoldOwnerUnrecognized { raw: r } => diff --git a/dag/gunbc/workspace_allocation_plan.dag b/dag/gunbc/workspace_allocation_plan.dag index a932825a016..14841a634e8 100644 --- a/dag/gunbc/workspace_allocation_plan.dag +++ b/dag/gunbc/workspace_allocation_plan.dag @@ -13,7 +13,7 @@ import std.types { NonEmptyStr, String, Int } import std.measure { money_amount_micro } import std.content_hash { content_hash_of_value, serialize_content_hash } import std.durable_compare_and_set { ExpectSlotAbsent, ExpectSlotGeneration, cas_generation_count, cas_generation_successor, CasSuccessorGeneration, CasSuccessorExhausted } -import std.durable_exclusive_hold { DurableHoldObservation, DurableHoldOwnerRef, HoldSlotHeld, HoldSlotAbsent, HoldSlotFree, HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldAcquireOccupied, HoldAcquireSlotUndecodable, HoldAcquireObservationUnavailable, HoldAcquireNamedWindowHeadMoved } +import std.durable_exclusive_hold { DurableHoldObservation, DurableHoldOwnerRef, HoldSlotHeld, HoldSlotAbsent, HoldSlotFree, HoldSlotUndecodable, HoldObservationUnavailable, HoldAcquireOccupied, HoldAcquireSlotUndecodable, HoldAcquireObservationUnavailable } import extdeps.languages.json.emit { JsonValue, json_object, json_kv, json_string } import product.fabric.demand { Demand } import product.fabric.identity { FabricIdentity, ExecutionAttemptKey, fabric_identity_eq } @@ -85,7 +85,6 @@ fn workspace_reservation_prepare( HoldAcquireOccupied { generation: _, holder: _ } => WorkspaceReservationPreparationRefused { reason: "selected slot is occupied; no acquisition plan issued" } HoldAcquireSlotUndecodable { generation: _, detail } => WorkspaceReservationPreparationRefused { reason: detail } HoldAcquireObservationUnavailable { cause: _ } => WorkspaceReservationPreparationRefused { reason: "allocation store observation unavailable; no acquisition plan issued" } - HoldAcquireNamedWindowHeadMoved { generation: _ } => WorkspaceReservationPreparationRefused { reason: "selected slot head moved; no acquisition plan issued" } } } MoneyRefused { refusal: _ } => WorkspaceReservationPreparationRefused { reason: "monetary reservation refused" } @@ -140,7 +139,6 @@ fn workspace_reservation_apply_under_exclusion(root: NonEmptyStr, plan: Workspac HoldSlotFree { generation: _, released_by: _ } => workspace_reservation_acquire(root: root, key: key, owner: owner, reservation: reservation) HoldSlotUndecodable { generation: _, detail: _ } => workspace_reservation_acquire(root: root, key: key, owner: owner, reservation: reservation) HoldObservationUnavailable { cause: _ } => workspace_reservation_acquire(root: root, key: key, owner: owner, reservation: reservation) - HoldNamedWindowHeadMoved { generation: _ } => workspace_reservation_acquire(root: root, key: key, owner: owner, reservation: reservation) } } } @@ -194,7 +192,6 @@ fn workspace_actor_takeover_decision(plan: WorkspaceReservationPlan, hold: Durab else { TakeoverRetain { reason: "slot is held by another reservation or incarnation" } } HoldSlotUndecodable { generation: _, detail: _ } => TakeoverRetain { reason: "slot hold is undecodable; the acquisition obligation is retained" } HoldObservationUnavailable { cause: _ } => TakeoverRetain { reason: "slot hold is unreadable; the acquisition obligation is retained" } - HoldNamedWindowHeadMoved { generation: _ } => TakeoverRetain { reason: "slot hold is unreadable; the acquisition obligation is retained" } HoldSlotAbsent => workspace_actor_takeover_unheld(prior: prior, intent: intent) HoldSlotFree { generation: _, released_by: _ } => workspace_actor_takeover_unheld(prior: prior, intent: intent) } @@ -231,7 +228,6 @@ fn workspace_reservation_recover(root: NonEmptyStr, plan: WorkspaceReservationPl HoldSlotFree { generation: _, released_by: _ } => WorkspaceReservationApplyRefused { reason: "no matching held incarnation observed; do not acquire retired intent" } HoldSlotUndecodable { generation: _, detail: _ } => WorkspaceReservationApplyRefused { reason: "no matching held incarnation observed; do not acquire retired intent" } HoldObservationUnavailable { cause: _ } => WorkspaceReservationApplyRefused { reason: "no matching held incarnation observed; do not acquire retired intent" } - HoldNamedWindowHeadMoved { generation: _ } => WorkspaceReservationApplyRefused { reason: "no matching held incarnation observed; do not acquire retired intent" } } } } diff --git a/dag/gunbc/workspace_allocation_supply.dag b/dag/gunbc/workspace_allocation_supply.dag index 5025b49ac01..0a197a47db1 100644 --- a/dag/gunbc/workspace_allocation_supply.dag +++ b/dag/gunbc/workspace_allocation_supply.dag @@ -8,7 +8,7 @@ import extdeps.shell.exec { ProcessOutputPresent, ProcessOutputAbsent, ProcessRe import gunbc.command_runner { run_shell_command_capture } import std.decl_ref { DeclarationRef } import std.durable_compare_and_set { cas_generation_count, cas_generation_successor, CasSuccessorGeneration, CasSuccessorExhausted } -import std.durable_exclusive_hold { HoldSlotAbsent, HoldSlotFree, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotHeld, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotAbsent, HoldSlotFree, HoldObservationUnavailable, HoldSlotHeld, HoldSlotUndecodable } import product.host_identity { HostIdentity } import product.fabric.identity { fabric_identity_eq } import product.fabric.provider_route { CompliancePostureRevision } @@ -102,7 +102,6 @@ fn workspace_slot_observation_refusal(root: NonEmptyStr, slot: RunnerSlotIdentit HoldSlotHeld { generation: _, owner: _ } => none HoldSlotUndecodable { generation: _, detail: _ } => none HoldObservationUnavailable { cause: _ } => none - HoldNamedWindowHeadMoved { generation: _ } => none } match next { Absent => Present { value: "allocation slot is held, unreadable, or generation exhausted" } diff --git a/dag/gunbc/workspace_commissioning_admission_observe.dag b/dag/gunbc/workspace_commissioning_admission_observe.dag index 67b4e0613a2..ecc4695d0a8 100644 --- a/dag/gunbc/workspace_commissioning_admission_observe.dag +++ b/dag/gunbc/workspace_commissioning_admission_observe.dag @@ -19,7 +19,7 @@ import gunbc.workspace_initial_sanitation { workspace_observe_initial_sanitation import gunbc.runner_microvm_network { SlotNetworkSanitationSubject } import gunbc.fabric_control_plane { fabric_control_plane_store_root, fabric_cell_allocation_key } import gunbc.durable_exclusive_hold_file_store { observe_file_hold } -import std.durable_exclusive_hold { HoldSlotAbsent, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotAbsent, HoldObservationUnavailable, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable } import gunbc.runner_microvm_cell_readiness { StoreRootOwned, StoreNotRootOwned } import gunbc.runner_microvm_cell_readiness_store { read_store_ownership, OwnershipRead, OwnershipUnread } import gunbc.fleet_host_budget { HostAllocationConserves } @@ -75,7 +75,6 @@ fn workspace_commissioning_no_hold_refusal(provenance: WorkspaceCommissioningPro HoldSlotHeld { generation: _, owner: _ } => Present { value: "slot has held, released or unreadable allocation history; initial commissioning cannot reset it" } HoldSlotUndecodable { generation: _, detail: _ } => Present { value: "slot has held, released or unreadable allocation history; initial commissioning cannot reset it" } HoldObservationUnavailable { cause: _ } => Present { value: "slot has held, released or unreadable allocation history; initial commissioning cannot reset it" } - HoldNamedWindowHeadMoved { generation: _ } => Present { value: "slot has held, released or unreadable allocation history; initial commissioning cannot reset it" } } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => Present { value: "allocation history root is missing, unreadable or outside root custody" } OwnershipUnread { reason: _ } => Present { value: "allocation history root is missing, unreadable or outside root custody" } diff --git a/dag/gunbc/workspace_commissioning_executor_fence.dag b/dag/gunbc/workspace_commissioning_executor_fence.dag index 44fdb924f79..5481576869f 100644 --- a/dag/gunbc/workspace_commissioning_executor_fence.dag +++ b/dag/gunbc/workspace_commissioning_executor_fence.dag @@ -4,7 +4,7 @@ import gunbc.workspace_commissioning_start_identity { workspace_commissioning_sn import std.types { NonEmptyStr, String, Bool } import std.decl_ref { decl_ref } -import std.durable_exclusive_hold { HoldSlotHeld, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotFree, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotHeld, HoldObservationUnavailable, HoldSlotFree, HoldSlotUndecodable } import gunbc.build_cache_instance { runner_slot_instance_name } import gunbc.workspace_commissioning_location { workspace_commissioning_store_location } import gunbc.workspace_slot_mutation { workspace_slot_mutation_key, workspace_commissioning_mutation_owner } @@ -39,7 +39,6 @@ fn workspace_executor_mutation_matches(record: WorkspaceExecutorRecord) -> Bool HoldSlotFree { generation: _, released_by: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false - HoldNamedWindowHeadMoved { generation: _ } => false } } diff --git a/dag/gunbc/workspace_commissioning_executor_submission.dag b/dag/gunbc/workspace_commissioning_executor_submission.dag index d3e1a763af1..ecb46d8ab9b 100644 --- a/dag/gunbc/workspace_commissioning_executor_submission.dag +++ b/dag/gunbc/workspace_commissioning_executor_submission.dag @@ -4,7 +4,7 @@ import gunbc.workspace_commissioning_apply_readback { workspace_commissioning_co import gunbc.runner_microvm_slot_start { MicrovmInvocationWait, InvocationStillRunning, InvocationTerminated, InvocationWaitRefused, microvm_invocation_observe, microvm_unit_start_observed, MicrovmSlotStartRun, SlotStartCompleted, SlotStartWrongHost, SlotStartRefused, SlotStartAlreadyActive, SlotStartInvocationUnread, SlotStartInvocationUnchanged, SlotStartStillActiveAtBound } import std.types { String, NonEmptyStr, Bool } import std.decl_ref { decl_ref } -import std.durable_exclusive_hold { HoldSlotHeld, HoldSlotAbsent, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotFree, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotHeld, HoldSlotAbsent, HoldObservationUnavailable, HoldSlotFree, HoldSlotUndecodable } import std.content_hash { content_hash_validate_lower_hex_length } import std.durable_compare_and_set { CasGeneration } import extdeps.filesystem.filesystem_io { Filesystem, filesystem_exact_read, FilesystemExactPathRead, FilesystemExactPathAbsent, FilesystemExactPathKindUnrecognized, FilesystemExactPathUnreadable } @@ -112,7 +112,6 @@ fn workspace_executor_initial_guard(slot: NonEmptyStr, plan: NonEmptyStr) -> Cas HoldSlotFree { generation: _, released_by: _ } => none HoldSlotUndecodable { generation: _, detail: _ } => none HoldObservationUnavailable { cause: _ } => none - HoldNamedWindowHeadMoved { generation: _ } => none } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => none OwnershipUnread { reason: _ } => none diff --git a/dag/gunbc/workspace_commissioning_fleet.dag b/dag/gunbc/workspace_commissioning_fleet.dag index bd8557d8220..14839d08417 100644 --- a/dag/gunbc/workspace_commissioning_fleet.dag +++ b/dag/gunbc/workspace_commissioning_fleet.dag @@ -35,7 +35,7 @@ import gunbc.workspace_commissioning_executor_input { workspace_executor_install import gunbc.workspace_commissioning_location { workspace_commissioning_store_location } import gunbc.workspace_slot_mutation { workspace_slot_mutation_key } import gunbc.durable_exclusive_hold_file_store { observe_file_hold } -import std.durable_exclusive_hold { HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotUndecodable } +import std.durable_exclusive_hold { HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldObservationUnavailable, HoldSlotUndecodable } import gunbc.workspace_slot_mutation { workspace_commissioning_mutation_owner, workspace_mutation_recovery_standing, WorkspaceMutationRecoveryReleased } import gunbc.runner_microvm_cell_readiness_store { read_cell_readiness_store } import gunbc.workspace_commissioning_apply_readback { workspace_commissioning_initial_ready, workspace_commissioning_credential_absent } @@ -158,7 +158,6 @@ fn workspace_commissioning_basis_observe(provenance: WorkspaceCommissioningProve HoldSlotFree { generation: _, released_by: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false - HoldNamedWindowHeadMoved { generation: _ } => false }) if !committed && !initial { WorkspaceCommissioningFleetRefused { reason: "commissioning has existing history; resume its exact reviewed executor rather than minting a replacement plan" } } else { match resolve_network_subject(slot: slot) { diff --git a/dag/gunbc/workspace_slot_mutation.dag b/dag/gunbc/workspace_slot_mutation.dag index cc37daf3fcd..c1f3c9c0fac 100644 --- a/dag/gunbc/workspace_slot_mutation.dag +++ b/dag/gunbc/workspace_slot_mutation.dag @@ -3,7 +3,7 @@ module gunbc.workspace_slot_mutation import std.types { Bool, NonEmptyStr, String } import std.decl_ref { decl_ref } import std.durable_compare_and_set { CasGeneration, cas_generation_successor, CasSuccessorGeneration, CasSuccessorExhausted } -import std.durable_exclusive_hold { DurableHoldObservation, DurableHoldOwnerRef, DurableHoldReleaseRef, HoldSlotFree, HoldSlotHeld, HoldObservationUnavailable, HoldNamedWindowHeadMoved, HoldSlotUndecodable, HoldSlotAbsent } +import std.durable_exclusive_hold { DurableHoldObservation, DurableHoldOwnerRef, DurableHoldReleaseRef, HoldSlotFree, HoldSlotHeld, HoldObservationUnavailable, HoldSlotUndecodable, HoldSlotAbsent } import gunbc.build_cache_instance { runner_slot_instance_name } import gunbc.runner_slot_allocation { workspace_execution_slot_identities } import gunbc.workspace_commissioning_location { workspace_commissioning_store_location } @@ -100,7 +100,6 @@ fn workspace_slot_mutation_release_fenced(lease: WorkspaceSlotMutationLease) -> HoldSlotHeld { generation: _, owner: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false - HoldNamedWindowHeadMoved { generation: _ } => false } FileHoldReleaseLost { slot_key: _ } => false FileHoldReleaseStoreUnavailable { slot_key: _, cause: _ } => false @@ -128,7 +127,6 @@ fn workspace_commissioning_mutation_held(slot_key: NonEmptyStr, plan_hash: NonEm HoldSlotFree { generation: _, released_by: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false - HoldNamedWindowHeadMoved { generation: _ } => false } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => false OwnershipUnread { reason: _ } => false @@ -146,7 +144,6 @@ fn workspace_slot_mutation_supply_open(slot_key: NonEmptyStr) -> Bool { HoldSlotHeld { generation: _, owner: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false - HoldNamedWindowHeadMoved { generation: _ } => false } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => false OwnershipUnread { reason: _ } => false @@ -162,7 +159,6 @@ fn workspace_slot_mutation_still_owned(held: WorkspaceSlotMutationHeld) -> Bool HoldSlotFree { generation: _, released_by: _ } => false HoldSlotUndecodable { generation: _, detail: _ } => false HoldObservationUnavailable { cause: _ } => false - HoldNamedWindowHeadMoved { generation: _ } => false } OwnershipRead { ownership: StoreNotRootOwned { owner_uid: _, mode: _ } } => false OwnershipUnread { reason: _ } => false @@ -192,7 +188,6 @@ fn workspace_mutation_recovery_standing(observed: DurableHoldObservation, HoldSlotAbsent => WorkspaceMutationRecoveryRefused HoldSlotUndecodable { generation: _, detail: _ } => WorkspaceMutationRecoveryRefused HoldObservationUnavailable { cause: _ } => WorkspaceMutationRecoveryRefused - HoldNamedWindowHeadMoved { generation: _ } => WorkspaceMutationRecoveryRefused } } diff --git a/dag/std/durable_exclusive_hold.dag b/dag/std/durable_exclusive_hold.dag index 877c4494393..b5d63aadb83 100644 --- a/dag/std/durable_exclusive_hold.dag +++ b/dag/std/durable_exclusive_hold.dag @@ -70,7 +70,6 @@ type DurableHoldObservation | HoldSlotHeld { generation: CasGeneration, owner: DurableHoldOwnerRef } | HoldSlotUndecodable { generation: CasGeneration, detail: NonEmptyStr } | HoldObservationUnavailable { cause: CasUnreadableSlot } - | HoldNamedWindowHeadMoved { generation: CasGeneration } // ACQUIRE DERIVES ITS EXPECTATION FROM THE OBSERVATION -- THE FIRST AMENDMENT. // @@ -92,7 +91,6 @@ type DurableHoldAcquireRefusal = HoldAcquireOccupied { generation: CasGeneration, holder: DurableHoldOwnerRef } | HoldAcquireSlotUndecodable { generation: CasGeneration, detail: NonEmptyStr } | HoldAcquireObservationUnavailable { cause: CasUnreadableSlot } - | HoldAcquireNamedWindowHeadMoved { generation: CasGeneration } type DurableHoldAcquireAssessment = HoldAcquireEligible { expected: CasExpectation } @@ -109,8 +107,6 @@ fn durable_hold_acquire_assess(observed: DurableHoldObservation) -> DurableHoldA HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: g, detail: d } } HoldObservationUnavailable { cause: c } => HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: c } } - HoldNamedWindowHeadMoved { generation: g } => - HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: g } } } } @@ -160,8 +156,6 @@ fn durable_hold_release_assess( HoldReleaseRefused { refusal: HoldReleaseSlotUndecodable { generation: g, detail: d } } HoldObservationUnavailable { cause: c } => HoldReleaseRefused { refusal: HoldReleaseObservationUnavailable { cause: c } } - HoldNamedWindowHeadMoved { generation: g } => - HoldReleaseRefused { refusal: HoldReleaseNamedWindowHeadMoved { named: g } } HoldSlotHeld { generation: g, owner: holder } => if !durable_hold_owner_equal(left: holder, right: owner) { HoldReleaseRefused { refusal: HoldReleaseHeldByAnotherOwner { generation: g, holder: holder } } @@ -229,8 +223,6 @@ fn durable_hold_recovery_assess(observed: DurableHoldObservation, report: Durabl HoldRecoveryRefused { refusal: HoldRecoverySlotUndecodable { generation: g, detail: d } } HoldObservationUnavailable { cause: c } => HoldRecoveryRefused { refusal: HoldRecoveryObservationUnavailable { cause: c } } - HoldNamedWindowHeadMoved { generation: g } => - HoldRecoveryRefused { refusal: HoldRecoveryNamedWindowHeadMoved { named: g } } HoldSlotHeld { generation: g, owner: holder } => if !durable_hold_owner_equal(left: holder, right: report.holder) { HoldRecoveryRefused { refusal: HoldRecoveryReportForAnotherHolder { holder: holder, reported: report.holder } } diff --git a/dag/test/claim/durable_exclusive_hold_witness_test.dag b/dag/test/claim/durable_exclusive_hold_witness_test.dag index 22c1a15d0b9..7bf4af250a4 100644 --- a/dag/test/claim/durable_exclusive_hold_witness_test.dag +++ b/dag/test/claim/durable_exclusive_hold_witness_test.dag @@ -13,10 +13,10 @@ import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldState, DurableHoldHeld, DurableHoldFree, DurableHoldObservation, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, - HoldSlotUndecodable, HoldObservationUnavailable, HoldNamedWindowHeadMoved, + HoldSlotUndecodable, HoldObservationUnavailable, DurableHoldAcquireAssessment, HoldAcquireEligible, HoldAcquireRefused, DurableHoldAcquireRefusal, HoldAcquireOccupied, HoldAcquireSlotUndecodable, - HoldAcquireObservationUnavailable, HoldAcquireNamedWindowHeadMoved, + HoldAcquireObservationUnavailable, DurableHoldReleaseAssessment, HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, @@ -32,7 +32,9 @@ import std.durable_exclusive_hold { import gunbc.durable_exclusive_hold_file_store { HoldPayloadDecode, HoldPayloadDecoded, HoldPayloadUndecodable, hold_slot_payload, decode_hold_slot_payload, hold_observation_of_cas, + windowed_hold_release_from_verify, windowed_hold_recovery_from_verify, } +import gunbc.durable_cas_file_store { NamedWindowHeadMoved } // THE PURE HALF OF THE HOLD PROTOCOL AND ITS CODEC. Every row here decides // from a supplied observation over the PURE assessments; nothing reaches a @@ -119,7 +121,6 @@ test fn a_bare_free_tag_is_not_an_acquirable_free_slot() -> Bool { HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: g, detail: _ } } => cas_generation_count(g: g) == 3 HoldAcquireRefused { refusal: HoldAcquireOccupied { generation: _, holder: _ } } => false HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: _ } } => false - HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: _ } } => false HoldAcquireEligible { expected: _ } => false } } @@ -192,7 +193,6 @@ fn eh_acquire_occupied_by(a: DurableHoldAcquireAssessment, holder: String) -> Bo HoldAcquireRefused { refusal: HoldAcquireOccupied { generation: _, holder: h } } => (h as String) == holder HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: _, detail: _ } } => false HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: _ } } => false - HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: _ } } => false HoldAcquireEligible { expected: _ } => false } } @@ -219,7 +219,6 @@ test fn an_undecodable_slot_is_neither_acquirable_nor_occupied() -> Bool { HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: g, detail: _ } } => cas_generation_count(g: g) == 2 HoldAcquireRefused { refusal: HoldAcquireOccupied { generation: _, holder: _ } } => false HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: _ } } => false - HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: _ } } => false HoldAcquireEligible { expected: _ } => false } } @@ -229,7 +228,6 @@ test fn an_unavailable_observation_is_not_acquirable() -> Bool { HoldAcquireRefused { refusal: HoldAcquireObservationUnavailable { cause: _ } } => true HoldAcquireRefused { refusal: HoldAcquireOccupied { generation: _, holder: _ } } => false HoldAcquireRefused { refusal: HoldAcquireSlotUndecodable { generation: _, detail: _ } } => false - HoldAcquireRefused { refusal: HoldAcquireNamedWindowHeadMoved { generation: _ } } => false HoldAcquireEligible { expected: _ } => false } } @@ -376,7 +374,11 @@ test fn a_dead_verdict_recovers_only_the_hold_it_was_taken_of() -> Bool { } test fn named_window_head_moved_release_does_not_claim_an_observed_current_generation() -> Bool { - match eh_release_assess(observed: HoldNamedWindowHeadMoved { generation: 3 }, owner: "demand-7|offer-1", acquired: 3) { + match windowed_hold_release_from_verify( + verify: NamedWindowHeadMoved { named_generation: 3 }, + owner: eh_owner(text: "demand-7|offer-1"), + acquired_generation: 3, + ) { HoldReleaseRefused { refusal: HoldReleaseNamedWindowHeadMoved { named: n } } => cas_generation_count(g: n) == 3 HoldReleaseRefused { refusal: HoldReleaseAgainstAnotherGeneration { acquired: _, observed: _ } } => false @@ -386,7 +388,7 @@ test fn named_window_head_moved_release_does_not_claim_an_observed_current_gener test fn named_window_head_moved_recovery_does_not_claim_an_observed_current_generation() -> Bool { let dead = eh_report(holder: "mtcollins1-boot:4242", generation: 3, dead: 1) - match durable_hold_recovery_assess(observed: HoldNamedWindowHeadMoved { generation: 3 }, report: dead) { + match windowed_hold_recovery_from_verify(verify: NamedWindowHeadMoved { named_generation: 3 }, report: dead) { HoldRecoveryRefused { refusal: HoldRecoveryNamedWindowHeadMoved { named: n } } => cas_generation_count(g: n) == 3 HoldRecoveryRefused { refusal: HoldRecoverySlotMoved { reported: _, observed: _ } } => false diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 88f96c30f85..07829df6f4a 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -39,7 +39,7 @@ import extdeps.realization.materialization_store_local { local_store_record_recency, local_store_is_object_name, local_store_is_store_file_name, local_store_object_bytes, local_store_family_budget, LocalStoreFamilyBudget, local_store_budget_sum, LocalStoreRecencyHoldRefused, LocalStoreRecencyReceipt, - local_store_with_family, LocalStoreBracket, LocalStoreBracketRan, LocalStoreBracketRefused, local_store_commit_under, LocalStoreHoldStale, LocalStoreHoldWrongFamily, + local_store_with_family, LocalStoreBracket, LocalStoreBracketRan, LocalStoreBracketRefused, local_store_commit_under, LocalStoreHoldStale, LocalStoreHoldNamedWindowHeadMoved, LocalStoreHoldWrongFamily, local_store_hold_key, local_store_hold_owner_prefix, LocalStoreHoldAcquisition, LocalStoreHoldAcquired, LocalStoreHoldRefused, LocalStoreHoldRefusal, LocalStoreFamilyBusy, LocalStoreHolderUnobservable, LocalStoreHoldRecoveryRefused, LocalStoreHoldContended, LocalStoreHoldStoreUnavailable, @@ -57,8 +57,6 @@ import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, Fil import gunbc.durable_cas_file_store { cas_file_slot_path } import std.durable_exclusive_hold { DurableHoldOwnerRef, - HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, - HoldNamedWindowHeadMoved, } import std.fabric_blob { fabric_blob_reading_path } import std.cache_identity { native_module_verdict_bundle_artifact_kind, typed_module_artifact_kind } @@ -635,6 +633,7 @@ fn hold_refusal_is_busy(r: LocalStoreHoldRefusal?) -> Bool { LocalStoreHoldWindowTooNarrow { k: _ } => false LocalStoreHoldUnsettled { settlement: _ } => false LocalStoreHoldStale { observed: _ } => false + LocalStoreHoldNamedWindowHeadMoved { named: _ } => false LocalStoreHoldWrongFamily { held: _, requested: _ } => false } Absent => false @@ -672,14 +671,8 @@ test fn a_commit_while_the_family_is_held_refuses_and_writes_nothing_by_real_exe fn hold_refusal_is_named_window_head_moved(r: LocalStoreHoldRefusal?) -> Bool { match r { Present { value: x } => match x { - LocalStoreHoldStale { observed: o } => match o { - HoldNamedWindowHeadMoved { generation: _ } => true - HoldObservationUnavailable { cause: _ } => false - HoldSlotAbsent => false - HoldSlotFree { generation: _, released_by: _ } => false - HoldSlotHeld { generation: _, owner: _ } => false - HoldSlotUndecodable { generation: _, detail: _ } => false - } + LocalStoreHoldNamedWindowHeadMoved { named: _ } => true + LocalStoreHoldStale { observed: _ } => false LocalStoreFamilyBusy { holder: _, evidence: _ } => false LocalStoreHolderUnobservable { holder: _, cause: _ } => false LocalStoreHoldRecoveryRefused { refusal: _ } => false @@ -710,6 +703,7 @@ fn hold_refusal_is_stale(r: LocalStoreHoldRefusal?) -> Bool { LocalStoreHoldSelfUnidentified { cause: _ } => false LocalStoreHoldWindowTooNarrow { k: _ } => false LocalStoreHoldUnsettled { settlement: _ } => false + LocalStoreHoldNamedWindowHeadMoved { named: _ } => false LocalStoreHoldWrongFamily { held: _, requested: _ } => false } Absent => false @@ -749,8 +743,8 @@ fn pin_generation_unlink(path: String, on: Bool) -> Bool { // PARTIAL RECLAIM MUST NOT RE-AUTHORIZE A LEAKED HOLD. Independent deletes could remove G+1 while // G survived, so named verify would treat leaked G as head. Pin G against unlink, run another // acquire/release (the reclaim that used to punch that hole), then reuse G. commit_under observes -// the hold first: named verify must answer HoldNamedWindowHeadMoved (G present, G+1 still there), mapped to -// LocalStoreHoldStale, before any occupancy write. occupancy_unavailable on +// the hold first: named verify must answer NamedWindowHeadMoved (G present, G+1 still there), mapped to +// LocalStoreHoldNamedWindowHeadMoved, before any occupancy write. occupancy_unavailable on // that path is the hold-refuse wrapper, not an occupancy fault from the pin. ioctl-set IMMUTABLE // must succeed (CAP_LINUX_IMMUTABLE); without it this claim is false, not vacuously green. test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution() -> Bool { From b5fa4c54583c1803ff2b5263171bb5723e48b77a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 14:10:15 +0000 Subject: [PATCH 19/30] Delete the typecheck_module_store_session forwarding alias. It was a second name for local_store_held_session with no production caller. The wet N-vs-2N inhabitance claim now calls the session operation; v2.compiler.compile is the named later consumer that will admit itself when typecheck writes leave per-commit local_store_commit. Co-authored-by: Cursor --- .../materialization_store_local.dag | 11 ++++++++--- dag/gunbc/typecheck_module_store.dag | 18 ------------------ ...ialization_store_local_wet_witness_test.dag | 7 ++----- 3 files changed, 10 insertions(+), 26 deletions(-) delete mode 100644 dag/gunbc/typecheck_module_store.dag diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag index 0d257ccdee4..b2330ef5157 100644 --- a/dag/extdeps/realization/materialization_store_local.dag +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -1273,8 +1273,13 @@ fn local_store_with_family_of(binding: LocalStoreBinding, family: ArtifactKin // A HELD SESSION: acquire the family once, run many lookups and commits under LocalStoreFamilyHeld, // release once. Distinct from local_store_commit, which still brackets every write; this operation -// does not change that signature. Sealed to gunbc.typecheck_module_store (and the inhabitance -// witness that proves N commits acquire once). +// does not change that signature. +// Consumed now by test.claim.materialization_store_local_wet_witness +// n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution (inhabitance). +// Named later consumer: v2.compiler.compile, when its typecheck writes leave per-commit +// local_store_commit (the host typed_module_cache still does) and take one session around the +// family's writes; that change adds an admit_callers row here. Until then this is not a second +// name for local_store_with_family_of -- it is the sealed session operation. // // DEADLINE AND RENEWAL follow std.durable_exclusive_hold, not a clock. That protocol recovers a // hold only when its holder is OBSERVED dead -- there is no timestamp, duration or silent expiry. @@ -1284,7 +1289,7 @@ fn local_store_with_family_of(binding: LocalStoreBinding, family: ArtifactKin // whose process has died is recovered by the next writer's liveness observation, never by age. fn local_store_held_session(capability: LocalStoreCapability, family: ArtifactKindId, run: fn(LocalStoreFamilyHeld) -> T) -> LocalStoreBracket admit_callers: [ - decl_ref(module_path: "gunbc.typecheck_module_store", decl_name: "typecheck_module_store_session"), + decl_ref(module_path: "test.claim.materialization_store_local_wet_witness", decl_name: "n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution"), ] { local_store_with_family_of(binding: capability.binding, family: family, run: run) diff --git a/dag/gunbc/typecheck_module_store.dag b/dag/gunbc/typecheck_module_store.dag deleted file mode 100644 index 5b5455286c4..00000000000 --- a/dag/gunbc/typecheck_module_store.dag +++ /dev/null @@ -1,18 +0,0 @@ -module gunbc.typecheck_module_store - -// THE TYPECHECK MODULE STORE SESSION: the admitted consumer of -// extdeps.realization.materialization_store_local local_store_held_session. -// Inhabitance is n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution. -// Production cutover trigger: the typecheck compile driver that still brackets each write -// through local_store_commit switches those writes to typecheck_module_store_session. -// This module is the seal's inhabitant, not a second store. - -import std.cache_identity { ArtifactKindId } -import extdeps.realization.materialization_store_local { - LocalStoreCapability, LocalStoreFamilyHeld, LocalStoreBracket, - local_store_held_session, -} - -fn typecheck_module_store_session(capability: LocalStoreCapability, family: ArtifactKindId, run: fn(LocalStoreFamilyHeld) -> T) -> LocalStoreBracket { - local_store_held_session(capability: capability, family: family, run: run) -} diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 07829df6f4a..894ed923b88 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -39,7 +39,7 @@ import extdeps.realization.materialization_store_local { local_store_record_recency, local_store_is_object_name, local_store_is_store_file_name, local_store_object_bytes, local_store_family_budget, LocalStoreFamilyBudget, local_store_budget_sum, LocalStoreRecencyHoldRefused, LocalStoreRecencyReceipt, - local_store_with_family, LocalStoreBracket, LocalStoreBracketRan, LocalStoreBracketRefused, local_store_commit_under, LocalStoreHoldStale, LocalStoreHoldNamedWindowHeadMoved, LocalStoreHoldWrongFamily, + local_store_with_family, LocalStoreBracket, LocalStoreBracketRan, LocalStoreBracketRefused, local_store_commit_under, local_store_held_session, LocalStoreHoldStale, LocalStoreHoldNamedWindowHeadMoved, LocalStoreHoldWrongFamily, local_store_hold_key, local_store_hold_owner_prefix, LocalStoreHoldAcquisition, LocalStoreHoldAcquired, LocalStoreHoldRefused, LocalStoreHoldRefusal, LocalStoreFamilyBusy, LocalStoreHolderUnobservable, LocalStoreHoldRecoveryRefused, LocalStoreHoldContended, LocalStoreHoldStoreUnavailable, @@ -48,9 +48,6 @@ import extdeps.realization.materialization_store_local { local_store_index_window, local_store_digits_int, LocalStoreRecordDeleteAttempt, LocalStoreRecordGone, LocalStoreRecordFileRefused, LocalStoreRecordPartsRefused, LocalStoreRecordPartsUnlisted, LocalStoreStagedPart, LocalStorePartsLookup, local_store_lookup_parts, local_store_part_name, } -import gunbc.typecheck_module_store { - typecheck_module_store_session, -} import gunbc.process_hold_identity { self_process_identity, SelfIdentified, SelfUnidentified, process_hold_text } import gunbc.build_cache_instance { ProcessIdentity } import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable } @@ -1450,7 +1447,7 @@ test fn n_commits_under_one_held_session_acquire_the_hold_once_by_real_execution Absent => false Present { value: store } => { let before = hold_slot_head_generation(root_path: session_root_path) - match typecheck_module_store_session(capability: store, family: native_module_verdict_bundle_artifact_kind, run: h => { + match local_store_held_session(capability: store, family: native_module_verdict_bundle_artifact_kind, run: h => { let writes = [1, 2, 3, 4, 5, 6] |> map(i => local_store_commit_under(held: h, req: bundle_req(module: concat("s", i as String)), payloads: verdicts(text: "s pass"), parts: [])) let hits = [1, 2, 3, 4, 5, 6] |> map(i => store_lookup_is_hit(l: local_store_lookup(capability: store, req: bundle_req(module: concat("s", i as String))))) writes |> all(c => session_commit_ok(c: c)) && hits |> all(x => x) From 0325b4838a01b27cf55d0769170eabcc6d1ee56f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 14:23:36 +0000 Subject: [PATCH 20/30] Drop CasReclaimProgress.stopped; the failed list already stops the walk. Stop-at-first-failure means a non-empty failed list is the same fact as the boolean, so the walk now keys off that list. Co-authored-by: Cursor --- dag/gunbc/durable_cas_file_store.dag | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dag/gunbc/durable_cas_file_store.dag b/dag/gunbc/durable_cas_file_store.dag index d0b4dd16796..99b3ec830c2 100644 --- a/dag/gunbc/durable_cas_file_store.dag +++ b/dag/gunbc/durable_cas_file_store.dag @@ -1101,7 +1101,6 @@ type CasFileWrite { type CasReclaimProgress { removed: List failed: List - stopped: Bool } // ASCENDING, AND STOP AT THE FIRST HOST REFUSAL. Independent deletes of every generation at or @@ -1121,16 +1120,15 @@ fn cas_reclaim_below(root: NonEmptyStr, key: NonEmptyStr, window: CasGenerationW CasWindowGenerationsListed { generations: gs } => { let floor = cas_generation_count(g: head) - window.k let ordered = gs |> filter(g => cas_generation_count(g: g) <= floor) |> sort_by(g => cas_generation_count(g: g)) - let walk = fold(ordered, init: CasReclaimProgress { removed: [], failed: [], stopped: false }, f: (acc, g) => - if acc.stopped { acc } else { + let walk = fold(ordered, init: CasReclaimProgress { removed: [], failed: [] }, f: (acc, g) => + if count(acc.failed) != 0 { acc } else { let d = Filesystem.Delete(path: cas_file_slot_path(root: root, key: key, generation: g)) if d.success { - CasReclaimProgress { removed: acc.removed |> list_push(g), failed: acc.failed, stopped: false } + CasReclaimProgress { removed: acc.removed |> list_push(g), failed: acc.failed } } else { CasReclaimProgress { removed: acc.removed, failed: [CasGenerationReclaimFailure { generation: g, host_error: d.error }], - stopped: true, } } } From caae4966bcb6364d160f7c12fa4c1fd9e0f597ce Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 14:42:17 +0000 Subject: [PATCH 21/30] Keep named-window head-moved off the generic hold protocol. Windowed named verify is a file-store realization outcome consumed by materialization_store_local; keep-all fabric and host matches no longer carry dead NamedWindow arms. BoundExceeded now renders as an attempt budget rather than a generation. Co-authored-by: Cursor --- .../materialization_store_local.dag | 15 +-- .../durable_exclusive_hold_file_store.dag | 102 ++++++++++++------ dag/gunbc/fabric/fabric_control_plane.dag | 9 -- dag/gunbc/host/managed_host_unit_hold.dag | 4 - dag/std/durable_compare_and_set.dag | 7 +- dag/std/durable_exclusive_hold.dag | 2 - .../durable_compare_and_set_witness_test.dag | 2 + .../durable_exclusive_hold_witness_test.dag | 13 ++- 8 files changed, 91 insertions(+), 63 deletions(-) diff --git a/dag/extdeps/realization/materialization_store_local.dag b/dag/extdeps/realization/materialization_store_local.dag index b2330ef5157..69c9b474502 100644 --- a/dag/extdeps/realization/materialization_store_local.dag +++ b/dag/extdeps/realization/materialization_store_local.dag @@ -92,9 +92,9 @@ import std.durable_exclusive_hold { import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquireOutcome, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable, - file_hold_release_assess_windowed_at, FileHoldReleasePlanned, FileHoldReleaseNotEligible, + file_hold_release_assess_windowed_at, FileHoldWindowedReleasePlanned, FileHoldWindowedReleaseNotEligible, FileHoldWindowedReleaseNamedHeadMoved, file_hold_release_commit, FileHoldReleased, FileHoldReleaseLost, FileHoldReleaseStoreUnavailable, FileHoldReleaseKeyNotSlotAddressable, - file_hold_recovery_assess_windowed, FileHoldRecoveryPlanned, FileHoldRecoveryNotEligible, + file_hold_recovery_assess_windowed, FileHoldWindowedRecoveryPlanned, FileHoldWindowedRecoveryNotEligible, FileHoldWindowedRecoveryNamedHeadMoved, hold_observation_of_cas, } import gunbc.process_hold_identity { @@ -1183,8 +1183,9 @@ fn local_store_hold_occupied(binding: LocalStoreBinding, family: ArtifactKindId, LocalStoreHoldRefused { refusal: LocalStoreHoldContended } } else { match file_hold_recovery_assess_windowed(root: root_path as NonEmptyStr, slot_key: key, report: DurableHoldHolderReport { holder: holder, generation: generation, liveness: liveness }, released_by: local_store_hold_recovery_ref, window: window) { - FileHoldRecoveryNotEligible { slot_key: _, refusal: r } => LocalStoreHoldRefused { refusal: LocalStoreHoldRecoveryRefused { refusal: r } } - FileHoldRecoveryPlanned { plan: pl, dead_holder: _, evidence: _ } => + FileHoldWindowedRecoveryNotEligible { slot_key: _, refusal: r } => LocalStoreHoldRefused { refusal: LocalStoreHoldRecoveryRefused { refusal: r } } + FileHoldWindowedRecoveryNamedHeadMoved { slot_key: _, named: n } => LocalStoreHoldRefused { refusal: LocalStoreHoldNamedWindowHeadMoved { named: n } } + FileHoldWindowedRecoveryPlanned { plan: pl, dead_holder: _, evidence: _ } => match file_hold_release_commit(plan: pl) { FileHoldReleased { slot_key: _, generation: _ } => local_store_acquire_family_hold(binding: binding, family: family, recovered: true) FileHoldReleaseLost { slot_key: _ } => LocalStoreHoldRefused { refusal: LocalStoreHoldContended } @@ -1300,6 +1301,7 @@ fn local_store_held_session(capability: LocalStoreCapability, family: Artifac type LocalStoreHoldRelease = LocalStoreHoldReleased | LocalStoreHoldReleaseNotEligible { refusal: DurableHoldReleaseRefusal } + | LocalStoreHoldReleaseNamedWindowHeadMoved { named: CasGeneration } | LocalStoreHoldReleaseLost | LocalStoreHoldReleaseStoreUnavailable { cause: CasStoreFailure } | LocalStoreHoldReleaseKeyNotAddressable @@ -1312,8 +1314,9 @@ fn local_store_release_family_hold(held: LocalStoreFamilyHeld) -> LocalStoreHold ] { match file_hold_release_assess_windowed_at(root: materialization_store_root_path(root: held.binding.root) as NonEmptyStr, slot_key: local_store_hold_key(family: held.family), owner: held.owner, acquired_generation: held.generation, released_by: local_store_hold_release_ref, window: held.window) { - FileHoldReleaseNotEligible { slot_key: _, refusal: r } => LocalStoreHoldReleaseNotEligible { refusal: r } - FileHoldReleasePlanned { plan: pl } => + FileHoldWindowedReleaseNotEligible { slot_key: _, refusal: r } => LocalStoreHoldReleaseNotEligible { refusal: r } + FileHoldWindowedReleaseNamedHeadMoved { slot_key: _, named: n } => LocalStoreHoldReleaseNamedWindowHeadMoved { named: n } + FileHoldWindowedReleasePlanned { plan: pl } => match file_hold_release_commit(plan: pl) { FileHoldReleased { slot_key: _, generation: _ } => LocalStoreHoldReleased FileHoldReleaseLost { slot_key: _ } => LocalStoreHoldReleaseLost diff --git a/dag/gunbc/durable_exclusive_hold_file_store.dag b/dag/gunbc/durable_exclusive_hold_file_store.dag index f97be05abc2..1b1f6f35ba9 100644 --- a/dag/gunbc/durable_exclusive_hold_file_store.dag +++ b/dag/gunbc/durable_exclusive_hold_file_store.dag @@ -37,10 +37,9 @@ import std.durable_exclusive_hold { HoldSlotUndecodable, HoldObservationUnavailable, DurableHoldAcquireRefusal, HoldAcquireEligible, HoldAcquireRefused, HoldAcquireOccupied, HoldAcquireSlotUndecodable, HoldAcquireObservationUnavailable, - HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseAssessment, DurableHoldReleaseRefusal, HoldReleaseNamedWindowHeadMoved, + HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseAssessment, DurableHoldReleaseRefusal, durable_hold_acquire_assess, durable_hold_release_assess, DurableHoldHolderReport, DurableHoldRecoveryRefusal, DurableHoldRecoveryAssessment, HoldRecoveryEligible, HoldRecoveryRefused, - HoldRecoveryNamedWindowHeadMoved, durable_hold_recovery_assess, } import gunbc.durable_cas_file_store { KeepAllGenerations, KeepGenerationWindow, CasSlotRetention, @@ -157,11 +156,22 @@ fn observe_file_hold(root: NonEmptyStr, key: NonEmptyStr) -> DurableHoldObservat observe_file_hold_retained(root: root, key: key, retention: KeepAllGenerations) } +// Named-window head movement is a file/window realization outcome, not a +// protocol refusal. Keep-all consumers of DurableHoldReleaseAssessment never +// see it; only windowed named verify mints these arms. +type WindowedHoldReleaseFromVerify + = WindowedHoldReleaseProtocol { assessment: DurableHoldReleaseAssessment } + | WindowedHoldReleaseNamedHeadMoved { named: CasGeneration } + +type WindowedHoldRecoveryFromVerify + = WindowedHoldRecoveryProtocol { assessment: DurableHoldRecoveryAssessment } + | WindowedHoldRecoveryNamedHeadMoved { named: CasGeneration } + fn windowed_hold_release_from_verify( verify: CasNamedWindowVerify, owner: DurableHoldOwnerRef, acquired_generation: CasGeneration, -) -> DurableHoldReleaseAssessment +) -> WindowedHoldReleaseFromVerify admit_callers: [ decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_release_assess_windowed_at"), decl_ref(module_path: "test.claim.durable_exclusive_hold_witness", decl_name: "named_window_head_moved_release_does_not_claim_an_observed_current_generation"), @@ -169,16 +179,18 @@ fn windowed_hold_release_from_verify( { match verify { NamedWindowHeadMoved { named_generation: n } => - HoldReleaseRefused { refusal: HoldReleaseNamedWindowHeadMoved { named: n } } + WindowedHoldReleaseNamedHeadMoved { named: n } NamedWindowHeadCurrent { observation: o } => - durable_hold_release_assess(observed: hold_observation_of_cas(observed: o), owner: owner, acquired_generation: acquired_generation) + WindowedHoldReleaseProtocol { + assessment: durable_hold_release_assess(observed: hold_observation_of_cas(observed: o), owner: owner, acquired_generation: acquired_generation), + } } } fn windowed_hold_recovery_from_verify( verify: CasNamedWindowVerify, report: DurableHoldHolderReport, -) -> DurableHoldRecoveryAssessment +) -> WindowedHoldRecoveryFromVerify admit_callers: [ decl_ref(module_path: "gunbc.durable_exclusive_hold_file_store", decl_name: "file_hold_recovery_assess_windowed"), decl_ref(module_path: "test.claim.durable_exclusive_hold_witness", decl_name: "named_window_head_moved_recovery_does_not_claim_an_observed_current_generation"), @@ -186,9 +198,11 @@ fn windowed_hold_recovery_from_verify( { match verify { NamedWindowHeadMoved { named_generation: n } => - HoldRecoveryRefused { refusal: HoldRecoveryNamedWindowHeadMoved { named: n } } + WindowedHoldRecoveryNamedHeadMoved { named: n } NamedWindowHeadCurrent { observation: o } => - durable_hold_recovery_assess(observed: hold_observation_of_cas(observed: o), report: report) + WindowedHoldRecoveryProtocol { + assessment: durable_hold_recovery_assess(observed: hold_observation_of_cas(observed: o), report: report), + } } } @@ -354,6 +368,11 @@ type FileHoldReleaseAssessment = FileHoldReleasePlanned { plan: FileHoldReleasePlan } | FileHoldReleaseNotEligible { slot_key: NonEmptyStr, refusal: DurableHoldReleaseRefusal } +type FileHoldWindowedReleaseAssessment + = FileHoldWindowedReleasePlanned { plan: FileHoldReleasePlan } + | FileHoldWindowedReleaseNotEligible { slot_key: NonEmptyStr, refusal: DurableHoldReleaseRefusal } + | FileHoldWindowedReleaseNamedHeadMoved { slot_key: NonEmptyStr, named: CasGeneration } + // THE ONLY MINT OF A RELEASE PLAN READS THE SLOT ITSELF. There is deliberately // no variant over a supplied observation: a synthetic "held by A at N" handed to // such a mint would produce a plan that frees a real hold of B's at N, and the @@ -395,7 +414,7 @@ fn file_hold_release_assess(root: NonEmptyStr, slot_key: NonEmptyStr, owner: Dur file_hold_release_assess_retained(root: root, slot_key: slot_key, owner: owner, acquired_generation: acquired_generation, released_by: released_by, retention: KeepAllGenerations) } -fn file_hold_release_assess_windowed_at(root: NonEmptyStr, slot_key: NonEmptyStr, owner: DurableHoldOwnerRef, acquired_generation: CasGeneration, released_by: DurableHoldReleaseRef, window: CasGenerationWindow) -> FileHoldReleaseAssessment +fn file_hold_release_assess_windowed_at(root: NonEmptyStr, slot_key: NonEmptyStr, owner: DurableHoldOwnerRef, acquired_generation: CasGeneration, released_by: DurableHoldReleaseRef, window: CasGenerationWindow) -> FileHoldWindowedReleaseAssessment admit_callers: [ decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_release_family_hold"), ] @@ -405,17 +424,22 @@ fn file_hold_release_assess_windowed_at(root: NonEmptyStr, slot_key: NonEmptyStr owner: owner, acquired_generation: acquired_generation, ) { - HoldReleaseRefused { refusal: r } => FileHoldReleaseNotEligible { slot_key: slot_key, refusal: r } - HoldReleaseEligible { generation: g } => - FileHoldReleasePlanned { - plan: FileHoldReleasePlan { - root: root, - slot_key: slot_key, - owner: owner, - generation: g, - free_payload: hold_slot_payload(state: DurableHoldFree { released_by: released_by }), - retention: KeepGenerationWindow { window: window }, - }, + WindowedHoldReleaseNamedHeadMoved { named: n } => + FileHoldWindowedReleaseNamedHeadMoved { slot_key: slot_key, named: n } + WindowedHoldReleaseProtocol { assessment: a } => + match a { + HoldReleaseRefused { refusal: r } => FileHoldWindowedReleaseNotEligible { slot_key: slot_key, refusal: r } + HoldReleaseEligible { generation: g } => + FileHoldWindowedReleasePlanned { + plan: FileHoldReleasePlan { + root: root, + slot_key: slot_key, + owner: owner, + generation: g, + free_payload: hold_slot_payload(state: DurableHoldFree { released_by: released_by }), + retention: KeepGenerationWindow { window: window }, + }, + } } } } @@ -461,6 +485,11 @@ type FileHoldRecoveryAssessment = FileHoldRecoveryPlanned { plan: FileHoldReleasePlan, dead_holder: DurableHoldOwnerRef, evidence: NonEmptyStr } | FileHoldRecoveryNotEligible { slot_key: NonEmptyStr, refusal: DurableHoldRecoveryRefusal } +type FileHoldWindowedRecoveryAssessment + = FileHoldWindowedRecoveryPlanned { plan: FileHoldReleasePlan, dead_holder: DurableHoldOwnerRef, evidence: NonEmptyStr } + | FileHoldWindowedRecoveryNotEligible { slot_key: NonEmptyStr, refusal: DurableHoldRecoveryRefusal } + | FileHoldWindowedRecoveryNamedHeadMoved { slot_key: NonEmptyStr, named: CasGeneration } + fn file_hold_recovery_assess_retained( root: NonEmptyStr, slot_key: NonEmptyStr, @@ -498,7 +527,7 @@ fn file_hold_recovery_assess(root: NonEmptyStr, slot_key: NonEmptyStr, report: D file_hold_recovery_assess_retained(root: root, slot_key: slot_key, report: report, released_by: released_by, retention: KeepAllGenerations) } -fn file_hold_recovery_assess_windowed(root: NonEmptyStr, slot_key: NonEmptyStr, report: DurableHoldHolderReport, released_by: DurableHoldReleaseRef, window: CasGenerationWindow) -> FileHoldRecoveryAssessment +fn file_hold_recovery_assess_windowed(root: NonEmptyStr, slot_key: NonEmptyStr, report: DurableHoldHolderReport, released_by: DurableHoldReleaseRef, window: CasGenerationWindow) -> FileHoldWindowedRecoveryAssessment admit_callers: [ decl_ref(module_path: "extdeps.realization.materialization_store_local", decl_name: "local_store_hold_occupied"), ] @@ -507,19 +536,24 @@ fn file_hold_recovery_assess_windowed(root: NonEmptyStr, slot_key: NonEmptyStr, verify: cas_verify_named_window_head(root: root, key: slot_key, generation: report.generation), report: report, ) { - HoldRecoveryRefused { refusal: r } => FileHoldRecoveryNotEligible { slot_key: slot_key, refusal: r } - HoldRecoveryEligible { generation: g, dead_holder: h, evidence: e } => - FileHoldRecoveryPlanned { - plan: FileHoldReleasePlan { - root: root, - slot_key: slot_key, - owner: h, - generation: g, - free_payload: hold_slot_payload(state: DurableHoldFree { released_by: released_by }), - retention: KeepGenerationWindow { window: window }, - }, - dead_holder: h, - evidence: e, + WindowedHoldRecoveryNamedHeadMoved { named: n } => + FileHoldWindowedRecoveryNamedHeadMoved { slot_key: slot_key, named: n } + WindowedHoldRecoveryProtocol { assessment: a } => + match a { + HoldRecoveryRefused { refusal: r } => FileHoldWindowedRecoveryNotEligible { slot_key: slot_key, refusal: r } + HoldRecoveryEligible { generation: g, dead_holder: h, evidence: e } => + FileHoldWindowedRecoveryPlanned { + plan: FileHoldReleasePlan { + root: root, + slot_key: slot_key, + owner: h, + generation: g, + free_payload: hold_slot_payload(state: DurableHoldFree { released_by: released_by }), + retention: KeepGenerationWindow { window: window }, + }, + dead_holder: h, + evidence: e, + } } } } diff --git a/dag/gunbc/fabric/fabric_control_plane.dag b/dag/gunbc/fabric/fabric_control_plane.dag index 7bd9149192a..185ecc3d22a 100644 --- a/dag/gunbc/fabric/fabric_control_plane.dag +++ b/dag/gunbc/fabric/fabric_control_plane.dag @@ -19,7 +19,6 @@ import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, - HoldReleaseNamedWindowHeadMoved, HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, } import product.fabric.identity { FabricIdentity, OfferKey, DemandKey, WorkKey, fabric_identity_eq } @@ -737,14 +736,6 @@ fn end_cell_hold( to_string(cas_generation_count(g: a)), ) as NonEmptyStr, } - HoldReleaseNamedWindowHeadMoved { named: n } => - HoldEndSlotNotHeld { - slot_key: slot_key, - detail: concat( - "named window head moved beyond generation ", - to_string(cas_generation_count(g: n)), - ) as NonEmptyStr, - } HoldReleaseSlotUndecodable { generation: _, detail: d } => HoldEndSlotNotHeld { slot_key: slot_key, detail: d } } diff --git a/dag/gunbc/host/managed_host_unit_hold.dag b/dag/gunbc/host/managed_host_unit_hold.dag index 1e99a71f336..e249d59d7f0 100644 --- a/dag/gunbc/host/managed_host_unit_hold.dag +++ b/dag/gunbc/host/managed_host_unit_hold.dag @@ -10,13 +10,11 @@ import std.optional { Present, Absent } import std.durable_exclusive_hold { DurableHoldOwnerRef, DurableHoldReleaseRef, DurableHoldObservation, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, - HoldReleaseNamedWindowHeadMoved, HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, HoldSlotAbsent, HoldSlotFree, HoldSlotHeld, HoldSlotUndecodable, HoldObservationUnavailable, DurableHoldHolderLiveness, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, DurableHoldHolderReport, DurableHoldRecoveryRefusal, HoldRecoveryNothingHeld, HoldRecoveryHolderLive, HoldRecoveryHolderUnobservable, HoldRecoverySlotMoved, - HoldRecoveryNamedWindowHeadMoved, HoldRecoveryReportForAnotherHolder, HoldRecoverySlotUndecodable, HoldRecoveryObservationUnavailable, } import std.nat { Nat } @@ -450,7 +448,6 @@ fn recovery_refusal_text(key: NonEmptyStr, r: DurableHoldRecoveryRefusal) -> Non HoldRecoveryHolderLive { generation: _, holder: h, evidence: e } => join(["is held by '", h as String, "', which is alive (", e as String, "); no BMC actuation until it is released"], "") HoldRecoveryHolderUnobservable { generation: _, holder: h, cause: c } => join(["is held by '", h as String, "', whose liveness could not be observed (", c as String, "); an unobserved holder is not a dead one, so the hold stands"], "") HoldRecoverySlotMoved { reported: _, observed: _ } => "moved while its holder was being observed; re-run" - HoldRecoveryNamedWindowHeadMoved { named: _ } => "named window head moved; the current head was not observed; re-run" HoldRecoveryReportForAnotherHolder { holder: h, reported: _ } => join(["changed hands to '", h as String, "' while its holder was being observed; re-run"], "") HoldRecoverySlotUndecodable { generation: _, detail: d } => concat("became undecodable: ", d as String) HoldRecoveryObservationUnavailable { cause: _ } => "could not be re-read for recovery; an unreadable interlock is not a released one" @@ -645,7 +642,6 @@ fn probe_hold_release_refused(refusal: DurableHoldReleaseRefusal) -> ProbeHoldRe HoldReleaseAlreadyFree { generation: _, released_by: _ } => ProbeHoldAlreadyFree HoldReleaseHeldByAnotherOwner { generation: _, holder: h } => ProbeHoldNotOurs { holder: h as String } HoldReleaseAgainstAnotherGeneration { acquired: _, observed: _ } => ProbeHoldStale { detail: "the slot moved to another generation after it was observed" } - HoldReleaseNamedWindowHeadMoved { named: _ } => ProbeHoldStale { detail: "the named window head moved; the current head was not observed" } HoldReleaseSlotUndecodable { generation: _, detail: d } => ProbeHoldUnreleased { cause: concat("slot undecodable: ", d as String) } HoldReleaseObservationUnavailable { cause: _ } => ProbeHoldUnreleased { cause: "the store could not be read" } } diff --git a/dag/std/durable_compare_and_set.dag b/dag/std/durable_compare_and_set.dag index 5b850827ecf..cb23b091d0e 100644 --- a/dag/std/durable_compare_and_set.dag +++ b/dag/std/durable_compare_and_set.dag @@ -148,6 +148,11 @@ type CasUnreadableSlot | CasUnreadableReadRefused { detail: NonEmptyStr } | CasUnreadableObservationBoundExceeded { bound: Int } +// `bound` is the exhausted observation-attempt budget (pass count), never a +// generation. A windowed list that ran eight verifies against a live head at +// generation 1500 reports bound 8. Rendering it as a generation is a meaning +// fork of the field and a fabricated diagnostic. + // THE ONE RENDERING OF AN UNREADABLE CAUSE, so a consumer that must report "the slot could not be // read" carries the kind it was refused with rather than a fixed sentence: the first approval // redemption on srv1 (2026-09-17) was followed by a poll that answered unreadable, and the cause @@ -157,7 +162,7 @@ fn cas_unreadable_slot_detail(cause: CasUnreadableSlot) -> String { CasUnreadableMalformed { detail: d } => concat("malformed: ", d as String) CasUnreadableContentMissing { referenced: h } => concat("content missing: ", serialize_content_hash(hash: h)) CasUnreadableReadRefused { detail: d } => concat("read refused: ", d as String) - CasUnreadableObservationBoundExceeded { bound: b } => concat("observation bound exceeded at generation ", to_string(b)) + CasUnreadableObservationBoundExceeded { bound: b } => concat("observation bound exceeded after ", concat(to_string(b), " attempts")) } } diff --git a/dag/std/durable_exclusive_hold.dag b/dag/std/durable_exclusive_hold.dag index b5d63aadb83..54e7072dfa7 100644 --- a/dag/std/durable_exclusive_hold.dag +++ b/dag/std/durable_exclusive_hold.dag @@ -132,7 +132,6 @@ type DurableHoldReleaseRefusal | HoldReleaseAlreadyFree { generation: CasGeneration, released_by: DurableHoldReleaseRef } | HoldReleaseHeldByAnotherOwner { generation: CasGeneration, holder: DurableHoldOwnerRef } | HoldReleaseAgainstAnotherGeneration { acquired: CasGeneration, observed: CasGeneration } - | HoldReleaseNamedWindowHeadMoved { named: CasGeneration } | HoldReleaseSlotUndecodable { generation: CasGeneration, detail: NonEmptyStr } | HoldReleaseObservationUnavailable { cause: CasUnreadableSlot } @@ -203,7 +202,6 @@ type DurableHoldRecoveryRefusal | HoldRecoveryHolderLive { generation: CasGeneration, holder: DurableHoldOwnerRef, evidence: NonEmptyStr } | HoldRecoveryHolderUnobservable { generation: CasGeneration, holder: DurableHoldOwnerRef, cause: NonEmptyStr } | HoldRecoverySlotMoved { reported: CasGeneration, observed: CasGeneration } - | HoldRecoveryNamedWindowHeadMoved { named: CasGeneration } | HoldRecoveryReportForAnotherHolder { holder: DurableHoldOwnerRef, reported: DurableHoldOwnerRef } | HoldRecoverySlotUndecodable { generation: CasGeneration, detail: NonEmptyStr } | HoldRecoveryObservationUnavailable { cause: CasUnreadableSlot } diff --git a/dag/test/claim/durable_compare_and_set_witness_test.dag b/dag/test/claim/durable_compare_and_set_witness_test.dag index 60709fa44d5..92834cc2ff0 100644 --- a/dag/test/claim/durable_compare_and_set_witness_test.dag +++ b/dag/test/claim/durable_compare_and_set_witness_test.dag @@ -161,6 +161,8 @@ test fn an_unreadable_slot_is_a_store_refusal_not_a_lost_race() -> Bool { && is_store_refusal(outcome: missing) && is_store_refusal(outcome: refused) && is_store_refusal(outcome: beyond) + && cas_unreadable_slot_detail(cause: CasUnreadableObservationBoundExceeded { bound: 8 }) + == "observation bound exceeded after 8 attempts" } fn is_store_refusal(outcome: CasOutcome) -> Bool { diff --git a/dag/test/claim/durable_exclusive_hold_witness_test.dag b/dag/test/claim/durable_exclusive_hold_witness_test.dag index 7bf4af250a4..58463c7747c 100644 --- a/dag/test/claim/durable_exclusive_hold_witness_test.dag +++ b/dag/test/claim/durable_exclusive_hold_witness_test.dag @@ -20,19 +20,19 @@ import std.durable_exclusive_hold { DurableHoldReleaseAssessment, HoldReleaseEligible, HoldReleaseRefused, DurableHoldReleaseRefusal, HoldReleaseSlotAbsent, HoldReleaseAlreadyFree, HoldReleaseHeldByAnotherOwner, HoldReleaseAgainstAnotherGeneration, - HoldReleaseNamedWindowHeadMoved, HoldReleaseSlotUndecodable, HoldReleaseObservationUnavailable, durable_hold_acquire_assess, durable_hold_release_assess, durable_hold_owner_equal, DurableHoldHolderReport, HolderObservedLive, HolderObservedDead, HolderLivenessUnobservable, DurableHoldRecoveryAssessment, HoldRecoveryEligible, HoldRecoveryRefused, HoldRecoveryNothingHeld, HoldRecoveryHolderLive, HoldRecoveryHolderUnobservable, HoldRecoverySlotMoved, - HoldRecoveryNamedWindowHeadMoved, HoldRecoveryReportForAnotherHolder, durable_hold_recovery_assess, } import gunbc.durable_exclusive_hold_file_store { HoldPayloadDecode, HoldPayloadDecoded, HoldPayloadUndecodable, hold_slot_payload, decode_hold_slot_payload, hold_observation_of_cas, windowed_hold_release_from_verify, windowed_hold_recovery_from_verify, + WindowedHoldReleaseNamedHeadMoved, WindowedHoldReleaseProtocol, + WindowedHoldRecoveryNamedHeadMoved, WindowedHoldRecoveryProtocol, } import gunbc.durable_cas_file_store { NamedWindowHeadMoved } @@ -252,7 +252,6 @@ fn eh_release_refusal_label(a: DurableHoldReleaseAssessment) -> String { HoldReleaseAlreadyFree { generation: _, released_by: _ } => "already-free" HoldReleaseHeldByAnotherOwner { generation: _, holder: _ } => "another-owner" HoldReleaseAgainstAnotherGeneration { acquired: _, observed: _ } => "another-generation" - HoldReleaseNamedWindowHeadMoved { named: _ } => "named-head-moved" HoldReleaseSlotUndecodable { generation: _, detail: _ } => "undecodable" HoldReleaseObservationUnavailable { cause: _ } => "unavailable" } @@ -379,9 +378,9 @@ test fn named_window_head_moved_release_does_not_claim_an_observed_current_gener owner: eh_owner(text: "demand-7|offer-1"), acquired_generation: 3, ) { - HoldReleaseRefused { refusal: HoldReleaseNamedWindowHeadMoved { named: n } } => + WindowedHoldReleaseNamedHeadMoved { named: n } => cas_generation_count(g: n) == 3 - HoldReleaseRefused { refusal: HoldReleaseAgainstAnotherGeneration { acquired: _, observed: _ } } => false + WindowedHoldReleaseProtocol { assessment: _ } => false _ => false } } @@ -389,9 +388,9 @@ test fn named_window_head_moved_release_does_not_claim_an_observed_current_gener test fn named_window_head_moved_recovery_does_not_claim_an_observed_current_generation() -> Bool { let dead = eh_report(holder: "mtcollins1-boot:4242", generation: 3, dead: 1) match windowed_hold_recovery_from_verify(verify: NamedWindowHeadMoved { named_generation: 3 }, report: dead) { - HoldRecoveryRefused { refusal: HoldRecoveryNamedWindowHeadMoved { named: n } } => + WindowedHoldRecoveryNamedHeadMoved { named: n } => cas_generation_count(g: n) == 3 - HoldRecoveryRefused { refusal: HoldRecoverySlotMoved { reported: _, observed: _ } } => false + WindowedHoldRecoveryProtocol { assessment: _ } => false _ => false } } From e14bd752c841ea3f2657958aec4cd774feb79d83 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 14:48:44 +0000 Subject: [PATCH 22/30] Render BoundExceeded as a unit-neutral observation budget. The shared detail string is now "observation bound exceeded: N" so a pass count or a fabric closure bound is not reported as a generation. Co-authored-by: Cursor --- dag/std/durable_compare_and_set.dag | 2 +- dag/test/claim/durable_compare_and_set_witness_test.dag | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dag/std/durable_compare_and_set.dag b/dag/std/durable_compare_and_set.dag index cb23b091d0e..9d7fc6acc02 100644 --- a/dag/std/durable_compare_and_set.dag +++ b/dag/std/durable_compare_and_set.dag @@ -162,7 +162,7 @@ fn cas_unreadable_slot_detail(cause: CasUnreadableSlot) -> String { CasUnreadableMalformed { detail: d } => concat("malformed: ", d as String) CasUnreadableContentMissing { referenced: h } => concat("content missing: ", serialize_content_hash(hash: h)) CasUnreadableReadRefused { detail: d } => concat("read refused: ", d as String) - CasUnreadableObservationBoundExceeded { bound: b } => concat("observation bound exceeded after ", concat(to_string(b), " attempts")) + CasUnreadableObservationBoundExceeded { bound: b } => concat("observation bound exceeded: ", to_string(b)) } } diff --git a/dag/test/claim/durable_compare_and_set_witness_test.dag b/dag/test/claim/durable_compare_and_set_witness_test.dag index 92834cc2ff0..03178a6dba7 100644 --- a/dag/test/claim/durable_compare_and_set_witness_test.dag +++ b/dag/test/claim/durable_compare_and_set_witness_test.dag @@ -162,7 +162,7 @@ test fn an_unreadable_slot_is_a_store_refusal_not_a_lost_race() -> Bool { && is_store_refusal(outcome: refused) && is_store_refusal(outcome: beyond) && cas_unreadable_slot_detail(cause: CasUnreadableObservationBoundExceeded { bound: 8 }) - == "observation bound exceeded after 8 attempts" + == "observation bound exceeded: 8" } fn is_store_refusal(outcome: CasOutcome) -> Bool { From 014baad3789513a5100155837c71b991a1160912 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 16:33:29 +0000 Subject: [PATCH 23/30] Match leaked-hold red to NamedWindowHeadMoved. After release, G+1 is present, so named verify refuses the leaked G as head-moved rather than decoding the new free head as LocalStoreHoldStale. Co-authored-by: Cursor --- ...alization_store_local_wet_witness_test.dag | 28 +++---------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 894ed923b88..77080c75d86 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -686,30 +686,10 @@ fn hold_refusal_is_named_window_head_moved(r: LocalStoreHoldRefusal?) -> Bool { } } -fn hold_refusal_is_stale(r: LocalStoreHoldRefusal?) -> Bool { - match r { - Present { value: x } => match x { - LocalStoreHoldStale { observed: _ } => true - LocalStoreFamilyBusy { holder: _, evidence: _ } => false - LocalStoreHolderUnobservable { holder: _, cause: _ } => false - LocalStoreHoldRecoveryRefused { refusal: _ } => false - LocalStoreHoldContended => false - LocalStoreHoldStoreUnavailable { cause: _ } => false - LocalStoreHoldSlotUndecodable { detail: _ } => false - LocalStoreHoldKeyNotAddressable => false - LocalStoreHoldSelfUnidentified { cause: _ } => false - LocalStoreHoldWindowTooNarrow { k: _ } => false - LocalStoreHoldUnsettled { settlement: _ } => false - LocalStoreHoldNamedWindowHeadMoved { named: _ } => false - LocalStoreHoldWrongFamily { held: _, requested: _ } => false - } - Absent => false - } -} - // RELEASE-THEN-MUTATE REFUSES. A bracket body returns the held value it was given, so it outlives its -// release. Committing under that leaked value re-observes the family's hold slot, finds it no longer -// held at that generation by that owner, and refuses as a stale hold; nothing is written. +// release. Committing under that leaked value named-verifies G with G+1 present (the free successor +// the release published) and refuses as NamedWindowHeadMoved; nothing is written. The listing path +// used to decode the new free head as LocalStoreHoldStale; named verify does not observe that head. test fn a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution() -> Bool { let root_path = fresh_root_path() let root = WitnessScratchRoot { root: root_path } @@ -721,7 +701,7 @@ test fn a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution() LocalStoreBracketRan { value: leaked, hold_release: _ } => { let rs = bundle_req(module: "s") let c = local_store_commit_under(held: leaked, req: rs, payloads: verdicts(text: "s pass"), parts: []) - commit_tag(c: c.commit) == "occupancy_unavailable" && hold_refusal_is_stale(r: c.hold_refusal) + commit_tag(c: c.commit) == "occupancy_unavailable" && hold_refusal_is_named_window_head_moved(r: c.hold_refusal) && !Filesystem.Read(path: object_path(root_path: root_path, req: rs)).success && store_lookup_is_miss(l: local_store_lookup(capability: store, req: rs)) } From 42d93b7a73036b8568c7a857a3955790f514be3f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 16:36:41 +0000 Subject: [PATCH 24/30] Drop the ioctl leaked-hold reclaim claim from the floor. The floor cannot set FS_IMMUTABLE (CAP_LINUX_IMMUTABLE); that member was false by construction there. The hole stays on reclaim_stops (mkdir unlink-refusal). Leaked-hold after a clean release stays on the NamedWindowHeadMoved control. Co-authored-by: Cursor --- ...alization_store_local_wet_witness_test.dag | 44 ------------------- src/v2/workflow/floor_route_gap.dag | 2 - src/v2/workflow/local_repo_wet_terminal.dag | 6 --- 3 files changed, 52 deletions(-) diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 77080c75d86..04c22fa8321 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -51,7 +51,6 @@ import extdeps.realization.materialization_store_local { import gunbc.process_hold_identity { self_process_identity, SelfIdentified, SelfUnidentified, process_hold_text } import gunbc.build_cache_instance { ProcessIdentity } import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable } -import gunbc.durable_cas_file_store { cas_file_slot_path } import std.durable_exclusive_hold { DurableHoldOwnerRef, } @@ -711,49 +710,6 @@ test fn a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution() verdict && removed.success } -data pin_ext2_immutable_script: String = "import fcntl, os, struct, sys\nGET = 0x80086601\nSET = 0x40086602\nIMM = 0x10\npath = sys.argv[1]\non = sys.argv[2] == \"1\"\nfd = os.open(path, os.O_RDONLY)\ncur = struct.unpack(\"I\", fcntl.ioctl(fd, GET, struct.pack(\"I\", 0)))[0]\nfcntl.ioctl(fd, SET, struct.pack(\"I\", (cur | IMM) if on else (cur & ~IMM)))\nos.close(fd)\n" - -fn pin_generation_unlink(path: String, on: Bool) -> Bool { - shell.Exec.RunArgv(program: "python3", arguments: ["-c", pin_ext2_immutable_script, path, if on { "1" } else { "0" }]).exit_code == 0 -} - -// PARTIAL RECLAIM MUST NOT RE-AUTHORIZE A LEAKED HOLD. Independent deletes could remove G+1 while -// G survived, so named verify would treat leaked G as head. Pin G against unlink, run another -// acquire/release (the reclaim that used to punch that hole), then reuse G. commit_under observes -// the hold first: named verify must answer NamedWindowHeadMoved (G present, G+1 still there), mapped to -// LocalStoreHoldNamedWindowHeadMoved, before any occupancy write. occupancy_unavailable on -// that path is the hold-refuse wrapper, not an occupancy fault from the pin. ioctl-set IMMUTABLE -// must succeed (CAP_LINUX_IMMUTABLE); without it this claim is false, not vacuously green. -test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution() -> Bool { - let root_path = fresh_root_path() - let root = WitnessScratchRoot { root: root_path } - let rs = bundle_req(module: "s") - let verdict = match opened(root: root) { - Absent => false - Present { value: store } => - match local_store_with_family(capability: store, of: rs, run: h => h) { - LocalStoreBracketRefused { refusal: _ } => false - LocalStoreBracketRan { value: leaked, hold_release: _ } => { - let pinned = cas_file_slot_path(root: root_path as NonEmptyStr, key: local_store_hold_key(family: leaked.family), generation: leaked.generation) - let immutable = pin_generation_unlink(path: pinned, on: true) - let advanced = match local_store_with_family(capability: store, of: rs, run: _ => true) { - LocalStoreBracketRefused { refusal: _ } => false - LocalStoreBracketRan { value: ok, hold_release: _ } => ok - } - let c = local_store_commit_under(held: leaked, req: rs, payloads: verdicts(text: "s pass"), parts: []) - let _unlock = pin_generation_unlink(path: pinned, on: false) - immutable && advanced - && commit_tag(c: c.commit) == "occupancy_unavailable" - && hold_refusal_is_named_window_head_moved(r: c.hold_refusal) - && !Filesystem.Read(path: object_path(root_path: root_path, req: rs)).success - && store_lookup_is_miss(l: local_store_lookup(capability: store, req: rs)) - } - } - } - let removed = shell.Remove.RecursiveForce(path: root_path) - verdict && removed.success -} - // A HOLDER THAT DIED HOLDING THE FAMILY IS FREED BY OBSERVATION, NEVER BY AGE. The family is held by an // owner naming this host's current boot and namespace but a pid with no process; the next commit looks // the holder up, observes it dead, recovers the hold at the generation it saw, and lands. diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 43dc07b914e..8b60b83a800 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1124,8 +1124,6 @@ fn floor_route_gap_expectation_chunk_09() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { - head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, - tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.reopening_with_a_larger_budget_than_the_store_was_created_with_refuses_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_refused_orphan_delete_leaves_the_store_unwritable_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, diff --git a/src/v2/workflow/local_repo_wet_terminal.dag b/src/v2/workflow/local_repo_wet_terminal.dag index b62a3efc096..579caa283c6 100644 --- a/src/v2/workflow/local_repo_wet_terminal.dag +++ b/src/v2/workflow/local_repo_wet_terminal.dag @@ -2119,12 +2119,6 @@ fn local_repo_wet_schedule() -> List { function: "a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution", expectation: ExpectedToHold {} }, - WetScheduledClaim { - identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution" }, - entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", - function: "a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution", - expectation: ExpectedToHold {} - }, WetScheduledClaim { identity: WitnessIdentity { module_path: "test.claim.materialization_store_local_wet_witness", function: "reopening_with_a_larger_budget_than_the_store_was_created_with_refuses_by_real_execution" }, entry: "dag/test/claim/materialization_store_local_wet_witness_test.dag", From 3b015d635247be2244b2f0325d3c5d975fc1997e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 16:38:44 +0000 Subject: [PATCH 25/30] Declare the ioctl leaked-hold claim as DirWithTemplate debt. Keep it off local_repo_wet ExpectedToHold: the CI wet runner lacks CAP_LINUX_IMMUTABLE. Restore the identity and enroll the hermetic gap like its siblings. Restoration is a wet lane that grants the capability, or an unlink-refusal that needs none; the floor wall stays reclaim_stops. Co-authored-by: Cursor --- ...alization_store_local_wet_witness_test.dag | 50 +++++++++++++++++++ src/v2/workflow/floor_route_gap.dag | 6 +++ 2 files changed, 56 insertions(+) diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 04c22fa8321..4f2113a782c 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -51,6 +51,7 @@ import extdeps.realization.materialization_store_local { import gunbc.process_hold_identity { self_process_identity, SelfIdentified, SelfUnidentified, process_hold_text } import gunbc.build_cache_instance { ProcessIdentity } import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable } +import gunbc.durable_cas_file_store { cas_file_slot_path } import std.durable_exclusive_hold { DurableHoldOwnerRef, } @@ -710,6 +711,55 @@ test fn a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution() verdict && removed.success } +data pin_ext2_immutable_script: String = "import fcntl, os, struct, sys\nGET = 0x80086601\nSET = 0x40086602\nIMM = 0x10\npath = sys.argv[1]\non = sys.argv[2] == \"1\"\nfd = os.open(path, os.O_RDONLY)\ncur = struct.unpack(\"I\", fcntl.ioctl(fd, GET, struct.pack(\"I\", 0)))[0]\nfcntl.ioctl(fd, SET, struct.pack(\"I\", (cur | IMM) if on else (cur & ~IMM)))\nos.close(fd)\n" + +fn pin_generation_unlink(path: String, on: Bool) -> Bool { + shell.Exec.RunArgv(program: "python3", arguments: ["-c", pin_ext2_immutable_script, path, if on { "1" } else { "0" }]).exit_code == 0 +} + +// PARTIAL RECLAIM MUST NOT RE-AUTHORIZE A LEAKED HOLD. Independent deletes could remove G+1 while +// G survived, so named verify would treat leaked G as head. Pin G against unlink, run another +// acquire/release (the reclaim that used to punch that hole), then reuse G. commit_under observes +// the hold first: named verify must answer NamedWindowHeadMoved (G present, G+1 still there). +// +// DECLARED DEBT, NOT SILENCE. ioctl-set IMMUTABLE needs CAP_LINUX_IMMUTABLE; the CI wet lane +// does not grant it, so this claim is not ExpectedToHold on local_repo_wet_terminal. Hermetic +// floor enrolls it as the DirWithTemplate gap (floor_route_gap), same as its siblings. Cause: +// runner lacks CAP_LINUX_IMMUTABLE. Restoration trigger: a wet lane that grants +// CAP_LINUX_IMMUTABLE, or an unlink-refusal construction that needs no capability. The floor +// wall for the invariant is reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution. +// Integration evidence (not CI): invocation bbdf835a-b6fa-4a51-a93b-26b140398306, FAIL then PASS +// with reclaim-stop reverted. +test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let rs = bundle_req(module: "s") + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => + match local_store_with_family(capability: store, of: rs, run: h => h) { + LocalStoreBracketRefused { refusal: _ } => false + LocalStoreBracketRan { value: leaked, hold_release: _ } => { + let pinned = cas_file_slot_path(root: root_path as NonEmptyStr, key: local_store_hold_key(family: leaked.family), generation: leaked.generation) + let immutable = pin_generation_unlink(path: pinned, on: true) + let advanced = match local_store_with_family(capability: store, of: rs, run: _ => true) { + LocalStoreBracketRefused { refusal: _ } => false + LocalStoreBracketRan { value: ok, hold_release: _ } => ok + } + let c = local_store_commit_under(held: leaked, req: rs, payloads: verdicts(text: "s pass"), parts: []) + let _unlock = pin_generation_unlink(path: pinned, on: false) + immutable && advanced + && commit_tag(c: c.commit) == "occupancy_unavailable" + && hold_refusal_is_named_window_head_moved(r: c.hold_refusal) + && !Filesystem.Read(path: object_path(root_path: root_path, req: rs)).success + && store_lookup_is_miss(l: local_store_lookup(capability: store, req: rs)) + } + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} + // A HOLDER THAT DIED HOLDING THE FAMILY IS FREED BY OBSERVATION, NEVER BY AGE. The family is held by an // owner naming this host's current boot and namespace but a pid with no process; the next commit looks // the holder up, observes it dead, recovers the hold at the generation it saw, and lands. diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 8b60b83a800..fb7d769a1ce 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1124,6 +1124,12 @@ fn floor_route_gap_expectation_chunk_09() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { + // Hermetic DirWithTemplate gap, same as siblings. Not ExpectedToHold on local_repo_wet_terminal: + // ioctl-IMMUTABLE needs CAP_LINUX_IMMUTABLE the CI wet lane does not grant. Restoration: a wet + // lane that grants that capability, or an unlink-refusal that needs none. Floor wall for the + // invariant: reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution. + head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, + tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.reopening_with_a_larger_budget_than_the_store_was_created_with_refuses_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_refused_orphan_delete_leaves_the_store_unwritable_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, From af71b94dc6a38c902dd2320d2c4f77ca924a299e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 17:11:00 +0000 Subject: [PATCH 26/30] Keep the ioctl debt annotation at module-item grain. Body // inside floor_route_gap_expectation_chunk_09 is DESIGN 4c-unmodeled and native emit-build refuses it. Co-authored-by: Cursor --- src/v2/workflow/floor_route_gap.dag | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index fb7d769a1ce..30aa61a8971 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1072,6 +1072,13 @@ fn floor_route_gap_expectation_chunk_08() -> List { // v2.workflow.local_repo_wet_terminal local_repo_wet_schedule, and the gunbc.ci_layer_roots // witness_exclusion_frontier row classifying the file LocalRepoWetLane. Each was executed wet and // held before these rows were written. +// +// a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution is the +// same hermetic DirWithTemplate gap as its siblings. It is not ExpectedToHold on +// local_repo_wet_terminal: ioctl-IMMUTABLE needs CAP_LINUX_IMMUTABLE the CI wet lane does not +// grant. Restoration: a wet lane that grants that capability, or an unlink-refusal that needs +// none. Floor wall for the invariant: +// reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution. fn floor_route_gap_expectation_chunk_09() -> List { Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.commit_then_read_back_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, @@ -1124,10 +1131,6 @@ fn floor_route_gap_expectation_chunk_09() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { - // Hermetic DirWithTemplate gap, same as siblings. Not ExpectedToHold on local_repo_wet_terminal: - // ioctl-IMMUTABLE needs CAP_LINUX_IMMUTABLE the CI wet lane does not grant. Restoration: a wet - // lane that grants that capability, or an unlink-refusal that needs none. Floor wall for the - // invariant: reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution. head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.reopening_with_a_larger_budget_than_the_store_was_created_with_refuses_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, From d0f99d93b6de01a334ccadfb5e9b44a8508f5f49 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 18:43:43 +0000 Subject: [PATCH 27/30] Decline the ioctl leaked-hold claim as DeclinedNoCiWetLane. A route-gap row still planned the changed identity with no terminal. Own file classified BinWitnessWet and named in the no-CI-wet-lane population, so the floor declines instead of requiring a verdict. Co-authored-by: Cursor --- dag/gunbc/ci/ci_layer_roots.dag | 12 +- ...in_witness_wet_rows_not_executed_by_ci.dag | 1 + ...e_local_immutable_pin_wet_witness_test.dag | 158 ++++++++++++++++++ ...alization_store_local_wet_witness_test.dag | 50 ------ docs/design-rung-drops.md | 2 +- src/v2/workflow/floor_route_gap.dag | 9 - 6 files changed, 171 insertions(+), 61 deletions(-) create mode 100644 dag/test/claim/materialization_store_local_immutable_pin_wet_witness_test.dag diff --git a/dag/gunbc/ci/ci_layer_roots.dag b/dag/gunbc/ci/ci_layer_roots.dag index c32209dddd5..4b933278fa3 100644 --- a/dag/gunbc/ci/ci_layer_roots.dag +++ b/dag/gunbc/ci/ci_layer_roots.dag @@ -321,6 +321,10 @@ data excl_bin_wet_reason: String = "bin-execution witness class: fns run compile data excl_bin_wet_dissolve: DissolutionCondition = unbound_dissolution(description: "the file migrates under the execution-corpus scope prefix and enrolls as ExecutionWitnessKind rows in gunbc.commit_workflow; then this row and the roster entries delete (bin_witness_wet_note)") +data excl_immutable_pin_no_ci_wet_lane_reason: String = "ioctl-set IMMUTABLE leaked-hold composition (test.claim.materialization_store_local_immutable_pin_wet_witness): pin G, second acquire/release, reuse leaked G. Needs CAP_LINUX_IMMUTABLE. No required wet lane grants that cap, so the file is BinWitnessWet and named in gunbc.rung_drop.edited_bin_witness_wet_rows_not_executed_by_ci_population: a changed identity is DeclinedNoCiWetLane, not PlannedAsChangedWitness without a terminal. The executing floor wall for the no-hole reclaim half is reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution on the LocalRepoWetLane sibling file." + +data excl_immutable_pin_no_ci_wet_lane_dissolve: DissolutionCondition = unbound_dissolution(description: "a required wet lane grants CAP_LINUX_IMMUTABLE, or an unlink-refusal construction that needs none; then this file re-enrolls on local_repo_wet_schedule as ExpectedToHold, this exclusion flips to LocalRepoWetLane or deletes, and the pattern leaves edited_bin_witness_wet_rows_not_executed_by_ci_population") + data excl_run_verdict_exit_status_reason: String = "SUBSTANTIATED per-row (found 2026-08-16, PR #8286 CI red, run 31922786399, batch 3 discovery-corpus, 3 of 9782): every fn in this file invokes the compiled `gunbc` seed as a SUBPROCESS and reads the status of that process, so the hermetic envelope refuses them — 'no mock_response for operation Run' and 'for operation IsExecutable', the class excl_install_media_reason names. The refusal is correct and the file cannot be made hermetic: the subject IS the process exit status, which no in-process assertion can observe, and mocking the invocation would let the witness pass against a fabricated status — precisely the defect it exists to detect. The file was authored into dag/test/claim/ and so was swept into unshrunk hermetic discovery by path; the witnesses were right and the enrolment was wrong. Per DESIGN 'witness cost derives from purpose', this row alone would only remove them from hermetic discovery, which is NOT an executing consumer — so all three fns are ALSO enrolled in bin_witness_wet_entries below, which is their real per-PR executing consumer under WitnessHasExecutingConsumer standing. Size is derived, not chosen: three subprocess invocations of a three-constructor fixture, one per arm of the driver's verdict map." data excl_run_verdict_exit_status_dissolve: DissolutionCondition = unbound_dissolution(description: "the driver's exit-status seam becomes observable without a subprocess — i.e. the verdict boundary migrates under the execution-corpus scope prefix and enrolls as ExecutionWitnessKind rows in gunbc.commit_workflow, the same terminal excl_bin_wet_dissolve names; then this row and the three roster entries delete together. Publishing a mock_response for Run/IsExecutable does NOT discharge it: a mocked status is authored data, so the assertions would be re-checked against the fixture rather than against the seam, which is the vacuous pass this file's refusal-polarity note already records once.") @@ -1110,6 +1114,11 @@ data witness_exclusion_frontier: List = [ classification: LocalRepoWetLane, reason: excl_local_repo_wet_tempdir_write_reason, dissolution: excl_local_repo_wet_dissolve}, + WitnessExclusionRow { + pattern: "materialization_store_local_immutable_pin_wet_witness_test.dag", + classification: BinWitnessWet, + reason: excl_immutable_pin_no_ci_wet_lane_reason, + dissolution: excl_immutable_pin_no_ci_wet_lane_dissolve}, WitnessExclusionRow { pattern: "host_capacity_wet_witness_test.dag", classification: LocalRepoWetLane, @@ -1723,7 +1732,8 @@ data bin_witness_wet_entries: List = [ bin_wet(entry: "dag/test/claim/push_event_witness_wet_test.dag", f: "witness_push_before_payload_read_refused_on_missing_path"), bin_wet(entry: "dag/test/claim/host/host_cli_dependency_wet_witness_test.dag", f: "observe_echo_wet_posix_command_v_check_does_not_crash"), bin_wet(entry: "dag/test/claim/host/host_cli_dependency_wet_witness_test.dag", f: "observe_npm_wet_posix_command_v_check_does_not_crash"), - bin_wet(entry: "dag/test/claim/stage0_rust_maintenance_census_report_live_witness_test.dag", f: "maintenance_census_report_positive_control_derives_from_current_head") + bin_wet(entry: "dag/test/claim/stage0_rust_maintenance_census_report_live_witness_test.dag", f: "maintenance_census_report_positive_control_derives_from_current_head"), + bin_wet(entry: "dag/test/claim/materialization_store_local_immutable_pin_wet_witness_test.dag", f: "a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution") ] data bin_witness_wet_per_row_wall_budget_seconds: Int = 60 diff --git a/dag/gunbc/rung_drop/edited_bin_witness_wet_rows_not_executed_by_ci.dag b/dag/gunbc/rung_drop/edited_bin_witness_wet_rows_not_executed_by_ci.dag index cfdddbed893..0740a1bfcdf 100644 --- a/dag/gunbc/rung_drop/edited_bin_witness_wet_rows_not_executed_by_ci.dag +++ b/dag/gunbc/rung_drop/edited_bin_witness_wet_rows_not_executed_by_ci.dag @@ -69,6 +69,7 @@ data edited_bin_witness_wet_rows_not_executed_by_ci_population: List = [ "repo_local_git_config_real_execution_witness_test.dag", "transport_script_stdin_byte_fidelity_witness_test.dag", "push_event_witness_wet_test.dag", + "materialization_store_local_immutable_pin_wet_witness_test.dag", ] data edited_bin_witness_wet_rows_not_executed_by_ci: RungDrop = RungDrop { diff --git a/dag/test/claim/materialization_store_local_immutable_pin_wet_witness_test.dag b/dag/test/claim/materialization_store_local_immutable_pin_wet_witness_test.dag new file mode 100644 index 00000000000..6940a163ce4 --- /dev/null +++ b/dag/test/claim/materialization_store_local_immutable_pin_wet_witness_test.dag @@ -0,0 +1,158 @@ +module test.claim.materialization_store_local_immutable_pin_wet_witness + +import extdeps.shell +import extdeps.filesystem.filesystem_io { Filesystem } +import std.types { Bool, List, String, NonEmptyStr } +import std.content_hash { Fnv1a64Structural, content_hash_atom } +import std.materialization_provider { + ArtifactRequest, NativeModuleVerdictBundleRequest, NativeModuleVerdictBundleSubject, + evaluation_store_address, evaluation_store_admitted_address, + output_module_test_verdicts, output_module_preparation_diagnostics, +} +import std.materialization_store_grant { MaterializationStoreRoot, WitnessScratchRoot } +import std.materialization_object { + ArtifactPayload, + StoreCommit, + store_object_name, + store_lookup_is_miss, + store_commit_refusal_tag, +} +import extdeps.realization.materialization_store_local { + LocalStoreCapability, + LocalStoreOpened, LocalStoreUnavailable, + LocalStoreInitialization, LocalStoreInitialized, LocalStoreInitializationRefused, + local_store_initialize, local_store_lookup, store_path, + LocalStoreFamilyBudget, + local_store_with_family, LocalStoreBracket, LocalStoreBracketRan, LocalStoreBracketRefused, local_store_commit_under, + LocalStoreHoldStale, LocalStoreHoldNamedWindowHeadMoved, LocalStoreHoldWrongFamily, + local_store_hold_key, + LocalStoreHoldRefusal, LocalStoreFamilyBusy, + LocalStoreHolderUnobservable, LocalStoreHoldRecoveryRefused, LocalStoreHoldContended, LocalStoreHoldStoreUnavailable, + LocalStoreHoldSlotUndecodable, LocalStoreHoldKeyNotAddressable, LocalStoreHoldSelfUnidentified, LocalStoreHoldWindowTooNarrow, LocalStoreHoldUnsettled, +} +import gunbc.durable_cas_file_store { cas_file_slot_path } +import std.cache_identity { native_module_verdict_bundle_artifact_kind, typed_module_artifact_kind } +import std.measure { byte_size } + +// PARTIAL RECLAIM MUST NOT RE-AUTHORIZE A LEAKED HOLD. Independent deletes could remove G+1 while +// G survived, so named verify would treat leaked G as head. Pin G against unlink, run another +// acquire/release (the reclaim that used to punch that hole), then reuse G. commit_under observes +// the hold first: named verify must answer NamedWindowHeadMoved (G present, G+1 still there). +// +// DECLARED NO-CI-WET-LANE DEBT. ioctl-set IMMUTABLE needs CAP_LINUX_IMMUTABLE; no required wet +// lane grants it. This file is BinWitnessWet in gunbc.ci_layer_roots and named in +// gunbc.rung_drop.edited_bin_witness_wet_rows_not_executed_by_ci_population, so a changed +// identity is DeclinedNoCiWetLane rather than PlannedAsChangedWitness without a terminal. +// Restoration: a wet lane that grants CAP_LINUX_IMMUTABLE, or an unlink-refusal construction +// that needs none. Floor wall for the invariant: +// reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution. +// Integration evidence (not CI): invocation bbdf835a-b6fa-4a51-a93b-26b140398306, FAIL then PASS +// with reclaim-stop reverted. + +data witness_family_budgets: List = [ + LocalStoreFamilyBudget { family: native_module_verdict_bundle_artifact_kind, budget: byte_size(count: 4096) }, + LocalStoreFamilyBudget { family: typed_module_artifact_kind, budget: byte_size(count: 4096) }, +] + +fn h(v: String) -> Fnv1a64Structural { + content_hash_atom(value: v as NonEmptyStr) +} + +fn bundle_req(module: String) -> ArtifactRequest { + NativeModuleVerdictBundleRequest(NativeModuleVerdictBundleSubject { + route_binary_identity: h(v: "route-binary-1"), + module_identity: h(v: module), + canonical_module_import_closure_identity: h(v: "closure-1"), + ordered_test_identity_population: h(v: "t1,t2"), + eval_interpretation_identity: h(v: "eval-1") + }) +} + +fn verdicts(text: String) -> List { + [ + ArtifactPayload { id: output_module_test_verdicts, content: text }, + ArtifactPayload { id: output_module_preparation_diagnostics, content: "" }, + ] +} + +fn object_path(root_path: String, req: ArtifactRequest) -> String { + match evaluation_store_admitted_address(admission: evaluation_store_address(req: req)) { + Absent => "" + Present { value: addr } => store_path(root_path: root_path, name: store_object_name(address: addr)) + } +} + +fn commit_tag(c: StoreCommit) -> String { + store_commit_refusal_tag(c: c) +} + +fn fresh_root_path() -> String { + shell.Mktemp.DirWithTemplate(template: "/tmp/gunbc_matstore.XXXXXX").path +} + +fn opened(root: MaterializationStoreRoot) -> LocalStoreCapability? { + match local_store_initialize(root: root, instance_label: "wet-witness", budgets: witness_family_budgets) { + LocalStoreInitialized { opening: o } => match o { + LocalStoreOpened { capability: c, durability: _ } => Present { value: c } + LocalStoreUnavailable { root_path: _, cause: _ } => none + } + LocalStoreInitializationRefused { root_path: _, cause: _ } => none + } +} + +fn hold_refusal_is_named_window_head_moved(r: LocalStoreHoldRefusal?) -> Bool { + match r { + Present { value: x } => match x { + LocalStoreHoldNamedWindowHeadMoved { named: _ } => true + LocalStoreHoldStale { observed: _ } => false + LocalStoreFamilyBusy { holder: _, evidence: _ } => false + LocalStoreHolderUnobservable { holder: _, cause: _ } => false + LocalStoreHoldRecoveryRefused { refusal: _ } => false + LocalStoreHoldContended => false + LocalStoreHoldStoreUnavailable { cause: _ } => false + LocalStoreHoldSlotUndecodable { detail: _ } => false + LocalStoreHoldKeyNotAddressable => false + LocalStoreHoldSelfUnidentified { cause: _ } => false + LocalStoreHoldWindowTooNarrow { k: _ } => false + LocalStoreHoldUnsettled { settlement: _ } => false + LocalStoreHoldWrongFamily { held: _, requested: _ } => false + } + Absent => false + } +} + +data pin_ext2_immutable_script: String = "import fcntl, os, struct, sys\nGET = 0x80086601\nSET = 0x40086602\nIMM = 0x10\npath = sys.argv[1]\non = sys.argv[2] == \"1\"\nfd = os.open(path, os.O_RDONLY)\ncur = struct.unpack(\"I\", fcntl.ioctl(fd, GET, struct.pack(\"I\", 0)))[0]\nfcntl.ioctl(fd, SET, struct.pack(\"I\", (cur | IMM) if on else (cur & ~IMM)))\nos.close(fd)\n" + +fn pin_generation_unlink(path: String, on: Bool) -> Bool { + shell.Exec.RunArgv(program: "python3", arguments: ["-c", pin_ext2_immutable_script, path, if on { "1" } else { "0" }]).exit_code == 0 +} + +test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution() -> Bool { + let root_path = fresh_root_path() + let root = WitnessScratchRoot { root: root_path } + let rs = bundle_req(module: "s") + let verdict = match opened(root: root) { + Absent => false + Present { value: store } => + match local_store_with_family(capability: store, of: rs, run: h => h) { + LocalStoreBracketRefused { refusal: _ } => false + LocalStoreBracketRan { value: leaked, hold_release: _ } => { + let pinned = cas_file_slot_path(root: root_path as NonEmptyStr, key: local_store_hold_key(family: leaked.family), generation: leaked.generation) + let immutable = pin_generation_unlink(path: pinned, on: true) + let advanced = match local_store_with_family(capability: store, of: rs, run: _ => true) { + LocalStoreBracketRefused { refusal: _ } => false + LocalStoreBracketRan { value: ok, hold_release: _ } => ok + } + let c = local_store_commit_under(held: leaked, req: rs, payloads: verdicts(text: "s pass"), parts: []) + let _unlock = pin_generation_unlink(path: pinned, on: false) + immutable && advanced + && commit_tag(c: c.commit) == "occupancy_unavailable" + && hold_refusal_is_named_window_head_moved(r: c.hold_refusal) + && !Filesystem.Read(path: object_path(root_path: root_path, req: rs)).success + && store_lookup_is_miss(l: local_store_lookup(capability: store, req: rs)) + } + } + } + let removed = shell.Remove.RecursiveForce(path: root_path) + verdict && removed.success +} diff --git a/dag/test/claim/materialization_store_local_wet_witness_test.dag b/dag/test/claim/materialization_store_local_wet_witness_test.dag index 4f2113a782c..04c22fa8321 100644 --- a/dag/test/claim/materialization_store_local_wet_witness_test.dag +++ b/dag/test/claim/materialization_store_local_wet_witness_test.dag @@ -51,7 +51,6 @@ import extdeps.realization.materialization_store_local { import gunbc.process_hold_identity { self_process_identity, SelfIdentified, SelfUnidentified, process_hold_text } import gunbc.build_cache_instance { ProcessIdentity } import gunbc.durable_exclusive_hold_file_store { file_hold_acquire_windowed, FileHoldAcquired, FileHoldOccupied, FileHoldAcquireLost, FileHoldAcquireStoreUnavailable, FileHoldSlotUndecodable, FileHoldKeyNotSlotAddressable } -import gunbc.durable_cas_file_store { cas_file_slot_path } import std.durable_exclusive_hold { DurableHoldOwnerRef, } @@ -711,55 +710,6 @@ test fn a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution() verdict && removed.success } -data pin_ext2_immutable_script: String = "import fcntl, os, struct, sys\nGET = 0x80086601\nSET = 0x40086602\nIMM = 0x10\npath = sys.argv[1]\non = sys.argv[2] == \"1\"\nfd = os.open(path, os.O_RDONLY)\ncur = struct.unpack(\"I\", fcntl.ioctl(fd, GET, struct.pack(\"I\", 0)))[0]\nfcntl.ioctl(fd, SET, struct.pack(\"I\", (cur | IMM) if on else (cur & ~IMM)))\nos.close(fd)\n" - -fn pin_generation_unlink(path: String, on: Bool) -> Bool { - shell.Exec.RunArgv(program: "python3", arguments: ["-c", pin_ext2_immutable_script, path, if on { "1" } else { "0" }]).exit_code == 0 -} - -// PARTIAL RECLAIM MUST NOT RE-AUTHORIZE A LEAKED HOLD. Independent deletes could remove G+1 while -// G survived, so named verify would treat leaked G as head. Pin G against unlink, run another -// acquire/release (the reclaim that used to punch that hole), then reuse G. commit_under observes -// the hold first: named verify must answer NamedWindowHeadMoved (G present, G+1 still there). -// -// DECLARED DEBT, NOT SILENCE. ioctl-set IMMUTABLE needs CAP_LINUX_IMMUTABLE; the CI wet lane -// does not grant it, so this claim is not ExpectedToHold on local_repo_wet_terminal. Hermetic -// floor enrolls it as the DirWithTemplate gap (floor_route_gap), same as its siblings. Cause: -// runner lacks CAP_LINUX_IMMUTABLE. Restoration trigger: a wet lane that grants -// CAP_LINUX_IMMUTABLE, or an unlink-refusal construction that needs no capability. The floor -// wall for the invariant is reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution. -// Integration evidence (not CI): invocation bbdf835a-b6fa-4a51-a93b-26b140398306, FAIL then PASS -// with reclaim-stop reverted. -test fn a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution() -> Bool { - let root_path = fresh_root_path() - let root = WitnessScratchRoot { root: root_path } - let rs = bundle_req(module: "s") - let verdict = match opened(root: root) { - Absent => false - Present { value: store } => - match local_store_with_family(capability: store, of: rs, run: h => h) { - LocalStoreBracketRefused { refusal: _ } => false - LocalStoreBracketRan { value: leaked, hold_release: _ } => { - let pinned = cas_file_slot_path(root: root_path as NonEmptyStr, key: local_store_hold_key(family: leaked.family), generation: leaked.generation) - let immutable = pin_generation_unlink(path: pinned, on: true) - let advanced = match local_store_with_family(capability: store, of: rs, run: _ => true) { - LocalStoreBracketRefused { refusal: _ } => false - LocalStoreBracketRan { value: ok, hold_release: _ } => ok - } - let c = local_store_commit_under(held: leaked, req: rs, payloads: verdicts(text: "s pass"), parts: []) - let _unlock = pin_generation_unlink(path: pinned, on: false) - immutable && advanced - && commit_tag(c: c.commit) == "occupancy_unavailable" - && hold_refusal_is_named_window_head_moved(r: c.hold_refusal) - && !Filesystem.Read(path: object_path(root_path: root_path, req: rs)).success - && store_lookup_is_miss(l: local_store_lookup(capability: store, req: rs)) - } - } - } - let removed = shell.Remove.RecursiveForce(path: root_path) - verdict && removed.success -} - // A HOLDER THAT DIED HOLDING THE FAMILY IS FREED BY OBSERVATION, NEVER BY AGE. The family is held by an // owner naming this host's current boot and namespace but a pid with no process; the next commit looks // the holder up, observes it dead, recovers the hold at the generation it saw, and lands. diff --git a/docs/design-rung-drops.md b/docs/design-rung-drops.md index 6c98862e5da..f80ca6171d6 100644 --- a/docs/design-rung-drops.md +++ b/docs/design-rung-drops.md @@ -112,7 +112,7 @@ Transitive determinism reachability over the fn-arrow call graph: RUNG DROP, mec ### an edited witness file classified BinWitnessWet in gunbc.ci_layer_roots witness_exclusion_frontier: the required floor's changed-witness sublane declines it as DeclinedNoCiWetLane instead of blocking, and no CI lane executes it — declared 2026-09-30 -an edited witness file classified BinWitnessWet in gunbc.ci_layer_roots witness_exclusion_frontier: the required floor's changed-witness sublane declines it as DeclinedNoCiWetLane instead of blocking, and no CI lane executes it: RUNG DROP, mechanically preventable -> mitigatable (lost as a passenger of the per-PR bin-witness wet batch, deleted with the floor cut of 2026-08-15, and .github/workflows/falsifier.yml, deleted at 611fd02770 (gunbc#8283)). Population: self_host_artifact_materialization_real_execution_witness_test.dag, stage0_regen_convergence_real_execution_witness_test.dag, typed_witness_invocation_test.dag, namespace_structural_root_exposure_generated_witness_test.dag, emit_host_typed_smoke_test.dag, build_artifact_corruption_probe_witness_test.dag, dag_collect_fingerprint_witness_test.dag, dag_compile_clean_perturb_receipts_test.dag, test/claim/diagnostics_test.dag, effects_rest_transport_parse_witness_test.dag, test/claim/parse_test.dag, v1_dag_parse_witness_test.dag, auth_declared_but_unwired_witness_test.dag, test/claim/bootstrap_test.dag, infer_semantics_witness_test.dag, dag_compile_clean_shard_a_witness_test.dag, dag_compile_clean_shard_totality_witness_test.dag, dag_compile_clean_seam_witness_test.dag, run_verdict_exit_status_witness_test.dag, http_client_get_real_execution_witness_test.dag, roadmap_belt_actuate_witness_test.dag, roadmap_closing_contract_execution_witness_test.dag, host_build_cache_provision_real_execution_witness_test.dag, materialized_ssh_key_file_real_execution_witness_test.dag, proc_self_cgroup_real_execution_witness_test.dag, repo_local_git_config_real_execution_witness_test.dag, transport_script_stdin_byte_fidelity_witness_test.dag, push_event_witness_wet_test.dag. Restored when: a REQUIRED lane executes changed BinWitnessWet rows for the same pull_request or merge_group -- builds their witness binaries and runs the edited entries to a terminal verdict on a host that admits their effects -- so the changed-witness sublane can plan them instead of declining; sufficient for deleting DeclinedNoCiWetLane and this row together. +an edited witness file classified BinWitnessWet in gunbc.ci_layer_roots witness_exclusion_frontier: the required floor's changed-witness sublane declines it as DeclinedNoCiWetLane instead of blocking, and no CI lane executes it: RUNG DROP, mechanically preventable -> mitigatable (lost as a passenger of the per-PR bin-witness wet batch, deleted with the floor cut of 2026-08-15, and .github/workflows/falsifier.yml, deleted at 611fd02770 (gunbc#8283)). Population: self_host_artifact_materialization_real_execution_witness_test.dag, stage0_regen_convergence_real_execution_witness_test.dag, typed_witness_invocation_test.dag, namespace_structural_root_exposure_generated_witness_test.dag, emit_host_typed_smoke_test.dag, build_artifact_corruption_probe_witness_test.dag, dag_collect_fingerprint_witness_test.dag, dag_compile_clean_perturb_receipts_test.dag, test/claim/diagnostics_test.dag, effects_rest_transport_parse_witness_test.dag, test/claim/parse_test.dag, v1_dag_parse_witness_test.dag, auth_declared_but_unwired_witness_test.dag, test/claim/bootstrap_test.dag, infer_semantics_witness_test.dag, dag_compile_clean_shard_a_witness_test.dag, dag_compile_clean_shard_totality_witness_test.dag, dag_compile_clean_seam_witness_test.dag, run_verdict_exit_status_witness_test.dag, http_client_get_real_execution_witness_test.dag, roadmap_belt_actuate_witness_test.dag, roadmap_closing_contract_execution_witness_test.dag, host_build_cache_provision_real_execution_witness_test.dag, materialized_ssh_key_file_real_execution_witness_test.dag, proc_self_cgroup_real_execution_witness_test.dag, repo_local_git_config_real_execution_witness_test.dag, transport_script_stdin_byte_fidelity_witness_test.dag, push_event_witness_wet_test.dag, materialization_store_local_immutable_pin_wet_witness_test.dag. Restored when: a REQUIRED lane executes changed BinWitnessWet rows for the same pull_request or merge_group -- builds their witness binaries and runs the edited entries to a terminal verdict on a host that admits their effects -- so the changed-witness sublane can plan them instead of declining; sufficient for deleting DeclinedNoCiWetLane and this row together. ### The EMIT-COST-QUAL-0 wet battery's only sanctioned executing consumer — declared 2026-09-04 diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 30aa61a8971..8b60b83a800 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1072,13 +1072,6 @@ fn floor_route_gap_expectation_chunk_08() -> List { // v2.workflow.local_repo_wet_terminal local_repo_wet_schedule, and the gunbc.ci_layer_roots // witness_exclusion_frontier row classifying the file LocalRepoWetLane. Each was executed wet and // held before these rows were written. -// -// a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution is the -// same hermetic DirWithTemplate gap as its siblings. It is not ExpectedToHold on -// local_repo_wet_terminal: ioctl-IMMUTABLE needs CAP_LINUX_IMMUTABLE the CI wet lane does not -// grant. Restoration: a wet lane that grants that capability, or an unlink-refusal that needs -// none. Floor wall for the invariant: -// reclaim_stops_before_a_later_generation_when_an_earlier_delete_fails_by_real_execution. fn floor_route_gap_expectation_chunk_09() -> List { Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.commit_then_read_back_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, @@ -1131,8 +1124,6 @@ fn floor_route_gap_expectation_chunk_09() -> List { tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_hold_leaked_out_of_its_bracket_is_refused_as_stale_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { - head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_leaked_hold_stays_stale_when_reclaim_cannot_remove_its_generation_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, - tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.reopening_with_a_larger_budget_than_the_store_was_created_with_refuses_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, tail: Cons { head: FloorRouteGapExpectation { identity: "test.claim.materialization_store_local_wet_witness.a_refused_orphan_delete_leaves_the_store_unwritable_by_real_execution", operation: "DirWithTemplate", ground: NoMockResponse {} }, From b272d5e8d93cc73408ae637a6ccd70c807b74ef6 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 19:28:25 +0000 Subject: [PATCH 28/30] Close floor_route_gap chunk_09 after the dropped Cons identity. Removing the leaked-hold row left an extra closer that ended the function early, so the next item was unparseable and CI refused the module index. Co-authored-by: Cursor --- src/v2/workflow/floor_route_gap.dag | 1 - 1 file changed, 1 deletion(-) diff --git a/src/v2/workflow/floor_route_gap.dag b/src/v2/workflow/floor_route_gap.dag index 8b60b83a800..848a9c57392 100644 --- a/src/v2/workflow/floor_route_gap.dag +++ b/src/v2/workflow/floor_route_gap.dag @@ -1191,7 +1191,6 @@ fn floor_route_gap_expectation_chunk_09() -> List { } } } - } } // MegaRAC committed-artifact sha256sum: first host effect is shell.Mktemp.Dir (checksum From 121565ac7a9c8ed6e581b34e386a0384df19e211 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Thu, 8 Oct 2026 19:39:12 +0000 Subject: [PATCH 29/30] Attach the BoundExceeded field comment to CasUnreadableSlot. A blank-separated block after the type was nearest-following the renderer, so the field's meaning sat on the wrong subject. Co-authored-by: Cursor --- dag/std/durable_compare_and_set.dag | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dag/std/durable_compare_and_set.dag b/dag/std/durable_compare_and_set.dag index 9d7fc6acc02..ddf8da11619 100644 --- a/dag/std/durable_compare_and_set.dag +++ b/dag/std/durable_compare_and_set.dag @@ -142,17 +142,16 @@ type CasSlotObservation // as an instruction to every other realization to describe itself in terms of scanning. What is NOT // claimed is that every CAS realization can produce this: most cannot, and one that never has to // discover a head should never construct it. +// `bound` is the exhausted observation-attempt budget (pass count), never a +// generation. A windowed list that ran eight verifies against a live head at +// generation 1500 reports bound 8. Rendering it as a generation is a meaning +// fork of the field and a fabricated diagnostic. type CasUnreadableSlot = CasUnreadableMalformed { detail: NonEmptyStr } | CasUnreadableContentMissing { referenced: ContentHash } | CasUnreadableReadRefused { detail: NonEmptyStr } | CasUnreadableObservationBoundExceeded { bound: Int } -// `bound` is the exhausted observation-attempt budget (pass count), never a -// generation. A windowed list that ran eight verifies against a live head at -// generation 1500 reports bound 8. Rendering it as a generation is a meaning -// fork of the field and a fabricated diagnostic. - // THE ONE RENDERING OF AN UNREADABLE CAUSE, so a consumer that must report "the slot could not be // read" carries the kind it was refused with rather than a fixed sentence: the first approval // redemption on srv1 (2026-09-17) was followed by a poll that answered unreadable, and the cause From cf2f5274139ac79d04816db4146aa748d3dfd71d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Fri, 9 Oct 2026 17:25:14 +0000 Subject: [PATCH 30/30] docs/design-rung-drops.md: regenerate through tools.docs_projection_gate regen after the #13569 merge Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/design-rung-drops.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design-rung-drops.md b/docs/design-rung-drops.md index 49916065be5..0ba3dbadb4e 100644 --- a/docs/design-rung-drops.md +++ b/docs/design-rung-drops.md @@ -112,7 +112,7 @@ Transitive determinism reachability over the fn-arrow call graph: RUNG DROP, mec ### an edited witness file classified BinWitnessWet in gunbc.ci_layer_roots witness_exclusion_frontier: the required floor's changed-witness sublane declines it as DeclinedNoCiWetLane instead of blocking, and no CI lane executes it — declared 2026-09-30 -an edited witness file classified BinWitnessWet in gunbc.ci_layer_roots witness_exclusion_frontier: the required floor's changed-witness sublane declines it as DeclinedNoCiWetLane instead of blocking, and no CI lane executes it: RUNG DROP, mechanically preventable -> mitigatable (lost as a passenger of the per-PR bin-witness wet batch, deleted with the floor cut of 2026-08-15, and .github/workflows/falsifier.yml, deleted at 611fd02770 (gunbc#8283)). Population: self_host_artifact_materialization_real_execution_witness_test.dag, stage0_regen_convergence_real_execution_witness_test.dag, typed_witness_invocation_test.dag, namespace_structural_root_exposure_generated_witness_test.dag, emit_host_typed_smoke_test.dag, build_artifact_corruption_probe_witness_test.dag, dag_collect_fingerprint_witness_test.dag, dag_compile_clean_perturb_receipts_test.dag, test/claim/diagnostics_test.dag, effects_rest_transport_parse_witness_test.dag, test/claim/parse_test.dag, v1_dag_parse_witness_test.dag, auth_declared_but_unwired_witness_test.dag, test/claim/bootstrap_test.dag, infer_semantics_witness_test.dag, dag_compile_clean_shard_a_witness_test.dag, dag_compile_clean_shard_totality_witness_test.dag, dag_compile_clean_seam_witness_test.dag, run_verdict_exit_status_witness_test.dag, http_client_get_real_execution_witness_test.dag, roadmap_belt_actuate_witness_test.dag, roadmap_closing_contract_execution_witness_test.dag, host_build_cache_provision_real_execution_witness_test.dag, materialized_ssh_key_file_real_execution_witness_test.dag, proc_self_cgroup_real_execution_witness_test.dag, repo_local_git_config_real_execution_witness_test.dag, transport_script_stdin_byte_fidelity_witness_test.dag, push_event_witness_wet_test.dag. Restored when: a REQUIRED lane executes changed BinWitnessWet rows for the same pull_request or merge_group -- builds their witness binaries and runs the edited entries to a terminal verdict on a host that admits their effects -- so the changed-witness sublane can plan them instead of declining; sufficient for deleting DeclinedNoCiWetLane and this row together. +an edited witness file classified BinWitnessWet in gunbc.ci_layer_roots witness_exclusion_frontier: the required floor's changed-witness sublane declines it as DeclinedNoCiWetLane instead of blocking, and no CI lane executes it: RUNG DROP, mechanically preventable -> mitigatable (lost as a passenger of the per-PR bin-witness wet batch, deleted with the floor cut of 2026-08-15, and .github/workflows/falsifier.yml, deleted at 611fd02770 (gunbc#8283)). Population: self_host_artifact_materialization_real_execution_witness_test.dag, stage0_regen_convergence_real_execution_witness_test.dag, typed_witness_invocation_test.dag, namespace_structural_root_exposure_generated_witness_test.dag, emit_host_typed_smoke_test.dag, build_artifact_corruption_probe_witness_test.dag, dag_collect_fingerprint_witness_test.dag, dag_compile_clean_perturb_receipts_test.dag, test/claim/diagnostics_test.dag, effects_rest_transport_parse_witness_test.dag, test/claim/parse_test.dag, v1_dag_parse_witness_test.dag, auth_declared_but_unwired_witness_test.dag, test/claim/bootstrap_test.dag, infer_semantics_witness_test.dag, dag_compile_clean_shard_a_witness_test.dag, dag_compile_clean_shard_totality_witness_test.dag, dag_compile_clean_seam_witness_test.dag, run_verdict_exit_status_witness_test.dag, http_client_get_real_execution_witness_test.dag, roadmap_belt_actuate_witness_test.dag, roadmap_closing_contract_execution_witness_test.dag, host_build_cache_provision_real_execution_witness_test.dag, materialized_ssh_key_file_real_execution_witness_test.dag, proc_self_cgroup_real_execution_witness_test.dag, repo_local_git_config_real_execution_witness_test.dag, transport_script_stdin_byte_fidelity_witness_test.dag, push_event_witness_wet_test.dag, materialization_store_local_immutable_pin_wet_witness_test.dag. Restored when: a REQUIRED lane executes changed BinWitnessWet rows for the same pull_request or merge_group -- builds their witness binaries and runs the edited entries to a terminal verdict on a host that admits their effects -- so the changed-witness sublane can plan them instead of declining; sufficient for deleting DeclinedNoCiWetLane and this row together. ### The EMIT-COST-QUAL-0 wet battery's only sanctioned executing consumer — declared 2026-09-04