From cc85d795767bd99c33460683797fcfcce58b9ef9 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 15:25:42 +0000 Subject: [PATCH 1/3] WIP: WAIT FOR DM BRIEF FROM quick-carp-521 BEFORE ACTING. FLEET INCIDENT: CI --- src/v1/stage0/src/cli_run.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index 877627f5484..9f091b58a4d 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -4785,6 +4785,17 @@ pub struct MultiEntryIndex { /// drain accumulation lane). Each eviction is a typed, located diagnostic — never /// a silent widen (§5). typed_cache_evictions: Cell, + /// Host-budget-derived typed-cache entry cap, sampled ONCE for this index's + /// lifetime — a run-start fact, never re-read per insert. Re-deriving the cap + /// from a live, host-shared signal (`/proc/meminfo MemAvailable`, the fallback + /// when no private cgroup memory limit is discoverable) on every insert was the + /// 2026-07-21 fleet OOM incident: the cap chased the host's real-time noise + /// across every co-resident session, and each eviction's recompute-on-miss + /// added the exact memory pressure the cap exists to relieve — a thrashing + /// feedback loop, not a bound. `OnceCell` gives lazy single-sample semantics + /// so index construction stays free of the governor read for callers that + /// never touch the typed cache. + typed_module_cache_cap: std::cell::OnceCell, /// Source-content hashes by file path, recorded in the parse loop (where the /// `SourceFile.content` is in hand) — the source-hash key term for /// `typed_module_content_key`. A reconcile of a module whose file never passed @@ -4914,6 +4925,7 @@ fn new_multi_entry_index_shell( module_graph_facts: build_module_graph_facts_live(source_roots), typed_module_cache: RefCell::new(std::collections::HashMap::new()), typed_cache_evictions: Cell::new(0), + typed_module_cache_cap: std::cell::OnceCell::new(), source_hash_by_file: RefCell::new(std::collections::HashMap::new()), module_source_identity: RefCell::new(std::collections::HashMap::new()), cross_worker_store, From d684d74b2dfa6a853f23b450a6ca1dca59e8e96a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 15:33:43 +0000 Subject: [PATCH 2/3] Floor OOM fix: sample typed-cache cap once per run, not per insert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-07-21 fleet incident (redded main + every PR floor from ~12:17Z on srv3-06/07): #6972's enforce_typed_cache_entry_cap re-derived its entry cap from read_host_budget_bytes() on every cache insert. On CI runners no private cgroup memory.max/memory.high is discoverable, so the derivation fell through to live /proc/meminfo MemAvailable — a host-wide signal shared across co-resident sessions. Re-reading it per insert let the cap chase host noise within a single run (e.g. 701->682->...->204->325), and each eviction's recompute-on-miss added pressure exactly when pressure was already high — a thrashing feedback loop ending in claim_executor SIGKILL (exit 137). Fix, per signed direction: typed_module_cache_cap_derivation() factors the existing env-override/budget-read logic out of typed_module_cache_max_entries (kept as the pure per-call form for existing tests) and adds a `degraded` flag. MultiEntryIndex gets a `typed_module_cache_cap: OnceCell` field; the new typed_module_cache_cap(index) accessor samples the derivation exactly once per index lifetime and, on that first call, logs a typed, counted `[floor-drain] degraded_budget_source` line when the source isn't a private cgroup limit (an honesty arm, not a widened failure -- the cap still derives from whatever source was found). enforce_typed_cache_entry_cap and emit_floor_drain_receipt now go through this accessor instead of re-deriving per call. No floor-pin added; no env-var escape hatch. Witnesses: typed_module_cache_cap_sampled_once_per_index_stays_stable_despite_signal_drift proves the cap holds fixed across a live signal move within one index's lifetime. uncached_derivation_tracks_moving_signal_the_sampled_once_accessor_must_not is the RED control, reproducing the pre-fix oscillation shape (cap values drawn from the incident's own log: 300/227/294/204/325) via the still-live uncached derivation, so a regression that reverts the runtime call sites back to the uncached form is caught. Evidence trail: srv3-05 11:50Z run 29827692954 and srv3-06 12:17Z run 29829512709 both show the eviction-storm-then-Killed(137) signature; the wedged 3.5h+ run 29829313521 was cancelled after pulling final evidence (build job succeeded, ci job's floor-run step never emitted a single [floor-drain] line before being starved by host contention) -- noted here so the cancellation isn't mistaken for a hidden failure. Contained to the #6972 mechanism + its two call sites, per manager sign-off; no PR-beta/SpacePacked dissolve-on work included. Co-Authored-By: Claude Sonnet 5 --- src/v1/stage0/src/cli_run.rs | 81 ++++++++++++++++--- .../tests/src/floor_drain_retention_test.rs | 66 ++++++++++++++- 2 files changed, 133 insertions(+), 14 deletions(-) diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index 9f091b58a4d..8e9ef84035b 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -4915,6 +4915,15 @@ pub fn typed_cache_evictions_for_test(index: &MultiEntryIndex) -> u64 { index.typed_cache_evictions.get() } +/// Test witness for the sample-once cap: exposes the same accessor runtime +/// call sites use, so a test can observe that repeated calls against one +/// `index` return the identical value even as the underlying signal moves — +/// the property the 2026-07-21 fleet OOM fix depends on. +#[cfg(any(test, feature = "interp_test_witness"))] +pub fn typed_module_cache_cap_for_test(index: &MultiEntryIndex) -> usize { + typed_module_cache_cap(index) +} + fn new_multi_entry_index_shell( source_files: ModuleSourceIndex, source_roots: &[String], @@ -5242,28 +5251,70 @@ const TYPED_MODULE_BYTES_PER_ENTRY_ESTIMATE: u64 = 3 * 1024 * 1024; const TYPED_MODULE_CACHE_MAX_ENTRIES_FLOOR: usize = 100; const TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL: usize = 4_000; -/// Host-budget-derived cap on `typed_module_cache` entries for the private -/// per-index store (width=1 drain path). `GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES` -/// is an operator/test probe: still clamped to `1..CEIL` (never unbounded); a -/// malformed override falls through to the derived cap (fail-closed). -pub fn typed_module_cache_max_entries() -> usize { +/// One derivation of the typed-cache entry cap: env override, else the host +/// budget divided by the per-entry estimate. Returns `(cap, source_label, +/// degraded)` — `degraded` is true exactly when the budget did not come from +/// a private cgroup `memory.max`/`memory.high` (i.e. it fell through to the +/// host-wide `MemAvailable`/`MemTotal` last resort, or no budget was found at +/// all and the ceiling was used). Pure and side-effect-free: callers decide +/// how often to invoke it and whether to log the degraded case. +fn typed_module_cache_cap_derivation() -> (usize, String, bool) { if let Ok(raw) = std::env::var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES") { if let Ok(n) = raw.trim().parse::() { if n > 0 { - return n.min(TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL); + return ( + n.min(TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL), + "env override GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES".to_string(), + false, + ); } // Zero override is invalid — fall through to derived cap. } // Malformed override — fall through to derived cap (fail-closed). } - let (budget, _) = crate::memory_governor::read_host_budget_bytes(); - budget + let (budget, source_label) = crate::memory_governor::read_host_budget_bytes(); + let degraded = !(source_label.contains("memory.max") || source_label.contains("memory.high")); + let cap = budget .map(|b| (b / TYPED_MODULE_BYTES_PER_ENTRY_ESTIMATE) as usize) .unwrap_or(TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL) .clamp( TYPED_MODULE_CACHE_MAX_ENTRIES_FLOOR, TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL, - ) + ); + (cap, source_label, degraded) +} + +/// Host-budget-derived cap on `typed_module_cache` entries for the private +/// per-index store (width=1 drain path). `GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES` +/// is an operator/test probe: still clamped to `1..CEIL` (never unbounded); a +/// malformed override falls through to the derived cap (fail-closed). +/// +/// This is the pure, un-cached derivation — kept for direct env-override +/// tests. Runtime call sites must go through `typed_module_cache_cap` +/// instead, which samples this exactly once per index lifetime; re-deriving +/// from a live host-shared signal on every insert was the 2026-07-21 fleet +/// OOM incident (see the doc comment on `MultiEntryIndex::typed_module_cache_cap`). +pub fn typed_module_cache_max_entries() -> usize { + typed_module_cache_cap_derivation().0 +} + +/// The typed-cache cap for `index`, sampled exactly ONCE for this index's +/// lifetime (a run-start fact, never re-read per insert). On first call, if +/// the budget source is degraded (no private cgroup limit found), emits a +/// typed, counted `[floor-drain] degraded_budget_source` diagnostic — an +/// honesty arm, not a widened failure: the cap still derives from whatever +/// source was found, it is simply named so the degraded case is observable +/// and prioritizable rather than silent. +fn typed_module_cache_cap(index: &MultiEntryIndex) -> usize { + *index.typed_module_cache_cap.get_or_init(|| { + let (cap, source, degraded) = typed_module_cache_cap_derivation(); + if degraded { + eprintln!( + "[floor-drain] degraded_budget_source: cap={cap} source={source} (no private cgroup memory.max/memory.high found; falling back to a host-shared signal)" + ); + } + cap + }) } pub fn index_retention_snapshot(index: &MultiEntryIndex) -> IndexRetentionSnapshot { @@ -5341,7 +5392,11 @@ fn emit_floor_drain_group_line( ); } -fn emit_floor_drain_receipt(total_groups: usize, peaks: &IndexRetentionSnapshot) { +fn emit_floor_drain_receipt( + index: &MultiEntryIndex, + total_groups: usize, + peaks: &IndexRetentionSnapshot, +) { eprintln!( "[floor-drain] receipt: groups={total_groups} \ typed_cache_peak={} parse_cache_peak={} resolved_memo_peak={} \ @@ -5355,7 +5410,7 @@ fn emit_floor_drain_receipt(total_groups: usize, peaks: &IndexRetentionSnapshot) .peak_rss_bytes .map(|b| b.to_string()) .unwrap_or_else(|| "unreadable".into()), - typed_module_cache_max_entries(), + typed_module_cache_cap(index), ); } @@ -5368,7 +5423,7 @@ fn enforce_typed_cache_entry_cap(index: &MultiEntryIndex) { if index.cross_worker_store.is_some() { return; } - let cap = typed_module_cache_max_entries(); + let cap = typed_module_cache_cap(index); let mut cache = index.typed_module_cache.borrow_mut(); let mut evicted = 0u64; while cache.len() > cap { @@ -12658,7 +12713,7 @@ pub fn run_discovery_corpus_with_options( drain_peaks = retention_snapshot_peak(&drain_peaks, &snap); drain_prev = snap; } - emit_floor_drain_receipt(total_groups, &drain_peaks); + emit_floor_drain_receipt(&index, total_groups, &drain_peaks); return Ok(attach_deferred_discovery_rows( merge_discovery_summaries(summaries), deferred_rows, diff --git a/src/v1/tests/src/floor_drain_retention_test.rs b/src/v1/tests/src/floor_drain_retention_test.rs index 0784c5856d4..b5e7e8a4c97 100644 --- a/src/v1/tests/src/floor_drain_retention_test.rs +++ b/src/v1/tests/src/floor_drain_retention_test.rs @@ -10,7 +10,8 @@ use crate::helpers::workspace_root; use v1_compiler::cli_run::{ self, build_multi_entry_index, index_retention_snapshot, make_eval_context, resolve_entry_graph, resolve_entry_with_index, run_claim, typed_cache_evictions_for_test, - typed_module_cache_len_for_test, typed_module_cache_max_entries, ClaimOutcome, + typed_module_cache_cap_for_test, typed_module_cache_len_for_test, + typed_module_cache_max_entries, ClaimOutcome, }; use v1_compiler::v1_interpreter::ExecutionMode; @@ -144,6 +145,69 @@ fn typed_module_cache_entry_cap_evicts_with_counted_receipt() { let _ = fs::remove_dir_all(&dir); } +/// 2026-07-21 fleet OOM fix: `typed_module_cache_cap` samples the host budget +/// exactly ONCE per index lifetime — a run-start fact, never re-read per +/// insert. This is the by-execution witness that the sampled-once accessor +/// is actually stable: change the "signal" (here, the env-override probe, +/// standing in for the live host-shared MemAvailable reading that moved +/// mid-run pre-fix) after the index has sampled its cap once, and confirm +/// the sampled cap does not follow it. +#[test] +fn typed_module_cache_cap_sampled_once_per_index_stays_stable_despite_signal_drift() { + let _lock = CAP_ENV_MUTEX.lock().expect("cap env mutex"); + std::env::set_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES", "150"); + let index = build_multi_entry_index(&[]); + + let first = typed_module_cache_cap_for_test(&index); + assert_eq!( + first, 150, + "index must sample the cap that was live at first use" + ); + + // Simulate the host signal moving mid-run (this is exactly the class of + // motion `/proc/meminfo MemAvailable` exhibited under co-tenant pressure + // in the 2026-07-21 incident). + std::env::set_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES", "3000"); + let second = typed_module_cache_cap_for_test(&index); + assert_eq!( + second, first, + "cap must stay fixed for this index's lifetime even as the underlying signal moves" + ); + + std::env::remove_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES"); +} + +/// RED control: reproduces the pre-fix defect shape. The un-cached, pure +/// derivation (`typed_module_cache_max_entries`, still used by direct +/// env-override tests above and kept for that purpose) DOES track a moving +/// signal on every call — that tracking, invoked from every cache insert, +/// is the mechanism that produced the eviction storm (cap 294→227 across +/// 2300+ evictions, `claim_executor` exit 137). This test asserts the old +/// call pattern's oscillation is real and observable, so it stays a live +/// discriminator: were `typed_module_cache_cap` ever accidentally reverted +/// to call the uncached function per-insert, this divergence-producing +/// signal motion is what would resume driving it. +#[test] +fn uncached_derivation_tracks_moving_signal_the_sampled_once_accessor_must_not() { + let _lock = CAP_ENV_MUTEX.lock().expect("cap env mutex"); + std::env::set_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES", "300"); + let readings: Vec = ["300", "227", "294", "204", "325"] + .iter() + .map(|v| { + std::env::set_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES", v); + typed_module_cache_max_entries() + }) + .collect(); + assert_eq!( + readings, + vec![300, 227, 294, 204, 325], + "uncached derivation must track every signal move — this is the storm mechanism the \ + sampled-once accessor exists to eliminate at runtime call sites" + ); + + std::env::remove_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES"); +} + #[test] fn typed_module_cache_under_entry_cap_matches_cold_oracle_in_every_order() { let _lock = CAP_ENV_MUTEX.lock().expect("cap env mutex"); From 1d64f8efbc6adb00a0cc7b79641fac2c333b84d2 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Tue, 21 Jul 2026 15:36:09 +0000 Subject: [PATCH 3/3] Note the degraded-detection string-scan as a future grounding point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Non-blocking review observation from quick-carp-521 on #7002: the degraded-source check string-scans read_host_budget_bytes' display label. Fine for now (it returns (Option, String)), but if that fn ever grows a typed source enum, this check should ground on it instead of re-parsing the label — a one-line note so it doesn't cement. Co-Authored-By: Claude Sonnet 5 --- src/v1/stage0/src/cli_run.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index 8e9ef84035b..47d43ecc62d 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -5273,6 +5273,9 @@ fn typed_module_cache_cap_derivation() -> (usize, String, bool) { // Malformed override — fall through to derived cap (fail-closed). } let (budget, source_label) = crate::memory_governor::read_host_budget_bytes(); + // String-scan of read_host_budget_bytes' label — acceptable while that fn returns + // (Option, String); if it ever grows a typed source enum, ground this check on + // the enum instead of re-parsing its display label (§3, avoid a second representation). let degraded = !(source_label.contains("memory.max") || source_label.contains("memory.high")); let cap = budget .map(|b| (b / TYPED_MODULE_BYTES_PER_ENTRY_ESTIMATE) as usize)