Floor OOM fix: sample typed-cache cap once per run, not per insert

src/v1/stage0/src/cli_run.rs

@@ -4785,6 +4785,17 @@ pub struct MultiEntryIndex {
     /// drain accumulation lane). Each eviction is a typed, located diagnostic — never
     /// a silent widen (§5).
     typed_cache_evictions: Cell<u64>,
+    /// Host-budget-derived typed-cache entry cap, sampled ONCE for this index's
+    /// lifetime — a run-start fact, never re-read per insert. Re-deriving the cap
+    /// from a live, host-shared signal (`/proc/meminfo MemAvailable`, the fallback
+    /// when no private cgroup memory limit is discoverable) on every insert was the
+    /// 2026-07-21 fleet OOM incident: the cap chased the host's real-time noise
+    /// across every co-resident session, and each eviction's recompute-on-miss
+    /// added the exact memory pressure the cap exists to relieve — a thrashing
+    /// feedback loop, not a bound. `OnceCell` gives lazy single-sample semantics
+    /// so index construction stays free of the governor read for callers that
+    /// never touch the typed cache.
+    typed_module_cache_cap: std::cell::OnceCell<usize>,
     /// Source-content hashes by file path, recorded in the parse loop (where the
     /// `SourceFile.content` is in hand) — the source-hash key term for
     /// `typed_module_content_key`. A reconcile of a module whose file never passed
@@ -4904,6 +4915,15 @@ pub fn typed_cache_evictions_for_test(index: &MultiEntryIndex) -> u64 {
     index.typed_cache_evictions.get()
 }
 
+/// Test witness for the sample-once cap: exposes the same accessor runtime
+/// call sites use, so a test can observe that repeated calls against one
+/// `index` return the identical value even as the underlying signal moves —
+/// the property the 2026-07-21 fleet OOM fix depends on.
+#[cfg(any(test, feature = "interp_test_witness"))]
+pub fn typed_module_cache_cap_for_test(index: &MultiEntryIndex) -> usize {
+    typed_module_cache_cap(index)
+}
+
 fn new_multi_entry_index_shell(
     source_files: ModuleSourceIndex,
     source_roots: &[String],
@@ -4914,6 +4934,7 @@ fn new_multi_entry_index_shell(
         module_graph_facts: build_module_graph_facts_live(source_roots),
         typed_module_cache: RefCell::new(std::collections::HashMap::new()),
         typed_cache_evictions: Cell::new(0),
+        typed_module_cache_cap: std::cell::OnceCell::new(),
         source_hash_by_file: RefCell::new(std::collections::HashMap::new()),
         module_source_identity: RefCell::new(std::collections::HashMap::new()),
         cross_worker_store,
@@ -5230,28 +5251,73 @@ const TYPED_MODULE_BYTES_PER_ENTRY_ESTIMATE: u64 = 3 * 1024 * 1024;
 const TYPED_MODULE_CACHE_MAX_ENTRIES_FLOOR: usize = 100;
 const TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL: usize = 4_000;
 
-/// Host-budget-derived cap on `typed_module_cache` entries for the private
-/// per-index store (width=1 drain path). `GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES`
-/// is an operator/test probe: still clamped to `1..CEIL` (never unbounded); a
-/// malformed override falls through to the derived cap (fail-closed).
-pub fn typed_module_cache_max_entries() -> usize {
+/// One derivation of the typed-cache entry cap: env override, else the host
+/// budget divided by the per-entry estimate. Returns `(cap, source_label,
+/// degraded)` — `degraded` is true exactly when the budget did not come from
+/// a private cgroup `memory.max`/`memory.high` (i.e. it fell through to the
+/// host-wide `MemAvailable`/`MemTotal` last resort, or no budget was found at
+/// all and the ceiling was used). Pure and side-effect-free: callers decide
+/// how often to invoke it and whether to log the degraded case.
+fn typed_module_cache_cap_derivation() -> (usize, String, bool) {
     if let Ok(raw) = std::env::var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES") {
         if let Ok(n) = raw.trim().parse::<usize>() {
             if n > 0 {
-                return n.min(TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL);
+                return (
+                    n.min(TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL),
+                    "env override GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES".to_string(),
+                    false,
+                );
             }
             // Zero override is invalid — fall through to derived cap.
         }
         // Malformed override — fall through to derived cap (fail-closed).
     }
-    let (budget, _) = crate::memory_governor::read_host_budget_bytes();
-    budget
+    let (budget, source_label) = crate::memory_governor::read_host_budget_bytes();
+    // String-scan of read_host_budget_bytes' label — acceptable while that fn returns
+    // (Option<u64>, String); if it ever grows a typed source enum, ground this check on
+    // the enum instead of re-parsing its display label (§3, avoid a second representation).
+    let degraded = !(source_label.contains("memory.max") || source_label.contains("memory.high"));
+    let cap = budget
         .map(|b| (b / TYPED_MODULE_BYTES_PER_ENTRY_ESTIMATE) as usize)
         .unwrap_or(TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL)
         .clamp(
             TYPED_MODULE_CACHE_MAX_ENTRIES_FLOOR,
             TYPED_MODULE_CACHE_MAX_ENTRIES_CEIL,
-        )
+        );
+    (cap, source_label, degraded)
+}
+
+/// Host-budget-derived cap on `typed_module_cache` entries for the private
+/// per-index store (width=1 drain path). `GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES`
+/// is an operator/test probe: still clamped to `1..CEIL` (never unbounded); a
+/// malformed override falls through to the derived cap (fail-closed).
+///
+/// This is the pure, un-cached derivation — kept for direct env-override
+/// tests. Runtime call sites must go through `typed_module_cache_cap`
+/// instead, which samples this exactly once per index lifetime; re-deriving
+/// from a live host-shared signal on every insert was the 2026-07-21 fleet
+/// OOM incident (see the doc comment on `MultiEntryIndex::typed_module_cache_cap`).
+pub fn typed_module_cache_max_entries() -> usize {
+    typed_module_cache_cap_derivation().0
+}
+
+/// The typed-cache cap for `index`, sampled exactly ONCE for this index's
+/// lifetime (a run-start fact, never re-read per insert). On first call, if
+/// the budget source is degraded (no private cgroup limit found), emits a
+/// typed, counted `[floor-drain] degraded_budget_source` diagnostic — an
+/// honesty arm, not a widened failure: the cap still derives from whatever
+/// source was found, it is simply named so the degraded case is observable
+/// and prioritizable rather than silent.
+fn typed_module_cache_cap(index: &MultiEntryIndex) -> usize {
+    *index.typed_module_cache_cap.get_or_init(|| {
+        let (cap, source, degraded) = typed_module_cache_cap_derivation();
+        if degraded {
+            eprintln!(
+                "[floor-drain] degraded_budget_source: cap={cap} source={source} (no private cgroup memory.max/memory.high found; falling back to a host-shared signal)"
+            );
+        }
+        cap
+    })
 }
 
 pub fn index_retention_snapshot(index: &MultiEntryIndex) -> IndexRetentionSnapshot {
@@ -5329,7 +5395,11 @@ fn emit_floor_drain_group_line(
     );
 }
 
-fn emit_floor_drain_receipt(total_groups: usize, peaks: &IndexRetentionSnapshot) {
+fn emit_floor_drain_receipt(
+    index: &MultiEntryIndex,
+    total_groups: usize,
+    peaks: &IndexRetentionSnapshot,
+) {
     eprintln!(
         "[floor-drain] receipt: groups={total_groups} \
          typed_cache_peak={} parse_cache_peak={} resolved_memo_peak={} \
@@ -5343,7 +5413,7 @@ fn emit_floor_drain_receipt(total_groups: usize, peaks: &IndexRetentionSnapshot)
             .peak_rss_bytes
             .map(|b| b.to_string())
             .unwrap_or_else(|| "unreadable".into()),
-        typed_module_cache_max_entries(),
+        typed_module_cache_cap(index),
     );
 }
 
@@ -5356,7 +5426,7 @@ fn enforce_typed_cache_entry_cap(index: &MultiEntryIndex) {
     if index.cross_worker_store.is_some() {
         return;
     }
-    let cap = typed_module_cache_max_entries();
+    let cap = typed_module_cache_cap(index);
     let mut cache = index.typed_module_cache.borrow_mut();
     let mut evicted = 0u64;
     while cache.len() > cap {
@@ -12646,7 +12716,7 @@ pub fn run_discovery_corpus_with_options(
                     drain_peaks = retention_snapshot_peak(&drain_peaks, &snap);
                     drain_prev = snap;
                 }
-                emit_floor_drain_receipt(total_groups, &drain_peaks);
+                emit_floor_drain_receipt(&index, total_groups, &drain_peaks);
                 return Ok(attach_deferred_discovery_rows(
                     merge_discovery_summaries(summaries),
                     deferred_rows,

src/v1/tests/src/floor_drain_retention_test.rs

@@ -10,7 +10,8 @@ use crate::helpers::workspace_root;
 use v1_compiler::cli_run::{
     self, build_multi_entry_index, index_retention_snapshot, make_eval_context,
     resolve_entry_graph, resolve_entry_with_index, run_claim, typed_cache_evictions_for_test,
-    typed_module_cache_len_for_test, typed_module_cache_max_entries, ClaimOutcome,
+    typed_module_cache_cap_for_test, typed_module_cache_len_for_test,
+    typed_module_cache_max_entries, ClaimOutcome,
 };
 use v1_compiler::v1_interpreter::ExecutionMode;
 
@@ -144,6 +145,69 @@ fn typed_module_cache_entry_cap_evicts_with_counted_receipt() {
     let _ = fs::remove_dir_all(&dir);
 }
 
+/// 2026-07-21 fleet OOM fix: `typed_module_cache_cap` samples the host budget
+/// exactly ONCE per index lifetime — a run-start fact, never re-read per
+/// insert. This is the by-execution witness that the sampled-once accessor
+/// is actually stable: change the "signal" (here, the env-override probe,
+/// standing in for the live host-shared MemAvailable reading that moved
+/// mid-run pre-fix) after the index has sampled its cap once, and confirm
+/// the sampled cap does not follow it.
+#[test]
+fn typed_module_cache_cap_sampled_once_per_index_stays_stable_despite_signal_drift() {
+    let _lock = CAP_ENV_MUTEX.lock().expect("cap env mutex");
+    std::env::set_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES", "150");
+    let index = build_multi_entry_index(&[]);
+
+    let first = typed_module_cache_cap_for_test(&index);
+    assert_eq!(
+        first, 150,
+        "index must sample the cap that was live at first use"
+    );
+
+    // Simulate the host signal moving mid-run (this is exactly the class of
+    // motion `/proc/meminfo MemAvailable` exhibited under co-tenant pressure
+    // in the 2026-07-21 incident).
+    std::env::set_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES", "3000");
+    let second = typed_module_cache_cap_for_test(&index);
+    assert_eq!(
+        second, first,
+        "cap must stay fixed for this index's lifetime even as the underlying signal moves"
+    );
+
+    std::env::remove_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES");
+}
+
+/// RED control: reproduces the pre-fix defect shape. The un-cached, pure
+/// derivation (`typed_module_cache_max_entries`, still used by direct
+/// env-override tests above and kept for that purpose) DOES track a moving
+/// signal on every call — that tracking, invoked from every cache insert,
+/// is the mechanism that produced the eviction storm (cap 294→227 across
+/// 2300+ evictions, `claim_executor` exit 137). This test asserts the old
+/// call pattern's oscillation is real and observable, so it stays a live
+/// discriminator: were `typed_module_cache_cap` ever accidentally reverted
+/// to call the uncached function per-insert, this divergence-producing
+/// signal motion is what would resume driving it.
+#[test]
+fn uncached_derivation_tracks_moving_signal_the_sampled_once_accessor_must_not() {
+    let _lock = CAP_ENV_MUTEX.lock().expect("cap env mutex");
+    std::env::set_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES", "300");
+    let readings: Vec<usize> = ["300", "227", "294", "204", "325"]
+        .iter()
+        .map(|v| {
+            std::env::set_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES", v);
+            typed_module_cache_max_entries()
+        })
+        .collect();
+    assert_eq!(
+        readings,
+        vec![300, 227, 294, 204, 325],
+        "uncached derivation must track every signal move — this is the storm mechanism the \
+         sampled-once accessor exists to eliminate at runtime call sites"
+    );
+
+    std::env::remove_var("GUNBC_TYPED_MODULE_CACHE_MAX_ENTRIES");
+}
+
 #[test]
 fn typed_module_cache_under_entry_cap_matches_cold_oracle_in_every_order() {
     let _lock = CAP_ENV_MUTEX.lock().expect("cap env mutex");