From e747cc630e28ce4eed90c1ffd21c4276def6e8e8 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 26 Jun 2026 19:44:25 +0000 Subject: [PATCH 1/5] WIP: phase 0 metrics --- src/v2/workflow/ci_floor_plan.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v2/workflow/ci_floor_plan.dag b/src/v2/workflow/ci_floor_plan.dag index 7f4cb14776e..c6b94a61bda 100644 --- a/src/v2/workflow/ci_floor_plan.dag +++ b/src/v2/workflow/ci_floor_plan.dag @@ -292,7 +292,7 @@ fn gunbc_ci_floor_spawn_width_from_tree() -> HardwareThreadCount { } fn gunbc_ci_floor_spawn_width(memory_budget_bytes: Int) -> HardwareThreadCount { - gunbc_ci_floor_spawn_width_for_budget(memory_budget: byte_size(memory_budget_bytes)) + hardware_thread_count(count: 3) } fn gunbc_ci_floor_run_envelope(memory_budget_bytes: Int) -> ResourceEnvelope { From 7db859c844e1ae454da875f6fc02c9c86a63800a Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 26 Jun 2026 20:00:54 +0000 Subject: [PATCH 2/5] WIP: phase 0 metrics --- src/v2/workflow/ci_floor_plan.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v2/workflow/ci_floor_plan.dag b/src/v2/workflow/ci_floor_plan.dag index c6b94a61bda..7f4cb14776e 100644 --- a/src/v2/workflow/ci_floor_plan.dag +++ b/src/v2/workflow/ci_floor_plan.dag @@ -292,7 +292,7 @@ fn gunbc_ci_floor_spawn_width_from_tree() -> HardwareThreadCount { } fn gunbc_ci_floor_spawn_width(memory_budget_bytes: Int) -> HardwareThreadCount { - hardware_thread_count(count: 3) + gunbc_ci_floor_spawn_width_for_budget(memory_budget: byte_size(memory_budget_bytes)) } fn gunbc_ci_floor_run_envelope(memory_budget_bytes: Int) -> ResourceEnvelope { From 83dea09a20ff40632457198925433eee996a5557 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 26 Jun 2026 22:14:53 +0000 Subject: [PATCH 3/5] WIP: phase 0 metrics --- src/v1/stage0/src/cli_run.rs | 10 ++++------ src/v1/stage0/src/resolved_graph_cache.rs | 24 ++++++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/v1/stage0/src/cli_run.rs b/src/v1/stage0/src/cli_run.rs index e2eda0d536a..90f3adce424 100644 --- a/src/v1/stage0/src/cli_run.rs +++ b/src/v1/stage0/src/cli_run.rs @@ -28,8 +28,8 @@ use crate::v1_std_core::{ use serde::Serialize; use crate::resolved_graph_cache::{ - lookup as cross_process_lookup, resolved_graph_cache_root, subject_digest_for_closure, - write as cross_process_write, CacheLookupResult, + lookup as cross_process_lookup, resolved_graph_cache_root_from_env, + subject_digest_for_closure, write as cross_process_write, CacheLookupResult, }; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -404,8 +404,7 @@ fn resolve_entry_with_parse_cache( > { let sources = load_sources_for_entry_with_index(&index.source_files, entry_file)?; - { - let cache_root = resolved_graph_cache_root(); + if let Some(cache_root) = resolved_graph_cache_root_from_env() { let subject = subject_digest_for_closure(&sources); match cross_process_lookup(&cache_root, &subject) { CacheLookupResult::Hit(hit) => { @@ -515,8 +514,7 @@ fn resolve_entry_with_parse_cache( return Err(format_error_nodes(&ownership_diags, &source_indices)); } - { - let cache_root = resolved_graph_cache_root(); + if let Some(cache_root) = resolved_graph_cache_root_from_env() { let subject = subject_digest_for_closure(&sources); let _ = cross_process_write(&cache_root, &subject, &typed, source_indices.as_ref()); } diff --git a/src/v1/stage0/src/resolved_graph_cache.rs b/src/v1/stage0/src/resolved_graph_cache.rs index c8b2d17d892..4b3798e2854 100644 --- a/src/v1/stage0/src/resolved_graph_cache.rs +++ b/src/v1/stage0/src/resolved_graph_cache.rs @@ -65,15 +65,21 @@ pub enum CacheWriteOutcome { AlreadyExists, } -pub fn resolved_graph_cache_root() -> PathBuf { - std::env::var_os("GUNBC_RESOLVED_GRAPH_CACHE_DIR") - .map(PathBuf::from) - .unwrap_or_else(|| { - let user_tag = std::env::var("USER") - .or_else(|_| std::env::var("USERNAME")) - .unwrap_or_else(|_| "shared".to_string()); - std::env::temp_dir().join(format!("gunbc-rg-cache-{user_tag}")) - }) +/// The cross-process resolved-graph disk cache is **opt-in**: it activates only +/// when `GUNBC_RESOLVED_GRAPH_CACHE_DIR` names a directory. With it unset the +/// cache is entirely off (`None` — no read and no write). +/// +/// Why opt-in and not a `temp_dir()` default: under the CI floor the cache is a +/// net loss. Every commit changes the compiler binary, which is part of the +/// content digest, so the floor re-colds each run and is overwhelmingly a miss; +/// and both paths buffer a whole cache file in memory (a hit `read_to_end`s the +/// verbose-JSON file, ~11x the packed graph; a miss `to_vec`s the whole JSON), +/// so multi-GiB entries resolved across concurrent shards OOM the runner. The +/// cache pays its worst cost exactly where it collects ~no benefit. (#5789 made +/// it always-on via a `temp_dir()` default; this reverts to opt-in — its IO +/// realization must stream, not buffer, before it is safe to default on.) +pub fn resolved_graph_cache_root_from_env() -> Option { + std::env::var_os("GUNBC_RESOLVED_GRAPH_CACHE_DIR").map(PathBuf::from) } fn extract_module_path(content: &str) -> Option { From e25d99c5d2d1ad097520c898de8d7134373bd0fb Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 26 Jun 2026 22:25:24 +0000 Subject: [PATCH 4/5] WIP: phase 0 metrics --- dsl/gunbc/roadmap_authority.dag | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dsl/gunbc/roadmap_authority.dag b/dsl/gunbc/roadmap_authority.dag index 54c42847b2a..45425a8c497 100644 --- a/dsl/gunbc/roadmap_authority.dag +++ b/dsl/gunbc/roadmap_authority.dag @@ -166,7 +166,7 @@ fn section_1() -> RoadmapSection { derivable_doc(id: "1-measure-foundation", prs: [5470, 5478], title: "**Section 1 spawn-width foundation — std.measure expressibility**", description: "— the FLOOR family (measure_scale_fraction_floor + measure_fit_count_floor) and the demand-CEIL family completing the ceil-not-equal-floor money-pair landed, dissolving the measure unwrap→raw-arith→rewrap §3 fork. Authority: the expressibility-frontier spec #5467", path: "docs/plans/expressibility-frontier.md"), authored(id: "1-placement-inert", done: false, content: "**scheduling: placement + materialization inert** — `Placement`/`Materialization` modeled + witness-passing, **no live consumer** (same band as the host-ops gap, substrate side)"), authored(id: "1-sched-resource-aware", done: false, content: "**scheduling: resource-aware (larger allocation → larger throughput)** — `spawn_width` is a side-channel derived from a static data row (`gunbc_ci_floor_measured_peak`, coarse whole_run÷concurrency estimate); goal: width derived from per-`Runnable` cost in the plan itself, self-calibrating from measured per-shard RSS. Four nodes: (A) measurement plumbing — claim_batch emits `[measurement] per-shard-peak-rss`, claim_executor collects max *(dispatched adhoc-97532cd3-dfe)*; (B) `CostEstimate.space` on `Runnable` in std; (C) scheduler reads cost → derives width, side-channel deleted; (D) calibration loop retires static data rows. Construction invariant: host memory increase → higher width → more throughput, no `.dag` edits required. [plan](docs/plans/resource-aware-scheduler.md)"), - authored(id: "1-caching-forked", done: false, content: "**caching forked** — sccache live · resolve-cache **always-on (#5789 — default `$TMPDIR/gunbc-rg-cache`, CI pending merge)** · ParseTable memo live · RecordedFixture · BuildBuddy opt-in → converge on `realize(subject)` (§2 P2)"), + authored(id: "1-caching-forked", done: false, content: "**caching forked** — sccache live · resolve-cache **opt-in (`GUNBC_RESOLVED_GRAPH_CACHE_DIR`; #5789 always-on default reverted — its whole-file JSON read/write OOMs the floor, must stream before it can default on)** · ParseTable memo live · RecordedFixture · BuildBuddy opt-in → converge on `realize(subject)` (§2 P2)"), ], edges: [], ), @@ -241,7 +241,7 @@ fn section_2() -> RoadmapSection { authored(id: "2-p1", done: true, content: "P1 honest keys by construction — warm==cold purity oracle **LANDED (#5429)**"), authored(id: "2-p2", done: false, content: "P2 one door: `realize(subject)` sole API — kernel inhabits `cache_interface.dag` (#5446); ParseTable dissolution is downstream of the dsl→v2 de-fork (§5)"), plain(id: "2-hermetic", content: "hermetic fixtures feed P2: [x] M4.1 universal hermetic corpus governance (#5236, [plan](docs/plans/m4-universal-hermetic-corpus.md)); [ ] M5 fixture-store onto one Realization kernel ([plan](docs/plans/m5-fixture-store-consolidation.md))"), - authored(id: "2-p3", done: true, content: "P3 **resolve-cache enable** — purity proven (616/616); #5789 makes cache **always-on** (default `$TMPDIR/gunbc-rg-cache`, no env var required; CI pending merge 2026-06-25) ← **core ask landed**"), + authored(id: "2-p3", done: false, content: "P3 **resolve-cache enable** — purity proven (616/616); cache is correct but **opt-in** (`GUNBC_RESOLVED_GRAPH_CACHE_DIR`): #5789's always-on `$TMPDIR` default was reverted because the whole-file JSON read (hit) and write (miss) buffer ~11x the packed graph in memory and OOM the concurrent-shard floor. Re-enabling by default is gated on a streaming IO realization (binary + `deserialize_from`/`serialize_into`, no whole-file `Vec`)"), authored(id: "2-p4", done: false, content: "P4 economic tier (measured cost → `Materialization`) — instrument done (#5431); remaining = (a) space arm: per-shard RSS plumbing → `CostEstimate.space` on `Runnable` (nodes A+B, [plan](docs/plans/resource-aware-scheduler.md)); (b) time arm: consumer feedback → `CostAccount.time = Measured` feeds width-fold (node C+D); dissolves `cost_account_predicted_zero()` on the floor path"), authored(id: "2-p5", done: false, content: "P5 native `content(T) = content_hash(subgraph)` — gated on B2"), plain(id: "2-blockers", content: "blockers: [ ] B1 #5295 generic-instantiation (gates cross-shard `Share`) · [ ] B2 cross-tree content-hash (gates P5)"), From 54d16c50e1f04650c3aab7d8016ac1699d9d872d Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 26 Jun 2026 22:50:53 +0000 Subject: [PATCH 5/5] WIP: phase 0 metrics --- ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 95d34e8aa19..bfe6f3faf76 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -107,7 +107,7 @@ A flaky or green-but-broken floor means no gate protects anything — so CI is u - [x] **Section 1 spawn-width foundation — std.measure expressibility** (#5470/#5478) — the FLOOR family (measure_scale_fraction_floor + measure_fit_count_floor) and the demand-CEIL family completing the ceil-not-equal-floor money-pair landed, dissolving the measure unwrap→raw-arith→rewrap §3 fork. Authority: the expressibility-frontier spec #5467 [plan](docs/plans/expressibility-frontier.md) - [ ] **scheduling: placement + materialization inert** — `Placement`/`Materialization` modeled + witness-passing, **no live consumer** (same band as the host-ops gap, substrate side) - [ ] **scheduling: resource-aware (larger allocation → larger throughput)** — `spawn_width` is a side-channel derived from a static data row (`gunbc_ci_floor_measured_peak`, coarse whole_run÷concurrency estimate); goal: width derived from per-`Runnable` cost in the plan itself, self-calibrating from measured per-shard RSS. Four nodes: (A) measurement plumbing — claim_batch emits `[measurement] per-shard-peak-rss`, claim_executor collects max *(dispatched adhoc-97532cd3-dfe)*; (B) `CostEstimate.space` on `Runnable` in std; (C) scheduler reads cost → derives width, side-channel deleted; (D) calibration loop retires static data rows. Construction invariant: host memory increase → higher width → more throughput, no `.dag` edits required. [plan](docs/plans/resource-aware-scheduler.md) -- [ ] **caching forked** — sccache live · resolve-cache **always-on (#5789 — default `$TMPDIR/gunbc-rg-cache`, CI pending merge)** · ParseTable memo live · RecordedFixture · BuildBuddy opt-in → converge on `realize(subject)` (§2 P2) +- [ ] **caching forked** — sccache live · resolve-cache **opt-in (`GUNBC_RESOLVED_GRAPH_CACHE_DIR`; #5789 always-on default reverted — its whole-file JSON read/write OOMs the floor, must stream before it can default on)** · ParseTable memo live · RecordedFixture · BuildBuddy opt-in → converge on `realize(subject)` (§2 P2) **Host-operation band — off-fabric, unmodeled, unenforced (G1–G3, NOW):** @@ -157,7 +157,7 @@ Gate: uncached non-redundant work is an ERROR, not "slow". The cache-key-from-in - [x] P1 honest keys by construction — warm==cold purity oracle **LANDED (#5429)** - [ ] P2 one door: `realize(subject)` sole API — kernel inhabits `cache_interface.dag` (#5446); ParseTable dissolution is downstream of the dsl→v2 de-fork (§5) - hermetic fixtures feed P2: [x] M4.1 universal hermetic corpus governance (#5236, [plan](docs/plans/m4-universal-hermetic-corpus.md)); [ ] M5 fixture-store onto one Realization kernel ([plan](docs/plans/m5-fixture-store-consolidation.md)) - - [x] P3 **resolve-cache enable** — purity proven (616/616); #5789 makes cache **always-on** (default `$TMPDIR/gunbc-rg-cache`, no env var required; CI pending merge 2026-06-25) ← **core ask landed** + - [ ] P3 **resolve-cache enable** — purity proven (616/616); cache is correct but **opt-in** (`GUNBC_RESOLVED_GRAPH_CACHE_DIR`): #5789's always-on `$TMPDIR` default was reverted because the whole-file JSON read (hit) and write (miss) buffer ~11x the packed graph in memory and OOM the concurrent-shard floor. Re-enabling by default is gated on a streaming IO realization (binary + `deserialize_from`/`serialize_into`, no whole-file `Vec`) - [ ] P4 economic tier (measured cost → `Materialization`) — instrument done (#5431); remaining = (a) space arm: per-shard RSS plumbing → `CostEstimate.space` on `Runnable` (nodes A+B, [plan](docs/plans/resource-aware-scheduler.md)); (b) time arm: consumer feedback → `CostAccount.time = Measured` feeds width-fold (node C+D); dissolves `cost_account_predicted_zero()` on the floor path - [ ] P5 native `content(T) = content_hash(subgraph)` — gated on B2